Beispiel #1
0
    def request(self, proposal, future):
        """
        makes phase 1a proposal
        manages successful ballots
        performs state updates
        returns succeed/fail code
            0: request succeeded
            1: request failed
            2: request failed + resynced state
        """
        print(self.maxBal)
        # bmsgs := bmsgs U ("1a", bal)
        #if self.Q:
        #    self.logger.error("Bad Client: prior quorum being managed for ballot {}".format(self.Q.N))
        self.logger.debug("request initiated: {}".format(proposal))
        res = {'code': 1}
        if not ',' in proposal:
            self.logger.error("db commit proposal not in key,value format")
            return res # invalid request failcase

        self.pending = future
        self.maxBal = self.maxVBal + 1
        recipients = self.peers.get_all()
        self.logger.debug("local group is {}".format(recipients))

        if not len(recipients): # no localgroup peers
            self.maxVBal += 1
            self.bmsgs.append(proposal)
            self.state.update(self.maxBal, proposal) # necessary to avoid including self in every round
            self.avs.append((self.maxBal, proposal))
            res = {'code': 0} # success case
            

        else:
            self.Q = Quorum(self.maxBal, self.peers.num_peers)
            self.logger.debug("creating Quorum object with N={}, num_peers={}".format(self.maxBal, self.peers.num_peers))
            self.logger.debug("sending 1a -> {}: {}".format(self.maxBal, proposal))
            msg_1a = "1a&{}&{}".format(str(time.time()),self.maxBal)
            yield from asyncio.async(self.send_msg(msg_1a, recipients, self.handle_msg)) # collect stats here
        
            if self.pending.done():
                # Error case (possibly unneccessary)
                self.logger.error("future done before voting complete")
            else:
                if self.Q.quorum_1b():
                    if self.Q.got_majority_accept():
                        self.logger.info("1: quorum of {} accepts".format(self.Q.quorum))
                        val_bytes = proposal.encode()
                        length = len(val_bytes)
                        prepped_val = str(length)+"<>"+str(int.from_bytes(val_bytes, byteorder='little'))
                        yield from self.send_msg(self.phase1c(prepped_val), recipients, self.handle_msg) # send 1c
                        self.maxVBal = self.Q.N
                        self.maxVVal = proposal
                        if self.Q.quorum_2b():
                            # Quorum Accepts and Commit succeeds case
                            self.logger.info("2: quorum accepts")
                            #self.logger.info("Quorum Accepted Value {} at Ballot {}".format(proposal,self.Q.N))
                            try:
                                self.state.update(self.Q.N, proposal)
                                res = {'code': 0} # succeeded
                                self.avs.append((self.Q.N, proposal))
                            except Exception as e:
                                self.logger.info("update failed!")
                        else:
                            # Quorum Accepts then Commit fails case
                            self.logger.info("2b failure: quorum1 accepts but quorum2 failed")
                    else:
                        # Quorum Rejects case -> reconfigure!
                        self.logger.info("failure: quorum1 rejects {} for ballot {}".format(proposal, self.Q.N))
                        # update avs structure here for newest ballot number for value
                        catchup_avs = self.Q.rejecting_quorum_avs() # list of (ballot#, database_operation) tuples
                        if len(catchup_avs):
                            self.logger.info("syncing state...")
                            for ballot, op in catchup_avs:
                                if ballot > self.maxBal:
                                    self.state.update(ballot, op)

                            self.maxVBal, self.maxVVal = catchup_avs[-1]   
                            res = {'code': 2} # corrupt state resync failcase
                            #res = {'code': 1} # rejected by quorum failcase
                else:
                    self.logger.info("failure: quorum1 not acquired")

        if not self.pending.done():
            self.pending.set_result(res)
        self.logger.debug("bpcon request finished")    
        return res 
Beispiel #2
0
    def request(self, proposal, future):
        """
        makes phase 1a proposal
        manages successful ballots
         state updates
        returns succeed/fail boolean
        """
        # bmsgs := bmsgs U ("1a", bal)
        #if self.Q:
        #    self.logger.error("Bad Client: prior quorum being managed for ballot {}".format(self.Q.N))
        if not ',' in proposal:
            self.logger.error("db commit proposal not in key,value format")
            return {'code': 1} # invalid request failcase

        self.pending = future
        self.maxBal = self.maxVBal + 1
        recipients = self.peers.get_all()

        if not len(recipients): # no localgroup peers
            self.maxVBal += 1
            self.bmsgs.append(proposal)
            self.state.update(proposal,self.maxBal) # necessary to avoid including self in every round
            res = {'code': 0} # success case

        else:
            self.Q = Quorum(self.maxBal, self.peers.num_peers)
            self.logger.debug("creating Quorum object with N={}, num_peers={}".format(self.maxBal, self.peers.num_peers))
            self.logger.debug("sending 1a -> {}: {}".format(self.maxBal, proposal))
            msg_1a = "1a&{}&{}".format(str(time.time()),self.maxBal)
            yield from asyncio.async(self.send_msg(msg_1a, recipients, self.handle_msg)) # collect stats here
        
            if self.pending.done():
                # Error case (possibly unneccessary)
                self.logger.error("future done before voting complete")
            else:
                if self.Q.quorum_1b():
                    if self.Q.got_majority_accept():
                        self.logger.info("1: quorum of {} accepts".format(self.Q.quorum))
                        val_bytes = proposal.encode()
                        length = len(val_bytes)
                        prepped_val = str(length)+"<>"+str(int.from_bytes(val_bytes, byteorder='little'))
                        yield from self.send_msg(self.phase1c(prepped_val), recipients, self.handle_msg) # send 1c
                        self.maxVBal = self.Q.N
                        self.maxVVal = proposal
                        if self.Q.quorum_2b():
                            # Quorum Accepts and Commit succeeds case
                            self.logger.info("2: quorum accepts")
                            #self.logger.info("Quorum Accepted Value {} at Ballot {}".format(proposal,self.Q.N))
                            try:
                                self.state.update(proposal, self.Q.N)
                                res = {'code': 0} # succeeded
                            except Exception as e:
                                self.logger.info("update failed!")
                                res = {'code': 1}
                        else:
                            # Quorum Accepts then Commit fails case
                            self.logger.info("2b failure: quorum1 accepts but quorum2 failed")
                    else:
                        # Quorum Rejects case -> reconfigure!
                        self.logger.info("failure: quorum1 rejects {} for ballot {}".format(proposal, self.Q.N))
                        good_peer = self.Q.rejecting_quorum_member()
                        res = {'code': 2, 'value': good_peer} # corrupt state failcase, wss of an up-to-date peer
                else:
                    self.logger.info("failure: quorum1 not acquired")

        if not self.pending.done():
            self.pending.set_result(res)
        return res 
Beispiel #3
0
class BPConProtocol:
    def __init__(self, conf, state):
        """
        conf  -- dictionary with configuration variables
        state -- class object with update() function
        """
        self.logger = conf['log']
        self.maxBal = -1
        self.maxVBal = -1       # highest ballot voted in
        self.maxVVal = None     # value voted for maxVBal
        self.avs = []           # msg dict keyed by value
        self.seen = {}          # 1b messages -> use to check if v is safe at b
        self.bmsgs = []         # log of values in 2b messages sent by this instance       
        self.Q = None
        
        with open(conf['keyfile'], 'r') as fh:
            key = RSA.importKey(fh.read())
        self.signer = PKCS1_v1_5.new(key)

        # initialize localgroup
        self.peers = GroupManager(conf)
        self.peers.init_local_group()
        self.state = state
        #self.state.groups['G1'] = self.peers

        self.ctx = get_ssl_context(conf['peer_certs'])
        self.pending = None
    
    def sentMsgs(self, type_, bal):
        pass

    def knowsSafeAt(ac, b, v):
        pass

    @asyncio.coroutine
    def request(self, proposal, future):
        """
        makes phase 1a proposal
        manages successful ballots
        performs state updates
        returns succeed/fail code
            0: request succeeded
            1: request failed
            2: request failed + resynced state
        """
        print(self.maxBal)
        # bmsgs := bmsgs U ("1a", bal)
        #if self.Q:
        #    self.logger.error("Bad Client: prior quorum being managed for ballot {}".format(self.Q.N))
        self.logger.debug("request initiated: {}".format(proposal))
        res = {'code': 1}
        if not ',' in proposal:
            self.logger.error("db commit proposal not in key,value format")
            return res # invalid request failcase

        self.pending = future
        self.maxBal = self.maxVBal + 1
        recipients = self.peers.get_all()
        self.logger.debug("local group is {}".format(recipients))

        if not len(recipients): # no localgroup peers
            self.maxVBal += 1
            self.bmsgs.append(proposal)
            self.state.update(self.maxBal, proposal) # necessary to avoid including self in every round
            self.avs.append((self.maxBal, proposal))
            res = {'code': 0} # success case
            

        else:
            self.Q = Quorum(self.maxBal, self.peers.num_peers)
            self.logger.debug("creating Quorum object with N={}, num_peers={}".format(self.maxBal, self.peers.num_peers))
            self.logger.debug("sending 1a -> {}: {}".format(self.maxBal, proposal))
            msg_1a = "1a&{}&{}".format(str(time.time()),self.maxBal)
            yield from asyncio.async(self.send_msg(msg_1a, recipients, self.handle_msg)) # collect stats here
        
            if self.pending.done():
                # Error case (possibly unneccessary)
                self.logger.error("future done before voting complete")
            else:
                if self.Q.quorum_1b():
                    if self.Q.got_majority_accept():
                        self.logger.info("1: quorum of {} accepts".format(self.Q.quorum))
                        val_bytes = proposal.encode()
                        length = len(val_bytes)
                        prepped_val = str(length)+"<>"+str(int.from_bytes(val_bytes, byteorder='little'))
                        yield from self.send_msg(self.phase1c(prepped_val), recipients, self.handle_msg) # send 1c
                        self.maxVBal = self.Q.N
                        self.maxVVal = proposal
                        if self.Q.quorum_2b():
                            # Quorum Accepts and Commit succeeds case
                            self.logger.info("2: quorum accepts")
                            #self.logger.info("Quorum Accepted Value {} at Ballot {}".format(proposal,self.Q.N))
                            try:
                                self.state.update(self.Q.N, proposal)
                                res = {'code': 0} # succeeded
                                self.avs.append((self.Q.N, proposal))
                            except Exception as e:
                                self.logger.info("update failed!")
                        else:
                            # Quorum Accepts then Commit fails case
                            self.logger.info("2b failure: quorum1 accepts but quorum2 failed")
                    else:
                        # Quorum Rejects case -> reconfigure!
                        self.logger.info("failure: quorum1 rejects {} for ballot {}".format(proposal, self.Q.N))
                        # update avs structure here for newest ballot number for value
                        catchup_avs = self.Q.rejecting_quorum_avs() # list of (ballot#, database_operation) tuples
                        if len(catchup_avs):
                            self.logger.info("syncing state...")
                            for ballot, op in catchup_avs:
                                if ballot > self.maxBal:
                                    self.state.update(ballot, op)

                            self.maxVBal, self.maxVVal = catchup_avs[-1]   
                            res = {'code': 2} # corrupt state resync failcase
                            #res = {'code': 1} # rejected by quorum failcase
                else:
                    self.logger.info("failure: quorum1 not acquired")

        if not self.pending.done():
            self.pending.set_result(res)
        self.logger.debug("bpcon request finished")    
        return res 
        
    def phase1b(self, N):
        # bmsgs := bmsgs U ("1b", bal, acceptor, self.avs, self.maxVBal, self.maxVVal)
        if (int(N) > int(self.maxBal)): #maxbal type undefined behavior sometimes 
            self.maxBal = N
            avs = "0"
        else:
            n = 256
            a = pickle.dumps(self.avs) # bytes instead of list
            #chunks = [a[i:i + n] for i in range(0, len(a), n)] # split into smaller bytestrings
            #avs = ','.join(map(encode_as_ints, chunks)) 
            avs = encode_for_transport(a)
        
        msg_1b = "1b&{}&{}&{}&{}&{}".format(str(time.time()),N,self.maxVBal,self.maxVVal,avs)
        msg_hash = SHA.new(msg_1b.encode())
        sig = self.signer.sign(msg_hash)
        #tosend = msg_1b + ";" + str(int.from_bytes(sig, byteorder='little'))
        tosend = msg_1b + ";" + encode_for_transport(sig)
        self.logger.debug("sending 1b -> {}".format(tosend))
        return tosend
            
    def phase1c(self, val):
        # bmsgs := bmsgs U ("1c", bal, val)
        self.logger.debug("sending 1c -> {}: {}".format(self.Q.N, val))
        tosend = "1c&{}&{}&{}&;{}".format(str(time.time()),self.Q.N, val, self.Q.get_msgs())
        return tosend

    @asyncio.coroutine
    def phase2av(self, b):
        if (self.maxBal <= b) and (r.bal < b for r in self.avs):
            # m in (1c, b) sentMsgs and b safe at vals from sentmsgs
            # bmsgs := bmsgs U ("2av", m.bal, m.val, acceptor)
            # remove r from avs where r.val == m.val
            self.maxBal = b

    def phase2b(self, b, v):
        # bmsgs := bmsgs U ("2b", m.bal, m.val, acceptor)
        if int(self.maxBal) <= int(b): 
            # exists (2av, b) pairing in sentMsgs that quorum of acceptors agree upon
            
            self.maxVVal = v
            self.maxBal = b
            self.maxVBal = b
            self.avs.append((b,v))
            #self.bmsgs.append(m.val) # saving state update values for bringing peers up-to-date

            #self.logger.info(self.bmsgs)
            self.state.update(b, v)
            tosend = "2b&{}&{}&{}".format(str(time.time()), b, v)
            return tosend

    @asyncio.coroutine
    def main_loop(self, websocket, path):
        """
        server socket
        """ 
        self.logger.debug("main loop")
        # idle until receives network input
        try:
            input_msg = yield from websocket.recv()
            self.logger.debug("< {}".format(input_msg))
            host,port = websocket.remote_address
            output_msg = self.handle_msg(str(input_msg), "wss://{}:{}".format(host,port))
            if output_msg:
                yield from websocket.send(output_msg)
                self.bmsgs.append(output_msg)
                #print(self.bmsgs)
            else:
                self.logger.error("got bad input from peer")
        except Exception as e:
            self.logger.error("mainloop exception: {}".format(e))
        self.logger.debug("Pending tasks after mainloop: %i" % len(asyncio.Task.all_tasks(asyncio.get_event_loop())))    


    def preprocess_msg(self, msg): # TODO verification
        return msg.split('&')

    def handle_msg(self, msg, peer_wss=None):    
        msg_type = msg[:2]
        if msg_type == "1c":
            msg,proofs = msg.split('&;')
        parts = self.preprocess_msg(msg)
        num_parts = len(parts)
        timestamp = parts[1]
        N = int(parts[2])

        
        if msg_type == "1a" and num_parts == 3:
            # a peer is leader for a ballot, requesting votes
            print('a')
            output_msg = self.phase1b(N)
            return output_msg
            
        elif msg_type == "1b" and num_parts == 6:
            # implies is leader for ballot, has quorum object
            self.logger.debug("got 1b!!!")
            [mb,mv,avsSig] = parts[3:6]
            avs, sig = avsSig.split(';')
            
            if N == self.Q.N:
                self.Q.add_1b(int(mb), msg, peer_wss)
                
                if int(mb) > self.maxBal:
                    try:
                        #b = pickle.loads(b''.join(map(decode_to_bytes, avs.split(','))))
                        b = pickle.loads(decode_to_bytes(avs))
                        self.logger.debug("recording mb and avs update: {}, {}".format(mb, b))
                        self.Q.add_avs(int(mb), b)
                    except:
                        self.logger.debug("could not decode avs sent by {}: {}".format(peer_wss, avs))
            else:
                self.logger.error("got bad 1b msg")
                 
        elif msg_type == "1c" and num_parts == 4:
            self.logger.debug("got 1c")
            v = parts[3]
            signed_msgs = proofs.split(',') # 1b proofs in wss;msg;sig format
            if len(signed_msgs) <= self.peers.num_peers:
                # test against pubkey
                self.logger.debug("testing sig here...")
                num_verified = self.peers.verify_sigs(signed_msgs)
                
                if num_verified >= self.peers.quorum_size():
                    self.logger.debug("signature verification succeeded")
                    output_msg = self.phase2b(N,v)
                    return output_msg
                else:
                    self.logger.error("signature verification failed")
            else:
                self.logger.error("too many signatures")
 
        elif msg_type == "2b" and num_parts == 4:
            self.logger.debug("got 2b")
            self.Q.add_2b(N)
        else:
            self.logger.info("non-paxos msg received")

    @asyncio.coroutine
    def send_msg(self, to_send, recipient_list, handler_function=None):
        """
        client socket 

        returns list of latencies
        """
        self.logger.debug("sending to recipients: {}".format(recipient_list))
        good_peers = 0
        peer_latencies = {}
        input_msg = None
        for ws in recipient_list:
            send_start = time.time()
            try:
                self.logger.info("sending {} to {}".format(to_send, ws))
                client_socket = yield from websockets.connect(ws, ssl=self.ctx)
                self.logger.debug("to_send: {}".format(to_send))
                yield from client_socket.send(to_send)
               
                input_msg = yield from client_socket.recv()
                peer_latencies[ws] = time.time() - send_start
                self.logger.debug("input_msg: {}".format(input_msg))
                if handler_function:
                    try:
                        handler_function(input_msg, ws)
                        good_peers += 1
                    except Exception as e:
                        self.logger.critical("could not process input from {}: {}".format(ws,e))
                else:
                    good_peers += 1
            except Exception as e: # custom error handling
                self.logger.debug("send to peer {} failed: {}".format(ws,e))
                peer_latencies[ws] = -1
            finally:
                try:
                    yield from client_socket.close()
                except Exception as e:
                    self.logger.debug("no socket to close")

        self.logger.info("Peers: {}/{}".format(good_peers, len(recipient_list)))     
        self.logger.info("Peer Latencies: {}".format(peer_latencies))
        
        if not handler_function: 
            return input_msg
Beispiel #4
0
class BPConProtocol:
    def __init__(self, conf, state):
        """
        conf  -- dictionary with configuration variables
        state -- class object with update() function
        """
        self.logger = conf['log']
        self.maxBal = -1
        self.maxVBal = -1       # highest ballot voted in
        self.maxVVal = None     # value voted for maxVBal
        self.avs = {}           # msg dict keyed by value (max ballot saved only)
        self.seen = {}          # 1b messages -> use to check if v is safe at b
        self.bmsgs = []         # log of values in 2b messages sent by this instance       
        self.Q = None
        
        with open(conf['keyfile'], 'r') as fh:
            key = RSA.importKey(fh.read())
        self.signer = PKCS1_v1_5.new(key)

        # initialize localgroup
        self.peers = GroupManager(conf)
        self.peers.init_local_group()
        self.state = state
        #self.state.groups['G1'] = self.peers

        self.ctx = get_ssl_context(conf['peer_certs'])
        self.pending = None
    
    def sentMsgs(self, type_, bal):
        pass

    def knowsSafeAt(ac, b, v):
        pass

    @asyncio.coroutine
    def request(self, proposal, future):
        """
        makes phase 1a proposal
        manages successful ballots
         state updates
        returns succeed/fail boolean
        """
        # bmsgs := bmsgs U ("1a", bal)
        #if self.Q:
        #    self.logger.error("Bad Client: prior quorum being managed for ballot {}".format(self.Q.N))
        if not ',' in proposal:
            self.logger.error("db commit proposal not in key,value format")
            return {'code': 1} # invalid request failcase

        self.pending = future
        self.maxBal = self.maxVBal + 1
        recipients = self.peers.get_all()

        if not len(recipients): # no localgroup peers
            self.maxVBal += 1
            self.bmsgs.append(proposal)
            self.state.update(proposal,self.maxBal) # necessary to avoid including self in every round
            res = {'code': 0} # success case

        else:
            self.Q = Quorum(self.maxBal, self.peers.num_peers)
            self.logger.debug("creating Quorum object with N={}, num_peers={}".format(self.maxBal, self.peers.num_peers))
            self.logger.debug("sending 1a -> {}: {}".format(self.maxBal, proposal))
            msg_1a = "1a&{}&{}".format(str(time.time()),self.maxBal)
            yield from asyncio.async(self.send_msg(msg_1a, recipients, self.handle_msg)) # collect stats here
        
            if self.pending.done():
                # Error case (possibly unneccessary)
                self.logger.error("future done before voting complete")
            else:
                if self.Q.quorum_1b():
                    if self.Q.got_majority_accept():
                        self.logger.info("1: quorum of {} accepts".format(self.Q.quorum))
                        val_bytes = proposal.encode()
                        length = len(val_bytes)
                        prepped_val = str(length)+"<>"+str(int.from_bytes(val_bytes, byteorder='little'))
                        yield from self.send_msg(self.phase1c(prepped_val), recipients, self.handle_msg) # send 1c
                        self.maxVBal = self.Q.N
                        self.maxVVal = proposal
                        if self.Q.quorum_2b():
                            # Quorum Accepts and Commit succeeds case
                            self.logger.info("2: quorum accepts")
                            #self.logger.info("Quorum Accepted Value {} at Ballot {}".format(proposal,self.Q.N))
                            try:
                                self.state.update(proposal, self.Q.N)
                                res = {'code': 0} # succeeded
                            except Exception as e:
                                self.logger.info("update failed!")
                                res = {'code': 1}
                        else:
                            # Quorum Accepts then Commit fails case
                            self.logger.info("2b failure: quorum1 accepts but quorum2 failed")
                    else:
                        # Quorum Rejects case -> reconfigure!
                        self.logger.info("failure: quorum1 rejects {} for ballot {}".format(proposal, self.Q.N))
                        good_peer = self.Q.rejecting_quorum_member()
                        res = {'code': 2, 'value': good_peer} # corrupt state failcase, wss of an up-to-date peer
                else:
                    self.logger.info("failure: quorum1 not acquired")

        if not self.pending.done():
            self.pending.set_result(res)
        return res 
        
    def phase1b(self, N):
        # bmsgs := bmsgs U ("1b", bal, acceptor, self.avs, self.maxVBal, self.maxVVal)
        
        if (int(N) > int(self.maxBal)): #maxbal type undefined behavior sometimes 
            self.maxBal = N
        msg_1b = "1b&{}&{}&{}&{}&{}".format(str(time.time()),N,self.maxVBal,self.maxVVal,self.avs)
        msg_hash = SHA.new(msg_1b.encode())
        sig = self.signer.sign(msg_hash)
        
        tosend = msg_1b + ";" + str(int.from_bytes(sig, byteorder='little'))
        self.logger.debug("sending 1b -> {}".format(tosend))
        return tosend
            
    def phase1c(self, val):
        # bmsgs := bmsgs U ("1c", bal, val)
        self.logger.debug("sending 1c -> {}: {}".format(self.Q.N, val))
        tosend = "1c&{}&{}&{}&;{}".format(str(time.time()),self.Q.N, val, self.Q.get_msgs())
        return tosend

    @asyncio.coroutine
    def phase2av(self, b):
        if (self.maxBal <= b) and (r.bal < b for r in self.avs):
            # m in (1c, b) sentMsgs and b safe at vals from sentmsgs
            # bmsgs := bmsgs U ("2av", m.bal, m.val, acceptor)
            # remove r from avs where r.val == m.val
            self.maxBal = b

    def phase2b(self, b, v):
        # bmsgs := bmsgs U ("2b", m.bal, m.val, acceptor)
        if int(self.maxBal) <= int(b): 
            # exists (2av, b) pairing in sentMsgs that quorum of acceptors agree upon
            
            self.maxVVal = v
            self.maxBal = b
            self.maxVBal = b

            self.bmsgs.append(m.val) # saving state update values for bringing peers up-to-date

            self.logger.info(self.bmsgs)
            self.state.update(v, b)
            tosend = "2b&{}&{}&{}".format(str(time.time()), b, v)
            return tosend

    @asyncio.coroutine
    def main_loop(self, websocket, path):
        """
        server socket
        """ 
        self.logger.debug("main loop")
        # idle until receives network input
        try:
            input_msg = yield from websocket.recv()
            self.logger.debug("< {}".format(input_msg))
            output_msg = self.handle_msg(input_msg)
            if output_msg:
                yield from websocket.send(output_msg)
                self.bmsgs.append(output_msg)
                #print(self.bmsgs)
            else:
                self.logger.error("got bad input from peer")
        except Exception as e:
            self.logger.error("mainloop exception: {}".format(e))
        self.logger.debug("Pending tasks after mainloop: %i" % len(asyncio.Task.all_tasks(asyncio.get_event_loop())))    


    def preprocess_msg(self, msg): # TODO verification
        return msg.split('&')

    def handle_msg(self, msg, peer_wss=None):    
        msg_type = msg[:2]
        if msg_type == "1c":
            msg,proofs = msg.split('&;')
        parts = self.preprocess_msg(msg)
        num_parts = len(parts)
        timestamp = parts[1]
        N = int(parts[2])

        
        if msg_type == "1a" and num_parts == 3:
            # a peer is leader for a ballot, requesting votes
            output_msg = self.phase1b(N)
            return output_msg
            
        elif msg_type == "1b" and num_parts == 6:
            # implies is leader for ballot, has quorum object
            self.logger.debug("got 1b!!!")
            [mb,mv,avs] = parts[3:6]
            # do stuff with v and 2avs
            # update avs structure here for newest ballot number for value
            if N == self.Q.N:
                self.Q.add_1b(int(mb), msg, peer_wss)
            else:
                self.logger.error("got bad 1b msg")
                 
        elif msg_type == "1c" and num_parts == 4:
            self.logger.debug("got 1c")
            v = parts[3]
            signed_msgs = proofs.split(',') # 1b proofs in wss;msg;sig format
            if len(signed_msgs) <= self.peers.num_peers:
                # test against pubkey
                self.logger.debug("testing sig here...")
                num_verified = self.peers.verify_sigs(signed_msgs)
                
                if num_verified >= self.peers.quorum_size():
                    self.logger.debug("signature verification succeeded")
                    output_msg = self.phase2b(N,v)
                    return output_msg
                else:
                    self.logger.error("signature verification failed")
            else:
                self.logger.error("too many signatures")
 
        elif msg_type == "2b" and num_parts == 4:
            self.logger.debug("got 2b")
            self.Q.add_2b(N)
        else:
            self.logger.info("non-paxos msg received")

    @asyncio.coroutine
    def send_msg(self, to_send, recipient_list, handler_function=None):
        """
        client socket 

        returns list of latencies
        """
        good_peers = 0
        peer_latencies = {}
        input_msg = None
        for ws in recipient_list:
            send_start = time.time()
            try:
                #self.logger.debug("sending {} to {}".format(to_send, ws))
                client_socket = yield from websockets.connect(ws, ssl=self.ctx)
                self.logger.debug("to_send: {}".format(to_send))
                yield from client_socket.send(to_send)
               
                input_msg = yield from client_socket.recv()
                peer_latencies[ws] = time.time() - send_start
                self.logger.debug("input_msg: {}".format(input_msg))
                if handler_function:
                    handler_function(input_msg, ws)
                good_peers += 1
            except Exception as e: # custom error handling
                self.logger.debug("send to peer {} failed: {}".format(ws,e))
                peer_latencies[ws] = -1
            finally:
                try:
                    yield from client_socket.close()
                except Exception as e:
                    self.logger.debug("no socket to close")

        self.logger.info("Peers: {}/{}".format(good_peers, len(recipient_list)))     
        self.logger.info("Peer Latencies: {}".format(peer_latencies))
        
        if not handler_function: # TODO what is this for?
            return input_msg