def _on_supplier_connector_state_changed(self, idurl, newstate, **kwargs): from customer import supplier_connector idurl = id_url.field(idurl) if _Debug: lg.out( _DebugLevel, 'fire_hire._on_supplier_connector_state_changed %s to %s, own state is %s ' % (idurl, newstate, self.state)) if supplier_connector.by_idurl(idurl): supplier_connector.by_idurl(idurl).remove_callback( 'fire_hire', self._on_supplier_connector_state_changed) if self.state == 'SUPPLIERS?': if idurl in self.connect_list: self.connect_list.remove(idurl) else: lg.warn('did not found %r in connect_list' % idurl) elif self.state == 'FIRE_MANY': if idurl not in self.dismiss_results: self.dismiss_results.append(idurl) else: lg.warn('did not found %r in dismiss_results' % idurl) else: return self.automat('supplier-state-changed', ( idurl, newstate, ))
def _on_supplier_connector_state_changed(self, idurl, newstate): lg.out( 14, 'fire_hire._supplier_connector_state_changed %s to %s, own state is %s' % (idurl, newstate, self.state)) supplier_connector.by_idurl(idurl).remove_callback( 'shared_access_coordinator') if newstate == 'CONNECTED': self.automat('supplier-connected', idurl)
def _on_supplier_connector_state_changed(self, idurl, newstate): lg.out(14, 'fire_hire._on_supplier_connector_state_changed %s to %s, own state is %s' % ( idurl, newstate, self.state)) supplier_connector.by_idurl(idurl).remove_callback('fire_hire') if self.state == 'SUPPLIERS?': self.connect_list.remove(idurl) elif self.state == 'FIRE_MANY': self.dismiss_results.append(idurl) else: return self.automat('supplier-state-changed', (idurl, newstate))
def doConnectCustomerSuppliers(self, *args, **kwargs): """ Action method. """ try: self.known_suppliers_list = [ _f for _f in args[0]['suppliers'] if _f ] except: lg.exc() return self.known_ecc_map = args[0].get('ecc_map') for supplier_idurl in self.known_suppliers_list: sc = supplier_connector.by_idurl( supplier_idurl, customer_idurl=self.customer_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl, # we only want to read the data at the moment, # so requesting 0 bytes from that supplier needed_bytes=0, key_id=self.key_id, queue_subscribe=False, ) if sc.state in [ 'CONNECTED', 'QUEUE?', ]: self.automat('supplier-connected', supplier_idurl) else: sc.set_callback('shared_access_coordinator', self._on_supplier_connector_state_changed) sc.automat('connect')
def doDisconnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status if _Debug: lg.out(_DebugLevel, 'fire_hire.doDisconnectSuppliers %r' % self.dismiss_list) self.dismiss_results = [] for supplier_idurl in self.dismiss_list: sc = supplier_connector.by_idurl(supplier_idurl) if sc: sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) sc.automat( 'disconnect', ecc_map=eccmap.Current().name, ) else: lg.warn('supplier_connector must exist, but not found %s' % supplier_idurl) online_status.remove_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if position is None or position == -1: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): supplier_idurl = current_suppliers[i].to_bin() if not supplier_idurl: position = i break if id_url.is_in(supplier_idurl, fire_hire.A().dismiss_list, as_field=False): position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getIDURL(), ) sc.set_callback('supplier_finder', self._supplier_connector_state) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, )
def _do_retrieve(self, x=None): packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) localID = my_id.getIDURL() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue pkt_out = p2p_service.SendRetreive(ownerID=localID, creatorID=localID, packetID=packetID, remoteID=supplier_idurl, response_timeout=60 * 2, callbacks={ commands.Data(): self._on_supplier_response, commands.Fail(): self._on_supplier_fail, }) if pkt_out: self.requesting_suppliers.add(supplier_idurl) self.requested_suppliers_number += 1 self.requests_packets_sent.append((packetID, supplier_idurl)) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplier_idurl)))
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range(settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if not position: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if current_suppliers[i] in fire_hire.A().dismiss_list: position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getLocalID(), ) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, ) sc.set_callback('supplier_finder', self._supplier_connector_state)
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status self.connect_list = [] my_current_family = contactsdb.suppliers() for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue if self.configs[0] and pos >= self.configs[0]: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getIDURL(), ) else: sc.needed_bytes = None sc.do_calculate_needed_bytes() sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(my_current_family), ) online_status.add_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ self.connect_list = [] my_current_family = list(contactsdb.suppliers()) for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getLocalID(), ) sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=my_current_family, ) supplier_contact_status = contact_status.getInstance( supplier_idurl) if supplier_contact_status: supplier_contact_status.addStateChangedCallback( self._on_supplier_contact_status_state_changed, newstate='OFFLINE', )
def _on_my_list_files_refreshed(self, evt): from access import shared_access_coordinator from customer import supplier_connector from p2p import p2p_service from p2p import commands for key_id in shared_access_coordinator.list_active_shares(): cur_share = shared_access_coordinator.get_active_share(key_id) if not cur_share: continue if cur_share.state != 'CONNECTED': continue for supplier_idurl in cur_share.known_suppliers_list: sc = supplier_connector.by_idurl( supplier_idurl, customer_idurl=cur_share.customer_idurl, ) if sc is not None and sc.state == 'CONNECTED': p2p_service.SendListFiles( target_supplier=supplier_idurl, customer_idurl=cur_share.customer_idurl, key_id=cur_share.key_id, callbacks={ commands.Files(): lambda r, i: self._on_list_files_response( r, i, cur_share.customer_idurl, supplier_idurl, cur_share.key_id), commands.Fail(): lambda r, i: self._on_list_files_failed( r, i, cur_share.customer_idurl, supplier_idurl, cur_share.key_id), })
def _do_connect_with_supplier(self, supplier_idurl): if _Debug: lg.args(_DebugLevel, supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl) sc = supplier_connector.by_idurl(supplier_idurl, customer_idurl=self.customer_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl, needed_bytes= 0, # we only want to read the data at the moment - requesting 0 bytes from the supplier key_id=self.key_id, queue_subscribe=False, ) if sc.state in [ 'CONNECTED', 'QUEUE?', ]: self.automat('supplier-connected', supplier_idurl) else: sc.set_callback('shared_access_coordinator', self._on_supplier_connector_state_changed) sc.automat('connect')
def start(self): from customer import supplier_connector from contacts import contactsdb for supplier_idurl in contactsdb.suppliers(): if supplier_idurl and not supplier_connector.by_idurl( supplier_idurl): supplier_connector.create(supplier_idurl) return True
def _supplier_acked(self, newpacket, info): from customer import supplier_connector self.sentSuppliers.discard(newpacket.OwnerID) self.automat('db-info-acked', newpacket.OwnerID) sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: sc.automat(newpacket.Command.lower(), newpacket) else: raise Exception('not found supplier connector')
def doCheckAllConnected(self, *args, **kwargs): """ Action method. """ for supplier_idurl in self.known_suppliers_list: sc = supplier_connector.by_idurl(supplier_idurl, customer_idurl=self.customer_idurl) if sc is None or sc.state != 'CONNECTED': return self.automat('all-suppliers-connected')
def isAnySupplierConnected(self, *args, **kwargs): """ Condition method. """ for supplier_idurl in self.known_suppliers_list: sc = supplier_connector.by_idurl(supplier_idurl, customer_idurl=self.customer_idurl) if sc is not None and sc.state == 'CONNECTED': return True return False
def _on_supplier_connector_state_changed(self, idurl, newstate, **kwargs): if _Debug: lg.out(_DebugLevel, 'shared_access_coordinator._supplier_connector_state_changed %s to %s, own state is %s' % ( idurl, newstate, self.state)) sc = supplier_connector.by_idurl(idurl) if sc: sc.remove_callback('shared_access_coordinator') if newstate == 'CONNECTED': self.automat('supplier-connected', idurl)
def doCleanPrevUser(self, arg): """ Action method. """ from customer import supplier_connector sc = supplier_connector.by_idurl(self.target_idurl) if sc: sc.remove_callback('supplier_finder') self.target_idurl = None
def doCloseConnectors(self, arg): """ Action method. """ for supplier_idurl in self.dismiss_list: sc = supplier_connector.by_idurl(supplier_idurl) if supplier_idurl in self.dismiss_list: self.dismiss_list.remove(supplier_idurl) if sc: sc.automat('shutdown')
def doCloseConnectors(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector for supplier_idurl in self.dismiss_list: sc = supplier_connector.by_idurl(supplier_idurl) if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): self.dismiss_list.remove(supplier_idurl) if sc: sc.automat('shutdown')
def doCleanPrevUser(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector if id_url.is_cached(self.target_idurl): sc = supplier_connector.by_idurl(self.target_idurl) if sc: sc.remove_callback('supplier_finder', self._supplier_connector_state) self.target_idurl = None
def doSupplierConnect(self, arg): """ Action method. """ from customer import supplier_connector sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create(self.target_idurl, customer_idurl=my_id.getLocalID()) sc.automat('connect') sc.set_callback('supplier_finder', self._supplier_connector_state)
def doCloseConnector(self, arg): """ Action method. """ supplier_idurl, _ = arg sc = supplier_connector.by_idurl(supplier_idurl) if supplier_idurl in self.dismiss_list: self.dismiss_list.remove(supplier_idurl) if sc: sc.automat('shutdown') else: raise Exception('supplier_connector must exist')
def _on_supplier_acked(self, newpacket, info): sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: sc.automat(newpacket.Command.lower(), newpacket) else: raise Exception('not found supplier connector') self.sending_suppliers.discard(newpacket.OwnerID) if _Debug: lg.out(_DebugLevel, 'index_synchronizer._on_supplier_acked %s, pending: %d, total: %d' % ( newpacket, len(self.sending_suppliers), self.sent_suppliers_number)) if len(self.sending_suppliers) == 0: self.automat('all-acked')
def OnFileSendAckReceived(self, newpacket, info): if self.shutdown: if _Debug: lg.out( _DebugLevel, "io_throttle.OnFileSendAckReceived finishing to %s, shutdown is True" % self.remoteName) return if not newpacket and not info: lg.warn('packet timed out during responding') return self.ackedCount += 1 packetID = global_id.CanonicalID(newpacket.PacketID) if packetID not in self.fileSendQueue: lg.warn("packet %s not in sending queue for %s" % (newpacket.PacketID, self.remoteName)) return if packetID not in list(self.fileSendDict.keys()): lg.warn("packet %s not in sending dict for %s" % (newpacket.PacketID, self.remoteName)) return self.fileSendDict[packetID].ackTime = time.time() if newpacket.Command == commands.Ack(): self.fileSendDict[packetID].result = 'acked' if self.fileSendDict[packetID].callOnAck: reactor.callLater(0, self.fileSendDict[packetID].callOnAck, newpacket, newpacket.OwnerID, packetID) # @UndefinedVariable elif newpacket.Command == commands.Fail(): self.fileSendDict[packetID].result = 'failed' if self.fileSendDict[packetID].callOnFail: reactor.callLater(0, self.fileSendDict[packetID].callOnFail, newpacket.CreatorID, packetID, 'failed') # @UndefinedVariable from customer import supplier_connector sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: if newpacket.Command == commands.Ack(): sc.automat('ack', newpacket) elif newpacket.Command == commands.Fail(): sc.automat('fail', newpacket) # elif newpacket.Command == commands.Data(): # sc.automat('data', newpacket) else: raise Exception('incorrect packet type received') self.DoSend() # self.RunSend() if _Debug: lg.out( _DebugLevel, "io_throttle.OnFileSendAckReceived %s from %s, queue=%d" % (str(newpacket), self.remoteName, len(self.fileSendQueue)))
def doSuppliersSendIndexFile(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) self.sending_suppliers.clear() self.outgoing_packets_ids = [] self.sent_suppliers_number = 0 localID = my_id.getIDURL() b = encrypted.Block( CreatorID=localID, BackupID=packetID, BlockNumber=0, SessionKey=key.NewSessionKey( session_key_type=key.SessionKeyType()), SessionKeyType=key.SessionKeyType(), LastBlock=True, Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()), ) Payload = b.Serialize() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue newpacket, pkt_out = p2p_service.SendData( raw_data=Payload, ownerID=localID, creatorID=localID, remoteID=supplier_idurl, packetID=packetID, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }, ) if pkt_out: self.sending_suppliers.add(supplier_idurl) self.sent_suppliers_number += 1 self.outgoing_packets_ids.append(packetID) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (newpacket, nameurl.GetName(supplier_idurl)))
def OnFileSendAckReceived(self, newpacket, info): if self.shutdown: if _Debug: lg.out( _DebugLevel, "io_throttle.OnFileSendAckReceived finishing to %s, shutdown is True" % self.remoteName) return if not newpacket and not info: lg.warn('packet timed out during responding') return if _Debug: lg.out(_DebugLevel, "io_throttle.OnFileSendAckReceived with %r" % newpacket) self.ackedCount += 1 packetID = global_id.CanonicalID(newpacket.PacketID) if packetID not in self.fileSendQueue: lg.warn("packet %s not in sending queue for %s" % (newpacket.PacketID, self.remoteName)) return if packetID not in list(self.fileSendDict.keys()): lg.warn("packet %s not in sending dict for %s" % (newpacket.PacketID, self.remoteName)) return f_up = self.fileSendDict[packetID] if newpacket.Command == commands.Ack(): f_up.event('ack-received', newpacket) elif newpacket.Command == commands.Fail(): f_up.event('fail-received', newpacket) else: raise Exception('wrong command received in response: %r' % newpacket) from customer import supplier_connector sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: if newpacket.Command == commands.Ack(): sc.automat('ack', newpacket) elif newpacket.Command == commands.Fail(): sc.automat('fail', newpacket) else: raise Exception('incorrect packet type received: %r' % newpacket) else: lg.warn('supplier connector for %r not found' % newpacket.OwnerID) if _Debug: lg.out( _DebugLevel, "io_throttle.OnFileSendAckReceived %s from %s, queue=%d" % (str(newpacket), self.remoteName, len(self.fileSendQueue)))
def _on_supplier_acked(self, newpacket, info): sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: sc.automat(newpacket.Command.lower(), newpacket) else: raise Exception('not found supplier connector') self.sending_suppliers.discard(newpacket.OwnerID) if _Debug: lg.out( _DebugLevel, 'index_synchronizer._on_supplier_acked %s, pending: %d, total: %d' % (newpacket, len( self.sending_suppliers), self.sent_suppliers_number)) if len(self.sending_suppliers) == 0: self.automat('all-acked')
def doDisconnectSuppliers(self, arg): """ Action method. """ lg.out(10, 'fire_hire.doDisconnectSuppliers %r' % self.dismiss_list) self.dismiss_results = [] for supplier_idurl in self.dismiss_list: sc = supplier_connector.by_idurl(supplier_idurl) if sc: sc.set_callback('fire_hire', self._supplier_connector_state_changed) sc.automat('disconnect') else: lg.warn('supplier_connector must exist, but not found %s' % supplier_idurl)
def start(self): from contacts import contactsdb from crypt import my_keys from customer import supplier_connector from userid import my_id from logs import lg customer_key_id = my_id.getGlobalID(key_alias='customer') if not my_keys.is_key_registered(customer_key_id): lg.warn('customer key was not found, generate new key: %s' % customer_key_id) my_keys.generate_key(customer_key_id) for supplier_idurl in contactsdb.suppliers(): if supplier_idurl and not supplier_connector.by_idurl(supplier_idurl, customer_idurl=my_id.getLocalID()): supplier_connector.create(supplier_idurl, customer_idurl=my_id.getLocalID()) # TODO: read from dht and connect to other suppliers - from other customers who shared data to me return True
def doDestroyMe(self, *args, **kwargs): """ Remove all references to the state machine object to destroy it. """ from customer import supplier_connector global _SupplierFinder del _SupplierFinder _SupplierFinder = None if self.target_idurl: sc = supplier_connector.by_idurl(self.target_idurl) if sc: sc.remove_callback('supplier_finder', self._supplier_connector_state) self.target_idurl = None self.destroy()
def _supplier_response(self, newpacket, pkt_out): from customer import supplier_connector if newpacket.Command == commands.Data(): self.requestedSuppliers.discard(newpacket.RemoteID) elif newpacket.Command == commands.Fail(): self.requestedSuppliers.discard(newpacket.OwnerID) sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: sc.automat('fail', newpacket) else: raise Exception('supplier connector was not found') else: raise Exception('wrong type of response') if len(self.requestedSuppliers) == 0: self.automat('all-responded')
def doConnectSuppliers(self, arg): """ Action method. """ self.connect_list = [] for supplier_idurl in contactsdb.suppliers(): if supplier_idurl == '': continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create(supplier_idurl) sc.set_callback('fire_hire', self._supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat('connect')
def _on_supplier_response(self, newpacket, pkt_out): if newpacket.Command == commands.Data(): self.requesting_suppliers.discard(newpacket.RemoteID) elif newpacket.Command == commands.Fail(): self.requesting_suppliers.discard(newpacket.OwnerID) sc = supplier_connector.by_idurl(newpacket.OwnerID) if sc: sc.automat('fail', newpacket) else: raise Exception('supplier connector was not found') else: raise Exception('wrong type of response') if _Debug: lg.out(_DebugLevel, 'index_synchronizer._on_supplier_response %s, pending: %d, total: %d' % ( newpacket, len(self.requesting_suppliers), self.requested_suppliers_number)) if len(self.requesting_suppliers) == 0: self.automat('all-responded')