def SupplierSendFile(self, fileName, packetID, ownerID, callOnAck=None, callOnFail=None): if self.shutdown: if _Debug: lg.out(_DebugLevel, "io_throttle.SupplierSendFile finishing to %s, shutdown is True" % self.remoteName) if callOnFail is not None: reactor.callLater(0, callOnFail, self.remoteID, packetID, 'shutdown') # @UndefinedVariable return False if online_status.isOffline(self.remoteID): if _Debug: lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s is offline, so packet %s is failed" % ( self.remoteName, packetID)) if callOnFail is not None: reactor.callLater(0, callOnFail, self.remoteID, packetID, 'offline') # @UndefinedVariable return False if packetID in self.fileSendQueue: lg.warn("packet %s already in the queue for %s" % (packetID, self.remoteName)) if callOnFail is not None: reactor.callLater(0, callOnFail, self.remoteID, packetID, 'in queue') # @UndefinedVariable return False from stream import file_up f_up = file_up.FileUp( self, fileName, packetID, self.remoteID, ownerID, callOnAck, callOnFail, ) f_up.event('init') if _Debug: lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s to %s, %d queued items" % ( packetID, self.remoteName, len(self.fileSendQueue))) self.DoSend() return True
def _do_retrieve(self, x=None): packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) localID = my_id.getIDURL() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue pkt_out = p2p_service.SendRetreive(ownerID=localID, creatorID=localID, packetID=packetID, remoteID=supplier_idurl, response_timeout=60 * 2, callbacks={ commands.Data(): self._on_supplier_response, commands.Fail(): self._on_supplier_fail, }) if pkt_out: self.requesting_suppliers.add(supplier_idurl) self.requested_suppliers_number += 1 self.requests_packets_sent.append((packetID, supplier_idurl)) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplier_idurl)))
def doSuppliersRequestIndexFile(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersRequestIndexFile') if driver.is_on('service_backups'): from storage import backup_control self.current_local_revision = backup_control.revision() else: self.current_local_revision = -1 self.latest_supplier_revision = -1 self.requesting_suppliers.clear() self.requested_suppliers_number = 0 packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) # packetID = settings.BackupIndexFileName() localID = my_id.getLocalIDURL() for supplierId in contactsdb.suppliers(): if not supplierId: continue if online_status.isOffline(supplierId): continue pkt_out = p2p_service.SendRetreive(localID, localID, packetID, supplierId, callbacks={ commands.Data(): self._on_supplier_response, commands.Fail(): self._on_supplier_response, }) # newpacket = signed.Packet( # commands.Retrieve(), # localID, # localID, # packetid.RemotePath(packetID), # '', # supplierId) # pkt_out = gateway.outbox(newpacket, callbacks={ # commands.Data(): self._on_supplier_response, # commands.Fail(): self._on_supplier_response, }) if pkt_out: self.requesting_suppliers.add(supplierId) self.requested_suppliers_number += 1 if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) self.sending_suppliers.clear() self.outgoing_packets_ids = [] self.sent_suppliers_number = 0 localID = my_id.getIDURL() b = encrypted.Block( CreatorID=localID, BackupID=packetID, BlockNumber=0, SessionKey=key.NewSessionKey( session_key_type=key.SessionKeyType()), SessionKeyType=key.SessionKeyType(), LastBlock=True, Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()), ) Payload = b.Serialize() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue newpacket, pkt_out = p2p_service.SendData( raw_data=Payload, ownerID=localID, creatorID=localID, remoteID=supplier_idurl, packetID=packetID, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }, ) if pkt_out: self.sending_suppliers.add(supplier_idurl) self.sent_suppliers_number += 1 self.outgoing_packets_ids.append(packetID) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (newpacket, nameurl.GetName(supplier_idurl)))
def doSuppliersSendIndexFile(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) self.sending_suppliers.clear() self.sent_suppliers_number = 0 localID = my_id.getLocalIDURL() b = encrypted.Block( CreatorID=localID, BackupID=packetID, BlockNumber=0, SessionKey=key.NewSessionKey(), SessionKeyType=key.SessionKeyType(), LastBlock=True, Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()), ) Payload = b.Serialize() for supplierId in contactsdb.suppliers(): if not supplierId: continue if online_status.isOffline(supplierId): continue newpacket, pkt_out = p2p_service.SendData( raw_data=Payload, ownerID=localID, creatorID=localID, remoteID=supplierId, packetID=packetID, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }, ) # newpacket = signed.Packet( # commands.Data(), localID, localID, packetID, # Payload, supplierId) # pkt_out = gateway.outbox(newpacket, callbacks={ # commands.Ack(): self._on_supplier_acked, # commands.Fail(): self._on_supplier_acked, }) if pkt_out: self.sending_suppliers.add(supplierId) self.sent_suppliers_number += 1 if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (newpacket, nameurl.GetName(supplierId)))
def doDecideToDismiss(self, *args, **kwargs): """ Action method. """ global _SuppliersToFire from p2p import p2p_connector from p2p import network_connector from customer import supplier_connector from p2p import online_status # take any actions only if I am connected to the network if not p2p_connector.A() or not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if p2p_connector.A().state != 'CONNECTED' or network_connector.A( ).state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP' ) self.automat('made-decision', []) return # if certain suppliers needs to be removed by manual/external request just do that to_be_fired = id_url.to_list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.info('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return # make sure to not go too far when i just want to decrease number of my suppliers number_desired = settings.getSuppliersNumberDesired() redundant_suppliers = set() if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.info('found REDUNDANT supplier %s at position %d' % ( idurl, supplier_index, )) redundant_suppliers.add(idurl) if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return # now I need to look more careful at my suppliers potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. my_suppliers = contactsdb.suppliers() if _Debug: lg.args(_DebugLevel, my_suppliers=my_suppliers) if id_url.is_some_empty(my_suppliers): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return for supplier_idurl in my_suppliers: sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if online_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif online_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif online_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if not disconnected_suppliers: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment' ) # TODO: add more conditions to fire "slow" suppliers - they are still connected but useless self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % (offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return max_offline_suppliers_count = eccmap.GetCorrectableErrors( number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn( 'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors( number_desired) if len(offline_suppliers) >= critical_offline_suppliers_count and len( offline_suppliers) > 0: if config.conf().getBool( 'services/employer/replace-critically-offline-enabled'): # TODO: check that issue # too aggressive replacing suppliers who still have the data is very dangerous !!! one_dead_supplier = offline_suppliers.pop() lg.warn( 'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!' ) self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [ result[0], ])
def _do_check_run_requests(self): if _Debug: lg.out( _DebugLevel, 'restore_worker._do_check_run_requests for %s at block %d' % ( self.backup_id, self.block_number, )) from stream import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data') if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandData is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) for SupplierNumber in range(self.EccMap.paritysegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity') if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandParity is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) requests_made = 0 # already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): # already_requested += 1 # if packetID not in self.AlreadyRequestedCounts: # self.AlreadyRequestedCounts[packetID] = 0 # self.AlreadyRequestedCounts[packetID] += 1 lg.warn('packet already in IO queue for supplier %s : %s' % ( SupplierID, packetID, )) continue self.block_requests[packetID] = None if io_throttle.QueueRequestFile( callOnReceived=self._on_packet_request_result, creatorID=self.creator_id, packetID=packetID, ownerID=self.creator_id, # self.customer_idurl, remoteID=SupplierID, ): requests_made += 1 else: self.block_requests[packetID] = False if _Debug: lg.dbg( _DebugLevel, 'sent request %r to %r, other requests: %r' % (packetID, SupplierID, list(self.block_requests.values()))) del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % ( requests_made, self.block_number, )) return current_block_requests_results = list(self.block_requests.values()) if _Debug: lg.args(_DebugLevel, current_results=current_block_requests_results) pending_count = current_block_requests_results.count(None) if pending_count > 0: if _Debug: lg.out( _DebugLevel, " nothing for request, currently %d pending packets for block %d" % ( pending_count, self.block_number, )) return failed_count = current_block_requests_results.count(False) if failed_count > self.max_errors: lg.err( 'all requests finished and %d packets failed, not possible to read data for block %d' % ( failed_count, self.block_number, )) reactor.callLater(0, self.automat, 'request-failed', None) # @UndefinedVariable return if _Debug: lg.out( _DebugLevel, " all requests finished for block %d : %r" % ( self.block_number, current_block_requests_results, )) reactor.callLater(0, self.automat, 'request-finished', None) # @UndefinedVariable
def doDecideToDismiss(self, *args, **kwargs): """ Action method. """ global _SuppliersToFire to_be_fired = list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.warn('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() redundant_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers() or None in contactsdb.suppliers() or b'' in contactsdb.suppliers(): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return number_desired = settings.getSuppliersNumberDesired() for supplier_idurl in contactsdb.suppliers(): sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if online_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif online_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif online_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.warn('found "REDUNDANT" supplier %s at position %d' % ( idurl, supplier_index, )) potentialy_fired.add(idurl) redundant_suppliers.add(idurl) else: lg.warn('supplier at position %d not exist' % supplier_index) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return if not disconnected_suppliers: lg.warn('SKIP, no OFFLINE suppliers found at the moment') # TODO: add more conditions to fire "slow" suppliers self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % ( offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return max_offline_suppliers_count = eccmap.GetCorrectableErrors(number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn('SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors(number_desired) if len(offline_suppliers) >= critical_offline_suppliers_count and len(offline_suppliers) > 0: # TODO: check that issue # too aggressive replacing suppliers who still have the data one_dead_supplier = offline_suppliers.pop() lg.warn('found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: lg.out(6, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!') self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [result[0], ])
def doRequestPackets(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'restore_worker.doRequestPackets for %s at block %d' % ( self.backup_id, self.block_number, )) from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandData is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandParity is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity'))) if _Debug: lg.out(_DebugLevel, ' packets to request: %s' % packetsToRequest) requests_made = 0 already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): already_requested += 1 if packetID not in self.AlreadyRequestedCounts: self.AlreadyRequestedCounts[packetID] = 0 self.AlreadyRequestedCounts[packetID] += 1 if _Debug: lg.out( _DebugLevel, ' packet already in request queue: %s %s' % ( SupplierID, packetID, )) continue io_throttle.QueueRequestFile(self._on_packet_request_result, self.creator_id, packetID, self.creator_id, SupplierID) requests_made += 1 del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % (requests_made, self.block_number)) else: if not already_requested: lg.warn('no requests made for block %d' % self.block_number) self.automat('request-failed', None) else: if _Debug: lg.out( _DebugLevel, " found %d already requested packets for block %d" % (already_requested, self.block_number)) if self.AlreadyRequestedCounts: all_counts = sorted(self.AlreadyRequestedCounts.values()) if all_counts[0] > 100: lg.warn('too much requests made for block %d' % self.block_number) self.automat('request-failed', None)