def doRequestPackets(self, arg): from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: continue if not self.OnHandData[SupplierNumber] and contact_status.isOnline( SupplierID): packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: continue if not self.OnHandParity[ SupplierNumber] and contact_status.isOnline(SupplierID): packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity'))) for SupplierID, packetID in packetsToRequest: io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID, packetID, self.CreatorID, SupplierID) lg.out( 6, "restore.doRequestPackets requested %d packets for block %d" % (len(packetsToRequest), self.BlockNumber)) del packetsToRequest self.automat('request-done')
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range(settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier') position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if supplier_idurl in self.dismiss_list: lg.info('going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, wide=False, callbacks={}): """ This is used as a request method from your supplier : if you send him a ListFiles() packet he will reply you with a list of stored files in a Files() packet. """ MyID = my_id.getLocalID() if not customer_idurl: customer_idurl = MyID if not str(target_supplier).isdigit(): RemoteID = target_supplier else: RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl) if not RemoteID: lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier)) return None if _Debug: lg.out(_DebugLevel, "p2p_service.SendListFiles to %s" % nameurl.GetName(RemoteID)) if not key_id: key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer') PacketID = "%s:%s" % (key_id, packetid.UniqueID(), ) Payload = settings.ListFilesFormat() result = signed.Packet( Command=commands.ListFiles(), OwnerID=MyID, CreatorID=MyID, PacketID=PacketID, Payload=Payload, RemoteID=RemoteID, ) gateway.outbox(result, wide=wide, callbacks=callbacks) return result
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range( settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def _send(index, payload, delay): global _SlowSendIsWorking idurl = contactsdb.supplier(index, customer_idurl=customer_idurl) if not idurl: _SlowSendIsWorking = False return # transport_control.ClearAliveTime(idurl) SendToID(idurl, Payload=payload, wide=True) reactor.callLater(delay, _send, index + 1, payload, delay)
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r' % (settings.getSuppliersNumberDesired(), contactsdb.suppliers())) from p2p import network_connector if network_connector.A().state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, ' network_connector is not CONNECTED at the moment, SKIP' ) self.automat('search-failed') return position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info( 'found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): lg.info( 'going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return from customer import supplier_finder for idurl_txt in strng.to_text(config.conf().getData( 'services/employer/candidates')).split(','): if idurl_txt.strip(): supplier_finder.AddSupplierToHire(idurl_txt) self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(contactsdb.suppliers()), )
def isRequestQueueEmpty(self, arg): """ Condition method. """ from customer import io_throttle # supplierSet = backup_matrix.suppliers_set() for supplierNum in range(contactsdb.num_suppliers()): supplierID = contactsdb.supplier(supplierNum) if io_throttle.HasBackupIDInRequestQueue( supplierID, self.currentBackupID): return False return True
def doRemoveTempFile(self, *args, **kwargs): """ Action method. """ if not args or not len(args) > 1: return filename = args[1] if filename: tmpfile.throw_out(filename, 'block restored') if settings.getBackupsKeepLocalCopies(): return from storage import backup_matrix from storage import backup_rebuilder if not backup_rebuilder.ReadStoppedFlag(): if backup_rebuilder.A().currentBackupID is not None: if backup_rebuilder.A().currentBackupID == self.backup_id: if _Debug: lg.out( _DebugLevel, 'restore_worker.doRemoveTempFile SKIP because rebuilding in process' ) return count = 0 for supplierNum in range( contactsdb.num_suppliers(customer_idurl=self.customer_idurl)): supplierIDURL = contactsdb.supplier( supplierNum, customer_idurl=self.customer_idurl) if not supplierIDURL: continue for dataORparity in [ 'Data', 'Parity', ]: packetID = packetid.MakePacketID(self.backup_id, self.block_number, supplierNum, dataORparity) customer, remotePath = packetid.SplitPacketID(packetID) filename = os.path.join(settings.getLocalBackupsDir(), customer, remotePath) if os.path.isfile(filename): try: os.remove(filename) except: lg.exc() continue count += 1 backup_matrix.LocalBlockReport(self.backup_id, self.block_number, *args, **kwargs) if _Debug: lg.out( _DebugLevel, 'restore_worker.doRemoveTempFile %d files were removed' % count)
def SuppliersChangedNumbers(oldSupplierList): """ Return list of positions of changed suppliers, say if suppliers 1 and 3 were changed it should return [1,3]. """ changedList = [] for i in xrange(len(oldSupplierList)): suplier_idurl = oldSupplierList[i] if not suplier_idurl: continue if contactsdb.supplier(i) != suplier_idurl: changedList.append(i) return changedList
def ScanBlocksToRemove(backupID, check_all_suppliers=True): """ This method compare both matrixes and found pieces which is present on both sides. If remote supplier got that file it can be removed from the local HDD. """ from customer import io_throttle lg.out(10, "backup_matrix.ScanBlocksToRemove for %s" % backupID) packets = [] localMaxBlockNum = local_max_block_numbers().get(backupID, -1) if backupID not in remote_files() or backupID not in local_files(): # no info about this backup yet - skip return packets for blockNum in xrange(localMaxBlockNum + 1): localArray = {"Data": GetLocalDataArray(backupID, blockNum), "Parity": GetLocalParityArray(backupID, blockNum)} remoteArray = { "Data": GetRemoteDataArray(backupID, blockNum), "Parity": GetRemoteParityArray(backupID, blockNum), } if (0 in remoteArray["Data"]) or (0 in remoteArray["Parity"]): # if some supplier do not have some data for that block - do not remove any local files for that block! # we do remove the local files only when we sure all suppliers got the all data pieces continue if (-1 in remoteArray["Data"]) or (-1 in remoteArray["Parity"]): # also if we do not have any info about this block for some supplier do not remove other local pieces continue for supplierNum in xrange(contactsdb.num_suppliers()): supplierIDURL = contactsdb.supplier(supplierNum) if not supplierIDURL: # supplier is unknown - skip continue for dataORparity in ["Data", "Parity"]: packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity) if io_throttle.HasPacketInSendQueue(supplierIDURL, packetID): # if we do sending the packet at the moment - skip continue if localArray[dataORparity][supplierNum] == 1: packets.append(packetID) # lg.out(10, ' mark to remove %s, blockNum:%d remote:%s local:%s' % (packetID, blockNum, str(remoteArray), str(localArray))) # if check_all_suppliers: # if localArray[dataORparity][supplierNum] == 1: # packets.append(packetID) # else: # if remoteArray[dataORparity][supplierNum] == 1 and localArray[dataORparity][supplierNum] == 1: # packets.append(packetID) return packets
def SendRequestListFiles(supplierNumORidurl): if not str(supplierNumORidurl).isdigit(): RemoteID = supplierNumORidurl else: RemoteID = contactsdb.supplier(supplierNumORidurl) if not RemoteID: lg.warn("RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl)) return None if _Debug: lg.out(_DebugLevel, "p2p_service.SendRequestListFiles [%s]" % nameurl.GetName(RemoteID)) MyID = my_id.getLocalID() PacketID = packetid.UniqueID() Payload = settings.ListFilesFormat() result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID) gateway.outbox(result) return result
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, query_items=[], wide=False, callbacks={}, timeout=None): """ This is used as a request method from your supplier : if you send him a ListFiles() packet he will reply you with a list of stored files in a Files() packet. """ MyID = my_id.getLocalID() if not customer_idurl: customer_idurl = MyID if not str(target_supplier).isdigit(): RemoteID = target_supplier else: RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl) if not RemoteID: lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier)) return None if not key_id: # key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer') # TODO: due to issue with "customer" key backup/restore decided to always use my "master" key # to retrieve my list files info from supplier # expect remote user always poses my master public key from my identity. # probably require more work to build more reliable solution without using my master key at all # when my identity rotated supplier first needs to receive my new identity and then sending ListFiles() key_id = my_id.getGlobalID(key_alias='master') if not my_keys.is_key_registered(key_id) or not my_keys.is_key_private(key_id): lg.warn('key %r not exist or public, my "master" key to be used with ListFiles() packet' % key_id) key_id = my_id.getGlobalID(key_alias='master') PacketID = "%s:%s" % (key_id, packetid.UniqueID(), ) if not query_items: query_items = ['*', ] Payload = serialization.DictToBytes({'items': query_items, }) if _Debug: lg.out(_DebugLevel, "p2p_service.SendListFiles %r to %s with query : %r" % ( PacketID, nameurl.GetName(RemoteID), query_items, )) result = signed.Packet( Command=commands.ListFiles(), OwnerID=MyID, CreatorID=MyID, PacketID=PacketID, Payload=Payload, RemoteID=RemoteID, ) gateway.outbox(result, wide=wide, callbacks=callbacks, response_timeout=timeout) return result
def GetActiveArray(): """ Loops all suppliers and returns who is alive at the moment. Return a list with integers: 0 for offline suppler and 1 if he is available right now. Uses ``p2p.contact_status.isOnline()`` to see the current state of supplier. """ from p2p import contact_status activeArray = [0] * contactsdb.num_suppliers() for i in xrange(contactsdb.num_suppliers()): suplier_idurl = contactsdb.supplier(i) if not suplier_idurl: continue if contact_status.isOnline(suplier_idurl): activeArray[i] = 1 else: activeArray[i] = 0 return activeArray
def doRemoveTempFile(self, arg): try: filename = arg[1] except: return tmpfile.throw_out(filename, 'block restored') if settings.getBackupsKeepLocalCopies(): return import backup_rebuilder import backup_matrix if not backup_rebuilder.ReadStoppedFlag(): if backup_rebuilder.A().currentBackupID is not None: if backup_rebuilder.A().currentBackupID == self.BackupID: lg.out( 6, 'restore.doRemoveTempFile SKIP because rebuilding in process' ) return count = 0 for supplierNum in xrange( contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)): supplierIDURL = contactsdb.supplier( supplierNum, customer_idurl=self.CustomerIDURL) if not supplierIDURL: continue for dataORparity in ['Data', 'Parity']: packetID = packetid.MakePacketID(self.BackupID, self.BlockNumber, supplierNum, dataORparity) customer, remotePath = packetid.SplitPacketID(packetID) filename = os.path.join(settings.getLocalBackupsDir(), customer, remotePath) if os.path.isfile(filename): try: os.remove(filename) except: lg.exc() continue count += 1 backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg) lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] disconnected_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) elif sc.state == 'DISCONNECTED': disconnected_suppliers.add(supplier_idurl) if contact_status.isOffline(supplier_idurl): disconnected_suppliers.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range( settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) if disconnected_suppliers: from raid import eccmap if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()): result.update(disconnected_suppliers) result = list(result) lg.out(6, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def SendRequestListFiles(supplierNumORidurl, customer_idurl=None): MyID = my_id.getLocalID() if not customer_idurl: customer_idurl = MyID if not str(supplierNumORidurl).isdigit(): RemoteID = supplierNumORidurl else: RemoteID = contactsdb.supplier(supplierNumORidurl, customer_idurl=customer_idurl) if not RemoteID: lg.warn("RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl)) return None if _Debug: lg.out( _DebugLevel, "p2p_service.SendRequestListFiles [%s]" % nameurl.GetName(RemoteID)) PacketID = "%s:%s" % (global_id.UrlToGlobalID(customer_idurl), packetid.UniqueID()) Payload = settings.ListFilesFormat() result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID) gateway.outbox(result) return result
def doRequestPackets(self, arg): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'restore_worker.doRequestPackets for %s at block %d' % ( self.BackupID, self.BlockNumber, )) from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: lg.warn('bad supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandData is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: lg.warn('bad supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandParity is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity'))) if _Debug: lg.out(_DebugLevel, ' packets to request: %s' % packetsToRequest) requests_made = 0 already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): already_requested += 1 if _Debug: lg.out( _DebugLevel, ' packet already in request queue: %s %s' % ( SupplierID, packetID, )) continue io_throttle.QueueRequestFile(self._on_packet_request_result, self.CreatorID, packetID, self.CreatorID, SupplierID) requests_made += 1 del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % (requests_made, self.BlockNumber)) else: if already_requested: if _Debug: lg.out( _DebugLevel, " found %d already requested packets for block %d" % (already_requested, self.BlockNumber)) else: lg.warn('no requests made for block %d' % self.BlockNumber) self.automat('request-failed', None)
def doRequestPackets(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'restore_worker.doRequestPackets for %s at block %d' % ( self.backup_id, self.block_number, )) from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandData is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandParity is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity'))) if _Debug: lg.out(_DebugLevel, ' packets to request: %s' % packetsToRequest) requests_made = 0 already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): already_requested += 1 if packetID not in self.AlreadyRequestedCounts: self.AlreadyRequestedCounts[packetID] = 0 self.AlreadyRequestedCounts[packetID] += 1 if _Debug: lg.out( _DebugLevel, ' packet already in request queue: %s %s' % ( SupplierID, packetID, )) continue io_throttle.QueueRequestFile(self._on_packet_request_result, self.creator_id, packetID, self.creator_id, SupplierID) requests_made += 1 del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % (requests_made, self.block_number)) else: if not already_requested: lg.warn('no requests made for block %d' % self.block_number) self.automat('request-failed', None) else: if _Debug: lg.out( _DebugLevel, " found %d already requested packets for block %d" % (already_requested, self.block_number)) if self.AlreadyRequestedCounts: all_counts = sorted(self.AlreadyRequestedCounts.values()) if all_counts[0] > 100: lg.warn('too much requests made for block %d' % self.block_number) self.automat('request-failed', None)
def _do_check_run_requests(self): if _Debug: lg.out( _DebugLevel, 'restore_worker._do_check_run_requests for %s at block %d' % ( self.backup_id, self.block_number, )) from stream import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data') if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandData is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) for SupplierNumber in range(self.EccMap.paritysegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity') if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandParity is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) requests_made = 0 # already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): # already_requested += 1 # if packetID not in self.AlreadyRequestedCounts: # self.AlreadyRequestedCounts[packetID] = 0 # self.AlreadyRequestedCounts[packetID] += 1 lg.warn('packet already in IO queue for supplier %s : %s' % ( SupplierID, packetID, )) continue self.block_requests[packetID] = None if io_throttle.QueueRequestFile( callOnReceived=self._on_packet_request_result, creatorID=self.creator_id, packetID=packetID, ownerID=self.creator_id, # self.customer_idurl, remoteID=SupplierID, ): requests_made += 1 else: self.block_requests[packetID] = False if _Debug: lg.dbg( _DebugLevel, 'sent request %r to %r, other requests: %r' % (packetID, SupplierID, list(self.block_requests.values()))) del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % ( requests_made, self.block_number, )) return current_block_requests_results = list(self.block_requests.values()) if _Debug: lg.args(_DebugLevel, current_results=current_block_requests_results) pending_count = current_block_requests_results.count(None) if pending_count > 0: if _Debug: lg.out( _DebugLevel, " nothing for request, currently %d pending packets for block %d" % ( pending_count, self.block_number, )) return failed_count = current_block_requests_results.count(False) if failed_count > self.max_errors: lg.err( 'all requests finished and %d packets failed, not possible to read data for block %d' % ( failed_count, self.block_number, )) reactor.callLater(0, self.automat, 'request-failed', None) # @UndefinedVariable return if _Debug: lg.out( _DebugLevel, " all requests finished for block %d : %r" % ( self.block_number, current_block_requests_results, )) reactor.callLater(0, self.automat, 'request-finished', None) # @UndefinedVariable
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire to_be_fired = list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.warn('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() redundant_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers() or None in contactsdb.suppliers(): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return number_desired = settings.getSuppliersNumberDesired() for supplier_idurl in contactsdb.suppliers(): sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if contact_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif contact_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif contact_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.warn('found "REDUNDANT" supplier %s at position %d' % ( idurl, supplier_index, )) potentialy_fired.add(idurl) redundant_suppliers.add(idurl) else: lg.warn('supplier at position %d not exist' % supplier_index) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return if not disconnected_suppliers: lg.warn('SKIP, no OFFLINE suppliers found at the moment') # TODO: add more conditions to fire "slow" suppliers self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % (offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return from raid import eccmap max_offline_suppliers_count = eccmap.GetCorrectableErrors( number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn( 'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors( number_desired) # TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data if False: # len(offline_suppliers) >= critical_offline_suppliers_count: one_dead_supplier = offline_suppliers.pop() lg.warn( 'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: lg.out( 6, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!' ) self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [ result[0], ])
def _request_files(self): from storage import backup_matrix from customer import io_throttle from customer import data_sender self.missingPackets = 0 # here we want to request some packets before we start working to # rebuild the missed blocks availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL) # remember how many requests we did on this iteration total_requests_count = 0 # at the moment I do download everything I have available and needed if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL): lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier') self.automat('no-requests') return for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)): supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL) if not supplierID: continue requests_count = 0 # we do requests in reverse order because we start rebuilding from # the last block for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1): blockNum = self.workingBlocksQueue[blockIndex] # do not keep too many requests in the queue if io_throttle.GetRequestQueueLength(supplierID) >= 16: break # also don't do too many requests at once if requests_count > 16: break remoteData = backup_matrix.GetRemoteDataArray( self.currentBackupID, blockNum) remoteParity = backup_matrix.GetRemoteParityArray( self.currentBackupID, blockNum) localData = backup_matrix.GetLocalDataArray( self.currentBackupID, blockNum) localParity = backup_matrix.GetLocalParityArray( self.currentBackupID, blockNum) if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity): break if supplierNum >= len(localData) or supplierNum >= len(localParity): break # if remote Data exist and is available because supplier is on-line, # but we do not have it on hand - do request if localData[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Data') if remoteData[supplierNum] == 1: if availableSuppliers[supplierNum]: # if supplier is not alive - we can't request from him if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID): requests_count += 1 else: # count this packet as missing self.missingPackets += 1 # also mark this guy as one who dont have any data - nor local nor remote else: # but if local Data already exists, but was not sent - do it now if remoteData[supplierNum] != 1: data_sender.A('new-data') # same for Parity if localParity[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Parity') if remoteParity[supplierNum] == 1: if availableSuppliers[supplierNum]: if not io_throttle.HasPacketInRequestQueue( supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID, ): requests_count += 1 else: self.missingPackets += 1 else: # but if local Parity already exists, but was not sent - do it now if remoteParity[supplierNum] != 1: data_sender.A('new-data') total_requests_count += requests_count if total_requests_count > 0: lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count) self.automat('requests-sent', total_requests_count) else: if self.missingPackets: lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets) self.automat('found-missing') else: lg.out(8, 'backup_rebuilder._request_files : nothing was requested') self.automat('no-requests')
def doDecideToDismiss(self, *args, **kwargs): """ Action method. """ global _SuppliersToFire from p2p import p2p_connector from p2p import network_connector from customer import supplier_connector from p2p import online_status # take any actions only if I am connected to the network if not p2p_connector.A() or not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if p2p_connector.A().state != 'CONNECTED' or network_connector.A( ).state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP' ) self.automat('made-decision', []) return # if certain suppliers needs to be removed by manual/external request just do that to_be_fired = id_url.to_list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.info('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return # make sure to not go too far when i just want to decrease number of my suppliers number_desired = settings.getSuppliersNumberDesired() redundant_suppliers = set() if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.info('found REDUNDANT supplier %s at position %d' % ( idurl, supplier_index, )) redundant_suppliers.add(idurl) if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return # now I need to look more careful at my suppliers potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. my_suppliers = contactsdb.suppliers() if _Debug: lg.args(_DebugLevel, my_suppliers=my_suppliers) if id_url.is_some_empty(my_suppliers): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return for supplier_idurl in my_suppliers: sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if online_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif online_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif online_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if not disconnected_suppliers: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment' ) # TODO: add more conditions to fire "slow" suppliers - they are still connected but useless self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % (offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return max_offline_suppliers_count = eccmap.GetCorrectableErrors( number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn( 'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors( number_desired) if len(offline_suppliers) >= critical_offline_suppliers_count and len( offline_suppliers) > 0: if config.conf().getBool( 'services/employer/replace-critically-offline-enabled'): # TODO: check that issue # too aggressive replacing suppliers who still have the data is very dangerous !!! one_dead_supplier = offline_suppliers.pop() lg.warn( 'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!' ) self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [ result[0], ])
def cmd_suppliers(opts, args, overDict): def _wait_replace_supplier_and_stop(src, supplier_name, count=0): suppliers = [] for s in find_comments(src): if s.count('[online ]') or s.count('[offline]'): suppliers.append(s[18:38].strip()) if supplier_name not in suppliers: print ' supplier %s is fired !' % supplier_name print_and_stop(src) return if count >= 60: print ' time is out\n' reactor.stop() return else: def _check_again(supplier_name, count): sys.stdout.write('.') run_url_command(webcontrol._PAGE_SUPPLIERS).addCallback(_wait_replace_supplier_and_stop, supplier_name, count) reactor.callLater(1, _check_again, supplier_name, count+1) if len(args) < 2 or args[1] in [ 'list', 'ls' ]: url = webcontrol._PAGE_SUPPLIERS run_url_command(url).addCallback(print_and_stop) reactor.run() return 0 elif args[1] in [ 'call', 'cl' ]: url = webcontrol._PAGE_SUPPLIERS + '?action=call' run_url_command(url).addCallback(print_and_stop) reactor.run() return 0 elif args[1] in [ 'replace', 'rep', 'rp' ] and len(args) >= 3: contactsdb.init() idurl = args[2].strip() if not idurl.startswith('http://'): try: idurl = contactsdb.supplier(int(idurl)) except: idurl = '' if not idurl: print 'supplier IDURL is unknown\n' return 0 name = nameurl.GetName(idurl) url = webcontrol._PAGE_SUPPLIERS + '?action=replace&idurl=%s' % misc.pack_url_param(idurl) run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0) reactor.run() return 0 elif args[1] in [ 'change', 'ch' ] and len(args) >= 4: contactsdb.init() idurl = args[2].strip() if not idurl.startswith('http://'): try: idurl = contactsdb.supplier(int(idurl)) except: idurl = '' if not idurl: print 'supplier IDURL is unknown\n' return 0 newidurl = args[3].strip() name = nameurl.GetName(idurl) newname = nameurl.GetName(newidurl) url = webcontrol._PAGE_SUPPLIERS + '?action=change&idurl=%s&newidurl=%s' % (misc.pack_url_param(idurl), misc.pack_url_param(newidurl)) run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0) reactor.run() return 0 return 2
def doScanAndQueue(self, arg): global _ShutdownFlag if _Debug: lg.out( _DebugLevel, 'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag) if _Debug: log = open(os.path.join(settings.LogsDir(), 'data_sender.log'), 'w') log.write('doScanAndQueue %s\n' % time.asctime()) if _ShutdownFlag: if _Debug: log.write('doScanAndQueue _ShutdownFlag is True\n') self.automat('scan-done') if _Debug: log.flush() log.close() return for customer_idurl in contactsdb.known_customers(): if '' not in contactsdb.suppliers(customer_idurl): from storage import backup_matrix for backupID in misc.sorted_backup_ids( backup_matrix.local_files().keys(), True): packetsBySupplier = backup_matrix.ScanBlocksToSend( backupID) if _Debug: log.write('%s\n' % packetsBySupplier) for supplierNum in packetsBySupplier.keys(): supplier_idurl = contactsdb.supplier( supplierNum, customer_idurl=customer_idurl) if not supplier_idurl: lg.warn('?supplierNum? %s for %s' % (supplierNum, backupID)) continue for packetID in packetsBySupplier[supplierNum]: backupID_, _, supplierNum_, _ = packetid.BidBnSnDp( packetID) if backupID_ != backupID: lg.warn('?backupID? %s for %s' % (packetID, backupID)) continue if supplierNum_ != supplierNum: lg.warn('?supplierNum? %s for %s' % (packetID, backupID)) continue if io_throttle.HasPacketInSendQueue( supplier_idurl, packetID): if _Debug: log.write( '%s already in sending queue for %s\n' % (packetID, supplier_idurl)) continue if not io_throttle.OkToSend(supplier_idurl): if _Debug: log.write('ok to send %s ? - NO!\n' % supplier_idurl) continue customerGlobalID, pathID = packetid.SplitPacketID( packetID) # tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, []) # if len(tranByID) > 3: # log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByID))) # continue customerGlobalID, pathID = packetid.SplitPacketID( packetID) filename = os.path.join( settings.getLocalBackupsDir(), customerGlobalID, pathID, ) if not os.path.isfile(filename): if _Debug: log.write('%s is not a file\n' % filename) continue if io_throttle.QueueSendFile( filename, packetID, supplier_idurl, my_id.getLocalID(), self._packetAcked, self._packetFailed, ): if _Debug: log.write( 'io_throttle.QueueSendFile %s\n' % packetID) else: if _Debug: log.write( 'io_throttle.QueueSendFile FAILED %s\n' % packetID) # lg.out(6, ' %s for %s' % (packetID, backupID)) # DEBUG # break self.automat('scan-done') if _Debug: log.flush() log.close()