def Load(filepath=None): """ This load the data from local file and call ``ReadIndex()`` method. """ global _LoadingFlag if _LoadingFlag: return False if filepath is None: filepath = settings.BackupIndexFilePath() if not os.path.isfile(filepath): lg.warn('file %s not exist' % filepath) WriteIndex(filepath) # return False src = bpio.ReadTextFile(filepath) if not src: lg.out(2, 'backup_control.Load ERROR reading file %s' % filepath) return False inpt = cStringIO.StringIO(src) try: known_revision = int(inpt.readline().rstrip('\n')) except: lg.exc() return False raw_data = inpt.read() inpt.close() ret = ReadIndex(raw_data) if ret: commit(known_revision) backup_fs.Scan() backup_fs.Calculate() else: lg.warn('catalog index reading failed') return ret
def WriteIndex(filepath=None, encoding='utf-8'): """ Write index data base to the local file .bitdust/metadata/index. """ global _LoadingFlag if _LoadingFlag: return if filepath is None: filepath = settings.BackupIndexFilePath() json_data = {} # json_data = backup_fs.Serialize(to_json=True, encoding=encoding) for customer_idurl in backup_fs.known_customers(): customer_id = global_id.UrlToGlobalID(customer_idurl) json_data[customer_id] = backup_fs.Serialize( iterID=backup_fs.fsID(customer_idurl), to_json=True, encoding=encoding, ) src = '%d\n' % revision() src += json.dumps(json_data, indent=1, separators=(',', ':'), encoding=encoding) if _Debug: import pprint lg.out(_DebugLevel, pprint.pformat(json_data)) return bpio.WriteTextFile(filepath, src)
def WriteIndex(filepath=None, encoding='utf-8'): """ Write index data base to the local file .bitdust/metadata/index. """ global _LoadingFlag if _LoadingFlag: return if filepath is None: filepath = settings.BackupIndexFilePath() json_data = {} for customer_idurl in backup_fs.known_customers(): customer_id = customer_idurl.to_id() json_data[customer_id] = backup_fs.Serialize( iterID=backup_fs.fsID(customer_idurl), to_json=True, encoding=encoding, ) src = '%d\n' % revision() src += jsn.dumps( json_data, indent=1, separators=(',', ':'), encoding=encoding, ) if _Debug: lg.args(_DebugLevel, size=len(src), filepath=filepath) return bpio.WriteTextFile(filepath, src)
def doSuppliersSendIndexFile(self, arg): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) self.sending_suppliers.clear() self.sent_suppliers_number = 0 src = bpio.ReadBinaryFile(settings.BackupIndexFilePath()) localID = my_id.getLocalID() b = encrypted.Block( localID, packetID, 0, key.NewSessionKey(), key.SessionKeyType(), True, src, ) Payload = b.Serialize() for supplierId in contactsdb.suppliers(): if not supplierId: continue if not contact_status.isOnline(supplierId): continue newpacket, pkt_out = p2p_service.SendData( raw_data=Payload, ownerID=localID, creatorID=localID, remoteID=supplierId, packetID=packetID, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }, ) # newpacket = signed.Packet( # commands.Data(), localID, localID, packetID, # Payload, supplierId) # pkt_out = gateway.outbox(newpacket, callbacks={ # commands.Ack(): self._on_supplier_acked, # commands.Fail(): self._on_supplier_acked, }) if pkt_out: self.sending_suppliers.add(supplierId) self.sent_suppliers_number += 1 if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (newpacket, nameurl.GetName(supplierId)))
def WriteIndex(filepath=None): """ Write index data base to the local file .bitdust/metadata/index. """ global _LoadingFlag if _LoadingFlag: return if filepath is None: filepath = settings.BackupIndexFilePath() src = '%d\n' % revision() src += backup_fs.Serialize(to_json=True) return bpio.AtomicWriteFile(filepath, src)
def doSuppliersSendIndexFile(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) self.sending_suppliers.clear() self.outgoing_packets_ids = [] self.sent_suppliers_number = 0 localID = my_id.getIDURL() b = encrypted.Block( CreatorID=localID, BackupID=packetID, BlockNumber=0, SessionKey=key.NewSessionKey( session_key_type=key.SessionKeyType()), SessionKeyType=key.SessionKeyType(), LastBlock=True, Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()), ) Payload = b.Serialize() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue newpacket, pkt_out = p2p_service.SendData( raw_data=Payload, ownerID=localID, creatorID=localID, remoteID=supplier_idurl, packetID=packetID, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }, ) if pkt_out: self.sending_suppliers.add(supplier_idurl) self.sent_suppliers_number += 1 self.outgoing_packets_ids.append(packetID) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (newpacket, nameurl.GetName(supplier_idurl)))