def Load(filepath=None):
"""
This load the data from local file and call ``ReadIndex()`` method.
"""
global _LoadingFlag
if _LoadingFlag:
return False
if filepath is None:
filepath = settings.BackupIndexFilePath()
if not os.path.isfile(filepath):
lg.warn('file %s not exist' % filepath)
WriteIndex(filepath)
# return False
src = bpio.ReadTextFile(filepath)
if not src:
lg.out(2, 'backup_control.Load ERROR reading file %s' % filepath)
return False
inpt = cStringIO.StringIO(src)
try:
known_revision = int(inpt.readline().rstrip('\n'))
except:
lg.exc()
return False
raw_data = inpt.read()
inpt.close()
ret = ReadIndex(raw_data)
if ret:
commit(known_revision)
backup_fs.Scan()
backup_fs.Calculate()
else:
lg.warn('catalog index reading failed')
return ret
def WriteIndex(filepath=None, encoding='utf-8'):
"""
Write index data base to the local file .bitdust/metadata/index.
"""
global _LoadingFlag
if _LoadingFlag:
return
if filepath is None:
filepath = settings.BackupIndexFilePath()
json_data = {}
# json_data = backup_fs.Serialize(to_json=True, encoding=encoding)
for customer_idurl in backup_fs.known_customers():
customer_id = global_id.UrlToGlobalID(customer_idurl)
json_data[customer_id] = backup_fs.Serialize(
iterID=backup_fs.fsID(customer_idurl),
to_json=True,
encoding=encoding,
)
src = '%d\n' % revision()
src += json.dumps(json_data,
indent=1,
separators=(',', ':'),
encoding=encoding)
if _Debug:
import pprint
lg.out(_DebugLevel, pprint.pformat(json_data))
return bpio.WriteTextFile(filepath, src)
def WriteIndex(filepath=None, encoding='utf-8'):
"""
Write index data base to the local file .bitdust/metadata/index.
"""
global _LoadingFlag
if _LoadingFlag:
return
if filepath is None:
filepath = settings.BackupIndexFilePath()
json_data = {}
for customer_idurl in backup_fs.known_customers():
customer_id = customer_idurl.to_id()
json_data[customer_id] = backup_fs.Serialize(
iterID=backup_fs.fsID(customer_idurl),
to_json=True,
encoding=encoding,
)
src = '%d\n' % revision()
src += jsn.dumps(
json_data,
indent=1,
separators=(',', ':'),
encoding=encoding,
)
if _Debug:
lg.args(_DebugLevel, size=len(src), filepath=filepath)
return bpio.WriteTextFile(filepath, src)
def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src,
)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplierId,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
# newpacket = signed.Packet(
# commands.Data(), localID, localID, packetID,
# Payload, supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Ack(): self._on_supplier_acked,
# commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplierId)))
def WriteIndex(filepath=None):
"""
Write index data base to the local file .bitdust/metadata/index.
"""
global _LoadingFlag
if _LoadingFlag:
return
if filepath is None:
filepath = settings.BackupIndexFilePath()
src = '%d\n' % revision()
src += backup_fs.Serialize(to_json=True)
return bpio.AtomicWriteFile(filepath, src)
def doSuppliersSendIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.outgoing_packets_ids = []
self.sent_suppliers_number = 0
localID = my_id.getIDURL()
b = encrypted.Block(
CreatorID=localID,
BackupID=packetID,
BlockNumber=0,
SessionKey=key.NewSessionKey(
session_key_type=key.SessionKeyType()),
SessionKeyType=key.SessionKeyType(),
LastBlock=True,
Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()),
)
Payload = b.Serialize()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplier_idurl,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
if pkt_out:
self.sending_suppliers.add(supplier_idurl)
self.sent_suppliers_number += 1
self.outgoing_packets_ids.append(packetID)
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplier_idurl)))
