def _do_retrieve(self, x=None):
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
localID = my_id.getIDURL()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
pkt_out = p2p_service.SendRetreive(ownerID=localID,
creatorID=localID,
packetID=packetID,
remoteID=supplier_idurl,
response_timeout=60 * 2,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_fail,
})
if pkt_out:
self.requesting_suppliers.add(supplier_idurl)
self.requested_suppliers_number += 1
self.requests_packets_sent.append((packetID, supplier_idurl))
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplier_idurl)))
def make_valid_filename(customerIDURL, glob_path):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
keyAlias = glob_path['key_alias'] or 'master'
packetID = glob_path['path']
customerGlobID = glob_path['customer']
if not customerGlobID:
lg.warn("customer id is empty")
return ''
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
lg.warn('invalid file path')
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not my customer" % (customerIDURL))
if customerGlobID:
if glob_path['idurl'] != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(glob_path['idurl'], customerIDURL))
filename = make_filename(customerGlobID, packetID, keyAlias)
return filename
def doCancelRequests(self, arg):
"""
Action method.
"""
packetID = global_id.MakeGlobalID(idurl=my_id.getLocalID(),
path=settings.BackupIndexFileName())
packetsToCancel = packet_out.search_by_backup_id(packetID)
for pkt_out in packetsToCancel:
if pkt_out.outpacket.Command == commands.Retrieve():
lg.warn('sending "cancel" to %s' % pkt_out)
pkt_out.automat('cancel')
def doSuppliersRequestIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel,
'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_on('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
# packetID = settings.BackupIndexFileName()
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
pkt_out = p2p_service.SendRetreive(localID,
localID,
packetID,
supplierId,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_response,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# localID,
# localID,
# packetid.RemotePath(packetID),
# '',
# supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Data(): self._on_supplier_response,
# commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src,
)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplierId,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
# newpacket = signed.Packet(
# commands.Data(), localID, localID, packetID,
# Payload, supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Ack(): self._on_supplier_acked,
# commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.outgoing_packets_ids = []
self.sent_suppliers_number = 0
localID = my_id.getIDURL()
b = encrypted.Block(
CreatorID=localID,
BackupID=packetID,
BlockNumber=0,
SessionKey=key.NewSessionKey(
session_key_type=key.SessionKeyType()),
SessionKeyType=key.SessionKeyType(),
LastBlock=True,
Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()),
)
Payload = b.Serialize()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplier_idurl,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
if pkt_out:
self.sending_suppliers.add(supplier_idurl)
self.sent_suppliers_number += 1
self.outgoing_packets_ids.append(packetID)
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplier_idurl)))
def _on_inbox_packet_received(self, newpacket, info, status,
error_message):
from logs import lg
from main import settings
from contacts import contactsdb
from userid import my_id
from userid import global_id
from storage import backup_control
from p2p import commands
if newpacket.Command == commands.Data():
if newpacket.OwnerID != my_id.getLocalID():
# only catch data belongs to me
return False
lg.out(
self.debug_level,
"service_backups._on_inbox_packet_received: %r for us from %s"
% (
newpacket,
newpacket.RemoteID,
))
if newpacket.PacketID == global_id.MakeGlobalID(
idurl=my_id.getLocalID(),
path=settings.BackupIndexFileName(),
):
# TODO: move to service_backup_db
backup_control.IncomingSupplierBackupIndex(newpacket)
return True
if newpacket.Command == commands.Files():
if not newpacket.PacketID.startswith(my_id.getGlobalID() + ':'):
# skip Files() which are from another customer
return False
if not contactsdb.is_supplier(newpacket.OwnerID):
# skip Files() if this is not my supplier
return False
lg.out(
self.debug_level,
"service_backups._on_inbox_packet_received: %r for us from %s"
% (
newpacket,
newpacket.RemoteID,
))
return backup_control.IncomingSupplierListFiles(newpacket)
return False
def makeFilename(customerIDURL, packetID):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
# lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID )
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not a customer" % (customerIDURL))
return ''
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
return constructFilename(customerIDURL, packetID)
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
if _Debug:
lg.out(
_DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' %
(len(request.Payload), request.PacketID, request.OwnerID,
request.CreatorID))
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data %r for us from %s" %
(request, nameurl.GetName(request.RemoteID)))
if driver.is_on('service_backups'):
# TODO: move this into callback
settings.BackupIndexFileName()
indexPacketID = global_id.MakeGlobalID(
idurl=my_id.getLocalID(), path=settings.BackupIndexFileName())
if request.PacketID == indexPacketID:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_on('service_supplier'):
return SendFail(request, 'supplier service is off')
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" %
(request.OwnerID, request.PacketID))
SendFail(request, 'not a customer')
return
glob_path = global_id.ParseGlobalID(request.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' +
request.PacketID)
if not glob_path['path']:
lg.warn("got incorrect PacketID")
SendFail(request, 'incorrect PacketID')
return
# TODO: process files from another customer : glob_path['idurl']
filename = makeFilename(request.OwnerID, glob_path['path'])
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, 'empty filename')
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, 'write error')
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': donated_bytes})
if _Debug:
lg.out(_DebugLevel, 'p2p_service.Data created a new space file')
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, 'no info about donated space')
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, 'no free space')
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, 'write error')
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % (
request.OwnerID,
request.CreatorID,
filename,
))
