def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src,
)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplierId,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
# newpacket = signed.Packet(
# commands.Data(), localID, localID, packetID,
# Payload, supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Ack(): self._on_supplier_acked,
# commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.outgoing_packets_ids = []
self.sent_suppliers_number = 0
localID = my_id.getIDURL()
b = encrypted.Block(
CreatorID=localID,
BackupID=packetID,
BlockNumber=0,
SessionKey=key.NewSessionKey(
session_key_type=key.SessionKeyType()),
SessionKeyType=key.SessionKeyType(),
LastBlock=True,
Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()),
)
Payload = b.Serialize()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplier_idurl,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
if pkt_out:
self.sending_suppliers.add(supplier_idurl)
self.sent_suppliers_number += 1
self.outgoing_packets_ids.append(packetID)
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplier_idurl)))
def doSendData(self, *args, **kwargs):
"""
Action method.
"""
payload = bpio.ReadBinaryFile(self.fileName)
if not payload:
self.event('error',
Exception('file %r reading error' % self.fileName))
return
p2p_service.SendData(
raw_data=payload,
ownerID=self.ownerID,
creatorID=self.parent.creatorID,
remoteID=self.remoteID,
packetID=self.packetID,
callbacks={
commands.Ack(): self.parent.OnFileSendAckReceived,
commands.Fail(): self.parent.OnFileSendAckReceived,
},
)
self.sendTime = time.time()
def RunSend(self):
if self._runSend:
return
self._runSend = True
#out(6, 'io_throttle.RunSend')
packetsFialed = {}
packetsToRemove = set()
packetsSent = 0
# let's check all packets in the queue
for i in range(len(self.fileSendQueue)):
try:
packetID = self.fileSendQueue[i]
except:
lg.warn("item at position %d not exist in send queue" % i)
continue
fileToSend = self.fileSendDict[packetID]
# we got notify that this packet was failed to send
if packetID in self.sendFailedPacketIDs:
self.sendFailedPacketIDs.remove(packetID)
packetsFialed[packetID] = 'failed'
continue
# we already sent the file
if fileToSend.sendTime is not None:
packetsSent += 1
# and we got ack
if fileToSend.ackTime is not None:
# deltaTime = fileToSend.ackTime - fileToSend.sendTime
# so remove it from queue
packetsToRemove.add(packetID)
# if we do not get an ack ...
else:
# ... we do not want to wait to long
if time.time() - fileToSend.sendTime > fileToSend.sendTimeout:
# so this packet is failed because no response on it
packetsFialed[packetID] = 'timeout'
# we sent this packet already - check next one
continue
# the data file to send no longer exists - it is failed situation
if not os.path.exists(fileToSend.fileName):
lg.warn("file %s not exist" % (fileToSend.fileName))
packetsFialed[packetID] = 'not exist'
continue
# do not send too many packets, need to wait for ack
# hold other packets in the queue and may be send next time
if packetsSent > self.fileSendMaxLength:
# if we sending big file - we want to wait
# other packets must go without waiting in the queue
# 10K seems fine, because we need to filter only Data and Parity packets here
try:
if os.path.getsize(fileToSend.fileName) > 1024 * 10:
continue
except:
lg.exc()
continue
# prepare the packet
# dt = time.time()
Payload = bpio.ReadBinaryFile(fileToSend.fileName)
# newpacket = signed.Packet(
# commands.Data(),
# fileToSend.ownerID,
# self.creatorID,
# fileToSend.packetID,
# Payload,
# fileToSend.remoteID,
# )
p2p_service.SendData(
raw_data=Payload,
ownerID=fileToSend.ownerID,
creatorID=self.creatorID,
remoteID=fileToSend.remoteID,
packetID=fileToSend.packetID,
callbacks={
commands.Ack(): self.OnFileSendAckReceived,
commands.Fail(): self.OnFileSendAckReceived,
},
)
# outbox will not resend, because no ACK, just data,
# need to handle resends on own
# transport_control.outboxNoAck(newpacket)
# gateway.outbox(newpacket, callbacks={
# commands.Ack(): self.OnFileSendAckReceived,
# commands.Fail(): self.OnFileSendAckReceived,
# })
# str(bpio.ReadBinaryFile(fileToSend.fileName))
# {commands.Ack(): self.OnFileSendAckReceived,
# commands.Fail(): self.OnFileSendAckReceived}
# transport_control.RegisterInterest(
# self.OnFileSendAckReceived,
# fileToSend.remoteID,
# fileToSend.packetID)
# callback.register_interest(self.OnFileSendAckReceived, fileToSend.remoteID, fileToSend.packetID)
# lg.out(12, 'io_throttle.RunSend %s to %s, dt=%s' % (
# str(newpacket), nameurl.GetName(fileToSend.remoteID), str(time.time()-dt)))
# mark file as been sent
fileToSend.sendTime = time.time()
packetsSent += 1
# process failed packets
for packetID, why in packetsFialed.items():
self.OnFileSendFailReceived(self.fileSendDict[packetID].remoteID, packetID, why)
packetsToRemove.add(packetID)
# remove finished packets
for packetID in packetsToRemove:
self.fileSendQueue.remove(packetID)
del self.fileSendDict[packetID]
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunSend removed %s from %s sending queue, %d more items" % (
packetID, self.remoteName, len(self.fileSendQueue)))
# if sending queue is empty - remove all records about packets failed to send
if len(self.fileSendQueue) == 0:
del self.sendFailedPacketIDs[:]
# remember results
result = max(len(packetsToRemove), packetsSent)
# erase temp lists
del packetsFialed
del packetsToRemove
self._runSend = False
return result
def _do_send_packets(self, backup_id, block_num):
customer_id, path_id, version_name = packetid.SplitBackupID(backup_id)
archive_snapshot_dir = os.path.join(settings.getLocalBackupsDir(),
customer_id, path_id, version_name)
if _Debug:
lg.args(_DebugLevel,
backup_id=backup_id,
block_num=block_num,
archive_snapshot_dir=archive_snapshot_dir)
if not os.path.isdir(archive_snapshot_dir):
self.block_failed = True
lg.err('archive snapshot folder was not found in %r' %
archive_snapshot_dir)
return None
failed_supliers = 0
for supplier_num in range(len(self.suppliers_list)):
supplier_idurl = self.suppliers_list[supplier_num]
if not supplier_idurl:
failed_supliers += 1
lg.warn('unknown supplier supplier_num=%d' % supplier_num)
continue
for dataORparity in (
'Data',
'Parity',
):
packet_id = packetid.MakePacketID(backup_id, block_num,
supplier_num, dataORparity)
packet_filename = os.path.join(
archive_snapshot_dir, '%d-%d-%s' % (
block_num,
supplier_num,
dataORparity,
))
if not os.path.isfile(packet_filename):
lg.err('%s is not a file' % packet_filename)
continue
packet_payload = bpio.ReadBinaryFile(packet_filename)
if not packet_payload:
lg.err('file %r reading error' % packet_filename)
continue
if block_num not in self.packets_out:
self.packets_out[block_num] = {}
self.packets_out[block_num][packet_id] = None
p2p_service.SendData(
raw_data=packet_payload,
ownerID=self.queue_owner_idurl,
creatorID=my_id.getIDURL(),
remoteID=supplier_idurl,
packetID=packet_id,
callbacks={
commands.Ack():
lambda newpacket, _: self.automat('ack',
newpacket=newpacket),
commands.Fail():
lambda newpacket, _: self.automat('fail',
newpacket=newpacket),
},
)
if failed_supliers > self.correctable_errors:
self.block_failed = True
lg.err('too many failed suppliers %d in block %d' % (
failed_supliers,
block_num,
))