def _do_retrieve(self, x=None):
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
localID = my_id.getIDURL()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
pkt_out = p2p_service.SendRetreive(ownerID=localID,
creatorID=localID,
packetID=packetID,
remoteID=supplier_idurl,
response_timeout=60 * 2,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_fail,
})
if pkt_out:
self.requesting_suppliers.add(supplier_idurl)
self.requested_suppliers_number += 1
self.requests_packets_sent.append((packetID, supplier_idurl))
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplier_idurl)))
def doSuppliersRequestIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel,
'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_on('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
# packetID = settings.BackupIndexFileName()
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
pkt_out = p2p_service.SendRetreive(localID,
localID,
packetID,
supplierId,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_response,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# localID,
# localID,
# packetid.RemotePath(packetID),
# '',
# supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Data(): self._on_supplier_response,
# commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSendRetreive(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.args(_DebugLevel, packetID=self.packetID, remoteID=self.remoteID)
p2p_service.SendRetreive(
self.ownerID,
self.parent.creatorID,
self.packetID,
self.remoteID,
callbacks={
commands.Data(): self.parent.OnDataReceived,
commands.Fail(): self.parent.OnDataReceived,
# None: lambda pkt_out: self.OnDataReceived(fileRequest.packetID, 'timeout'), # timeout
},
# response_timeout=10,
)
self.requestTime = time.time()
def RunRequest(self):
#out(6, 'io_throttle.RunRequest')
packetsToRemove = {}
for i in range(0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
# we got notify that this packet was failed to send
if packetID in self.requestFailedPacketIDs:
self.requestFailedPacketIDs.remove(packetID)
packetsToRemove[packetID] = 'failed'
continue
# request timeouts are disabled for now
# currentTime = time.time()
# if self.fileRequestDict[packetID].requestTime is not None:
# # the packet was requested
# if self.fileRequestDict[packetID].fileReceivedTime is None:
# # but no answer yet ...
# if currentTime - self.fileRequestDict[packetID].requestTime > self.fileRequestDict[packetID].requestTimeout:
# # and time is out!!!
# self.fileRequestDict[packetID].report = 'timeout'
# packetsToRemove[packetID] = 'timeout'
# else:
# # the packet were received (why it is not removed from the queue yet ???)
# self.fileRequestDict[packetID].result = 'received'
# packetsToRemove[packetID] = 'received'
# the packet was not requested yet
if self.fileRequestDict[packetID].requestTime is None:
customer, pathID = packetid.SplitPacketID(packetID)
if not os.path.exists(os.path.join(settings.getLocalBackupsDir(), customer, pathID)):
fileRequest = self.fileRequestDict[packetID]
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest for packetID " + fileRequest.packetID)
# transport_control.RegisterInterest(self.DataReceived,fileRequest.creatorID,fileRequest.packetID)
# callback.register_interest(self.DataReceived, fileRequest.creatorID, fileRequest.packetID)
p2p_service.SendRetreive(
fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
fileRequest.remoteID,
callbacks={
commands.Data(): self.OnDataReceived,
commands.Fail(): self.OnDataReceived,
# None: lambda pkt_out: self.OnDataReceived(fileRequest.packetID, 'timeout'), # timeout
},
# response_timeout=10,
)
# newpacket = signed.Packet(
# commands.Retrieve(),
# fileRequest.ownerID,
# fileRequest.creatorID,
# packetid.RemotePath(fileRequest.packetID),
# "",
# fileRequest.remoteID)
# gateway.outbox(newpacket, callbacks={
# commands.Data(): self.DataReceived,
# commands.Fail(): self.DataReceived})
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove[packetID] = 'exist'
# if request queue is empty - remove all records about packets failed to request
if len(self.fileRequestQueue) == 0:
del self.requestFailedPacketIDs[:]
# remember requests results
result = len(packetsToRemove)
# remove finished requests
for packetID, why in packetsToRemove.items():
# self.fileRequestQueue.remove(packetID)
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest removed %s from %s receiving queue, %d more items" % (
packetID, self.remoteName, len(self.fileRequestQueue)))
self.OnDataRequestFailed(packetID, why)
del packetsToRemove
return result
def RunRequest(self):
#out(6, 'io_throttle.RunRequest')
packetsToRemove = set()
for i in range(
0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
currentTime = time.time()
if self.fileRequestDict[packetID].requestTime is not None:
# the packet were requested
if self.fileRequestDict[packetID].fileReceivedTime is None:
# but no answer yet ...
if currentTime - self.fileRequestDict[
packetID].requestTime > self.fileRequestDict[
packetID].requestTimeout:
# and time is out!!!
self.fileRequestDict[packetID].report = 'timeout'
packetsToRemove.add(packetID)
else:
# the packet were received (why it is not removed from the queue yet ???)
self.fileRequestDict[packetID].result = 'received'
packetsToRemove.add(packetID)
if self.fileRequestDict[packetID].requestTime is None:
customer, pathID = packetid.SplitPacketID(packetID)
if not os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
pathID)):
fileRequest = self.fileRequestDict[packetID]
lg.out(
10, "io_throttle.RunRequest for packetID " +
fileRequest.packetID)
# transport_control.RegisterInterest(self.DataReceived,fileRequest.creatorID,fileRequest.packetID)
# callback.register_interest(self.DataReceived, fileRequest.creatorID, fileRequest.packetID)
p2p_service.SendRetreive(fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
fileRequest.remoteID,
callbacks={
commands.Data():
self.DataReceived,
commands.Fail():
self.DataReceived,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# fileRequest.ownerID,
# fileRequest.creatorID,
# packetid.RemotePath(fileRequest.packetID),
# "",
# fileRequest.remoteID)
# gateway.outbox(newpacket, callbacks={
# commands.Data(): self.DataReceived,
# commands.Fail(): self.DataReceived})
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove.add(packetID)
# remember requests results
result = len(packetsToRemove)
# remove finished requests
if len(packetsToRemove) > 0:
for packetID in packetsToRemove:
self.fileRequestQueue.remove(packetID)
del packetsToRemove
return result
