def SendDeleteBackup(SupplierID, BackupID):
dhnio.Dprint(6, "p2p_service.SendDeleteBackup SupplierID=%s BackupID=%s " % (SupplierID, BackupID))
MyID = misc.getLocalID()
PacketID = BackupID
RemoteID = SupplierID
result = dhnpacket.dhnpacket(commands.DeleteBackup(), MyID, MyID, PacketID, "", RemoteID)
transport_control.outboxAck(result)
def SendSingleCustomer(idurl, response_callback=None):
dhnio.Dprint(6, "identitypropagate.SendSingleCustomer [%s]" % nameurl.GetName(idurl))
MyID = misc.getLocalID()
packet = dhnpacket.dhnpacket(commands.Identity(), MyID, MyID, MyID, misc.getLocalIdentity().serialize(), idurl)
transport_control.outboxNoAck(packet)
if response_callback is not None:
transport_control.RegisterInterest(response_callback, packet.RemoteID, packet.PacketID)
def SendDeleteListBackups(SupplierID, ListBackupIDs):
dhnio.Dprint(6, "p2p_service.SendDeleteListBackups SupplierID=%s BackupIDs number: %d" % (SupplierID, len(ListBackupIDs)))
MyID = misc.getLocalID()
PacketID = packetid.UniqueID()
RemoteID = SupplierID
Payload = '\n'.join(ListBackupIDs)
result = dhnpacket.dhnpacket(commands.DeleteBackup(), MyID, MyID, PacketID, Payload, RemoteID)
transport_control.outboxAck(result)
def RequestIdentity(request):
dhnio.Dprint(6, "p2p_service.RequestIdentity starting")
MyID = misc.getLocalID()
RemoteID = request.OwnerID
PacketID = request.PacketID
identitystr = misc.getLocalIdentity().serialize()
dhnio.Dprint(12, "p2p_service.RequestIdentity returning ")
result = dhnpacket.dhnpacket(commands.Identity(), MyID, MyID, PacketID, identitystr, RemoteID)
transport_control.outboxNoAck(result)
def ListFiles(request):
MyID = misc.getLocalID()
RemoteID = request.OwnerID
PacketID = request.PacketID
Payload = request.Payload
dhnio.Dprint(8, "p2p_service.ListFiles from [%s], format is %s" % (nameurl.GetName(request.OwnerID), Payload))
custdir = settings.getCustomersFilesDir()
ownerdir = os.path.join(custdir, nameurl.UrlFilename(request.OwnerID))
if not os.path.isdir(ownerdir):
dhnio.Dprint(8, "p2p_service.ListFiles did not find customer dir " + ownerdir)
src = PackListFiles('', Payload)
result = dhnpacket.dhnpacket(commands.Files(), MyID, MyID, PacketID, src, RemoteID)
transport_control.outboxNoAck(result)
return
plaintext = TreeSummary(ownerdir)
src = PackListFiles(plaintext, Payload)
outpacket = dhnpacket.dhnpacket(commands.Files(), MyID, MyID, PacketID, src, RemoteID)
transport_control.outboxNoAck(outpacket)
def RequestListFiles(supplierNumORidurl):
if isinstance(supplierNumORidurl, str):
RemoteID = supplierNumORidurl
else:
RemoteID = contacts.getSupplierID(supplierNumORidurl)
if not RemoteID:
dhnio.Dprint(4, "p2p_service.RequestListFiles WARNING RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl))
return
dhnio.Dprint(8, "p2p_service.RequestListFiles [%s]" % nameurl.GetName(RemoteID))
MyID = misc.getLocalID()
PacketID = packetid.UniqueID()
Payload = settings.ListFilesFormat()
result = dhnpacket.dhnpacket(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID)
transport_control.outboxNoAck(result)
return PacketID
def RunRequest(self):
#dhnio.Dprint(6, 'io_throttle.RunRequest')
packetsToRemove = set()
for i in range(0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
currentTime = time.time()
if self.fileRequestDict[packetID].requestTime is not None:
# the packet were requested
if self.fileRequestDict[packetID].fileReceivedTime is None:
# but no answer yet ...
if currentTime - self.fileRequestDict[packetID].requestTime > self.fileRequestDict[packetID].requestTimeout:
# and time is out!!!
self.fileRequestDict[packetID].report = 'timeout'
packetsToRemove.add(packetID)
else:
# the packet were received (why it is not removed from the queue yet ???)
self.fileRequestDict[packetID].result = 'received'
packetsToRemove.add(packetID)
if self.fileRequestDict[packetID].requestTime is None:
if not os.path.exists(os.path.join(settings.getLocalBackupsDir(), packetID)):
fileRequest = self.fileRequestDict[packetID]
dhnio.Dprint(10, "io_throttle.RunRequest for packetID " + fileRequest.packetID)
transport_control.RegisterInterest(
self.DataReceived,
fileRequest.creatorID,
fileRequest.packetID)
newpacket = dhnpacket.dhnpacket(
commands.Retrieve(),
fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
"",
fileRequest.remoteID)
transport_control.outboxNoAck(newpacket)
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove.add(packetID)
# remember requests results
result = len(packetsToRemove)
# remove finished requests
if len(packetsToRemove) > 0:
for packetID in packetsToRemove:
self.fileRequestQueue.remove(packetID)
del packetsToRemove
return result
def send2market(command, data, doAck=False, PacketID=None):
"""
Same - send a packet to Market server.
"""
MyID = misc.getLocalID()
RemoteID = settings.MarketServerID()
if PacketID is None:
PacketID = packetid.UniqueID()
packet = dhnpacket.dhnpacket(
command,
MyID,
MyID,
PacketID,
data,
RemoteID,)
transport_control.outbox(packet, doAck)
return PacketID
def send2central(command, data, doAck=False, PacketID=None):
"""
A method to prepare a packet for Central server and send it to the transport gate.
"""
MyID = misc.getLocalID()
RemoteID = settings.CentralID()
if PacketID is None:
PacketID = packetid.UniqueID()
packet = dhnpacket.dhnpacket(
command,
MyID,
MyID,
PacketID,
data,
RemoteID,)
transport_control.outbox(packet, doAck)
#del packet
return PacketID
def doSuppliersRequestDBInfo(self, arg):
# dhnio.Dprint(4, 'backup_db_keeper.doSuppliersRequestDBInfo')
# packetID_ = settings.BackupInfoFileName()
# packetID = settings.BackupInfoEncryptedFileName()
packetID = settings.BackupIndexFileName()
for supplierId in contacts.getSupplierIDs():
if supplierId:
transport_control.RemoveInterest(supplierId, packetID)
self.requestedSuppliers.clear()
Payload = ''
localID = misc.getLocalID()
for supplierId in contacts.getSupplierIDs():
if not supplierId:
continue
newpacket = dhnpacket.dhnpacket(commands.Retrieve(), localID, localID, packetID, Payload, supplierId)
transport_control.outboxAck(newpacket)
transport_control.RegisterInterest(self.SupplierResponse, supplierId, packetID)
self.requestedSuppliers.add(supplierId)
def SendMessage(RemoteID, messagebody, PacketID=""):
dhnio.Dprint(6, "message.SendMessage to: " + str(RemoteID))
# TODO ERROR HERE (return Defer)
if not identitycache.scheduleForCaching(RemoteID):
dhnio.Dprint(1, "message.SendMessage ERROR. Can't find identity: " + str(RemoteID))
return
RemoteIdentity = identitycache.FromCache(RemoteID)
if RemoteIdentity == "":
dhnio.Dprint(1, "message.SendMessage ERROR. Can't retreive identity: " + str(RemoteID))
return
Amessage = MessageClass(RemoteIdentity, messagebody)
MyID = misc.getLocalID()
if PacketID == "":
PacketID = packetid.UniqueID()
Payload = misc.ObjectToString(Amessage)
dhnio.Dprint(6, "message.SendMessage about to send to " + RemoteID)
result = dhnpacket.dhnpacket(commands.Message(), MyID, MyID, PacketID, Payload, RemoteID)
transport_control.outboxAck(result)
def OneFromList(filename):
WholeFile=dhnio.ReadBinaryFile(filename)
FileList=WholeFile.split("\n")
num=len(FileList)
dhnio.Dprint(7, "supplierpatrol.OneFromList number of items is " + str(num))
rnd=random.randint(0,num-1)
item=FileList[rnd]
command=commands.Data()
OwnerID=misc.getLocalID()
CreatorID=misc.getLocalID()
PacketID=item
Payload=""
supnum=packetid.SupplierNumber(item)
RemoteID=contacts.getSupplierID(supnum)
request=dhnpacket.dhnpacket(command, OwnerID, CreatorID, PacketID, Payload, RemoteID)
global DataResultsOutstanding
DataResultsOutstanding.append(item)
transport_control.RegisterInterest(DataResult, RemoteID, PacketID)
transport_control.outboxAck(request)
def SendToID(idurl, AckHandler=None, Payload=None, NeedAck=False, wide=False):
dhnio.Dprint(8, "identitypropagate.SendToID [%s] NeedAck=%s" % (nameurl.GetName(idurl), str(NeedAck)))
thePayload = Payload
if thePayload is None:
thePayload = misc.getLocalIdentity().serialize()
packet = dhnpacket.dhnpacket(
commands.Identity(),
misc.getLocalID(), #MyID,
misc.getLocalID(), #MyID,
misc.getLocalID(), #PacketID,
thePayload,
idurl)
if AckHandler is not None:
transport_control.RegisterInterest(AckHandler, packet.RemoteID, packet.PacketID)
transport_control.outbox(packet, NeedAck, wide)
if wide:
# this is a ping packet - need to clear old info
transport_control.ErasePeerProtosStates(idurl)
transport_control.EraseMyProtosStates(idurl)
def UpdateListFiles():
if (not os.path.exists(settings.FileListDir())):
os.mkdir(settings.FileListDir())
for supnum in range(0, contacts.numSuppliers()):
filename= os.path.join(settings.FileListDir(), str(supnum))
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles looking at = " + filename)
if (not os.path.exists(filename) or (fileAgeInSeconds(filename) > 3600*24)):
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles found one to update " + filename)
command=commands.ListFiles()
OwnerID=misc.getLocalID()
CreatorID=misc.getLocalID()
PacketID="ListFiles" + str(supnum)
Payload=""
RemoteID= contacts.getSupplierID(supnum)
request=dhnpacket.dhnpacket(command, OwnerID, CreatorID, PacketID, Payload, RemoteID)
transport_control.RegisterInterest(ListResult, RemoteID, PacketID)
transport_control.outboxAck(request)
global NumRequestsOutstanding
NumRequestsOutstanding += 1
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles sent request - now outstanding=" + str(NumRequestsOutstanding))
def SendToIDs(idlist, AckHandler=None, wide=False):
dhnio.Dprint(8, "identitypropagate.SendToIDs to %d users" % len(idlist))
MyID = misc.getLocalID()
PacketID = MyID
LocalIdentity = misc.getLocalIdentity()
Payload = LocalIdentity.serialize()
Hash = dhncrypto.Hash(Payload)
alreadysent = set()
for contact in idlist:
if not contact:
continue
if contact in alreadysent:
# just want to send once even if both customer and supplier
continue
found_previous_packets = 0
for transfer_id in transport_control.transfers_by_idurl(contact):
ti = transport_control.transfer_by_id(transfer_id)
if ti and ti.description.count('Identity'):
found_previous_packets += 1
break
if found_previous_packets >= 3:
dhnio.Dprint(8, ' skip sending to %s' % contact)
continue
packet = dhnpacket.dhnpacket(
commands.Identity(),
misc.getLocalID(), #MyID,
misc.getLocalID(), #MyID,
misc.getLocalID(), #PacketID,
Payload,
contact)
dhnio.Dprint(8, " sending [Identity] to %s" % nameurl.GetName(contact))
if AckHandler is not None:
transport_control.RegisterInterest(AckHandler, packet.RemoteID, packet.PacketID)
transport_control.outboxNoAck(packet, wide)
if wide:
# this is a ping packet - need to clear old info
transport_control.ErasePeerProtosStates(contact)
transport_control.EraseMyProtosStates(contact)
alreadysent.add(contact)
del alreadysent
def doSuppliersSendDBInfo(self, arg):
# dhnio.Dprint(4, 'backup_db_keeper.doSuppliersSendDBInfo')
# packetID = settings.BackupInfoEncryptedFileName()
packetID = settings.BackupIndexFileName()
for supplierId in contacts.getSupplierIDs():
if supplierId:
transport_control.RemoveInterest(supplierId, packetID)
self.sentSuppliers.clear()
# src = dhnio.ReadBinaryFile(settings.BackupInfoFileFullPath())
src = dhnio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = misc.getLocalID()
block = dhnblock.dhnblock(localID, packetID, 0, dhncrypto.NewSessionKey(), dhncrypto.SessionKeyType(), True, src)
Payload = block.Serialize()
for supplierId in contacts.getSupplierIDs():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = dhnpacket.dhnpacket(commands.Data(), localID, localID, packetID, Payload, supplierId)
transport_control.outboxAck(newpacket)
transport_control.RegisterInterest(self.SupplierAcked, supplierId, packetID)
self.sentSuppliers.add(supplierId)
def RunSend(self):
if self._runSend:
return
self._runSend = True
#dhnio.Dprint(6, 'io_throttle.RunSend')
packetsFialed = {}
packetsToRemove = set()
packetsSent = 0
# let's check all packets in the queue
for i in xrange(len(self.fileSendQueue)):
packetID = self.fileSendQueue[i]
fileToSend = self.fileSendDict[packetID]
# we got notify that this packet was failed to send
if packetID in self.sendFailedPacketIDs:
self.sendFailedPacketIDs.remove(packetID)
packetsFialed[packetID] = 'failed'
continue
# we already sent the file
if fileToSend.sendTime is not None:
packetsSent += 1
# and we got ack
if fileToSend.ackTime is not None:
# deltaTime = fileToSend.ackTime - fileToSend.sendTime
# so remove it from queue
packetsToRemove.add(packetID)
# if we do not get an ack ...
else:
# ... we do not want to wait to long
if time.time() - fileToSend.sendTime > fileToSend.sendTimeout:
# so this packet is failed because no response on it
packetsFialed[packetID] = 'timeout'
# we sent this packet already - check next one
continue
# the data file to send no longer exists - it is failed situation
if not os.path.exists(fileToSend.fileName):
dhnio.Dprint(4, "io_throttle.RunSend WARNING file %s not exist" % (fileToSend.fileName))
packetsFialed[packetID] = 'not exist'
continue
# do not send too many packets, need to wait for ack
# hold other packets in the queue and may be send next time
if packetsSent > self.fileSendMaxLength:
# if we sending big file - we want to wait
# other packets must go without waiting in the queue
# 10K seems fine, because we need to filter only Data and Parity packets here
try:
if os.path.getsize(fileToSend.fileName) > 1024 * 10:
continue
except:
dhnio.DprintException()
continue
# prepare the packet
dt = time.time()
Payload = str(dhnio.ReadBinaryFile(fileToSend.fileName))
newpacket = dhnpacket.dhnpacket(
commands.Data(),
fileToSend.ownerID,
self.creatorID,
fileToSend.packetID,
Payload,
fileToSend.remoteID)
# outbox will not resend, because no ACK, just data,
# need to handle resends on own
transport_control.outboxNoAck(newpacket)
transport_control.RegisterInterest(
self.FileSendAck,
fileToSend.remoteID,
fileToSend.packetID)
dhnio.Dprint(12, 'io_throttle.RunSend %s to %s, dt=%s' % (
str(newpacket), nameurl.GetName(fileToSend.remoteID), str(time.time()-dt)))
# mark file as been sent
fileToSend.sendTime = time.time()
packetsSent += 1
# process failed packets
for packetID, why in packetsFialed.items():
self.FileSendFailed(self.fileSendDict[packetID].remoteID, packetID, why)
packetsToRemove.add(packetID)
# remove finished packets
for packetID in packetsToRemove:
self.fileSendQueue.remove(packetID)
del self.fileSendDict[packetID]
# if sending queue is empty - remove all records about packets failed to send
if len(self.fileSendQueue) == 0:
del self.sendFailedPacketIDs[:]
# remember results
result = max(len(packetsToRemove), packetsSent)
# erase temp lists
del packetsFialed
del packetsToRemove
self._runSend = False
return result