def request_files_from_mirrors(requestedfilelist, redundancy, rng, parallel, manifestdict):
"""
<Purpose>
Reconstitutes files by privately contacting mirrors
<Arguments>
requestedfilelist: the files to acquire
redundancy: use chunks and overlap this often
rng: use rnd to generate latter chunks
parallel: query one block per chunk
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve files. They are written to disk
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
None
"""
neededblocks = []
#print "Request Files:"
# let's figure out what blocks we need
for filename in requestedfilelist:
theseblocks = lib.get_blocklist_for_file(filename, manifestdict)
# add the blocks we don't already know we need to request
for blocknum in theseblocks:
if blocknum not in neededblocks:
neededblocks.append(blocknum)
# do the actual retrieval work
blockdict = request_blocks_from_mirrors(neededblocks, manifestdict, redundancy, rng, parallel)
# now we should write out the files
for filename in requestedfilelist:
filedata = lib.extract_file_from_blockdict(filename, manifestdict, blockdict)
# let's check the hash
thisfilehash = lib.find_hash(filedata, manifestdict['hashalgorithm'])
for fileinfo in manifestdict['fileinfolist']:
# find this entry
if fileinfo['filename'] == filename:
if thisfilehash == fileinfo['hash']:
# we found it and it checks out!
break
else:
raise Exception("Corrupt manifest has incorrect file hash despite passing block hash checks!")
else:
raise Exception("Internal Error: Cannot locate fileinfo in manifest!")
# open the filename w/o the dir and write it
filenamewithoutpath = os.path.basename(filename)
open(filenamewithoutpath, "w").write(filedata)
print "wrote", filenamewithoutpath
def request_files_from_mirrors(requestedfilelist, redundancy, rng, parallel, manifestdict):
"""
<Purpose>
Reconstitutes files by privately contacting mirrors
<Arguments>
requestedfilelist: the files to acquire
redundancy: use chunks and overlap this often
rng: use rnd to generate latter chunks
parallel: query one block per chunk
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve files. They are written to disk
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
None
"""
neededblocks = []
#print "Request Files:"
# let's figure out what blocks we need
for filename in requestedfilelist:
theseblocks = lib.get_blocklist_for_file(filename, manifestdict)
# add the blocks we don't already know we need to request
for blocknum in theseblocks:
if blocknum not in neededblocks:
neededblocks.append(blocknum)
# do the actual retrieval work
blockdict = request_blocks_from_mirrors(neededblocks, manifestdict, redundancy, rng, parallel)
# now we should write out the files
for filename in requestedfilelist:
filedata = lib.extract_file_from_blockdict(filename, manifestdict, blockdict)
# let's check the hash
thisfilehash = lib.find_hash(filedata, manifestdict['hashalgorithm'])
for fileinfo in manifestdict['fileinfolist']:
# find this entry
if fileinfo['filename'] == filename:
if thisfilehash == fileinfo['hash']:
# we found it and it checks out!
break
else:
raise Exception("Corrupt manifest has incorrect file hash despite passing block hash checks!")
else:
raise Exception("Internal Error: Cannot locate fileinfo in manifest!")
# open the filename w/o the dir and write it
filenamewithoutpath = os.path.basename(filename)
open(filenamewithoutpath, "wb").write(filedata)
print("wrote", filenamewithoutpath)
def notify_success(self, thismirrorsinfo, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
if self.timing:
stime = _timer()
# acquire the lock...
self.tablelock.acquire()
try:
# now, let's find the activemirror this corresponds to.
for mirror in self.activemirrors:
if mirror['info'] == thismirrorsinfo:
if self.parallel:
#use blocknumbers[0] as index from now on
blocknumbers = mirror['blocksrequested'].pop(0)
# add the xorblocks to the dict
self.returnedxorblocksdict[blocknumbers[0]].append(
msgpack.unpackb(xorblock, raw=False))
#print "Appended blocknumber", blocknumbers[0], "from", thismirrorsinfo['port']
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[
blocknumbers[0]]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblockdict = _reconstruct_block_parallel(
self.returnedxorblocksdict[blocknumbers[0]],
self.chunklen, self.privacythreshold,
self.manifestdict['blocksize'], blocknumbers)
#parse resultingblocks into single blocks
for blocknumber in blocknumbers:
index = min(int(blocknumber / self.chunklen),
self.privacythreshold - 1)
# let's check the hash...
resultingblockhash = lib.find_hash(
resultingblockdict[index],
self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict[
'blockhashlist'][blocknumber]:
print(mirror)
# TODO: We should notify the vendor!
raise Exception(
'Should notify vendor that one of the mirrors or manifest is corrupt, for blocknumber '
+ str(blocknumber))
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[
blocknumber] = resultingblockdict[index]
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumbers[0]]
return
#single block query:
else:
# remove the block and bitstring (asserting they match what we said before)
blocknumber = mirror['blocksrequested'].pop(0)
# add the xorblock to the dict
self.returnedxorblocksdict[blocknumber].append(
xorblock)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]
) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(
self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = lib.find_hash(
resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict[
'blockhashlist'][blocknumber]:
print(mirror)
# TODO: We should notify the vendor!
raise Exception(
'Should notify vendor that one of the mirrors or manifest is corrupt'
)
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_success")
finally:
# release the lock
self.tablelock.release()
if self.timing:
self.recons_time = self.recons_time + _timer() - stime
def notify_success(self, thismirrorsinfo, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
if self.timing:
stime = _timer()
# acquire the lock...
self.tablelock.acquire()
#... but always release it
try:
# now, let's find the activemirror this corresponds to.
for mirror in self.activemirrors:
if mirror['info'] == thismirrorsinfo:
# remove the block and bitstring (asserting they match what we said before)
blocknumber = mirror['blocksrequested'].pop(0)
# add the xorblockinfo to the dict
self.returnedxorblocksdict[blocknumber].append(xorblock)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]
) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(
self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = lib.find_hash(
resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict[
'blockhashlist'][blocknumber]:
# TODO: We should notify the vendor!
raise Exception(
'Should notify vendor that one of the mirrors or manifest is corrupt'
)
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_success")
finally:
# release the lock
self.tablelock.release()
if self.timing:
self.recons_time = self.recons_time + _timer() - stime
def notify_success(self, thismirrorsinfo, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
if self.timing:
stime = _timer()
# acquire the lock...
self.tablelock.acquire()
try:
# now, let's find the activemirror this corresponds to.
for mirror in self.activemirrors:
if mirror['info'] == thismirrorsinfo:
if self.parallel:
#use blocknumbers[0] as index from now on
blocknumbers = mirror['blocksrequested'].pop(0)
# add the xorblocks to the dict
self.returnedxorblocksdict[blocknumbers[0]].append(msgpack.unpackb(xorblock, raw=False))
#print "Appended blocknumber", blocknumbers[0], "from", thismirrorsinfo['port']
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumbers[0]]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblockdict = _reconstruct_block_parallel(self.returnedxorblocksdict[blocknumbers[0]], self.chunklen, self.privacythreshold, self.manifestdict['blocksize'], blocknumbers)
#parse resultingblocks into single blocks
for blocknumber in blocknumbers:
index = min(int(blocknumber/self.chunklen), self.privacythreshold-1)
# let's check the hash...
resultingblockhash = lib.find_hash(resultingblockdict[index], self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict['blockhashlist'][blocknumber]:
print(mirror)
# TODO: We should notify the vendor!
raise Exception('Should notify vendor that one of the mirrors or manifest is corrupt, for blocknumber ' + str(blocknumber))
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblockdict[index]
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumbers[0]]
return
#single block query:
else:
# remove the block and bitstring (asserting they match what we said before)
blocknumber = mirror['blocksrequested'].pop(0)
# add the xorblock to the dict
self.returnedxorblocksdict[blocknumber].append(xorblock)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = lib.find_hash(resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict['blockhashlist'][blocknumber]:
print(mirror)
# TODO: We should notify the vendor!
raise Exception('Should notify vendor that one of the mirrors or manifest is corrupt')
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_success")
finally:
# release the lock
self.tablelock.release()
if self.timing:
self.recons_time = self.recons_time + _timer() - stime
def notify_success(self, thismirrorsinfo, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
if self.timing:
stime = _timer()
# acquire the lock...
self.tablelock.acquire()
#... but always release it
try:
# now, let's find the activemirror this corresponds to.
for mirror in self.activemirrors:
if mirror['info'] == thismirrorsinfo:
# remove the block and bitstring (asserting they match what we said before)
blocknumber = mirror['blocksrequested'].pop(0)
# add the xorblockinfo to the dict
self.returnedxorblocksdict[blocknumber].append(xorblock)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = lib.find_hash(resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict['blockhashlist'][blocknumber]:
# TODO: We should notify the vendor!
raise Exception('Should notify vendor that one of the mirrors or manifest is corrupt')
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_success")
finally:
# release the lock
self.tablelock.release()
if self.timing:
self.recons_time = self.recons_time + _timer() - stime