def request_files_from_mirrors(requestedfilelist, manifestdict):
"""
<Purpose>
Reconstitutes files by privately contacting mirrors
<Arguments>
requestedfilelist: the files to acquire
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve files. They are written to disk
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
None
"""
neededblocks = []
# let's figure out what blocks we need
for filename in requestedfilelist:
theseblocks = uppirlib.get_blocklist_for_file(filename, manifestdict)
print filename, theseblocks
# add the blocks we don't already know we need to request
for blocknum in theseblocks:
if blocknum not in neededblocks:
neededblocks.append(blocknum)
# do the actual retrieval work
blockdict = request_blocks_from_mirrors(neededblocks, manifestdict)
# now we should write out the files
for filename in requestedfilelist:
filedata = uppirlib.extract_file_from_blockdict(filename, manifestdict, blockdict)#this is the point
# let's check the hash
thisfilehash = uppirlib.find_hash(filedata, manifestdict['hashalgorithm'])
for fileinfo in manifestdict['fileinfolist']:
# find this entry
if fileinfo['filename'] == filename:
if thisfilehash == fileinfo['hash']:
# we found it and it checks out!
break
else:
raise Exception("Corrupt manifest has incorrect file hash despite passing block hash checks")
else:
raise Exception("Internal Error: Cannot locate fileinfo in manifest")
# open the filename w/o the dir and write it
filenamewithoutpath = os.path.basename(filename)
open(filenamewithoutpath,"w").write(filedata)
print "wrote",filenamewithoutpath
def notify_success(self, xorrequesttuple, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
# acquire the lock...
self.tablelock.acquire()
#... but always release it
try:
thismirrorsinfo = xorrequesttuple[0]
# now, let's find the activemirror this corresponds ro.
for activemirrorinfo in self.activemirrorinfolist:
if activemirrorinfo['mirrorinfo'] == thismirrorsinfo:
# let's mark it as inactive and pop off the blocks, etc.
activemirrorinfo['servingrequest'] = False
# remove the block and bitstring (asserting they match what we said
# before)
blocknumber = activemirrorinfo['blocksneeded'].pop(0)
bitstring = activemirrorinfo['blockbitstringlist'].pop(0)
assert(blocknumber == xorrequesttuple[1])
assert(bitstring == xorrequesttuple[2])
# add the xorblockinfo to the dict
xorblockdict = {}
xorblockdict['bitstring'] = bitstring
xorblockdict['mirrorinfo'] = thismirrorsinfo
xorblockdict['xorblock'] = xorblock
self.returnedxorblocksdict[blocknumber].append(xorblockdict)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = uppirlib.find_hash(resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict['blockhashlist'][blocknumber]:
# TODO: We should notify the vendor!
raise Exception('Should notify vendor that one of the mirrors or manifest is corrupt')
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_failure")
finally:
# release the lock
self.tablelock.release()
def notify_success(self, xorrequesttuple, xorblock):
"""
<Purpose>
Handles the receipt of an xorblock
<Arguments>
xorrequesttuple: The tuple that was returned by get_next_xorrequest
xorblock: the data returned by the mirror
<Exceptions>
Assertions / IndexError / TypeError / InternalError if the
XORrequesttuple is bogus
<Returns>
None
"""
# acquire the lock...
self.tablelock.acquire()
#... but always release it
try:
thismirrorsinfo = xorrequesttuple[0]
# now, let's find the activemirror this corresponds ro.
for activemirrorinfo in self.activemirrorinfolist:
if activemirrorinfo['mirrorinfo'] == thismirrorsinfo:
# let's mark it as inactive and pop off the blocks, etc.
activemirrorinfo['servingrequest'] = False
# remove the block and bitstring (asserting they match what we said
# before)
blocknumber = activemirrorinfo['blocksneeded'].pop(0)
bitstring = activemirrorinfo['blockbitstringlist'].pop(0)
assert(blocknumber == xorrequesttuple[1])
assert(bitstring == xorrequesttuple[2])
# add the xorblockinfo to the dict
xorblockdict = {}
xorblockdict['bitstring'] = bitstring
xorblockdict['mirrorinfo'] = thismirrorsinfo
xorblockdict['xorblock'] = xorblock
self.returnedxorblocksdict[blocknumber].append(xorblockdict)
# if we don't have all of the pieces, continue
if len(self.returnedxorblocksdict[blocknumber]) != self.privacythreshold:
return
# if we have all of the pieces, reconstruct it
resultingblock = _reconstruct_block(self.returnedxorblocksdict[blocknumber])
# let's check the hash...
resultingblockhash = uppirlib.find_hash(resultingblock, self.manifestdict['hashalgorithm'])
if resultingblockhash != self.manifestdict['blockhashlist'][blocknumber]:
# TODO: We should notify the vendor!
raise Exception('Should notify vendor that one of the mirrors or manifest is corrupt')
# otherwise, let's put this in the finishedblockdict
self.finishedblockdict[blocknumber] = resultingblock
# it should be safe to delete this
del self.returnedxorblocksdict[blocknumber]
return
raise Exception("InternalError: Unknown mirror in notify_failure")
finally:
# release the lock
self.tablelock.release()
def request_files_from_mirrors(requestedfilelist, manifestdict):
"""
<Purpose>
Reconstitutes files by privately contacting mirrors
<Arguments>
requestedfilelist: the files to acquire
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve files. They are written to disk
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
None
"""
neededblocks = []
# let's figure out what blocks we need
for filename in requestedfilelist:
theseblocks = uppirlib.get_blocklist_for_file(filename, manifestdict)
print filename, theseblocks
# add the blocks we don't already know we need to request
for blocknum in theseblocks:
if blocknum not in neededblocks:
neededblocks.append(blocknum)
# do the actual retrieval work
blockdict = request_blocks_from_mirrors(neededblocks, manifestdict)
# now we should write out the files
for filename in requestedfilelist:
filedata = uppirlib.extract_file_from_blockdict(
filename, manifestdict, blockdict)
# let's check the hash
thisfilehash = uppirlib.find_hash(filedata,
manifestdict['hashalgorithm'])
for fileinfo in manifestdict['fileinfolist']:
# find this entry
if fileinfo['filename'] == filename:
if thisfilehash == fileinfo['hash']:
# we found it and it checks out!
break
else:
raise Exception(
"Corrupt manifest has incorrect file hash despite passing block hash checks"
)
else:
raise Exception(
"Internal Error: Cannot locate fileinfo in manifest")
# open the filename w/o the dir and write it
filenamewithoutpath = os.path.basename(filename)
open(filenamewithoutpath, "w").write(filedata)
print "wrote", filenamewithoutpath
