# this is a bunch of macro tests. If everything passes, there is no output.
import fastsimplexordatastore
size = 64
letterxordatastore = fastsimplexordatastore.XORDatastore(size, 16)
startpos = 0
for char in range(ord("A"), ord("Q")):
# put 1K of those chars in...
letterxordatastore.set_data(startpos, chr(char) * size)
startpos = startpos + size
# can read data out...
assert (letterxordatastore.get_data(size, 1) == 'B')
# let's create a bitstring that uses A, C, and P.
bitstring = chr(int('10100000', 2)) + chr(int('00000001', 2))
xorresult = letterxordatastore.produce_xor_from_bitstring(bitstring)
assert (xorresult[0] == 'R')
letterxordatastore.set_data(10, "Hello there")
mystring = letterxordatastore.get_data(9, 13)
assert (mystring == 'AHello thereA')
letterxordatastore.set_data(1, "Hello there" * size)
mystring = letterxordatastore.get_data(size * 2 - (size * 2 % 11) + 1, 11)
def main():
global _global_myxordatastore
global _global_manifestdict
global _batchlock
global _batchevent
global _xorstrings
global _batchrequests
global _request_restart
manifestdict = retrieve_manifest_dict()
# We should detach here. I don't do it earlier so that error
# messages are written to the terminal... I don't do it later so that any
# threads don't exist already. If I do put it much later, the code hangs...
if _commandlineoptions.daemonize:
daemon.daemonize()
if _commandlineoptions.database != None:
print("Using mmap datastore")
dstype = "mmap"
source = _commandlineoptions.database
else:
print("Using RAM datastore")
dstype = "RAM"
source = _commandlineoptions.files
myxordatastore = fastsimplexordatastore.XORDatastore(
manifestdict['blocksize'], manifestdict['blockcount'], dstype, source,
_commandlineoptions.use_precomputed_data)
if dstype == "RAM":
# now let's put the content in the datastore in preparation to serve it
print("Loading data into RAM datastore...")
start = _timer()
lib.populate_xordatastore(manifestdict, myxordatastore, source, dstype,
_commandlineoptions.use_precomputed_data)
elapsed = (_timer() - start)
print("Datastore initialized. Took %f seconds." % elapsed)
# we're now ready to handle clients!
#_log('ready to start servers!')
# an ugly hack, but Python's request handlers don't have an easy way to pass arguments
_global_myxordatastore = myxordatastore
_global_manifestdict = manifestdict
_batchlock = threading.Lock()
_batchevent = threading.Event()
_batchrequests = 0
_xorstrings = b''
# first, let's fire up the RAID-PIR server
xorserver = service_raidpir_clients(myxordatastore, _commandlineoptions.ip,
_commandlineoptions.port)
# If I should serve legacy clients via HTTP, let's start that up...
if _commandlineoptions.http:
service_http_clients(myxordatastore, manifestdict,
_commandlineoptions.ip,
_commandlineoptions.httpport)
#_log('servers started!')
print("Mirror Server started at", _commandlineoptions.ip, ":",
_commandlineoptions.port)
# let's send the mirror information periodically...
# we should log any errors...
_send_mirrorinfo()
counter = 0
while True:
if counter > _commandlineoptions.mirrorlistadvertisedelay:
counter = 0
try:
_send_mirrorinfo()
except Exception as e:
_log(
str(e) + "\n" +
str(traceback.format_tb(sys.exc_info()[2])))
if _request_restart:
print("Shutting down")
xorserver.shutdown()
sys.exit(0)
counter = counter + 1
time.sleep(1)
def main():
global _global_myxordatastore
global _global_manifestdict
# If we were asked to retrieve the mainfest file, do so...
if _commandlineoptions.retrievemanifestfrom:
# We need to download this file...
rawmanifestdata = uppirlib.retrieve_rawmanifest(_commandlineoptions.retrievemanifestfrom)
# ...make sure it is valid...
manifestdict = uppirlib.parse_manifest(rawmanifestdata)
# ...and write it out if it's okay
open(_commandlineoptions.manifestfilename, "w").write(rawmanifestdata)
else:
# Simply read it in from disk
rawmanifestdata = open(_commandlineoptions.manifestfilename).read()
manifestdict = uppirlib.parse_manifest(rawmanifestdata)
# We should detach here. I don't do it earlier so that error
# messages are written to the terminal... I don't do it later so that any
# threads don't exist already. If I do put it much later, the code hangs...
if _commandlineoptions.daemonize:
daemon.daemonize()
myxordatastore = fastsimplexordatastore.XORDatastore(manifestdict['blocksize'], manifestdict['blockcount'])
# now let's put the content in the datastore in preparation to serve it
uppirlib.populate_xordatastore(manifestdict, myxordatastore, rootdir = _commandlineoptions.mirrorroot)
# we're now ready to handle clients!
_log('ready to start servers!')
# an ugly hack, but Python's request handlers don't have an easy way to
# pass arguments
_global_myxordatastore = myxordatastore
_global_manifestdict = manifestdict
# first, let's fire up the upPIR server
service_uppir_clients(myxordatastore, _commandlineoptions.ip, _commandlineoptions.port)
# If I should serve legacy clients via HTTP, let's start that up...
if _commandlineoptions.http:
service_http_clients(myxordatastore, manifestdict, _commandlineoptions.ip, _commandlineoptions.httpport)
_log('servers started!')
# let's send the mirror information periodically...
# we should log any errors...
while True:
try:
_send_mirrorinfo()
except Exception, e:
_log(str(e)+"\n"+str(traceback.format_tb(sys.exc_info()[2])))
time.sleep(_commandlineoptions.mirrorlistadvertisedelay)
# this is a bunch of macro tests. If everything passes, there is no output.
import fastsimplexordatastore
size = 64 # block size in Byte
num_blocks = 16 # number of blocks
letterxordatastore = fastsimplexordatastore.XORDatastore(
size, num_blocks, "ram", "db_name")
startpos = 0
for char in range(ord("A"), ord("Q")):
# put 1K of those chars in...
letterxordatastore.set_data(startpos, chr(char) * size)
startpos = startpos + size
# can read data out...
assert (letterxordatastore.get_data(size, 1) == 'B')
letterxordatastore.finalize()
# let's create a bitstring that uses A, C, and P.
bitstring = chr(int('10100000', 2)) + chr(int('00000001', 2))
xorresult = letterxordatastore.produce_xor_from_bitstring(bitstring)
assert (xorresult[0] == 'R')
xorresult = letterxordatastore.produce_xor_from_multiple_bitstrings(
bitstring, 1)
assert (xorresult[0] == 'R')
def create_manifest(
rootdir=".",
hashalgorithm="sha1-base64",
block_size=1024 * 1024,
offset_assignment_function=nogaps_offset_assignment_function,
vendorhostname=None,
vendorport=62293):
"""
<Purpose>
Create a manifest (and an xordatastore ?)
<Arguments>
rootdir: The area to walk looking for files to add to the manifest
hashalgorithm: The hash algorithm to use to validate file contents
block_size: The size of a block of data.
offset_assignment_function: specifies how to lay out the files in blocks.
<Exceptions>
TypeError if the arguments are corrupt or of the wrong type
FileNotFound if the rootdir does not contain a manifest file.
IncorrectFileContents if the file listed in the manifest file has the
wrong size or hash
<Side Effects>
This function creates an XORdatastore while processing. This may use
a very large amount of memory. (This is not fundamental, and is done only
for convenience).
<Returns>
The manifest dictionary
"""
if vendorhostname == None:
raise TypeError("Must specify vendor server name")
if ':' in vendorhostname:
raise TypeError("Vendor server name must not contain ':'")
# general workflow:
# set the global parameters
# build an xordatastore and add file information as you go...
# derive hash information from the xordatastore
manifestdict = {}
manifestdict['manifestversion'] = "1.0"
manifestdict['hashalgorithm'] = hashalgorithm
manifestdict['blocksize'] = block_size
manifestdict['vendorhostname'] = vendorhostname
manifestdict['vendorport'] = vendorport
# first get the file information
fileinfolist = _generate_fileinfolist(rootdir,
manifestdict['hashalgorithm'])
# now let's assign the files to offsets as the caller requess...
offset_assignment_function(fileinfolist, rootdir,
manifestdict['blocksize'])
# let's ensure the offsets are valid...
# build a list of tuples with offset, etc. info...
offsetlengthtuplelist = []
for fileinfo in fileinfolist:
offsetlengthtuplelist.append((fileinfo['offset'], fileinfo['length']))
# ...sort the tuples so that it's easy to walk down them and check for
# overlapping entries...
offsetlengthtuplelist.sort()
# ...now, we need to ensure the values don't overlap.
nextfreeoffset = 0
for offset, length in offsetlengthtuplelist:
if offset < 0:
raise TypeError("Offset generation led to negative offset!")
if length < 0:
raise TypeError("File lengths must be positive!")
if nextfreeoffset > offset:
raise TypeError(
"Error! Offset generation led to overlapping files!")
# since this list is sorted by offset, this should ensure the property we
# want is upheld.
nextfreeoffset = offset + length
# great! The fileinfolist is okay!
manifestdict['fileinfolist'] = fileinfolist
# The nextfreeoffset value is the end of the datastore... Let's see how
# many blocks we need
manifestdict['blockcount'] = int(
math.ceil(nextfreeoffset * 1.0 / manifestdict['blocksize']))
# TODO: Improve this. It really shouldn't use a datastore...
import fastsimplexordatastore
xordatastore = fastsimplexordatastore.XORDatastore(
manifestdict['blocksize'], manifestdict['blockcount'])
# now let's put the files in the datastore
_add_data_to_datastore(xordatastore, manifestdict['fileinfolist'], rootdir,
manifestdict['hashalgorithm'])
# and it is time to get the blockhashlist...
manifestdict['blockhashlist'] = _compute_block_hashlist(
xordatastore, manifestdict['blockcount'], manifestdict['blocksize'],
manifestdict['hashalgorithm'])
# let's generate the manifest's hash
rawmanifest = json.dumps(manifestdict)
manifestdict['manifesthash'] = find_hash(rawmanifest,
manifestdict['hashalgorithm'])
# we are done!
return manifestdict