def handle(self):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
global _global_myxordatastore
global _global_manifestdict
global _request_restart
_finish = False
comp_time = 0
_batch_comp_time = 0
_batchrequests = 0
_xorstrings = b''
parallel = False
requeststring = b'0'
while requeststring != b'Q':
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
# remoteip, remoteport = self.request.getpeername()
start_time = _timer()
# if it's a request for a XORBLOCK
if requeststring.startswith(b'X'):
bitstring = requeststring[len(b'X'):]
expectedbitstringlength = lib.bits_to_bytes(_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
_finish = True
_batchevent.set()
return
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and immediately send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
# done!
elif requeststring.startswith(b'C'):
payload = requeststring[len(b'C'):]
chunks = msgpack.unpackb(payload, raw=False)
bitstring = lib.build_bitstring_from_chunks(chunks, k, chunklen, lastchunklen)
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring.startswith(b'R'):
payload = requeststring[len(b'R'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstring = lib.build_bitstring_from_chunks(chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring == b'MANIFEST UPDATE':
print("MANIFEST UPDATE")
_request_restart = True
elif requeststring.startswith(b'M'):
parallel = True
payload = requeststring[len(b'M'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstrings = lib.build_bitstring_from_chunks_parallel(chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
result = {}
for c in chunknumbers:
result[c] = _global_myxordatastore.produce_xor_from_bitstring(bitstrings[c])
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, msgpack.packb(result, use_bin_type=True))
else:
with _batchlock:
for c in chunknumbers:
_xorstrings += bitstrings[c]
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
#done!
elif requeststring.startswith(b'P'):
payload = requeststring[len(b'P'):]
params = msgpack.unpackb(payload, raw=False)
chunknumbers = params['cn']
k = params['k']
r = params['r']
chunklen = params['cl']
lastchunklen = params['lcl']
batch = params['b']
parallel = params['p']
if 's' in params:
cipher = lib.initAES(params['s'])
if batch:
# create batch xor thread
t = threading.Thread(target=BatchAnswer, args=[parallel, chunknumbers, self.request], name="RAID-PIR Batch XOR")
t.daemon = True
t.start()
# and send the reply.
session.sendmessage(self.request, b"PARAMS OK")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" PARAMS received " + str(params))
#done!
#Timing Request
elif requeststring == b'T':
session.sendmessage(self.request, b"T" + str(comp_time + _batch_comp_time))
comp_time = 0
_batch_comp_time = 0
#Debug Hello
elif requeststring == b'HELLO':
session.sendmessage(self.request, b"HI!")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
#the client asked to close the connection
elif requeststring == b'Q':
comp_time = 0
_finish = True
_batchevent.set()
return
#this happens if the client closed the socket unexpectedly
elif requeststring == b'':
comp_time = 0
_finish = True
_batchevent.set()
return
else:
# we don't know what this is! Log and tell the requestor
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
_finish = True
_batchevent.set()
return
def request_blocks_from_mirrors(requestedblocklist, manifestdict, redundancy, rng, parallel):
"""
<Purpose>
Retrieves blocks from mirrors
<Arguments>
requestedblocklist: the blocks to acquire
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve blocks. It uses some global options
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
A dict mapping blocknumber -> blockcontents.
"""
# let's get the list of mirrors...
if _commandlineoptions.vendorip == None:
# use data from manifest
mirrorinfolist = lib.retrieve_mirrorinfolist(manifestdict['vendorhostname'], manifestdict['vendorport'])
else:
# use commandlineoption
mirrorinfolist = lib.retrieve_mirrorinfolist(_commandlineoptions.vendorip)
print "Mirrors: ", mirrorinfolist
if _commandlineoptions.timing:
setup_start = _timer()
# no chunks (regular upPIR / Chor)
if redundancy == None:
# let's set up a requestor object...
rxgobj = simplexorrequestor.RandomXORRequestor(mirrorinfolist, requestedblocklist, manifestdict, _commandlineoptions.numberofmirrors, _commandlineoptions.batch, _commandlineoptions.timing)
if _commandlineoptions.timing:
setup_time = _timer() - setup_start
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockbitstringlist']))+"\n")
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockbitstringlist']))+"\n")
print "Blocks to request:", len(rxgobj.activemirrors[0]['blockbitstringlist'])
if _commandlineoptions.timing:
req_start = _timer()
# let's fire up the requested number of threads. Our thread will also participate (-1 because of us!)
for tid in xrange(_commandlineoptions.numberofmirrors - 1):
threading.Thread(target=_request_helper, args=[rxgobj, tid]).start()
_request_helper(rxgobj, _commandlineoptions.numberofmirrors - 1)
# wait for receiving threads to finish
for mirror in rxgobj.activemirrors:
mirror['rt'].join()
else: # chunks
# let's set up a chunk requestor object...
rxgobj = simplexorrequestor.RandomXORRequestorChunks(mirrorinfolist, requestedblocklist, manifestdict, _commandlineoptions.numberofmirrors, redundancy, rng, parallel, _commandlineoptions.batch, _commandlineoptions.timing)
if _commandlineoptions.timing:
setup_time = _timer() - setup_start
_timing_log.write(str(len(rxgobj.activemirrors[0]['blocksneeded']))+"\n")
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockchunklist']))+"\n")
print "# Blocks needed:", len(rxgobj.activemirrors[0]['blocksneeded'])
if parallel:
print "# Requests:", len(rxgobj.activemirrors[0]['blockchunklist'])
#chunk lengths in BYTE
global chunklen
global lastchunklen
chunklen = (manifestdict['blockcount'] / 8) / _commandlineoptions.numberofmirrors
lastchunklen = lib.bits_to_bytes(manifestdict['blockcount']) - (_commandlineoptions.numberofmirrors-1)*chunklen
if _commandlineoptions.timing:
req_start = _timer()
# let's fire up the requested number of threads. Our thread will also participate (-1 because of us!)
for tid in xrange(_commandlineoptions.numberofmirrors - 1):
threading.Thread(target=_request_helper_chunked, args=[rxgobj, tid]).start()
_request_helper_chunked(rxgobj, _commandlineoptions.numberofmirrors - 1)
# wait for receiving threads to finish
for mirror in rxgobj.activemirrors:
mirror['rt'].join()
rxgobj.cleanup()
if _commandlineoptions.timing:
req_time = _timer() - req_start
recons_time, comptimes, pings = rxgobj.return_timings()
avg_ping = sum(pings) / _commandlineoptions.numberofmirrors
avg_comptime = sum(comptimes) / _commandlineoptions.numberofmirrors
_timing_log.write(str(setup_time)+ "\n")
_timing_log.write(str(req_time)+ "\n")
_timing_log.write(str(recons_time)+ "\n")
_timing_log.write(str(avg_comptime)+ " " + str(comptimes)+ "\n")
_timing_log.write(str(avg_ping)+ " " + str(pings)+ "\n")
# okay, now we have them all. Let's get the returned dict ready.
retdict = {}
for blocknum in requestedblocklist:
retdict[blocknum] = rxgobj.return_block(blocknum)
return retdict
def handle(self):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
global _global_myxordatastore
global _global_manifestdict
global _request_restart
_finish = False
comp_time = 0
_batch_comp_time = 0
_batchrequests = 0
_xorstrings = b''
parallel = False
requeststring = b'0'
while requeststring != b'Q':
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
# remoteip, remoteport = self.request.getpeername()
start_time = _timer()
# if it's a request for a XORBLOCK
if requeststring.startswith(b'X'):
bitstring = requeststring[len(b'X'):]
expectedbitstringlength = lib.bits_to_bytes(
_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
_finish = True
_batchevent.set()
return
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and immediately send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
# done!
elif requeststring.startswith(b'C'):
payload = requeststring[len(b'C'):]
chunks = msgpack.unpackb(payload, raw=False)
bitstring = lib.build_bitstring_from_chunks(
chunks, k, chunklen, lastchunklen)
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring.startswith(b'R'):
payload = requeststring[len(b'R'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstring = lib.build_bitstring_from_chunks(
chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring == b'MANIFEST UPDATE':
print("MANIFEST UPDATE")
_request_restart = True
elif requeststring.startswith(b'M'):
parallel = True
payload = requeststring[len(b'M'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstrings = lib.build_bitstring_from_chunks_parallel(
chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
result = {}
for c in chunknumbers:
result[
c] = _global_myxordatastore.produce_xor_from_bitstring(
bitstrings[c])
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(
self.request, msgpack.packb(result, use_bin_type=True))
else:
with _batchlock:
for c in chunknumbers:
_xorstrings += bitstrings[c]
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
#done!
elif requeststring.startswith(b'P'):
payload = requeststring[len(b'P'):]
params = msgpack.unpackb(payload, raw=False)
chunknumbers = params['cn']
k = params['k']
r = params['r']
chunklen = params['cl']
lastchunklen = params['lcl']
batch = params['b']
parallel = params['p']
if 's' in params:
cipher = lib.initAES(params['s'])
if batch:
# create batch xor thread
t = threading.Thread(
target=BatchAnswer,
args=[parallel, chunknumbers, self.request],
name="RAID-PIR Batch XOR")
t.daemon = True
t.start()
# and send the reply.
session.sendmessage(self.request, b"PARAMS OK")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" PARAMS received " + str(params))
#done!
#Timing Request
elif requeststring == b'T':
session.sendmessage(self.request,
b"T" + str(comp_time + _batch_comp_time))
comp_time = 0
_batch_comp_time = 0
#Debug Hello
elif requeststring == b'HELLO':
session.sendmessage(self.request, b"HI!")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
#the client asked to close the connection
elif requeststring == b'Q':
comp_time = 0
_finish = True
_batchevent.set()
return
#this happens if the client closed the socket unexpectedly
elif requeststring == b'':
comp_time = 0
_finish = True
_batchevent.set()
return
else:
# we don't know what this is! Log and tell the requestor
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
_finish = True
_batchevent.set()
return
def request_blocks_from_mirrors(requestedblocklist, manifestdict, redundancy, rng, parallel):
"""
<Purpose>
Retrieves blocks from mirrors
<Arguments>
requestedblocklist: the blocks to acquire
manifestdict: the manifest with information about the release
<Side Effects>
Contacts mirrors to retrieve blocks. It uses some global options
<Exceptions>
TypeError may be raised if the provided lists are invalid.
socket errors may be raised if communications fail.
<Returns>
A dict mapping blocknumber -> blockcontents.
"""
# let's get the list of mirrors...
if _commandlineoptions.vendorip == None:
# use data from manifest
mirrorinfolist = lib.retrieve_mirrorinfolist(manifestdict['vendorhostname'], manifestdict['vendorport'])
else:
# use commandlineoption
mirrorinfolist = lib.retrieve_mirrorinfolist(_commandlineoptions.vendorip)
print("Mirrors: ", mirrorinfolist)
if _commandlineoptions.timing:
setup_start = _timer()
# no chunks (regular upPIR / Chor)
if redundancy == None:
# let's set up a requestor object...
rxgobj = simplexorrequestor.RandomXORRequestor(mirrorinfolist, requestedblocklist, manifestdict, _commandlineoptions.numberofmirrors, _commandlineoptions.batch, _commandlineoptions.timing)
if _commandlineoptions.timing:
setup_time = _timer() - setup_start
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockbitstringlist']))+"\n")
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockbitstringlist']))+"\n")
print("Blocks to request:", len(rxgobj.activemirrors[0]['blockbitstringlist']))
if _commandlineoptions.timing:
req_start = _timer()
# let's fire up the requested number of threads. Our thread will also participate (-1 because of us!)
for tid in range(_commandlineoptions.numberofmirrors - 1):
threading.Thread(target=_request_helper, args=[rxgobj, tid]).start()
_request_helper(rxgobj, _commandlineoptions.numberofmirrors - 1)
# wait for receiving threads to finish
for mirror in rxgobj.activemirrors:
mirror['rt'].join()
else: # chunks
# let's set up a chunk requestor object...
rxgobj = simplexorrequestor.RandomXORRequestorChunks(mirrorinfolist, requestedblocklist, manifestdict, _commandlineoptions.numberofmirrors, redundancy, rng, parallel, _commandlineoptions.batch, _commandlineoptions.timing)
if _commandlineoptions.timing:
setup_time = _timer() - setup_start
_timing_log.write(str(len(rxgobj.activemirrors[0]['blocksneeded']))+"\n")
_timing_log.write(str(len(rxgobj.activemirrors[0]['blockchunklist']))+"\n")
print("# Blocks needed:", len(rxgobj.activemirrors[0]['blocksneeded']))
if parallel:
print("# Requests:", len(rxgobj.activemirrors[0]['blockchunklist']))
#chunk lengths in BYTE
global chunklen
global lastchunklen
chunklen = (manifestdict['blockcount'] / 8) / _commandlineoptions.numberofmirrors
lastchunklen = lib.bits_to_bytes(manifestdict['blockcount']) - (_commandlineoptions.numberofmirrors-1)*chunklen
if _commandlineoptions.timing:
req_start = _timer()
# let's fire up the requested number of threads. Our thread will also participate (-1 because of us!)
for tid in range(_commandlineoptions.numberofmirrors - 1):
threading.Thread(target=_request_helper_chunked, args=[rxgobj, tid]).start()
_request_helper_chunked(rxgobj, _commandlineoptions.numberofmirrors - 1)
# wait for receiving threads to finish
for mirror in rxgobj.activemirrors:
mirror['rt'].join()
rxgobj.cleanup()
if _commandlineoptions.timing:
req_time = _timer() - req_start
recons_time, comptimes, pings = rxgobj.return_timings()
avg_ping = sum(pings) / _commandlineoptions.numberofmirrors
avg_comptime = sum(comptimes) / _commandlineoptions.numberofmirrors
_timing_log.write(str(setup_time)+ "\n")
_timing_log.write(str(req_time)+ "\n")
_timing_log.write(str(recons_time)+ "\n")
_timing_log.write(str(avg_comptime)+ " " + str(comptimes)+ "\n")
_timing_log.write(str(avg_ping)+ " " + str(pings)+ "\n")
# okay, now we have them all. Let's get the returned dict ready.
retdict = {}
for blocknum in requestedblocklist:
retdict[blocknum] = rxgobj.return_block(blocknum)
return retdict
def __init__(self, mirrorinfolist, blocklist, manifestdict,
privacythreshold, redundancy, rng, parallel, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
This is meant to be used for queries partitioned in chunks
(parallel or SB queries with redundancy parameter)
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold # aka k, the number of mirrors to use
self.redundancy = redundancy # aka r
self.rng = rng
self.parallel = parallel
self.blockcount = manifestdict['blockcount']
self.timing = timing
if timing:
self.recons_time = 0
#length of one chunk in BITS (1 bit per block)
#chunk length of the first chunks must be a multiple of 8, last chunk can be longer than first chunks
self.chunklen = int(self.blockcount / 8 / privacythreshold) * 8
self.lastchunklen = self.blockcount - (privacythreshold -
1) * self.chunklen
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " +
str(self.privacythreshold) +
" mirrors, but only " +
str(len(mirrorinfolist)) +
" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
#initialize queries for mirrors
i = 0
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirror = {}
mirror['info'] = mirrorinfo
mirror[
'blocksneeded'] = blocklist[:] # only for the client, obviously
mirror['blocksrequested'] = []
if parallel:
mirror['parallelblocksneeded'] = []
mirror['blockchunklist'] = []
# chunk numbers [0, ..., r-1]
mirror['chunknumbers'] = [i]
for j in range(1, redundancy):
mirror['chunknumbers'].append((i + j) % privacythreshold)
i = i + 1
#open a socket once:
mirror['info']['sock'] = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
mirror['info']['sock'].connect(
(mirrorinfo['ip'], mirrorinfo['port']))
if rng:
#pick a random seed (key) and initialize AES
seed = _randomnumberfunction(16) # random 128 bit key
mirror['seed'] = seed
mirror['cipher'] = lib.initAES(seed)
self.activemirrors.append(mirror)
for mirror in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = mirror['chunknumbers']
params['k'] = privacythreshold
params['r'] = redundancy
params['cl'] = self.chunklen
params['lcl'] = self.lastchunklen
params['b'] = batch
params['p'] = parallel
if rng:
params['s'] = mirror['seed']
#send the params, rcvlet will check response
session.sendmessage(
mirror['info']['sock'],
b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(
target=rcvlet,
args=[mirror, self],
name=("rcv_thread_" + str(
(mirror['info']['ip'], mirror['info']['port']))))
mirror['rt'] = t
t.start()
#multi block query. map the blocks to the minimum amount of queries
if parallel:
#create dictionary for each chunk, will hold block indices per chunk
blockchunks = {}
for i in range(0, privacythreshold):
blockchunks[i] = []
#map block numbers to chunks
for blocknum in blocklist:
index = min(int(blocknum / self.chunklen),
privacythreshold - 1)
blockchunks[index].append(blocknum)
#remove chunks that are still empty
for i in range(0, privacythreshold):
if len(blockchunks[i]) == 0:
del blockchunks[i]
#do until all blocks are in queries
while len(blockchunks) > 0:
#iterate through mirrors
for mirror in self.activemirrors:
#dicitonary of chunk requests
chunks = {}
#iterate through r-1 random chunks, skipping the head (flip) chunk
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
#set random bytes for the latter chunk(s) from AES (will be deleted later)
chunks[c] = lib.nextrandombitsAES(
mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s) randomly
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
#list of blocknumbers
blocks = []
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length) * b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][
c] #remove the pre-computed random chunk from the packet to send
#if there is a block within this chunk, then add it to the bitstring by flipping the bit
if c in blockchunks:
blocknum = blockchunks[c].pop(0)
thisbitstring = lib.flip_bitstring_bit(
thisbitstring, blocknum - c * self.chunklen)
blocks.append(blocknum)
if len(blockchunks[c]) == 0:
del blockchunks[c]
mirror['parallelblocksneeded'].append(blocks)
mirror['blockchunklist'][-1][c] = thisbitstring
#single block query:
else:
#iterate through all blocks
for blocknum in blocklist:
#iterate through mirrors
for mirror in self.activemirrors:
chunks = {}
#iterate through r-1 random chunks
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
chunks[c] = lib.nextrandombitsAES(
mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s)
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length) * b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][
c] #remove the pre-computed random chunk from the packet to send
#if the desired block is within this chunk, flip the bit
if c * self.chunklen <= blocknum and blocknum < c * self.chunklen + length:
thisbitstring = lib.flip_bitstring_bit(
thisbitstring, blocknum - c * self.chunklen)
mirror['blockchunklist'][-1][c] = thisbitstring
########################################
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.
privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def __init__(self, mirrorinfolist, blocklist, manifestdict,
privacythreshold, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
<Arguments>
mirrorinfolist: a list of dictionaries with information about mirrors
blocklist: the blocks that need to be retrieved
manifestdict: the manifest with information about the release
privacythreshold: the number of mirrors that would need to collude to break privacy
timing: collect timing info
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold
self.timing = timing
if timing:
self.recons_time = 0
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " +
str(self.privacythreshold) +
" mirrors, but only " +
str(len(mirrorinfolist)) +
" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirrors = {}
mirrors['info'] = mirrorinfo
mirrors['blocksneeded'] = blocklist[:]
mirrors['blockbitstringlist'] = []
mirrors['blocksrequested'] = []
# open a socket once:
mirrors['info']['sock'] = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
mirrors['info']['sock'].setsockopt(
socket.IPPROTO_TCP, socket.TCP_NODELAY,
1) #TODO check this in the cloud
mirrors['info']['sock'].connect(
(mirrorinfo['ip'], mirrorinfo['port']))
self.activemirrors.append(mirrors)
for thisrequestinfo in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = 1 # chunk numbers, here fixed to 1
params['k'] = privacythreshold
params[
'r'] = privacythreshold # r is irrelevant here, thus fixed to k
params['cl'] = 1 # chunk length, here fixed to 1
params['lcl'] = 1 # last chunk length, here fixed to 1
params['b'] = batch
params['p'] = False
#send the params, rcvlet will check response
session.sendmessage(
thisrequestinfo['info']['sock'],
b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet,
args=[thisrequestinfo, self],
name=("rcv_thread_" + str(
(thisrequestinfo['info']['ip'],
thisrequestinfo['info']['port']))))
thisrequestinfo['rt'] = t
t.start()
bitstringlength = lib.bits_to_bytes(manifestdict['blockcount'])
# let's generate the random bitstrings for k-1 mirrors
for thisrequestinfo in self.activemirrors[:-1]:
for _ in blocklist:
thisrequestinfo['blockbitstringlist'].append(
lib.randombits(manifestdict['blockcount']))
# now, let's do the 'derived' ones...
for blocknum in range(len(blocklist)):
thisbitstring = b'\0' * bitstringlength
# xor the random strings together
for requestinfo in self.activemirrors[:-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, requestinfo['blockbitstringlist'][blocknum])
# flip the appropriate bit for the block we want
thisbitstring = lib.flip_bitstring_bit(thisbitstring,
blocklist[blocknum])
# store the result for the last mirror
self.activemirrors[-1]['blockbitstringlist'].append(thisbitstring)
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.
privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def __init__(self, mirrorinfolist, blocklist, manifestdict, privacythreshold, redundancy, rng, parallel, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
This is meant to be used for queries partitioned in chunks
(parallel or SB queries with redundancy parameter)
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold # aka k, the number of mirrors to use
self.redundancy = redundancy # aka r
self.rng = rng
self.parallel = parallel
self.blockcount = manifestdict['blockcount']
self.timing = timing
if timing:
self.recons_time = 0
#length of one chunk in BITS (1 bit per block)
#chunk length of the first chunks must be a multiple of 8, last chunk can be longer than first chunks
self.chunklen = int(self.blockcount/8/privacythreshold) * 8
self.lastchunklen = self.blockcount - (privacythreshold-1)*self.chunklen
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " + str(self.privacythreshold) + " mirrors, but only " + str(len(mirrorinfolist)) + " were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
#initialize queries for mirrors
i = 0
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirror = {}
mirror['info'] = mirrorinfo
mirror['blocksneeded'] = blocklist[:] # only for the client, obviously
mirror['blocksrequested'] = []
if parallel:
mirror['parallelblocksneeded'] = []
mirror['blockchunklist'] = []
# chunk numbers [0, ..., r-1]
mirror['chunknumbers'] = [i]
for j in range(1, redundancy):
mirror['chunknumbers'].append((i+j) % privacythreshold)
i = i + 1
#open a socket once:
mirror['info']['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mirror['info']['sock'].connect((mirrorinfo['ip'], mirrorinfo['port']))
if rng:
#pick a random seed (key) and initialize AES
seed = _randomnumberfunction(16) # random 128 bit key
mirror['seed'] = seed
mirror['cipher'] = lib.initAES(seed)
self.activemirrors.append(mirror)
for mirror in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = mirror['chunknumbers']
params['k'] = privacythreshold
params['r'] = redundancy
params['cl'] = self.chunklen
params['lcl'] = self.lastchunklen
params['b'] = batch
params['p'] = parallel
if rng:
params['s'] = mirror['seed']
#send the params, rcvlet will check response
session.sendmessage(mirror['info']['sock'], b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet, args=[mirror, self], name=("rcv_thread_" + str((mirror['info']['ip'], mirror['info']['port']))))
mirror['rt'] = t
t.start()
#multi block query. map the blocks to the minimum amount of queries
if parallel:
#create dictionary for each chunk, will hold block indices per chunk
blockchunks = {}
for i in range(0, privacythreshold):
blockchunks[i] = []
#map block numbers to chunks
for blocknum in blocklist:
index = min(int(blocknum/self.chunklen), privacythreshold-1)
blockchunks[index].append(blocknum)
#remove chunks that are still empty
for i in range(0, privacythreshold):
if len(blockchunks[i]) == 0:
del blockchunks[i]
#do until all blocks are in queries
while len(blockchunks)>0:
#iterate through mirrors
for mirror in self.activemirrors:
#dicitonary of chunk requests
chunks = {}
#iterate through r-1 random chunks, skipping the head (flip) chunk
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
#set random bytes for the latter chunk(s) from AES (will be deleted later)
chunks[c] = lib.nextrandombitsAES(mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s) randomly
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
#list of blocknumbers
blocks = []
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length)*b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][c] #remove the pre-computed random chunk from the packet to send
#if there is a block within this chunk, then add it to the bitstring by flipping the bit
if c in blockchunks:
blocknum = blockchunks[c].pop(0)
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocknum - c*self.chunklen)
blocks.append(blocknum)
if len(blockchunks[c]) == 0:
del blockchunks[c]
mirror['parallelblocksneeded'].append(blocks)
mirror['blockchunklist'][-1][c] = thisbitstring
#single block query:
else:
#iterate through all blocks
for blocknum in blocklist:
#iterate through mirrors
for mirror in self.activemirrors:
chunks = {}
#iterate through r-1 random chunks
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
chunks[c] = lib.nextrandombitsAES(mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s)
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length)*b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][c] #remove the pre-computed random chunk from the packet to send
#if the desired block is within this chunk, flip the bit
if c*self.chunklen <= blocknum and blocknum < c*self.chunklen + length:
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocknum - c*self.chunklen)
mirror['blockchunklist'][-1][c] = thisbitstring
########################################
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def __init__(self, mirrorinfolist, blocklist, manifestdict, privacythreshold, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
<Arguments>
mirrorinfolist: a list of dictionaries with information about mirrors
blocklist: the blocks that need to be retrieved
manifestdict: the manifest with information about the release
privacythreshold: the number of mirrors that would need to collude to break privacy
timing: collect timing info
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold
self.timing = timing
if timing:
self.recons_time = 0
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of "+str(self.privacythreshold)+" mirrors, but only "+str(len(mirrorinfolist))+" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirrors = {}
mirrors['info'] = mirrorinfo
mirrors['blocksneeded'] = blocklist[:]
mirrors['blockbitstringlist'] = []
mirrors['blocksrequested'] = []
# open a socket once:
mirrors['info']['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mirrors['info']['sock'].setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) #TODO check this in the cloud
mirrors['info']['sock'].connect((mirrorinfo['ip'], mirrorinfo['port']))
self.activemirrors.append(mirrors)
for thisrequestinfo in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = 1 # chunk numbers, here fixed to 1
params['k'] = privacythreshold
params['r'] = privacythreshold # r is irrelevant here, thus fixed to k
params['cl'] = 1 # chunk length, here fixed to 1
params['lcl'] = 1 # last chunk length, here fixed to 1
params['b'] = batch
params['p'] = False
#send the params, rcvlet will check response
session.sendmessage(thisrequestinfo['info']['sock'], b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet, args=[thisrequestinfo, self], name=("rcv_thread_" + str((thisrequestinfo['info']['ip'], thisrequestinfo['info']['port']))))
thisrequestinfo['rt'] = t
t.start()
bitstringlength = lib.bits_to_bytes(manifestdict['blockcount'])
# let's generate the random bitstrings for k-1 mirrors
for thisrequestinfo in self.activemirrors[:-1]:
for _ in blocklist:
thisrequestinfo['blockbitstringlist'].append(lib.randombits(manifestdict['blockcount']))
# now, let's do the 'derived' ones...
for blocknum in range(len(blocklist)):
thisbitstring = b'\0'*bitstringlength
# xor the random strings together
for requestinfo in self.activemirrors[:-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, requestinfo['blockbitstringlist'][blocknum])
# flip the appropriate bit for the block we want
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocklist[blocknum])
# store the result for the last mirror
self.activemirrors[-1]['blockbitstringlist'].append(thisbitstring)
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}