def _remote_query_helper_sock(socket, command):
# issue the relevant command
session.sendmessage(socket, command)
# receive and return the answer
rawanswer = session.recvmessage(socket)
return rawanswer
def _remote_query_helper_sock(socket, command): # issue the relevant command session.sendmessage(socket, command) # receive and return the answer rawanswer = session.recvmessage(socket) return rawanswer
def _remote_query_helper(serverlocation, command, defaultserverport):
# private function that contains the guts of server communication. It
# issues a single query and then closes the connection. This is used
# both to talk to the vendor and also to talk to mirrors
if type(serverlocation) != str and type(serverlocation) != unicode:
raise TypeError("Server location must be a string, not "+str(type(serverlocation)))
# now let's split it and ensure there are 0 or 1 colons
splitlocationlist = serverlocation.split(':')
if len(splitlocationlist) >2:
raise TypeError("Server location may not contain more than one colon")
# now either set the port or use the default
if len(splitlocationlist) == 2:
serverport = int(splitlocationlist[1])
else:
serverport = defaultserverport
# check that this port is in the right range
if serverport <= 0 or serverport > 65535:
raise TypeError("Server location's port is not in the allowed range")
serverhostname = splitlocationlist[0]
# now we actually download the information...
# first open the socket
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# if serverport == 60443, use SSL
if (serverport == 60443):
ssl_serversocket = ssl.wrap_socket(serversocket, server_side=False,
ca_certs="CA_Certs/ssl.crt",
ssl_version=ssl.PROTOCOL_TLSv1,
cert_reqs=ssl.CERT_REQUIRED)
ssl_serversocket.connect((serverhostname, serverport))
#session.sendmessage(ssl_serversocket, command)
ssl_serversocket.sendall(command)
#rawanswer = session.recvmessage(ssl_serversocket)
rawanswer = ssl_serversocket.recv(4096)
ssl_serversocket.close()
return rawanswer
# else connect normally
serversocket.connect((serverhostname, serverport))
# then issue the relevant command
session.sendmessage(serversocket, command)
# and return the answer
rawanswer = session.recvmessage(serversocket)
serversocket.close()
return rawanswer
def cleanup(self): """cleanup. here: maybe request debug timing info and always close sockets""" for mirror in self.activemirrors: if self.timing: # request total computation time and measure delay ping_start = _timer() session.sendmessage(mirror['info']['sock'], "T") mirror['info']['comptime'] = float(session.recvmessage(mirror['info']['sock'])[1:]) mirror['info']['ping'] = _timer() - ping_start session.sendmessage(mirror['info']['sock'], "Q") mirror['info']['sock'].close()
def cleanup(self): """cleanup. here: maybe request debug timing info and always close sockets""" for mirror in self.activemirrors: if self.timing: # request total computation time and measure delay ping_start = _timer() session.sendmessage(mirror['info']['sock'], "T") mirror['info']['comptime'] = float(session.recvmessage(mirror['info']['sock'])[1:]) mirror['info']['ping'] = _timer() - ping_start session.sendmessage(mirror['info']['sock'], "Q") mirror['info']['sock'].close()
def BatchAnswer(parallel, chunknumbers, sock):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
blocksize = _global_myxordatastore.sizeofblocks
_batch_comp_time = 0
# while a client is connected
while not _finish:
# wait for signal to start
_batchevent.wait()
# create local copies and reset global values
with _batchlock:
batchrequests = _batchrequests
xorstrings = _xorstrings
_batchrequests = 0
_xorstrings = b''
if batchrequests == 0:
# all request answered, remove flag and wait/return
_batchevent.clear()
else: # answer requests
start_time = _timer()
if parallel:
xoranswer = _global_myxordatastore.produce_xor_from_multiple_bitstrings(
xorstrings, batchrequests * len(chunknumbers))
_batch_comp_time = _batch_comp_time + _timer() - start_time
i = 0
for _ in range(batchrequests):
result = {}
for c in chunknumbers:
result[c] = xoranswer[i * blocksize:(i + 1) *
blocksize]
i = i + 1
session.sendmessage(
sock, msgpack.packb(result, use_bin_type=True))
else:
xoranswer = _global_myxordatastore.produce_xor_from_multiple_bitstrings(
xorstrings, batchrequests)
_batch_comp_time = _batch_comp_time + _timer() - start_time
for i in range(batchrequests):
session.sendmessage(
sock, xoranswer[i * blocksize:(i + 1) * blocksize])
def BatchAnswer(parallel, chunknumbers, sock):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
blocksize = _global_myxordatastore.sizeofblocks
_batch_comp_time = 0;
# while a client is connected
while not _finish:
# wait for signal to start
_batchevent.wait()
# create local copies and reset global values
with _batchlock:
batchrequests = _batchrequests
xorstrings = _xorstrings
_batchrequests = 0
_xorstrings = b''
if batchrequests == 0:
# all request answered, remove flag and wait/return
_batchevent.clear()
else: # answer requests
start_time = _timer()
if parallel:
xoranswer = _global_myxordatastore.produce_xor_from_multiple_bitstrings(xorstrings, batchrequests*len(chunknumbers))
_batch_comp_time = _batch_comp_time + _timer() - start_time
i = 0
for _ in range(batchrequests):
result = {}
for c in chunknumbers:
result[c] = xoranswer[i*blocksize : (i+1)*blocksize]
i = i + 1
session.sendmessage(sock, msgpack.packb(result, use_bin_type=True))
else:
xoranswer = _global_myxordatastore.produce_xor_from_multiple_bitstrings(xorstrings, batchrequests)
_batch_comp_time = _batch_comp_time + _timer() - start_time
for i in range(batchrequests):
session.sendmessage(sock, xoranswer[i*blocksize : (i+1)*blocksize])
def _remote_query_helper(serverlocation, command, defaultserverport):
# private function that contains the guts of server communication. It
# issues a single query and then closes the connection. This is used
# both to talk to the vendor and also to talk to mirrors
if type(serverlocation) != str and type(serverlocation) != str:
raise TypeError("Server location must be a string, not " +
str(type(serverlocation)))
# now let's split it and ensure there are 0 or 1 colons
splitlocationlist = serverlocation.split(':')
if len(splitlocationlist) > 2:
raise TypeError("Server location may not contain more than one colon")
# now either set the port or use the default
if len(splitlocationlist) == 2:
serverport = int(splitlocationlist[1])
else:
serverport = defaultserverport
# check that this port is in the right range
if serverport <= 0 or serverport > 65535:
raise TypeError("Server location's port is not in the allowed range")
serverhostname = splitlocationlist[0]
# now we actually download the information...
# first open the socket
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.connect((serverhostname, serverport))
# then issue the relevant command
session.sendmessage(serversocket, command)
# and return the answer
rawanswer = session.recvmessage(serversocket)
serversocket.close()
return rawanswer
def _remote_query_helper(serverlocation, command, defaultserverport):
# private function that contains the guts of server communication. It
# issues a single query and then closes the connection. This is used
# both to talk to the vendor and also to talk to mirrors
if type(serverlocation) != str and type(serverlocation) != str:
raise TypeError("Server location must be a string, not " + str(type(serverlocation)))
# now let's split it and ensure there are 0 or 1 colons
splitlocationlist = serverlocation.split(':')
if len(splitlocationlist) > 2:
raise TypeError("Server location may not contain more than one colon")
# now either set the port or use the default
if len(splitlocationlist) == 2:
serverport = int(splitlocationlist[1])
else:
serverport = defaultserverport
# check that this port is in the right range
if serverport <= 0 or serverport > 65535:
raise TypeError("Server location's port is not in the allowed range")
serverhostname = splitlocationlist[0]
# now we actually download the information...
# first open the socket
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.connect((serverhostname, serverport))
# then issue the relevant command
session.sendmessage(serversocket, command)
# and return the answer
rawanswer = session.recvmessage(serversocket)
serversocket.close()
return rawanswer
def _testmirror(rh, testinfodict):
manifestdict = uppirlib.parse_manifest(_global_rawmanifestdata)
myxordatastore = fastsimplexordatastore.XORDatastore(manifestdict['blocksize'], manifestdict['blockcount'])
uppirlib.populate_xordatastore(manifestdict, myxordatastore, rootdir = _commandlineoptions.rootdir)
bitstring = base64.b64decode(testinfodict['chunklist'])
expectedData = base64.b64decode(testinfodict['data'])
expectedbitstringlength = uppirlib.compute_bitstring_length(myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
_log("UPPIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(rh.request, 'Invalid request length')
return
mirrorip = testinfodict['ip']
mirrorport = testinfodict['port']
#print "bitstring"+testinfodict['chunklist']+"\n"
xoranswer = myxordatastore.produce_xor_from_bitstring(bitstring)
if xoranswer != expectedData: # or True
session.sendmessage(rh.request, 'TEST: Invalid mirror: '+str(mirrorip)+":"+str(mirrorport))
#print "xor"+base64.b64encode(xoranswer)+"\n"
#print "mir"+base64.b64encode(expectedData)+"\n"
_remove_mirror(mirrorip, mirrorport)
else:
session.sendmessage(rh.request, 'TEST: Correct mirror: '+str(mirrorip)+":"+str(mirrorport))
return
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring.startswith('XORBLOCK'):
bitstring = requeststring[len('XORBLOCK'):]
expectedbitstringlength = uppirlib.compute_bitstring_length(_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
_log("UPPIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
return
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
# and send the reply.
session.sendmessage(self.request, xoranswer)
_log("UPPIR "+remoteip+" "+str(remoteport)+" GOOD")
# done!
return
elif requeststring == 'HELLO':
# send a reply.
session.sendmessage(self.request, "HI!")
_log("UPPIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log("UPPIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
return
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring.startswith('XORBLOCK'):
bitstring = requeststring[len('XORBLOCK'):]
expectedbitstringlength = uppirlib.compute_bitstring_length(_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
_log("UPPIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
return
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
# and send the reply.
session.sendmessage(self.request, xoranswer)
_log("UPPIR "+remoteip+" "+str(remoteport)+" GOOD")
# done!
return
elif requeststring == 'HELLO':
# send a reply.
session.sendmessage(self.request, "HI!")
_log("UPPIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log("UPPIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
return
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring == 'GET MANIFEST':
session.sendmessage(self.request, _global_rawmanifestdata)
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" manifest request")
# done!
return
elif requeststring == 'GET MIRRORLIST':
# let's try to clean up the list. If we are busy with another attempt
# to do this, the latter will be a NOOP
_check_for_expired_mirrorinfo()
# reply with the mirror list
session.sendmessage(self.request, _global_rawmirrorlist)
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorlist request")
# done!
return
elif requeststring.startswith('RUN TEST'):
if random.random() < RANDOM_THRESHOLD:
return
testrawdata = requeststring[len('RUN TEST'):]
try:
testinfodict = json.loads(testrawdata)
except (TypeError, ValueError), e:
session.sendmessage(self.request, "Error cannot deserialize testinfo!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" cannot deserialize testinfo!"+str(e))
return
if type(testinfodict) != dict or 'ip' not in testinfodict or 'port' not in testinfodict or 'data' not in testinfodict or 'chunklist' not in testinfodict:
session.sendmessage(self.request, "Error, testinfodict has an invalid format.")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" testinfodict has an invalid format")
return
_testmirror(self, testinfodict)
def request_xorblock_chunked(socket, chunks):
session.sendmessage(socket,
b"C" + msgpack.packb(chunks, use_bin_type=True))
def get_response(requeststring): s = socket.socket() s.connect((mirrortocheck,62294)) session.sendmessage(s,requeststring) return session.recvmessage(s)
def get_response(requeststring):
s = socket.socket()
s.connect((mirrortocheck, 62294))
session.sendmessage(s, requeststring)
return session.recvmessage(s)
def request_xorblock(socket, bitstring):
session.sendmessage(socket, b"X" + bitstring)
def request_xorblock_chunked(socket, chunks): session.sendmessage(socket, b"C" + msgpack.packb(chunks, use_bin_type=True))
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring == 'GET MANIFEST':
session.sendmessage(self.request, _global_rawmanifestdata)
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" manifest request")
# done!
return
elif requeststring == 'GET MIRRORLIST':
# let's try to clean up the list. If we are busy with another attempt
# to do this, the latter will be a NOOP
_check_for_expired_mirrorinfo()
# reply with the mirror list
session.sendmessage(self.request, _global_rawmirrorlist)
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorlist request")
# done!
return
elif requeststring.startswith('MIRRORADVERTISE'):
# This is a mirror telling us it's ready to serve clients.
mirrorrawdata = requeststring[len('MIRRORADVERTISE'):]
# handle the case where the mirror provides data that is larger than
# we want to serve
if len(mirrorrawdata) > _commandlineoptions.maxmirrorinfo:
session.sendmessage(self.request, "Error, mirrorinfo too large!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo too large: "+str(len(mirrorrawdata)))
return
# Let's sanity check the data...
# can we deserialize it?
try:
mirrorinfodict = json.loads(mirrorrawdata)
except (TypeError, ValueError), e:
session.sendmessage(self.request, "Error cannot deserialize mirrorinfo!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" cannot deserialize mirrorinfo!"+str(e))
return
# is it a dictionary and does it have the required keys?
if type(mirrorinfodict) != dict or 'ip' not in mirrorinfodict or 'port' not in mirrorinfodict:
session.sendmessage(self.request, "Error, mirrorinfo has an invalid format.")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo has an invalid format")
return
# is it a dictionary and does it have the required keys?
if mirrorinfodict['ip'] != remoteip:
session.sendmessage(self.request, "Error, must provide mirrorinfo from the mirror's IP")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo provided from the wrong IP")
return
# add the information to the mirrorlist
_add_mirrorinfo_to_list(mirrorinfodict)
# and notify the user
session.sendmessage(self.request, 'OK')
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo update "+str(len(mirrorrawdata)))
# done!
return
def request_xorblock_chunked_rng_parallel(socket, chunks):
session.sendmessage(socket, "M" + msgpack.packb(chunks))
def request_xorblock_chunked_rng_parallel(socket, chunks):
session.sendmessage(socket,
b"M" + msgpack.packb(chunks, use_bin_type=True))
def __init__(self, mirrorinfolist, blocklist, manifestdict, privacythreshold, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
<Arguments>
mirrorinfolist: a list of dictionaries with information about mirrors
blocklist: the blocks that need to be retrieved
manifestdict: the manifest with information about the release
privacythreshold: the number of mirrors that would need to collude to break privacy
timing: collect timing info
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold
self.timing = timing
if timing:
self.recons_time = 0
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of "+str(self.privacythreshold)+" mirrors, but only "+str(len(mirrorinfolist))+" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirrors = {}
mirrors['info'] = mirrorinfo
mirrors['blocksneeded'] = blocklist[:]
mirrors['blockbitstringlist'] = []
mirrors['blocksrequested'] = []
# open a socket once:
mirrors['info']['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mirrors['info']['sock'].setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) #TODO check this in the cloud
mirrors['info']['sock'].connect((mirrorinfo['ip'], mirrorinfo['port']))
self.activemirrors.append(mirrors)
for thisrequestinfo in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = 1 # chunk numbers, here fixed to 1
params['k'] = privacythreshold
params['r'] = privacythreshold # r is irrelevant here, thus fixed to k
params['cl'] = 1 # chunk length, here fixed to 1
params['lcl'] = 1 # last chunk length, here fixed to 1
params['b'] = batch
params['p'] = False
#send the params, rcvlet will check response
session.sendmessage(thisrequestinfo['info']['sock'], b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet, args=[thisrequestinfo, self], name=("rcv_thread_" + str((thisrequestinfo['info']['ip'], thisrequestinfo['info']['port']))))
thisrequestinfo['rt'] = t
t.start()
bitstringlength = lib.bits_to_bytes(manifestdict['blockcount'])
# let's generate the random bitstrings for k-1 mirrors
for thisrequestinfo in self.activemirrors[:-1]:
for _ in blocklist:
thisrequestinfo['blockbitstringlist'].append(lib.randombits(manifestdict['blockcount']))
# now, let's do the 'derived' ones...
for blocknum in range(len(blocklist)):
thisbitstring = b'\0'*bitstringlength
# xor the random strings together
for requestinfo in self.activemirrors[:-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, requestinfo['blockbitstringlist'][blocknum])
# flip the appropriate bit for the block we want
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocklist[blocknum])
# store the result for the last mirror
self.activemirrors[-1]['blockbitstringlist'].append(thisbitstring)
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
session.sendmessage(self.request, "Error, mirrorinfo too large!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo too large: "+str(len(mirrorrawdata)))
return
# Let's sanity check the data...
# can we deserialize it?
try:
mirrorinfodict = json.loads(mirrorrawdata)
except (TypeError, ValueError), e:
session.sendmessage(self.request, "Error cannot deserialize mirrorinfo!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" cannot deserialize mirrorinfo!"+str(e))
return
# is it a dictionary and does it have the required keys?
if type(mirrorinfodict) != dict or 'ip' not in mirrorinfodict or 'port' not in mirrorinfodict:
session.sendmessage(self.request, "Error, mirrorinfo has an invalid format.")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo has an invalid format")
return
# is it a dictionary and does it have the required keys?
if mirrorinfodict['ip'] != remoteip:
session.sendmessage(self.request, "Error, must provide mirrorinfo from the mirror's IP")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo provided from the wrong IP")
return
# add the information to the mirrorlist
_add_mirrorinfo_to_list(mirrorinfodict)
# and notify the user
session.sendmessage(self.request, 'OK')
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" mirrorinfo update "+str(len(mirrorrawdata)))
def __init__(self, mirrorinfolist, blocklist, manifestdict,
privacythreshold, redundancy, rng, parallel, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
This is meant to be used for queries partitioned in chunks
(parallel or SB queries with redundancy parameter)
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold # aka k, the number of mirrors to use
self.redundancy = redundancy # aka r
self.rng = rng
self.parallel = parallel
self.blockcount = manifestdict['blockcount']
self.timing = timing
if timing:
self.recons_time = 0
#length of one chunk in BITS (1 bit per block)
#chunk length of the first chunks must be a multiple of 8, last chunk can be longer than first chunks
self.chunklen = int(self.blockcount / 8 / privacythreshold) * 8
self.lastchunklen = self.blockcount - (privacythreshold -
1) * self.chunklen
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " +
str(self.privacythreshold) +
" mirrors, but only " +
str(len(mirrorinfolist)) +
" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
#initialize queries for mirrors
i = 0
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirror = {}
mirror['info'] = mirrorinfo
mirror[
'blocksneeded'] = blocklist[:] # only for the client, obviously
mirror['blocksrequested'] = []
if parallel:
mirror['parallelblocksneeded'] = []
mirror['blockchunklist'] = []
# chunk numbers [0, ..., r-1]
mirror['chunknumbers'] = [i]
for j in range(1, redundancy):
mirror['chunknumbers'].append((i + j) % privacythreshold)
i = i + 1
#open a socket once:
mirror['info']['sock'] = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
mirror['info']['sock'].connect(
(mirrorinfo['ip'], mirrorinfo['port']))
if rng:
#pick a random seed (key) and initialize AES
seed = _randomnumberfunction(16) # random 128 bit key
mirror['seed'] = seed
mirror['cipher'] = lib.initAES(seed)
self.activemirrors.append(mirror)
for mirror in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = mirror['chunknumbers']
params['k'] = privacythreshold
params['r'] = redundancy
params['cl'] = self.chunklen
params['lcl'] = self.lastchunklen
params['b'] = batch
params['p'] = parallel
if rng:
params['s'] = mirror['seed']
#send the params, rcvlet will check response
session.sendmessage(
mirror['info']['sock'],
b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(
target=rcvlet,
args=[mirror, self],
name=("rcv_thread_" + str(
(mirror['info']['ip'], mirror['info']['port']))))
mirror['rt'] = t
t.start()
#multi block query. map the blocks to the minimum amount of queries
if parallel:
#create dictionary for each chunk, will hold block indices per chunk
blockchunks = {}
for i in range(0, privacythreshold):
blockchunks[i] = []
#map block numbers to chunks
for blocknum in blocklist:
index = min(int(blocknum / self.chunklen),
privacythreshold - 1)
blockchunks[index].append(blocknum)
#remove chunks that are still empty
for i in range(0, privacythreshold):
if len(blockchunks[i]) == 0:
del blockchunks[i]
#do until all blocks are in queries
while len(blockchunks) > 0:
#iterate through mirrors
for mirror in self.activemirrors:
#dicitonary of chunk requests
chunks = {}
#iterate through r-1 random chunks, skipping the head (flip) chunk
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
#set random bytes for the latter chunk(s) from AES (will be deleted later)
chunks[c] = lib.nextrandombitsAES(
mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s) randomly
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
#list of blocknumbers
blocks = []
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length) * b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][
c] #remove the pre-computed random chunk from the packet to send
#if there is a block within this chunk, then add it to the bitstring by flipping the bit
if c in blockchunks:
blocknum = blockchunks[c].pop(0)
thisbitstring = lib.flip_bitstring_bit(
thisbitstring, blocknum - c * self.chunklen)
blocks.append(blocknum)
if len(blockchunks[c]) == 0:
del blockchunks[c]
mirror['parallelblocksneeded'].append(blocks)
mirror['blockchunklist'][-1][c] = thisbitstring
#single block query:
else:
#iterate through all blocks
for blocknum in blocklist:
#iterate through mirrors
for mirror in self.activemirrors:
chunks = {}
#iterate through r-1 random chunks
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
chunks[c] = lib.nextrandombitsAES(
mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s)
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length) * b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][
c] #remove the pre-computed random chunk from the packet to send
#if the desired block is within this chunk, flip the bit
if c * self.chunklen <= blocknum and blocknum < c * self.chunklen + length:
thisbitstring = lib.flip_bitstring_bit(
thisbitstring, blocknum - c * self.chunklen)
mirror['blockchunklist'][-1][c] = thisbitstring
########################################
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.
privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def __init__(self, mirrorinfolist, blocklist, manifestdict,
privacythreshold, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
<Arguments>
mirrorinfolist: a list of dictionaries with information about mirrors
blocklist: the blocks that need to be retrieved
manifestdict: the manifest with information about the release
privacythreshold: the number of mirrors that would need to collude to break privacy
timing: collect timing info
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold
self.timing = timing
if timing:
self.recons_time = 0
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " +
str(self.privacythreshold) +
" mirrors, but only " +
str(len(mirrorinfolist)) +
" were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirrors = {}
mirrors['info'] = mirrorinfo
mirrors['blocksneeded'] = blocklist[:]
mirrors['blockbitstringlist'] = []
mirrors['blocksrequested'] = []
# open a socket once:
mirrors['info']['sock'] = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
mirrors['info']['sock'].setsockopt(
socket.IPPROTO_TCP, socket.TCP_NODELAY,
1) #TODO check this in the cloud
mirrors['info']['sock'].connect(
(mirrorinfo['ip'], mirrorinfo['port']))
self.activemirrors.append(mirrors)
for thisrequestinfo in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = 1 # chunk numbers, here fixed to 1
params['k'] = privacythreshold
params[
'r'] = privacythreshold # r is irrelevant here, thus fixed to k
params['cl'] = 1 # chunk length, here fixed to 1
params['lcl'] = 1 # last chunk length, here fixed to 1
params['b'] = batch
params['p'] = False
#send the params, rcvlet will check response
session.sendmessage(
thisrequestinfo['info']['sock'],
b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet,
args=[thisrequestinfo, self],
name=("rcv_thread_" + str(
(thisrequestinfo['info']['ip'],
thisrequestinfo['info']['port']))))
thisrequestinfo['rt'] = t
t.start()
bitstringlength = lib.bits_to_bytes(manifestdict['blockcount'])
# let's generate the random bitstrings for k-1 mirrors
for thisrequestinfo in self.activemirrors[:-1]:
for _ in blocklist:
thisrequestinfo['blockbitstringlist'].append(
lib.randombits(manifestdict['blockcount']))
# now, let's do the 'derived' ones...
for blocknum in range(len(blocklist)):
thisbitstring = b'\0' * bitstringlength
# xor the random strings together
for requestinfo in self.activemirrors[:-1]:
thisbitstring = xordatastore.do_xor(
thisbitstring, requestinfo['blockbitstringlist'][blocknum])
# flip the appropriate bit for the block we want
thisbitstring = lib.flip_bitstring_bit(thisbitstring,
blocklist[blocknum])
# store the result for the last mirror
self.activemirrors[-1]['blockbitstringlist'].append(thisbitstring)
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.
privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def handle(self):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
global _global_myxordatastore
global _global_manifestdict
global _request_restart
_finish = False
comp_time = 0
_batch_comp_time = 0
_batchrequests = 0
_xorstrings = b''
parallel = False
requeststring = b'0'
while requeststring != b'Q':
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
# remoteip, remoteport = self.request.getpeername()
start_time = _timer()
# if it's a request for a XORBLOCK
if requeststring.startswith(b'X'):
bitstring = requeststring[len(b'X'):]
expectedbitstringlength = lib.bits_to_bytes(
_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
_finish = True
_batchevent.set()
return
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and immediately send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
# done!
elif requeststring.startswith(b'C'):
payload = requeststring[len(b'C'):]
chunks = msgpack.unpackb(payload, raw=False)
bitstring = lib.build_bitstring_from_chunks(
chunks, k, chunklen, lastchunklen)
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring.startswith(b'R'):
payload = requeststring[len(b'R'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstring = lib.build_bitstring_from_chunks(
chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(
bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring == b'MANIFEST UPDATE':
print("MANIFEST UPDATE")
_request_restart = True
elif requeststring.startswith(b'M'):
parallel = True
payload = requeststring[len(b'M'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstrings = lib.build_bitstring_from_chunks_parallel(
chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
result = {}
for c in chunknumbers:
result[
c] = _global_myxordatastore.produce_xor_from_bitstring(
bitstrings[c])
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(
self.request, msgpack.packb(result, use_bin_type=True))
else:
with _batchlock:
for c in chunknumbers:
_xorstrings += bitstrings[c]
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
#done!
elif requeststring.startswith(b'P'):
payload = requeststring[len(b'P'):]
params = msgpack.unpackb(payload, raw=False)
chunknumbers = params['cn']
k = params['k']
r = params['r']
chunklen = params['cl']
lastchunklen = params['lcl']
batch = params['b']
parallel = params['p']
if 's' in params:
cipher = lib.initAES(params['s'])
if batch:
# create batch xor thread
t = threading.Thread(
target=BatchAnswer,
args=[parallel, chunknumbers, self.request],
name="RAID-PIR Batch XOR")
t.daemon = True
t.start()
# and send the reply.
session.sendmessage(self.request, b"PARAMS OK")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" PARAMS received " + str(params))
#done!
#Timing Request
elif requeststring == b'T':
session.sendmessage(self.request,
b"T" + str(comp_time + _batch_comp_time))
comp_time = 0
_batch_comp_time = 0
#Debug Hello
elif requeststring == b'HELLO':
session.sendmessage(self.request, b"HI!")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
#the client asked to close the connection
elif requeststring == b'Q':
comp_time = 0
_finish = True
_batchevent.set()
return
#this happens if the client closed the socket unexpectedly
elif requeststring == b'':
comp_time = 0
_finish = True
_batchevent.set()
return
else:
# we don't know what this is! Log and tell the requestor
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
_finish = True
_batchevent.set()
return
return
# add HELLO
elif requeststring == 'HELLO':
# send a reply.
session.sendmessage(self.request, "VENDORHI!")
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" VENDORHI!")
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log("UPPIRVendor "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
return
def start_vendor_service(manifestdict, ip, port):
# this should be done before we are called
assert(_global_rawmanifestdata != None)
# create the handler / server
vendorserver = ThreadedVendorServer((ip, port), ThreadedVendorRequestHandler)
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring == 'GET MANIFEST':
session.sendmessage(self.request, _global_rawmanifestdata)
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" manifest request")
# done!
return
elif requeststring == 'GET MIRRORLIST':
# let's try to clean up the list. If we are busy with another attempt
# to do this, the latter will be a NOOP
_check_for_expired_mirrorinfo()
# reply with the mirror list
session.sendmessage(self.request, _global_rawmirrorlist)
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" mirrorlist request")
# done!
return
elif requeststring.startswith('MIRRORADVERTISE'):
# This is a mirror telling us it's ready to serve clients.
mirrorrawdata = requeststring[len('MIRRORADVERTISE'):]
# handle the case where the mirror provides data that is larger than
# we want to serve
if len(mirrorrawdata) > _commandlineoptions.maxmirrorinfo:
session.sendmessage(self.request,
"Error, mirrorinfo too large!")
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" mirrorinfo too large: " + str(len(mirrorrawdata)))
return
# Let's sanity check the data...
# can we deserialize it?
try:
mirrorinfodict = json.loads(mirrorrawdata)
except (TypeError, ValueError), e:
session.sendmessage(self.request,
"Error cannot deserialize mirrorinfo!")
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" cannot deserialize mirrorinfo!" + str(e))
return
# is it a dictionary and does it have the required keys?
if type(
mirrorinfodict
) != dict or 'ip' not in mirrorinfodict or 'port' not in mirrorinfodict:
session.sendmessage(
self.request, "Error, mirrorinfo has an invalid format.")
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" mirrorinfo has an invalid format")
return
# is it a dictionary and does it have the required keys?
if mirrorinfodict['ip'] != remoteip:
session.sendmessage(
self.request,
"Error, must provide mirrorinfo from the mirror's IP")
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" mirrorinfo provided from the wrong IP")
return
# add the information to the mirrorlist
_add_mirrorinfo_to_list(mirrorinfodict)
# and notify the user
session.sendmessage(self.request, 'OK')
_log("UPPIRVendor " + remoteip + " " + str(remoteport) +
" mirrorinfo update " + str(len(mirrorrawdata)))
# done!
return
def request_xorblock(socket, bitstring): session.sendmessage(socket, b"X" + bitstring)
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log(
"UPPIRVendor "
+ remoteip
+ " "
+ str(remoteport)
+ " Invalid request type starts:'"
+ requeststring[:5]
+ "'"
)
session.sendmessage(self.request, "Invalid request type")
return
def start_vendor_service(manifestdict, ip, port):
# this should be done before we are called
assert _global_rawmanifestdata != None
# create the handler / server
vendorserver = ThreadedVendorServer((ip, port), ThreadedVendorRequestHandler)
# and serve forever! This call will not return which is why we spawn a new
# thread to handle it
threading.Thread(target=vendorserver.serve_forever, name="upPIR vendor server").start()
def request_xorblock_chunked_rng_parallel(socket, chunks): session.sendmessage(socket, b"M" + msgpack.packb(chunks, use_bin_type=True))
# add HELLO
elif requeststring == 'HELLO':
# send a reply.
session.sendmessage(self.request, "VENDORHI!")
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" VENDORHI!")
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" Invalid request type starts:'" + requeststring[:5] + "'")
session.sendmessage(self.request, 'Invalid request type')
return
def start_vendor_service(manifestdict, ip, port):
# this should be done before we are called
assert _global_rawmanifestdata != None
# create the handler / server
vendorserver = ThreadedVendorServer((ip, port),
ThreadedVendorRequestHandler)
_log('vendor servers started at' + str(ip) + ':' + str(port))
print "Vendor Server started at", ip, ":", port
print "Manifest contains", len(
def handle(self):
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
remoteip, remoteport = self.request.getpeername()
# if it's a request for a XORBLOCK
if requeststring == b'GET MANIFEST':
print("GET MANIFEST")
rawmanifestdata = open(_commandlineoptions.manifestfilename,
'rb').read()
_global_rawmanifestdata = rawmanifestdata
session.sendmessage(self.request, _global_rawmanifestdata)
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" manifest request")
# done!
return
elif requeststring == b'MANIFEST UPDATE':
print('MANIFEST UPDATE')
rawmanifestdata = open(_commandlineoptions.manifestfilename,
'rb').read()
_global_rawmanifestdata = rawmanifestdata
# get a copy of the mirrorlist
mirrorlist = list()
_global_mirrorinfolock.acquire()
try:
for mirror in _global_mirrorinfodict:
mirrorlist.append(
_global_mirrorinfodict[mirror]['mirrorinfo'])
finally:
_global_mirrorinfolock.release()
for mirror in mirrorlist:
sock = None
try:
# Connect to server and send data
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((mirror['ip'], mirror['port']))
session.sendmessage(sock, 'MANIFEST UPDATE')
except:
print("Could not connect to mirror", mirror)
pass
finally:
sock.close()
elif requeststring == b'GET MIRRORLIST':
# let's try to clean up the list. If we are busy with another attempt
# to do this, the latter will be a NOOP
_check_for_expired_mirrorinfo()
# reply with the mirror list
session.sendmessage(self.request, _global_rawmirrorlist)
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" mirrorlist request")
# done!
return
elif requeststring.startswith(b'MIRRORADVERTISE'):
# This is a mirror telling us it's ready to serve clients.
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" mirror advertise")
mirrorrawdata = requeststring[len(b'MIRRORADVERTISE'):]
# handle the case where the mirror provides data that is larger than
# we want to serve
if len(mirrorrawdata) > _commandlineoptions.maxmirrorinfo:
session.sendmessage(self.request,
"Error, mirrorinfo too large!")
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" mirrorinfo too large: " + str(len(mirrorrawdata)))
return
# Let's sanity check the data...
# can we unpack it?
try:
mirrorinfodict = msgpack.unpackb(mirrorrawdata, raw=False)
except (TypeError, ValueError) as e:
session.sendmessage(self.request,
"Error cannot unpack mirrorinfo!")
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" cannot unpack mirrorinfo!" + str(e))
return
# is it a dictionary and does it have the required keys?
if type(
mirrorinfodict
) != dict or 'ip' not in mirrorinfodict or 'port' not in mirrorinfodict:
session.sendmessage(
self.request, "Error, mirrorinfo has an invalid format.")
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" mirrorinfo has an invalid format")
return
#is the mirror to add coming from the same ip?
if _commandlineoptions.checkmirrorip:
if mirrorinfodict['ip'] != remoteip:
session.sendmessage(
self.request,
"Error, must provide mirrorinfo from the mirror's IP")
_log("RAID-PIR Vendor " + remoteip + " " +
str(remoteport) +
" mirrorinfo provided from the wrong IP")
return
# add the information to the mirrorlist
_add_mirrorinfo_to_list(mirrorinfodict)
# and notify the user
session.sendmessage(self.request, 'OK')
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" mirrorinfo update " + str(len(mirrorrawdata)))
# done!
return
# add HELLO
elif requeststring == b'HELLO':
# send a reply.
session.sendmessage(self.request, "VENDORHI!")
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" VENDORHI!")
# done!
return
else:
# we don't know what this is! Log and tell the requestor
_log("RAID-PIR Vendor " + remoteip + " " + str(remoteport) +
" Invalid request type starts:'" + requeststring[:5] + "'")
session.sendmessage(self.request, 'Invalid request type')
return
def __init__(self, mirrorinfolist, blocklist, manifestdict, privacythreshold, redundancy, rng, parallel, batch, timing):
"""
<Purpose>
Get ready to handle requests for XOR block strings, etc.
This is meant to be used for queries partitioned in chunks
(parallel or SB queries with redundancy parameter)
<Exceptions>
TypeError may be raised if invalid parameters are given.
InsufficientMirrors if there are not enough mirrors
"""
self.blocklist = blocklist
self.manifestdict = manifestdict
self.privacythreshold = privacythreshold # aka k, the number of mirrors to use
self.redundancy = redundancy # aka r
self.rng = rng
self.parallel = parallel
self.blockcount = manifestdict['blockcount']
self.timing = timing
if timing:
self.recons_time = 0
#length of one chunk in BITS (1 bit per block)
#chunk length of the first chunks must be a multiple of 8, last chunk can be longer than first chunks
self.chunklen = int(self.blockcount/8/privacythreshold) * 8
self.lastchunklen = self.blockcount - (privacythreshold-1)*self.chunklen
if len(mirrorinfolist) < self.privacythreshold:
raise InsufficientMirrors("Requested the use of " + str(self.privacythreshold) + " mirrors, but only " + str(len(mirrorinfolist)) + " were available.")
# now we do the 'random' part. I copy the mirrorinfolist to avoid changing the list in place.
self.fullmirrorinfolist = mirrorinfolist[:]
random.shuffle(self.fullmirrorinfolist)
# let's make a list of mirror information (what has been retrieved, etc.)
self.activemirrors = []
#initialize queries for mirrors
i = 0
for mirrorinfo in self.fullmirrorinfolist[:self.privacythreshold]:
mirror = {}
mirror['info'] = mirrorinfo
mirror['blocksneeded'] = blocklist[:] # only for the client, obviously
mirror['blocksrequested'] = []
if parallel:
mirror['parallelblocksneeded'] = []
mirror['blockchunklist'] = []
# chunk numbers [0, ..., r-1]
mirror['chunknumbers'] = [i]
for j in range(1, redundancy):
mirror['chunknumbers'].append((i+j) % privacythreshold)
i = i + 1
#open a socket once:
mirror['info']['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mirror['info']['sock'].connect((mirrorinfo['ip'], mirrorinfo['port']))
if rng:
#pick a random seed (key) and initialize AES
seed = _randomnumberfunction(16) # random 128 bit key
mirror['seed'] = seed
mirror['cipher'] = lib.initAES(seed)
self.activemirrors.append(mirror)
for mirror in self.activemirrors:
#send parameters to mirrors once
params = {}
params['cn'] = mirror['chunknumbers']
params['k'] = privacythreshold
params['r'] = redundancy
params['cl'] = self.chunklen
params['lcl'] = self.lastchunklen
params['b'] = batch
params['p'] = parallel
if rng:
params['s'] = mirror['seed']
#send the params, rcvlet will check response
session.sendmessage(mirror['info']['sock'], b"P" + msgpack.packb(params, use_bin_type=True))
# start separate receiving thread for this socket
t = threading.Thread(target=rcvlet, args=[mirror, self], name=("rcv_thread_" + str((mirror['info']['ip'], mirror['info']['port']))))
mirror['rt'] = t
t.start()
#multi block query. map the blocks to the minimum amount of queries
if parallel:
#create dictionary for each chunk, will hold block indices per chunk
blockchunks = {}
for i in range(0, privacythreshold):
blockchunks[i] = []
#map block numbers to chunks
for blocknum in blocklist:
index = min(int(blocknum/self.chunklen), privacythreshold-1)
blockchunks[index].append(blocknum)
#remove chunks that are still empty
for i in range(0, privacythreshold):
if len(blockchunks[i]) == 0:
del blockchunks[i]
#do until all blocks are in queries
while len(blockchunks)>0:
#iterate through mirrors
for mirror in self.activemirrors:
#dicitonary of chunk requests
chunks = {}
#iterate through r-1 random chunks, skipping the head (flip) chunk
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
#set random bytes for the latter chunk(s) from AES (will be deleted later)
chunks[c] = lib.nextrandombitsAES(mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s) randomly
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
#list of blocknumbers
blocks = []
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length)*b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][c] #remove the pre-computed random chunk from the packet to send
#if there is a block within this chunk, then add it to the bitstring by flipping the bit
if c in blockchunks:
blocknum = blockchunks[c].pop(0)
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocknum - c*self.chunklen)
blocks.append(blocknum)
if len(blockchunks[c]) == 0:
del blockchunks[c]
mirror['parallelblocksneeded'].append(blocks)
mirror['blockchunklist'][-1][c] = thisbitstring
#single block query:
else:
#iterate through all blocks
for blocknum in blocklist:
#iterate through mirrors
for mirror in self.activemirrors:
chunks = {}
#iterate through r-1 random chunks
for c in mirror['chunknumbers'][1:]:
#pick correct length in bits
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
if rng:
chunks[c] = lib.nextrandombitsAES(mirror['cipher'], length)
else:
#set random bytes for the latter chunk(s)
chunks[c] = lib.randombits(length)
mirror['blockchunklist'].append(chunks)
# now derive the first chunks
for mirror in self.activemirrors:
#number of the first chunk
c = mirror['chunknumbers'][0]
#pick correct length for the chunk
if c == self.privacythreshold - 1:
length = self.lastchunklen
else:
length = self.chunklen
#fill it with zero
thisbitstring = lib.bits_to_bytes(length)*b'\0'
#xor all other rnd chunks onto it
for rqi in self.activemirrors:
if c in rqi['blockchunklist'][-1]:
thisbitstring = xordatastore.do_xor(thisbitstring, rqi['blockchunklist'][-1][c])
if rng:
del rqi['blockchunklist'][-1][c] #remove the pre-computed random chunk from the packet to send
#if the desired block is within this chunk, flip the bit
if c*self.chunklen <= blocknum and blocknum < c*self.chunklen + length:
thisbitstring = lib.flip_bitstring_bit(thisbitstring, blocknum - c*self.chunklen)
mirror['blockchunklist'][-1][c] = thisbitstring
########################################
# want to have a structure for locking
self.tablelock = threading.Lock()
# and we'll keep track of the ones that are waiting in the wings...
self.backupmirrorinfolist = self.fullmirrorinfolist[self.privacythreshold:]
# the returned blocks are put here...
self.returnedxorblocksdict = {}
for blocknum in blocklist:
# make these all empty lists to start with
self.returnedxorblocksdict[blocknum] = []
# and here is where they are put when reconstructed
self.finishedblockdict = {}
def handle(self):
global _batchrequests
global _xorstrings
global _finish
global _batch_comp_time
global _global_myxordatastore
global _global_manifestdict
global _request_restart
_finish = False
comp_time = 0
_batch_comp_time = 0
_batchrequests = 0
_xorstrings = b''
parallel = False
requeststring = b'0'
while requeststring != b'Q':
# read the request from the socket...
requeststring = session.recvmessage(self.request)
# for logging purposes, get the remote info
# remoteip, remoteport = self.request.getpeername()
start_time = _timer()
# if it's a request for a XORBLOCK
if requeststring.startswith(b'X'):
bitstring = requeststring[len(b'X'):]
expectedbitstringlength = lib.bits_to_bytes(_global_myxordatastore.numberofblocks)
if len(bitstring) != expectedbitstringlength:
# Invalid request length...
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request with length: "+str(len(bitstring)))
session.sendmessage(self.request, 'Invalid request length')
_finish = True
_batchevent.set()
return
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and immediately send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
# done!
elif requeststring.startswith(b'C'):
payload = requeststring[len(b'C'):]
chunks = msgpack.unpackb(payload, raw=False)
bitstring = lib.build_bitstring_from_chunks(chunks, k, chunklen, lastchunklen)
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring.startswith(b'R'):
payload = requeststring[len(b'R'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstring = lib.build_bitstring_from_chunks(chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
# Now let's process this...
xoranswer = _global_myxordatastore.produce_xor_from_bitstring(bitstring)
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, xoranswer)
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
else:
with _batchlock:
_xorstrings += bitstring
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#done!
elif requeststring == b'MANIFEST UPDATE':
print("MANIFEST UPDATE")
_request_restart = True
elif requeststring.startswith(b'M'):
parallel = True
payload = requeststring[len(b'M'):]
chunks = msgpack.unpackb(payload, raw=False)
#iterate through r-1 random chunks
for c in chunknumbers[1:]:
if c == k - 1:
length = lastchunklen
else:
length = chunklen
chunks[c] = lib.nextrandombitsAES(cipher, length)
bitstrings = lib.build_bitstring_from_chunks_parallel(chunks, k, chunklen, lastchunklen) #the expanded query
if not batch:
result = {}
for c in chunknumbers:
result[c] = _global_myxordatastore.produce_xor_from_bitstring(bitstrings[c])
comp_time = comp_time + _timer() - start_time
# and send the reply.
session.sendmessage(self.request, msgpack.packb(result, use_bin_type=True))
else:
with _batchlock:
for c in chunknumbers:
_xorstrings += bitstrings[c]
_batchrequests = _batchrequests + 1
# notify batch thread
_batchevent.set()
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" GOOD")
#done!
elif requeststring.startswith(b'P'):
payload = requeststring[len(b'P'):]
params = msgpack.unpackb(payload, raw=False)
chunknumbers = params['cn']
k = params['k']
r = params['r']
chunklen = params['cl']
lastchunklen = params['lcl']
batch = params['b']
parallel = params['p']
if 's' in params:
cipher = lib.initAES(params['s'])
if batch:
# create batch xor thread
t = threading.Thread(target=BatchAnswer, args=[parallel, chunknumbers, self.request], name="RAID-PIR Batch XOR")
t.daemon = True
t.start()
# and send the reply.
session.sendmessage(self.request, b"PARAMS OK")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" PARAMS received " + str(params))
#done!
#Timing Request
elif requeststring == b'T':
session.sendmessage(self.request, b"T" + str(comp_time + _batch_comp_time))
comp_time = 0
_batch_comp_time = 0
#Debug Hello
elif requeststring == b'HELLO':
session.sendmessage(self.request, b"HI!")
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" HI!")
# done!
#the client asked to close the connection
elif requeststring == b'Q':
comp_time = 0
_finish = True
_batchevent.set()
return
#this happens if the client closed the socket unexpectedly
elif requeststring == b'':
comp_time = 0
_finish = True
_batchevent.set()
return
else:
# we don't know what this is! Log and tell the requestor
#_log("RAID-PIR "+remoteip+" "+str(remoteport)+" Invalid request type starts:'"+requeststring[:5]+"'")
session.sendmessage(self.request, 'Invalid request type')
_finish = True
_batchevent.set()
return
def request_xorblock_chunked_rng(socket, chunks):
session.sendmessage(socket, "R" + msgpack.packb(chunks))