def write_work_client(writer_key):
client_id = writer_key['i']
mapId = rounds * taskId + writer_key['roundIdx']
key_per_client = writer_key['key-per-client']
key_per_client = int(key_per_client)
client_id = int(client_id)
numPartitions = int(writer_key['num_partitions'])
print("(" + str(taskId) + ")" + "range" + str(key_per_client) +
" " + str(client_id) + " " + str(numPartitions))
for i in range(
key_per_client * client_id,
min(key_per_client * (client_id + 1), numPartitions)):
keyname = "shuffle-part-" + str(mapId) + "-" + str(i)
m = hashlib.md5()
m.update(keyname.encode('utf-8'))
randomized_keyname = "shuffle-" + m.hexdigest(
)[:8] + "-part-" + str(mapId) + "-" + str(i)
print("The name of the key to write is: " +
randomized_keyname)
bytes_body = np.asarray(outputs[ps[i]]).tobytes()
print("Hey top top " + str(len(bytes_body)))
datasize = 1700000
#print(body)
#body = bytes_body.decode('ascii')
body = b64encode(bytes_body).decode('utf-8')
body = body.ljust(datasize, '=')
print("Byte to be written: " + str(len(body)))
print("Last ten bits after padding: " + body[-10:])
pocket.put_buffer(pocket_namenode, body, len(body),
randomized_keyname, jobid)
def pocket_write_buffer(p, jobid, iter, src, size):
for i in range(iter):
dst_filename = 'tmp' + str(random.randint(
1, 100000000000000)) + '-' + str(i)
r = pocket.put_buffer(p, src, size, dst_filename, jobid)
if r != 0:
raise Exception("put buffer failed: " + dst_filename + " " + jobid)
def pocket_write_buffer(p, jobid, iter, src, size):
for i in range(iter):
dst_filename = 'tmp1' + '-' + str(i)
r = pocket.put_buffer(p, src, size, dst_filename, jobid)
if r != 0:
raise Exception("put buffer failed: " + dst_filename)