def handle_register_job(reader, writer): print( "-------------------------- REGISTER JOB --------------------------------", time.time()) jobname_len = yield from reader.read(INT) jobname_len, = struct.Struct("!i").unpack(jobname_len) jobname = yield from reader.read(jobname_len + 3 * INT + SHORT) jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct( "!" + str(jobname_len) + "siiih").unpack(jobname) jobname = jobname.decode('utf-8') # generate jobid if 'gg' in jobname: jobid = jobname + '-1234' jobid_int = 1234 else: jobid_int = randint(0, 1000000) jobid = jobname + "-" + str(jobid_int) print("received hints ", jobid, num_lambdas, jobGB, peakMbps, latency_sensitive) # create dir named jobid createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return pocket.create_dir(createdirsock, None, jobid) #pocket.close(createdirsock) if jobGB == 0 or peakMbps == 0: jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB, peakMbps, latency_sensitive) # generate weightmask wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps, latency_sensitive) # wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)] # register job in table err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str) # send wmask to metadata server ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT) if ioctlsock is None: return yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i") resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE, jobid_int) pkt = resp_packer.pack(*resp) writer.write(pkt) print( "-------------------------- REGISTERED JOB --------------------------------" ) return
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 100 datasize = 1024 #bytes jobid = "latency-test".join( random.sample(string.ascii_letters + string.digits, 6)) namenode_ip = "10.1.47.178" file_tmp = '/tmp/file_tmp2' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) # test read/write through buffer dir = jobid + "microbenchmark" pocket.create_dir(p, dir, "") jobid = dir t0 = time.time() pocket_write_buffer(p, jobid, iter, text, datasize) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes write_buffer:") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") text_back = " " * datasize t0 = time.time() pocket_read_buffer(p, jobid, iter, text_back, datasize) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes read_buffer:") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes lookup (metadata RPC):") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") os.remove(file_tmp) return
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 50000 datasize = 1024 #bytes jobid = "latency-test" namenode_ip = "10.1.0.10" file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) # test read/write through buffer dir = jobid + "1" pocket.create_dir(p, dir, "") jobid = dir t0 = time.time() pocket_write_buffer(p, jobid, iter, text, datasize) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes write_buffer:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" text_back = " " * datasize t0 = time.time() pocket_read_buffer(p, jobid, iter, text_back, datasize) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes read_buffer:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes lookup (metadata RPC):" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" os.remove(file_tmp) return
def handle_register_job(reader, writer): print( "-------------------------- REGISTER JOB --------------------------------", time.time()) jobname_len = yield from reader.read(INT) jobname_len, = struct.Struct("!i").unpack(jobname_len) jobname = yield from reader.read(jobname_len + 3 * INT + SHORT) jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct( "!" + str(jobname_len) + "siiih").unpack(jobname) jobname = jobname.decode('utf-8') # generate jobid if 'gg' in jobname: jobid = jobname + '-1234' jobid_int = 1234 else: jobid_int = randint(0, 1000000) jobid = jobname + "-" + str(jobid_int) print("received hints ", jobid, num_lambdas, jobGB, peakMbps, latency_sensitive) # create dir named jobid # NOTE: this is blocking but we are not yielding createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return ret = pocket.create_dir(createdirsock, None, jobid) if jobGB == 0 or peakMbps == 0: jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB, peakMbps, latency_sensitive) nvm_ip = [] for each_ip in nvm_ip: print("Adding this nvme server" + each_ip) add_nvme_datanodes(each_ip) # generate weightmask print("Generating weightmask") wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps, latency_sensitive) # wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)] # register job in table # nvm_ip = ['10.1.80.147', '10.1.71.111'] # for each_ip in nvm_ip: # print("Adding this nvme server" + each_ip) # add_nvme_datanodes(each_ip) #wmask[1] = (ioctlcmd.calculate_datanode_hash(nvm_ip, 1234), 0.25) #wmask_str[1] = (nvm_ip + ':1234', 0.24999999999999994) err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str) # send wmask to metadata server ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT) if ioctlsock is None: return yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i") resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE, jobid_int) pkt = resp_packer.pack(*resp) writer.write(pkt) print( "-------------------------- REGISTERED JOB --------------------------------" ) return
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 10000 datasize = 32 #bytes file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # write to crail p = pocket.connect("10.1.129.91", 9070) jobid = 'lambda3' print pocket.create_dir(p, 'new_dir', jobid) print pocket.put(p, file_tmp, "test", jobid) return r = pocket.register_job(p, jobid) # works if return 0 if r != 0: print "registration failed" return time_list = [] t0 = time.time() pocket_write(p, jobid, iter, file_tmp) t1 = time.time() print "==========================================" #print np.percentile(time_list, 90) print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes write:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_read(p, jobid, iter, file_tmp) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes read:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes lookup:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" os.remove(file_tmp) return