def handle_deregister_job(reader, writer): jobid_len = yield from reader.read(INT) jobid_len, = struct.Struct("!i").unpack(jobid_len) jobid = yield from reader.read(jobid_len) jobid, = struct.Struct("!" + str(jobid_len) + "s").unpack(jobid) jobid = jobid.decode('utf-8') print( "------------------------- DEREGISTER JOB --------------------------------", time.time()) # clear weight of this job for (datanodeip_port, weight) in job_table.loc[jobid, 'wmask_str']: datanode_alloc.at[datanodeip_port, 'net'] = datanode_alloc.at[datanodeip_port, 'net'] - weight # delete job from table err = remove_job(jobid) if err == 0: # delete dir named jobid createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return pocket.delete(createdirsock, None, "/" + jobid) #pocket.close(createdirsock) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT) resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, DEREGISTER_OPCODE) pkt = resp_packer.pack(*resp) writer.write(pkt) print( "------------------------- DEREGISTERED JOB --------------------------------" ) return
def handle_register_job(reader, writer): print( "-------------------------- REGISTER JOB --------------------------------", time.time()) jobname_len = yield from reader.read(INT) jobname_len, = struct.Struct("!i").unpack(jobname_len) jobname = yield from reader.read(jobname_len + 3 * INT + SHORT) jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct( "!" + str(jobname_len) + "siiih").unpack(jobname) jobname = jobname.decode('utf-8') # generate jobid if 'gg' in jobname: jobid = jobname + '-1234' jobid_int = 1234 else: jobid_int = randint(0, 1000000) jobid = jobname + "-" + str(jobid_int) print("received hints ", jobid, num_lambdas, jobGB, peakMbps, latency_sensitive) # create dir named jobid createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return pocket.create_dir(createdirsock, None, jobid) #pocket.close(createdirsock) if jobGB == 0 or peakMbps == 0: jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB, peakMbps, latency_sensitive) # generate weightmask wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps, latency_sensitive) # wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)] # register job in table err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str) # send wmask to metadata server ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT) if ioctlsock is None: return yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i") resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE, jobid_int) pkt = resp_packer.pack(*resp) writer.write(pkt) print( "-------------------------- REGISTERED JOB --------------------------------" ) return
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 100 datasize = 1024 #bytes jobid = "latency-test".join( random.sample(string.ascii_letters + string.digits, 6)) namenode_ip = "10.1.47.178" file_tmp = '/tmp/file_tmp2' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) # test read/write through buffer dir = jobid + "microbenchmark" pocket.create_dir(p, dir, "") jobid = dir t0 = time.time() pocket_write_buffer(p, jobid, iter, text, datasize) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes write_buffer:") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") text_back = " " * datasize t0 = time.time() pocket_read_buffer(p, jobid, iter, text_back, datasize) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes read_buffer:") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print("==========================================") print("Stats for " + str(iter) + " iter of " + str(datasize) + " bytes lookup (metadata RPC):") throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print("throughput (Gb/s) = " + str(throughput)) print("latency (us) = " + str((t1 - t0) / iter * 1e6)) print("==========================================") os.remove(file_tmp) return
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 50000 datasize = 1024 #bytes jobid = "latency-test" namenode_ip = "10.1.0.10" file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) # test read/write through buffer dir = jobid + "1" pocket.create_dir(p, dir, "") jobid = dir t0 = time.time() pocket_write_buffer(p, jobid, iter, text, datasize) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes write_buffer:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" text_back = " " * datasize t0 = time.time() pocket_read_buffer(p, jobid, iter, text_back, datasize) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes read_buffer:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes lookup (metadata RPC):" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" os.remove(file_tmp) return
def lambda_handler(event, context): id = int(event['id']) n = num_workers = int(event['n']) type = event['type'] iter = int(event['iter']) datasize = int(event['datasize']) #bytes namenode_ip = "10.1.22.136" # create a file of size (datasize) bytes file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) jobid = "" if type == "write": t1 = time.time() pocket_write(p, jobid, iter, file_tmp, id) t2 = time.time() elif type == "read": t1 = time.time() pocket_read(p, jobid, iter, file_tmp, id) t2 = time.time() elif type == "lookup": t1 = time.time() pocket_lookup(p, jobid, iter, id) t2 = time.time() elif type == "writebuffer": t1 = time.time() pocket_write_buffer(p, jobid, iter, text, datasize, id) t2 = time.time() elif type == "readbuffer": text_back = " " * datasize t1 = time.time() pocket_read_buffer(p, jobid, iter, text_back, datasize, id) t2 = time.time() else: return "Illegal type" # upload network data redis_host = "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com" startup_nodes = [{"host": redis_host, "port": "6379"}] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) log = {'t': t2 - t1, 't_start': t1} log_str = pickle.dumps(log) key = '/throughput-log' + '-' + str(n) + '-' + str(id) redis_client.set(key, log_str) print key + " logged" os.remove(file_tmp) return type + " finished"
def handle_deregister_job(reader, writer): jobid_len = yield from reader.read(INT) jobid_len, = struct.Struct("!i").unpack(jobid_len) jobid = yield from reader.read(jobid_len) jobid, = struct.Struct("!" + str(jobid_len) + "s").unpack(jobid) jobid = jobid.decode('utf-8') print("------------------------- DEREGISTER JOB --------------------------------", time.time()) # clear weight of this job if jobid not in job_datanode_net_allocations: print("ERROR: job to deregister no net allocation recognized!\n") print(job_datanode_net_allocations) for (datanodeip_port, weight) in job_datanode_net_allocations[jobid]: datanode_alloc.at[datanodeip_port, 'net'] -= weight if datanode_alloc.at[datanodeip_port, 'net'] < 0.0: # could happen due to rounding datanode_alloc.at[datanodeip_port, 'net'] = 0.0 if jobid in job_datanode_dramGB_allocations: for (datanodeip_port, weight) in job_datanode_dramGB_allocations[jobid]: datanode_alloc.at[datanodeip_port, 'DRAM_GB'] -= weight if datanode_alloc.at[datanodeip_port, 'DRAM_GB'] < 0.0: # could happen due to rounding datanode_alloc.at[datanodeip_port, 'DRAM_GB'] = 0.0 elif jobid in job_datanode_flashGB_allocations: for (datanodeip_port, weight) in job_datanode_flashGB_allocations[jobid]: datanode_alloc.at[datanodeip_port, 'Flash_GB'] -= weight if datanode_alloc.at[datanodeip_port, 'Flash_GB'] < 0.0: # could happen due to rounding datanode_alloc.at[datanodeip_port, 'Flash_GB'] = 0.0 else: print("ERROR: job to deregister no capacity allocation recognized!\n") print(job_datanode_dramGB_allocations) print(job_datanode_flashGB_allocations) # for (datanodeip_port, weight) in job_table.loc[jobid,'wmask_str']: # datanode_alloc.at[datanodeip_port, 'net'] = datanode_alloc.at[datanodeip_port, 'net'] - weight # delete job from table err = remove_job(jobid) if err == 0: # delete dir named jobid # NOTE: this is blocking but we are not yielding createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return pocket.delete(createdirsock, None, "/" + jobid) #pocket.close(createdirsock) print(datanode_alloc) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT) resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, DEREGISTER_OPCODE) pkt = resp_packer.pack(*resp) writer.write(pkt) print("------------------------- DEREGISTERED JOB --------------------------------") return
def lambda_handler(event, context): id = int(event['id']) n = num_workers = int(event['n']) bucket_name = str(event['bucket_name']) path = str(event['path']) n_tasks = n t0 = time.time() #[s3] read from input file: input<id> s3 = boto3.resource('s3') file_local = '/tmp/input_tmp' lines = [] # read 4 100MB files m = 1000 / n_tasks for i in xrange(m): i += id * m key = path + 'input' + str(i) s3.Bucket(bucket_name).download_file(key, file_local) with open(file_local, "r") as f: lines += f.readlines() #each line contains a 100b record os.remove(file_local) t1 = time.time() #partition p_list = [[] for x in xrange(n_tasks)] #list of n partitions #hardcode for line in lines: key1 = ord(line[0]) - 32 # key range 32-126 key2 = ord(line[1]) - 32 #126-32+1=95 #p = n/95 # 2500/(126-32+1) ~ 26.3 = 26 #index = int(26.3*(key1+key2/95.0)) p = n_tasks / 95.0 # total of 250 tasks index = int(p * (key1 + key2 / 95.0)) p_list[index].append(line) t2 = time.time() #write to output files: shuffle<id 0> shuffle<id 1> shuffle<id num_workers-1> # connect to crail p = pocket.connect("10.1.0.10", 9070) #jobid = "" jobid = str(event['id']) file_tmp = file_local for i in xrange(n_tasks): with open(file_tmp, "w") as f: f.writelines(p_list[i]) key = 'shuffle' + str(id) + '-' + str(i) src_filename = file_tmp dst_filename = '/' + key r = pocket.put(p, src_filename, dst_filename, jobid) if r != 0: raise Exception("put failed: " + dst_filename) return -1 t3 = time.time() # upload log startup_nodes = [{ "host": "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com", "port": "6379" }] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) log = {'id': id, 't0': t0, 't1': t1, 't2': t2, 't3': t3} log_str = pickle.dumps(log) key = '/map-log' + '-' + '100GB' + '-' + str(n) + '-' + str(id) redis_client.set(key, log_str) print key + " logged" os.remove(file_tmp) #crail.close(socket, ticket, p) r = 'map finished ' + str(id) print r return r
def run_command(key): begin_of_function = time.time() logger = logging.getLogger(__name__) print("taskId = " + str(key['taskId'])) print("number of inputs = " + str(key['inputs'])) print("number of output partitions = " + str(key['parts'])) # TODO: make the parameters configurable taskId = key['taskId'] # 1T #totalInputs = 10000 totalInputs = key['total_input'] inputsPerTask = key['inputs'] taskPerRound = key['taskPerRound'] rounds = (inputsPerTask + taskPerRound - 1) / taskPerRound numPartitions = key['parts'] bucketName = key['bucket'] jobid_int = int(key['job_number']) pocket_job_name = "job" + str(jobid_int) print("Pocket job name " + pocket_job_name) #jobid = pocket.register_job(pocket_job_name, capacityGB=1) jobid = pocket_job_name print("(" + str(taskId) + ")" + "Finish registering job") pocket_namenode = pocket.connect("10.1.0.10", 9070) print("(" + str(taskId) + ")" + "Connecting namenode job") print("See the number of partitions: " + str(numPartitions)) min_value = struct.unpack(">I", b"\x00\x00\x00\x00")[0] max_value = struct.unpack(">I", b"\xff\xff\xff\xff")[0] rangePerPart = int((max_value - min_value) / numPartitions) print("here 1 " + str(rangePerPart)) keyType = np.dtype([('key', 'S4')]) # 4 bytes good enough for partitioning recordType = np.dtype([('key', 'S4'), ('value', 'S96')]) print("here 2") boundaries = [] # (numPartitions-1) boundaries for i in range(1, numPartitions): # 4 bytes unsigned integers b = struct.pack('>I', rangePerPart * i) boundaries.append(b) client = boto3.client('s3', 'us-west-2') print("(" + str(taskId) + ")" + "Connected s3 client") [t1, t2, t3] = [time.time()] * 3 [read_time, work_time, write_time] = [0] * 3 # a total of 10 threads read_pool = ThreadPool(1) number_of_clients = 1 write_pool = ThreadPool(number_of_clients) clients = [] for client_id in range(number_of_clients): clients.append(boto3.client('s3', 'us-west-2')) write_pool_handler_container = [] print("(" + str(taskId) + ")" + "rounds" + str(rounds)) # manager = Manager() rounds = int(rounds) for roundIdx in range(rounds): inputs = [] def read_work(read_key): inputId = read_key['inputId'] keyname = "input/part-" + str(inputId) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "input/" + m.hexdigest( )[:8] + "-part-" + str(inputId) print("(" + str(taskId) + ")" + "fetching " + randomized_keyname) obj = client.get_object(Bucket=bucketName, Key=randomized_keyname) print("(" + str(taskId) + ")" + "fetching " + randomized_keyname + " done") fileobj = obj['Body'] #data = np.fromstring(fileobj.read(), dtype=recordType) data = np.frombuffer(fileobj.read(), dtype=recordType) print("(" + str(taskId) + ")" + "conversion " + randomized_keyname + " done") print("(" + str(taskId) + ")" + "size " + randomized_keyname + " " + str(len(data))) inputs.append(data) startId = taskId * inputsPerTask + roundIdx * taskPerRound endId = min( taskId * inputsPerTask + min( (roundIdx + 1) * taskPerRound, inputsPerTask), totalInputs) inputIds = range(startId, endId) if len(inputIds) == 0: break print("(" + str(taskId) + ")" + "Range for round " + str(roundIdx) + " is (" + str(startId) + "," + str(endId) + ")") read_keylist = [] for i in range(len(inputIds)): read_keylist.append({'inputId': inputIds[i], 'i': i}) # before processing, make sure all data is read read_pool.map(read_work, read_keylist) print("(" + str(taskId) + ")" + "read call done ") print("(" + str(taskId) + ")" + "size of inputs" + str(len(inputs))) records = np.concatenate(inputs) gc.collect() t1 = time.time() print("(" + str(taskId) + ")" + 'read time ' + str(t1 - t3)) read_time = t1 - t3 if numPartitions == 1: ps = [0] * len(records) else: ps = np.searchsorted(boundaries, records['key']) t2 = time.time() print("(" + str(taskId) + ")" + 'calculating partitions time: ' + str(t2 - t1)) # before processing the newly read data, make sure outputs are all written out if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() twait_start = time.time() write_pool_handler.wait() twait_end = time.time() if twait_end - twait_start > 0.5: print("(" + str(taskId) + ")" + 'write time = ' + str(twait_end - t3) + " slower than read " + str(t1 - t3)) else: print("(" + str(taskId) + ")" + 'write time < ' + str(twait_end - t3) + " faster than read " + str(t1 - t3)) t2 = time.time() gc.collect() numPartitions = int(numPartitions) outputs = [[] for i in range(0, numPartitions)] for idx, record in enumerate(records): outputs[ps[idx]].append(record) t3 = time.time() print("(" + str(taskId) + ")" + 'paritioning time: ' + str(t3 - t2)) work_time = t3 - t1 def write_work_client(writer_key): client_id = writer_key['i'] mapId = rounds * taskId + writer_key['roundIdx'] key_per_client = writer_key['key-per-client'] key_per_client = int(key_per_client) client_id = int(client_id) numPartitions = int(writer_key['num_partitions']) print("(" + str(taskId) + ")" + "range" + str(key_per_client) + " " + str(client_id) + " " + str(numPartitions)) for i in range( key_per_client * client_id, min(key_per_client * (client_id + 1), numPartitions)): keyname = "shuffle-part-" + str(mapId) + "-" + str(i) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "shuffle-" + m.hexdigest( )[:8] + "-part-" + str(mapId) + "-" + str(i) print("The name of the key to write is: " + randomized_keyname) bytes_body = np.asarray(outputs[ps[i]]).tobytes() print("Hey top top " + str(len(bytes_body))) datasize = 1700000 #print(body) #body = bytes_body.decode('ascii') body = b64encode(bytes_body).decode('utf-8') body = body.ljust(datasize, '=') print("Byte to be written: " + str(len(body))) print("Last ten bits after padding: " + body[-10:]) pocket.put_buffer(pocket_namenode, body, len(body), randomized_keyname, jobid) writer_keylist = [] key_per_client = (numPartitions + number_of_clients - 1) / number_of_clients number_of_clients = int(number_of_clients) for i in range(number_of_clients): writer_keylist.append({ 'roundIdx': roundIdx, 'i': i, 'key-per-client': key_per_client, 'num_partitions': numPartitions }) for i in range(number_of_clients): write_work_client(writer_keylist[i]) #write_pool_handler = write_pool.map_async(write_work_client, writer_keylist) #write_pool_handler_container.append(write_pool_handler) #pocket.deregister_job(jobid) if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() write_pool_handler.wait() twait_end = time.time() print("(" + str(taskId) + ")" + 'last write time = ' + str(twait_end - t3)) write_time = twait_end - t3 read_pool.close() write_pool.close() read_pool.join() write_pool.join() end_of_function = time.time() print("(" + str(taskId) + ")" + "Exciting this function") return begin_of_function, end_of_function, read_time, work_time, write_time
def lambda_handler(event, context): rid = int(event['rid']) n = num_mapper = int(event['num_mapper']) t0 = time.time() # read shuffle files # connect to crail p = pocket.connect("10.1.129.91", 9070) jobid = "" #jobid = str(event['id']) word_count_list = [] for i in xrange(num_mapper): #shuffle_file = 'shuffle/shuffle-' + str(i) + '-' + str(rid) #body = pickle.loads(redis_client.get(shuffle_file)) #word_count_list += body key = 'shuffle-' + str(i) + '-' + str(rid) src_filename = '/tmp/shuffle' dst_filename = '/' + key r = pocket.get(p, dst_filename, src_filename, jobid) if r != 0: raise Exception("get failed: " + dst_filename) return -1 with open(src_filename, 'r') as f: word_count_list += json.load(f) os.remove(src_filename) t1 = time.time() # add up word count word_count = {} for (word, count) in word_count_list: if word in word_count: word_count[word] += count else: word_count[word] = count t2 = time.time() # write output to s3 s3 = boto3.resource('s3') file_tmp = '/tmp/output' with open(file_tmp, "w+") as f: for k, v in word_count.items(): f.write(str(k) + ' ' + str(v) + '\n') key = 'output/output' + str(rid) bucket_name = 'wordcount-yawen' s3.Bucket(bucket_name).upload_file(file_tmp, key) os.remove(file_tmp) t3 = time.time() # upload log startup_nodes = [{ "host": "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com", "port": "6379" }] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) # t1-t0: intermediate read # t2-t1: adding word count # t3-t2: s3 write log = {'id': rid, 't0': t0, 't1': t1, 't2': t2, 't3': t3} key = 'reduce-log-' + str(n) + '-' + str(rid) redis_client.set(key, pickle.dumps(log)) #print "reducer"+str(rid)+' finished' print key + ' logged'
def run_command(key): global concat_time begin_of_function = time.time() logger = logging.getLogger(__name__) print("taskId = " + str(key['taskId'])) #print("number of works = " + str(key['works'])) #print("number of input partitions = " + str(key['parts'])) bucketName = key['bucket'] taskId = key['taskId'] rounds = key['works'] numPartitions = int(key['parts']) jobid_int = int(key['job_number']) pocket_job_name = "job" + str(jobid_int) # jobid = pocket.register_job(pocket_job_name, capacityGB=1) jobid = pocket_job_name pocket_namenode = pocket.connect("10.1.0.10", 9070) # 10 bytes for sorting recordType = np.dtype([('key', 'S10'), ('value', 'S90')]) client = boto3.client('s3', 'us-west-2') # rs = [] # for hostname in key['redis'].split(";"): # r1 = StrictRedis(host=hostname, port=6379, db=0).pipeline() # rs.append(r1) # nrs = len(rs) [t1, t2, t3] = [time.time()] * 3 [read_time, work_time, write_time] = [0] * 3 # a total of 10 threads write_pool = ThreadPool(1) number_of_clients = 1 read_pool = ThreadPool(number_of_clients) clients = [] number_of_clients = int(number_of_clients) for client_id in range(number_of_clients): clients.append(boto3.client('s3', 'us-west-2')) write_pool_handler_container = [] rounds = int(rounds) for roundIdx in range(rounds): inputs = [] def read_work(reader_key): client_id = reader_key['client_id'] reduceId = rounds * taskId + reader_key['roundIdx'] key_per_client = reader_key['key-per-client'] key_per_client = int(key_per_client) client_id = int(client_id) objs = [] for mapId in range(key_per_client * client_id, min(key_per_client * (client_id + 1), numPartitions)): # for mapId in range(1): keyname = "shuffle-part-" + str(mapId) + "-" + str(reduceId) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "shuffle-" + m.hexdigest()[:8] + "-part-" + str(mapId) + "-" + str(reduceId) print("The name of the key to read is: " + randomized_keyname) try: datasize = 17000000 textback = " "*datasize pocket.get_buffer(pocket_namenode, randomized_keyname, textback, datasize, jobid) print("Successfully read") #pos = textback.find('.') #print("Padding position: " + str(pos)) original_text = b64decode(textback.encode('utf-8')) print("last ten bytes after padding: " + textback[-10:]) objs.append(original_text) except Exception: print("reading error key " + randomized_keyname) raise data = [np.fromstring(obj, dtype=recordType) for obj in objs] [d.sort(order='key') for d in data] inputs.extend(data) reader_keylist = [] key_per_client = (numPartitions + number_of_clients - 1) / number_of_clients number_of_clients = int(number_of_clients) for client_id in range(number_of_clients): reader_keylist.append({'roundIdx': roundIdx, 'client_id': client_id, 'key-per-client': key_per_client}) for i in range(number_of_clients): read_work(reader_keylist[i]) t1 = time.time() print('read time ' + str(t1 - t3)) read_time = t1 - t3 if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() twait_start = time.time() write_pool_handler.wait() twait_end = time.time() if twait_end - twait_start > 0.5: print('write time = ' + str(twait_end - t3) + " slower than read " + str(t1 - t3)) else: print('write time < ' + str(twait_end - t3) + " faster than read " + str(t1 - t3)) t2 = time.time() records = np.concatenate(inputs) gc.collect() concat_time = len(records) records.sort(order='key', kind='mergesort') t3 = time.time() print('sort time: ' + str(t3 - t2)) work_time = t3 - t2 def write_work(reduceId): keyname = "output/part-" + str(reduceId) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "output/" + m.hexdigest()[:8] + "-part-" + str(reduceId) body = records.tobytes() client.put_object(Bucket=bucketName, Key=randomized_keyname, Body=body) write_pool_handler = write_pool.map_async(write_work, [taskId * rounds + roundIdx]) write_pool_handler_container.append(write_pool_handler) if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() write_pool_handler.wait() twait_end = time.time() print('last write time = ' + str(twait_end - t3)) write_time = twait_end - t3 read_pool.close() write_pool.close() read_pool.join() write_pool.join() end_of_function = time.time() return begin_of_function, end_of_function, read_time, work_time, write_time, concat_time
def lambda_handler(event, context): ''' startup_nodes = [{"host": "rediscluster.a9ith3.clustercfg.usw2.cache.amazonaws.com", "port": "6379"}] client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) client.flushall() print "Redis cluster flushed" return ''' mid = int(event['mid']) n = num_reducer = int(event['num_reducer']) #100 t0 = time.time() #[s3] read from input file: input<id> s3 = boto3.resource('s3') bucket_name = 'wordcount-yawen' #key = 'input/input' + str(mid) key = 'input/input_5MB' file_tmp = '/tmp/input' s3.Bucket(bucket_name).download_file(key, file_tmp) with open(file_tmp, "r") as f: lines = f.readlines() t1 = time.time() words = [] for line in lines: words += re.split(r'[^\w]+', line) words = list(filter(None, words)) # count word frequency word_count = {} #(word, count) for word in words: if word in word_count: word_count[word] += 1 else: word_count[word] = 1 # partition among reducers shuffle = {} #(rid, [(word,count), ...]) for k, v in word_count.items(): rid = int(hashlib.md5(k).hexdigest(), 16) % num_reducer if rid in shuffle: shuffle[rid].append((k, v)) else: shuffle[rid] = [(k, v)] t2 = time.time() # connect to crail p = pocket.connect("10.1.129.91", 9070) jobid = "" #jobid = str(event['id']) for k, v in shuffle.items(): #shuffle_file = 'shuffle/shuffle-'+str(mid)+'-'+str(k) #value = pickle.dumps(v) #redis_client.set(shuffle_file, value) key = 'shuffle-' + str(mid) + '-' + str(k) src_filename = file_tmp #'/tmp/shuffle' dst_filename = '/' + key with open(src_filename, 'w') as f: json.dump(v, f) r = pocket.put(p, src_filename, dst_filename, jobid) t3 = time.time() # upload log startup_nodes = [{ "host": "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com", "port": "6379" }] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) # t1-t0: s3 read # t2-t1: word counting & partition # t3-t2: intermediate write log = {'id': mid, 't0': t0, 't1': t1, 't2': t2, 't3': t3} key = 'map-log-' + str(n) + '-' + str(mid) redis_client.set(key, pickle.dumps(log)) os.remove(file_tmp) #print "mapper"+str(mid)+' finished' print key + ' logged'
def lambda_handler(event, context): id = int(event['id']) n = num_workers = int(event['n']) type = event['type'] iter = int(event['iter']) datasize = int(event['datasize']) #bytes namenode_ip = "10.1.22.136" LOGS_PATH = 'logs-'+str(n) STOP = threading.Event() class TimeLog: def __init__(self, enabled=True): self.enabled = enabled self.start = time.time() self.prev = self.start self.points = [] self.sizes = [] def add_point(self, title): if not self.enabled: return now = time.time() self.points += [(title, now - self.prev)] self.prev = now def upload_net_bytes(rclient, rxbytes_per_s, txbytes_per_s, cpu_util, timelogger, reqid): #rclient = redis.Redis(host=REDIS_HOSTADDR_PRIV, port=6379, db=0) netstats = LOGS_PATH + '/netstats-' + reqid rclient.set(netstats, str({'lambda': reqid, 'started': timelogger.start, 'rx': rxbytes_per_s, 'tx': txbytes_per_s, 'cpu': cpu_util}).encode('utf-8')) print "wrote netstats" return STOP.set() timelogger = TimeLog(enabled=True) # create a file of size (datasize) bytes file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a'*datasize f.write(text) # connect to pocket p = pocket.connect(namenode_ip, 9070) jobid = "" t1=time.time() req_per_s = pocket_lookup(p, jobid, iter, id) t2=time.time() # upload network data redis_host = "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com" startup_nodes = [{"host": redis_host, "port": "6379"}] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) rclient = redis_client STOP.clear() place_holder = [1]*len(req_per_s) upload_net_bytes(rclient, req_per_s, place_holder, place_holder, timelogger, str(id)) print "iops stats uploaded" # upload network data redis_host = "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com" startup_nodes = [{"host": redis_host, "port": "6379"}] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) log = {'t':t2-t1, 't_start':t1} log_str = pickle.dumps(log) key = '/throughput-log'+'-'+str(n)+'-'+str(id) redis_client.set(key, log_str) print key + " logged" os.remove(file_tmp) return type+" finished"
def handle_register_job(reader, writer): print( "-------------------------- REGISTER JOB --------------------------------", time.time()) jobname_len = yield from reader.read(INT) jobname_len, = struct.Struct("!i").unpack(jobname_len) jobname = yield from reader.read(jobname_len + 3 * INT + SHORT) jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct( "!" + str(jobname_len) + "siiih").unpack(jobname) jobname = jobname.decode('utf-8') # generate jobid if 'gg' in jobname: jobid = jobname + '-1234' jobid_int = 1234 else: jobid_int = randint(0, 1000000) jobid = jobname + "-" + str(jobid_int) print("received hints ", jobid, num_lambdas, jobGB, peakMbps, latency_sensitive) # create dir named jobid # NOTE: this is blocking but we are not yielding createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT) if createdirsock is None: return ret = pocket.create_dir(createdirsock, None, jobid) if jobGB == 0 or peakMbps == 0: jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB, peakMbps, latency_sensitive) nvm_ip = [] for each_ip in nvm_ip: print("Adding this nvme server" + each_ip) add_nvme_datanodes(each_ip) # generate weightmask print("Generating weightmask") wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps, latency_sensitive) # wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)] # register job in table # nvm_ip = ['10.1.80.147', '10.1.71.111'] # for each_ip in nvm_ip: # print("Adding this nvme server" + each_ip) # add_nvme_datanodes(each_ip) #wmask[1] = (ioctlcmd.calculate_datanode_hash(nvm_ip, 1234), 0.25) #wmask_str[1] = (nvm_ip + ':1234', 0.24999999999999994) err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str) # send wmask to metadata server ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT) if ioctlsock is None: return yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask) # reply to client with jobid int resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i") resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE, jobid_int) pkt = resp_packer.pack(*resp) writer.write(pkt) print( "-------------------------- REGISTERED JOB --------------------------------" ) return
def run_command(key): """ keylist.append({'taskId': i, 'job_number': job_number, 'total_input': numTasks, 'write_element_size': write_element_size, 'process_time': process_time, 'total_time': total_time}) """ pywren.wrenlogging.default_config('INFO') begin_of_function = time.time() logger = logging.getLogger(__name__) logger.info("taskId = " + str(key['taskId'])) taskId = key['taskId'] jobid_int = int(key['job_number']) pocket_job_name = key['pocket_job_name'] write_element_size = int(key['write_element_size']) process_time = int(key['process_time']) total_time = int(key['total_time']) pocket_namenode = pocket.connect("10.1.0.10", 9070) [read_time, work_time, write_time] = [0] * 3 start_time = time.time() # a total of 10 threads number_of_clients = 1 write_pool = ThreadPool(number_of_clients) time.sleep(process_time) logger.info("Process finish here: " + str(time.time())) def write_work_client(writer_key): start_time = time.time() client_id = int(writer_key['client_id']) taskID = writer_key['taskId'] jobID = writer_key['jobid'] datasize = writer_key['write_element_size'] #datasize = 1310720 total_time = writer_key['total_time'] body = b'a' * datasize client_id = int(client_id) count = 0 throughput_step = 1 throughput_count = 1 throughput_total = 0 throughput_nops = 0 ret = [] while time.time() < start_time + total_time: count = count + 1 keyname = str(jobID) + "-" + str(taskID) + "-" + str(count) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = str(jobID) + "-" + str(taskID) + '-' + m.hexdigest()[:8] + '-' + str(count) #logger.info("(" + str(taskId) + ")" + "The name of the key to write is: " + randomized_keyname) start = time.time() #logger.info("[POCKET] [" + str(jobID) + "] " + str(start) + " " + str(taskID) + " " + str(len(body)) + " write " + "S") r = pocket.put_buffer_bytes(pocket_namenode, body, len(body), randomized_keyname, pocket_job_name) end = time.time() #logger.info("[POCKET] [" + str(jobID) + "] " + str(end) + " " + str(taskID) + " " + str(len(body)) + " write " + "E " + str(r) ) throughput_total += end - start throughput_nops += 1 if end - start_time >= throughput_count: throughput = throughput_nops / throughput_total ret.append((end, throughput)) throughput_nops = 0 throughput_count += throughput_step throughput_total = 0 logger.info("Write finish here: " + str(time.time())) return ret writer_keylist = [] number_of_clients = int(number_of_clients) for i in range(number_of_clients): writer_keylist.append({'client_id': i, 'taskId': taskId, 'jobid': jobid_int, 'write_element_size': write_element_size, 'total_time': total_time}) start_time = time.time() write_pool_handler_container = [] write_pool_handler = write_pool.map_async(write_work_client, writer_keylist) write_pool_handler_container.append(write_pool_handler) if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() ret = write_pool_handler.get() twait_end = time.time() write_time = twait_end - start_time write_pool.close() write_pool.join() end_of_function = time.time() return begin_of_function, end_of_function, write_time, ret
def run_command(key): global concat_time pywren.wrenlogging.default_config('INFO') begin_of_function = time.time() logger = logging.getLogger(__name__) logger.info("taskId = " + str(key['taskId'])) #logger.info("number of works = " + str(key['works'])) #logger.info("number of input partitions = " + str(key['parts'])) bucketName = key['bucket'] taskId = key['taskId'] rounds = key['works'] numPartitions = int(key['parts']) jobid_int = int(key['job_number']) pocket_job_name = "job" + str(jobid_int) # jobid = pocket.register_job(pocket_job_name, capacityGB=1) jobid = pocket_job_name pocket_namenode = pocket.connect("10.1.0.10", 9070) # 10 bytes for sorting recordType = np.dtype([('key', 'S10'), ('value', 'S90')]) client = boto3.client('s3', 'us-west-2') [t1, t2, t3] = [time.time()] * 3 [read_time, work_time, write_time] = [0] * 3 # a total of 10 threads write_pool = ThreadPool(1) number_of_clients = 1 read_pool = ThreadPool(number_of_clients) clients = [] number_of_clients = int(number_of_clients) for client_id in range(number_of_clients): clients.append(boto3.client('s3', 'us-west-2')) write_pool_handler_container = [] rounds = int(rounds) for roundIdx in range(rounds): inputs = [] def read_work(reader_key): client_id = reader_key['client_id'] reduceId = rounds * taskId + reader_key['roundIdx'] key_per_client = reader_key['key-per-client'] key_per_client = int(key_per_client) client_id = int(client_id) taskID = int(reader_key['taskId']) jobID = int(reader_key['jobid']) objs = [] for mapId in range(key_per_client * client_id, min(key_per_client * (client_id + 1), numPartitions)): # for mapId in range(1): keyname = "shuffle-part-" + str(mapId) + "-" + str(reduceId) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "shuffle-" + m.hexdigest()[:8] + "-part-" + str(mapId) + "-" + str(reduceId) #logger.info("The name of the key to read is: " + randomized_keyname) try: # FIXME Need to set this stuff #datasize = 1300 * 1000 datasize = 1310720 logger.info("[POCKET] [" + str(jobID) +"] " + str(time.time_ns()) + " " + str(taskID) + " 0 read " + "S") r = pocket.get_buffer_bytes(pocket_namenode, randomized_keyname, datasize, jobid, DELETE_AFTER_READ=False) end_time = time.time_ns() #logger.info("Successfully read") #pos = textback.find('.') #logger.info("Padding position: " + str(pos)) #logger.info("last ten bytes after padding: " + textback[-10:]) #original_text = b64decode(textback.encode('utf-8')) logger.info("[POCKET] [" + str(jobID) + "] " + str(end_time) + " " + str(taskID) + " " + str(len(r)) + " read " + "E ") #logger.info("Size of original text: " + str(len(original_text))) objs.append(r[:1300000]) except Exception: logger.info("reading error key " + randomized_keyname) raise data = [np.fromstring(obj, dtype=recordType) for obj in objs] [d.sort(order='key') for d in data] inputs.extend(data) reader_keylist = [] key_per_client = (numPartitions + number_of_clients - 1) / number_of_clients number_of_clients = int(number_of_clients) for client_id in range(number_of_clients): reader_keylist.append({'roundIdx': roundIdx, 'client_id': client_id, 'key-per-client': key_per_client, 'taskId': taskId, 'jobid': jobid_int}) read_pool.map(read_work, reader_keylist) t1 = time.time() #logger.info('read time ' + str(t1 - t3)) read_time = t1 - t3 if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() twait_start = time.time() write_pool_handler.wait() twait_end = time.time() if twait_end - twait_start > 0.5: logger.info('write time = ' + str(twait_end - t3) + " slower than read " + str(t1 - t3)) else: logger.info('write time < ' + str(twait_end - t3) + " faster than read " + str(t1 - t3)) t2 = time.time() records = np.concatenate(inputs) gc.collect() concat_time = len(records) records.sort(order='key', kind='mergesort') t3 = time.time() #logger.info('sort time: ' + str(t3 - t2)) work_time = t3 - t2 def write_work(reduceId): keyname = "output/part-" + str(reduceId) m = hashlib.md5() m.update(keyname.encode('utf-8')) randomized_keyname = "output/" + m.hexdigest()[:8] + "-part-" + str(reduceId) body = records.tobytes() client.put_object(Bucket=bucketName, Key=randomized_keyname, Body=body) write_pool_handler = write_pool.map_async(write_work, [taskId * rounds + roundIdx]) write_pool_handler_container.append(write_pool_handler) if len(write_pool_handler_container) > 0: write_pool_handler = write_pool_handler_container.pop() write_pool_handler.wait() twait_end = time.time() #logger.info('last write time = ' + str(twait_end - t3)) write_time = twait_end - t3 read_pool.close() write_pool.close() read_pool.join() write_pool.join() end_of_function = time.time() return begin_of_function, end_of_function, read_time, work_time, write_time, concat_time
def lambda_handler(event, context): id = int(event['id']) n = num_workers = int(event['n']) bucket_name = str(event['bucket_name']) n_tasks = n log_file = [] t0 = time.time() # connect to crail #p = pocket.connect("10.1.12.156", 9070) p = pocket.connect("10.1.0.10", 9070) print "connected" jobid = "" #jobid = str(event['id']) #read from input file: shuffle<0 id> shuffle<1 id> ... shuffle<id num_workers-1> #''' file_tmp = '/tmp/tmp' all_lines = [] for i in xrange(n_tasks): #key = 'shuffle' + str(id) +'-'+ str(i) # wrong one just for testing key = 'shuffle' + str(i) + '-' + str(id) src_filename = key dst_filename = file_tmp #print src_filename r = pocket.get(p, src_filename, dst_filename, jobid) if r != 0: raise Exception("get failed: " + src_filename) return -1 #log_file.append((key, time.time())) with open(dst_filename, "r") as f: all_lines += f.readlines() #print src_filename + " read success" os.remove(file_tmp) #''' t1 = time.time() #print "read all from pocket" #merge & sort for i in xrange(len(all_lines)): all_lines[i] = (all_lines[i][:10], all_lines[i][12:]) all_lines.sort(key=lambda x: x[0]) for i in xrange(len(all_lines)): all_lines[i] = all_lines[i][0] + " " + all_lines[i][1] t2 = time.time() #[s3] write to output file: output<id> s3 = boto3.resource('s3') file_name = 'output/sorted_output' m = 1000 / n_tasks size = len(all_lines) / m for i in xrange(m): with open(file_tmp, "w+") as f: start = size * i end = start + size f.writelines(all_lines[start:end]) f.seek(0) body = f.read() key = file_name + str(id * m + i) s3.Bucket(bucket_name).upload_file(file_tmp, key) os.remove(file_tmp) t3 = time.time() # upload log startup_nodes = [{ "host": "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com", "port": "6379" }] redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True) log = {'id': id, 't0': t0, 't1': t1, 't2': t2, 't3': t3} log_str = pickle.dumps(log) key = '/reduce-log' + '-' + '100GB' + '-' + str(n) + '-' + str(id) redis_client.set(key, log_str) print key + " logged" ''' log_file_str = pickle.dumps(log_file) key = '/reduce-log-time'+'-'+'100GB'+'-'+str(n)+'-'+str(id) redis_client.set(key, log_file_str) print key + " logged" ''' #crail.close(socket, ticket, p) r = 'reduce finished ' + str(id) print r return r
def lambda_handler(event, context): # create a file of size (datasize) in bytes iter = 10000 datasize = 32 #bytes file_tmp = '/tmp/file_tmp' with open(file_tmp, 'w') as f: text = 'a' * datasize f.write(text) # write to crail p = pocket.connect("10.1.129.91", 9070) jobid = 'lambda3' print pocket.create_dir(p, 'new_dir', jobid) print pocket.put(p, file_tmp, "test", jobid) return r = pocket.register_job(p, jobid) # works if return 0 if r != 0: print "registration failed" return time_list = [] t0 = time.time() pocket_write(p, jobid, iter, file_tmp) t1 = time.time() print "==========================================" #print np.percentile(time_list, 90) print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes write:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_read(p, jobid, iter, file_tmp) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes read:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" t0 = time.time() pocket_lookup(p, jobid, iter) t1 = time.time() print "==========================================" print "Stats for " + str(iter) + " iter of " + str( datasize) + " bytes lookup:" throughput = iter * datasize * 8 / (t1 - t0) / 1e9 print "throughput (Gb/s) = " + str(throughput) print "time (s) = " + str(t1 - t0) print "latency (us) = " + str((t1 - t0) / iter * 1e6) print "==========================================" os.remove(file_tmp) return