def _EvictCache(): with Cons.MT("Drop caches ..."): Util.RunSubp("sudo sh -c \"echo 3 >/proc/sys/vm/drop_caches\"") if False: # Evict the DB data files from cache with Cons.MT("Evicting DB data ..."): Util.RunSubp("%s/work/vmtouch/vmtouch -e %s" % (os.path.expanduser("~"), Conf.GetDir("db_path"))) # Evict the input data files from cache so that the read IO is consistent # throughput the experiment. # Caching them would have been nice, but you can do it only when you limit # the memory smaller than 3.4 GB (= 15 (total ram) - 0.4 (for the OS) - 11.2 # (the 100% quizup data size) evict = True if evict: with Cons.MT("Evicting workload data ..."): Util.RunSubp( "%s/work/vmtouch/vmtouch -e %s" % (os.path.expanduser("~"), Conf.GetDir("workload_dir"))) else: with Cons.MT("Caching workload data ..."): Util.RunSubp( "%s/work/vmtouch/vmtouch -t %s" % (os.path.expanduser("~"), Conf.GetDir("workload_dir")))
def Restart(): with Cons.MT("Restarting dstat ...", print_time=False): Dstat._Stop() dn = "%s/dstat" % Conf.GetDir("log_archive_dn") Util.MkDirs(dn) # Get a list of all block devices devs = [] for f in os.listdir("/dev"): mo = None if socket.gethostname() == "node3": mo = re.match(r"sd\w$", f) else: mo = re.match(r"xvd\w$", f) if mo is not None: devs.append(f) global _cur_datetime _cur_datetime = datetime.datetime.now().strftime( "%y%m%d-%H%M%S.%f")[:-3] Cons.P(_cur_datetime) # Run dstat as a daemon fn_out = "%s/%s.csv" % (dn, _cur_datetime) cmd = "dstat -tcdnrmy -D %s --output %s" % (",".join(devs), fn_out) Util.RunDaemon(cmd)
def ParseArgs(): parser = argparse.ArgumentParser( description="Plot SSTables access frequencies and temperatures", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--simulation_time_begin", type=str, default="170101-005737.017", help="Simulation time begin") parser.add_argument("--log_dir", type=str, default="~/work/mutant/misc/rocksdb/log", help="Mutant simulation log directory") parser.add_argument("--dn_result", type=str, default=("%s/.result" % os.path.dirname(__file__)), help="Result directory") global _args _args = parser.parse_args() Cons.P("Parameters:") for a in vars(_args): Cons.P("%s: %s" % (a, getattr(_args, a)), ind=2)
def _DeleteOldAmisSnapshots(self): try: for img in self.imgs_acorn_to_delete: try: r = self._Bc().deregister_image(ImageId=img.image_id) Cons.P("%-20s deregistered AMI %s" % (self.region, img.image_id)) except botocore.exceptions.ClientError as e: if e.response["Error"][ "Code"] == "InvalidAMIID.Unavailable": pass else: raise e if len(self.ss_to_delete) > 0: time.sleep(1) for ss in self.ss_to_delete: r = self._Bc().delete_snapshot(SnapshotId=ss.snapshot_id) Cons.P("%-20s deleted snapshot %s" % (self.region, ss.snapshot_id)) except Exception as e: Cons.P("%s\n%s\nregion=%s" % (e, traceback.format_exc(), self.region)) os._exit(1)
def PlotWorkload(workload_type): Util.MkDirs(_dn_output) #(fn_plot_data_m_ls_st1, fn_plot_data_ind) = GetPlotDataMutant(workload_type, "ls-st1", "~/work/mutant/log/ycsb/workload-%s/mutant-ls-st1" % workload_type) (fn_plot_data_m_ls_st1, fn_plot_data_ind) = GetPlotDataMutant( workload_type, "ls-st1", "~/work/mutant/log/ycsb/workload-%s/170822-022606-d-ls-st1-short-exps" % workload_type) sys.exit(1) (fn_plot_data_r_st1, fn_plot_data_ind) = GetPlotDataRocksdb( workload_type, "st1", "~/work/mutant/log/ycsb/workload-%s/rocksdb-st1" % workload_type) (fn_plot_data_r_ls, fn_plot_data_ind) = GetPlotDataRocksdb( workload_type, "ls", "~/work/mutant/log/ycsb/workload-%s/rocksdb-ls" % workload_type) fn_out = "%s/ycsb-%s-thp-vs-latency.pdf" % (_dn_output, workload_type) with Cons.MT("Plotting ..."): env = os.environ.copy() env["FN_ROCKSDB_ST1"] = fn_plot_data_r_st1 env["FN_ROCKSDB_LS"] = fn_plot_data_r_ls env["FN_MUTANT_LS_ST1"] = fn_plot_data_m_ls_st1 env["FN_OUT"] = fn_out Util.RunSubp("gnuplot %s/thrp-vs-lat.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Stop(): with Cons.MT("Stopping dstat ...", print_time=False): Dstat._Stop() # Change the current dstat log file name to the simulation_time_begin of # the simulator. if _cur_datetime is not None: with Cons.MT("Renaming the log file and zipping ..."): dn_client = "%s/quizup" % Conf.GetDir("log_archive_dn") global _latest_client_log_dt _latest_client_log_dt = None for f in os.listdir(dn_client): mo = re.match( r"(?P<dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)$", f) if mo is not None: if _latest_client_log_dt is None: _latest_client_log_dt = mo.group("dt") else: _latest_client_log_dt = max( _latest_client_log_dt, mo.group("dt")) # There should be a client log file whose dt is bigger than # _cur_datetime if _latest_client_log_dt <= _cur_datetime: raise RuntimeError("Unexpected") fn0 = "%s/dstat/%s.csv" % (Conf.GetDir("log_archive_dn"), _cur_datetime) fn1 = "%s/dstat/%s.csv" % (Conf.GetDir("log_archive_dn"), _latest_client_log_dt) Cons.P("renaming %s to %s" % (fn0, fn1)) os.rename(fn0, fn1) Util.RunSubp("7z a -mx %s.7z %s" % (fn1, fn1))
def Deleted(self, line): mo = re.match(r"(?P<ts>(\d|\/|-|:|\.)+) .+EVENT_LOG_v1 (?P<json>.+)", line) if mo is None: raise RuntimeError("Unexpected: [%s]" % line) j = mo.group("json") try: j1 = json.loads(j) except ValueError as e: Cons.P("%s [%s]" % (e, line)) sys.exit(1) ts0 = mo.group("ts") sst_id = int(j1["file_number"]) ts1 = self._GetRelTs(ts0) si = self.rocks_log_reader.sst_info.Get(sst_id) if si is None: Cons.P( "Interesting: Sst (id %d) deleted without a creation. Ignore." % sst_id) return path_id = si.PathId() self.cur_sstsize[path_id] -= si.Size() self.cur_numssts[path_id] -= 1 self.ts_sstsize[ts1] = list(self.cur_sstsize) self.ts_numssts[ts1] = list(self.cur_numssts)
def __init__(self): self.conn = None # Open or create DB fn_db = Conf.GetFn("whole_foods_loc_db") conn = None if os.path.exists(fn_db): with Cons.MT("Opening the existing db ..."): conn = sqlite3.connect(fn_db) if conn is None: raise RuntimeError( "Error! cannot create the database connection.") conn.row_factory = sqlite3.Row cur = conn.cursor() q = "SELECT count(*) as cnt FROM whole_foods_loc" cur.execute(q) r = cur.fetchone() Cons.P("There are %d records" % r["cnt"]) else: with Cons.MT("Creating a new db ..."): conn = sqlite3.connect(fn_db) if conn is None: raise RuntimeError( "Error! cannot create the database connection.") conn.row_factory = sqlite3.Row cur = conn.cursor() q = """CREATE TABLE IF NOT EXISTS whole_foods_loc ( addr text NOT NULL , lat real NOT NULL , lon real NOT NULL , PRIMARY KEY (addr) ); """ cur.execute(q) self.conn = conn
def GetRunningAvg(): fn = "%s/num-reads-by-obj-age" % _dn_out if os.path.isfile(fn): return fn with Cons.MT("Generating running average ..."): # Running average of the last n items, not in the last n-minute time # window. Should generate smoother curve for plotting. q = Q(60) with open(_fn_in) as fo, open(fn, "w") as fo_out: for line in fo: t = line.strip().split(" ") if len(t) != 2: raise RuntimeError("Unexpected [%s]" % line) ts = int(t[0]) num_reads = int(t[1]) q.Enq(num_reads) if False: if ts < 2 * 24 * 60: fo_out.write("%d %d -\n" % (ts, num_reads)) else: fo_out.write("%d %d %.1f\n" % (ts, num_reads, q.Avg())) else: fo_out.write("%d %d %.1f\n" % (ts, num_reads, q.Avg())) Cons.P("Created %s %d" % (fn, os.path.getsize(fn))) return fn
def PlotCompareTwo(): (fns_rocksdb, fn_sst_creation_stat) = RocksdbLog.GenDataFilesForGnuplot() #fn_cpu_stat_by_time = CompareCpu.GetHourlyFn() fn_cpu_1min_avg = CompareCpu.Get1minAvgFn() fn_mem_stat_by_time = CompareMem.GetHourlyFn() fn_mem_1min_avg = CompareMem.Get1minAvgFn() #time_max = "09:00:00" #time_max = "08:00:00" time_max = "07:50:00" exp_dts = [] for i in range(2): mo = re.match(r".+/(?P<exp_dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)-d", Conf.Get(i)) exp_dts.append(mo.group("exp_dt")) fn_out = "%s/mutant-overhead-%s.pdf" % (Conf.GetOutDir(), "-".join(exp_dts)) with Cons.MT("Plotting ..."): env = os.environ.copy() env["TIME_MAX"] = str(time_max) #env["CPU_STAT"] = fn_cpu_stat_by_time env["FN_CPU_1MIN_AVG"] = fn_cpu_1min_avg #env["MEM_STAT"] = fn_mem_stat_by_time env["FN_MEM_1MIN_AVG"] = fn_mem_1min_avg env["ROCKSDB0"] = fns_rocksdb[0] env["ROCKSDB1"] = fns_rocksdb[1] env["OUT_FN"] = fn_out Util.RunSubp("gnuplot %s/compare-two-exps.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def _Poll(jc_q): q = _GetQ() while True: try: msgs = q.receive_messages( #AttributeNames=[ # 'Policy'|'VisibilityTimeout'|'MaximumMessageSize'|'MessageRetentionPeriod'|'ApproximateNumberOfMessages'|'ApproximateNumberOfMessagesNotVisible'|'CreatedTimestamp'|'LastModifiedTimestamp'|'QueueArn'|'ApproximateNumberOfMessagesDelayed'|'DelaySeconds'|'ReceiveMessageWaitTimeSeconds'|'RedrivePolicy', # ], MessageAttributeNames=["All"], MaxNumberOfMessages=1, VisibilityTimeout=60, WaitTimeSeconds=1) for m in msgs: # put the job completion msg. Wait when the queue is full. jc_q.put(Msg(m), block=True, timeout=None) except botocore.exceptions.EndpointConnectionError as e: # Could not connect to the endpoint URL: "https://queue.amazonaws.com/" # Retrying after 1 sec doesn't seem to help. Might be the server being # unreliable. Just kill the server. Cons.P("%s\n%s" % (e, traceback.format_exc())) os._exit(1) except Exception as e: Cons.P("%s\n%s" % (e, traceback.format_exc())) os._exit(1)
def _DescInsts(self): fmt = "%-15s %19s %10s %13s %15s %10s %6.4f" Cons.P( Util.BuildHeader( fmt, "Placement:AvailabilityZone" " InstanceId" " InstanceType" " LaunchTime" #" PrivateIpAddress" " PublicIpAddress" " State:Name" " CurSpotPrice")) with self.status_by_regions_lock: for region, status in sorted(self.status_by_regions.iteritems()): for s in reversed(status): if isinstance(s, DescInstResp): # Print only the last desc instance response per region r = s.r["Reservations"][0]["Instances"][0] az = _Value(_Value(r, "Placement"), "AvailabilityZone") Cons.P(fmt % ( az, _Value(r, "InstanceId"), _Value(r, "InstanceType"), _Value(r, "LaunchTime").strftime("%y%m%d-%H%M%S") #, _Value(r, "PrivateIpAddress") , _Value(r, "PublicIpAddress"), _Value(_Value(r, "State"), "Name"), SpotPrice.GetCurPrice(az))) break
def _KeepCheckingSpotReq(self): fulfilled = False while not fulfilled: try: r = BotoClient.Get( self.params["region"]).describe_spot_instance_requests( SpotInstanceRequestIds=self.spot_req_infos.SpotReqIds( )) #Cons.P(Util.Indent(pprint.pformat(r, indent=2, width=100), 2)) for e in r["SpotInstanceRequests"]: spot_req_id = e["SpotInstanceRequestId"] status = e["Status"]["Code"] if status != "fulfilled": Cons.P("spot_req_id=%s status=%s" % (spot_req_id, status)) continue inst_id = e["InstanceId"] Cons.P("spot_req_id=%s status=%s inst_id=%s" % (spot_req_id, status, inst_id)) self.spot_req_infos.SetSpotReqFulfilled( spot_req_id, inst_id) fulfilled = True if not fulfilled: time.sleep(1) except botocore.exceptions.ClientError as e: if e.response["Error"][ "Code"] == "InvalidSpotInstanceRequestID.NotFound": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(1) else: raise e self._KeepCheckingInstAndTag()
def RunEc2Inst(self): try: # This is run as root user_data = \ """#!/bin/bash cd /home/ubuntu/work rm -rf /home/ubuntu/work/acorn-tools sudo -i -u ubuntu bash -c 'git clone https://github.com/hobinyoon/acorn-tools.git /home/ubuntu/work/acorn-tools' sudo -i -u ubuntu /home/ubuntu/work/acorn-tools/ec2/ec2-init.py {0} {1} {2} {3} """ user_data = user_data.format(_init_script, _jr_sqs_url, _jr_sqs_msg_receipt_handle, _num_regions) placement = {} if self.az != None: placement['AvailabilityZone'] = self.az response = None while True: try: response = BotoClient.Get(self.region_name).run_instances( DryRun=False, ImageId=self.ami_id, MinCount=1, MaxCount=1, SecurityGroups=["cass-server"], EbsOptimized=True, InstanceType=_inst_type, Placement=placement # User data is passed as a string. I don't see an option of specifying a file. , UserData=user_data, InstanceInitiatedShutdownBehavior='terminate') break except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "RequestLimitExceeded": InstLaunchProgMon.Update(self.inst_id, e) # TODO Cons.P("%s. Retrying in 5 sec ..." % e) time.sleep(5) else: raise e #Cons.P("Response:") #Cons.P(Util.Indent(pprint.pformat(response, indent=2, width=100), 2)) if len(response["Instances"]) != 1: raise RuntimeError("len(response[\"Instances\"])=%d" % len(response["Instances"])) self.inst_id = response["Instances"][0]["InstanceId"] #Cons.P("region=%s inst_id=%s" % (self.region_name, self.inst_id)) InstLaunchProgMon.SetRegion(self.inst_id, self.region_name) self._KeepCheckingInst() except Exception as e: Cons.P("%s\nRegion=%s\n%s" % (e, self.region_name, traceback.format_exc())) os._exit(1)
def ParseArgs(): parser = argparse.ArgumentParser( description="Plot system resource usage", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--simulation_time_begin", type=str, default="161229-211308.518", help="Simulation time begin") parser.add_argument("--log_dir", type=str, default="~/work/mutant/misc/rocksdb/log", help="Mutant simulation log directory") parser.add_argument("--output_dir", type=str, default=("%s/.output" % os.path.dirname(__file__)), help="Output directory") global _args _args = parser.parse_args() Cons.P("Parameters:") for a in vars(_args): Cons.P("%s: %s" % (a, getattr(_args, a)), ind=2)
def GenDataFileForGnuplot(dt): SimTime.Init(dt) dn = "%s/%s" % (Conf.GetDir("output_dir"), dt) Util.MkDirs(dn) fn = "%s/dstat-data" % dn if os.path.isfile(fn): return fn with Cons.MT("Generating data file for plot ..."): global _header_idx global _body_rows _header_idx = None _body_rows = None _Parse(dt) fmt = "%9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f" \ " %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f" \ " %8.0f %8.0f %8.0f %8.0f" \ " %3.0f %3.0f" \ " %3.0f %3.0f %11s" \ " %3.1f %6.2f %3.1f %6.2f %6.2f %6.3f" header = Util.BuildHeader( fmt, " ".join(k for k, v in sorted(_header_idx.iteritems()))) with open(fn, "w") as fo: i = 0 for r in _body_rows: if i % 50 == 0: fo.write("%s\n" % header) i += 1 #Cons.P(fmt % tuple(r.Prepared())) fo.write((fmt + "\n") % tuple(r.Prepared())) Cons.P("Created %s %d" % (fn, os.path.getsize(fn))) return fn
def StopDstat(): with Cons.MT("Stopping dstat ...", print_time=False): cmd = "ps -e -o pid,ppid,user,args" lines = Util.RunSubp(cmd, print_cmd=False, print_output=False) #Cons.P(lines) pids = [] for line in lines.split("\n"): line = line.strip() if "dstat" not in line: continue if "csv" not in line: continue # Get the second-level processes, skipping the root-level ones. t = re.split(" +", line) if t[1] == "1": continue pids.append(t[0]) #Cons.P("[%s]" % line) if len(pids) > 0: #Cons.P("[%s]" % " ".join(pids)) Util.RunSubp("kill %s" % " ".join(pids)) # Make sure each of the processes has terminated for pid in pids: cmd = "kill -0 %s" % pid while True: r = 0 with open(os.devnull, "w") as devnull: r = subprocess.Popen(cmd, shell=True, stdin=devnull, stdout=devnull, stderr=devnull) if r != 0: Cons.P("Process %s has terminated" % pid) break time.sleep(0.1)
def Extract(fn): if _stop_requested: return fn_out = "%s/%s" % (_dn_out, os.path.splitext(os.path.basename(fn))[0]) Cons.P("Extracing %s %d to %s" % (fn, os.path.getsize(fn), fn_out)) pattern = re.compile("(?P<op>[sg]et) full-players:(?P<id>\d+)") # Check if there's anything other than get or set. doen't look like. #pattern = re.compile("^([sg]et) full-players:(?P<id>\d+)") with PcapReader(fn) as pr, open(fn_out, "w") as fo: try: i = 0 for p in pr: if _stop_requested: return i += 1 if i % 10000 == 0: sys.stdout.write(".") sys.stdout.flush() fo.flush() bp = bytes(p.payload) for m in pattern.finditer(bp): fo.write("%s %s %s\n" % (p.time, m.group("op"), m.group("id"))) except Exception as e: sys.stderr.write("Error: %s\n" % format(e)) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Check(fn_pair): try: start_time = time.time() # 160718-144303 ts_lower = fn_pair[0] if fn_pair[1] is None: ts_upper = None else: ts_upper = fn_pair[1] fn1 = "%s/%s" % (_dn_in, fn_pair[0]) ts_prev = None # Object IDs with the same timestamp obj_ids_w = sets.Set() obj_ids_r = sets.Set() with open(fn1) as fo: for line in fo: line = line.strip() t = line.split(" ") if len(t) != 3: raise RuntimeError("Unexpected [%s]" % line) ts = t[0] op = t[1].replace("get", "G").replace("set", "S") obj_id = t[2] if (ts < ts_lower) or ((ts_upper is not None) and (ts_upper < ts)): #Cons.P("ts_lower ts_upper ts %s %s %s" \ # % (ts_lower, ts_upper, ts)) raise RuntimeError("Unexpected: ts_lower ts_upper ts %s %s %s" \ % (ts_lower, ts_upper, ts)) if (ts_prev is not None) and (ts < ts_prev): raise RuntimeError("Unexpected: ts_prev ts %s %s" % (ts_prev, ts)) if ts != ts_prev: obj_ids_w.clear() obj_ids_r.clear() if op == "S": obj_ids_w.add(obj_id) elif op == "G": obj_ids_r.add(obj_id) ts_prev = ts else: if op == "S": if obj_id in obj_ids_w: raise RuntimeError("Unexpected: [%s]" % line) obj_ids_w.add(obj_id) elif op == "G": if obj_id in obj_ids_r: raise RuntimeError("Unexpected: [%s]" % line) obj_ids_r.add(obj_id) Cons.P("Checked %s %d in %.0f ms" % \ (fn1, os.path.getsize(fn1), (time.time() - start_time) * 1000.0)) except Exception as e: Cons.P("Error while checking %s\n%s\n%s\n" % (fn1, e, traceback.format_exc()))
def SstLevelInfo(): # Set Conf.ExpStartTime(), if not already set. if Conf.ExpStartTime() is None: MutantLogReader.Get() fn = "%s/sst-info-by-time-by-levels-level-seps-%s" % (Conf.dn_result, Conf.ExpStartTime()) if os.path.isfile(fn): return fn sst_y_cord_level_sep_highs = SstYCord.LevelSepHigh() with Cons.MT( "Generating Sst info by time by levels: level separators data file ..." ): with open(fn, "w") as fo: fmt = "%1d %10d %10s" fo.write("%s\n" % Util.BuildHeader( fmt, "level level_mid_for_labels level_low_for_separators")) lh_prev = 0 for l, lh in sorted(sst_y_cord_level_sep_highs.iteritems()): lm = (lh + lh_prev) / 2 fo.write((fmt + "\n") % (l, lm, lh_prev)) lh_prev = lh Cons.P("Created %s %d" % (fn, os.path.getsize(fn))) return fn
def SstInfo(): # Set Conf.ExpStartTime(), if not already set. if Conf.ExpStartTime() is None: MutantLogReader.Get() fn = "%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result, Conf.ExpStartTime()) if os.path.isfile(fn): return fn (sst_lives, memt_lives) = MemtSstLife.Get() with Cons.MT("Generating Sst info by time by levels data file ..."): #with open(fn_m, "w") as fo: # fo.write("%s\n" % Memt.Header()) # for addr, l in sorted(_memt_lives.iteritems()): # fo.write("%s\n" % l) #Cons.P("Created %s %d" % (fn_m, os.path.getsize(fn_m))) with open(fn, "w") as fo: fo.write("%s\n" % MemtSstLife.SstLife.Header()) for sst_gen, l in sorted(sst_lives.iteritems()): fo.write("%s\n" % l) Cons.P("Created %s %d" % (fn, os.path.getsize(fn))) return fn
def RunSubp(cmd, env=os.environ.copy(), shell=True, print_cmd=True, print_output=True): if print_cmd: Cons.P(cmd) lines = "" p = None if shell: p = subprocess.Popen(cmd, shell=shell, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: p = subprocess.Popen(cmd.split(" "), shell=shell, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # http://stackoverflow.com/questions/18421757/live-output-from-subprocess-command # It can read char by char depending on the requirements. for line in iter(p.stdout.readline, ''): if print_output: Cons.P(Indent(line.rstrip(), 2)) lines += line p.wait() if p.returncode != 0: raise RuntimeError("Error: cmd=[%s] rc=%d" % (cmd, p.returncode)) return lines
def _KeepCheckingInstAndTag(self): while True: r = None while True: try: r = BotoClient.Get(self.params["region"]).describe_instances(InstanceIds=self.inst_info.GetAllInstIds()) break except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(1) else: raise e num_running = 0 for e in r["Reservations"]: for e1 in e["Instances"]: inst_id = e1["InstanceId"] state = e1["State"]["Name"] Cons.P("inst_id=%s state=%s" % (inst_id, state)) if state in ["shutting-down", "terminated"]: raise RuntimeError("Unexpected: %s" % pprint.pformat(e1)) elif state == "running": num_running += 1 # Some doesn't have it. Might be not assigned yet. #pub_ip = e1["PublicIpAddress"] elif state == "pending": self._TagInst(inst_id) if num_running == 1: Cons.P("job_id: %s. %d instances are created." % (self.job_id, num_running)) return time.sleep(1)
def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.GetDir("output_dir")) dist_sq_thresholds = [0, 0.008, 0.01, 0.02, 0.05] #dist_sq_thresholds = [0.02] dist_sq_thresholds_str = [] for d in dist_sq_thresholds: dist_sq_thresholds_str.append(_NoTrailing0s(d)) reduced_files = [] reduced_file_sizes = [] with Cons.MT("Generating reduced size usa map ..."): for d in dist_sq_thresholds_str: fn_co_loc = "filter-out-almost-duplicate-points/.output/usa-map-smallsize-%s" % d reduced_files.append(fn_co_loc) if not os.path.exists(fn_co_loc): cmd = "cd filter-out-almost-duplicate-points && ./build-and-run.sh --dist_sq_threshold=%s" % d Util.RunSubp(cmd) reduced_file_sizes.append(os.path.getsize(fn_co_loc)) dn_out = "%s/.output" % os.path.dirname(__file__) fn_out = "%s/usa-map.pdf" % dn_out with Cons.MT("Plotting ..."): env = os.environ.copy() env["IN_FNS"] = " ".join(reduced_files) env["IN_FN_SIZES"] = " ".join(str(s) for s in reduced_file_sizes) env["DIST_SQ_THRESHOLDS"] = " ".join(dist_sq_thresholds_str) env["OUT_FN"] = fn_out Util.RunSubp("gnuplot %s/usa-map.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def _TagInst(self, inst_id): if self.inst_info.IsInstTagged(inst_id): return # Go with minimal tags for now. The numbers are limited, like 10. tags = {"job_id": self.job_id} tags_boto = [] for k, v in tags.iteritems(): tags_boto.append({"Key": k, "Value": v}) while True: try: BotoClient.Get(self.params["region"]).create_tags(Resources=[inst_id], Tags=tags_boto) self.inst_info.SetInstTagged(inst_id) return except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(1) elif e.response["Error"]["Code"] == "RequestLimitExceeded": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(5) else: raise e
def ByJobIdTermSelfLast(): job_id = Ec2Util.JobId() Cons.P("Terminating running instances of job_id %s" % job_id) _TermInst.Init(term_by_job_id_self_last=True) tags = {} tags["job_id"] = job_id tis = [] for r in Ec2Region.All(): tis.append(_TermInst(r, tags)) threads = [] for ti in tis: t = threading.Thread(target=ti.Run) t.daemon = True threads.append(t) t.start() for t in threads: t.join() print "" Cons.P(_TermInst.Header()) for ti in tis: ti.PrintResult()
def PlotSstAccDistAtSpecificTimes(): # At the time m sec after the n-th SSTable is created (time t). To get the # max_plot_hgieht, all plot data files need to be generated before plotting # the first one. plot_data_fns_at_n = {} with Cons.MT( "Generating plot data for SSTables by levels with heat at specific times ..." ): for (n, m) in Conf.times_sst_by_levels_with_heat: (fn_in_boxes, fn_in_level_seps) = SstByLevelsWithHeatAtSpecificTimes.Boxes( n, m) plot_data_fns_at_n[n] = (fn_in_boxes, fn_in_level_seps) with Cons.MT( "Plotting SSTables by levels with heat at specific times ..."): dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result for n, (fn_in_boxes, fn_in_level_seps) in sorted(plot_data_fns_at_n.iteritems()): env = os.environ.copy() env["FN_IN_BOXES"] = fn_in_boxes env["FN_IN_LEVEL_INFO"] = fn_in_level_seps env["MAX_PLOT_HEIGHT"] = str( SstByLevelsWithHeatAtSpecificTimes.max_plot_height) fn_out = "%s/sst-by-level-by-ks-range-with-heat-%s-%s.pdf" % ( dn, Conf.ExpStartTime(), n) env["FN_OUT"] = fn_out Util.RunSubp( "gnuplot %s/sst-by-level-by-ks-range-with-heat-at-specific-time.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Get1minAvgFn(): exp_dts = [] for i in range(2): #Cons.P(Conf.Get(i)) mo = re.match(r".+/(?P<exp_dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)-d", Conf.Get(i)) exp_dts.append(mo.group("exp_dt")) fn_out = "%s/mem-1minavg-%s" % (Conf.GetOutDir(), "-".join(exp_dts)) if os.path.exists(fn_out): return fn_out with Cons.MT("Creating avg memory usage comparison file for plotting ..."): records = [] dn_base = Conf.GetDir("dn_base") for i in range(2): fn_ycsb_log = "%s/%s" % (dn_base, Conf.Get(i)) hm_mem = _GetHmMem(fn_ycsb_log) for hm, mem in hm_mem.iteritems(): records.append(_RecordMemAvg(hm, i * 30, mem, i)) records.sort(key=operator.attrgetter("ts")) fmt = "%8s %6.3f %1d" header = Util.BuildHeader(fmt, "timestamp mem_avg_in_gb exp_type") with open(fn_out, "w") as fo: i = 0 for r in records: if i % 40 == 0: fo.write(header + "\n") i += 1 fo.write("%s\n" % r.ToStr(fmt)) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out))) return fn_out
def _ReadStoredLog(): if Conf.ExpStartTime() is None: return None dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~") fn = "%s/system-%s" % (dn, Conf.ExpStartTime()) if not os.path.isfile(fn): # If there is a 7z file, uncompress it fn_7z = "%s.7z" % fn if os.path.isfile(fn_7z): with Cons.MT("Found a 7z file. Uncompressing"): Util.RunSubp("7z e -o%s %s" % (dn, fn_7z)) else: return None with Cons.MT("Reading the stored Cassandra Mutant log file %s" % fn, print_time=False): lines = [] with open(fn) as fo: for line in fo.readlines(): lines.append(line.strip()) # Stop after reading n lines for testing if 0 < Conf.MaxCassLogLines(): if Conf.MaxCassLogLines() < len(lines): break #Cons.P(len(lines)) return lines
def _TagInst(self, inst_id): if self.spot_req_infos.IsInstTagged(inst_id): return # Go with minimal tags for now. tags = { "job_id": self.job_id # , "name": self.spot_req_infos.NodeName(inst_id) # Experiment name to prevent your collaborator accidentally killing your VM , "Name": self.params["ec2_tag_Name"] # Note: node expiration time can be added here for auto cleaning. dev # nodes don't have them. } tags_boto = [] for k, v in tags.iteritems(): tags_boto.append({"Key": k, "Value": v}) while True: try: BotoClient.Get(self.params["region"]).create_tags( Resources=[inst_id], Tags=tags_boto) self.spot_req_infos.SetInstTagged(inst_id) return except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(1) elif e.response["Error"]["Code"] == "RequestLimitExceeded": Cons.P("region=%s error=%s" % (self.params["region"], e)) time.sleep(5) else: raise e
def main(): # Takes in the file and parses into datum's. D = cData(sys.argv[1]) # Starting points from the pickle or not. EmAlg = parseCommandLine(D, sys.argv) prevCons = 0 totalcons = 0 nmiResult = evaluateEM_NMI(D, EmAlg) print "Initial nmi: ",nmiResult consobj = Cons.cCons(D) for numCons in range(1,len(D.data)/4,1): consobj.constype = Cons.cCons.eConsType.TripCenterChunk consobj.centerChunkSize = 0.2 cons = consobj.tripCons(EmAlg.mGammas,numCons-prevCons) prevCons = numCons totalcons += len(cons) for i in cons: EmAlg.mCij[i[0]][i[1]] = i[2] EmAlg.mCij[i[1]][i[0]] = i[2] EmAlg.EM(len(D.classlist)) nmiresult = evaluateEM_NMI(D, EmAlg) print numCons, ",", nmiresult, ",", totalcons if(nmiresult > 0.999 or len(D.data)==numCons): break
def Run(self): bc = boto3.session.Session().client("ec2", region_name = self.region) response = bc.describe_account_attributes( AttributeNames=[ "max-instances", ] ) #Cons.P(pprint.pformat(response, indent=2)) for r in response["AccountAttributes"]: if r["AttributeName"] != "max-instances": continue if len(r["AttributeValues"]) != 1: raise RuntimeError("len(r[\"AttributeValues\"])=%d" % len(r["AttributeValues"])) self.max_inst = int(r["AttributeValues"][0]["AttributeValue"]) #Cons.P(self.max_inst) Cons.sys_stdout_write(" %s" % self.region)
def Run(self): try: # http://boto3.readthedocs.io/en/latest/guide/session.html session = boto3.session.Session() bc = BotoClient.Get(self.region) if self.tags is None: self.response = bc.describe_instances() else: filters = [] for k, v in self.tags.iteritems(): d = {} d["Name"] = ("tag:%s" % k) d["Values"] = [v] filters.append(d) self.response = bc.describe_instances(Filters = filters) except Exception as e: Cons.P("%s\n%s\nregion=%s" % (e, traceback.format_exc(), self.region)) os._exit(1) Cons.sys_stdout_write(" %s" % self.region)
def cons_to_list(cons): """Converts a Lisp cons object to a Python list :rtype : Atom """ result = [] while cons: if cons.first and cons.first.type == Atom.CONS: item = Cons.orphan(cons.first) if cons.quote: item.quote = cons.quote else: item = cons.first result.append(item) cons = cons.second.data return result
def TripConsTest(D, sNum, EMStarts, fp): fp.write("trips,queries,cons,likelihood,NMI\n") for option in TripConsOptions.lOptions: optname = TripConsOptions.lOptionNames[option] fp.write(optname + "\n") for iOutertrial in range(numOuterTrials): print "scenario ", sNum, " options ", optname, " outertrial ", iOutertrial fp.write("outertrial: %d\n" % iOutertrial) em = copy.deepcopy(EMStarts[iOutertrial]) em.bPPC = True prevTrips = 0 totalCons = 0 nmiResult = utils.evaluateEM_NMI(D, em) fp.write("Initial nmi: %f\n" % nmiResult) consobj = Cons.cCons(D) for numTrips in range(1,len(D.data)/4,1): if option == TripConsOptions.CenterChunkCons: consobj.constype = Cons.cCons.eConsType.TripCenterChunk elif option == TripConsOptions.MidCons: consobj.constype = Cons.cCons.eConsType.TripMids print em.mLikelihood_il cons = consobj.tripCons(em.mGammas,numTrips-prevTrips) prevTrips = numTrips totalCons += len(cons) for i in cons: em.mCij[i[0]][i[1]] = i[2] em.mCij[i[1]][i[0]] = i[2] em.EM(len(D.classlist)) nmiresult = utils.evaluateEM_NMI(D, em) fp.write("%d,%d,%d,%f,%f\n" % (numTrips, numTrips*14, totalCons, em.dEMLikelihood, nmiresult) ) fp.flush() if (nmiresult > 0.999 or len(D.data)==numTrips): break