def _EvictCache():
with Cons.MT("Drop caches ..."):
Util.RunSubp("sudo sh -c \"echo 3 >/proc/sys/vm/drop_caches\"")
if False:
# Evict the DB data files from cache
with Cons.MT("Evicting DB data ..."):
Util.RunSubp("%s/work/vmtouch/vmtouch -e %s" %
(os.path.expanduser("~"), Conf.GetDir("db_path")))
# Evict the input data files from cache so that the read IO is consistent
# throughput the experiment.
# Caching them would have been nice, but you can do it only when you limit
# the memory smaller than 3.4 GB (= 15 (total ram) - 0.4 (for the OS) - 11.2
# (the 100% quizup data size)
evict = True
if evict:
with Cons.MT("Evicting workload data ..."):
Util.RunSubp(
"%s/work/vmtouch/vmtouch -e %s" %
(os.path.expanduser("~"), Conf.GetDir("workload_dir")))
else:
with Cons.MT("Caching workload data ..."):
Util.RunSubp(
"%s/work/vmtouch/vmtouch -t %s" %
(os.path.expanduser("~"), Conf.GetDir("workload_dir")))
def Restart():
with Cons.MT("Restarting dstat ...", print_time=False):
Dstat._Stop()
dn = "%s/dstat" % Conf.GetDir("log_archive_dn")
Util.MkDirs(dn)
# Get a list of all block devices
devs = []
for f in os.listdir("/dev"):
mo = None
if socket.gethostname() == "node3":
mo = re.match(r"sd\w$", f)
else:
mo = re.match(r"xvd\w$", f)
if mo is not None:
devs.append(f)
global _cur_datetime
_cur_datetime = datetime.datetime.now().strftime(
"%y%m%d-%H%M%S.%f")[:-3]
Cons.P(_cur_datetime)
# Run dstat as a daemon
fn_out = "%s/%s.csv" % (dn, _cur_datetime)
cmd = "dstat -tcdnrmy -D %s --output %s" % (",".join(devs), fn_out)
Util.RunDaemon(cmd)
def ParseArgs():
parser = argparse.ArgumentParser(
description="Plot SSTables access frequencies and temperatures",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--simulation_time_begin",
type=str,
default="170101-005737.017",
help="Simulation time begin")
parser.add_argument("--log_dir",
type=str,
default="~/work/mutant/misc/rocksdb/log",
help="Mutant simulation log directory")
parser.add_argument("--dn_result",
type=str,
default=("%s/.result" % os.path.dirname(__file__)),
help="Result directory")
global _args
_args = parser.parse_args()
Cons.P("Parameters:")
for a in vars(_args):
Cons.P("%s: %s" % (a, getattr(_args, a)), ind=2)
def _DeleteOldAmisSnapshots(self):
try:
for img in self.imgs_acorn_to_delete:
try:
r = self._Bc().deregister_image(ImageId=img.image_id)
Cons.P("%-20s deregistered AMI %s" %
(self.region, img.image_id))
except botocore.exceptions.ClientError as e:
if e.response["Error"][
"Code"] == "InvalidAMIID.Unavailable":
pass
else:
raise e
if len(self.ss_to_delete) > 0:
time.sleep(1)
for ss in self.ss_to_delete:
r = self._Bc().delete_snapshot(SnapshotId=ss.snapshot_id)
Cons.P("%-20s deleted snapshot %s" %
(self.region, ss.snapshot_id))
except Exception as e:
Cons.P("%s\n%s\nregion=%s" %
(e, traceback.format_exc(), self.region))
os._exit(1)
def PlotWorkload(workload_type):
Util.MkDirs(_dn_output)
#(fn_plot_data_m_ls_st1, fn_plot_data_ind) = GetPlotDataMutant(workload_type, "ls-st1", "~/work/mutant/log/ycsb/workload-%s/mutant-ls-st1" % workload_type)
(fn_plot_data_m_ls_st1, fn_plot_data_ind) = GetPlotDataMutant(
workload_type, "ls-st1",
"~/work/mutant/log/ycsb/workload-%s/170822-022606-d-ls-st1-short-exps"
% workload_type)
sys.exit(1)
(fn_plot_data_r_st1, fn_plot_data_ind) = GetPlotDataRocksdb(
workload_type, "st1",
"~/work/mutant/log/ycsb/workload-%s/rocksdb-st1" % workload_type)
(fn_plot_data_r_ls, fn_plot_data_ind) = GetPlotDataRocksdb(
workload_type, "ls",
"~/work/mutant/log/ycsb/workload-%s/rocksdb-ls" % workload_type)
fn_out = "%s/ycsb-%s-thp-vs-latency.pdf" % (_dn_output, workload_type)
with Cons.MT("Plotting ..."):
env = os.environ.copy()
env["FN_ROCKSDB_ST1"] = fn_plot_data_r_st1
env["FN_ROCKSDB_LS"] = fn_plot_data_r_ls
env["FN_MUTANT_LS_ST1"] = fn_plot_data_m_ls_st1
env["FN_OUT"] = fn_out
Util.RunSubp("gnuplot %s/thrp-vs-lat.gnuplot" %
os.path.dirname(__file__),
env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Stop():
with Cons.MT("Stopping dstat ...", print_time=False):
Dstat._Stop()
# Change the current dstat log file name to the simulation_time_begin of
# the simulator.
if _cur_datetime is not None:
with Cons.MT("Renaming the log file and zipping ..."):
dn_client = "%s/quizup" % Conf.GetDir("log_archive_dn")
global _latest_client_log_dt
_latest_client_log_dt = None
for f in os.listdir(dn_client):
mo = re.match(
r"(?P<dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)$", f)
if mo is not None:
if _latest_client_log_dt is None:
_latest_client_log_dt = mo.group("dt")
else:
_latest_client_log_dt = max(
_latest_client_log_dt, mo.group("dt"))
# There should be a client log file whose dt is bigger than
# _cur_datetime
if _latest_client_log_dt <= _cur_datetime:
raise RuntimeError("Unexpected")
fn0 = "%s/dstat/%s.csv" % (Conf.GetDir("log_archive_dn"),
_cur_datetime)
fn1 = "%s/dstat/%s.csv" % (Conf.GetDir("log_archive_dn"),
_latest_client_log_dt)
Cons.P("renaming %s to %s" % (fn0, fn1))
os.rename(fn0, fn1)
Util.RunSubp("7z a -mx %s.7z %s" % (fn1, fn1))
def Deleted(self, line):
mo = re.match(r"(?P<ts>(\d|\/|-|:|\.)+) .+EVENT_LOG_v1 (?P<json>.+)",
line)
if mo is None:
raise RuntimeError("Unexpected: [%s]" % line)
j = mo.group("json")
try:
j1 = json.loads(j)
except ValueError as e:
Cons.P("%s [%s]" % (e, line))
sys.exit(1)
ts0 = mo.group("ts")
sst_id = int(j1["file_number"])
ts1 = self._GetRelTs(ts0)
si = self.rocks_log_reader.sst_info.Get(sst_id)
if si is None:
Cons.P(
"Interesting: Sst (id %d) deleted without a creation. Ignore."
% sst_id)
return
path_id = si.PathId()
self.cur_sstsize[path_id] -= si.Size()
self.cur_numssts[path_id] -= 1
self.ts_sstsize[ts1] = list(self.cur_sstsize)
self.ts_numssts[ts1] = list(self.cur_numssts)
def __init__(self):
self.conn = None
# Open or create DB
fn_db = Conf.GetFn("whole_foods_loc_db")
conn = None
if os.path.exists(fn_db):
with Cons.MT("Opening the existing db ..."):
conn = sqlite3.connect(fn_db)
if conn is None:
raise RuntimeError(
"Error! cannot create the database connection.")
conn.row_factory = sqlite3.Row
cur = conn.cursor()
q = "SELECT count(*) as cnt FROM whole_foods_loc"
cur.execute(q)
r = cur.fetchone()
Cons.P("There are %d records" % r["cnt"])
else:
with Cons.MT("Creating a new db ..."):
conn = sqlite3.connect(fn_db)
if conn is None:
raise RuntimeError(
"Error! cannot create the database connection.")
conn.row_factory = sqlite3.Row
cur = conn.cursor()
q = """CREATE TABLE IF NOT EXISTS whole_foods_loc (
addr text NOT NULL
, lat real NOT NULL
, lon real NOT NULL
, PRIMARY KEY (addr)
); """
cur.execute(q)
self.conn = conn
def GetRunningAvg():
fn = "%s/num-reads-by-obj-age" % _dn_out
if os.path.isfile(fn):
return fn
with Cons.MT("Generating running average ..."):
# Running average of the last n items, not in the last n-minute time
# window. Should generate smoother curve for plotting.
q = Q(60)
with open(_fn_in) as fo, open(fn, "w") as fo_out:
for line in fo:
t = line.strip().split(" ")
if len(t) != 2:
raise RuntimeError("Unexpected [%s]" % line)
ts = int(t[0])
num_reads = int(t[1])
q.Enq(num_reads)
if False:
if ts < 2 * 24 * 60:
fo_out.write("%d %d -\n" % (ts, num_reads))
else:
fo_out.write("%d %d %.1f\n" % (ts, num_reads, q.Avg()))
else:
fo_out.write("%d %d %.1f\n" % (ts, num_reads, q.Avg()))
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def PlotCompareTwo():
(fns_rocksdb, fn_sst_creation_stat) = RocksdbLog.GenDataFilesForGnuplot()
#fn_cpu_stat_by_time = CompareCpu.GetHourlyFn()
fn_cpu_1min_avg = CompareCpu.Get1minAvgFn()
fn_mem_stat_by_time = CompareMem.GetHourlyFn()
fn_mem_1min_avg = CompareMem.Get1minAvgFn()
#time_max = "09:00:00"
#time_max = "08:00:00"
time_max = "07:50:00"
exp_dts = []
for i in range(2):
mo = re.match(r".+/(?P<exp_dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)-d",
Conf.Get(i))
exp_dts.append(mo.group("exp_dt"))
fn_out = "%s/mutant-overhead-%s.pdf" % (Conf.GetOutDir(),
"-".join(exp_dts))
with Cons.MT("Plotting ..."):
env = os.environ.copy()
env["TIME_MAX"] = str(time_max)
#env["CPU_STAT"] = fn_cpu_stat_by_time
env["FN_CPU_1MIN_AVG"] = fn_cpu_1min_avg
#env["MEM_STAT"] = fn_mem_stat_by_time
env["FN_MEM_1MIN_AVG"] = fn_mem_1min_avg
env["ROCKSDB0"] = fns_rocksdb[0]
env["ROCKSDB1"] = fns_rocksdb[1]
env["OUT_FN"] = fn_out
Util.RunSubp("gnuplot %s/compare-two-exps.gnuplot" %
os.path.dirname(__file__),
env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def _Poll(jc_q):
q = _GetQ()
while True:
try:
msgs = q.receive_messages(
#AttributeNames=[
# 'Policy'|'VisibilityTimeout'|'MaximumMessageSize'|'MessageRetentionPeriod'|'ApproximateNumberOfMessages'|'ApproximateNumberOfMessagesNotVisible'|'CreatedTimestamp'|'LastModifiedTimestamp'|'QueueArn'|'ApproximateNumberOfMessagesDelayed'|'DelaySeconds'|'ReceiveMessageWaitTimeSeconds'|'RedrivePolicy',
# ],
MessageAttributeNames=["All"],
MaxNumberOfMessages=1,
VisibilityTimeout=60,
WaitTimeSeconds=1)
for m in msgs:
# put the job completion msg. Wait when the queue is full.
jc_q.put(Msg(m), block=True, timeout=None)
except botocore.exceptions.EndpointConnectionError as e:
# Could not connect to the endpoint URL: "https://queue.amazonaws.com/"
# Retrying after 1 sec doesn't seem to help. Might be the server being
# unreliable. Just kill the server.
Cons.P("%s\n%s" % (e, traceback.format_exc()))
os._exit(1)
except Exception as e:
Cons.P("%s\n%s" % (e, traceback.format_exc()))
os._exit(1)
def _DescInsts(self):
fmt = "%-15s %19s %10s %13s %15s %10s %6.4f"
Cons.P(
Util.BuildHeader(
fmt,
"Placement:AvailabilityZone"
" InstanceId"
" InstanceType"
" LaunchTime"
#" PrivateIpAddress"
" PublicIpAddress"
" State:Name"
" CurSpotPrice"))
with self.status_by_regions_lock:
for region, status in sorted(self.status_by_regions.iteritems()):
for s in reversed(status):
if isinstance(s, DescInstResp):
# Print only the last desc instance response per region
r = s.r["Reservations"][0]["Instances"][0]
az = _Value(_Value(r, "Placement"), "AvailabilityZone")
Cons.P(fmt % (
az,
_Value(r, "InstanceId"),
_Value(r, "InstanceType"),
_Value(r, "LaunchTime").strftime("%y%m%d-%H%M%S")
#, _Value(r, "PrivateIpAddress")
,
_Value(r, "PublicIpAddress"),
_Value(_Value(r, "State"), "Name"),
SpotPrice.GetCurPrice(az)))
break
def _KeepCheckingSpotReq(self):
fulfilled = False
while not fulfilled:
try:
r = BotoClient.Get(
self.params["region"]).describe_spot_instance_requests(
SpotInstanceRequestIds=self.spot_req_infos.SpotReqIds(
))
#Cons.P(Util.Indent(pprint.pformat(r, indent=2, width=100), 2))
for e in r["SpotInstanceRequests"]:
spot_req_id = e["SpotInstanceRequestId"]
status = e["Status"]["Code"]
if status != "fulfilled":
Cons.P("spot_req_id=%s status=%s" %
(spot_req_id, status))
continue
inst_id = e["InstanceId"]
Cons.P("spot_req_id=%s status=%s inst_id=%s" %
(spot_req_id, status, inst_id))
self.spot_req_infos.SetSpotReqFulfilled(
spot_req_id, inst_id)
fulfilled = True
if not fulfilled:
time.sleep(1)
except botocore.exceptions.ClientError as e:
if e.response["Error"][
"Code"] == "InvalidSpotInstanceRequestID.NotFound":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(1)
else:
raise e
self._KeepCheckingInstAndTag()
def RunEc2Inst(self):
try:
# This is run as root
user_data = \
"""#!/bin/bash
cd /home/ubuntu/work
rm -rf /home/ubuntu/work/acorn-tools
sudo -i -u ubuntu bash -c 'git clone https://github.com/hobinyoon/acorn-tools.git /home/ubuntu/work/acorn-tools'
sudo -i -u ubuntu /home/ubuntu/work/acorn-tools/ec2/ec2-init.py {0} {1} {2} {3}
"""
user_data = user_data.format(_init_script, _jr_sqs_url,
_jr_sqs_msg_receipt_handle,
_num_regions)
placement = {}
if self.az != None:
placement['AvailabilityZone'] = self.az
response = None
while True:
try:
response = BotoClient.Get(self.region_name).run_instances(
DryRun=False,
ImageId=self.ami_id,
MinCount=1,
MaxCount=1,
SecurityGroups=["cass-server"],
EbsOptimized=True,
InstanceType=_inst_type,
Placement=placement
# User data is passed as a string. I don't see an option of specifying a file.
,
UserData=user_data,
InstanceInitiatedShutdownBehavior='terminate')
break
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "RequestLimitExceeded":
InstLaunchProgMon.Update(self.inst_id, e)
# TODO
Cons.P("%s. Retrying in 5 sec ..." % e)
time.sleep(5)
else:
raise e
#Cons.P("Response:")
#Cons.P(Util.Indent(pprint.pformat(response, indent=2, width=100), 2))
if len(response["Instances"]) != 1:
raise RuntimeError("len(response[\"Instances\"])=%d" %
len(response["Instances"]))
self.inst_id = response["Instances"][0]["InstanceId"]
#Cons.P("region=%s inst_id=%s" % (self.region_name, self.inst_id))
InstLaunchProgMon.SetRegion(self.inst_id, self.region_name)
self._KeepCheckingInst()
except Exception as e:
Cons.P("%s\nRegion=%s\n%s" %
(e, self.region_name, traceback.format_exc()))
os._exit(1)
def ParseArgs():
parser = argparse.ArgumentParser(
description="Plot system resource usage",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--simulation_time_begin",
type=str,
default="161229-211308.518",
help="Simulation time begin")
parser.add_argument("--log_dir",
type=str,
default="~/work/mutant/misc/rocksdb/log",
help="Mutant simulation log directory")
parser.add_argument("--output_dir",
type=str,
default=("%s/.output" % os.path.dirname(__file__)),
help="Output directory")
global _args
_args = parser.parse_args()
Cons.P("Parameters:")
for a in vars(_args):
Cons.P("%s: %s" % (a, getattr(_args, a)), ind=2)
def GenDataFileForGnuplot(dt):
SimTime.Init(dt)
dn = "%s/%s" % (Conf.GetDir("output_dir"), dt)
Util.MkDirs(dn)
fn = "%s/dstat-data" % dn
if os.path.isfile(fn):
return fn
with Cons.MT("Generating data file for plot ..."):
global _header_idx
global _body_rows
_header_idx = None
_body_rows = None
_Parse(dt)
fmt = "%9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f" \
" %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f" \
" %8.0f %8.0f %8.0f %8.0f" \
" %3.0f %3.0f" \
" %3.0f %3.0f %11s" \
" %3.1f %6.2f %3.1f %6.2f %6.2f %6.3f"
header = Util.BuildHeader(
fmt, " ".join(k for k, v in sorted(_header_idx.iteritems())))
with open(fn, "w") as fo:
i = 0
for r in _body_rows:
if i % 50 == 0:
fo.write("%s\n" % header)
i += 1
#Cons.P(fmt % tuple(r.Prepared()))
fo.write((fmt + "\n") % tuple(r.Prepared()))
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def StopDstat():
with Cons.MT("Stopping dstat ...", print_time=False):
cmd = "ps -e -o pid,ppid,user,args"
lines = Util.RunSubp(cmd, print_cmd=False, print_output=False)
#Cons.P(lines)
pids = []
for line in lines.split("\n"):
line = line.strip()
if "dstat" not in line:
continue
if "csv" not in line:
continue
# Get the second-level processes, skipping the root-level ones.
t = re.split(" +", line)
if t[1] == "1":
continue
pids.append(t[0])
#Cons.P("[%s]" % line)
if len(pids) > 0:
#Cons.P("[%s]" % " ".join(pids))
Util.RunSubp("kill %s" % " ".join(pids))
# Make sure each of the processes has terminated
for pid in pids:
cmd = "kill -0 %s" % pid
while True:
r = 0
with open(os.devnull, "w") as devnull:
r = subprocess.Popen(cmd, shell=True, stdin=devnull, stdout=devnull, stderr=devnull)
if r != 0:
Cons.P("Process %s has terminated" % pid)
break
time.sleep(0.1)
def Extract(fn):
if _stop_requested:
return
fn_out = "%s/%s" % (_dn_out, os.path.splitext(os.path.basename(fn))[0])
Cons.P("Extracing %s %d to %s" % (fn, os.path.getsize(fn), fn_out))
pattern = re.compile("(?P<op>[sg]et) full-players:(?P<id>\d+)")
# Check if there's anything other than get or set. doen't look like.
#pattern = re.compile("^([sg]et) full-players:(?P<id>\d+)")
with PcapReader(fn) as pr, open(fn_out, "w") as fo:
try:
i = 0
for p in pr:
if _stop_requested:
return
i += 1
if i % 10000 == 0:
sys.stdout.write(".")
sys.stdout.flush()
fo.flush()
bp = bytes(p.payload)
for m in pattern.finditer(bp):
fo.write("%s %s %s\n" %
(p.time, m.group("op"), m.group("id")))
except Exception as e:
sys.stderr.write("Error: %s\n" % format(e))
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Check(fn_pair):
try:
start_time = time.time()
# 160718-144303
ts_lower = fn_pair[0]
if fn_pair[1] is None:
ts_upper = None
else:
ts_upper = fn_pair[1]
fn1 = "%s/%s" % (_dn_in, fn_pair[0])
ts_prev = None
# Object IDs with the same timestamp
obj_ids_w = sets.Set()
obj_ids_r = sets.Set()
with open(fn1) as fo:
for line in fo:
line = line.strip()
t = line.split(" ")
if len(t) != 3:
raise RuntimeError("Unexpected [%s]" % line)
ts = t[0]
op = t[1].replace("get", "G").replace("set", "S")
obj_id = t[2]
if (ts < ts_lower) or ((ts_upper is not None) and (ts_upper < ts)):
#Cons.P("ts_lower ts_upper ts %s %s %s" \
# % (ts_lower, ts_upper, ts))
raise RuntimeError("Unexpected: ts_lower ts_upper ts %s %s %s" \
% (ts_lower, ts_upper, ts))
if (ts_prev is not None) and (ts < ts_prev):
raise RuntimeError("Unexpected: ts_prev ts %s %s" % (ts_prev, ts))
if ts != ts_prev:
obj_ids_w.clear()
obj_ids_r.clear()
if op == "S":
obj_ids_w.add(obj_id)
elif op == "G":
obj_ids_r.add(obj_id)
ts_prev = ts
else:
if op == "S":
if obj_id in obj_ids_w:
raise RuntimeError("Unexpected: [%s]" % line)
obj_ids_w.add(obj_id)
elif op == "G":
if obj_id in obj_ids_r:
raise RuntimeError("Unexpected: [%s]" % line)
obj_ids_r.add(obj_id)
Cons.P("Checked %s %d in %.0f ms" % \
(fn1, os.path.getsize(fn1), (time.time() - start_time) * 1000.0))
except Exception as e:
Cons.P("Error while checking %s\n%s\n%s\n" % (fn1, e, traceback.format_exc()))
def SstLevelInfo():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn = "%s/sst-info-by-time-by-levels-level-seps-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn):
return fn
sst_y_cord_level_sep_highs = SstYCord.LevelSepHigh()
with Cons.MT(
"Generating Sst info by time by levels: level separators data file ..."
):
with open(fn, "w") as fo:
fmt = "%1d %10d %10s"
fo.write("%s\n" % Util.BuildHeader(
fmt, "level level_mid_for_labels level_low_for_separators"))
lh_prev = 0
for l, lh in sorted(sst_y_cord_level_sep_highs.iteritems()):
lm = (lh + lh_prev) / 2
fo.write((fmt + "\n") % (l, lm, lh_prev))
lh_prev = lh
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def SstInfo():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn = "%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn):
return fn
(sst_lives, memt_lives) = MemtSstLife.Get()
with Cons.MT("Generating Sst info by time by levels data file ..."):
#with open(fn_m, "w") as fo:
# fo.write("%s\n" % Memt.Header())
# for addr, l in sorted(_memt_lives.iteritems()):
# fo.write("%s\n" % l)
#Cons.P("Created %s %d" % (fn_m, os.path.getsize(fn_m)))
with open(fn, "w") as fo:
fo.write("%s\n" % MemtSstLife.SstLife.Header())
for sst_gen, l in sorted(sst_lives.iteritems()):
fo.write("%s\n" % l)
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def RunSubp(cmd,
env=os.environ.copy(),
shell=True,
print_cmd=True,
print_output=True):
if print_cmd:
Cons.P(cmd)
lines = ""
p = None
if shell:
p = subprocess.Popen(cmd,
shell=shell,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
else:
p = subprocess.Popen(cmd.split(" "),
shell=shell,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# http://stackoverflow.com/questions/18421757/live-output-from-subprocess-command
# It can read char by char depending on the requirements.
for line in iter(p.stdout.readline, ''):
if print_output:
Cons.P(Indent(line.rstrip(), 2))
lines += line
p.wait()
if p.returncode != 0:
raise RuntimeError("Error: cmd=[%s] rc=%d" % (cmd, p.returncode))
return lines
def _KeepCheckingInstAndTag(self):
while True:
r = None
while True:
try:
r = BotoClient.Get(self.params["region"]).describe_instances(InstanceIds=self.inst_info.GetAllInstIds())
break
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(1)
else:
raise e
num_running = 0
for e in r["Reservations"]:
for e1 in e["Instances"]:
inst_id = e1["InstanceId"]
state = e1["State"]["Name"]
Cons.P("inst_id=%s state=%s" % (inst_id, state))
if state in ["shutting-down", "terminated"]:
raise RuntimeError("Unexpected: %s" % pprint.pformat(e1))
elif state == "running":
num_running += 1
# Some doesn't have it. Might be not assigned yet.
#pub_ip = e1["PublicIpAddress"]
elif state == "pending":
self._TagInst(inst_id)
if num_running == 1:
Cons.P("job_id: %s. %d instances are created." % (self.job_id, num_running))
return
time.sleep(1)
def main(argv):
Conf.ParseArgs()
Util.MkDirs(Conf.GetDir("output_dir"))
dist_sq_thresholds = [0, 0.008, 0.01, 0.02, 0.05]
#dist_sq_thresholds = [0.02]
dist_sq_thresholds_str = []
for d in dist_sq_thresholds:
dist_sq_thresholds_str.append(_NoTrailing0s(d))
reduced_files = []
reduced_file_sizes = []
with Cons.MT("Generating reduced size usa map ..."):
for d in dist_sq_thresholds_str:
fn_co_loc = "filter-out-almost-duplicate-points/.output/usa-map-smallsize-%s" % d
reduced_files.append(fn_co_loc)
if not os.path.exists(fn_co_loc):
cmd = "cd filter-out-almost-duplicate-points && ./build-and-run.sh --dist_sq_threshold=%s" % d
Util.RunSubp(cmd)
reduced_file_sizes.append(os.path.getsize(fn_co_loc))
dn_out = "%s/.output" % os.path.dirname(__file__)
fn_out = "%s/usa-map.pdf" % dn_out
with Cons.MT("Plotting ..."):
env = os.environ.copy()
env["IN_FNS"] = " ".join(reduced_files)
env["IN_FN_SIZES"] = " ".join(str(s) for s in reduced_file_sizes)
env["DIST_SQ_THRESHOLDS"] = " ".join(dist_sq_thresholds_str)
env["OUT_FN"] = fn_out
Util.RunSubp("gnuplot %s/usa-map.gnuplot" % os.path.dirname(__file__),
env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def _TagInst(self, inst_id):
if self.inst_info.IsInstTagged(inst_id):
return
# Go with minimal tags for now. The numbers are limited, like 10.
tags = {"job_id": self.job_id}
tags_boto = []
for k, v in tags.iteritems():
tags_boto.append({"Key": k, "Value": v})
while True:
try:
BotoClient.Get(self.params["region"]).create_tags(Resources=[inst_id], Tags=tags_boto)
self.inst_info.SetInstTagged(inst_id)
return
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(1)
elif e.response["Error"]["Code"] == "RequestLimitExceeded":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(5)
else:
raise e
def ByJobIdTermSelfLast():
job_id = Ec2Util.JobId()
Cons.P("Terminating running instances of job_id %s" % job_id)
_TermInst.Init(term_by_job_id_self_last=True)
tags = {}
tags["job_id"] = job_id
tis = []
for r in Ec2Region.All():
tis.append(_TermInst(r, tags))
threads = []
for ti in tis:
t = threading.Thread(target=ti.Run)
t.daemon = True
threads.append(t)
t.start()
for t in threads:
t.join()
print ""
Cons.P(_TermInst.Header())
for ti in tis:
ti.PrintResult()
def PlotSstAccDistAtSpecificTimes():
# At the time m sec after the n-th SSTable is created (time t). To get the
# max_plot_hgieht, all plot data files need to be generated before plotting
# the first one.
plot_data_fns_at_n = {}
with Cons.MT(
"Generating plot data for SSTables by levels with heat at specific times ..."
):
for (n, m) in Conf.times_sst_by_levels_with_heat:
(fn_in_boxes,
fn_in_level_seps) = SstByLevelsWithHeatAtSpecificTimes.Boxes(
n, m)
plot_data_fns_at_n[n] = (fn_in_boxes, fn_in_level_seps)
with Cons.MT(
"Plotting SSTables by levels with heat at specific times ..."):
dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result
for n, (fn_in_boxes,
fn_in_level_seps) in sorted(plot_data_fns_at_n.iteritems()):
env = os.environ.copy()
env["FN_IN_BOXES"] = fn_in_boxes
env["FN_IN_LEVEL_INFO"] = fn_in_level_seps
env["MAX_PLOT_HEIGHT"] = str(
SstByLevelsWithHeatAtSpecificTimes.max_plot_height)
fn_out = "%s/sst-by-level-by-ks-range-with-heat-%s-%s.pdf" % (
dn, Conf.ExpStartTime(), n)
env["FN_OUT"] = fn_out
Util.RunSubp(
"gnuplot %s/sst-by-level-by-ks-range-with-heat-at-specific-time.gnuplot"
% os.path.dirname(__file__),
env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def Get1minAvgFn():
exp_dts = []
for i in range(2):
#Cons.P(Conf.Get(i))
mo = re.match(r".+/(?P<exp_dt>\d\d\d\d\d\d-\d\d\d\d\d\d\.\d\d\d)-d", Conf.Get(i))
exp_dts.append(mo.group("exp_dt"))
fn_out = "%s/mem-1minavg-%s" % (Conf.GetOutDir(), "-".join(exp_dts))
if os.path.exists(fn_out):
return fn_out
with Cons.MT("Creating avg memory usage comparison file for plotting ..."):
records = []
dn_base = Conf.GetDir("dn_base")
for i in range(2):
fn_ycsb_log = "%s/%s" % (dn_base, Conf.Get(i))
hm_mem = _GetHmMem(fn_ycsb_log)
for hm, mem in hm_mem.iteritems():
records.append(_RecordMemAvg(hm, i * 30, mem, i))
records.sort(key=operator.attrgetter("ts"))
fmt = "%8s %6.3f %1d"
header = Util.BuildHeader(fmt, "timestamp mem_avg_in_gb exp_type")
with open(fn_out, "w") as fo:
i = 0
for r in records:
if i % 40 == 0:
fo.write(header + "\n")
i += 1
fo.write("%s\n" % r.ToStr(fmt))
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
return fn_out
def _ReadStoredLog():
if Conf.ExpStartTime() is None:
return None
dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~")
fn = "%s/system-%s" % (dn, Conf.ExpStartTime())
if not os.path.isfile(fn):
# If there is a 7z file, uncompress it
fn_7z = "%s.7z" % fn
if os.path.isfile(fn_7z):
with Cons.MT("Found a 7z file. Uncompressing"):
Util.RunSubp("7z e -o%s %s" % (dn, fn_7z))
else:
return None
with Cons.MT("Reading the stored Cassandra Mutant log file %s" % fn, print_time=False):
lines = []
with open(fn) as fo:
for line in fo.readlines():
lines.append(line.strip())
# Stop after reading n lines for testing
if 0 < Conf.MaxCassLogLines():
if Conf.MaxCassLogLines() < len(lines):
break
#Cons.P(len(lines))
return lines
def _TagInst(self, inst_id):
if self.spot_req_infos.IsInstTagged(inst_id):
return
# Go with minimal tags for now.
tags = {
"job_id": self.job_id
# , "name": self.spot_req_infos.NodeName(inst_id)
# Experiment name to prevent your collaborator accidentally killing your VM
,
"Name": self.params["ec2_tag_Name"]
# Note: node expiration time can be added here for auto cleaning. dev
# nodes don't have them.
}
tags_boto = []
for k, v in tags.iteritems():
tags_boto.append({"Key": k, "Value": v})
while True:
try:
BotoClient.Get(self.params["region"]).create_tags(
Resources=[inst_id], Tags=tags_boto)
self.spot_req_infos.SetInstTagged(inst_id)
return
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(1)
elif e.response["Error"]["Code"] == "RequestLimitExceeded":
Cons.P("region=%s error=%s" % (self.params["region"], e))
time.sleep(5)
else:
raise e
def main():
# Takes in the file and parses into datum's.
D = cData(sys.argv[1])
# Starting points from the pickle or not.
EmAlg = parseCommandLine(D, sys.argv)
prevCons = 0
totalcons = 0
nmiResult = evaluateEM_NMI(D, EmAlg)
print "Initial nmi: ",nmiResult
consobj = Cons.cCons(D)
for numCons in range(1,len(D.data)/4,1):
consobj.constype = Cons.cCons.eConsType.TripCenterChunk
consobj.centerChunkSize = 0.2
cons = consobj.tripCons(EmAlg.mGammas,numCons-prevCons)
prevCons = numCons
totalcons += len(cons)
for i in cons:
EmAlg.mCij[i[0]][i[1]] = i[2]
EmAlg.mCij[i[1]][i[0]] = i[2]
EmAlg.EM(len(D.classlist))
nmiresult = evaluateEM_NMI(D, EmAlg)
print numCons, ",", nmiresult, ",", totalcons
if(nmiresult > 0.999 or len(D.data)==numCons):
break
def Run(self):
bc = boto3.session.Session().client("ec2", region_name = self.region)
response = bc.describe_account_attributes(
AttributeNames=[
"max-instances",
]
)
#Cons.P(pprint.pformat(response, indent=2))
for r in response["AccountAttributes"]:
if r["AttributeName"] != "max-instances":
continue
if len(r["AttributeValues"]) != 1:
raise RuntimeError("len(r[\"AttributeValues\"])=%d" % len(r["AttributeValues"]))
self.max_inst = int(r["AttributeValues"][0]["AttributeValue"])
#Cons.P(self.max_inst)
Cons.sys_stdout_write(" %s" % self.region)
def Run(self):
try:
# http://boto3.readthedocs.io/en/latest/guide/session.html
session = boto3.session.Session()
bc = BotoClient.Get(self.region)
if self.tags is None:
self.response = bc.describe_instances()
else:
filters = []
for k, v in self.tags.iteritems():
d = {}
d["Name"] = ("tag:%s" % k)
d["Values"] = [v]
filters.append(d)
self.response = bc.describe_instances(Filters = filters)
except Exception as e:
Cons.P("%s\n%s\nregion=%s" % (e, traceback.format_exc(), self.region))
os._exit(1)
Cons.sys_stdout_write(" %s" % self.region)
def cons_to_list(cons): """Converts a Lisp cons object to a Python list :rtype : Atom """ result = [] while cons: if cons.first and cons.first.type == Atom.CONS: item = Cons.orphan(cons.first) if cons.quote: item.quote = cons.quote else: item = cons.first result.append(item) cons = cons.second.data return result
def TripConsTest(D, sNum, EMStarts, fp):
fp.write("trips,queries,cons,likelihood,NMI\n")
for option in TripConsOptions.lOptions:
optname = TripConsOptions.lOptionNames[option]
fp.write(optname + "\n")
for iOutertrial in range(numOuterTrials):
print "scenario ", sNum, " options ", optname, " outertrial ", iOutertrial
fp.write("outertrial: %d\n" % iOutertrial)
em = copy.deepcopy(EMStarts[iOutertrial])
em.bPPC = True
prevTrips = 0
totalCons = 0
nmiResult = utils.evaluateEM_NMI(D, em)
fp.write("Initial nmi: %f\n" % nmiResult)
consobj = Cons.cCons(D)
for numTrips in range(1,len(D.data)/4,1):
if option == TripConsOptions.CenterChunkCons:
consobj.constype = Cons.cCons.eConsType.TripCenterChunk
elif option == TripConsOptions.MidCons:
consobj.constype = Cons.cCons.eConsType.TripMids
print em.mLikelihood_il
cons = consobj.tripCons(em.mGammas,numTrips-prevTrips)
prevTrips = numTrips
totalCons += len(cons)
for i in cons:
em.mCij[i[0]][i[1]] = i[2]
em.mCij[i[1]][i[0]] = i[2]
em.EM(len(D.classlist))
nmiresult = utils.evaluateEM_NMI(D, em)
fp.write("%d,%d,%d,%f,%f\n" % (numTrips,
numTrips*14,
totalCons,
em.dEMLikelihood,
nmiresult) )
fp.flush()
if (nmiresult > 0.999 or len(D.data)==numTrips):
break
