def gen_cdfs(experiment_name):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = "select * from `{}` where experiment_name like'{}%' and num_nodes='4'".format(
ec2config.sdb_sweep_domain, experiment_name, experiment_name)
rs = dom.select(query)
to_plot = defaultdict(list)
for r in rs:
# selected workloads. Skip those for which env set (threshold sweep)
# print r
if r['tracefile'] in tracefile_to_workload and 'log_fn' in r:
if 'env' in r and r['env'] != u'{}':
print 'skipping', r
continue
workload = tracefile_to_workload[r['tracefile']]
scheduler = r['scheduler']
log_fn = r['log_fn']
print workload, scheduler, log_fn
if scheduler in include_schedulers:
to_plot[workload].append(
(scheduler,
'sweep/' + r['experiment_name'] + '/' + log_fn))
print to_plot
for workload, data in sorted(to_plot.items()):
plot_cdfs.build_submit_phase0_cdf_multi_scheduler(
workload, data, 'poster_figs/{}-cdf'.format(workload))
def dump_activity(experiment_name):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = _getquery(experiment_name)
rs = dom.select(query)
end_times = []
elapsed_times = []
for r in reversed(list(rs)):
print r.name, r
sdb_conn.close()
def dump_activity(experiment_name):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = "select * from `{}` where experiment_name like '{}%' and hostname ='50aaaa988e94'".format(
ec2config.sdb_sweep_domain, experiment_name, experiment_name)
rs = dom.select(query)
end_times = []
elapsed_times = []
for r in reversed(list(rs)):
print r.name, r
sdb_conn.close()
def get_simulator_runs(experiment_name, filters):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = "select * from `{}` where experiment_name like '{}%' and end_time is not null".format(
ec2config.sdb_sweep_domain, experiment_name, experiment_name)
for filter in filters:
query += " and {}='{}'".format(filter[0], filter[1])
rs = dom.select(query)
simulator_runs = []
for r in list(rs):
simulator_runs.append(dict(r.items()))
sdb_conn.close()
return simulator_runs
def do_iter():
try:
m = queue.read()
if m is None:
print "waiting for more input"
time.sleep(sleep_time)
else:
queue.delete_message(m)
config = json.loads(m.get_body())
print config
replay_id = config['replay_id']
config_start = copy.copy(config)
config_start['hostname'] = socket.gethostname()
config_start['gitrev'] = gitrev.get_rev()
config_start['start_time'] = time.time()
config_start['is-start'] = True
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
dom.put_attributes(replay_id + '-start', config)
sdb_conn.close()
config_etc = replay_trace(config_start)
if 'env' in config_etc:
config_etc['env'] = json.dumps(config_etc['env'])
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
dom.put_attributes(replay_id, config_etc)
sdb_conn.close()
except ValueError as err:
print err
traceback.print_exc()
except RuntimeError as err:
print err
traceback.print_exc()
def measure_activity(experiment_name):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = "select start_time, end_time from `{}` where end_time > '1480000000' and experiment_name like '{}%' order by end_time desc".format(
ec2config.sdb_sweep_domain, experiment_name, experiment_name)
rs = dom.select(query)
end_times = []
elapsed_times = []
for r in reversed(list(rs)):
end_time = float(r['end_time'])
start_time = float(r['start_time'])
elapsed_time = end_time - start_time
end_times.append(end_time)
elapsed_times.append(elapsed_time)
print '{} simulations run for experiment {}'.format(
len(end_times), experiment_name)
elapsed_sec = max(end_times) - min(end_times)
print 'elapsed time {:.0f} seconds ({:.2f} hours)'.format(
elapsed_sec, elapsed_sec / 3600)
print 'avg simulation duartion {:.2f} seconds'.format(
sum(elapsed_times) / float(len(elapsed_times)))
sdb_conn.close()
def measure_activity(experiment_name):
sdb_conn = ec2config.sdb_connect()
dom = sdb_conn.get_domain(ec2config.sdb_sweep_domain)
query = _getquery(experiment_name)
rs = dom.select(query)
end_times = []
elapsed_times = []
for r in reversed(list(rs)):
if not 'end_time' in r:
continue
end_time = float(r['end_time'])
start_time = float(r['start_time'])
elapsed_time = end_time - start_time
end_times.append(end_time)
elapsed_times.append(elapsed_time)
print '{} simulations run for experiment {}'.format(
len(end_times), experiment_name)
elapsed_sec = max(end_times) - min(end_times)
print 'elapsed time {:.0f} seconds ({:.2f} hours)'.format(
elapsed_sec, elapsed_sec / 3600)
print 'avg simulation duartion {:.2f} seconds'.format(
sum(elapsed_times) / float(len(elapsed_times)))
sdb_conn.close()