def perf(opts, conf, args):
def task_count_last(i):
s = i[1].split()
try:
count = int(s[0])
last = int(s[1])
except:
return None
else:
return count, last
script = ['if', '[', '-f', 'task_count', ']', '&&', '[', '-f', 'task_last', '];', 'then', 'cat', 'task_count;', 'cat', 'task_last;', 'else', 'echo', '0;', 'fi']
instances = aws.filter_instances(opts, conf)
idict = dict([(i.dns_name, i) for i in instances])
sdict = aws.get_spot_request_dict(conf)
data = {}
for i in run_cmd_list(opts, conf, ssh_cmd_list(opts, conf, script, instances), show_output=False, capture_stderr=False):
host = i[0]
inst = idict.get(host)
if inst:
sir = sdict.get(inst.spot_instance_request_id)
price = None
if sir:
price = float(sir.price)
tasks = task_count_last(i)
if tasks:
task_count, task_last = tasks
uptime = aws.get_uptime(task_last, inst.launch_time) / 3600.0
stat = data.setdefault(inst.instance_type, dict(n=0, uptime_sum=0.0, task_sum=0, price_sum=0.0))
stat['n'] += 1
stat['uptime_sum'] += uptime
stat['task_sum'] += task_count
if price is not None:
stat['price_sum'] += price
tph= []
tpd = []
total_tasks = 0.0
total_uptime = 0
total_n = 0
for itype, stat in data.items():
total_tasks += stat['task_sum']
total_uptime += stat['uptime_sum']
total_n += stat['n']
tasks_per_hour = stat['task_sum'] / stat['uptime_sum']
tph.append((tasks_per_hour, itype))
if 'price_sum' in stat:
mprice = stat['price_sum'] / stat['n']
tasks_per_dollar = tasks_per_hour / mprice
tpd.append((tasks_per_dollar, itype))
tph.sort(reverse=True)
tpd.sort(reverse=True)
if total_n:
print "Tasks per hour (%.02f)" % (total_tasks / total_uptime * total_n,)
for tasks_per_hour, itype in tph:
print " %s %.02f" % (itype, tasks_per_hour)
print "Tasks per US$"
for tasks_per_dollar, itype in tpd:
print " %s %.02f" % (itype, tasks_per_dollar)
def ssh_cmd_list(opts, conf, args, instances=None):
if instances is None:
instances = aws.filter_instances(opts, conf)
for i in instances:
node = i.public_dns_name
cmd = ssh_args(opts, conf)
cmd.append(node)
cmd.extend(args)
yield node, cmd
def ssh_cmd_list(opts, conf, args, instances=None):
if instances is None:
instances = aws.filter_instances(opts, conf)
for i in instances:
node = i.public_dns_name
cmd = ssh_args(opts, conf)
cmd.append(node)
cmd.extend(args)
yield node, cmd
def status(opts, conf):
ec2 = aws.get_ec2_conn(conf)
instances = aws.filter_instances(opts, conf)
if instances:
print "Active Instances"
now = time.time()
for i in instances:
uptime = aws.get_uptime(now, i.launch_time)
print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name
requests = ec2.get_all_spot_instance_requests()
if requests:
print "Spot Requests"
for r in requests:
dns_name = ''
print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def status(opts, conf):
ec2 = aws.get_ec2_conn(conf)
instances = aws.filter_instances(opts, conf)
if instances:
print "Active Instances"
now = time.time()
for i in instances:
uptime = aws.get_uptime(now, i.launch_time)
print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name
requests = ec2.get_all_spot_instance_requests()
if requests:
print "Spot Requests"
for r in requests:
dns_name = ''
print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def status(opts, conf):
now = time.time()
instances = aws.filter_instances(opts, conf,
{'instance-state-name': 'running'})
if instances:
print "Running Instances"
for i in instances:
uptime = aws.get_uptime(now, i.launch_time)
print ' ', i.image_id, aws.format_uptime(
uptime), i.public_dns_name, i.tags
requests = aws.get_all_spot_instance_requests(
opts, conf, {'state': ['active', 'open']})
if requests:
print "Active Spot Requests"
for r in requests:
print " %s %s %s %s $%s %s %s %s" % (r.id, r.region, r.type,
r.create_time, r.price, r.state,
r.status, r.tags)
def stop(opts, conf):
instances = aws.filter_instances(opts, conf)
iids = [i.id for i in instances]
aws.shutdown(opts, conf, iids)
def rsync_cmd_list(opts, conf, args):
for i in aws.filter_instances(opts, conf):
node = i.public_dns_name
cmd = ['rsync', '-e', ' '.join(ssh_args(opts, conf))
] + [a.replace('HOST', node) for a in args]
yield node, cmd
def instances(opts, conf):
now = time.time()
for i in aws.filter_instances(opts, conf):
uptime = aws.get_uptime(now, i.launch_time)
print i.state, i.image_id, aws.format_uptime(
uptime), i.public_dns_name, i.tags
def stop(opts, conf):
instances = aws.filter_instances(opts, conf)
iids = [i.id for i in instances]
aws.shutdown(opts, conf, iids)
def rsync_cmd_list(opts, conf, args, hostset=None):
for i in aws.filter_instances(opts, conf, hostset=hostset):
node = i.public_dns_name
cmd = ["rsync", "-e", " ".join(ssh_args(opts, conf))] + [a.replace("HOST", node) for a in args]
yield node, cmd
def instances(opts, conf):
now = time.time()
for i in aws.filter_instances(opts, conf):
uptime = aws.get_uptime(now, i.launch_time)
print i.image_id, aws.format_uptime(uptime), i.public_dns_name
def perf(opts, conf, args):
def task_count_last(i):
s = i[1].split()
try:
count = int(s[0])
last = int(s[1])
except:
return None
else:
return count, last
script = [
"if",
"[",
"-f",
"task_count",
"]",
"&&",
"[",
"-f",
"task_last",
"];",
"then",
"cat",
"task_count;",
"cat",
"task_last;",
"else",
"echo",
"0;",
"fi",
]
instances = aws.filter_instances(opts, conf)
idict = dict([(i.dns_name, i) for i in instances])
sdict = aws.get_spot_request_dict(conf)
data = {}
for i in run_cmd_list(
opts, conf, ssh_cmd_list(opts, conf, script, instances), show_output=False, capture_stderr=False
):
host = i[0]
inst = idict.get(host)
if inst:
sir = sdict.get(inst.spot_instance_request_id)
price = None
if sir:
price = float(sir.price)
tasks = task_count_last(i)
if tasks:
task_count, task_last = tasks
uptime = aws.get_uptime(task_last, inst.launch_time) / 3600.0
stat = data.setdefault(inst.instance_type, dict(n=0, uptime_sum=0.0, task_sum=0, price_sum=0.0))
stat["n"] += 1
stat["uptime_sum"] += uptime
stat["task_sum"] += task_count
if price is not None:
stat["price_sum"] += price
tph = []
tpd = []
total_tasks = 0.0
total_uptime = 0
total_n = 0
for itype, stat in data.items():
total_tasks += stat["task_sum"]
total_uptime += stat["uptime_sum"]
total_n += stat["n"]
tasks_per_hour = stat["task_sum"] / stat["uptime_sum"]
tph.append((tasks_per_hour, itype))
if "price_sum" in stat:
mprice = stat["price_sum"] / stat["n"]
tasks_per_dollar = tasks_per_hour / mprice
tpd.append((tasks_per_dollar, itype))
tph.sort(reverse=True)
tpd.sort(reverse=True)
if total_n:
print "Tasks per hour (%.02f)" % (total_tasks / total_uptime * total_n,)
for tasks_per_hour, itype in tph:
print " %s %.02f" % (itype, tasks_per_hour)
print "Tasks per US$"
for tasks_per_dollar, itype in tpd:
print " %s %.02f" % (itype, tasks_per_dollar)