def perf(opts, conf, args): def task_count_last(i): s = i[1].split() try: count = int(s[0]) last = int(s[1]) except: return None else: return count, last script = ['if', '[', '-f', 'task_count', ']', '&&', '[', '-f', 'task_last', '];', 'then', 'cat', 'task_count;', 'cat', 'task_last;', 'else', 'echo', '0;', 'fi'] instances = aws.filter_instances(opts, conf) idict = dict([(i.dns_name, i) for i in instances]) sdict = aws.get_spot_request_dict(conf) data = {} for i in run_cmd_list(opts, conf, ssh_cmd_list(opts, conf, script, instances), show_output=False, capture_stderr=False): host = i[0] inst = idict.get(host) if inst: sir = sdict.get(inst.spot_instance_request_id) price = None if sir: price = float(sir.price) tasks = task_count_last(i) if tasks: task_count, task_last = tasks uptime = aws.get_uptime(task_last, inst.launch_time) / 3600.0 stat = data.setdefault(inst.instance_type, dict(n=0, uptime_sum=0.0, task_sum=0, price_sum=0.0)) stat['n'] += 1 stat['uptime_sum'] += uptime stat['task_sum'] += task_count if price is not None: stat['price_sum'] += price tph= [] tpd = [] total_tasks = 0.0 total_uptime = 0 total_n = 0 for itype, stat in data.items(): total_tasks += stat['task_sum'] total_uptime += stat['uptime_sum'] total_n += stat['n'] tasks_per_hour = stat['task_sum'] / stat['uptime_sum'] tph.append((tasks_per_hour, itype)) if 'price_sum' in stat: mprice = stat['price_sum'] / stat['n'] tasks_per_dollar = tasks_per_hour / mprice tpd.append((tasks_per_dollar, itype)) tph.sort(reverse=True) tpd.sort(reverse=True) if total_n: print "Tasks per hour (%.02f)" % (total_tasks / total_uptime * total_n,) for tasks_per_hour, itype in tph: print " %s %.02f" % (itype, tasks_per_hour) print "Tasks per US$" for tasks_per_dollar, itype in tpd: print " %s %.02f" % (itype, tasks_per_dollar)
def ssh_cmd_list(opts, conf, args, instances=None): if instances is None: instances = aws.filter_instances(opts, conf) for i in instances: node = i.public_dns_name cmd = ssh_args(opts, conf) cmd.append(node) cmd.extend(args) yield node, cmd
def status(opts, conf): ec2 = aws.get_ec2_conn(conf) instances = aws.filter_instances(opts, conf) if instances: print "Active Instances" now = time.time() for i in instances: uptime = aws.get_uptime(now, i.launch_time) print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name requests = ec2.get_all_spot_instance_requests() if requests: print "Spot Requests" for r in requests: dns_name = '' print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def status(opts, conf): now = time.time() instances = aws.filter_instances(opts, conf, {'instance-state-name': 'running'}) if instances: print "Running Instances" for i in instances: uptime = aws.get_uptime(now, i.launch_time) print ' ', i.image_id, aws.format_uptime( uptime), i.public_dns_name, i.tags requests = aws.get_all_spot_instance_requests( opts, conf, {'state': ['active', 'open']}) if requests: print "Active Spot Requests" for r in requests: print " %s %s %s %s $%s %s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status, r.tags)
def stop(opts, conf): instances = aws.filter_instances(opts, conf) iids = [i.id for i in instances] aws.shutdown(opts, conf, iids)
def rsync_cmd_list(opts, conf, args): for i in aws.filter_instances(opts, conf): node = i.public_dns_name cmd = ['rsync', '-e', ' '.join(ssh_args(opts, conf)) ] + [a.replace('HOST', node) for a in args] yield node, cmd
def instances(opts, conf): now = time.time() for i in aws.filter_instances(opts, conf): uptime = aws.get_uptime(now, i.launch_time) print i.state, i.image_id, aws.format_uptime( uptime), i.public_dns_name, i.tags
def rsync_cmd_list(opts, conf, args, hostset=None): for i in aws.filter_instances(opts, conf, hostset=hostset): node = i.public_dns_name cmd = ["rsync", "-e", " ".join(ssh_args(opts, conf))] + [a.replace("HOST", node) for a in args] yield node, cmd
def instances(opts, conf): now = time.time() for i in aws.filter_instances(opts, conf): uptime = aws.get_uptime(now, i.launch_time) print i.image_id, aws.format_uptime(uptime), i.public_dns_name
def perf(opts, conf, args): def task_count_last(i): s = i[1].split() try: count = int(s[0]) last = int(s[1]) except: return None else: return count, last script = [ "if", "[", "-f", "task_count", "]", "&&", "[", "-f", "task_last", "];", "then", "cat", "task_count;", "cat", "task_last;", "else", "echo", "0;", "fi", ] instances = aws.filter_instances(opts, conf) idict = dict([(i.dns_name, i) for i in instances]) sdict = aws.get_spot_request_dict(conf) data = {} for i in run_cmd_list( opts, conf, ssh_cmd_list(opts, conf, script, instances), show_output=False, capture_stderr=False ): host = i[0] inst = idict.get(host) if inst: sir = sdict.get(inst.spot_instance_request_id) price = None if sir: price = float(sir.price) tasks = task_count_last(i) if tasks: task_count, task_last = tasks uptime = aws.get_uptime(task_last, inst.launch_time) / 3600.0 stat = data.setdefault(inst.instance_type, dict(n=0, uptime_sum=0.0, task_sum=0, price_sum=0.0)) stat["n"] += 1 stat["uptime_sum"] += uptime stat["task_sum"] += task_count if price is not None: stat["price_sum"] += price tph = [] tpd = [] total_tasks = 0.0 total_uptime = 0 total_n = 0 for itype, stat in data.items(): total_tasks += stat["task_sum"] total_uptime += stat["uptime_sum"] total_n += stat["n"] tasks_per_hour = stat["task_sum"] / stat["uptime_sum"] tph.append((tasks_per_hour, itype)) if "price_sum" in stat: mprice = stat["price_sum"] / stat["n"] tasks_per_dollar = tasks_per_hour / mprice tpd.append((tasks_per_dollar, itype)) tph.sort(reverse=True) tpd.sort(reverse=True) if total_n: print "Tasks per hour (%.02f)" % (total_tasks / total_uptime * total_n,) for tasks_per_hour, itype in tph: print " %s %.02f" % (itype, tasks_per_hour) print "Tasks per US$" for tasks_per_dollar, itype in tpd: print " %s %.02f" % (itype, tasks_per_dollar)