def grow_ebs_for_task(task_fragment, target_size_gb):
"""Grows EBS volume for given task."""
ec2 = u.create_ec2_resource()
client = u.create_ec2_client()
# todo: don't crash on missing/duplicate names
instances = {u.get_name(i.tags): i for i in ec2.instances.all()}
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
sorted_instances = sorted(instances, key=itemgetter(0))
for (seconds, instance) in sorted_instances:
task_name = u.get_name(instance.tags)
hours_ago = (time.time() - seconds) / 3600
hours_ago += 8 # adjust for time being in UTC
if task_fragment in task_name:
print("Found instance %s launched %.1f hours ago" %
(task_name, hours_ago))
break
print(instance.id)
volumes = list(instance.volumes.all())
assert len(volumes) == 1, "Must have 1 volume"
print("Growing %s to %s" % (volumes[0].id, target_size_gb))
response = client.modify_volume(
VolumeId=volumes[0].id,
Size=target_size_gb,
)
assert u.is_good_response(response)
def get_instance(fragment):
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
# latest instance first
sorted_instances = reversed(sorted(instances, key=itemgetter(0)))
for (seconds, instance) in sorted_instances:
name = u.get_name(instance.tags)
if fragment in u.get_name(instance.tags):
hours_ago = (time.time() - seconds) / 3600
hours_ago += 8 # adjust for time being in UTC
print("Found instance %s launched %.1f hours ago" % (
name,
hours_ago,
))
return instance
print("Found nothing matching", fragment)
def list_instances():
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
sorted_instances = sorted(instances, key=itemgetter(0))
for (seconds, instance) in sorted_instances:
hours_ago = (time.time() - seconds) / 3600
hours_ago += 8 # adjust for time being in UTC
if instance.state['Name'] != 'running':
continue
if not (LIMIT_TO_KEY in instance.key_name):
continue
print("%4s %20s %10s %20s %s %s" %
(int(hours_ago), u.get_name(instance.tags),
instance.instance_type, instance.public_ip_address,
instance.private_ip_address, instance.id))
def list_ebss():
"""Print list of instances with their attached volume id/size to console, ie
master-us-east-1a.masters.df86c4e8-pachydermcluster.kubernetes.com: vol-0f0e841d0cc657002 (20),vol-06fb03280cf2598fb (20),vol-0e7ef0896b234db53 (64)
nodes.df86c4e8-pachydermcluster.kubernetes.com: vol-012367900cd8dae8c (128)
nodes.df86c4e8-pachydermcluster.kubernetes.com: vol-0a98ee5f7f155b2b7 (128),vol-048e29f604d2900a7 (100)
imagenet: vol-024347797a6ab11e8 (1500)
api_service_prod: vol-0c36c9f21bb6be8a6 (8)
box00.gpubox.0: vol-0c69c68295a89cde5 (50)
"""
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
sorted_instances = sorted(instances, key=itemgetter(0))
for (seconds, instance) in sorted_instances:
volumes = instance.volumes.all()
volume_strs = []
for v in volumes:
volume_strs.append("%s (%s)" % (v.id, v.size))
print("%s: %s" % (u.get_name(instance.tags), ','.join(volume_strs)))