def validate_image_id(self, image_id):
ec2 = self.connect_ec2()
try:
ec2.get_all_images(image_ids=[image_id])
return True, ''
except boto.exception.EC2ResponseError as e:
return False, e.error_message
def has_image(self, image_id):
"""
Checks to see if the given ami is available in the account
"""
ec2 = self.connect_ec2()
try:
ec2.get_all_images(image_id)
return True, ''
except boto.exception.EC2ResponseError:
return False, 'The image id \'{0}\' does not exist in this ' \
'account.'.format(image_id)
def _lookup_ami_id(ec2, name):
"""Returns AMI id for `name`"""
images = ec2.get_all_images(filters={'name': [name]})
if len(images) != 1:
raise RuntimeError('cannot find exactly one image')
return images[0]
def process_region(region):
print "Processing region %s" % region.name
ec2 = region.connect()
print "Scanning instances...\n"
instances = ec2.get_only_instances()
for inst in instances:
inst_accessor = Instance(inst)
process_instance(inst_accessor)
print "\nScanning images...\n"
images = ec2.get_all_images(None, ("self"))
for image in images:
image_accessor = Image(image)
process_image(image_accessor)
print "\nScanning volumes...\n"
volumes = ec2.get_all_volumes()
for volume in volumes:
volume_accessor = Volume(volume)
process_volume(volume_accessor)
print "\nScanning snapshots...\n"
snapshots = ec2.get_all_snapshots(owner="self")
for snapshot in snapshots:
process_snapshot(Snapshot(snapshot))
def ec2_deregister_ami(ec2): images = ec2.get_all_images(owners="self") for i in images: try: print "---Deregistering AMI: " + i.id ec2.deregister_image(i.id, delete_snapshot=True) except Exception, e: print(e)
def register_image(module, ec2):
"""
Registers AMI
"""
name = module.params.get('name')
description = module.params.get('description')
image_location = module.params.get('image_location')
architecture = module.params.get('architecture')
kernel_id = module.params.get('kernel_id')
root_device_name = module.params.get('root_device_name')
virtualization_type = module.params.get('virtualization_type')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
device_mapping = module.params.get('device_mapping')
try:
params = {'name': name,
'description': description,
'architecture': architecture,
'virtualization_type': virtualization_type}
if image_location:
params['image_location'] = image_location
if kernel_id:
params['kernel_id'] = kernel_id
if root_device_name:
params['root_device_name'] = root_device_name
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if 'device_name' not in device:
module.fail_json(msg = 'Device name must be set for volume')
device_name = device['device_name']
del device['device_name']
bd = BlockDeviceType(**device)
bdm[device_name] = bd
params['block_device_map'] = bdm
image_id = ec2.register_image(**params)
except boto.exception.BotoServerError, e:
if e.error_code == 'InvalidAMIName.Duplicate':
images = ec2.get_all_images()
for img in images:
if img.name == name:
module.exit_json(msg="AMI name already present", image_id=img.id, state=img.state, changed=False)
sys.exit(0)
else:
module.fail_json(msg="Error in retrieving duplicate AMI details")
else:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def get_blessed_ami():
images = ec2.get_all_images(
filters={
'tag:environment': args.environment,
'tag:deployment': args.deployment,
'tag:play': args.play,
'tag:blessed': True
}
)
if len(images) != 1:
raise Exception("ERROR: Expected only one blessed ami, got {}\n".format(
len(images)))
return images[0].id
def create_image(module, ec2):
"""
Creates new AMI
module : AnsibleModule object
ec2: authenticated ec2 connection object
"""
instance_id = module.params.get('instance_id')
name = module.params.get('name')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
description = module.params.get('description')
no_reboot = module.params.get('no_reboot')
device_mapping = module.params.get('device_mapping')
tags = module.params.get('tags')
launch_permissions = module.params.get('launch_permissions')
try:
params = {'instance_id': instance_id,
'name': name,
'description': description,
'no_reboot': no_reboot}
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if 'device_name' not in device:
module.fail_json(msg = 'Device name must be set for volume')
device_name = device['device_name']
del device['device_name']
bd = BlockDeviceType(**device)
bdm[device_name] = bd
params['block_device_mapping'] = bdm
image_id = ec2.create_image(**params)
except boto.exception.BotoServerError, e:
if e.error_code == 'InvalidAMIName.Duplicate':
images = ec2.get_all_images()
for img in images:
if img.name == name:
module.exit_json(msg="AMI name already present", image_id=img.id, state=img.state, changed=False)
sys.exit(0)
else:
module.fail_json(msg="Error in retrieving duplicate AMI details")
else:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def create_image(module, ec2):
"""
Creates new AMI
module : AnsibleModule object
ec2: authenticated ec2 connection object
"""
instance_id = module.params.get('instance_id')
name = module.params.get('name')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
description = module.params.get('description')
no_reboot = module.params.get('no_reboot')
device_mapping = module.params.get('device_mapping')
tags = module.params.get('tags')
launch_permissions = module.params.get('launch_permissions')
try:
params = {'instance_id': instance_id,
'name': name,
'description': description,
'no_reboot': no_reboot}
images = ec2.get_all_images(filters={'name': name})
if images and images[0]:
module.exit_json(msg="AMI name already present", image_id=images[0].id, state=images[0].state, changed=False)
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if 'device_name' not in device:
module.fail_json(msg = 'Device name must be set for volume')
device_name = device['device_name']
del device['device_name']
bd = BlockDeviceType(**device)
bdm[device_name] = bd
params['block_device_mapping'] = bdm
image_id = ec2.create_image(**params)
except boto.exception.BotoServerError, e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def get_ec2_marketplace_prices(filter_region=None, filter_instance_type=None):
get_specific_region = (filter_region is not None)
pool = Pool(processes=WORKERS)
queue = Queue.Queue(maxsize=WORKERS)
result = {
"regions" : {}
}
for region in EC2_REGIONS:
if get_specific_region and filter_region != region :
continue
if not region in result["regions"] :
result["regions"][region] = []
verbose("Connecting to " + region)
ec2 = boto.ec2.connect_to_region(region)
verbose("Getting images")
images = ec2.get_all_images(owners=["aws-marketplace"])
totalCount = len(images)
currentImage = 1
region_result = result["regions"][region]
for image in images :
verbose("Processing " + image.id + " [" + str(currentImage) + "/" + str(totalCount) + "]")
currentImage = currentImage + 1
match = re.search(r".*-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})-ami.*", image.location)
if queue.full():
process_image(queue, region, region_result, filter_instance_type)
if match:
code = match.group(1)
if code in PRODUCT_CACHE:
process_image2(region, PRODUCT_CACHE[code], image, region_result, filter_instance_type)
else:
queue.put(pool.apply_async(prepare_product, [code, image]))
while not queue.empty() :
process_image(queue, region, region_result, filter_instance_type)
return result
def build_ami_list():
# AMI list
ami = dict()
for fname in os.listdir('ami'):
fname = os.path.join('ami', fname)
cfg = ConfigParser.ConfigParser()
cfg.read(fname)
s = str(ec2.region.name)
if cfg.has_section(s):
for o in cfg.options(s):
ami[o] = cfg.get(s, o)
# Add custom AMIs
for img in ec2.get_all_images(owners=[owner_id]):
n = img.location.split('/')[-1]
n = re.sub(r'\.manifest\.xml$', '', n)
n = to_slug(n)
ami[n] = str(img.id)
return ami
def build_ami_list():
# AMI list
ami = dict()
for fname in os.listdir('ami'):
fname = os.path.join('ami', fname)
cfg = ConfigParser.ConfigParser()
cfg.read(fname)
s = str(ec2.region.name)
if cfg.has_section(s):
for o in cfg.options(s):
ami[o] = cfg.get(s, o)
# Add custom AMIs
for img in ec2.get_all_images(owners=[owner_id]):
n = img.location.split('/')[-1]
n = re.sub(r'\.manifest\.xml$', '', n)
n = to_slug(n)
ami[n] = str(img.id)
return ami
def get_all_images_filtered(ec2, module):
"""Retrieve a list of EC2 images, based on user filters.
"""
ami_id = module.params.get('ami_id')
ami_tags = module.params.get('ami_tags')
architecture = module.params.get('architecture')
hypervisor = module.params.get('hypervisor')
is_public = module.params.get('is_public')
name = module.params.get('name')
owner = module.params.get('owner')
platform = module.params.get('platform')
state = module.params.get('state')
virtualization_type = module.params.get('virtualization_type')
filters = {'state': state}
if ami_id:
filters['image_id'] = ami_id
if ami_tags:
for tag in ami_tags:
filters['tag:' + tag] = ami_tags[tag]
if architecture:
filters['architecture'] = architecture
if hypervisor:
filters['hypervisor'] = hypervisor
if is_public:
filters['is_public'] = is_public
if name:
filters['name'] = name
if platform:
filters['platform'] = platform
if virtualization_type:
filters['virtualization_type'] = virtualization_type
images_result = ec2.get_all_images(owners=owner, filters=filters)
return [images_result, filters]
def get_all_images_filtered(ec2, module):
"""Retrieve a list of EC2 images, based on user filters.
"""
ami_id = module.params.get('ami_id')
ami_tags = module.params.get('ami_tags')
architecture = module.params.get('architecture')
hypervisor = module.params.get('hypervisor')
is_public = module.params.get('is_public')
name = module.params.get('name')
owner = module.params.get('owner')
platform = module.params.get('platform')
state = module.params.get('state')
virtualization_type = module.params.get('virtualization_type')
filters = {'state': state}
if ami_id:
filters['image_id'] = ami_id
if ami_tags:
for tag in ami_tags:
filters['tag:' + tag] = ami_tags[tag]
if architecture:
filters['architecture'] = architecture
if hypervisor:
filters['hypervisor'] = hypervisor
if is_public:
filters['is_public'] = is_public
if name:
filters['name'] = name
if platform:
filters['platform'] = platform
if virtualization_type:
filters['virtualization_type'] = virtualization_type
images_result = ec2.get_all_images(owners=owner, filters=filters)
return [images_result, filters]
def prepare_ami(region,
zone,
nodename,
es,
client,
image=None,
dag=False,
progress=None):
"""
Prepare client AMI
"""
if image is None:
image = "ethereum/client-%s" % client
global completed
if progress:
completed += 5
progress.update(completed)
# Get our Instance ID
inspect = json.loads(machine("inspect %s" % nodename))
instance_id = inspect['Driver']['InstanceId']
if progress:
completed += 5
progress.update(completed)
# Pull base image
pull_on(nodename, image)
if progress:
completed += 10
progress.update(completed)
# Create docker container w/ logstash-forwarder
# Build logstash-forwarder directly, docker-compose doesn't seem to
# like getting called concurrently
# with lcd('logstash-forwarder'):
# compose_on(nodename, "build")
build_on(nodename, "logstash-forwarder", "forwarder")
if progress:
completed += 20
progress.update(completed)
# Run logstash-forwarder, using run_on so we can pass --add-host
# with lcd('logstash-forwarder'):
# compose_on(nodename, "up -d") # ElasticSearch IP
run_on(nodename,
"forwarder", ("-d "
"-v /var/log/syslog:/var/log/syslog "
"--add-host logs.ethdev.com:%s "
"--restart always" % es),
name="forwarder")
if progress:
completed += 5
progress.update(completed)
# Generate DAG
if client == 'cpp':
ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag()
elif client == 'go':
ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag()
if dag and client != 'python':
generate_dag(nodename, client, image)
# FIXME For some reason, 'docker run' exits with 0
# but never returns, so somewhere between futures,
# Fabric and docker, there's an unhandled timeout
# while generating DAG caches and getting no output...
# We poll for 'Exited' in 'docker ps -a' and run with
# -d in generate_dag() for now...
dag_done = False
logging.info("Generating DAG on %s..." % nodename)
while dag_done is False:
time.sleep(5)
ps = docker_on(nodename, "ps -a")
if "Exited" in ps:
logging.info("DAG done on %s" % nodename)
dag_done = True
if progress:
completed += 20
progress.update(completed)
# Stop the instance
machine("stop %s" % nodename)
if progress:
completed += 5
progress.update(completed)
# Create EC2 connection with boto
ec2 = boto.ec2.connect_to_region(region)
# Cleanup old AMIs
images = ec2.get_all_images(filters={'tag:Name': "prepared-%s" % client})
for image in images:
image.deregister(delete_snapshot=True)
logger.info("Deleted AMI %s" % image.id)
# Create new AMI
ami_id = ec2.create_image(instance_id,
"prepared-%s" % client,
description="Prepared %s AMI" % client)
# Tag new AMI
image = ec2.get_all_images(image_ids=ami_id)[0]
image.add_tag("Name", "prepared-%s" % client)
if progress:
completed += 10
progress.update(completed)
# Wait until the image is ready
logger.info("Waiting for AMI to be available")
while image.state == 'pending':
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(5)
image.update()
if image.state == 'available':
return ami_id
else:
raise ValueError("Created AMI returned non-available state",
image.state)
return True if region.name.find(args.region) != -1 else False
# execute business logic
credentials = {'aws_access_key_id': args.aws_access_key_id, 'aws_secret_access_key': args.aws_secret_access_key}
heading = "Describing EBS images"
regions = boto.ec2.regions()
if args.region:
heading += " (filtered by region '" + args.region + "')"
regions = filter(isSelected, regions)
filters = None
if args.filter:
filters = dict([filter.split('=') for filter in args.filter])
log.info(args.filter)
log.debug(filters)
log.info(args.resource_ids)
print heading + ":"
for region in regions:
try:
ec2 = boto.connect_ec2(region=region, **credentials)
resources = ec2.get_all_images(image_ids=args.resource_ids, owners=['self'], filters=filters)
print region.name + ": " + str(len(resources)) + " images"
for resource in resources:
if args.verbose:
pprint(vars(resource))
else:
print resource.id
except boto.exception.BotoServerError, e:
log.error(e.error_message)
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(dict(
region = dict(required=True,
aliases = ['aws_region', 'ec2_region']),
owner = dict(required=False, default=None),
ami_id = dict(required=False),
ami_tags = dict(required=False, type='dict',
aliases = ['search_tags', 'image_tags']),
architecture = dict(required=False),
hypervisor = dict(required=False),
is_public = dict(required=False),
name = dict(required=False),
platform = dict(required=False),
sort = dict(required=False, default=None,
choices=['name', 'description', 'tag']),
sort_tag = dict(required=False),
sort_order = dict(required=False, default='ascending',
choices=['ascending', 'descending']),
sort_start = dict(required=False),
sort_end = dict(required=False),
state = dict(required=False, default='available'),
virtualization_type = dict(required=False),
no_result_action = dict(required=False, default='success',
choices = ['success', 'fail']),
)
)
module = AnsibleModule(
argument_spec=argument_spec,
)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module, install via pip or your package manager')
ami_id = module.params.get('ami_id')
ami_tags = module.params.get('ami_tags')
architecture = module.params.get('architecture')
hypervisor = module.params.get('hypervisor')
is_public = module.params.get('is_public')
name = module.params.get('name')
owner = module.params.get('owner')
platform = module.params.get('platform')
sort = module.params.get('sort')
sort_tag = module.params.get('sort_tag')
sort_order = module.params.get('sort_order')
sort_start = module.params.get('sort_start')
sort_end = module.params.get('sort_end')
state = module.params.get('state')
virtualization_type = module.params.get('virtualization_type')
no_result_action = module.params.get('no_result_action')
filter = {'state': state}
if ami_id:
filter['image_id'] = ami_id
if ami_tags:
for tag in ami_tags:
filter['tag:'+tag] = ami_tags[tag]
if architecture:
filter['architecture'] = architecture
if hypervisor:
filter['hypervisor'] = hypervisor
if is_public:
filter['is_public'] = is_public
if name:
filter['name'] = name
if platform:
filter['platform'] = platform
if virtualization_type:
filter['virtualization_type'] = virtualization_type
ec2 = ec2_connect(module)
images_result = ec2.get_all_images(owners=owner, filters=filter)
if no_result_action == 'fail' and len(images_result) == 0:
module.fail_json(msg="No AMIs matched the attributes: %s" % json.dumps(filter))
results = []
for image in images_result:
data = {
'ami_id': image.id,
'architecture': image.architecture,
'description': image.description,
'is_public': image.is_public,
'name': image.name,
'owner_id': image.owner_id,
'platform': image.platform,
'root_device_name': image.root_device_name,
'root_device_type': image.root_device_type,
'state': image.state,
'tags': image.tags,
'virtualization_type': image.virtualization_type,
}
if image.kernel_id:
data['kernel_id'] = image.kernel_id
if image.ramdisk_id:
data['ramdisk_id'] = image.ramdisk_id
results.append(data)
if sort == 'tag':
if not sort_tag:
module.fail_json(msg="'sort_tag' option must be given with 'sort=tag'")
results.sort(key=lambda e: e['tags'][sort_tag], reverse=(sort_order=='descending'))
elif sort:
results.sort(key=lambda e: e[sort], reverse=(sort_order=='descending'))
try:
if sort and sort_start and sort_end:
results = results[int(sort_start):int(sort_end)]
elif sort and sort_start:
results = results[int(sort_start):]
elif sort and sort_end:
results = results[:int(sort_end)]
except TypeError:
module.fail_json(msg="Please supply numeric values for sort_start and/or sort_end")
module.exit_json(results=results)
parser = argparse.ArgumentParser(description="AutoScale Launch Config updater")
parser.add_argument('--image_id', metavar='image_id')
parser.add_argument('--instance-type', metavar='instance_type')
parser.add_argument('--region', default='eu-west-1', metavar='region')
parser.add_argument('launch_config_name')
parser.add_argument('autoscale_group_name')
args = parser.parse_args()
if not args.image_id and not args.instance_type:
print("Specify at least one of image_id or instance_type")
sys.exit(0)
ec2 = ec2.connect_to_region(args.region)
try:
if args.image_id:
ec2.get_all_images(image_ids=[args.image_id])
except exception.EC2ResponseError:
print("It seems that '{0}' is not a valid image_id name or it does not exist ".format(args.image_id))
sys.exit(1)
autoscale = autoscale.connect_to_region(args.region)
try:
as_launch_config = autoscale.get_all_launch_configurations(names = [args.launch_config_name]).pop()
except IndexError:
print ("Couldn't found AutoScaling Launch Configuration")
sys.exit(1)
try:
as_group = autoscale.get_all_groups(names=[args.autoscale_group_name])[0]
except IndexError:
def create_image(module, ec2):
"""
Creates new AMI
module : AnsibleModule object
ec2: authenticated ec2 connection object
"""
instance_id = module.params.get("instance_id")
name = module.params.get("name")
wait = module.params.get("wait")
wait_timeout = int(module.params.get("wait_timeout"))
description = module.params.get("description")
no_reboot = module.params.get("no_reboot")
device_mapping = module.params.get("device_mapping")
tags = module.params.get("tags")
launch_permissions = module.params.get("launch_permissions")
try:
params = {"instance_id": instance_id, "name": name, "description": description, "no_reboot": no_reboot}
images = ec2.get_all_images(filters={"name": name})
if images and images[0]:
module.exit_json(
msg="AMI name already present", image_id=images[0].id, state=images[0].state, changed=False
)
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if "device_name" not in device:
module.fail_json(msg="Device name must be set for volume")
device_name = device["device_name"]
del device["device_name"]
bd = BlockDeviceType(**device)
bdm[device_name] = bd
params["block_device_mapping"] = bdm
image_id = ec2.create_image(**params)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
# Wait until the image is recognized. EC2 API has eventual consistency,
# such that a successful CreateImage API call doesn't guarantee the success
# of subsequent DescribeImages API call using the new image id returned.
for i in range(wait_timeout):
try:
img = ec2.get_image(image_id)
if img.state == "available":
break
elif img.state == "failed":
module.fail_json(msg="AMI creation failed, please see the AWS console for more details")
except boto.exception.EC2ResponseError as e:
if (
("InvalidAMIID.NotFound" not in e.error_code and "InvalidAMIID.Unavailable" not in e.error_code)
and wait
and i == wait_timeout - 1
):
module.fail_json(
msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help. %s: %s"
% (e.error_code, e.error_message)
)
finally:
time.sleep(1)
if img.state != "available":
module.fail_json(
msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help."
)
if tags:
try:
ec2.create_tags(image_id, tags)
except boto.exception.EC2ResponseError as e:
module.fail_json(msg="Image tagging failed => %s: %s" % (e.error_code, e.error_message))
if launch_permissions:
try:
img = ec2.get_image(image_id)
img.set_launch_permissions(**launch_permissions)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message), image_id=image_id)
module.exit_json(msg="AMI creation operation complete", changed=True, **get_ami_info(img))
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
region=dict(required=True, aliases=['aws_region', 'ec2_region']),
owner=dict(required=False, default=None),
ami_id=dict(required=False),
ami_tags=dict(required=False,
type='dict',
aliases=['search_tags', 'image_tags']),
architecture=dict(required=False),
hypervisor=dict(required=False),
is_public=dict(required=False),
name=dict(required=False),
platform=dict(required=False),
sort=dict(required=False,
default=None,
choices=['name', 'description', 'tag']),
sort_tag=dict(required=False),
sort_order=dict(required=False,
default='ascending',
choices=['ascending', 'descending']),
sort_start=dict(required=False),
sort_end=dict(required=False),
state=dict(required=False, default='available'),
virtualization_type=dict(required=False),
no_result_action=dict(required=False,
default='success',
choices=['success', 'fail']),
))
module = AnsibleModule(argument_spec=argument_spec, )
if not HAS_BOTO:
module.fail_json(
msg=
'boto required for this module, install via pip or your package manager'
)
ami_id = module.params.get('ami_id')
ami_tags = module.params.get('ami_tags')
architecture = module.params.get('architecture')
hypervisor = module.params.get('hypervisor')
is_public = module.params.get('is_public')
name = module.params.get('name')
owner = module.params.get('owner')
platform = module.params.get('platform')
sort = module.params.get('sort')
sort_tag = module.params.get('sort_tag')
sort_order = module.params.get('sort_order')
sort_start = module.params.get('sort_start')
sort_end = module.params.get('sort_end')
state = module.params.get('state')
virtualization_type = module.params.get('virtualization_type')
no_result_action = module.params.get('no_result_action')
filter = {'state': state}
if ami_id:
filter['image_id'] = ami_id
if ami_tags:
for tag in ami_tags:
filter['tag:' + tag] = ami_tags[tag]
if architecture:
filter['architecture'] = architecture
if hypervisor:
filter['hypervisor'] = hypervisor
if is_public:
filter['is_public'] = is_public
if name:
filter['name'] = name
if platform:
filter['platform'] = platform
if virtualization_type:
filter['virtualization_type'] = virtualization_type
ec2 = ec2_connect(module)
images_result = ec2.get_all_images(owners=owner, filters=filter)
if no_result_action == 'fail' and len(images_result) == 0:
module.fail_json(msg="No AMIs matched the attributes: %s" %
json.dumps(filter))
results = []
for image in images_result:
data = {
'ami_id': image.id,
'architecture': image.architecture,
'description': image.description,
'is_public': image.is_public,
'name': image.name,
'owner_id': image.owner_id,
'platform': image.platform,
'root_device_name': image.root_device_name,
'root_device_type': image.root_device_type,
'state': image.state,
'tags': image.tags,
'virtualization_type': image.virtualization_type,
}
if image.kernel_id:
data['kernel_id'] = image.kernel_id
if image.ramdisk_id:
data['ramdisk_id'] = image.ramdisk_id
results.append(data)
if sort == 'tag':
if not sort_tag:
module.fail_json(
msg="'sort_tag' option must be given with 'sort=tag'")
results.sort(key=lambda e: e['tags'][sort_tag],
reverse=(sort_order == 'descending'))
elif sort:
results.sort(key=lambda e: e[sort],
reverse=(sort_order == 'descending'))
try:
if sort and sort_start and sort_end:
results = results[int(sort_start):int(sort_end)]
elif sort and sort_start:
results = results[int(sort_start):]
elif sort and sort_end:
results = results[:int(sort_end)]
except TypeError:
module.fail_json(
msg="Please supply numeric values for sort_start and/or sort_end")
module.exit_json(results=results)
def get_my_images(self, filters=None): """Get list of AMI images owned by us""" ec2 = self.get_ec2() return ec2.get_all_images(owners=_aws_account_number, filters=filters)
parser = argparse.ArgumentParser(description="AutoScale Launch Config updater")
parser.add_argument('--image_id', metavar='image_id')
parser.add_argument('--instance-type', metavar='instance_type')
parser.add_argument('--region', default='eu-west-1', metavar='region')
parser.add_argument('launch_config_name')
parser.add_argument('autoscale_group_name')
args = parser.parse_args()
if not args.image_id and not args.instance_type:
print("Specify at least one of image_id or instance_type")
sys.exit(0)
ec2 = ec2.connect_to_region(args.region)
try:
if args.image_id:
ec2.get_all_images(image_ids=[args.image_id])
except exception.EC2ResponseError:
print(
"It seems that '{0}' is not a valid image_id name or it does not exist "
.format(args.image_id))
sys.exit(1)
autoscale = autoscale.connect_to_region(args.region)
try:
as_launch_config = autoscale.get_all_launch_configurations(
names=[args.launch_config_name]).pop()
except IndexError:
print("Couldn't found AutoScaling Launch Configuration")
sys.exit(1)
#
import boto.ec2
import boto.ec2.image
import boto.exception
import os
import re
import sys
ec2 = boto.ec2.connect_to_region(
'ap-northeast-1',
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'))
snapshot_to_ami = {}
for ami in ec2.get_all_images(owners=['self']):
for (device, volume) in ami.block_device_mapping.iteritems():
if volume.snapshot_id:
snapshot_to_ami[volume.snapshot_id] = ami
snapshots = dict([(snapshot.id, snapshot)
for snapshot in ec2.get_all_snapshots(owner='self')])
for (snapshot_id, snapshot) in snapshots.iteritems():
if not snapshot_to_ami.has_key(snapshot_id):
if snapshot.description: # confirmation
matched = re.search(r'ami-[0-9A-Fa-f]+', snapshot.description)
if matched:
ami = ec2.get_image(matched.group())
if ami:
continue
def create_image(module, ec2):
"""
Creates new AMI
module : AnsibleModule object
ec2: authenticated ec2 connection object
"""
instance_id = module.params.get('instance_id')
name = module.params.get('name')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
description = module.params.get('description')
architecture = module.params.get('architecture')
kernel_id = module.params.get('kernel_id')
root_device_name = module.params.get('root_device_name')
virtualization_type = module.params.get('virtualization_type')
no_reboot = module.params.get('no_reboot')
device_mapping = module.params.get('device_mapping')
tags = module.params.get('tags')
launch_permissions = module.params.get('launch_permissions')
try:
params = {'name': name,
'description': description}
images = ec2.get_all_images(filters={'name': name})
if images and images[0]:
# ensure that launch_permissions are up to date
update_image(module, ec2, images[0].id)
bdm = None
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if 'device_name' not in device:
module.fail_json(msg='Device name must be set for volume')
device_name = device['device_name']
del device['device_name']
bd = BlockDeviceType(**device)
bdm[device_name] = bd
if instance_id:
params['instance_id'] = instance_id
params['no_reboot'] = no_reboot
if bdm:
params['block_device_mapping'] = bdm
image_id = ec2.create_image(**params)
else:
params['architecture'] = architecture
params['virtualization_type'] = virtualization_type
if kernel_id:
params['kernel_id'] = kernel_id
if root_device_name:
params['root_device_name'] = root_device_name
if bdm:
params['block_device_map'] = bdm
image_id = ec2.register_image(**params)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
# Wait until the image is recognized. EC2 API has eventual consistency,
# such that a successful CreateImage API call doesn't guarantee the success
# of subsequent DescribeImages API call using the new image id returned.
for i in range(wait_timeout):
try:
img = ec2.get_image(image_id)
if img.state == 'available':
break
elif img.state == 'failed':
module.fail_json(msg="AMI creation failed, please see the AWS console for more details")
except boto.exception.EC2ResponseError as e:
if ('InvalidAMIID.NotFound' not in e.error_code and 'InvalidAMIID.Unavailable' not in e.error_code) and wait and i == wait_timeout - 1:
module.fail_json(msg="Error while trying to find the new image. Using wait=yes and/or a longer "
"wait_timeout may help. %s: %s" % (e.error_code, e.error_message))
finally:
time.sleep(1)
if img.state != 'available':
module.fail_json(msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help.")
if tags:
try:
ec2.create_tags(image_id, tags)
except boto.exception.EC2ResponseError as e:
module.fail_json(msg="Image tagging failed => %s: %s" % (e.error_code, e.error_message))
if launch_permissions:
try:
img = ec2.get_image(image_id)
img.set_launch_permissions(**launch_permissions)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message), image_id=image_id)
module.exit_json(msg="AMI creation operation complete", changed=True, **get_ami_info(img))
#print instance.__dict__.keys()
instance_name = instance.tags['Name']
instance_id = instance.id
print "\n" + server_name + ": " + instance_name + " (" + instance_id + ")"
current_datetime = datetime.datetime.now()
date_stamp = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
ami_name = instance_name + signature + date_stamp
try:
ami_id = instance.create_image(ami_name, description='Created by AMIBackup', no_reboot=True, dry_run=False)
except Exception, e:
logger.error("Backup " + server_name + ": " + e.message)
continue
logger.info("Backup " + server_name + ": " + ami_name)
print "AMI creation started"
print "AMI name: " + ami_name
images = ec2.get_all_images(image_ids = ami_id)
image = images[0]
image.add_tag("Name", ami_name)
# deregister old images
print "Deletion of old AMIs"
images = ec2.get_all_images(filters = {'tag:Name':server_pattern + signature + '*'})
for image in images:
image_name = image.tags['Name']
image_id = image.id
image_stamp = image_name.replace(server_pattern + signature, "")
image_timestamp = mktime(time.strptime(image_stamp, "%Y-%m-%d_%H-%M-%S"))
current_timestamp = mktime(current_datetime.timetuple())
diff_minutes = (current_timestamp - image_timestamp) / 60
if ( diff_minutes > backup_retention ):
image.deregister(delete_snapshot=True, dry_run=False)
def get_my_images(): return ec2.get_all_images(owners=['self'])
parser = argparse.ArgumentParser(description='Describe EC2 images in all/some available EC2 regions',
parents=[bc.build_region_parser(), bc.build_filter_parser('EC2 image'), bc.build_common_parser()])
args = parser.parse_args()
# process common command line arguments
log = logging.getLogger('botocross')
bc.configure_logging(log, args.log_level)
credentials = bc.parse_credentials(args)
regions = bc.filter_regions(boto.ec2.regions(), args.region)
filter = bc.build_filter(args.filter, args.exclude)
log.info(args.resource_ids)
# execute business logic
log.info("Describing EC2 images")
for region in regions:
try:
ec2 = boto.connect_ec2(region=region, **credentials)
images = ec2.get_all_images(image_ids=args.resource_ids, owners=['self'], filters=filter['filters'])
if filter['excludes']:
exclusions = ec2.get_all_images(owners=['self'], filters=filter['excludes'])
images = bc.filter_list_by_attribute(images, exclusions, 'id')
print region.name + ": " + str(len(images)) + " images"
for image in images:
if args.verbose:
pprint(vars(image))
else:
print image.id
except boto.exception.BotoServerError, e:
log.error(e.error_message)
def prepare_ami(region, zone, nodename, es, client, image=None, dag=False, progress=None):
"""
Prepare client AMI
"""
if image is None:
image = "ethereum/client-%s" % client
global completed
if progress:
completed += 5
progress.update(completed)
# Get our Instance ID
inspect = json.loads(machine("inspect %s" % nodename))
instance_id = inspect['Driver']['InstanceId']
if progress:
completed += 5
progress.update(completed)
# Pull base image
pull_on(nodename, image)
if progress:
completed += 10
progress.update(completed)
# Create docker container w/ logstash-forwarder
# Build logstash-forwarder directly, docker-compose doesn't seem to
# like getting called concurrently
# with lcd('logstash-forwarder'):
# compose_on(nodename, "build")
build_on(nodename, "logstash-forwarder", "forwarder")
if progress:
completed += 20
progress.update(completed)
# Run logstash-forwarder, using run_on so we can pass --add-host
# with lcd('logstash-forwarder'):
# compose_on(nodename, "up -d") # ElasticSearch IP
run_on(
nodename,
"forwarder",
("-d "
"-v /var/log/syslog:/var/log/syslog "
"--add-host logs.ethdev.com:%s "
"--restart always" % es),
name="forwarder")
if progress:
completed += 5
progress.update(completed)
# Generate DAG
if client == 'cpp':
ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag()
elif client == 'go':
ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag()
if dag and client != 'python':
generate_dag(nodename, client, image)
# FIXME For some reason, 'docker run' exits with 0
# but never returns, so somewhere between futures,
# Fabric and docker, there's an unhandled timeout
# while generating DAG caches and getting no output...
# We poll for 'Exited' in 'docker ps -a' and run with
# -d in generate_dag() for now...
dag_done = False
logging.info("Generating DAG on %s..." % nodename)
while dag_done is False:
time.sleep(5)
ps = docker_on(nodename, "ps -a")
if "Exited" in ps:
logging.info("DAG done on %s" % nodename)
dag_done = True
if progress:
completed += 20
progress.update(completed)
# Stop the instance
machine("stop %s" % nodename)
if progress:
completed += 5
progress.update(completed)
# Create EC2 connection with boto
ec2 = boto.ec2.connect_to_region(region)
# Cleanup old AMIs
images = ec2.get_all_images(filters={'tag:Name': "prepared-%s" % client})
for image in images:
image.deregister(delete_snapshot=True)
logger.info("Deleted AMI %s" % image.id)
# Create new AMI
ami_id = ec2.create_image(instance_id, "prepared-%s" % client, description="Prepared %s AMI" % client)
# Tag new AMI
image = ec2.get_all_images(image_ids=ami_id)[0]
image.add_tag("Name", "prepared-%s" % client)
if progress:
completed += 10
progress.update(completed)
# Wait until the image is ready
logger.info("Waiting for AMI to be available")
while image.state == 'pending':
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(5)
image.update()
if image.state == 'available':
return ami_id
else:
raise ValueError("Created AMI returned non-available state", image.state)
def create_image(module, ec2):
"""
Creates new AMI
module : AnsibleModule object
ec2: authenticated ec2 connection object
"""
instance_id = module.params.get('instance_id')
name = module.params.get('name')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
description = module.params.get('description')
architecture = module.params.get('architecture')
kernel_id = module.params.get('kernel_id')
root_device_name = module.params.get('root_device_name')
virtualization_type = module.params.get('virtualization_type')
no_reboot = module.params.get('no_reboot')
device_mapping = module.params.get('device_mapping')
tags = module.params.get('tags')
launch_permissions = module.params.get('launch_permissions')
try:
params = {'name': name, 'description': description}
images = ec2.get_all_images(filters={'name': name})
if images and images[0]:
# ensure that launch_permissions are up to date
update_image(module, ec2, images[0].id)
bdm = None
if device_mapping:
bdm = BlockDeviceMapping()
for device in device_mapping:
if 'device_name' not in device:
module.fail_json(msg='Device name must be set for volume')
device_name = device['device_name']
del device['device_name']
bd = BlockDeviceType(**device)
bdm[device_name] = bd
if instance_id:
params['instance_id'] = instance_id
params['no_reboot'] = no_reboot
if bdm:
params['block_device_mapping'] = bdm
image_id = ec2.create_image(**params)
else:
params['architecture'] = architecture
params['virtualization_type'] = virtualization_type
if kernel_id:
params['kernel_id'] = kernel_id
if root_device_name:
params['root_device_name'] = root_device_name
if bdm:
params['block_device_map'] = bdm
image_id = ec2.register_image(**params)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
# Wait until the image is recognized. EC2 API has eventual consistency,
# such that a successful CreateImage API call doesn't guarantee the success
# of subsequent DescribeImages API call using the new image id returned.
for i in range(wait_timeout):
try:
img = ec2.get_image(image_id)
if img.state == 'available':
break
elif img.state == 'failed':
module.fail_json(
msg=
"AMI creation failed, please see the AWS console for more details"
)
except boto.exception.EC2ResponseError as e:
if ('InvalidAMIID.NotFound' not in e.error_code
and 'InvalidAMIID.Unavailable'
not in e.error_code) and wait and i == wait_timeout - 1:
module.fail_json(
msg=
"Error while trying to find the new image. Using wait=yes and/or a longer "
"wait_timeout may help. %s: %s" %
(e.error_code, e.error_message))
finally:
time.sleep(1)
if img.state != 'available':
module.fail_json(
msg=
"Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help."
)
if tags:
try:
ec2.create_tags(image_id, tags)
except boto.exception.EC2ResponseError as e:
module.fail_json(msg="Image tagging failed => %s: %s" %
(e.error_code, e.error_message))
if launch_permissions:
try:
img = ec2.get_image(image_id)
img.set_launch_permissions(**launch_permissions)
except boto.exception.BotoServerError as e:
module.fail_json(msg="%s: %s" % (e.error_code, e.error_message),
image_id=image_id)
module.exit_json(msg="AMI creation operation complete",
changed=True,
**get_ami_info(img))
def get_images():
ec2 = get_aws_connection('ec2')
filters = {'tag:tcm': 'image'}
images = ec2.get_all_images(owners=['self'], filters=filters)
return images
