def validate_image_id(self, image_id): ec2 = self.connect_ec2() try: ec2.get_all_images(image_ids=[image_id]) return True, '' except boto.exception.EC2ResponseError as e: return False, e.error_message
def has_image(self, image_id): """ Checks to see if the given ami is available in the account """ ec2 = self.connect_ec2() try: ec2.get_all_images(image_id) return True, '' except boto.exception.EC2ResponseError: return False, 'The image id \'{0}\' does not exist in this ' \ 'account.'.format(image_id)
def _lookup_ami_id(ec2, name): """Returns AMI id for `name`""" images = ec2.get_all_images(filters={'name': [name]}) if len(images) != 1: raise RuntimeError('cannot find exactly one image') return images[0]
def process_region(region): print "Processing region %s" % region.name ec2 = region.connect() print "Scanning instances...\n" instances = ec2.get_only_instances() for inst in instances: inst_accessor = Instance(inst) process_instance(inst_accessor) print "\nScanning images...\n" images = ec2.get_all_images(None, ("self")) for image in images: image_accessor = Image(image) process_image(image_accessor) print "\nScanning volumes...\n" volumes = ec2.get_all_volumes() for volume in volumes: volume_accessor = Volume(volume) process_volume(volume_accessor) print "\nScanning snapshots...\n" snapshots = ec2.get_all_snapshots(owner="self") for snapshot in snapshots: process_snapshot(Snapshot(snapshot))
def ec2_deregister_ami(ec2): images = ec2.get_all_images(owners="self") for i in images: try: print "---Deregistering AMI: " + i.id ec2.deregister_image(i.id, delete_snapshot=True) except Exception, e: print(e)
def register_image(module, ec2): """ Registers AMI """ name = module.params.get('name') description = module.params.get('description') image_location = module.params.get('image_location') architecture = module.params.get('architecture') kernel_id = module.params.get('kernel_id') root_device_name = module.params.get('root_device_name') virtualization_type = module.params.get('virtualization_type') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) device_mapping = module.params.get('device_mapping') try: params = {'name': name, 'description': description, 'architecture': architecture, 'virtualization_type': virtualization_type} if image_location: params['image_location'] = image_location if kernel_id: params['kernel_id'] = kernel_id if root_device_name: params['root_device_name'] = root_device_name if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if 'device_name' not in device: module.fail_json(msg = 'Device name must be set for volume') device_name = device['device_name'] del device['device_name'] bd = BlockDeviceType(**device) bdm[device_name] = bd params['block_device_map'] = bdm image_id = ec2.register_image(**params) except boto.exception.BotoServerError, e: if e.error_code == 'InvalidAMIName.Duplicate': images = ec2.get_all_images() for img in images: if img.name == name: module.exit_json(msg="AMI name already present", image_id=img.id, state=img.state, changed=False) sys.exit(0) else: module.fail_json(msg="Error in retrieving duplicate AMI details") else: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def get_blessed_ami(): images = ec2.get_all_images( filters={ 'tag:environment': args.environment, 'tag:deployment': args.deployment, 'tag:play': args.play, 'tag:blessed': True } ) if len(images) != 1: raise Exception("ERROR: Expected only one blessed ami, got {}\n".format( len(images))) return images[0].id
def create_image(module, ec2): """ Creates new AMI module : AnsibleModule object ec2: authenticated ec2 connection object """ instance_id = module.params.get('instance_id') name = module.params.get('name') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) description = module.params.get('description') no_reboot = module.params.get('no_reboot') device_mapping = module.params.get('device_mapping') tags = module.params.get('tags') launch_permissions = module.params.get('launch_permissions') try: params = {'instance_id': instance_id, 'name': name, 'description': description, 'no_reboot': no_reboot} if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if 'device_name' not in device: module.fail_json(msg = 'Device name must be set for volume') device_name = device['device_name'] del device['device_name'] bd = BlockDeviceType(**device) bdm[device_name] = bd params['block_device_mapping'] = bdm image_id = ec2.create_image(**params) except boto.exception.BotoServerError, e: if e.error_code == 'InvalidAMIName.Duplicate': images = ec2.get_all_images() for img in images: if img.name == name: module.exit_json(msg="AMI name already present", image_id=img.id, state=img.state, changed=False) sys.exit(0) else: module.fail_json(msg="Error in retrieving duplicate AMI details") else: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def create_image(module, ec2): """ Creates new AMI module : AnsibleModule object ec2: authenticated ec2 connection object """ instance_id = module.params.get('instance_id') name = module.params.get('name') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) description = module.params.get('description') no_reboot = module.params.get('no_reboot') device_mapping = module.params.get('device_mapping') tags = module.params.get('tags') launch_permissions = module.params.get('launch_permissions') try: params = {'instance_id': instance_id, 'name': name, 'description': description, 'no_reboot': no_reboot} images = ec2.get_all_images(filters={'name': name}) if images and images[0]: module.exit_json(msg="AMI name already present", image_id=images[0].id, state=images[0].state, changed=False) if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if 'device_name' not in device: module.fail_json(msg = 'Device name must be set for volume') device_name = device['device_name'] del device['device_name'] bd = BlockDeviceType(**device) bdm[device_name] = bd params['block_device_mapping'] = bdm image_id = ec2.create_image(**params) except boto.exception.BotoServerError, e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message))
def get_ec2_marketplace_prices(filter_region=None, filter_instance_type=None): get_specific_region = (filter_region is not None) pool = Pool(processes=WORKERS) queue = Queue.Queue(maxsize=WORKERS) result = { "regions" : {} } for region in EC2_REGIONS: if get_specific_region and filter_region != region : continue if not region in result["regions"] : result["regions"][region] = [] verbose("Connecting to " + region) ec2 = boto.ec2.connect_to_region(region) verbose("Getting images") images = ec2.get_all_images(owners=["aws-marketplace"]) totalCount = len(images) currentImage = 1 region_result = result["regions"][region] for image in images : verbose("Processing " + image.id + " [" + str(currentImage) + "/" + str(totalCount) + "]") currentImage = currentImage + 1 match = re.search(r".*-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})-ami.*", image.location) if queue.full(): process_image(queue, region, region_result, filter_instance_type) if match: code = match.group(1) if code in PRODUCT_CACHE: process_image2(region, PRODUCT_CACHE[code], image, region_result, filter_instance_type) else: queue.put(pool.apply_async(prepare_product, [code, image])) while not queue.empty() : process_image(queue, region, region_result, filter_instance_type) return result
def build_ami_list(): # AMI list ami = dict() for fname in os.listdir('ami'): fname = os.path.join('ami', fname) cfg = ConfigParser.ConfigParser() cfg.read(fname) s = str(ec2.region.name) if cfg.has_section(s): for o in cfg.options(s): ami[o] = cfg.get(s, o) # Add custom AMIs for img in ec2.get_all_images(owners=[owner_id]): n = img.location.split('/')[-1] n = re.sub(r'\.manifest\.xml$', '', n) n = to_slug(n) ami[n] = str(img.id) return ami
def get_all_images_filtered(ec2, module): """Retrieve a list of EC2 images, based on user filters. """ ami_id = module.params.get('ami_id') ami_tags = module.params.get('ami_tags') architecture = module.params.get('architecture') hypervisor = module.params.get('hypervisor') is_public = module.params.get('is_public') name = module.params.get('name') owner = module.params.get('owner') platform = module.params.get('platform') state = module.params.get('state') virtualization_type = module.params.get('virtualization_type') filters = {'state': state} if ami_id: filters['image_id'] = ami_id if ami_tags: for tag in ami_tags: filters['tag:' + tag] = ami_tags[tag] if architecture: filters['architecture'] = architecture if hypervisor: filters['hypervisor'] = hypervisor if is_public: filters['is_public'] = is_public if name: filters['name'] = name if platform: filters['platform'] = platform if virtualization_type: filters['virtualization_type'] = virtualization_type images_result = ec2.get_all_images(owners=owner, filters=filters) return [images_result, filters]
def prepare_ami(region, zone, nodename, es, client, image=None, dag=False, progress=None): """ Prepare client AMI """ if image is None: image = "ethereum/client-%s" % client global completed if progress: completed += 5 progress.update(completed) # Get our Instance ID inspect = json.loads(machine("inspect %s" % nodename)) instance_id = inspect['Driver']['InstanceId'] if progress: completed += 5 progress.update(completed) # Pull base image pull_on(nodename, image) if progress: completed += 10 progress.update(completed) # Create docker container w/ logstash-forwarder # Build logstash-forwarder directly, docker-compose doesn't seem to # like getting called concurrently # with lcd('logstash-forwarder'): # compose_on(nodename, "build") build_on(nodename, "logstash-forwarder", "forwarder") if progress: completed += 20 progress.update(completed) # Run logstash-forwarder, using run_on so we can pass --add-host # with lcd('logstash-forwarder'): # compose_on(nodename, "up -d") # ElasticSearch IP run_on(nodename, "forwarder", ("-d " "-v /var/log/syslog:/var/log/syslog " "--add-host logs.ethdev.com:%s " "--restart always" % es), name="forwarder") if progress: completed += 5 progress.update(completed) # Generate DAG if client == 'cpp': ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag() elif client == 'go': ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag() if dag and client != 'python': generate_dag(nodename, client, image) # FIXME For some reason, 'docker run' exits with 0 # but never returns, so somewhere between futures, # Fabric and docker, there's an unhandled timeout # while generating DAG caches and getting no output... # We poll for 'Exited' in 'docker ps -a' and run with # -d in generate_dag() for now... dag_done = False logging.info("Generating DAG on %s..." % nodename) while dag_done is False: time.sleep(5) ps = docker_on(nodename, "ps -a") if "Exited" in ps: logging.info("DAG done on %s" % nodename) dag_done = True if progress: completed += 20 progress.update(completed) # Stop the instance machine("stop %s" % nodename) if progress: completed += 5 progress.update(completed) # Create EC2 connection with boto ec2 = boto.ec2.connect_to_region(region) # Cleanup old AMIs images = ec2.get_all_images(filters={'tag:Name': "prepared-%s" % client}) for image in images: image.deregister(delete_snapshot=True) logger.info("Deleted AMI %s" % image.id) # Create new AMI ami_id = ec2.create_image(instance_id, "prepared-%s" % client, description="Prepared %s AMI" % client) # Tag new AMI image = ec2.get_all_images(image_ids=ami_id)[0] image.add_tag("Name", "prepared-%s" % client) if progress: completed += 10 progress.update(completed) # Wait until the image is ready logger.info("Waiting for AMI to be available") while image.state == 'pending': sys.stdout.write('.') sys.stdout.flush() time.sleep(5) image.update() if image.state == 'available': return ami_id else: raise ValueError("Created AMI returned non-available state", image.state)
return True if region.name.find(args.region) != -1 else False # execute business logic credentials = {'aws_access_key_id': args.aws_access_key_id, 'aws_secret_access_key': args.aws_secret_access_key} heading = "Describing EBS images" regions = boto.ec2.regions() if args.region: heading += " (filtered by region '" + args.region + "')" regions = filter(isSelected, regions) filters = None if args.filter: filters = dict([filter.split('=') for filter in args.filter]) log.info(args.filter) log.debug(filters) log.info(args.resource_ids) print heading + ":" for region in regions: try: ec2 = boto.connect_ec2(region=region, **credentials) resources = ec2.get_all_images(image_ids=args.resource_ids, owners=['self'], filters=filters) print region.name + ": " + str(len(resources)) + " images" for resource in resources: if args.verbose: pprint(vars(resource)) else: print resource.id except boto.exception.BotoServerError, e: log.error(e.error_message)
def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( region = dict(required=True, aliases = ['aws_region', 'ec2_region']), owner = dict(required=False, default=None), ami_id = dict(required=False), ami_tags = dict(required=False, type='dict', aliases = ['search_tags', 'image_tags']), architecture = dict(required=False), hypervisor = dict(required=False), is_public = dict(required=False), name = dict(required=False), platform = dict(required=False), sort = dict(required=False, default=None, choices=['name', 'description', 'tag']), sort_tag = dict(required=False), sort_order = dict(required=False, default='ascending', choices=['ascending', 'descending']), sort_start = dict(required=False), sort_end = dict(required=False), state = dict(required=False, default='available'), virtualization_type = dict(required=False), no_result_action = dict(required=False, default='success', choices = ['success', 'fail']), ) ) module = AnsibleModule( argument_spec=argument_spec, ) if not HAS_BOTO: module.fail_json(msg='boto required for this module, install via pip or your package manager') ami_id = module.params.get('ami_id') ami_tags = module.params.get('ami_tags') architecture = module.params.get('architecture') hypervisor = module.params.get('hypervisor') is_public = module.params.get('is_public') name = module.params.get('name') owner = module.params.get('owner') platform = module.params.get('platform') sort = module.params.get('sort') sort_tag = module.params.get('sort_tag') sort_order = module.params.get('sort_order') sort_start = module.params.get('sort_start') sort_end = module.params.get('sort_end') state = module.params.get('state') virtualization_type = module.params.get('virtualization_type') no_result_action = module.params.get('no_result_action') filter = {'state': state} if ami_id: filter['image_id'] = ami_id if ami_tags: for tag in ami_tags: filter['tag:'+tag] = ami_tags[tag] if architecture: filter['architecture'] = architecture if hypervisor: filter['hypervisor'] = hypervisor if is_public: filter['is_public'] = is_public if name: filter['name'] = name if platform: filter['platform'] = platform if virtualization_type: filter['virtualization_type'] = virtualization_type ec2 = ec2_connect(module) images_result = ec2.get_all_images(owners=owner, filters=filter) if no_result_action == 'fail' and len(images_result) == 0: module.fail_json(msg="No AMIs matched the attributes: %s" % json.dumps(filter)) results = [] for image in images_result: data = { 'ami_id': image.id, 'architecture': image.architecture, 'description': image.description, 'is_public': image.is_public, 'name': image.name, 'owner_id': image.owner_id, 'platform': image.platform, 'root_device_name': image.root_device_name, 'root_device_type': image.root_device_type, 'state': image.state, 'tags': image.tags, 'virtualization_type': image.virtualization_type, } if image.kernel_id: data['kernel_id'] = image.kernel_id if image.ramdisk_id: data['ramdisk_id'] = image.ramdisk_id results.append(data) if sort == 'tag': if not sort_tag: module.fail_json(msg="'sort_tag' option must be given with 'sort=tag'") results.sort(key=lambda e: e['tags'][sort_tag], reverse=(sort_order=='descending')) elif sort: results.sort(key=lambda e: e[sort], reverse=(sort_order=='descending')) try: if sort and sort_start and sort_end: results = results[int(sort_start):int(sort_end)] elif sort and sort_start: results = results[int(sort_start):] elif sort and sort_end: results = results[:int(sort_end)] except TypeError: module.fail_json(msg="Please supply numeric values for sort_start and/or sort_end") module.exit_json(results=results)
parser = argparse.ArgumentParser(description="AutoScale Launch Config updater") parser.add_argument('--image_id', metavar='image_id') parser.add_argument('--instance-type', metavar='instance_type') parser.add_argument('--region', default='eu-west-1', metavar='region') parser.add_argument('launch_config_name') parser.add_argument('autoscale_group_name') args = parser.parse_args() if not args.image_id and not args.instance_type: print("Specify at least one of image_id or instance_type") sys.exit(0) ec2 = ec2.connect_to_region(args.region) try: if args.image_id: ec2.get_all_images(image_ids=[args.image_id]) except exception.EC2ResponseError: print("It seems that '{0}' is not a valid image_id name or it does not exist ".format(args.image_id)) sys.exit(1) autoscale = autoscale.connect_to_region(args.region) try: as_launch_config = autoscale.get_all_launch_configurations(names = [args.launch_config_name]).pop() except IndexError: print ("Couldn't found AutoScaling Launch Configuration") sys.exit(1) try: as_group = autoscale.get_all_groups(names=[args.autoscale_group_name])[0] except IndexError:
def create_image(module, ec2): """ Creates new AMI module : AnsibleModule object ec2: authenticated ec2 connection object """ instance_id = module.params.get("instance_id") name = module.params.get("name") wait = module.params.get("wait") wait_timeout = int(module.params.get("wait_timeout")) description = module.params.get("description") no_reboot = module.params.get("no_reboot") device_mapping = module.params.get("device_mapping") tags = module.params.get("tags") launch_permissions = module.params.get("launch_permissions") try: params = {"instance_id": instance_id, "name": name, "description": description, "no_reboot": no_reboot} images = ec2.get_all_images(filters={"name": name}) if images and images[0]: module.exit_json( msg="AMI name already present", image_id=images[0].id, state=images[0].state, changed=False ) if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if "device_name" not in device: module.fail_json(msg="Device name must be set for volume") device_name = device["device_name"] del device["device_name"] bd = BlockDeviceType(**device) bdm[device_name] = bd params["block_device_mapping"] = bdm image_id = ec2.create_image(**params) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message)) # Wait until the image is recognized. EC2 API has eventual consistency, # such that a successful CreateImage API call doesn't guarantee the success # of subsequent DescribeImages API call using the new image id returned. for i in range(wait_timeout): try: img = ec2.get_image(image_id) if img.state == "available": break elif img.state == "failed": module.fail_json(msg="AMI creation failed, please see the AWS console for more details") except boto.exception.EC2ResponseError as e: if ( ("InvalidAMIID.NotFound" not in e.error_code and "InvalidAMIID.Unavailable" not in e.error_code) and wait and i == wait_timeout - 1 ): module.fail_json( msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help. %s: %s" % (e.error_code, e.error_message) ) finally: time.sleep(1) if img.state != "available": module.fail_json( msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help." ) if tags: try: ec2.create_tags(image_id, tags) except boto.exception.EC2ResponseError as e: module.fail_json(msg="Image tagging failed => %s: %s" % (e.error_code, e.error_message)) if launch_permissions: try: img = ec2.get_image(image_id) img.set_launch_permissions(**launch_permissions) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message), image_id=image_id) module.exit_json(msg="AMI creation operation complete", changed=True, **get_ami_info(img))
def main(): argument_spec = ec2_argument_spec() argument_spec.update( dict( region=dict(required=True, aliases=['aws_region', 'ec2_region']), owner=dict(required=False, default=None), ami_id=dict(required=False), ami_tags=dict(required=False, type='dict', aliases=['search_tags', 'image_tags']), architecture=dict(required=False), hypervisor=dict(required=False), is_public=dict(required=False), name=dict(required=False), platform=dict(required=False), sort=dict(required=False, default=None, choices=['name', 'description', 'tag']), sort_tag=dict(required=False), sort_order=dict(required=False, default='ascending', choices=['ascending', 'descending']), sort_start=dict(required=False), sort_end=dict(required=False), state=dict(required=False, default='available'), virtualization_type=dict(required=False), no_result_action=dict(required=False, default='success', choices=['success', 'fail']), )) module = AnsibleModule(argument_spec=argument_spec, ) if not HAS_BOTO: module.fail_json( msg= 'boto required for this module, install via pip or your package manager' ) ami_id = module.params.get('ami_id') ami_tags = module.params.get('ami_tags') architecture = module.params.get('architecture') hypervisor = module.params.get('hypervisor') is_public = module.params.get('is_public') name = module.params.get('name') owner = module.params.get('owner') platform = module.params.get('platform') sort = module.params.get('sort') sort_tag = module.params.get('sort_tag') sort_order = module.params.get('sort_order') sort_start = module.params.get('sort_start') sort_end = module.params.get('sort_end') state = module.params.get('state') virtualization_type = module.params.get('virtualization_type') no_result_action = module.params.get('no_result_action') filter = {'state': state} if ami_id: filter['image_id'] = ami_id if ami_tags: for tag in ami_tags: filter['tag:' + tag] = ami_tags[tag] if architecture: filter['architecture'] = architecture if hypervisor: filter['hypervisor'] = hypervisor if is_public: filter['is_public'] = is_public if name: filter['name'] = name if platform: filter['platform'] = platform if virtualization_type: filter['virtualization_type'] = virtualization_type ec2 = ec2_connect(module) images_result = ec2.get_all_images(owners=owner, filters=filter) if no_result_action == 'fail' and len(images_result) == 0: module.fail_json(msg="No AMIs matched the attributes: %s" % json.dumps(filter)) results = [] for image in images_result: data = { 'ami_id': image.id, 'architecture': image.architecture, 'description': image.description, 'is_public': image.is_public, 'name': image.name, 'owner_id': image.owner_id, 'platform': image.platform, 'root_device_name': image.root_device_name, 'root_device_type': image.root_device_type, 'state': image.state, 'tags': image.tags, 'virtualization_type': image.virtualization_type, } if image.kernel_id: data['kernel_id'] = image.kernel_id if image.ramdisk_id: data['ramdisk_id'] = image.ramdisk_id results.append(data) if sort == 'tag': if not sort_tag: module.fail_json( msg="'sort_tag' option must be given with 'sort=tag'") results.sort(key=lambda e: e['tags'][sort_tag], reverse=(sort_order == 'descending')) elif sort: results.sort(key=lambda e: e[sort], reverse=(sort_order == 'descending')) try: if sort and sort_start and sort_end: results = results[int(sort_start):int(sort_end)] elif sort and sort_start: results = results[int(sort_start):] elif sort and sort_end: results = results[:int(sort_end)] except TypeError: module.fail_json( msg="Please supply numeric values for sort_start and/or sort_end") module.exit_json(results=results)
def get_my_images(self, filters=None): """Get list of AMI images owned by us""" ec2 = self.get_ec2() return ec2.get_all_images(owners=_aws_account_number, filters=filters)
parser = argparse.ArgumentParser(description="AutoScale Launch Config updater") parser.add_argument('--image_id', metavar='image_id') parser.add_argument('--instance-type', metavar='instance_type') parser.add_argument('--region', default='eu-west-1', metavar='region') parser.add_argument('launch_config_name') parser.add_argument('autoscale_group_name') args = parser.parse_args() if not args.image_id and not args.instance_type: print("Specify at least one of image_id or instance_type") sys.exit(0) ec2 = ec2.connect_to_region(args.region) try: if args.image_id: ec2.get_all_images(image_ids=[args.image_id]) except exception.EC2ResponseError: print( "It seems that '{0}' is not a valid image_id name or it does not exist " .format(args.image_id)) sys.exit(1) autoscale = autoscale.connect_to_region(args.region) try: as_launch_config = autoscale.get_all_launch_configurations( names=[args.launch_config_name]).pop() except IndexError: print("Couldn't found AutoScaling Launch Configuration") sys.exit(1)
# import boto.ec2 import boto.ec2.image import boto.exception import os import re import sys ec2 = boto.ec2.connect_to_region( 'ap-northeast-1', aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY')) snapshot_to_ami = {} for ami in ec2.get_all_images(owners=['self']): for (device, volume) in ami.block_device_mapping.iteritems(): if volume.snapshot_id: snapshot_to_ami[volume.snapshot_id] = ami snapshots = dict([(snapshot.id, snapshot) for snapshot in ec2.get_all_snapshots(owner='self')]) for (snapshot_id, snapshot) in snapshots.iteritems(): if not snapshot_to_ami.has_key(snapshot_id): if snapshot.description: # confirmation matched = re.search(r'ami-[0-9A-Fa-f]+', snapshot.description) if matched: ami = ec2.get_image(matched.group()) if ami: continue
def create_image(module, ec2): """ Creates new AMI module : AnsibleModule object ec2: authenticated ec2 connection object """ instance_id = module.params.get('instance_id') name = module.params.get('name') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) description = module.params.get('description') architecture = module.params.get('architecture') kernel_id = module.params.get('kernel_id') root_device_name = module.params.get('root_device_name') virtualization_type = module.params.get('virtualization_type') no_reboot = module.params.get('no_reboot') device_mapping = module.params.get('device_mapping') tags = module.params.get('tags') launch_permissions = module.params.get('launch_permissions') try: params = {'name': name, 'description': description} images = ec2.get_all_images(filters={'name': name}) if images and images[0]: # ensure that launch_permissions are up to date update_image(module, ec2, images[0].id) bdm = None if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if 'device_name' not in device: module.fail_json(msg='Device name must be set for volume') device_name = device['device_name'] del device['device_name'] bd = BlockDeviceType(**device) bdm[device_name] = bd if instance_id: params['instance_id'] = instance_id params['no_reboot'] = no_reboot if bdm: params['block_device_mapping'] = bdm image_id = ec2.create_image(**params) else: params['architecture'] = architecture params['virtualization_type'] = virtualization_type if kernel_id: params['kernel_id'] = kernel_id if root_device_name: params['root_device_name'] = root_device_name if bdm: params['block_device_map'] = bdm image_id = ec2.register_image(**params) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message)) # Wait until the image is recognized. EC2 API has eventual consistency, # such that a successful CreateImage API call doesn't guarantee the success # of subsequent DescribeImages API call using the new image id returned. for i in range(wait_timeout): try: img = ec2.get_image(image_id) if img.state == 'available': break elif img.state == 'failed': module.fail_json(msg="AMI creation failed, please see the AWS console for more details") except boto.exception.EC2ResponseError as e: if ('InvalidAMIID.NotFound' not in e.error_code and 'InvalidAMIID.Unavailable' not in e.error_code) and wait and i == wait_timeout - 1: module.fail_json(msg="Error while trying to find the new image. Using wait=yes and/or a longer " "wait_timeout may help. %s: %s" % (e.error_code, e.error_message)) finally: time.sleep(1) if img.state != 'available': module.fail_json(msg="Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help.") if tags: try: ec2.create_tags(image_id, tags) except boto.exception.EC2ResponseError as e: module.fail_json(msg="Image tagging failed => %s: %s" % (e.error_code, e.error_message)) if launch_permissions: try: img = ec2.get_image(image_id) img.set_launch_permissions(**launch_permissions) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message), image_id=image_id) module.exit_json(msg="AMI creation operation complete", changed=True, **get_ami_info(img))
#print instance.__dict__.keys() instance_name = instance.tags['Name'] instance_id = instance.id print "\n" + server_name + ": " + instance_name + " (" + instance_id + ")" current_datetime = datetime.datetime.now() date_stamp = current_datetime.strftime("%Y-%m-%d_%H-%M-%S") ami_name = instance_name + signature + date_stamp try: ami_id = instance.create_image(ami_name, description='Created by AMIBackup', no_reboot=True, dry_run=False) except Exception, e: logger.error("Backup " + server_name + ": " + e.message) continue logger.info("Backup " + server_name + ": " + ami_name) print "AMI creation started" print "AMI name: " + ami_name images = ec2.get_all_images(image_ids = ami_id) image = images[0] image.add_tag("Name", ami_name) # deregister old images print "Deletion of old AMIs" images = ec2.get_all_images(filters = {'tag:Name':server_pattern + signature + '*'}) for image in images: image_name = image.tags['Name'] image_id = image.id image_stamp = image_name.replace(server_pattern + signature, "") image_timestamp = mktime(time.strptime(image_stamp, "%Y-%m-%d_%H-%M-%S")) current_timestamp = mktime(current_datetime.timetuple()) diff_minutes = (current_timestamp - image_timestamp) / 60 if ( diff_minutes > backup_retention ): image.deregister(delete_snapshot=True, dry_run=False)
def get_my_images(): return ec2.get_all_images(owners=['self'])
parser = argparse.ArgumentParser(description='Describe EC2 images in all/some available EC2 regions', parents=[bc.build_region_parser(), bc.build_filter_parser('EC2 image'), bc.build_common_parser()]) args = parser.parse_args() # process common command line arguments log = logging.getLogger('botocross') bc.configure_logging(log, args.log_level) credentials = bc.parse_credentials(args) regions = bc.filter_regions(boto.ec2.regions(), args.region) filter = bc.build_filter(args.filter, args.exclude) log.info(args.resource_ids) # execute business logic log.info("Describing EC2 images") for region in regions: try: ec2 = boto.connect_ec2(region=region, **credentials) images = ec2.get_all_images(image_ids=args.resource_ids, owners=['self'], filters=filter['filters']) if filter['excludes']: exclusions = ec2.get_all_images(owners=['self'], filters=filter['excludes']) images = bc.filter_list_by_attribute(images, exclusions, 'id') print region.name + ": " + str(len(images)) + " images" for image in images: if args.verbose: pprint(vars(image)) else: print image.id except boto.exception.BotoServerError, e: log.error(e.error_message)
def prepare_ami(region, zone, nodename, es, client, image=None, dag=False, progress=None): """ Prepare client AMI """ if image is None: image = "ethereum/client-%s" % client global completed if progress: completed += 5 progress.update(completed) # Get our Instance ID inspect = json.loads(machine("inspect %s" % nodename)) instance_id = inspect['Driver']['InstanceId'] if progress: completed += 5 progress.update(completed) # Pull base image pull_on(nodename, image) if progress: completed += 10 progress.update(completed) # Create docker container w/ logstash-forwarder # Build logstash-forwarder directly, docker-compose doesn't seem to # like getting called concurrently # with lcd('logstash-forwarder'): # compose_on(nodename, "build") build_on(nodename, "logstash-forwarder", "forwarder") if progress: completed += 20 progress.update(completed) # Run logstash-forwarder, using run_on so we can pass --add-host # with lcd('logstash-forwarder'): # compose_on(nodename, "up -d") # ElasticSearch IP run_on( nodename, "forwarder", ("-d " "-v /var/log/syslog:/var/log/syslog " "--add-host logs.ethdev.com:%s " "--restart always" % es), name="forwarder") if progress: completed += 5 progress.update(completed) # Generate DAG if client == 'cpp': ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag() elif client == 'go': ssh_on(nodename, "sudo mkdir /opt/dag") # see generate_dag() if dag and client != 'python': generate_dag(nodename, client, image) # FIXME For some reason, 'docker run' exits with 0 # but never returns, so somewhere between futures, # Fabric and docker, there's an unhandled timeout # while generating DAG caches and getting no output... # We poll for 'Exited' in 'docker ps -a' and run with # -d in generate_dag() for now... dag_done = False logging.info("Generating DAG on %s..." % nodename) while dag_done is False: time.sleep(5) ps = docker_on(nodename, "ps -a") if "Exited" in ps: logging.info("DAG done on %s" % nodename) dag_done = True if progress: completed += 20 progress.update(completed) # Stop the instance machine("stop %s" % nodename) if progress: completed += 5 progress.update(completed) # Create EC2 connection with boto ec2 = boto.ec2.connect_to_region(region) # Cleanup old AMIs images = ec2.get_all_images(filters={'tag:Name': "prepared-%s" % client}) for image in images: image.deregister(delete_snapshot=True) logger.info("Deleted AMI %s" % image.id) # Create new AMI ami_id = ec2.create_image(instance_id, "prepared-%s" % client, description="Prepared %s AMI" % client) # Tag new AMI image = ec2.get_all_images(image_ids=ami_id)[0] image.add_tag("Name", "prepared-%s" % client) if progress: completed += 10 progress.update(completed) # Wait until the image is ready logger.info("Waiting for AMI to be available") while image.state == 'pending': sys.stdout.write('.') sys.stdout.flush() time.sleep(5) image.update() if image.state == 'available': return ami_id else: raise ValueError("Created AMI returned non-available state", image.state)
def create_image(module, ec2): """ Creates new AMI module : AnsibleModule object ec2: authenticated ec2 connection object """ instance_id = module.params.get('instance_id') name = module.params.get('name') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) description = module.params.get('description') architecture = module.params.get('architecture') kernel_id = module.params.get('kernel_id') root_device_name = module.params.get('root_device_name') virtualization_type = module.params.get('virtualization_type') no_reboot = module.params.get('no_reboot') device_mapping = module.params.get('device_mapping') tags = module.params.get('tags') launch_permissions = module.params.get('launch_permissions') try: params = {'name': name, 'description': description} images = ec2.get_all_images(filters={'name': name}) if images and images[0]: # ensure that launch_permissions are up to date update_image(module, ec2, images[0].id) bdm = None if device_mapping: bdm = BlockDeviceMapping() for device in device_mapping: if 'device_name' not in device: module.fail_json(msg='Device name must be set for volume') device_name = device['device_name'] del device['device_name'] bd = BlockDeviceType(**device) bdm[device_name] = bd if instance_id: params['instance_id'] = instance_id params['no_reboot'] = no_reboot if bdm: params['block_device_mapping'] = bdm image_id = ec2.create_image(**params) else: params['architecture'] = architecture params['virtualization_type'] = virtualization_type if kernel_id: params['kernel_id'] = kernel_id if root_device_name: params['root_device_name'] = root_device_name if bdm: params['block_device_map'] = bdm image_id = ec2.register_image(**params) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message)) # Wait until the image is recognized. EC2 API has eventual consistency, # such that a successful CreateImage API call doesn't guarantee the success # of subsequent DescribeImages API call using the new image id returned. for i in range(wait_timeout): try: img = ec2.get_image(image_id) if img.state == 'available': break elif img.state == 'failed': module.fail_json( msg= "AMI creation failed, please see the AWS console for more details" ) except boto.exception.EC2ResponseError as e: if ('InvalidAMIID.NotFound' not in e.error_code and 'InvalidAMIID.Unavailable' not in e.error_code) and wait and i == wait_timeout - 1: module.fail_json( msg= "Error while trying to find the new image. Using wait=yes and/or a longer " "wait_timeout may help. %s: %s" % (e.error_code, e.error_message)) finally: time.sleep(1) if img.state != 'available': module.fail_json( msg= "Error while trying to find the new image. Using wait=yes and/or a longer wait_timeout may help." ) if tags: try: ec2.create_tags(image_id, tags) except boto.exception.EC2ResponseError as e: module.fail_json(msg="Image tagging failed => %s: %s" % (e.error_code, e.error_message)) if launch_permissions: try: img = ec2.get_image(image_id) img.set_launch_permissions(**launch_permissions) except boto.exception.BotoServerError as e: module.fail_json(msg="%s: %s" % (e.error_code, e.error_message), image_id=image_id) module.exit_json(msg="AMI creation operation complete", changed=True, **get_ami_info(img))
def get_images(): ec2 = get_aws_connection('ec2') filters = {'tag:tcm': 'image'} images = ec2.get_all_images(owners=['self'], filters=filters) return images