def main():
argument_spec = ec2_argument_spec()
argument_spec.update(dict(
resource = dict(required=True),
tags = dict(),
state = dict(default='present', choices=['present', 'absent', 'list']),
)
)
module = AnsibleModule(argument_spec=argument_spec)
resource = module.params.get('resource')
tags = module.params.get('tags')
state = module.params.get('state')
ec2 = ec2_connect(module)
# We need a comparison here so that we can accurately report back changed status.
# Need to expand the gettags return format and compare with "tags" and then tag or detag as appropriate.
filters = {'resource-id' : resource}
gettags = ec2.get_all_tags(filters=filters)
dictadd = {}
dictremove = {}
baddict = {}
tagdict = {}
for tag in gettags:
tagdict[tag.name] = tag.value
if state == 'present':
if not tags:
module.fail_json(msg="tags argument is required when state is present")
if set(tags.items()).issubset(set(tagdict.items())):
module.exit_json(msg="Tags already exists in %s." %resource, changed=False)
else:
for (key, value) in set(tags.items()):
if (key, value) not in set(tagdict.items()):
dictadd[key] = value
tagger = ec2.create_tags(resource, dictadd)
gettags = ec2.get_all_tags(filters=filters)
module.exit_json(msg="Tags %s created for resource %s." % (dictadd,resource), changed=True)
if state == 'absent':
if not tags:
module.fail_json(msg="tags argument is required when state is absent")
for (key, value) in set(tags.items()):
if (key, value) not in set(tagdict.items()):
baddict[key] = value
if set(baddict) == set(tags):
module.exit_json(msg="Nothing to remove here. Move along.", changed=False)
for (key, value) in set(tags.items()):
if (key, value) in set(tagdict.items()):
dictremove[key] = value
tagger = ec2.delete_tags(resource, dictremove)
gettags = ec2.get_all_tags(filters=filters)
module.exit_json(msg="Tags %s removed for resource %s." % (dictremove,resource), changed=True)
if state == 'list':
module.exit_json(changed=False, tags=tagdict)
sys.exit(0)
def delete_vpn_connection(module, ec2, vpc_conn):
"""
Deletes a vpn connection
module: Ansible module object
ec2: authenticated ec2 connection object
vpc_conn: authenticated VPCConnection connection object
name: the option set name
Returns a list of the DHCP options that have been deleted
"""
name = module.params.get('name')
vpn_connections = vpc_conn.get_all_vpn_connections()
removed_vpn_connections = []
for vpn_connection in vpn_connections:
filters = {'resource-id': vpn_connection.id}
gettags = ec2.get_all_tags(filters=filters)
tagdict = {}
for tag in gettags:
tagdict[tag.name] = tag.value
if ('Name', name) in set(tagdict.items()):
vpc_conn.delete_vpn_connection(vpn_connection.id)
removed_vpn_connections.append(vpn_connection.id)
return removed_vpn_connections
def create_ami(instance_id, name, description):
params = {
'instance_id': instance_id,
'name': name,
'description': description,
'no_reboot': True
}
AWS_API_WAIT_TIME = 1
image_id = ec2.create_image(**params)
print("Checking if image is ready.")
for _ in xrange(AMI_TIMEOUT):
try:
img = ec2.get_image(image_id)
if img.state == 'available':
print("Tagging image.")
img.add_tag("environment", args.environment)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("deployment", args.deployment)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("cluster", args.play)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("play", args.play)
time.sleep(AWS_API_WAIT_TIME)
conf_tag = "{} {}".format(
"http://github.com/edx/configuration",
args.configuration_version)
img.add_tag("version:configuration", conf_tag)
time.sleep(AWS_API_WAIT_TIME)
conf_secure_tag = "{} {}".format(
args.configuration_secure_repo,
args.configuration_secure_version)
img.add_tag("version:configuration_secure", conf_secure_tag)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("cache_id", args.cache_id)
time.sleep(AWS_API_WAIT_TIME)
# Get versions from the instance.
tags = ec2.get_all_tags(filters={'resource-id': instance_id})
for tag in tags:
if tag.name.startswith('version:'):
img.add_tag(tag.name, tag.value)
time.sleep(AWS_API_WAIT_TIME)
break
else:
time.sleep(1)
except EC2ResponseError as e:
if e.error_code == 'InvalidAMIID.NotFound':
time.sleep(1)
else:
raise Exception("Unexpected error code: {}".format(
e.error_code))
time.sleep(1)
else:
raise Exception("Timeout waiting for AMI to finish")
return image_id
def create_vpn_connection(module, ec2, vpc_conn):
"""
Creates a vpn connection
module: Ansible module object
ec2: authenticated ec2 connection object
vpc_conn: authenticated VPCConnection connection object
name: the option set name
type: the type of vpn connection
customer_gateway_id: the id of the customer gateway
vpn_gateway_id: the id of the vpn gateway
static_routes_only: indicates whether the VPN connection requires static routes.
wait: True/False indicating to wait for the VPN to be in the available state
wait_timeout: The timeout for waiting
Returns a tuple
"""
name = module.params.get('name')
type = module.params.get('type')
customer_gateway_id = module.params.get('customer_gateway_id')
vpn_gateway_id = module.params.get('vpn_gateway_id')
static_routes_only = module.params.get('static_routes_only')
wait = module.params.get('wait')
wait_timeout = module.params.get('wait_timeout')
vpn_connections = vpc_conn.get_all_vpn_connections()
for vpn_connection in vpn_connections:
filters = {'resource-id': vpn_connection.id}
gettags = ec2.get_all_tags(filters=filters)
tagdict = {}
for tag in gettags:
tagdict[tag.name] = tag.value
if ('Name', name) in set(tagdict.items()):
changed = False
if type != vpn_connection.type:
changed = True
if customer_gateway_id != vpn_connection.customer_gateway_id:
changed = True
if vpn_gateway_id != vpn_connection.vpn_gateway_id:
changed = True
# cant seem to get the routing type to check if that has changed
if changed:
module.fail_json(msg='VPN connection cannot be modified')
return (vpn_connection.id, False)
vpn_connection = vpc_conn.create_vpn_connection(type, customer_gateway_id, vpn_gateway_id, static_routes_only)
ec2.create_tags(vpn_connection.id, {'Name': name})
if wait:
await_vpn_connection_state(module, vpc_conn, vpn_connection, 'available', wait_timeout)
return (vpn_connection.id, True)
def create_ami(instance_id, name, description):
params = {'instance_id': instance_id,
'name': name,
'description': description,
'no_reboot': False}
AWS_API_WAIT_TIME = 1
image_id = ec2.create_image(**params)
print("Checking if image is ready.")
for _ in xrange(AMI_TIMEOUT):
try:
img = ec2.get_image(image_id)
if img.state == 'available':
print("Tagging image.")
img.add_tag("environment", args.environment)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("deployment", args.deployment)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("cluster", args.play)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("play", args.play)
time.sleep(AWS_API_WAIT_TIME)
conf_tag = "{} {}".format("http://github.com/edx/configuration", args.configuration_version)
img.add_tag("version:configuration", conf_tag)
time.sleep(AWS_API_WAIT_TIME)
conf_secure_tag = "{} {}".format(args.configuration_secure_repo, args.configuration_secure_version)
img.add_tag("version:configuration_secure", conf_secure_tag)
time.sleep(AWS_API_WAIT_TIME)
conf_internal_tag = "{} {}".format(args.configuration_internal_repo, args.configuration_internal_version)
img.add_tag("version:configuration_internal", conf_internal_tag)
time.sleep(AWS_API_WAIT_TIME)
img.add_tag("cache_id", args.cache_id)
time.sleep(AWS_API_WAIT_TIME)
# Get versions from the instance.
tags = ec2.get_all_tags(filters={'resource-id': instance_id})
for tag in tags:
if tag.name.startswith('version:'):
img.add_tag(tag.name, tag.value)
time.sleep(AWS_API_WAIT_TIME)
break
else:
time.sleep(1)
except EC2ResponseError as e:
if e.error_code == 'InvalidAMIID.NotFound':
time.sleep(1)
else:
raise Exception("Unexpected error code: {}".format(
e.error_code))
time.sleep(1)
else:
raise Exception("Timeout waiting for AMI to finish")
return image_id
def tag_instances():
for region in regions():
region_name = region.name
if region_name != 'cn-north-1' and region_name != 'us-gov-west-1': # Skip these regions
print "Region: %s" % (region_name)
# Connect
ec2 = boto.ec2.connect_to_region(
region_name,
aws_access_key_id=access_key,
aws_secret_access_key=access_secret)
cloudtrail = boto.cloudtrail.connect_to_region(
region_name,
aws_access_key_id=access_key,
aws_secret_access_key=access_secret)
reservations = ec2.get_all_reservations()
tags = ec2.get_all_tags()
for reservation in reservations:
for instance in reservation.instances:
events_dict = cloudtrail.lookup_events(
lookup_attributes=[{
'AttributeKey': 'ResourceName',
'AttributeValue': instance.id
}])
if len(events_dict['Events']) == 0:
print("No CloudTrail events for instance: %s - %s" %
(instance.id, instance.instance_type))
else:
for data in events_dict['Events']:
json_file = json.loads(data['CloudTrailEvent'])
# Only interested in RunInstances (e.g. created instances) to find owners
# There's also StartInstances, but that event is fired if someone else
# restarts an instance, which isn't what we're really looking for
if json_file['eventName'] == 'RunInstances':
arn = json_file['userIdentity']['arn']
username = json_file['userIdentity'][
'userName']
user_type = json_file['userIdentity']['type']
print(
"Tagging Instance: %s, Username: %s, ARN: %s, Type: %s, eventName: %s"
% (instance.id, username, arn, user_type,
json_file['eventName']))
# Tag the instance
ec2.create_tags(
[instance.id], {
"IAM Username": username,
"IAM ARN": arn,
"IAM Type": user_type
})
# CloudTrail calls are throttled if there's more than 1 req/s
time.sleep(1)
def ping(self):
metadata = boto.utils.get_instance_metadata()
instance_id = metadata['instance-id']
az = metadata['placement']['availability-zone']
# hacky - is there a better way to find the region name?
region_name = az[0:-1]
ec2 = boto.ec2.connect_to_region(region_name)
tags = ec2.get_all_tags(filters={'resource-id': instance_id})
for tag in tags:
if tag.name == self.tag_name:
self.send_to_stackdriver(tag.value, instance_id)
def instances_by_user(args, ec2, user_set=None):
result = {}
instance_tags = ec2.get_all_tags({'resource_type': 'instance', 'key': 'saved_for_user'})
for tag in instance_tags:
user = tag.value
if user_set and user not in user_set:
continue
instance = ec2.get_only_instances(instance_ids=[tag.res_id])[0]
if instance.state == 'terminated' or instance.state == 'shutting-down':
continue
result.setdefault(user, []).append(instance)
return result
def should_format_volume(region, partition, erase_on_boot):
"""
We need to take a safe decision whether to format a volume or not
based on two inputs: value of user data flag and EBS volume tag. The
tag can either be or not be there, which we model with values True and
False. The user data flag can have 3 possible values: True, False and
None (when not given at all).
In the following table we mark the decision to format with exclamation
mark:
Data \ Tag | T | F
-----------+---+---
T | ! | !
-----------+---+---
F | - | -
-----------+---+---
N | ! | -
"""
erase_tag_set = False
#
# We should only try to query the EBS tags if user data doesn't
# tell us anything: otherwise this will crash the instances which
# don't have any role attached.
#
if erase_on_boot is None:
ec2 = ec2_client(region)
volumes = list(
ec2.get_all_volumes(
filters={
'attachment.instance-id': instance_id(),
'attachment.device': partition
}))
if volumes:
volume_id = volumes[0].id
logging.info("%s: volume_id=%s", partition, volume_id)
tags = ec2.get_all_tags(
filters={
'resource-id': volume_id,
'key': ERASE_ON_BOOT_TAG_NAME,
'value': 'True'
})
if list(tags):
ec2.delete_tags(volume_id, [ERASE_ON_BOOT_TAG_NAME])
erase_tag_set = True
logging.info("%s: erase_on_boot=%s, erase_tag_set=%s", partition,
erase_on_boot, erase_tag_set)
return erase_on_boot or (erase_on_boot is None and erase_tag_set)
def get_fallback_slurm_s3_root(region=None):
"""
Get the SLURM S3 root by examining the tags applied to the instance.
This is used if the node has not (yet) been configured with an
/etc/slurm-ec2.conf file (the bootstrapping problem).
"""
if region is None:
region = get_region()
instance_id = get_instance_id()
ec2 = boto.ec2.connect_to_region(region)
if ec2 is None:
raise ValueError("Unable to connect to EC2 endpoint in region %r" %
(region,))
tags = ec2.get_all_tags(filters={"resource-type": "instance",
"resource-id": instance_id,
"key": "SLURMEC2Root"})
if len(tags) == 0:
return None
return tags[0].value
def get_fallback_slurm_s3_root(region=None):
"""
Get the SLURM S3 root by examining the tags applied to the instance.
This is used if the node has not (yet) been configured with an
/etc/slurm-ec2.conf file (the bootstrapping problem).
"""
if region is None:
region = get_region()
instance_id = get_instance_id()
ec2 = boto.ec2.connect_to_region(region)
if ec2 is None:
raise ValueError("Unable to connect to EC2 endpoint in region %r" %
(region, ))
tags = ec2.get_all_tags(
filters={
"resource-type": "instance",
"resource-id": instance_id,
"key": "SLURMEC2Root"
})
if len(tags) == 0:
return None
return tags[0].value
import boto.ec2
import json
import sys
ec2 = boto.ec2.connect_to_region('us-east-1')
#reservations= ec2.get_all_instances(filters={'tag:krole': 'npe'})
reservations = ec2.get_all_instances()
#print instances
for reservation in reservations:
instances = reservation.instances
for instance in instances:
name = ec2.get_all_tags(filters={
'resource-id': instance.id,
'key': 'Name'
})[0].value
print name, ":", instance.instance_type, ":", instance.id, ":", instance.private_ip_address
def get_all_tags(ec2, filters):
return ec2.get_all_tags(filters=filters)
bc.build_common_parser()])
parser.add_argument(
"-f",
"--filter",
action="append",
help="An AWS resource filter. [can be used multiple times]")
args = parser.parse_args()
# process common command line arguments
log = logging.getLogger('botocross')
bc.configure_logging(log, args.log_level)
credentials = bc.parse_credentials(args)
regions = bc.filter_regions(boto.ec2.regions(), args.region)
filters = bc.build_filter_params(args.filter)
# execute business logic
log.info("Describing AWS resource tags:")
for region in regions:
try:
ec2 = boto.connect_ec2(region=region, **credentials)
resources = ec2.get_all_tags(filters=filters)
print region.name + ": " + str(len(resources)) + " resources with tags"
for resource in resources:
if args.verbose:
pprint(vars(resource))
else:
print "type: " + resource.res_type + ", id: " + resource.res_id + ", key: " + resource.name + ", value: " + resource.value
except boto.exception.BotoServerError, e:
log.error(e.error_message)
import boto.ec2
import json
import sys
ec2 = boto.ec2.connect_to_region('us-east-1')
#reservations= ec2.get_all_instances(filters={'tag:krole': 'npe'})
reservations= ec2.get_all_instances()
#print instances
for reservation in reservations:
instances = reservation.instances
for instance in instances:
name = ec2.get_all_tags(filters={'resource-id': instance.id, 'key':'Name'})[0].value
print name,":",instance.instance_type,":",instance.id,":",instance.private_ip_address
snaps = list(sys.argv[1:])
ec2 = ec2.connect_to_region(region)
if snaps:
snapshots = ec2.get_all_snapshots(snapshot_ids=snaps, owner='self')
else:
snapshots = ec2.get_all_snapshots(owner='self')
force_tagging = False
for snapshot in snapshots:
snap_tags = ec2.get_all_tags(filters={
'resource-id': snapshot.id,
'resource-type': 'snapshot'
})
is_named = False
snap_name = "unnamed"
is_tagged = False
lincx_env = None
lincx_type = None
for tag in snap_tags:
if tag.name == "Name":
snap_name = tag.value
is_named = True
elif tag.name == "lincx_environment":
lincx_env = tag.value
is_tagged = True
elif tag.name == "lincx_type":
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(dict(
resource = dict(required=True),
tags = dict(type='dict'),
state = dict(default='present', choices=['present', 'absent', 'list']),
)
)
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
resource = module.params.get('resource')
tags = module.params.get('tags')
state = module.params.get('state')
ec2 = ec2_connect(module)
# We need a comparison here so that we can accurately report back changed status.
# Need to expand the gettags return format and compare with "tags" and then tag or detag as appropriate.
filters = {'resource-id' : resource}
gettags = ec2.get_all_tags(filters=filters)
dictadd = {}
dictremove = {}
baddict = {}
tagdict = {}
for tag in gettags:
tagdict[tag.name] = tag.value
if state == 'present':
if not tags:
module.fail_json(msg="tags argument is required when state is present")
if set(tags.items()).issubset(set(tagdict.items())):
module.exit_json(msg="Tags already exists in %s." %resource, changed=False)
else:
for (key, value) in set(tags.items()):
if (key, value) not in set(tagdict.items()):
dictadd[key] = value
tagger = ec2.create_tags(resource, dictadd)
gettags = ec2.get_all_tags(filters=filters)
module.exit_json(msg="Tags %s created for resource %s." % (dictadd,resource), changed=True)
if state == 'absent':
if not tags:
module.fail_json(msg="tags argument is required when state is absent")
for (key, value) in set(tags.items()):
if (key, value) not in set(tagdict.items()):
baddict[key] = value
if set(baddict) == set(tags):
module.exit_json(msg="Nothing to remove here. Move along.", changed=False)
for (key, value) in set(tags.items()):
if (key, value) in set(tagdict.items()):
dictremove[key] = value
tagger = ec2.delete_tags(resource, dictremove)
gettags = ec2.get_all_tags(filters=filters)
module.exit_json(msg="Tags %s removed for resource %s." % (dictremove,resource), changed=True)
if state == 'list':
module.exit_json(changed=False, tags=tagdict)
if vols:
volumes = ec2.get_all_volumes(volume_ids=vols)
else:
volumes = ec2.get_all_volumes()
snapshots = ec2.get_all_snapshots(owner='self')
for volume in volumes:
if available_only:
if volume.attach_data.status:
print "SKIPPING: Attached volume", volume.id
continue
vol_tags = ec2.get_all_tags(filters={'resource-id': volume.id, 'resource-type': 'volume'})
is_archived = False
is_named = False
vol_name = 'unnamed'
for tag in vol_tags:
if tag.name == 'Name':
vol_name = tag.value
is_named = True
elif tag.name == 'archived' and tag.value == 'yes':
is_archived = True
if is_archived:
print "SKIPPING: Already archived:", volume.id, vol_name
continue
def get_all_tags(ec2, filters):
return ec2.get_all_tags(filters=filters)
configure_logging(log, args.log_level)
def isSelected(region):
return True if region.name.find(args.region) != -1 else False
# execute business logic
credentials = {'aws_access_key_id': args.aws_access_key_id, 'aws_secret_access_key': args.aws_secret_access_key}
heading = "Describing AWS resource tags"
regions = boto.ec2.regions()
if args.region:
heading += " (filtered by region '" + args.region + "')"
regions = filter(isSelected, regions)
filters = None
if args.filter:
filters = dict([filter.split('=') for filter in args.filter])
log.info(args.filter)
log.debug(filters)
print heading + ":"
for region in regions:
try:
ec2 = boto.connect_ec2(region=region, **credentials)
resources = ec2.get_all_tags(filters=filters)
print region.name + ": " + str(len(resources)) + " resources with tags"
for resource in resources:
print "type: " + resource.res_type + ", id: " + resource.res_id + ", key: " + resource.name + ", value: " + resource.value
except boto.exception.BotoServerError, e:
log.error(e.error_message)
