def check_tag(key_value_tuple):
key, value = key_value_tuple
if not _TAG_KEY_REGEX.match(key):
raise UsageError("invalid tag key: " + key)
if not _TAG_VALUE_REGEX.match(value):
raise UsageError("invalid tag value: " + value)
return (key, value)
def parse_tag_filter(tagspec):
"""Parses a tag filter specification in the format NAME=VALUE
into a dict with 'Name' and 'Values' entries."""
if sum([1 if c == '=' else 0 for c in tagspec]) != 1:
raise UsageError("tag spec must contain exactly one '=' character")
name, value = tagspec.split('=')
return {'Name': 'tag:' + name, 'Values': [value]}
def act_on_rule(security_groups, cidrip, port, action, verbose, dry_run,
ignore_not_found):
"""Adds or removes a rule. Use action='authorize_ingress' to
add a rule; use action='revoke_ingress' to remove a rule."""
if action not in ('authorize_ingress', 'revoke_ingress'):
raise ValueError("invalid action: " + action)
if '/' not in cidrip:
raise UsageError(
'CIDR does not specify subnet; use /32 to restrict to only the explicit IP address'
)
cidrip_obj = ipcalc.Network(cidrip)
from_port, to_port = parse_port_range(port)
_log.debug(
"executing %s to %d security groups allowing TCP traffic on port(s) %d-%d from %d addresses (%s)",
action, len(security_groups), from_port, to_port, cidrip_obj.size(),
cidrip)
num_successes = 0
for secgroup in security_groups:
try:
fn = type(secgroup).__dict__[action]
fn(secgroup,
DryRun=dry_run,
FromPort=from_port,
ToPort=to_port,
CidrIp=cidrip,
IpProtocol='tcp')
num_successes += 1
except botocore.exceptions.ClientError as e:
error_code = e.response['Error']['Code']
if error_code == 'DryRunOperation':
num_successes += 1
elif error_code == 'InvalidPermission.NotFound' and not ignore_not_found:
raise
else:
raise
_log.debug("%s '%s' on rule of security group %s (%s)",
"dry-ran" if dry_run else "executed", action,
secgroup.group_name, secgroup.group_id)
_log.debug("%d successful actions executed on %d security groups",
num_successes, len(security_groups))
return num_successes
def is_suspended(config, instance_id):
"""Check the configuration to see if the limits defined are
temporarily suspended."""
suspensions = get_instance_criterion(config,
instance_id,
'suspensions',
default_value=[])
now = datetime.datetime.now()
for bounds in suspensions:
try:
earliest = dateparser.parse(
'1996-08-29T02:14:00-04:00' if bounds['from'] ==
'*' else bounds['from'])
latest = dateparser.parse('2038-01-19T03:14:07' if bounds['to'] ==
'*' else bounds['to'])
except KeyError:
raise UsageError(
"suspensions objects in config must have 'from' and 'to' fields"
)
if now >= earliest and now <= latest:
return True
return False
def main(argv):
from argparse import ArgumentParser
parser = ArgumentParser(description="""Creates security groups in
multiple VPCs with common name and tags.""")
myawscommon.add_log_level_option(parser)
parser.add_argument("--verbose",
help="print more messages on stdout",
action='store_true',
default=False)
myawscommon.add_credentials_options(parser)
myawscommon.add_region_option(parser)
parser.add_argument("sg_name",
metavar="NAME",
help="name for the security groups")
parser.add_argument("vpcs",
nargs='+',
metavar="VPC_ID",
help="one or more VPC IDs")
parser.add_argument("--dry-run",
action='store_true',
default=False,
help="execute in dry-run mode")
parser.add_argument("--description",
metavar="TEXT",
help="specify description for security groups")
parser.add_argument(
"--tags",
nargs='+',
metavar="KEY=VALUE",
default=(),
help="one or more key=value pairs to add as tags to each group created"
)
args = parser.parse_args(argv[1:])
myawscommon.configure_logging(_LOGGER_NAME, args.log_level)
check_security_group_name(args.sg_name)
tags = [check_tag(pair.split('=', 1)) for pair in args.tags]
session = boto3.session.Session(
aws_access_key_id=args.aws_access_key_id,
aws_secret_access_key=args.aws_secret_access_key,
profile_name=args.profile)
regions = myawscommon.filter_regions(session, args.regions)
vpcs_by_region = collections.defaultdict(list)
region_by_vpc_id = {}
vpc_filters = [] # [{'Name': 'isDefault', 'Values': ['False']}]
for region in regions:
_log.debug("gathering VPCs from region %s", region)
ec2 = session.client('ec2', region_name=region)
vpcs = ec2.describe_vpcs(Filters=vpc_filters)['Vpcs']
vpcs = [vpc for vpc in vpcs if not vpc['IsDefault']
] # the IsDefault query filter doesn't seem to work
for vpc in vpcs:
vpc_id = vpc['VpcId']
region_by_vpc_id[vpc_id] = region
if len(vpcs) > 0:
vpcs_by_region[region] = vpcs
if args.verbose:
print("%d VPCs in %s%s%s" % (len(vpcs), region, ': ' if
(len(vpcs) > 0) else '', '. '.join(
[vpc['VpcId'] for vpc in vpcs])))
_log.debug("%d VPCs across %d regions", len(region_by_vpc_id),
len(vpcs_by_region))
for vpc_id in args.vpcs:
if vpc_id not in region_by_vpc_id:
raise UsageError("vpc id not found in region scope: " + vpc_id)
_log.debug("creating security group named '%s' in %d VPCs with %d tags",
args.sg_name, len(args.vpcs), len(tags))
client_cache = SessionProductCache(
lambda region: session.client('ec2', region_name=region))
resource_cache = SessionProductCache(
lambda region: session.resource('ec2', region_name=region))
for vpc_id in args.vpcs:
region = region_by_vpc_id[vpc_id]
ec2 = client_cache[region]
ec2_resource = resource_cache[region]
_log.debug("using client %s for region %s", ec2, region)
try:
group_id = ec2.create_security_group(
DryRun=args.dry_run,
GroupName=args.sg_name,
Description=construct_description(args, vpc_id, region),
VpcId=vpc_id)['GroupId']
except botocore.exceptions.ClientError as e:
if myawscommon.client_error_has_code(e, 'DryRunOperation'):
group_id = "sg-%012x" % random.getrandbits(48)
else:
raise
print(group_id,
"%screated" % "(dry-run) " if args.dry_run else '',
end="")
if len(tags) > 0:
security_group = ec2_resource.SecurityGroup(group_id)
try:
response = security_group.create_tags(DryRun=args.dry_run,
Tags=[{
'Key': key,
'Value': value
} for key, value in tags
])
except botocore.exceptions.ClientError as e:
if myawscommon.client_error_has_code(e, 'DryRunOperation'):
response = [
ec2_resource.Tag(group_id, key, value)
for key, value in tags
]
else:
raise
print('with', len(response), 'tag(s)', end="")
_log.debug("created tags: %s", response)
print()
return 0
def check_security_group_name(sg_name):
if len(sg_name) > _SG_DESC_LEN_MAX:
raise UsageError(
f"security group name is invalid; must be no more than {_SG_NAME_LEN_MAX} characters"
)
return sg_name
def main(argv):
parser = ArgumentParser(description="""\
Perform operations on security groups. Print security groups,
check that they have identical ingress rules, and add/remove
ingress rules.""")
myawscommon.add_log_level_option(parser)
myawscommon.add_credentials_options(parser)
myawscommon.add_region_option(parser)
parser.add_argument("--group-ids",
nargs="+",
help="filter target groups by group id",
metavar="ID")
parser.add_argument("--group-names",
nargs="+",
help="filter target groups by group name",
metavar="NAME")
parser.add_argument("--tag",
metavar="NAME=VALUE",
help="filter target groups by tag value")
parser.add_argument("--verbose",
help="print more messages on stdout",
action='store_true',
default=False)
parser.add_argument(
"--check-in-use",
action='store_true',
default=False,
help=
"check whether security groups are in use (and mark those not in use with * in list)"
)
parser.add_argument(
"--dry-run",
action='store_true',
default=False,
help=
"set DryRun flag to true for actions that would modify security groups"
)
parser.add_argument(
"--check",
nargs="?",
metavar="TAGNAME",
const='ALL',
help=
"checks that security group subsets contain the same set of ingress IP permission rules; "
+
"security groups can be partitioned into subsets based on the value of the tag specified"
)
parser.add_argument(
"--add-rule",
nargs=2,
action='append',
metavar="ARG",
help=
"for each security group, adds a rule (formatted as two args, PORT SPEC) allowing ingress "
+
"via TCP on port PORT from IP addresses within the range CIDR; e.g. --add-rule 8080 10.0.0.0/16"
)
parser.add_argument(
"--remove-rule",
nargs=2,
action='append',
metavar="ARG",
help=
"each security group, removes the rule (formatted as two args, PORT SPEC) that allows "
+
"ingress via TCP on port PORT from IP addresses within the range CIDR")
parser.add_argument(
"--ignore-rule-not-found",
action="store_true",
default=False,
help=
"with --remove-rule, ignore errors that are due to absence of the specified rule in any security group"
)
parser.add_argument(
"--allow-action-on-all",
action='store_true',
default=False,
help=
"allows --add-rule or --remove-rule to operate on all security groups; by default, as "
+
"a precaution, it is assumed that the user erred in requesting a rule be added to every "
+ "security group; this option overrides that assumption")
parser.add_argument(
"--delete-unused",
action="store_true",
default=False,
help=
"delete security groups that are not in use by any instances (running or stopped)"
)
parser.add_argument(
"--ignore-empty-target-list",
action='store_true',
default=False,
help=
"do not exit dirty when an action is specified but the target list is empty"
)
args = parser.parse_args(argv[1:])
myawscommon.configure_logging(_LOGGER_NAME, args.log_level)
session = boto3.session.Session(
aws_access_key_id=args.aws_access_key_id,
aws_secret_access_key=args.aws_secret_access_key,
profile_name=args.profile)
try:
regions = myawscommon.filter_regions(session, args.regions)
_log.debug(
"regions filtered to %s according to user specification %s; tasks: %s",
regions, args.regions, _get_task_argument_values(args))
filters = []
if args.tag:
filters.append(parse_tag_filter(args.tag))
security_groups = fetch_security_groups(
session, regions, args.group_ids, args.group_names, filters,
print_security_group
if args.verbose or not _has_task_argument(args) else _NOOP,
args.check_in_use, args.delete_unused, args.dry_run)
if len(security_groups) == 0:
_log.info(
"target security group list is empty (%d regions searched)",
len(regions))
if _has_task_argument(args) and not args.ignore_empty_target_list:
return ERR_USAGE
if args.check is not None:
_log.debug(
"checking synchronization of security groups based on partition spec %s",
args.check)
secgroups_by_key = defaultdict(list)
if args.check == 'ALL':
secgroups_by_key['ALL'] += security_groups
else:
for secgroup in security_groups:
for tag in secgroup.tags or ():
if args.check == tag['Key']:
secgroups_by_key[tag['Value']].append(secgroup)
if len(secgroups_by_key) == 0:
raise UsageError(
"the set of partitions created from tag '%s' is empty (no security groups have this tag)"
% args.check)
not_in_sync = list()
for sync_key in secgroups_by_key:
expect_in_sync = secgroups_by_key[sync_key]
_log.debug(
"expecting %d security groups (annotated as %s) to be synchronized",
len(expect_in_sync), sync_key)
if not are_synchronized(expect_in_sync,
verbose=args.verbose,
annotation=sync_key):
not_in_sync.append(sync_key)
if len(not_in_sync) > 0:
_log.info("not in sync: %s", ','.join(not_in_sync))
return ERR_NOT_SYNCHRONIZED
for option in ((args.add_rule, 'authorize_ingress'),
(args.remove_rule, 'revoke_ingress')):
option_value, action = option
if option_value is not None:
for unexpanded_rule_spec in option_value:
if not is_any_restriction_specified(args):
_log.error(
"denying request to add or remove rule to/from all security groups; "
+
"use --group-ids, --group-names, or --tag to restrict the security "
+
"groups list, or use --allow-action-on-all to override this check"
)
return ERR_UNRESTRICTED_RULE_APPLICATION_REQUESTED
rule_specs = expand_rule_spec(unexpanded_rule_spec)
for rule_spec in rule_specs:
port, cidrip = rule_spec[0], rule_spec[1]
act_on_rule(security_groups, cidrip, port, action,
args.verbose, args.dry_run,
args.ignore_rule_not_found)
if args.verbose:
print("dry-ran" if args.dry_run else "executed",
action, port, cidrip, "on",
len(security_groups), "security groups")
except UsageError as e:
print(e, file=sys.stderr)
return ERR_USAGE
return 0
def fetch_security_groups(session, regions, group_ids, group_names, filters,
foreach, check_in_use, delete_unused, dry_run):
"""Fetches a list of security groups, compiled from multiple
regions and filtered by group IDs and other optional filters."""
if delete_unused and not check_in_use:
raise UsageError("--delete-unused requires --check-in-use")
if not callable(foreach):
raise ValueError("'foreach' parameter must be a function")
secgroups = []
group_ids, group_names = group_ids or [], group_names or []
for region in regions:
ec2 = session.resource('ec2', region_name=region)
try:
secgroups_in_region = list(
ec2.security_groups.filter(GroupIds=group_ids,
GroupNames=group_names,
Filters=filters))
except botocore.exceptions.ClientError as e:
error_code = e.response['Error']['Code']
if error_code == 'InvalidGroup.NotFound':
secgroups_in_region = []
elif error_code == 'InvalidParameterValue' and 'You may not reference Amazon VPC security groups by name.' in str(
e):
_log.info(
"adjusting fetch strategy for inconsistent server behavior ('%s' client error)",
error_code)
secgroups_in_region = [
sg for sg in ec2.security_groups.filter(
GroupIds=group_ids, GroupNames=[], Filters=filters)
if sg.group_name in group_names
]
else:
raise
groups_in_use = None
if check_in_use:
groups_in_use = fetch_security_groups_in_use(ec2)
for sg in sorted(secgroups_in_region,
key=operator.attrgetter('vpc_id', 'group_name')):
foreach(sg, region, groups_in_use)
if delete_unused:
not_deleted = []
for sg in secgroups_in_region:
if sg.group_name != 'default' and sg.group_id not in groups_in_use:
try:
sg.delete(DryRun=dry_run, GroupId=sg.group_id)
except botocore.exceptions.ClientError as e:
error_code = e.response['Error']['Code']
if error_code == 'DryRunOperation':
_log.debug("error_code=%s operation_name=%s",
error_code, e.operation_name)
elif error_code == 'DependencyViolation':
_log.warning(
"failed to delete %s (%s) due to dependency violation",
sg.group_id, sg.group_name)
else:
raise
else:
not_deleted.append(sg)
_log.info("%sdeleted %d unused security groups (out of %d)",
"(dry run) " if dry_run else "",
len(secgroups_in_region) - len(not_deleted),
len(secgroups_in_region))
secgroups_in_region = secgroups_in_region if dry_run else not_deleted
_log.debug("fetched %d security group(s) in region %s",
len(secgroups_in_region), region)
secgroups += secgroups_in_region
if len(group_ids) > 0 and (len(secgroups) >= len(group_ids)):
_log.debug(
"already found %d security groups, and %d group IDs were specified; not searching any more regions",
len(secgroups), len(group_ids))
break
return secgroups