def fetch_password_policy(self, credentials):
"""
Fetch the password policy that applies to all IAM users within the AWS account
"""
self.fetchstatuslogger.counts['password_policy']['discovered'] = 0
self.fetchstatuslogger.counts['password_policy']['fetched'] = 0
try:
api_client = connect_service('iam', credentials, silent = True)
self.password_policy = api_client.get_account_password_policy()['PasswordPolicy']
if 'PasswordReusePrevention' not in self.password_policy:
self.password_policy['PasswordReusePrevention'] = False
else:
self.password_policy['PreviousPasswordPrevented'] = self.password_policy['PasswordReusePrevention']
self.password_policy['PasswordReusePrevention'] = True
# There is a bug in the API: ExpirePasswords always returns false
if 'MaxPasswordAge' in self.password_policy:
self.password_policy['ExpirePasswords'] = True
self.fetchstatuslogger.counts['password_policy']['discovered'] = 1
self.fetchstatuslogger.counts['password_policy']['fetched'] = 1
except ClientError as e:
if e.response['Error']['Code'] == 'NoSuchEntity':
self.password_policy = {}
self.password_policy['MinimumPasswordLength'] = '1' # As of 10/10/2016, 1-character passwords were authorized when no policy exists, even though the console displays 6
self.password_policy['RequireUppercaseCharacters'] = False
self.password_policy['RequireLowercaseCharacters'] = False
self.password_policy['RequireNumbers'] = False
self.password_policy['RequireSymbols'] = False
self.password_policy['PasswordReusePrevention'] = False
self.password_policy['ExpirePasswords'] = False
else:
raise e
except Exception as e:
printError(str(e))
def fetch(self, credentials, services = [], regions = [], partition_name = ''):
"""
:param credentials:
:param services:
:param regions:
:param partition_name:
:return:
"""
for service in vars(self):
try:
if services != [] and service not in services:
continue
service_config = getattr(self, service)
if 'fetch_all' in dir(service_config):
method_args = {}
method_args['credentials'] = credentials
if service != 'iam':
method_args['regions'] = regions
method_args['partition_name'] = partition_name
service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
service_config.finalize()
except Exception as e:
printError('Error: could not fetch %s configuration.' % service)
printException(e)
def get_s3_acls(api_client, bucket_name, bucket, key_name = None):
try:
grantees = {}
if key_name:
grants = api_client.get_object_acl(Bucket = bucket_name, Key = key_name)
else:
grants = api_client.get_bucket_acl(Bucket = bucket_name)
for grant in grants['Grants']:
if 'ID' in grant['Grantee']:
grantee = grant['Grantee']['ID']
display_name = grant['Grantee']['DisplayName'] if 'DisplayName' in grant['Grantee'] else grant['Grantee']['ID']
elif 'URI' in grant['Grantee']:
grantee = grant['Grantee']['URI'].split('/')[-1]
display_name = s3_group_to_string(grant['Grantee']['URI'])
else:
grantee = display_name = 'Unknown'
permission = grant['Permission']
manage_dictionary(grantees, grantee, {})
grantees[grantee]['DisplayName'] = display_name
if 'URI' in grant['Grantee']:
grantees[grantee]['URI'] = grant['Grantee']['URI']
manage_dictionary(grantees[grantee], 'permissions', init_s3_permissions())
set_s3_permissions(grantees[grantee]['permissions'], permission)
return grantees
except Exception as e:
printError('Failed to get ACL configuration for %s: %s' % (bucket_name, e))
return {}
def fetch_credential_report(self, credentials, ignore_exception = False):
"""
Fetch the credential report
:param: api_client
:type: FOO
:param: ignore_exception : initiate credential report creation as not always ready
:type: Boolean
"""
iam_report = {}
try:
api_client = connect_service('iam', credentials, silent = True)
response = api_client.generate_credential_report()
if response['State'] != 'COMPLETE':
if not ignore_exception:
printError('Failed to generate a credential report.')
return
report = api_client.get_credential_report()['Content']
lines = report.splitlines()
keys = lines[0].decode('utf-8').split(',')
for line in lines[1:]:
values = line.decode('utf-8').split(',')
manage_dictionary(iam_report, values[0], {})
for key, value in zip(keys, values):
iam_report[values[0]][key] = value
self.credential_report = iam_report
self.fetchstatuslogger.counts['credential_report']['fetched'] = 1
except Exception as e:
if ignore_exception:
return
printError('Failed to download a credential report.')
printException(e)
def __update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, policy_name):
allowed_buckets = []
# By default, all buckets are allowed
for bucket_name in s3_info['buckets']:
allowed_buckets.append(bucket_name)
if policy_type == 'InlinePolicies':
policy = iam_info[iam_entity.title()][allowed_iam_entity]['Policies'][policy_name]['PolicyDocument']
elif policy_type == 'ManagedPolicies':
policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument']
else:
printError('Error, found unknown policy type.')
for statement in policy['Statement']:
for target_path in statement['NotResource']:
parts = target_path.split('/')
bucket_name = parts[0].split(':')[-1]
path = '/' + '/'.join(parts[1:]) if len(parts) > 1 else '/'
if (path == '/' or path == '/*') and (bucket_name in allowed_buckets):
# Remove bucket from list
allowed_buckets.remove(bucket_name)
elif bucket_name == '*':
allowed_buckets = []
policy_info = {}
policy_info[policy_type] = {}
policy_info[policy_type][policy_name] = iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][full_path][policy_type][policy_name]
for bucket_name in allowed_buckets:
__update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info)
def load(self, rule_type, quiet = False):
"""
Open a JSON file definiting a ruleset and load it into a Ruleset object
:param quiet:
:return:
"""
if self.filename and os.path.exists(self.filename):
try:
with open(self.filename, 'rt') as f:
ruleset = json.load(f)
self.about = ruleset['about'] if 'about' in ruleset else ''
self.rules = {}
for filename in ruleset['rules']:
self.rules[filename] = []
for rule in ruleset['rules'][filename]:
self.handle_rule_versions(filename, rule_type, rule)
except Exception as e:
printException(e)
printError('Error: ruleset file %s contains malformed JSON.' % self.filename)
self.rules = []
self.about = ''
else:
self.rules = []
if not quiet:
printError('Error: the file %s does not exist.' % self.filename)
def get_s3_bucket_policy(api_client, bucket_name, bucket_info):
try:
bucket_info['policy'] = json.loads(api_client.get_bucket_policy(Bucket = bucket_name)['Policy'])
return True
except Exception as e:
if not (type(e) == ClientError and e.response['Error']['Code'] == 'NoSuchBucketPolicy'):
printError('Failed to get bucket policy for %s: %s' % (bucket_name, e))
return False
def get_s3_bucket_logging(api_client, bucket_name, bucket_info):
try:
logging = api_client.get_bucket_logging(Bucket = bucket_name)
if 'LoggingEnabled' in logging:
bucket_info['logging'] = logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix']
bucket_info['logging_stuff'] = logging
else:
bucket_info['logging'] = 'Disabled'
return True
except Exception as e:
printError('Failed to get logging configuration for %s: %s' % (bucket_name, e))
bucket_info['logging'] = 'Unknown'
return False
def get_s3_bucket_default_encryption(api_client, bucket_name, bucket_info):
try:
default_encryption = api_client.get_bucket_encryption(Bucket = bucket_name)
bucket_info['default_encryption'] = 'Enabled'
return True
except ClientError as e:
if 'ServerSideEncryptionConfigurationNotFoundError' in e.response['Error']['Code']:
bucket_info['default_encryption'] = 'Disabled'
return True
else:
printError('Failed to get encryption configuration for %s: %s' % (bucket_name, e))
bucket_info['default_encryption'] = 'Unknown'
return False
except Exception as e:
printError('Failed to get encryption configuration for %s: %s' % (bucket_name, e))
bucket_info['default_encryption'] = 'Unknown'
return False
def pass_conditions(all_info, current_path, conditions, unknown_as_pass_condition = False):
"""
Pass all conditions?
:param all_info:
:param current_path:
:param conditions:
:param unknown_as_pass_condition: Consider an undetermined condition as passed
:return:
"""
result = False
if len(conditions) == 0:
return True
condition_operator = conditions.pop(0)
for condition in conditions:
if condition[0] in condition_operators:
res = pass_conditions(all_info, current_path, condition, unknown_as_pass_condition)
else:
# Conditions are formed as "path to value", "type of test", "value(s) for test"
path_to_value, test_name, test_values = condition
path_to_value = fix_path_string(all_info, current_path, path_to_value)
target_obj = get_value_at(all_info, current_path, path_to_value)
if type(test_values) != list:
dynamic_value = re_get_value_at.match(test_values)
if dynamic_value:
test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True)
try:
res = pass_condition(target_obj, test_name, test_values)
except Exception as e:
res = True if unknown_as_pass_condition else False
printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % (test_name, str(target_obj), res))
printException(e, True)
# Quick exit and + false
if condition_operator == 'and' and not res:
return False
# Quick exit or + true
if condition_operator == 'or' and res:
return True
# Still here ?
# or -> false
# and -> true
if condition_operator == 'or':
return False
else:
return True
def parse_parameter_group(self, global_params, region, parameter_group):
parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn')
parameter_group['name'] = parameter_group.pop('DBParameterGroupName')
api_client = api_clients[region]
try:
parameters = handle_truncated_response(api_client.describe_db_parameters, {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])['Parameters']
manage_dictionary(parameter_group, 'parameters', {})
for parameter in parameters:
if not parameter['IsModifiable']:
# Discard non-modifiable parameters
continue
parameter_name = parameter.pop('ParameterName')
parameter_group['parameters'][parameter_name] = parameter
except Exception as e:
printException(e)
printError('Failed fetching DB parameters for %s' % parameter_group['name'])
# Save
parameter_group_id = self.get_non_aws_id(parameter_group['name'])
(self).parameter_groups[parameter_group_id] = parameter_group
def recurse(all_info, current_info, target_path, current_path, config, add_suffix = False):
"""
:param all_info:
:param current_info:
:param target_path:
:param current_path:
:param config:
:param add_suffix:
:return:
"""
results = []
if len(target_path) == 0:
# Dashboard: count the number of processed resources here
setattr(config, 'checked_items', getattr(config, 'checked_items') + 1)
# Test for conditions...
if pass_conditions(all_info, current_path, copy.deepcopy(config.conditions)):
if add_suffix and hasattr(config, 'id_suffix'):
suffix = fix_path_string(all_info, current_path, config.id_suffix)
current_path.append(suffix)
results.append('.'.join(current_path))
# Return the flagged items...
return results
target_path = copy.deepcopy(target_path)
dbg_target_path = copy.deepcopy(target_path)
current_path = copy.deepcopy(current_path)
attribute = target_path.pop(0)
if type(current_info) == dict:
if attribute in current_info:
split_path = copy.deepcopy(current_path)
split_path.append(attribute)
results = results + recurse(all_info, current_info[attribute], target_path, split_path, config, add_suffix)
elif attribute == 'id':
for key in current_info:
split_target_path = copy.deepcopy(target_path)
split_current_path = copy.deepcopy(current_path)
split_current_path.append(key)
split_current_info = current_info[key]
results = results + recurse(all_info, split_current_info, split_target_path, split_current_path, config, add_suffix)
# To handle lists properly, I would have to make sure the list is properly ordered and I can use the index to consistently access an object... Investigate (or do not use lists)
elif type(current_info) == list:
for index, split_current_info in enumerate(current_info):
split_current_path = copy.deepcopy(current_path)
split_current_path.append(str(index))
results = results + recurse(all_info, split_current_info, copy.deepcopy(target_path), split_current_path, config, add_suffix)
else:
printError('Error: unhandled case, typeof(current_info) = %s' % type(current_info))
printError('Path: %s' % current_path)
printError('Object: %s' % str(current_info))
printError('Entry target path: %s' % str(dbg_target_path))
raise Exception
return results
def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info):
try:
if 'policy' in bucket_info:
bucket_info['secure_transport'] = 'Disabled'
for statement in bucket_info['policy']['Statement']:
# evaluate statement to see if it contains a condition disallowing HTTP transport
# TODO this might not cover all cases
if 'Condition' in statement and \
'Bool' in statement['Condition'] and \
'aws:SecureTransport' in statement['Condition']['Bool'] and \
statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and \
statement['Effect'] == 'Deny':
bucket_info['secure_transport'] = 'Enabled'
return True
else:
bucket_info['secure_transport'] = 'Disabled'
return True
except Exception as e:
printError('Failed to get evaluate bucket policy for %s: %s' % (bucket_name, e))
bucket_info['secure_transport'] = 'Unknown'
return False
def run(self, aws_config, skip_dashboard = False):
# Clean up existing findings
for service in aws_config['services']:
aws_config['services'][service][self.ruleset.rule_type] = {}
# Process each rule
for finding_path in self.rules:
for rule in self.rules[finding_path]:
if not rule.enabled: # or rule.service not in []: # TODO: handle this...
continue
printDebug('Processing %s rule[%s]: "%s"' % (rule.service, rule.filename, rule.description))
finding_path = rule.path
path = finding_path.split('.')
service = path[0]
manage_dictionary(aws_config['services'][service], self.ruleset.rule_type, {})
aws_config['services'][service][self.ruleset.rule_type][rule.key] = {}
aws_config['services'][service][self.ruleset.rule_type][rule.key]['description'] = rule.description
aws_config['services'][service][self.ruleset.rule_type][rule.key]['path'] = rule.path
for attr in ['level', 'id_suffix', 'display_path']:
if hasattr(rule, attr):
aws_config['services'][service][self.ruleset.rule_type][rule.key][attr] = getattr(rule, attr)
try:
setattr(rule, 'checked_items', 0)
aws_config['services'][service][self.ruleset.rule_type][rule.key]['items'] = recurse(aws_config['services'], aws_config['services'], path, [], rule, True)
if skip_dashboard:
continue
aws_config['services'][service][self.ruleset.rule_type][rule.key]['dashboard_name'] = rule.dashboard_name
aws_config['services'][service][self.ruleset.rule_type][rule.key]['checked_items'] = rule.checked_items
aws_config['services'][service][self.ruleset.rule_type][rule.key]['flagged_items'] = len(aws_config['services'][service][self.ruleset.rule_type][rule.key]['items'])
aws_config['services'][service][self.ruleset.rule_type][rule.key]['service'] = rule.service
aws_config['services'][service][self.ruleset.rule_type][rule.key]['rationale'] = rule.rationale if hasattr(rule, 'rationale') else 'N/A'
except Exception as e:
printException(e)
printError('Failed to process rule defined in %s' % rule.filename)
# Fallback if process rule failed to ensure report creation and data dump still happen
aws_config['services'][service][self.ruleset.rule_type][rule.key]['checked_items'] = 0
aws_config['services'][service][self.ruleset.rule_type][rule.key]['flagged_items'] = 0
def fetch_password_policy(self, credentials):
"""
Fetch the password policy that applies to all IAM users within the AWS account
"""
self.fetchstatuslogger.counts['password_policy']['discovered'] = 0
self.fetchstatuslogger.counts['password_policy']['fetched'] = 0
try:
api_client = connect_service('iam', credentials)
self.password_policy = api_client.get_account_password_policy(
)['PasswordPolicy']
if 'PasswordReusePrevention' not in self.password_policy:
self.password_policy['PasswordReusePrevention'] = False
else:
self.password_policy[
'PreviousPasswordPrevented'] = self.password_policy[
'PasswordReusePrevention']
self.password_policy['PasswordReusePrevention'] = True
# There is a bug in the API: ExpirePasswords always returns false
if 'MaxPasswordAge' in self.password_policy:
self.password_policy['ExpirePasswords'] = True
self.fetchstatuslogger.counts['password_policy']['discovered'] = 1
self.fetchstatuslogger.counts['password_policy']['fetched'] = 1
except Exception as e:
if type(e) == botocore.exceptions.ClientError:
if e.response['Error']['Code'] == 'NoSuchEntity':
self.password_policy = {}
self.password_policy[
'MinimumPasswordLength'] = '1' # As of 10/10/2016, 1-character passwords were authorized when no policy exists, even though the console displays 6
self.password_policy['RequireUppercaseCharacters'] = False
self.password_policy['RequireLowercaseCharacters'] = False
self.password_policy['RequireNumbers'] = False
self.password_policy['RequireSymbols'] = False
self.password_policy['PasswordReusePrevention'] = False
self.password_policy['ExpirePasswords'] = False
else:
printError("Unexpected error: %s" % e)
else:
printError(str(e))
def sort_vpc_flow_logs_callback(vpc_config, current_config, path, current_path,
flow_log_id, callback_args):
attached_resource = current_config['ResourceId']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(current_path[0:2],
['vpcs', attached_resource])
try:
attached_vpc = get_object_at(vpc_config, vpc_path)
except Exception as e:
printDebug(
'It appears that the flow log %s is attached to a resource that was previously deleted (%s).'
% (flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(attached_vpc['subnets'][subnet_id], 'flow_logs',
[])
if flow_log_id not in attached_vpc['subnets'][subnet_id][
'flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(
flow_log_id)
elif attached_resource.startswith('subnet-'):
all_vpcs = get_object_at(vpc_config,
combine_paths(current_path[0:2], ['vpcs']))
for vpc in all_vpcs:
if attached_resource in all_vpcs[vpc]['subnets']:
manage_dictionary(all_vpcs[vpc]['subnets'][attached_resource],
'flow_logs', [])
if flow_log_id not in all_vpcs[vpc]['subnets'][
attached_resource]['flow_logs']:
all_vpcs[vpc]['subnets'][attached_resource][
'flow_logs'].append(flow_log_id)
break
else:
printError('Resource %s attached to flow logs is not handled' %
attached_resource)
def parse_parameter_group(self, global_params, region, parameter_group):
parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn')
parameter_group['name'] = parameter_group.pop('DBParameterGroupName')
api_client = api_clients[region]
try:
parameters = handle_truncated_response(
api_client.describe_db_parameters,
{'DBParameterGroupName': parameter_group['name']},
['Parameters'])['Parameters']
for parameter in parameters:
param = {}
param['value'] = parameter[
'ParameterValue'] if 'ParameterValue' in parameter else None
param['source'] = parameter['Source']
manage_dictionary(parameter_group, 'parameters', {})
parameter_group['parameters'][
parameter['ParameterName']] = param
except Exception as e:
printException(e)
printError('Failed fetching DB parameters for %s' %
parameter_group['name'])
# Save
(self).parameter_groups[parameter_group['name']] = parameter_group
def parse_parameter_group(self, global_params, region, parameter_group):
parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn')
parameter_group['name'] = parameter_group.pop('DBParameterGroupName')
api_client = api_clients[region]
try:
parameters = handle_truncated_response(
api_client.describe_db_parameters,
{'DBParameterGroupName': parameter_group['name']},
['Parameters'])['Parameters']
manage_dictionary(parameter_group, 'parameters', {})
for parameter in parameters:
if not parameter['IsModifiable']:
# Discard non-modifiable parameters
continue
parameter_name = parameter.pop('ParameterName')
parameter_group['parameters'][parameter_name] = parameter
except Exception as e:
printException(e)
printError('Failed fetching DB parameters for %s' %
parameter_group['name'])
# Save
parameter_group_id = self.get_non_provider_id(parameter_group['name'])
(self).parameter_groups[parameter_group_id] = parameter_group
def set_emr_vpc_ids_callback(aws_config, current_config, path, current_path, vpc_id, callback_args):
if vpc_id != 'TODO':
return
region = current_path[3]
vpc_id = sg_id = subnet_id = None
pop_list = []
for cluster_id in current_config['clusters']:
cluster = current_config['clusters'][cluster_id]
if 'EmrManagedMasterSecurityGroup' in cluster['Ec2InstanceAttributes']:
sg_id = cluster['Ec2InstanceAttributes']['EmrManagedMasterSecurityGroup']
elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']:
subnet_id = cluster['Ec2InstanceAttributes']['RequestedEc2SubnetIds']
else:
printError('Unable to determine VPC id for EMR cluster %s' % str(cluster_id))
continue
if sg_id in sg_map:
vpc_id = sg_map[sg_id]['vpc_id']
pop_list.append(cluster_id)
else:
sid_found = False
if subnet_id:
for sid in subnet_id:
if sid in subnet_map:
vpc_id = subnet_map[sid]['vpc_id']
pop_list.append(cluster_id)
sid_found = True
if not sid_found:
printError('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id)))
continue
if vpc_id:
region_vpcs_config = get_object_at(aws_config, current_path)
manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}})
region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster
for cluster_id in pop_list:
current_config['clusters'].pop(cluster_id)
if len(current_config['clusters']) == 0:
callback_args['clear_list'].append(region)
def prepare_cloudformation_params(stack_name,
template_path,
template_parameters,
resource_type,
tags=[],
need_on_failure=False):
"""
:param api_client:
:param stack_name:
:param template_path:
:param template_parameters: List of parameter keys and values
:param quiet:
:return:
"""
printDebug('Reading CloudFormation template from %s' % template_path)
template_body = read_file(template_path)
params = {}
params['%sName' % resource_type] = stack_name
params['TemplateBody'] = template_body
if len(template_parameters):
params['Parameters'] = []
it = iter(template_parameters)
for param in it:
printError('Param:: %s' % param)
params['Parameters'].append({
'ParameterKey': param,
'ParameterValue': next(it)
})
if len(tags):
params['Tags'] = tags
if re_iam_capability.match(template_body):
params['Capabilities'] = ['CAPABILITY_NAMED_IAM']
if need_on_failure:
params['OnFailure'] = 'ROLLBACK'
return params
def set_emr_vpc_ids_callback(aws_config, current_config, path, current_path, vpc_id, callback_args):
if vpc_id != 'TODO':
return
region = current_path[3]
vpc_id = sg_id = subnet_id = None
pop_list = []
for cluster_id in current_config['clusters']:
cluster = current_config['clusters'][cluster_id]
if 'EmrManagedMasterSecurityGroup' in cluster['Ec2InstanceAttributes']:
sg_id = cluster['Ec2InstanceAttributes']['EmrManagedMasterSecurityGroup']
elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']:
subnet_id = cluster['Ec2InstanceAttributes']['RequestedEc2SubnetIds']
else:
printError('Unable to determine VPC id for EMR cluster %s' % str(cluster_id))
continue
if sg_id in sg_map:
vpc_id = sg_map[sg_id]['vpc_id']
pop_list.append(cluster_id)
else:
sid_found = False
if subnet_id:
for sid in subnet_id:
if sid in subnet_map:
vpc_id = subnet_map[sid]['vpc_id']
pop_list.append(cluster_id)
sid_found = True
if not sid_found:
printError('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id)))
continue
if vpc_id:
region_vpcs_config = get_object_at(aws_config, current_path)
manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}})
region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster
for cluster_id in pop_list:
current_config['clusters'].pop(cluster_id)
if len(current_config['clusters']) == 0:
callback_args['clear_list'].append(region)
def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info):
try:
if 'policy' in bucket_info:
bucket_info['secure_transport'] = 'Disabled'
for statement in bucket_info['policy']['Statement']:
# evaluate statement to see if it contains a condition disallowing HTTP transport
# TODO this might not cover all cases
if 'Condition' in statement and \
'Bool' in statement['Condition'] and \
'aws:SecureTransport' in statement['Condition']['Bool'] and \
((statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and \
statement['Effect'] == 'Deny') or
(statement['Condition']['Bool']['aws:SecureTransport'] == 'true' and \
statement['Effect'] == 'Allow')):
bucket_info['secure_transport'] = 'Enabled'
return True
else:
bucket_info['secure_transport'] = 'Disabled'
return True
except Exception as e:
printError('Failed to get evaluate bucket policy for %s: %s' %
(bucket_name, e))
bucket_info['secure_transport'] = 'Unknown'
return False
def sort_vpc_flow_logs_callback(aws_config, current_config, path, current_path, flow_log_id, callback_args):
attached_resource = current_config['ResourceId']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(current_path[0:4], ['vpcs', attached_resource])
try:
attached_vpc = get_object_at(aws_config, vpc_path)
except Exception as e:
printDebug('It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % (flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(attached_vpc['subnets'][subnet_id], 'flow_logs', [])
if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id)
elif attached_resource.startswith('subnet-'):
subnet_path = combine_paths(current_path[0:4], ['vpcs', subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource])
subnet = get_object_at(aws_config, subnet_path)
manage_dictionary(subnet, 'flow_logs', [])
if flow_log_id not in subnet['flow_logs']:
subnet['flow_logs'].append(flow_log_id)
else:
printError('Resource %s attached to flow logs is not handled' % attached_resource)
def wait_for_operation(api_client,
stack_set_name,
operation_id,
timeout=5 * 60,
increment=5):
printDebug('Waiting for operation %s on stack set %s...' %
(operation_id, stack_set_name))
timer = 0
status = ''
while True:
if timer >= timeout:
printError('Timed out.')
break
info = api_client.describe_stack_set_operation(
StackSetName=stack_set_name, OperationId=operation_id)
status = info['StackSetOperation']['Status']
if status not in ['RUNNING', 'STOPPING']:
break
printError(
'Operation status is \'%s\'... waiting %d seconds until next check...'
% (status, increment))
time.sleep(increment)
timer += increment
return 'Operation %s is %s' % (operation_id, status)
def load(self):
"""
Load the definition of the rule, searching in the specified rule dirs first, then in the built-in definitions
:return: None
"""
file_name_valid = False
rule_type_valid = False
file_path = None
# Look for a locally-defined rule
for rule_dir in self.rule_dirs:
try:
file_path = os.path.join(
rule_dir, self.file_name) if rule_dir else self.file_name
except Exception as e:
printError('Failed to load file %s: %e' % (self.file_name, e))
if os.path.isfile(file_path):
self.file_path = file_path
file_name_valid = True
break
# Look for a built-in rule
if not file_name_valid:
for rule_type in self.rule_types:
if self.file_name.startswith(rule_type):
self.file_path = os.path.join(self.rules_data_path,
self.file_name)
rule_type_valid = True
file_name_valid = True
break
if not rule_type_valid:
for rule_type in self.rule_types:
self.file_path = os.path.join(self.rules_data_path,
rule_type, self.file_name)
if os.path.isfile(self.file_path):
file_name_valid = True
break
else:
if os.path.isfile(self.file_path):
file_name_valid = True
if not file_name_valid:
printError('Error: could not find %s' % self.file_name)
else:
try:
with open(self.file_path, 'rt') as f:
self.string_definition = f.read()
self.load_from_string_definition()
except Exception as e:
# printException(e)
printError('Failed to load rule defined in %s: %s' %
(self.file_name, e))
def check_versions(min_version, installed_version, max_version, package_name, strict = False):
"""
:param min_version:
:param installed_version:
:param max_version:
:param package_name:
:return:
"""
if not min_version:
# If no minimum version was specified, pass
return True
if StrictVersion(installed_version) < StrictVersion(min_version):
printError('Error: the version of %s installed on this system (%s) is too old. You need at least version %s to run this tool.' % (package_name, OPINEL_VERSION, min_version))
return False
if max_version and StrictVersion(installed_version) >= StrictVersion(max_version):
printError('Warning: ther version of %s installed on this system (%s) is too recent; you may experience unexpected runtime errors as versions above %s have not been tested.' % (package_name, installed_version, max_version))
if strict:
printError('Warning treated as error.')
return False
return True
def main():
# Parse arguments
parser = OpinelArgumentParser()
parser.add_argument('debug')
parser.add_argument('profile')
parser.add_argument('csv-credentials')
parser.add_argument('mfa-serial')
parser.add_argument('mfa-code')
parser.parser.add_argument('--role-arn',
dest='role_arn',
default=None,
help='ARN of the assumed role.')
parser.parser.add_argument('--external-id',
dest='external_id',
default=None,
help='External ID to use when assuming the role.')
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Arguments
profile_name = args.profile[0]
if args.csv_credentials:
# Read credentials from a CSV file
credentials = {}
credentials['AccessKeyId'], credentials['SecretAccessKey'], credentials['SerialNumber'] = read_creds_from_csv(args.csv_credentials)
if not credentials['AccessKeyId'] or not credentials['SecretAccessKey']:
printError('Failed to read credentials from %s' % args.csv_credentials)
return 42
use_found_credentials = True
else:
# Check for migration from existing profile to no-mfa profile
use_found_credentials = False
credentials = read_creds_from_aws_credentials_file(profile_name)
if 'AccessKeyId' in credentials and credentials['AccessKeyId'] != None and credentials['SecretAccessKey'] != None and credentials['SerialNumber'] == None and credentials['SessionToken'] == None:
if prompt_4_yes_no('Found long-lived credentials for the profile \'%s\'. Do you want to use those when configuring MFA' % profile_name):
use_found_credentials = True
iam_client = connect_service('iam', credentials)
try:
printInfo('Trying to read the MFA serial number associated with this IAM user...')
user_name = iam_client.get_user()['User']['UserName']
mfa_devices = iam_client.list_mfa_devices(UserName = user_name)['MFADevices']
credentials['SerialNumber'] = mfa_devices[0]['SerialNumber']
except Exception as e:
printException(e)
pass
if not use_found_credentials:
# Get values
credentials['AccessKeyId'] = prompt_4_value('AWS Access Key ID: ', no_confirm = True)
credentials['SecretAccessKey'] = prompt_4_value('AWS Secret Access Key: ', no_confirm = True)
if 'SerialNumber' not in credentials or not credentials['SerialNumber']:
credentials['SerialNumber'] = prompt_4_mfa_serial()
# Check for overwrite
while True:
c = read_creds_from_aws_credentials_file(profile_name)
if 'AccessKeyId' in c and c['AccessKeyId']:
if not prompt_4_yes_no('The profile \'%s\' already exists. Do you want to overwrite the existing values' % profile_name):
if not prompt_4_yes_no('Do you want to create a new profile with these credentials'):
printError('Configuration aborted.')
return
profile_name = prompt_4_value('Profile name: ')
else:
break
else:
break
# Write values to credentials file
write_creds_to_aws_credentials_file(profile_name, credentials)
# Delete CSV file?
if args.csv_credentials and prompt_4_yes_no('Do you want to delete the CSV file that contains your long-lived credentials?'):
os.remove(args.csv_credentials)
def update_ruleset(self, rules_dir):
if rules_dir == None:
return
self.available_rules = {}
parameterized_rules = []
self.services = []
for rule in self.ruleset['rules']:
rule['filename'] = rule['filename'].replace('rules/', '')
if not 'args' in rule:
self.available_rules[rule['filename']] = rule
else:
parameterized_rules.append(rule)
# Add default location
rules_dir.append(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'data/findings'))
for dir in rules_dir:
rule_filenames = [
f for f in os.listdir(dir)
if os.path.isfile(os.path.join(dir, f))
]
for rule_filename in rule_filenames:
self.services.append(rule_filename.split('-')[0].lower())
printDebug('Loading %s' % rule_filename)
with open('%s/%s' % (dir, rule_filename), 'rt') as f:
rule = json.load(f)
if not 'key' in rule and not 'arg_names' in rule:
# Non-parameterized rule, save it
if rule_filename in self.available_rules:
self.available_rules[rule_filename].update(rule)
else:
self.available_rules[rule_filename] = rule
self.available_rules[rule_filename][
'enabled'] = False
if 'level' not in self.available_rules[
rule_filename]:
self.available_rules[rule_filename][
'level'] = 'danger'
self.available_rules[rule_filename][
'filename'] = rule_filename
else:
# Parameterized rules, find all occurences and save N times
parameterized_rule_found = False
for prule in parameterized_rules:
if prule['filename'] == rule_filename:
parameterized_rule_found = True
for k in rule:
prule[k] = set_argument_values(
rule[k], prule['args'], convert=True
) if k != 'conditions' else rule[k]
key = prule.pop(
'key'
) if 'key' in prule else prule['filename']
args = prule.pop('args')
if not 'arg_names' in prule:
printError('No arg names key in %s' %
rule_filename)
continue
arg_names = prule.pop('arg_names')
if len(args) != len(arg_names):
printError(
'Error: rule %s expects %d arguments but was provided %d.'
% (rule_filename, len(arg_names),
len(args)))
continue
prule['args'] = []
for (arg_name,
arg_value) in zip(arg_names, args):
prule['args'].append({
'arg_name': arg_name,
'arg_value': arg_value
})
if 'level' not in prule:
prule['level'] = 'danger'
self.available_rules[key] = prule
if not parameterized_rule_found:
# Save once with no parameters
self.available_rules[rule_filename] = rule
self.available_rules[rule_filename][
'enabled'] = False
if 'level' not in self.available_rules[
rule_filename]:
self.available_rules[rule_filename][
'level'] = 'danger'
self.available_rules[rule_filename][
'filename'] = rule_filename
args = []
for a in rule['arg_names']:
args.append({'arg_name': a, 'arg_value': ''})
self.available_rules[rule_filename][
'args'] = args
printDebug(
'Saving rule without parameter value: %s' %
rule_filename)
def authenticate(self, key_file=None, user_account=None, service_account=None, azure_cli=None, azure_msi=None,
azure_service_principal=None, azure_file_auth=None, azure_user_credentials=None, **kargs):
"""
Implements authentication for the Azure provider using azure-cli.
Refer to https://docs.microsoft.com/en-us/python/azure/python-sdk-azure-authenticate?view=azure-python.
:return:
"""
try:
if azure_cli:
cli_credentials, self.aws_account_id = get_azure_cli_credentials() # TODO: Remove aws_account_id
self.credentials = AzureCredentials(cli_credentials, self.aws_account_id)
return True
elif azure_msi:
credentials = MSIAuthentication()
# Get the subscription ID
subscription_client = SubscriptionClient(credentials)
try:
# Tries to read the subscription list
subscription = next(subscription_client.subscriptions.list())
self.aws_account_id = subscription.subscription_id
except StopIteration:
# If the VM cannot read subscription list, ask Subscription ID:
self.aws_account_id = input('Subscription ID: ')
self.credentials = AzureCredentials(credentials, self.aws_account_id)
return True
elif azure_file_auth:
with open(azure_file_auth) as f:
data = json.loads(f.read())
subscription_id = data.get('subscriptionId')
tenant_id = data.get('tenantId')
client_id = data.get('clientId')
client_secret = data.get('clientSecret')
self.aws_account_id = tenant_id # TODO this is for AWS
credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
self.credentials = AzureCredentials(credentials, subscription_id)
return True
elif azure_service_principal:
subscription_id = input("Subscription ID: ")
tenant_id = input("Tenant ID: ")
client_id = input("Client ID: ")
client_secret = getpass("Client secret: ")
self.aws_account_id = tenant_id # TODO this is for AWS
credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
self.credentials = AzureCredentials(credentials, subscription_id)
return True
elif azure_user_credentials:
username = input("Username: "******"Password: "******"" # TODO this is for AWS
self.credentials = AzureCredentials(credentials, self.aws_account_id)
return True
except Exception as e:
printError('Failed to authenticate to Azure')
printException(e)
return False
def analyze(self, aws_config):
"""
:param aws_config:
"""
printInfo('Analyzing AWS config...')
# TODO: reset violations for all services in scope (maybe this can be done somewhere else (e.g. loading)
for finding_path in self.rules:
for rule in self.rules[finding_path]:
printDebug('Processing %s rule[%s]: "%s"' %
(finding_path.split('.')[0], self.rule_type[:-1],
self.rules[finding_path][rule]['description']))
path = finding_path.split('.')
service = path[0]
manage_dictionary(aws_config['services'][service],
self.rule_type, {})
aws_config['services'][service][self.rule_type][rule] = {}
aws_config['services'][service][
self.rule_type][rule]['description'] = self.rules[
finding_path][rule]['description']
aws_config['services'][service][self.rule_type][rule][
'path'] = self.rules[finding_path][rule]['path']
if self.rule_type == 'findings':
aws_config['services'][service][self.rule_type][rule][
'level'] = self.rules[finding_path][rule]['level']
if 'id_suffix' in self.rules[finding_path][rule]:
aws_config['services'][service][
self.rule_type][rule]['id_suffix'] = self.rules[
finding_path][rule]['id_suffix']
if 'display_path' in self.rules[finding_path][rule]:
aws_config['services'][service][
self.rule_type][rule]['display_path'] = self.rules[
finding_path][rule]['display_path']
try:
aws_config['services'][service][
self.rule_type][rule]['items'] = recurse(
aws_config['services'], aws_config['services'],
path, [], self.rules[finding_path][rule], True)
aws_config['services'][service][self.rule_type][rule][
'dashboard_name'] = self.rules[finding_path][rule][
'dashboard_name'] if 'dashboard_name' in self.rules[
finding_path][rule] else '??'
aws_config['services'][service][self.rule_type][rule][
'checked_items'] = self.rules[finding_path][rule][
'checked_items'] if 'checked_items' in self.rules[
finding_path][rule] else 0
aws_config['services'][service][
self.rule_type][rule]['flagged_items'] = len(
aws_config['services'][service][
self.rule_type][rule]['items'])
aws_config['services'][service][
self.rule_type][rule]['service'] = service
aws_config['services'][service][self.rule_type][rule][
'rationale'] = self.rules[finding_path][rule][
'rationale'] if 'rationale' in self.rules[
finding_path][rule] else 'N/A'
except Exception as e:
printError('Failed to process rule defined in %s.json' %
rule)
# Fallback if process rule failed to ensure report creation and data dump still happen
aws_config['services'][service][
self.rule_type][rule]['checked_items'] = 0
aws_config['services'][service][
self.rule_type][rule]['flagged_items'] = 0
printException(e)
def recurse(all_info,
current_info,
target_path,
current_path,
config,
add_suffix=False):
"""
:param all_info:
:param current_info:
:param target_path:
:param current_path:
:param config:
:param add_suffix:
:return:
"""
results = []
if len(target_path) == 0:
# Dashboard: count the number of processed resources here
manage_dictionary(config, 'checked_items', 0)
config['checked_items'] = config['checked_items'] + 1
# Test for conditions...
if pass_conditions(all_info, current_path,
copy.deepcopy(config['conditions'])):
if add_suffix and 'id_suffix' in config:
current_path.append(config['id_suffix'])
results.append('.'.join(current_path))
# Return the flagged items...
config['flagged_items'] = len(results)
return results
target_path = copy.deepcopy(target_path)
current_path = copy.deepcopy(current_path)
attribute = target_path.pop(0)
if type(current_info) == dict:
if attribute in current_info:
split_path = copy.deepcopy(current_path)
split_path.append(attribute)
results = results + recurse(all_info, current_info[attribute],
target_path, split_path, config,
add_suffix)
elif attribute == 'id':
for key in current_info:
split_target_path = copy.deepcopy(target_path)
split_current_path = copy.deepcopy(current_path)
split_current_path.append(key)
split_current_info = current_info[key]
results = results + recurse(
all_info, split_current_info, split_target_path,
split_current_path, config, add_suffix)
# To handle lists properly, I would have to make sure the list is properly ordered and I can use the index to consistently access an object... Investigate (or do not use lists)
elif type(current_info) == list:
for index, split_current_info in enumerate(current_info):
split_current_path = copy.deepcopy(current_path)
split_current_path.append(str(index))
results = results + recurse(all_info, split_current_info,
copy.deepcopy(target_path),
split_current_path, config, add_suffix)
else:
printError('Error: unhandled case, typeof(current_info) = %s' %
type(current_info))
printError(str(current_info))
raise Exception
return results
def delete_user(iam_client,
user,
mfa_serial=None,
keep_user=False,
terminated_groups=[]):
"""
Delete IAM user
:param iam_client:
:param user:
:param mfa_serial:
:param keep_user:
:param terminated_groups:
:return:
"""
errors = []
printInfo('Deleting user %s...' % user)
# Delete access keys
try:
aws_keys = get_access_keys(iam_client, user)
for aws_key in aws_keys:
try:
printInfo(
'Deleting access key ID %s... ' % aws_key['AccessKeyId'],
False)
iam_client.delete_access_key(
AccessKeyId=aws_key['AccessKeyId'], UserName=user)
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
except Exception as e:
printException(e)
printError('Failed to get access keys for user %s.' % user)
# Deactivate and delete MFA devices
try:
mfa_devices = iam_client.list_mfa_devices(UserName=user)['MFADevices']
for mfa_device in mfa_devices:
serial = mfa_device['SerialNumber']
try:
printInfo('Deactivating MFA device %s... ' % serial, False)
iam_client.deactivate_mfa_device(SerialNumber=serial,
UserName=user)
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
delete_virtual_mfa_device(iam_client, serial)
if mfa_serial:
delete_virtual_mfa_device(iam_client, mfa_serial)
except Exception as e:
printException(e)
printError(
'Faile to fetch/delete MFA device serial number for user %s.' %
user)
errors.append(e.response['Error']['Code'])
# Remove IAM user from groups
try:
groups = iam_client.list_groups_for_user(UserName=user)['Groups']
for group in groups:
try:
printInfo('Removing from group %s... ' % group['GroupName'],
False)
iam_client.remove_user_from_group(GroupName=group['GroupName'],
UserName=user)
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
except Exception as e:
printException(e)
printError('Failed to fetch IAM groups for user %s.' % user)
errors.append(e.response['Error']['Code'])
# Delete login profile
login_profile = []
try:
login_profile = iam_client.get_login_profile(
UserName=user)['LoginProfile']
except Exception as e:
pass
try:
if len(login_profile):
printInfo('Deleting login profile... ', False)
iam_client.delete_login_profile(UserName=user)
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
# Delete inline policies
try:
printInfo('Deleting inline policies... ', False)
policies = iam_client.list_user_policies(UserName=user)
for policy in policies['PolicyNames']:
iam_client.delete_user_policy(UserName=user, PolicyName=policy)
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
# Detach managed policies
try:
printInfo('Detaching managed policies... ', False)
policies = iam_client.list_attached_user_policies(UserName=user)
for policy in policies['AttachedPolicies']:
iam_client.detach_user_policy(UserName=user,
PolicyArn=policy['PolicyArn'])
printInfo('Success')
except Exception as e:
printInfo('Failed')
printException(e)
errors.append(e.response['Error']['Code'])
# Delete IAM user
try:
if not keep_user:
iam_client.delete_user(UserName=user)
printInfo('User %s deleted.' % user)
else:
for group in terminated_groups:
add_user_to_group(iam_client, group, user)
except Exception as e:
printException(e)
printError('Failed to delete user.')
errors.append(e.response['Error']['Code'])
pass
return errors
def set_definition(self,
rule_definitions,
attributes=None,
ip_ranges=None,
params=None):
"""
Update every attribute of the rule by setting the argument values as necessary
:param parameterized_input:
:param arg_values:
:param convert:
:return:
"""
attributes = [] if attributes is None else attributes
ip_ranges = [] if ip_ranges is None else ip_ranges
params = {} if params is None else params
try:
string_definition = rule_definitions[
self.filename].string_definition
# Load condition dependencies
definition = json.loads(string_definition)
definition['conditions'] += self.conditions
loaded_conditions = []
for condition in definition['conditions']:
if condition[0].startswith('_INCLUDE_('):
include = re.findall(r'_INCLUDE_\((.*?)\)',
condition[0])[0]
#new_conditions = load_data(include, key_name = 'conditions')
rules_path = '%s/%s' % (self.data_path, include)
with open(rules_path, 'rt') as f:
new_conditions = f.read()
for (i, value) in enumerate(condition[1]):
new_conditions = re.sub(condition[1][i],
condition[2][i],
new_conditions)
new_conditions = json.loads(
new_conditions)['conditions']
loaded_conditions.append(new_conditions)
else:
loaded_conditions.append(condition)
definition['conditions'] = loaded_conditions
string_definition = json.dumps(definition)
# Set parameters
parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)',
string_definition)
for param in parameters:
index = int(param[1])
if len(self.args) <= index:
string_definition = string_definition.replace(param[0], '')
elif type(self.args[index]) == list:
value = '[ %s ]' % ', '.join('"%s"' % v
for v in self.args[index])
string_definition = string_definition.replace(
'"%s"' % param[0], value)
else:
string_definition = string_definition.replace(
param[0], self.args[index])
# Strip dots if necessary
stripdots = re_strip_dots.findall(string_definition)
for value in stripdots:
string_definition = string_definition.replace(
value[0], value[1].replace('.', ''))
definition = json.loads(string_definition)
# Set special values (IP ranges, AWS account ID, ...)
for condition in definition['conditions']:
if type(condition) != list or len(condition) == 1 or type(
condition[2]) == list:
continue
for testcase in testcases:
result = testcase['regex'].match(condition[2])
if result and (testcase['name'] == 'ip_ranges_from_file'
or testcase['name']
== 'ip_ranges_from_local_file'):
filename = result.groups()[0]
conditions = result.groups()[1] if len(
result.groups()) > 1 else []
# TODO :: handle comma here...
if filename == ip_ranges_from_args:
prefixes = []
for filename in ip_ranges:
prefixes += read_ip_ranges(
filename,
local_file=True,
ip_only=True,
conditions=conditions)
condition[2] = prefixes
break
else:
local_file = True if testcase[
'name'] == 'ip_ranges_from_local_file' else False
condition[2] = read_ip_ranges(
filename,
local_file=local_file,
ip_only=True,
conditions=conditions)
break
break
elif result:
condition[2] = params[testcase['name']]
break
if len(attributes) == 0:
attributes = [attr for attr in definition]
for attr in attributes:
if attr in definition:
setattr(self, attr, definition[attr])
if hasattr(self, 'path'):
self.service = format_service_name(self.path.split('.')[0])
if not hasattr(self, 'key'):
setattr(self, 'key', self.filename)
setattr(self, 'key', self.key.replace('.json', ''))
if self.key_suffix:
setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix))
except Exception as e:
# printException(e)
printError('Failed to set definition %s: %s' % (self.filename, e))
def test_all_finding_rules(self):
test_dir = os.path.dirname(os.path.realpath(__file__))
test_ruleset_file_name = os.path.join(test_dir, 'data/ruleset-test.json')
#FIXME this is only for AWS
with open(os.path.join(test_dir, '../ScoutSuite/providers/aws/rules/rulesets/default.json'), 'rt') as f:
ruleset = json.load(f)
rule_counters = {'found': 0, 'tested': 0, 'verified': 0}
for file_name in ruleset['rules']:
rule_counters['found'] += 1
test_config_file_name = os.path.join(test_dir, 'data/rule-configs/%s' % file_name)
if not os.path.isfile(test_config_file_name):
continue
rule_counters['tested'] += 1
test_ruleset = {'rules': {}, 'about': 'regression test'}
test_ruleset['rules'][file_name] = []
rule = ruleset['rules'][file_name][0]
rule['enabled'] = True
test_ruleset['rules'][file_name].append(rule)
with open(test_ruleset_file_name, 'wt') as f:
f.write(json.dumps(test_ruleset, indent=4))
# printError('Ruleset ::')
# printError(str(test_ruleset))
rules = Ruleset(filename=test_ruleset_file_name)
pe = ProcessingEngine(rules)
with open(test_config_file_name, 'rt') as f:
dummy_provider = DummyObject()
test_config_dict = json.load(f)
for key in test_config_dict:
setattr(dummy_provider, key, test_config_dict[key])
pe.run(dummy_provider)
service = file_name.split('-')[0]
findings = dummy_provider.services[service]['findings']
findings = findings[list(findings.keys())[0]]['items']
test_result_file_name = os.path.join(test_dir, 'data/rule-results/%s' % file_name)
if not os.path.isfile(test_result_file_name):
printError('Expected findings:: ')
printError(json.dumps(findings, indent=4))
continue
rule_counters['verified'] += 1
with open(test_result_file_name, 'rt') as f:
items = json.load(f)
try:
assert (set(sorted(findings)) == set(sorted(items)))
except Exception as e:
printError('Expected items:\n %s' % json.dumps(sorted(items)))
printError('Reported items:\n %s' % json.dumps(sorted(findings)))
assert (False)
printError('Existing rules: %d' % rule_counters['found'])
printError('Processed rules: %d' % rule_counters['tested'])
printError('Verified rules: %d' % rule_counters['verified'])
def _get_projects(self, parent_type, parent_id):
"""
Returns all the projects in a given organization or folder. For a project_id it only returns the project
details.
"""
if parent_type not in ['project', 'organization', 'folder', 'all']:
return None
projects = []
# FIXME can't currently be done with API client library as it consumes v1 which doesn't support folders
"""
resource_manager_client = resource_manager.Client(credentials=self.credentials)
project_list = resource_manager_client.list_projects()
for p in project_list:
if p.parent['id'] == self.organization_id and p.status == 'ACTIVE':
projects.append(p.project_id)
"""
resource_manager_client_v1 = gcp_connect_service(
service='cloudresourcemanager', credentials=self.credentials)
resource_manager_client_v2 = gcp_connect_service(
service='cloudresourcemanager-v2', credentials=self.credentials)
try:
if parent_type == 'project':
project_response = resource_manager_client_v1.projects().list(
filter='id:%s' % parent_id).execute()
if 'projects' in project_response.keys():
for project in project_response['projects']:
if project['lifecycleState'] == "ACTIVE":
projects.append(project)
elif parent_type == 'all':
project_response = resource_manager_client_v1.projects().list(
).execute()
if 'projects' in project_response.keys():
for project in project_response['projects']:
if project['lifecycleState'] == "ACTIVE":
projects.append(project)
else:
# get parent children projects
request = resource_manager_client_v1.projects().list(
filter='parent.id:%s' % parent_id)
while request is not None:
response = request.execute()
if 'projects' in response.keys():
for project in response['projects']:
if project['lifecycleState'] == "ACTIVE":
projects.append(project)
request = resource_manager_client_v1.projects().list_next(
previous_request=request, previous_response=response)
# get parent children projects in children folders recursively
folder_response = resource_manager_client_v2.folders().list(
parent='%ss/%s' % (parent_type, parent_id)).execute()
if 'folders' in folder_response.keys():
for folder in folder_response['folders']:
projects.extend(
self._get_projects(
"folder", folder['name'].strip(u'folders/')))
printInfo("Found {} project(s) to scan.".format(len(projects)))
except Exception as e:
printError('Unable to list accessible Projects')
printException(e)
finally:
return projects
def match_security_groups_and_resources_callback(aws_config, current_config, path, current_path, resource_id, callback_args):
service = current_path[1]
original_resource_path = combine_paths(copy.deepcopy(current_path), [ resource_id ])
resource = get_object_at(aws_config, original_resource_path)
if not 'resource_id_path' in callback_args:
resource_type = current_path[-1]
resource_path = copy.deepcopy(current_path)
resource_path.append(resource_id)
else:
resource_path = combine_paths(copy.deepcopy(current_path), callback_args['resource_id_path'])
resource_id = resource_path[-1]
resource_type = resource_path[-2]
if 'status_path' in callback_args:
status_path = combine_paths(copy.deepcopy(original_resource_path), callback_args['status_path'])
resource_status = get_object_at(aws_config, status_path).replace('.', '_')
else:
resource_status = None
unknown_vpc_id = True if current_path[4] != 'vpcs' else False
# Issue 89 & 91 : can instances have no security group?
try:
try:
sg_attribute = get_object_at(resource, callback_args['sg_list_attribute_name'])
except:
return
if type(sg_attribute) != list:
sg_attribute = [ sg_attribute ]
for resource_sg in sg_attribute:
if type(resource_sg) == dict:
sg_id = resource_sg[callback_args['sg_id_attribute_name']]
else:
sg_id = resource_sg
if unknown_vpc_id:
vpc_id = sg_map[sg_id]['vpc_id']
sg_base_path = copy.deepcopy(current_path[0:4])
sg_base_path[1] = 'ec2'
sg_base_path = sg_base_path + [ 'vpcs', vpc_id, 'security_groups' ]
else:
sg_base_path = copy.deepcopy(current_path[0:6])
sg_base_path[1] = 'ec2'
sg_base_path.append('security_groups')
sg_path = copy.deepcopy(sg_base_path)
sg_path.append(sg_id)
sg = get_object_at(aws_config, sg_path)
# Add usage information
manage_dictionary(sg, 'used_by', {})
manage_dictionary(sg['used_by'], service, {})
manage_dictionary(sg['used_by'][service], 'resource_type', {})
manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {} if resource_status else [])
if resource_status:
manage_dictionary(sg['used_by'][service]['resource_type'][resource_type], resource_status, [])
if not resource_id in sg['used_by'][service]['resource_type'][resource_type][resource_status]:
sg['used_by'][service]['resource_type'][resource_type][resource_status].append(resource_id)
else:
sg['used_by'][service]['resource_type'][resource_type].append(resource_id)
except Exception as e:
if resource_type in ['elbs', 'functions']:
pass
else:
region = current_path[3]
vpc_id = current_path[5]
if vpc_id == ec2_classic:
pass
else:
printError('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region))
printException(e)
def authenticate(self, key_file=None, user_account=None, service_account=None, **kargs):
"""
Implement authentication for the GCP provider
Refer to https://google-auth.readthedocs.io/en/stable/reference/google.auth.html.
:return:
"""
if user_account:
# disable GCP warning about using User Accounts
warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")
pass # Nothing more to do
elif service_account:
client_secrets_path = os.path.abspath(key_file) # TODO this is probably wrong
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path
else:
printError('Failed to authenticate to GCP - no supported account type')
return False
try:
self.credentials, project_id = google.auth.default()
if self.credentials:
if self.project_id:
# service_account credentials with project_id will follow this path
self.projects = self._get_projects(parent_type='project',
parent_id=self.project_id)
self.aws_account_id = self.project_id # FIXME this is for AWS
self.profile = self.project_id # FIXME this is for AWS
elif self.organization_id:
self.projects = self._get_projects(parent_type='organization',
parent_id=self.organization_id)
self.aws_account_id = self.organization_id # FIXME this is for AWS
self.profile = self.organization_id # FIXME this is for AWS
elif self.folder_id:
self.projects = self._get_projects(parent_type='folder',
parent_id=self.folder_id)
self.aws_account_id = self.folder_id # FIXME this is for AWS
self.profile = self.folder_id # FIXME this is for AWS
elif service_account: # We know that project_id hasn't been provided and that we have a service account
self.projects = self._get_projects(parent_type='service-account',
parent_id=self.project_id)
self.aws_account_id = self.credentials.service_account_email # FIXME this is for AWS
self.profile = self.credentials.service_account_email # FIXME this is for AWS
else:
# FIXME this will fail if no default project is set in gcloud config. This is caused because html.py is looking for a profile to build the report
self.project_id = project_id
self.projects = self._get_projects(parent_type='project',
parent_id=self.project_id)
self.aws_account_id = self.project_id # FIXME this is for AWS
self.profile = self.project_id # FIXME this is for AWS
# TODO this shouldn't be done here? but it has to in order to init with projects...
self.services.set_projects(projects=self.projects)
return True
else:
return False
except google.auth.exceptions.DefaultCredentialsError as e:
printError('Failed to authenticate to GCP')
printException(e)
return False
except googleapiclient.errors.HttpError as e:
printError('Failed to authenticate to GCP')
printException(e)
return False
def run(self, cloud_provider, skip_dashboard=False):
# Clean up existing findings
for service in cloud_provider.services:
cloud_provider.services[service][self.ruleset.rule_type] = {}
# Process each rule
for finding_path in self._filter_rules(self.rules,
cloud_provider.service_list):
for rule in self.rules[finding_path]:
if not rule.enabled: # or rule.service not in []: # TODO: handle this...
continue
printDebug('Processing %s rule[%s]: "%s"' %
(rule.service, rule.filename, rule.description))
finding_path = rule.path
path = finding_path.split('.')
service = path[0]
manage_dictionary(cloud_provider.services[service],
self.ruleset.rule_type, {})
cloud_provider.services[service][self.ruleset.rule_type][
rule.key] = {}
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['description'] = rule.description
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['path'] = rule.path
for attr in ['level', 'id_suffix', 'display_path']:
if hasattr(rule, attr):
cloud_provider.services[service][
self.ruleset.rule_type][rule.key][attr] = getattr(
rule, attr)
try:
setattr(rule, 'checked_items', 0)
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['items'] = recurse(cloud_provider.services,
cloud_provider.services,
path, [], rule, True)
if skip_dashboard:
continue
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['dashboard_name'] = rule.dashboard_name
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['checked_items'] = rule.checked_items
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['flagged_items'] = len(
cloud_provider.services[service][
self.ruleset.rule_type][rule.key]['items'])
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['service'] = rule.service
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['rationale'] = rule.rationale if hasattr(
rule, 'rationale') else 'No description available.'
except Exception as e:
printException(e)
printError('Failed to process rule defined in %s' %
rule.filename)
# Fallback if process rule failed to ensure report creation and data dump still happen
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['checked_items'] = 0
cloud_provider.services[service][self.ruleset.rule_type][
rule.key]['flagged_items'] = 0
def match_security_groups_and_resources_callback(aws_config, current_config,
path, current_path,
resource_id, callback_args):
service = current_path[1]
original_resource_path = combine_paths(copy.deepcopy(current_path),
[resource_id])
resource = get_object_at(aws_config, original_resource_path)
if not 'resource_id_path' in callback_args:
resource_type = current_path[-1]
resource_path = copy.deepcopy(current_path)
resource_path.append(resource_id)
else:
resource_path = combine_paths(copy.deepcopy(current_path),
callback_args['resource_id_path'])
resource_id = resource_path[-1]
resource_type = resource_path[-2]
#print('Resource path: %s' % resource_path)
#print('Resource type: %s' % resource_type)
#print('Resource id: %s' % resource_id)
if 'status_path' in callback_args:
status_path = combine_paths(copy.deepcopy(original_resource_path),
callback_args['status_path'])
#print('Status path: %s' % status_path)
resource_status = get_object_at(aws_config, status_path)
else:
resource_status = None
sg_base_path = copy.deepcopy(current_path[0:6])
sg_base_path[1] = 'ec2'
sg_base_path.append('security_groups')
# Issue 89 & 91 : can instances have no security group?
try:
for resource_sg in resource[callback_args['sg_list_attribute_name']]:
sg_id = resource_sg[callback_args['sg_id_attribute_name']]
sg_path = copy.deepcopy(sg_base_path)
sg_path.append(sg_id)
sg = get_object_at(aws_config, sg_path)
# Add usage information
manage_dictionary(sg, 'used_by', {})
manage_dictionary(sg['used_by'], service, {})
manage_dictionary(sg['used_by'][service], 'resource_type', {})
manage_dictionary(sg['used_by'][service]['resource_type'],
resource_type, {} if resource_status else [])
if resource_status:
manage_dictionary(
sg['used_by'][service]['resource_type'][resource_type],
resource_status, [])
if not resource_id in sg['used_by'][service]['resource_type'][
resource_type][resource_status]:
sg['used_by'][service]['resource_type'][resource_type][
resource_status].append(resource_id)
else:
sg['used_by'][service]['resource_type'][resource_type].append(
resource_id)
except Exception as e:
region = current_path[3]
vpc_id = current_path[5]
if vpc_id == ec2_classic and resource_type == 'elbs':
pass
else:
printError('Failed to parse %s in %s in %s' %
(resource_type, vpc_id, region))
printException(e)
def main():
# Parse arguments
parser = ListallArgumentParser()
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Support multiple environments
for profile_name in args.profile:
# Load the config
try:
report = Scout2Report(args.provider, profile_name, args.report_dir,
args.timestamp)
aws_config = report.jsrw.load_from_file(AWSCONFIG)
services = aws_config['service_list']
except Exception as e:
printException(e)
printError(
'Error, failed to load the configuration for profile %s' %
profile_name)
continue
# Create a ruleset with only whatever rules were specified...
if args.config:
rule_filename = args.config
ruleset = TmpRuleset(rule_dirs=[os.getcwd()],
rule_filename=args.config,
rule_args=args.config_args)
elif len(args.path) > 0:
# Create a local tmp rule
rule_dict = {'description': 'artifact'}
rule_dict['path'] = args.path[0]
rule_dict['conditions'] = []
rule_filename = 'listall-artifact.json'
with open(os.path.join(os.getcwd(), rule_filename), 'wt') as f:
f.write(json.dumps(rule_dict))
ruleset = TmpRuleset(rule_dirs=[os.getcwd()],
rule_filename=rule_filename,
rule_args=[])
else:
printError(
'Error, you must provide either a rule configuration file or the path to the resources targeted.'
)
continue
# Process the rule
pe = ProcessingEngine(ruleset)
pe.run(aws_config, skip_dashboard=True)
# Retrieve items
rule = ruleset.rules[rule_filename][0]
rule_service = rule.service.lower()
rule_key = rule.key
rule_type = rule.rule_type
resources = aws_config['services'][rule_service][rule_type][rule_key][
'items']
# Set the keys to output
if len(args.keys):
# 1. Explicitly provided on the CLI
rule.keys = args.keys
elif len(args.keys_file):
# 2. Explicitly provided files that contain the list of keys
rule.keys = []
for filename in args.keys_file:
with open(filename, 'rt') as f:
rule.keys += json.load(f)['keys']
else:
try:
# 3. Load default set of keys based on path
target_path = rule.display_path if hasattr(
rule, 'display_path') else rule.path
listall_configs_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'output/data/listall-configs')
target_file = os.path.join(listall_configs_dir,
'%s.json' % target_path)
if os.path.isfile(target_file):
with open(target_file, 'rt') as f:
rule.keys = json.load(f)['keys']
except:
# 4. Print the object name
rule.keys = ['name']
# Prepare the output format
(lines, template) = format_listall_output(args.format_file[0], None,
args.format, rule)
# Print the output
printInfo(
generate_listall_output(lines, resources, aws_config, template,
[]))
def get_value_at(all_info, current_path, key, to_string=False):
"""
Get value located at a given path.
:param all_info: All of the services' data
:param current_path: The value of the `path` variable defined in the finding file
:param key: The key that is being requested
:param to_string: Whether or not the returned value should be casted as a string
:return: The value in `all_info` indicated by the `key` in `current_path`
"""
keys = key.split('.')
if keys[-1] == 'id':
target_obj = current_path[len(keys) - 1]
else:
if key == 'this':
target_path = current_path
elif '.' in key:
target_path = []
for i, key in enumerate(keys):
# If 'id', replace by value
if key == 'id':
target_path.append(current_path[i])
# If empty key and value is an index, keep the index
elif key == '' and i < len(
current_path) and current_path[i].isdigit():
target_path.append(int(current_path[i]))
# Otherwise, use key
else:
target_path.append(key)
if len(keys) > len(current_path):
target_path = target_path + keys[len(target_path):]
else:
target_path = copy.deepcopy(current_path)
target_path.append(key)
target_obj = all_info
for p in target_path:
try:
if type(target_obj) == list and type(target_obj[0]) == dict:
target_obj = target_obj[int(p)]
# TODO ensure this additional condition didn't break anything
elif type(target_obj) == list and type(p) == int:
target_obj = target_obj[p]
elif type(target_obj) == list:
target_obj = p
elif p == '':
target_obj = target_obj
else:
try:
target_obj = target_obj[p]
except Exception as e:
printError('Current path: %s' % str(current_path))
printException(e)
raise Exception
except Exception as e:
printError('Current path: %s' % str(current_path))
printException(e)
raise Exception
if to_string:
return str(target_obj)
else:
return target_obj
def test_all_finding_rules(self):
test_dir = os.path.dirname(os.path.realpath(__file__))
test_ruleset_file_name = os.path.join(test_dir, 'data/ruleset-test.json')
with open(os.path.join(test_dir, '../AWSScout2/rules/data/rulesets/default.json'), 'rt') as f:
ruleset = json.load(f)
rule_counters = {'found': 0, 'tested': 0, 'verified': 0}
for file_name in ruleset['rules']:
rule_counters['found'] += 1
test_config_file_name = os.path.join(test_dir, 'data/rule-configs/%s' % file_name)
if not os.path.isfile(test_config_file_name):
continue
rule_counters['tested'] += 1
test_ruleset = {'rules': {}, 'about': 'regression test'}
test_ruleset['rules'][file_name] = []
rule = ruleset['rules'][file_name][0]
rule['enabled'] = True
test_ruleset['rules'][file_name].append(rule)
with open(test_ruleset_file_name, 'wt') as f:
f.write(json.dumps(test_ruleset, indent = 4))
# printError('Ruleset ::')
# printError(str(test_ruleset))
rules = Ruleset(filename = test_ruleset_file_name)
pe = ProcessingEngine(rules)
with open(test_config_file_name, 'rt') as f:
aws_config = json.load(f)
pe.run(aws_config)
service = file_name.split('-')[0]
findings = aws_config['services'][service]['findings']
findings = findings[list(findings.keys())[0]]['items']
test_result_file_name = os.path.join(test_dir, 'data/rule-results/%s' % file_name)
if not os.path.isfile(test_result_file_name):
printError('Expected findings:: ')
printError(json.dumps(findings, indent = 4))
continue
rule_counters['verified'] += 1
with open(test_result_file_name, 'rt') as f:
items = json.load(f)
try:
assert(set(sorted(findings)) == set(sorted(items)))
except Exception as e:
printError('Expected items:\n %s' % json.dumps(sorted(items)))
printError('Reported items:\n %s' % json.dumps(sorted(findings)))
assert(False)
printError('Existing rules: %d' % rule_counters['found'])
printError('Processed rules: %d' % rule_counters['tested'])
printError('Verified rules: %d' % rule_counters['verified'])
def main():
# Parse arguments
parser = ListallArgumentParser()
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Support multiple environments
for profile_name in args.profile:
# Load the config
try:
report = Scout2Report(profile_name, args.report_dir, args.timestamp)
aws_config = report.jsrw.load_from_file(AWSCONFIG)
services = aws_config['service_list']
except Exception as e:
printException(e)
printError('Error, failed to load the configuration for profile %s' % profile_name)
continue
# Create a ruleset with only whatever rules were specified...
if args.config:
rule_filename = args.config
ruleset = TmpRuleset(rule_dirs = [os.getcwd()], rule_filename = args.config, rule_args = args.config_args)
elif len(args.path) > 0:
# Create a local tmp rule
rule_dict = {'description': 'artifact'}
rule_dict['path'] = args.path[0]
rule_dict['conditions'] = []
rule_filename = 'listall-artifact.json'
with open(os.path.join(os.getcwd(), rule_filename), 'wt') as f:
f.write(json.dumps(rule_dict))
ruleset = TmpRuleset(rule_dirs = [os.getcwd()], rule_filename = rule_filename, rule_args = [])
else:
printError('Error, you must provide either a rule configuration file or the path to the resources targeted.')
continue
# Process the rule
pe = ProcessingEngine(ruleset)
pe.run(aws_config, skip_dashboard = True)
# Retrieve items
rule = ruleset.rules[rule_filename][0]
rule_service = rule.service.lower()
rule_key = rule.key
rule_type = rule.rule_type
resources = aws_config['services'][rule_service][rule_type][rule_key]['items']
# Set the keys to output
if len(args.keys):
# 1. Explicitly provided on the CLI
rule.keys = args.keys
elif len(args.keys_file):
# 2. Explicitly provided files that contain the list of keys
rule.keys = []
for filename in args.keys_file:
with open(filename, 'rt') as f:
rule.keys += json.load(f)['keys']
else:
try:
# 3. Load default set of keys based on path
target_path = rule.display_path if hasattr(rule, 'display_path') else rule.path
listall_configs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'output/data/listall-configs')
target_file = os.path.join(listall_configs_dir, '%s.json' % target_path)
if os.path.isfile(target_file):
with open(target_file, 'rt') as f:
rule.keys = json.load(f)['keys']
except:
# 4. Print the object name
rule.keys = ['name']
# Prepare the output format
(lines, template) = format_listall_output(args.format_file[0], None, args.format, rule)
# Print the output
printInfo(generate_listall_output(lines, resources, aws_config, template, []))
def read_creds(profile_name,
csv_file=None,
mfa_serial_arg=None,
mfa_code=None,
force_init=False,
role_session_name='opinel'):
"""
Read credentials from anywhere (CSV, Environment, Instance metadata, config/credentials)
:param profile_name:
:param csv_file:
:param mfa_serial_arg:
:param mfa_code:
:param force_init:
:param role_session_name:
:return:
"""
first_sts_session = False
expiration = None
credentials = init_creds()
if csv_file:
# Read credentials from a CSV file that was provided
credentials['AccessKeyId'], credentials[
'SecretAccessKey'], credentials[
'SerialNumber'] = read_creds_from_csv(csv_file)
elif profile_name == 'default':
# Try reading credentials from environment variables (Issue #11) if the profile name is 'default'
credentials = read_creds_from_environment_variables()
if ('AccessKeyId' not in credentials or not credentials['AccessKeyId']
) and not csv_file and profile_name == 'default':
credentials = read_creds_from_ec2_instance_metadata()
if not credentials['AccessKeyId'] and not csv_file:
# Lookup if a role is defined in ~/.aws/config
role_arn, source_profile, role_mfa_serial = read_profile_from_aws_config_file(
profile_name)
if role_arn and source_profile:
# Lookup cached credentials
try:
cached_credentials_filename = get_cached_credentials_filename(
profile_name, role_arn)
with open(cached_credentials_filename, 'rt') as f:
assume_role_data = json.load(f)
credentials = assume_role_data['Credentials']
expiration = dateutil.parser.parse(
credentials['Expiration'])
expiration = expiration.replace(tzinfo=None)
current = datetime.datetime.utcnow()
if expiration < current:
print('Role\'s credentials have expired on %s' %
credentials['Expiration'])
except Exception as e:
pass
if not expiration or expiration < current or credentials[
'AccessKeyId'] == None:
credentials = read_creds(source_profile)
if role_mfa_serial:
credentials['SerialNumber'] = role_mfa_serial
# Auto prompt for a code...
if not mfa_code:
credentials['TokenCode'] = prompt_4_mfa_code()
credentials = assume_role(profile_name, credentials, role_arn,
role_session_name)
# Read from ~/.aws/credentials
else:
credentials = read_creds_from_aws_credentials_file(profile_name)
if credentials['SessionToken']:
if 'Expiration' in credentials and credentials['Expiration']:
expiration = dateutil.parser.parse(
credentials['Expiration'])
expiration = expiration.replace(tzinfo=None)
current = datetime.datetime.utcnow()
if expiration < current:
printInfo('Saved STS credentials expired on %s' %
credentials['Expiration'])
force_init = True
else:
force_init = True
sts_credentials = credentials
else:
first_sts_session = True
if force_init or (mfa_serial_arg and mfa_code):
credentials = read_creds_from_aws_credentials_file(
profile_name if first_sts_session else '%s-nomfa' %
profile_name)
if not credentials['AccessKeyId']:
printInfo(
'Warning: Unable to determine STS token expiration; later API calls may fail.'
)
credentials = sts_credentials
else:
if mfa_serial_arg:
credentials['SerialNumber'] = mfa_serial_arg
if mfa_code:
credentials['TokenCode'] = mfa_code
if 'AccessKeyId' in credentials and credentials[
'AccessKeyId']:
credentials = init_sts_session(profile_name,
credentials)
# If we don't have valid creds by now, print an error message
if 'AccessKeyId' not in credentials or credentials[
'AccessKeyId'] == None or 'SecretAccessKey' not in credentials or credentials[
'SecretAccessKey'] == None:
printError(
'Error: could not find AWS credentials. Use the --help option for more information.'
)
if not 'AccessKeyId' in credentials:
credentials = {'AccessKeyId': None}
return credentials
def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error):
"""
Fetch the targets, required as each provider may have particularities
:return:
"""
targets = []
try:
regions = self.get_regions()
zones = self.get_zones()
# Create a list with all combinations for method parameters
list_params_list = []
# Dict for all the elements to combine
combination_elements = {'project_placeholder': [project['projectId'] for project in self.projects],
'region_placeholder': regions,
'zone_placeholder': zones}
# Get a list of {{}} terms
sources = re.findall("{{(.*?)}}", str(list_params.values()))
# Remove keys from combinations if they aren't in the sources
confirmed_combination_elements = {}
for source in sources:
confirmed_combination_elements[source] = combination_elements[source]
# Build a list of the possible combinations
combinations = self._dict_product(confirmed_combination_elements)
for combination in combinations:
l = list_params.copy()
for k, v in l.items():
k1 = re.findall("{{(.*?)}}", v)
if k1:
l[k] = l[k].replace('{{%s}}' % k1[0], combination[k1[0]])
list_params_list.append(l)
for list_params_combination in list_params_list:
try:
if self.library_type == 'cloud_client_library':
# TODO this should be more modular
# this is only for stackdriverlogging
if self.service == 'stackdriverlogging':
api_client.project = list_params_combination.pop('project')
response = method(**list_params_combination)
# TODO this should be more modular
# this is only for kubernetesengine
if isinstance(response, container_v1.types.ListClustersResponse):
targets += response.clusters
else:
targets += list(response)
# Remove client as it's unpickleable and adding the object to the Queue will pickle
# The client is later re-inserted in each Config
for t in targets:
if hasattr(t, '_client'):
t._client = None
if self.library_type == 'api_client_library':
# TODO need to handle long responses
request = method(**list_params_combination)
while request is not None:
response = request.execute()
if 'items' in response:
targets += response['items']
# TODO this should be more modular
# this is only for cloudresourcemanager
if 'bindings' in response:
targets += response['bindings']
# TODO this should be more modular
# this is only for IAM
if 'accounts' in response:
targets += response['accounts']
# TODO need to define the _next to handle long responses
# request = method_next(previous_request=request,
# previous_response=response)
request = None
except HttpError as e:
error_json = json.loads(e.content)
if error_json['error']['message'] not in self.error_list:
self.error_list.append(error_json['error']['message'])
printError(error_json['error']['message'])
except PermissionDenied as e:
printError("%s: %s - %s" % (e.message, self.service, self.targets))
except Exception as e:
printException(e)
except HttpError as e:
error_json = json.loads(e.content)
if error_json['error']['message'] not in self.error_list:
self.error_list.append(error_json['error']['message'])
printError(error_json['error']['message'])
except Exception as e:
printException(e)
finally:
return targets
def match_security_groups_and_resources_callback(aws_config, current_config, path, current_path, resource_id, callback_args):
service = current_path[1]
original_resource_path = combine_paths(copy.deepcopy(current_path), [ resource_id ])
resource = get_object_at(aws_config, original_resource_path)
if not 'resource_id_path' in callback_args:
resource_type = current_path[-1]
resource_path = copy.deepcopy(current_path)
resource_path.append(resource_id)
else:
resource_path = combine_paths(copy.deepcopy(current_path), callback_args['resource_id_path'])
resource_id = resource_path[-1]
resource_type = resource_path[-2]
if 'status_path' in callback_args:
status_path = combine_paths(copy.deepcopy(original_resource_path), callback_args['status_path'])
resource_status = get_object_at(aws_config, status_path).replace('.', '_')
else:
resource_status = None
unknown_vpc_id = True if current_path[4] != 'vpcs' else False
# Issue 89 & 91 : can instances have no security group?
try:
try:
sg_attribute = get_object_at(resource, callback_args['sg_list_attribute_name'])
except:
return
if type(sg_attribute) != list:
sg_attribute = [ sg_attribute ]
for resource_sg in sg_attribute:
if type(resource_sg) == dict:
sg_id = resource_sg[callback_args['sg_id_attribute_name']]
else:
sg_id = resource_sg
if unknown_vpc_id:
vpc_id = sg_map[sg_id]['vpc_id']
sg_base_path = copy.deepcopy(current_path[0:4])
sg_base_path[1] = 'ec2'
sg_base_path = sg_base_path + [ 'vpcs', vpc_id, 'security_groups' ]
else:
sg_base_path = copy.deepcopy(current_path[0:6])
sg_base_path[1] = 'ec2'
sg_base_path.append('security_groups')
sg_path = copy.deepcopy(sg_base_path)
sg_path.append(sg_id)
sg = get_object_at(aws_config, sg_path)
# Add usage information
manage_dictionary(sg, 'used_by', {})
manage_dictionary(sg['used_by'], service, {})
manage_dictionary(sg['used_by'][service], 'resource_type', {})
manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {} if resource_status else [])
if resource_status:
manage_dictionary(sg['used_by'][service]['resource_type'][resource_type], resource_status, [])
if not resource_id in sg['used_by'][service]['resource_type'][resource_type][resource_status]:
sg['used_by'][service]['resource_type'][resource_type][resource_status].append(resource_id)
else:
sg['used_by'][service]['resource_type'][resource_type].append(resource_id)
except Exception as e:
region = current_path[3]
vpc_id = current_path[5]
if vpc_id == ec2_classic and resource_type == 'elbs':
pass
else:
printError('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region))
printException(e)
def main():
# Parse arguments
parser = OpinelArgumentParser()
parser.add_argument('debug')
parser.add_argument('profile')
parser.add_argument('regions')
parser.add_argument('partition-name')
parser.add_argument('bucket-name')
parser.parser.add_argument('--aws-account-id',
dest='aws_account_id',
default=[ None ],
nargs='+',
help='Bleh.')
parser.parser.add_argument('--from',
dest='from_date',
default=[ None ],
nargs='+',
help='Bleh.')
parser.parser.add_argument('--to',
dest='to_date',
default=[ None ],
nargs='+',
help='Bleh.')
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Arguments
profile_name = args.profile[0]
try:
from_date = datetime.datetime.strptime(args.from_date[0], "%Y/%m/%d").date()
to_date = datetime.datetime.strptime(args.to_date[0], "%Y/%m/%d").date()
delta = to_date - from_date
except Exception as e:
printException(e)
printError('Error: dates must be formatted of the following format YYYY/MM/DD')
return 42
if delta.days < 0:
printError('Error: your \'to\' date is earlier than your \'from\' date')
return 42
# Search for AWS credentials
credentials = read_creds(profile_name)
if not credentials['AccessKeyId']:
return 42
# Fetch AWS account ID
if not args.aws_account_id[0]:
printInfo('Fetching the AWS account ID...')
aws_account_id = get_aws_account_id(credentials)
else:
aws_account_id = args.aws_account_id[0]
global cloudtrail_log_path
cloudtrail_log_path = cloudtrail_log_path.replace('AWS_ACCOUNT_ID', aws_account_id)
# Create download dir
if not os.path.exists(download_folder):
os.makedirs(download_folder)
# Iterate through regions
s3_clients = {}
for region in build_region_list('cloudtrail', args.regions, args.partition_name):
# Connect to CloudTrail
cloudtrail_client = connect_service('cloudtrail', credentials, region)
if not cloudtrail_client:
continue
# Get information about the S3 bucket that receives CloudTrail logs
trails = cloudtrail_client.describe_trails()
for trail in trails['trailList']:
bucket_name = trail['S3BucketName']
prefix = trail['S3KeyPrefix'] if 'S3KeyPrefix' in trail else ''
# Connect to S3
manage_dictionary(s3_clients, region, connect_service('s3', credentials, region))
target_bucket_region = get_s3_bucket_location(s3_clients[region], bucket_name)
manage_dictionary(s3_clients, target_bucket_region, connect_service('s3', credentials, target_bucket_region))
s3_client = s3_clients[target_bucket_region]
# Generate base path for files
log_path = os.path.join(prefix, cloudtrail_log_path.replace('REGION', region))
# Download files
printInfo('Downloading log files in %s... ' % region, False)
keys = []
for i in range(delta.days + 1):
day = from_date + timedelta(days=i)
folder_path = os.path.join(log_path, day.strftime("%Y/%m/%d"))
try:
objects = handle_truncated_response(s3_client.list_objects, {'Bucket': bucket_name, 'Prefix': folder_path}, ['Contents'])
for o in objects['Contents']:
keys.append([o['Key'], 0])
except Exception as e:
printException(e)
pass
thread_work(keys, download_object, params = {'Bucket': bucket_name, 'S3Client': s3_client}, num_threads = 100)
printInfo('Done')
# Iterate through files and gunzip 'em
printInfo('Decompressing files...')
gzlogs = []
for root, dirnames, filenames in os.walk(download_folder):
for filename in filenames:
gzlogs.append(filename)
thread_work(gzlogs, gunzip_file, num_threads = 30)
def display_qr_code(png, seed):
"""
Display MFA QR code
:param png:
:param seed:
:return:
"""
# This NamedTemporaryFile is deleted as soon as it is closed, so
# return it to caller, who must close it (or program termination
# could cause it to be cleaned up, that's fine too).
# If we don't keep the file around until after the user has synced
# his MFA, the file will possibly be already deleted by the time
# the operating system gets around to execing the browser, if
# we're using a browser.
qrcode_file = tempfile.NamedTemporaryFile(suffix='.png',
delete=True,
mode='wt')
qrcode_file.write(png)
qrcode_file.flush()
if _fabulous_available:
fabulous.utils.term.bgcolor = 'white'
with open(qrcode_file.name, 'rb') as png_file:
print(fabulous.image.Image(png_file, 100))
else:
graphical_browsers = [
webbrowser.BackgroundBrowser, webbrowser.Mozilla,
webbrowser.Galeon, webbrowser.Chrome, webbrowser.Opera,
webbrowser.Konqueror
]
if sys.platform[:3] == 'win':
graphical_browsers.append(webbrowser.WindowsDefault)
elif sys.platform == 'darwin':
graphical_browsers.append(webbrowser.MacOSXOSAScript)
browser_type = None
try:
browser_type = type(webbrowser.get())
except webbrowser.Error:
pass
if browser_type in graphical_browsers:
printError(
"Unable to print qr code directly to your terminal, trying a web browser."
)
webbrowser.open('file://' + qrcode_file.name)
else:
printInfo(
"Unable to print qr code directly to your terminal, and no graphical web browser seems available."
)
printInfo(
"But, the qr code file is temporarily available as this file:")
printInfo("\n %s\n" % qrcode_file.name)
printInfo(
"Alternately, if you feel like typing the seed manually into your MFA app:"
)
# this is a base32-encoded binary string (for case
# insensitivity) which is then dutifully base64-encoded by
# amazon before putting it on the wire. so the actual
# secret is b32decode(b64decode(seed)), and what users
# will need to type in to their app is just
# b64decode(seed). print that out so users can (if
# desperate) type in their MFA app.
printInfo("\n %s\n" % base64.b64decode(seed))
return qrcode_file
def authenticate(self, user_account=None, service_account=None, **kargs):
"""
Implement authentication for the GCP provider
Refer to https://google-auth.readthedocs.io/en/stable/reference/google.auth.html.
:return:
"""
if user_account:
# disable GCP warning about using User Accounts
warnings.filterwarnings(
"ignore",
"Your application has authenticated using end user credentials"
)
pass # Nothing more to do
elif service_account:
client_secrets_path = os.path.abspath(service_account)
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path
else:
printError(
'Failed to authenticate to GCP - no supported account type')
return False
try:
self.credentials, project_id = google.auth.default()
if self.credentials:
# All projects to which the user / Service Account has access to
if self.all_projects:
self.projects = self._get_projects(parent_type='all',
parent_id=None)
if service_account and hasattr(self.credentials,
'service_account_email'):
self.aws_account_id = self.credentials.service_account_email # FIXME this is for AWS
else:
self.aws_account_id = 'GCP' # FIXME this is for AWS
self.profile = 'GCP' # FIXME this is for AWS
# Project passed through the CLI
elif self.project_id:
self.projects = self._get_projects(
parent_type='project', parent_id=self.project_id)
self.aws_account_id = self.project_id # FIXME this is for AWS
self.profile = self.project_id # FIXME this is for AWS
# Folder passed through the CLI
elif self.folder_id:
self.projects = self._get_projects(
parent_type='folder', parent_id=self.folder_id)
self.aws_account_id = self.folder_id # FIXME this is for AWS
self.profile = self.folder_id # FIXME this is for AWS
# Organization passed through the CLI
elif self.organization_id:
self.projects = self._get_projects(
parent_type='organization',
parent_id=self.organization_id)
self.aws_account_id = self.organization_id # FIXME this is for AWS
self.profile = self.organization_id # FIXME this is for AWS
# Project inferred from default configuration
elif project_id:
self.projects = self._get_projects(parent_type='project',
parent_id=project_id)
self.aws_account_id = project_id # FIXME this is for AWS
self.profile = project_id # FIXME this is for AWS
# Raise exception if none of the above
else:
printInfo(
"Could not infer the Projects to scan and no default Project ID was found."
)
return False
# TODO this shouldn't be done here? but it has to in order to init with projects...
self.services.set_projects(projects=self.projects)
return True
else:
return False
except google.auth.exceptions.DefaultCredentialsError as e:
printError('Failed to authenticate to GCP')
printException(e)
return False
except googleapiclient.errors.HttpError as e:
printError('Failed to authenticate to GCP')
printException(e)
return False
