async def _fetch(self, service, regions):
try:
print_info('Fetching resources for the {} service'.format(
format_service_name(service)))
service_config = getattr(self, service)
# call fetch method for the service
if 'fetch_all' in dir(service_config):
method_args = {
'credentials': self.credentials,
'regions': regions
}
if self._is_provider('aws'):
if service != 'iam':
method_args['partition_name'] = get_partition_name(
self.credentials)
await service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
await service_config.finalize()
else:
print_debug('No method to fetch service %s.' % service)
except Exception as e:
print_error('Error: could not fetch %s configuration.' % service)
print_exception(e)
async def get_credential_reports(self):
client = AWSFacadeUtils.get_client('iam', self.session)
response = await run_concurrently(client.generate_credential_report)
if response['State'] != 'COMPLETE':
print_error('Failed to generate a credential report.')
return []
report = (await
run_concurrently(client.get_credential_report))['Content']
# The report is a CSV string. The first row contains the name of each column. The next rows
# each represent an individual account. This algorithm provides a simple initial parsing.
lines = report.splitlines()
keys = lines[0].decode('utf-8').split(',')
credential_reports = []
for line in lines[1:]:
credential_report = {}
values = line.decode('utf-8').split(',')
for key, value in zip(keys, values):
credential_report[key] = value
credential_reports.append(credential_report)
return credential_reports
def authenticate(self, user_account=None, service_account=None, **kargs):
"""
Implements authentication for the GCP provider
Refer to https://google-auth.readthedocs.io/en/stable/reference/google.auth.html.
"""
if user_account:
# disable GCP warning about using User Accounts
warnings.filterwarnings(
"ignore",
"Your application has authenticated using end user credentials"
)
pass # Nothing more to do
elif service_account:
client_secrets_path = os.path.abspath(service_account)
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path
else:
print_error(
'Failed to authenticate to GCP - no supported account type')
raise AuthenticationException()
credentials, default_project_id = google.auth.default()
if not credentials:
raise AuthenticationException()
credentials.is_service_account = service_account is not None
credentials.default_project_id = default_project_id
return credentials
def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity,
allowed_iam_entity, full_path, policy_type,
policy_name):
global policy
allowed_buckets = []
# By default, all buckets are allowed
for bucket_name in s3_info['buckets']:
allowed_buckets.append(bucket_name)
if policy_type == 'InlinePolicies':
policy = iam_info[iam_entity.title(
)][allowed_iam_entity]['Policies'][policy_name]['PolicyDocument']
elif policy_type == 'ManagedPolicies':
policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument']
else:
print_error('Error, found unknown policy type.')
for statement in policy['Statement']:
for target_path in statement['NotResource']:
parts = target_path.split('/')
bucket_name = parts[0].split(':')[-1]
path = '/' + '/'.join(parts[1:]) if len(parts) > 1 else '/'
if (path == '/' or path == '/*') and (bucket_name
in allowed_buckets):
# Remove bucket from list
allowed_buckets.remove(bucket_name)
elif bucket_name == '*':
allowed_buckets = []
policy_info = {policy_type: {}}
policy_info[policy_type][policy_name] = \
iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][
full_path][
policy_type][policy_name]
for bucket_name in allowed_buckets:
self._update_iam_permissions(s3_info, bucket_name, iam_entity,
allowed_iam_entity, policy_info)
def sort_vpc_flow_logs_callback(self, current_config, path, current_path,
flow_log_id, callback_args):
attached_resource = current_config['ResourceId']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(current_path[0:4],
['vpcs', attached_resource])
try:
attached_vpc = get_object_at(self, vpc_path)
except Exception:
print_debug(
'It appears that the flow log %s is attached to a resource that was previously deleted (%s).'
% (flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(attached_vpc['subnets'][subnet_id],
'flow_logs', [])
if flow_log_id not in attached_vpc['subnets'][subnet_id][
'flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(
flow_log_id)
elif attached_resource.startswith('subnet-'):
subnet_path = combine_paths(current_path[0:4], [
'vpcs', self.subnet_map[attached_resource]['vpc_id'],
'subnets', attached_resource
])
subnet = get_object_at(self, subnet_path)
manage_dictionary(subnet, 'flow_logs', [])
if flow_log_id not in subnet['flow_logs']:
subnet['flow_logs'].append(flow_log_id)
else:
print_error('Resource %s attached to flow logs is not handled' %
attached_resource)
def get_s3_acls(api_client, bucket_name, bucket, key_name=None):
try:
grantees = {}
if key_name:
grants = api_client.get_object_acl(Bucket=bucket_name, Key=key_name)
else:
grants = api_client.get_bucket_acl(Bucket=bucket_name)
for grant in grants['Grants']:
if 'ID' in grant['Grantee']:
grantee = grant['Grantee']['ID']
display_name = grant['Grantee']['DisplayName'] if \
'DisplayName' in grant['Grantee'] else grant['Grantee']['ID']
elif 'URI' in grant['Grantee']:
grantee = grant['Grantee']['URI'].split('/')[-1]
display_name = s3_group_to_string(grant['Grantee']['URI'])
else:
grantee = display_name = 'Unknown'
permission = grant['Permission']
manage_dictionary(grantees, grantee, {})
grantees[grantee]['DisplayName'] = display_name
if 'URI' in grant['Grantee']:
grantees[grantee]['URI'] = grant['Grantee']['URI']
manage_dictionary(grantees[grantee], 'permissions', init_s3_permissions())
set_s3_permissions(grantees[grantee]['permissions'], permission)
return grantees
except Exception as e:
print_error('Failed to get ACL configuration for %s: %s' % (bucket_name, e))
return {}
def load_from_string_definition(self):
try:
definition = json.loads(self.string_definition)
for attr in definition:
setattr(self, attr, definition[attr])
except Exception as e:
print_error('Failed to load string definition %s: %s' % (self.string_definition, str(e)))
def load(self, rule_type, quiet=False):
"""
Open a JSON file defining a ruleset and load it into a Ruleset object
:param rule_type: TODO
:param quiet: TODO
:return:
"""
if self.filename and os.path.exists(self.filename):
try:
with open(self.filename) as f:
ruleset = json.load(f)
self.about = ruleset['about'] if 'about' in ruleset else ''
self.rules = {}
for filename in ruleset['rules']:
self.rules[filename] = []
for rule in ruleset['rules'][filename]:
self.handle_rule_versions(filename, rule_type, rule)
except Exception as e:
print_exception(f'Ruleset file {self.filename} contains malformed JSON: {e}')
self.rules = []
self.about = ''
else:
self.rules = []
if not quiet:
print_error('Error: the file %s does not exist.' % self.filename)
def _get_instance_backups(self, instance, params):
backups_dict = {}
try:
backups = params['api_client'].backupRuns().list(
project=instance['project'],
instance=instance['name']).execute()
# this is triggered when there are no backups
except AttributeError as e:
return backups_dict
except Exception as e:
print_error('Failed to fetch backups for SQL instance %s: %s' %
(instance['name'], e))
return backups_dict
try:
if 'items' in backups:
for backup in backups['items']:
if backup['status'] == 'SUCCESSFUL':
backups_dict[backup['id']] = {
'backup_url': backup['selfLink'],
'creation_timestamp': backup['endTime'],
'status': backup['status'],
'type': backup['type']
}
return backups_dict
except Exception as e:
print_error('Failed to parse backups for SQL instance %s: %s' %
(instance['name'], e))
return None
def fetch(self, credentials, services=None, regions=None):
services = [] if services is None else services
regions = [] if regions is None else regions
for service in vars(self):
try:
# skip services
if services != [] and service not in services:
continue
service_config = getattr(self, service)
# call fetch method for the service
if 'fetch_all' in dir(service_config):
method_args = {
'credentials': credentials,
'regions': regions
}
if self._is_provider('aws'):
if service != 'iam':
method_args['partition_name'] = get_partition_name(
credentials)
service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
service_config.finalize()
else:
print_debug('No method to fetch service %s.' % service)
except Exception as e:
print_error('Error: could not fetch %s configuration.' %
service)
print_exception(e)
async def is_api_enabled(self, project_id, service):
"""
Given a project ID and service name, this method tries to determine if the service's API is enabled
"""
serviceusage_client = self._build_arbitrary_client('serviceusage',
'v1',
force_new=True)
services = serviceusage_client.services()
try:
request = services.list(parent=f'projects/{project_id}')
services_response = await GCPFacadeUtils.get_all(
'services', request, services)
except Exception as e:
print_exception(
f'Could not fetch the state of services for project \"{project_id}\", '
f'including {format_service_name(service.lower())} in the execution',
{'exception': e})
return True
# These are hardcoded endpoint correspondences as there's no easy way to do this.
if service == 'IAM':
endpoint = 'iam'
elif service == 'KMS':
endpoint = 'cloudkms'
elif service == 'CloudStorage':
endpoint = 'storage-component'
elif service == 'CloudSQL':
endpoint = 'sql-component'
elif service == 'ComputeEngine':
endpoint = 'compute'
elif service == 'KubernetesEngine':
endpoint = 'container'
elif service == 'StackdriverLogging':
endpoint = 'logging'
elif service == 'StackdriverMonitoring':
endpoint = 'monitoring'
else:
print_debug(
'Could not validate the state of the {} API for project \"{}\", '
'including it in the execution'.format(
format_service_name(service.lower()), project_id))
return True
for s in services_response:
if endpoint in s.get('name'):
if s.get('state') == 'ENABLED':
return True
else:
print_info(
'{} API not enabled for project \"{}\", skipping'.
format(format_service_name(service.lower()),
project_id))
return False
print_error(
f'Could not validate the state of the {format_service_name(service.lower())} API '
f'for project \"{project_id}\", including it in the execution')
return True
def get_s3_bucket_policy(api_client, bucket_name, bucket_info):
try:
bucket_info['policy'] = json.loads(api_client.get_bucket_policy(Bucket=bucket_name)['Policy'])
return True
except Exception as e:
if not (type(e) == ClientError and e.response['Error']['Code'] == 'NoSuchBucketPolicy'):
print_error('Failed to get bucket policy for %s: %s' % (bucket_name, e))
return False
def get_cloudstorage_bucket_logging(bucket, bucket_dict):
try:
bucket_dict['logging_enabled'] = bucket.get_logging() is not None
return True
except Exception as e:
print_error('Failed to get bucket logging configuration for %s: %s' % (bucket.name, e))
bucket_dict['logging_enabled'] = None
return False
def fetch_credential_reports(self, credentials, ignore_exception=False):
"""
Fetch the credential report
:param: api_client
:type: FOO
:param: ignore_exception : initiate credential report creation as not always ready
:type: Boolean
"""
credential_reports = {}
try:
api_client = connect_service('iam', credentials, silent=True)
response = api_client.generate_credential_report()
if response['State'] != 'COMPLETE':
if not ignore_exception:
print_error('Failed to generate a credential report.')
return
report = api_client.get_credential_report()['Content']
lines = report.splitlines()
keys = lines[0].decode('utf-8').split(',')
self.fetchstatuslogger.counts['credential_reports'][
'discovered'] = len(lines) - 1
for line in lines[1:]:
credential_report = {}
values = line.decode('utf-8').split(',')
user_id = values[0]
for key, value in zip(keys, values):
credential_report[key] = value
credential_report['password_last_used'] = self._sanitize_date(
credential_report['password_last_used'])
credential_report[
'access_key_1_last_used_date'] = self._sanitize_date(
credential_report['access_key_1_last_used_date'])
credential_report[
'access_key_2_last_used_date'] = self._sanitize_date(
credential_report['access_key_2_last_used_date'])
credential_report['last_used'] = self._compute_last_used(
credential_report)
credential_report['name'] = user_id
credential_report['id'] = user_id
manage_dictionary(credential_reports, user_id,
credential_report)
self.fetchstatuslogger.counts['credential_reports'][
'fetched'] = len(credential_reports)
self.credential_reports = credential_reports
except Exception as e:
if ignore_exception:
return
print_error('Failed to download a credential report.')
print_exception(e)
async def _get_and_set_s3_bucket_policy(self, bucket):
client = AWSFacadeUtils.get_client('s3', self.session,
bucket['region'])
try:
bucket_policy = await run_concurrently(
lambda: client.get_bucket_policy(Bucket=bucket['Name']))
bucket['policy'] = json.loads(bucket_policy['Policy'])
except ClientError as e:
if e.response['Error']['Code'] != 'NoSuchBucketPolicy':
print_error('Failed to get bucket policy for %s: %s' %
(bucket['Name'], e))
def _build_services_list(supported_services, services, skipped_services):
# Ensure services and skipped services exist, otherwise log exception
error = False
for service in services + skipped_services:
if service not in supported_services:
print_error('Service \"{}\" does not exist, skipping'.format(service))
error = True
if error:
print_info('Available services are: {}'.format(str(list(supported_services)).strip('[]')))
return [s for s in supported_services if (services == [] or s in services) and s not in skipped_services]
def __init__(self, ruleset):
# Organize rules by path
self.ruleset = ruleset
self.rules = {}
for filename in self.ruleset.rules:
for rule in self.ruleset.rules[filename]:
if not rule.enabled:
continue
try:
manage_dictionary(self.rules, rule.path, [])
self.rules[rule.path].append(rule)
except Exception as e:
print_error('Failed to create rule %s: %s' %
(rule.path, e))
def load(self):
"""
Load the definition of the rule, searching in the specified rule dirs first, then in the built-in definitions
:return: None
"""
file_name_valid = False
rule_type_valid = False
file_path = None
# Look for a locally-defined rule
for rule_dir in self.rule_dirs:
try:
file_path = os.path.join(
rule_dir, self.file_name) if rule_dir else self.file_name
except Exception as e:
print_exception('Failed to load file %s: %s' %
(self.file_name, str(e)))
if os.path.isfile(file_path):
self.file_path = file_path
file_name_valid = True
break
# Look for a built-in rule
if not file_name_valid:
for rule_type in self.rule_types:
if self.file_name.startswith(rule_type):
self.file_path = os.path.join(self.rules_data_path,
self.file_name)
rule_type_valid = True
file_name_valid = True
break
if not rule_type_valid:
for rule_type in self.rule_types:
self.file_path = os.path.join(self.rules_data_path,
rule_type, self.file_name)
if os.path.isfile(self.file_path):
file_name_valid = True
break
else:
if os.path.isfile(self.file_path):
file_name_valid = True
if not file_name_valid:
print_error('Error: could not find %s' % self.file_name)
else:
try:
with open(self.file_path, 'rt') as f:
self.string_definition = f.read()
self.load_from_string_definition()
except Exception as e:
print_exception('Failed to load rule defined in %s: %s' %
(self.file_name, str(e)))
def __init__(self, data_path, file_name=None, rule_dirs=None, string_definition=None):
rule_dirs = [] if rule_dirs is None else rule_dirs
self.rules_data_path = data_path
self.file_name = file_name
self.rule_dirs = rule_dirs
self.rule_types = ['findings', 'filters']
if self.file_name:
self.load()
elif string_definition:
self.string_definition = string_definition
self.load_from_string_definition()
else:
print_error('Error')
def pass_conditions(all_info,
current_path,
conditions,
unknown_as_pass_condition=False):
"""
Check that all conditions are passed for the current path.
:param all_info: All of the services' data
:param current_path: The value of the `path` variable defined in the finding file
:param conditions: The conditions to check as defined in the finding file
:param unknown_as_pass_condition: Consider an undetermined condition as passed
:return:
"""
# Fixes circular dependency
from ScoutSuite.providers.base.configs.browser import get_value_at
if len(conditions) == 0:
return True
condition_operator = conditions.pop(0)
for condition in conditions:
if condition[0] in condition_operators:
res = pass_conditions(all_info, current_path, condition,
unknown_as_pass_condition)
else:
# Conditions are formed as "path to value", "type of test", "value(s) for test"
path_to_value, test_name, test_values = condition
path_to_value = fix_path_string(all_info, current_path,
path_to_value)
target_obj = get_value_at(all_info, current_path, path_to_value)
if type(test_values) != list:
dynamic_value = re_get_value_at.match(test_values)
if dynamic_value:
test_values = get_value_at(all_info, current_path,
dynamic_value.groups()[0], True)
try:
res = pass_condition(target_obj, test_name, test_values)
except Exception as e:
res = True if unknown_as_pass_condition else False
print_error(
'Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.'
% (test_name, str(target_obj), res))
print_exception(e, True)
# Quick exit and + false
if condition_operator == 'and' and not res:
return False
# Quick exit or + true
if condition_operator == 'or' and res:
return True
return not condition_operator == 'or'
def get_s3_bucket_logging(api_client, bucket_name, bucket_info):
try:
logging = api_client.get_bucket_logging(Bucket=bucket_name)
if 'LoggingEnabled' in logging:
bucket_info['logging'] = \
logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix']
bucket_info['logging_stuff'] = logging
else:
bucket_info['logging'] = 'Disabled'
return True
except Exception as e:
print_error('Failed to get logging configuration for %s: %s' % (bucket_name, e))
bucket_info['logging'] = 'Unknown'
return False
def _get_users(self, instance, params):
users_dict = {}
try:
users = params['api_client'].users().list(
project=instance['project'],
instance=instance['name']).execute()
for user in users['items']:
users_dict[user['name']] = self._parse_user(user)
except Exception as e:
print_error('Failed to fetch users for SQL instance %s: %s' %
(instance['name'], e))
return users_dict
def __prepare_age_test(a, b):
if type(a) != list:
print_error('Error: olderThan requires a list such as [ N , \'days\' ] or [ M, \'hours\'].')
raise Exception
number = int(a[0])
unit = a[1]
if unit not in ['days', 'hours', 'minutes', 'seconds']:
print_error('Error: only days, hours, minutes, and seconds are supported.')
raise Exception
if unit == 'hours':
number *= 3600
unit = 'seconds'
elif unit == 'minutes':
number *= 60
unit = 'seconds'
age = getattr((datetime.datetime.today() - dateutil.parser.parse(str(b)).replace(tzinfo=None)), unit)
return age, number
def get_s3_bucket_default_encryption(api_client, bucket_name, bucket_info):
try:
default_encryption = api_client.get_bucket_encryption(Bucket=bucket_name)
bucket_info['default_encryption_enabled'] = True
return True
except ClientError as e:
if 'ServerSideEncryptionConfigurationNotFoundError' in e.response['Error']['Code']:
bucket_info['default_encryption_enabled'] = False
return True
else:
print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e))
bucket_info['default_encryption_enabled'] = None
return False
except Exception as e:
print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e))
bucket_info['default_encryption'] = 'Unknown'
return False
def prompt_4_yes_no(question):
"""
Ask a question and prompt for yes or no
:param question: Question to ask; answer is yes/no
:return: :boolean
"""
while True:
sys.stdout.write(question + ' (y/n)? ')
choice = input().lower()
if choice == 'yes' or choice == 'y':
return True
elif choice == 'no' or choice == 'n':
return False
else:
print_error(
'\'%s\' is not a valid answer. Enter \'yes\'(y) or \'no\'(n).'
% choice)
def get_cloudstorage_bucket_acl(bucket, bucket_dict):
try:
bucket_acls = bucket.get_iam_policy()
bucket_dict['acl_configuration'] = {}
for role in bucket_acls._bindings:
for member in bucket_acls[role]:
if member.split(':')[0] not in ['projectEditor', 'projectViewer', 'projectOwner']:
if member not in bucket_dict['acl_configuration']:
bucket_dict['acl_configuration'][member] = [role]
else:
bucket_dict['acl_configuration'][member].append(role)
return True
except Exception as e:
print_error('Failed to get bucket ACL configuration for %s: %s' % (bucket.name, e))
bucket_dict['acls'] = 'Unknown'
return False
def set_emr_vpc_ids_callback(self, current_config, path, current_path,
vpc_id, callback_args):
if vpc_id != 'TODO':
return
region = current_path[3]
vpc_id = sg_id = subnet_id = None
pop_list = []
for cluster_id in current_config['clusters']:
cluster = current_config['clusters'][cluster_id]
if 'EmrManagedMasterSecurityGroup' in cluster[
'Ec2InstanceAttributes']:
sg_id = cluster['Ec2InstanceAttributes'][
'EmrManagedMasterSecurityGroup']
elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']:
subnet_id = cluster['Ec2InstanceAttributes'][
'RequestedEc2SubnetIds']
else:
print_error('Unable to determine VPC id for EMR cluster %s' %
str(cluster_id))
continue
if sg_id in self.sg_map:
vpc_id = self.sg_map[sg_id]['vpc_id']
pop_list.append(cluster_id)
else:
sid_found = False
if subnet_id:
for sid in subnet_id:
if sid in self.subnet_map:
vpc_id = self.subnet_map[sid]['vpc_id']
pop_list.append(cluster_id)
sid_found = True
if not sid_found:
print_error('Unable to determine VPC id for %s' %
(str(subnet_id) if subnet_id else str(sg_id)))
continue
if vpc_id:
region_vpcs_config = get_object_at(current_path)
manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}})
region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster
for cluster_id in pop_list:
current_config['clusters'].pop(cluster_id)
if len(current_config['clusters']) == 0:
callback_args['clear_list'].append(region)
def _get_service_account_iam_policy(self, api_client, project_id,
service_account_email):
try:
#FIXME for some reason using the api_client fails, creating a new client doesn't generate an error...
client = discovery.build('iam', 'v1')
# response = api_client.projects().serviceAccounts().getIamPolicy(
response = client.projects().serviceAccounts().getIamPolicy(
resource='projects/%s/serviceAccounts/%s' %
(project_id, service_account_email)).execute()
if 'bindings' in response:
return response['bindings']
else:
return None
except Exception as e:
print_error('Failed to get IAM policy for service account %s: %s' %
(service_account_email, e))
return None
async def _get_and_set_s3_bucket_logging(self, bucket):
client = AWSFacadeUtils.get_client(
's3',
self.session,
bucket['region'],
)
try:
logging = await run_concurrently(
lambda: client.get_bucket_logging(Bucket=bucket['Name']))
except Exception as e:
print_error('Failed to get logging configuration for %s: %s' %
(bucket['Name'], e))
bucket['logging'] = 'Unknown'
if 'LoggingEnabled' in logging:
bucket['logging'] =\
logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix']
else:
bucket['logging'] = 'Disabled'
def _get_service_account_keys(self, api_client, project_id,
service_account_email):
try:
#FIXME for some reason using the api_client fails, creating a new client doesn't generate an error...
client = discovery.build('iam', 'v1', cache_discovery=False)
# client = gcp_connect_service(service='iam')
# response = api_client.projects().serviceAccounts().keys().list(
response = client.projects().serviceAccounts().keys().list(
name='projects/%s/serviceAccounts/%s' %
(project_id, service_account_email)).execute()
if 'keys' in response:
return response['keys']
else:
return None
except Exception as e:
print_error('Failed to get keys for service account %s: %s' %
(service_account_email, e))
return None
