async def _fetch(self, service, regions):
try:
print_info('Fetching resources for the {} service'.format(
format_service_name(service)))
service_config = getattr(self, service)
# call fetch method for the service
if 'fetch_all' in dir(service_config):
method_args = {
'credentials': self.credentials,
'regions': regions
}
if self._is_provider('aws'):
if service != 'iam':
method_args['partition_name'] = get_partition_name(
self.credentials)
await service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
await service_config.finalize()
else:
print_debug('No method to fetch service %s.' % service)
except Exception as e:
print_error('Error: could not fetch %s configuration.' % service)
print_exception(e)
async def _fetch(self, service, regions=None, excluded_regions=None):
try:
print_info('Fetching resources for the {} service'.format(
format_service_name(service)))
service_config = getattr(self, service)
# call fetch method for the service
if 'fetch_all' in dir(service_config):
method_args = {}
if regions:
method_args['regions'] = regions
if excluded_regions:
method_args['excluded_regions'] = excluded_regions
if self._is_provider('aws'):
if service != 'iam':
method_args['partition_name'] = get_partition_name(
self.credentials.session)
await service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
await service_config.finalize()
else:
print_debug('No method to fetch service %s.' % service)
except Exception as e:
print_exception(f'Could not fetch {service} configuration: {e}')
def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args):
attached_resource = current_config['resource_id']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(
current_path[0:4], ['vpcs', attached_resource])
try:
attached_vpc = get_object_at(self, vpc_path)
except Exception:
print_debug(
'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % (
flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(
attached_vpc['subnets'][subnet_id], 'flow_logs', [])
if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(
flow_log_id)
elif attached_resource.startswith('subnet-'):
subnet_path = combine_paths(current_path[0:4],
['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets',
attached_resource])
subnet = get_object_at(self, subnet_path)
manage_dictionary(subnet, 'flow_logs', [])
if flow_log_id not in subnet['flow_logs']:
subnet['flow_logs'].append(flow_log_id)
else:
print_exception('Resource %s attached to flow logs is not handled' % attached_resource)
def __init__(self,
cloud_provider,
environment_name='default',
filename=None,
name=None,
rules_dir=None,
rule_type='findings',
ip_ranges=None,
account_id=None,
ruleset_generator=False):
rules_dir = [] if rules_dir is None else rules_dir
ip_ranges = [] if ip_ranges is None else ip_ranges
self.rules_data_path = os.path.dirname(
os.path.dirname(os.path.abspath(__file__))) + '/providers/%s/rules' % cloud_provider
self.environment_name = environment_name
self.rule_type = rule_type
# Ruleset filename
self.filename = self.find_file(filename)
if not self.filename:
self.search_ruleset(environment_name)
print_debug('Loading ruleset %s' % self.filename)
self.name = os.path.basename(self.filename).replace('.json', '') if not name else name
self.load(self.rule_type)
self.shared_init(ruleset_generator, rules_dir, account_id, ip_ranges)
def fetch(self, credentials, services=None, regions=None):
services = [] if services is None else services
regions = [] if regions is None else regions
for service in vars(self):
try:
# skip services
if services != [] and service not in services:
continue
service_config = getattr(self, service)
# call fetch method for the service
if 'fetch_all' in dir(service_config):
method_args = {
'credentials': credentials,
'regions': regions
}
if self._is_provider('aws'):
if service != 'iam':
method_args['partition_name'] = get_partition_name(
credentials)
service_config.fetch_all(**method_args)
if hasattr(service_config, 'finalize'):
service_config.finalize()
else:
print_debug('No method to fetch service %s.' % service)
except Exception as e:
print_error('Error: could not fetch %s configuration.' %
service)
print_exception(e)
async def is_api_enabled(self, project_id, service):
"""
Given a project ID and service name, this method tries to determine if the service's API is enabled
"""
serviceusage_client = self._build_arbitrary_client('serviceusage',
'v1',
force_new=True)
services = serviceusage_client.services()
try:
request = services.list(parent=f'projects/{project_id}')
services_response = await GCPFacadeUtils.get_all(
'services', request, services)
except Exception as e:
print_exception(
f'Could not fetch the state of services for project \"{project_id}\", '
f'including {format_service_name(service.lower())} in the execution',
{'exception': e})
return True
# These are hardcoded endpoint correspondences as there's no easy way to do this.
if service == 'IAM':
endpoint = 'iam'
elif service == 'KMS':
endpoint = 'cloudkms'
elif service == 'CloudStorage':
endpoint = 'storage-component'
elif service == 'CloudSQL':
endpoint = 'sql-component'
elif service == 'ComputeEngine':
endpoint = 'compute'
elif service == 'KubernetesEngine':
endpoint = 'container'
elif service == 'StackdriverLogging':
endpoint = 'logging'
elif service == 'StackdriverMonitoring':
endpoint = 'monitoring'
else:
print_debug(
'Could not validate the state of the {} API for project \"{}\", '
'including it in the execution'.format(
format_service_name(service.lower()), project_id))
return True
for s in services_response:
if endpoint in s.get('name'):
if s.get('state') == 'ENABLED':
return True
else:
print_info(
'{} API not enabled for project \"{}\", skipping'.
format(format_service_name(service.lower()),
project_id))
return False
print_error(
f'Could not validate the state of the {format_service_name(service.lower())} API '
f'for project \"{project_id}\", including it in the execution')
return True
def refresh_credential(self, credentials):
"""
Refresh credentials
"""
print_debug('Refreshing credentials')
authority_uri = AUTHORITY_HOST_URI + '/' + self.get_tenant_id()
existing_cache = self.context.cache
context = adal.AuthenticationContext(authority_uri, cache=existing_cache)
new_token = context.acquire_token(credentials.token['resource'],
credentials.token['user_id'],
credentials.token['_client_id'])
new_credentials = AADTokenCredentials(new_token, credentials.token.get('_client_id'))
return new_credentials
def run(self, cloud_provider, skip_dashboard=False):
# Clean up existing findings
for service in cloud_provider.services:
cloud_provider.services[service][self.ruleset.rule_type] = {}
# Process each rule
for finding_path in self._filter_rules(self.rules, cloud_provider.service_list):
for rule in self.rules[finding_path]:
if not rule.enabled: # or rule.service not in []: # TODO: handle this...
continue
print_debug(f'Processing {rule.service} rule "{rule.description}" ({rule.filename})')
finding_path = rule.path
path = finding_path.split('.')
service = path[0]
manage_dictionary(cloud_provider.services[service], self.ruleset.rule_type, {})
cloud_provider.services[service][self.ruleset.rule_type][rule.key] = {}
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['description'] = rule.description
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['path'] = rule.path
for attr in ['level', 'id_suffix', 'class_suffix', 'display_path']:
if hasattr(rule, attr):
cloud_provider.services[service][self.ruleset.rule_type][rule.key][attr] = getattr(rule, attr)
try:
setattr(rule, 'checked_items', 0)
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items'] = recurse(
cloud_provider.services, cloud_provider.services, path, [], rule, True)
print(cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items'])
if skip_dashboard:
continue
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['dashboard_name'] = \
rule.dashboard_name
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = \
rule.checked_items
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = \
len(cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items'])
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['service'] = rule.service
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['rationale'] = \
rule.rationale if hasattr(rule, 'rationale') else None
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['remediation'] = \
rule.remediation if hasattr(rule, 'remediation') else None
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['compliance'] = \
rule.compliance if hasattr(rule, 'compliance') else None
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['references'] = \
rule.references if hasattr(rule, 'references') else None
except Exception as e:
print_exception(f'Failed to process rule defined in {rule.filename}: {e}')
# Fallback if process rule failed to ensure report creation and data dump still happen
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = 0
cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = 0
def process(self, cloud_provider):
for service in self.exceptions:
for rule in self.exceptions[service]:
filtered_items = []
if rule not in cloud_provider.services[service]['findings']:
print_debug('Warning:: key error should not be happening')
continue
for item in cloud_provider.services[service]['findings'][rule][
'items']:
if item not in self.exceptions[service][rule]:
filtered_items.append(item)
cloud_provider.services[service]['findings'][rule][
'items'] = filtered_items
cloud_provider.services[service]['findings'][rule]['flagged_items'] = \
len(cloud_provider.services[service]['findings'][rule]['items'])
async def fetch(self, services=None, regions=None):
# If services is set to None, fetch all services:
services = vars(self) if services is None else services
regions = [] if regions is None else regions
# First, print services that are going to get skipped:
for service in vars(self):
if service not in services:
print_debug('Skipping the {} service'.format(
format_service_name(service)))
# Then, fetch concurrently all services:
tasks = {
asyncio.ensure_future(self._fetch(service, regions))
for service in services
}
await asyncio.wait(tasks)
def _get_and_set_s3_bucket_creationdate(self, buckets):
# When using region other than 'us-east-1', the 'CreationDate' is the last modified time according to bucket's
# last replication in the respective region
# Source: https://github.com/aws/aws-cli/issues/3597#issuecomment-424167129
# Fixes issue https://github.com/nccgroup/ScoutSuite/issues/858
client = AWSFacadeUtils.get_client('s3', self.session, 'us-east-1')
try:
buckets_useast1 = client.list_buckets()['Buckets']
for bucket in buckets:
# Find the bucket with the same name and update 'CreationDate' from the 'us-east-1' region data,
# if doesn't exist keep the original value
bucket['CreationDate'] = next(
(b['CreationDate']
for b in buckets_useast1 if b['Name'] == bucket['Name']),
bucket['CreationDate'])
except Exception as e:
# Only output exception when in debug mode
print_debug(
'Failed to get bucket creation date from "us-east-1" region')
async def fetch(self, services: list, regions: list):
if not services:
print_debug('No services to scan')
else:
# Print services that are going to get skipped:
for service in vars(self):
if service not in services:
print_debug('Skipping the {} service'.format(
format_service_name(service)))
# Remove "credentials" as it isn't a service
if 'credentials' in services: services.remove('credentials')
# Then, fetch concurrently all services:
if services:
tasks = {
asyncio.ensure_future(self._fetch(service, regions))
for service in services
}
await asyncio.wait(tasks)
def test_ruleset_class(self, printError):
test001 = Ruleset(filename=self.test_ruleset_001)
assert (os.path.isdir(test001.rules_data_path))
assert (os.path.isfile(test001.filename))
assert (test001.name == "test-ruleset")
assert (test001.about == "regression test")
test_file_key = 'iam-password-policy-no-expiration.json'
assert (test_file_key in test001.rules)
assert (type(test001.rules[test_file_key]) == list)
assert (type(test001.rules[test_file_key][0] == Rule))
assert (hasattr(test001.rules[test_file_key][0], 'path'))
for rule in test001.rules:
print_debug(test001.rules[rule][0].to_string())
assert (test_file_key in test001.rule_definitions)
assert (test001.rule_definitions[test_file_key].description ==
"Password expiration disabled")
for rule_def in test001.rule_definitions:
print_debug(str(test001.rule_definitions[rule_def]))
assert (printError.call_count == 0)
test002 = Ruleset(filename=self.test_ruleset_002)
for rule in test002.rules:
print_debug(test002.rules[rule][0].to_string())
assert (printError.call_count == 1) # is this expected ??
assert ("test-ruleset-absolute-path.json does not exist."
in printError.call_args_list[0][0][0])
test005 = Ruleset(filename=self.test_ruleset_001,
ruleset_generator=True)
async def get_regulatory_compliance_results(self, subscription_id: str):
try:
client = self.get_client(subscription_id)
results = []
try:
compliance_standards = await run_concurrently(lambda: list(
client.regulatory_compliance_standards.list()))
except Exception as e:
if 'as it has no standard pricing bundle' in str(e):
print_debug(
'Failed to retrieve regulatory compliance standards: {}'
.format(e))
else:
print_exception(
'Failed to retrieve regulatory compliance standards: {}'
.format(e))
return {}
else:
for standard in compliance_standards:
try:
compliance_controls = await run_concurrently(
lambda: list(
client.regulatory_compliance_controls.
list(regulatory_compliance_standard_name=
standard.name)))
for control in compliance_controls:
control.standard_name = standard.name
results.append(control)
except Exception as e:
print_exception(
'Failed to retrieve compliance controls: {}'.
format(e))
finally:
return results
except Exception as e:
print_exception(
'Failed to retrieve regulatory compliance results: {}'.format(
e))
return []
def find_profiles_in_file(filename, names=None, quiet=True):
if names is None:
names = []
profiles = []
if type(names) != list:
names = [names]
if not quiet:
print_debug('Searching for profiles matching %s in %s ... ' %
(str(names), filename))
name_filters = []
for name in names:
name_filters.append(re.compile('^%s$' % name))
if os.path.isfile(filename):
with open(filename, 'rt') as f:
aws_credentials = f.read()
existing_profiles = re_profile_name.findall(aws_credentials)
profile_count = len(existing_profiles) - 1
for i, profile in enumerate(existing_profiles):
matching_profile = False
raw_profile = None
for name_filter in name_filters:
if name_filter.match(profile[2]):
matching_profile = True
i1 = aws_credentials.index(profile[0])
if i < profile_count:
i2 = aws_credentials.index(
existing_profiles[i + 1][0])
raw_profile = aws_credentials[i1:i2]
else:
raw_profile = aws_credentials[i1:]
if len(name_filters) == 0 or matching_profile:
profiles.append(
AWSProfile(filename=filename,
raw_profile=raw_profile,
name=profile[2]))
return profiles
async def run_scan(args):
# Configure the debug level
set_config_debug_level(args.get('debug'))
print_info('Launching Scout')
credentials = None
if not args.get('fetch_local'):
auth_strategy = get_authentication_strategy(args.get('provider'))
credentials = auth_strategy.authenticate(
profile=args.get('profile'),
user_account=args.get('user_account'),
service_account=args.get('service_account'),
cli=args.get('cli'),
msi=args.get('msi'),
service_principal=args.get('service_principal'),
file_auth=args.get('file_auth'),
tenant_id=args.get('tenant_id'),
subscription_id=args.get('subscription_id'),
client_id=args.get('client_id'),
client_secret=args.get('client_secret'),
username=args.get('username'),
password=args.get('password'))
if not credentials:
return 401
# Create a cloud provider object
cloud_provider = get_provider(
provider=args.get('provider'),
profile=args.get('profile'),
project_id=args.get('project_id'),
folder_id=args.get('folder_id'),
organization_id=args.get('organization_id'),
all_projects=args.get('all_projects'),
report_dir=args.get('report_dir'),
timestamp=args.get('timestamp'),
services=args.get('services'),
skipped_services=args.get('skipped_services'),
thread_config=args.get('thread_config'),
credentials=credentials)
report_file_name = generate_report_name(cloud_provider.provider_code, args)
# TODO: move this to after authentication, so that the report can be more specific to what's being scanned.
# For example if scanning with a GCP service account, the SA email can only be known after authenticating...
# Create a new report
report = Scout2Report(args.get('provider'), report_file_name,
args.get('report_dir'), args.get('timestamp'))
# Complete run, including pulling data from provider
if not args.get('fetch_local'):
# Fetch data from provider APIs
try:
print_info('Gathering data from APIs')
await cloud_provider.fetch(regions=args.get('regions'))
except KeyboardInterrupt:
print_info('\nCancelled by user')
return 130
# Update means we reload the whole config and overwrite part of it
if args.get('update'):
print_info('Updating existing data')
current_run_services = copy.deepcopy(cloud_provider.services)
last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE)
cloud_provider.services = last_run_dict['services']
for service in cloud_provider.service_list:
cloud_provider.services[service] = current_run_services[
service]
# Partial run, using pre-pulled data
else:
print_info('Using local data')
# Reload to flatten everything into a python dictionary
last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE)
for key in last_run_dict:
setattr(cloud_provider, key, last_run_dict[key])
# Pre processing
cloud_provider.preprocessing(args.get('ip_ranges'),
args.get('ip_ranges_name_key'))
# Analyze config
print_info('Running rule engine')
finding_rules = Ruleset(environment_name=args.get('profile'),
cloud_provider=args.get('provider'),
filename=args.get('ruleset'),
ip_ranges=args.get('ip_ranges'),
aws_account_id=cloud_provider.aws_account_id)
processing_engine = ProcessingEngine(finding_rules)
processing_engine.run(cloud_provider)
# Create display filters
print_info('Applying display filters')
filter_rules = Ruleset(cloud_provider=args.get('provider'),
filename='filters.json',
rule_type='filters',
aws_account_id=cloud_provider.aws_account_id)
processing_engine = ProcessingEngine(filter_rules)
processing_engine.run(cloud_provider)
if args.get('exceptions')[0]:
print_info('Applying exceptions')
try:
exceptions = RuleExceptions(args.get('profile'),
args.get('exceptions')[0])
exceptions.process(cloud_provider)
exceptions = exceptions.exceptions
except Exception as e:
print_debug(
'Failed to load exceptions. The file may not exist or may have an invalid format.'
)
exceptions = {}
else:
exceptions = {}
# Handle exceptions
try:
exceptions = RuleExceptions(args.get('profile'),
args.get('exceptions')[0])
exceptions.process(cloud_provider)
exceptions = exceptions.exceptions
except Exception as e:
print_debug(
'Warning, failed to load exceptions. The file may not exist or may have an invalid format.'
)
exceptions = {}
# Finalize
cloud_provider.postprocessing(report.current_time, finding_rules)
# Save config and create HTML report
html_report_path = report.save(cloud_provider, exceptions,
args.get('force_write'), args.get('debug'))
# Open the report by default
if not args.get('no_browser'):
print_info('Opening the HTML report')
url = 'file://%s' % os.path.abspath(html_report_path)
webbrowser.open(url, new=2)
return 0
def main(args=None):
"""
Main method that runs a scan
:return:
"""
if not args:
parser = ScoutSuiteArgumentParser()
args = parser.parse_args()
# Get the dictionnary to get None instead of a crash
args = args.__dict__
# Configure the debug level
config_debug_level(args.get('debug'))
# Create a cloud provider object
cloud_provider = get_provider(
provider=args.get('provider'),
profile=args.get('profile'),
project_id=args.get('project_id'),
folder_id=args.get('folder_id'),
organization_id=args.get('organization_id'),
all_projects=args.get('all_projects'),
report_dir=args.get('report_dir'),
timestamp=args.get('timestamp'),
services=args.get('services'),
skipped_services=args.get('skipped_services'),
thread_config=args.get('thread_config'))
report_file_name = generate_report_name(cloud_provider.provider_code, args)
# TODO: move this to after authentication, so that the report can be more specific to what's being scanned.
# For example if scanning with a GCP service account, the SA email can only be known after authenticating...
# Create a new report
report = Scout2Report(args.get('provider'), report_file_name,
args.get('report_dir'), args.get('timestamp'))
# Complete run, including pulling data from provider
if not args.get('fetch_local'):
# Authenticate to the cloud provider
authenticated = cloud_provider.authenticate(
profile=args.get('profile'),
user_account=args.get('user_account'),
service_account=args.get('service_account'),
cli=args.get('cli'),
msi=args.get('msi'),
service_principal=args.get('service_principal'),
file_auth=args.get('file_auth'),
tenant_id=args.get('tenant_id'),
subscription_id=args.get('subscription_id'),
client_id=args.get('client_id'),
client_secret=args.get('client_secret'),
username=args.get('username'),
password=args.get('password'))
if not authenticated:
return 401
# Fetch data from provider APIs
try:
cloud_provider.fetch(regions=args.get('regions'))
except KeyboardInterrupt:
print_info('\nCancelled by user')
return 130
# Update means we reload the whole config and overwrite part of it
if args.get('update'):
current_run_services = copy.deepcopy(cloud_provider.services)
last_run_dict = report.jsrw.load_from_file(AWSCONFIG)
cloud_provider.services = last_run_dict['services']
for service in cloud_provider.service_list:
cloud_provider.services[service] = current_run_services[
service]
# Partial run, using pre-pulled data
else:
# Reload to flatten everything into a python dictionary
last_run_dict = report.jsrw.load_from_file(AWSCONFIG)
for key in last_run_dict:
setattr(cloud_provider, key, last_run_dict[key])
# Pre processing
cloud_provider.preprocessing(args.get('ip_ranges'),
args.get('ip_ranges_name_key'))
# Analyze config
finding_rules = Ruleset(environment_name=args.get('profile'),
cloud_provider=args.get('provider'),
filename=args.get('ruleset'),
ip_ranges=args.get('ip_ranges'),
aws_account_id=cloud_provider.aws_account_id)
processing_engine = ProcessingEngine(finding_rules)
processing_engine.run(cloud_provider)
# Create display filters
filter_rules = Ruleset(cloud_provider=args.get('provider'),
filename='filters.json',
rule_type='filters',
aws_account_id=cloud_provider.aws_account_id)
processing_engine = ProcessingEngine(filter_rules)
processing_engine.run(cloud_provider)
# Handle exceptions
try:
exceptions = RuleExceptions(args.get('profile'),
args.get('exceptions')[0])
exceptions.process(cloud_provider)
exceptions = exceptions.exceptions
except Exception as e:
print_debug(
'Warning, failed to load exceptions. The file may not exist or may have an invalid format.'
)
exceptions = {}
# Finalize
cloud_provider.postprocessing(report.current_time, finding_rules)
# TODO: this is AWS-specific - move to postprocessing?
# This is partially implemented
# Get organization data if it exists
try:
profile = AWSProfiles.get(args.get('profile'))[0]
if 'source_profile' in profile.attributes:
organization_info_file = os.path.join(
os.path.expanduser('~/.aws/recipes/%s/organization.json' %
profile.attributes['source_profile']))
if os.path.isfile(organization_info_file):
with open(organization_info_file, 'rt') as f:
org = {}
accounts = json.load(f)
for account in accounts:
account_id = account.pop('Id')
org[account_id] = account
setattr(cloud_provider, 'organization', org)
except Exception as e:
pass
# Save config and create HTML report
html_report_path = report.save(cloud_provider, exceptions,
args.get('force_write'), args.get('debug'))
# Open the report by default
if not args.get('no_browser'):
print_info('Opening the HTML report...')
url = 'file://%s' % os.path.abspath(html_report_path)
webbrowser.open(url, new=2)
return 0
