async def _fetch(self, service, regions): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = { 'credentials': self.credentials, 'regions': regions } if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_error('Error: could not fetch %s configuration.' % service) print_exception(e)
async def _fetch(self, service, regions=None, excluded_regions=None): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = {} if regions: method_args['regions'] = regions if excluded_regions: method_args['excluded_regions'] = excluded_regions if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials.session) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_exception(f'Could not fetch {service} configuration: {e}')
def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args): attached_resource = current_config['resource_id'] if attached_resource.startswith('vpc-'): vpc_path = combine_paths( current_path[0:4], ['vpcs', attached_resource]) try: attached_vpc = get_object_at(self, vpc_path) except Exception: print_debug( 'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % ( flow_log_id, attached_resource)) return manage_dictionary(attached_vpc, 'flow_logs', []) if flow_log_id not in attached_vpc['flow_logs']: attached_vpc['flow_logs'].append(flow_log_id) for subnet_id in attached_vpc['subnets']: manage_dictionary( attached_vpc['subnets'][subnet_id], 'flow_logs', []) if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']: attached_vpc['subnets'][subnet_id]['flow_logs'].append( flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource]) subnet = get_object_at(self, subnet_path) manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) else: print_exception('Resource %s attached to flow logs is not handled' % attached_resource)
def __init__(self, cloud_provider, environment_name='default', filename=None, name=None, rules_dir=None, rule_type='findings', ip_ranges=None, account_id=None, ruleset_generator=False): rules_dir = [] if rules_dir is None else rules_dir ip_ranges = [] if ip_ranges is None else ip_ranges self.rules_data_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) + '/providers/%s/rules' % cloud_provider self.environment_name = environment_name self.rule_type = rule_type # Ruleset filename self.filename = self.find_file(filename) if not self.filename: self.search_ruleset(environment_name) print_debug('Loading ruleset %s' % self.filename) self.name = os.path.basename(self.filename).replace('.json', '') if not name else name self.load(self.rule_type) self.shared_init(ruleset_generator, rules_dir, account_id, ip_ranges)
def fetch(self, credentials, services=None, regions=None): services = [] if services is None else services regions = [] if regions is None else regions for service in vars(self): try: # skip services if services != [] and service not in services: continue service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = { 'credentials': credentials, 'regions': regions } if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( credentials) service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_error('Error: could not fetch %s configuration.' % service) print_exception(e)
async def is_api_enabled(self, project_id, service): """ Given a project ID and service name, this method tries to determine if the service's API is enabled """ serviceusage_client = self._build_arbitrary_client('serviceusage', 'v1', force_new=True) services = serviceusage_client.services() try: request = services.list(parent=f'projects/{project_id}') services_response = await GCPFacadeUtils.get_all( 'services', request, services) except Exception as e: print_exception( f'Could not fetch the state of services for project \"{project_id}\", ' f'including {format_service_name(service.lower())} in the execution', {'exception': e}) return True # These are hardcoded endpoint correspondences as there's no easy way to do this. if service == 'IAM': endpoint = 'iam' elif service == 'KMS': endpoint = 'cloudkms' elif service == 'CloudStorage': endpoint = 'storage-component' elif service == 'CloudSQL': endpoint = 'sql-component' elif service == 'ComputeEngine': endpoint = 'compute' elif service == 'KubernetesEngine': endpoint = 'container' elif service == 'StackdriverLogging': endpoint = 'logging' elif service == 'StackdriverMonitoring': endpoint = 'monitoring' else: print_debug( 'Could not validate the state of the {} API for project \"{}\", ' 'including it in the execution'.format( format_service_name(service.lower()), project_id)) return True for s in services_response: if endpoint in s.get('name'): if s.get('state') == 'ENABLED': return True else: print_info( '{} API not enabled for project \"{}\", skipping'. format(format_service_name(service.lower()), project_id)) return False print_error( f'Could not validate the state of the {format_service_name(service.lower())} API ' f'for project \"{project_id}\", including it in the execution') return True
def refresh_credential(self, credentials): """ Refresh credentials """ print_debug('Refreshing credentials') authority_uri = AUTHORITY_HOST_URI + '/' + self.get_tenant_id() existing_cache = self.context.cache context = adal.AuthenticationContext(authority_uri, cache=existing_cache) new_token = context.acquire_token(credentials.token['resource'], credentials.token['user_id'], credentials.token['_client_id']) new_credentials = AADTokenCredentials(new_token, credentials.token.get('_client_id')) return new_credentials
def run(self, cloud_provider, skip_dashboard=False): # Clean up existing findings for service in cloud_provider.services: cloud_provider.services[service][self.ruleset.rule_type] = {} # Process each rule for finding_path in self._filter_rules(self.rules, cloud_provider.service_list): for rule in self.rules[finding_path]: if not rule.enabled: # or rule.service not in []: # TODO: handle this... continue print_debug(f'Processing {rule.service} rule "{rule.description}" ({rule.filename})') finding_path = rule.path path = finding_path.split('.') service = path[0] manage_dictionary(cloud_provider.services[service], self.ruleset.rule_type, {}) cloud_provider.services[service][self.ruleset.rule_type][rule.key] = {} cloud_provider.services[service][self.ruleset.rule_type][rule.key]['description'] = rule.description cloud_provider.services[service][self.ruleset.rule_type][rule.key]['path'] = rule.path for attr in ['level', 'id_suffix', 'class_suffix', 'display_path']: if hasattr(rule, attr): cloud_provider.services[service][self.ruleset.rule_type][rule.key][attr] = getattr(rule, attr) try: setattr(rule, 'checked_items', 0) cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items'] = recurse( cloud_provider.services, cloud_provider.services, path, [], rule, True) print(cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items']) if skip_dashboard: continue cloud_provider.services[service][self.ruleset.rule_type][rule.key]['dashboard_name'] = \ rule.dashboard_name cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = \ rule.checked_items cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = \ len(cloud_provider.services[service][self.ruleset.rule_type][rule.key]['items']) cloud_provider.services[service][self.ruleset.rule_type][rule.key]['service'] = rule.service cloud_provider.services[service][self.ruleset.rule_type][rule.key]['rationale'] = \ rule.rationale if hasattr(rule, 'rationale') else None cloud_provider.services[service][self.ruleset.rule_type][rule.key]['remediation'] = \ rule.remediation if hasattr(rule, 'remediation') else None cloud_provider.services[service][self.ruleset.rule_type][rule.key]['compliance'] = \ rule.compliance if hasattr(rule, 'compliance') else None cloud_provider.services[service][self.ruleset.rule_type][rule.key]['references'] = \ rule.references if hasattr(rule, 'references') else None except Exception as e: print_exception(f'Failed to process rule defined in {rule.filename}: {e}') # Fallback if process rule failed to ensure report creation and data dump still happen cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = 0 cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = 0
def process(self, cloud_provider): for service in self.exceptions: for rule in self.exceptions[service]: filtered_items = [] if rule not in cloud_provider.services[service]['findings']: print_debug('Warning:: key error should not be happening') continue for item in cloud_provider.services[service]['findings'][rule][ 'items']: if item not in self.exceptions[service][rule]: filtered_items.append(item) cloud_provider.services[service]['findings'][rule][ 'items'] = filtered_items cloud_provider.services[service]['findings'][rule]['flagged_items'] = \ len(cloud_provider.services[service]['findings'][rule]['items'])
async def fetch(self, services=None, regions=None): # If services is set to None, fetch all services: services = vars(self) if services is None else services regions = [] if regions is None else regions # First, print services that are going to get skipped: for service in vars(self): if service not in services: print_debug('Skipping the {} service'.format( format_service_name(service))) # Then, fetch concurrently all services: tasks = { asyncio.ensure_future(self._fetch(service, regions)) for service in services } await asyncio.wait(tasks)
def _get_and_set_s3_bucket_creationdate(self, buckets): # When using region other than 'us-east-1', the 'CreationDate' is the last modified time according to bucket's # last replication in the respective region # Source: https://github.com/aws/aws-cli/issues/3597#issuecomment-424167129 # Fixes issue https://github.com/nccgroup/ScoutSuite/issues/858 client = AWSFacadeUtils.get_client('s3', self.session, 'us-east-1') try: buckets_useast1 = client.list_buckets()['Buckets'] for bucket in buckets: # Find the bucket with the same name and update 'CreationDate' from the 'us-east-1' region data, # if doesn't exist keep the original value bucket['CreationDate'] = next( (b['CreationDate'] for b in buckets_useast1 if b['Name'] == bucket['Name']), bucket['CreationDate']) except Exception as e: # Only output exception when in debug mode print_debug( 'Failed to get bucket creation date from "us-east-1" region')
async def fetch(self, services: list, regions: list): if not services: print_debug('No services to scan') else: # Print services that are going to get skipped: for service in vars(self): if service not in services: print_debug('Skipping the {} service'.format( format_service_name(service))) # Remove "credentials" as it isn't a service if 'credentials' in services: services.remove('credentials') # Then, fetch concurrently all services: if services: tasks = { asyncio.ensure_future(self._fetch(service, regions)) for service in services } await asyncio.wait(tasks)
def test_ruleset_class(self, printError): test001 = Ruleset(filename=self.test_ruleset_001) assert (os.path.isdir(test001.rules_data_path)) assert (os.path.isfile(test001.filename)) assert (test001.name == "test-ruleset") assert (test001.about == "regression test") test_file_key = 'iam-password-policy-no-expiration.json' assert (test_file_key in test001.rules) assert (type(test001.rules[test_file_key]) == list) assert (type(test001.rules[test_file_key][0] == Rule)) assert (hasattr(test001.rules[test_file_key][0], 'path')) for rule in test001.rules: print_debug(test001.rules[rule][0].to_string()) assert (test_file_key in test001.rule_definitions) assert (test001.rule_definitions[test_file_key].description == "Password expiration disabled") for rule_def in test001.rule_definitions: print_debug(str(test001.rule_definitions[rule_def])) assert (printError.call_count == 0) test002 = Ruleset(filename=self.test_ruleset_002) for rule in test002.rules: print_debug(test002.rules[rule][0].to_string()) assert (printError.call_count == 1) # is this expected ?? assert ("test-ruleset-absolute-path.json does not exist." in printError.call_args_list[0][0][0]) test005 = Ruleset(filename=self.test_ruleset_001, ruleset_generator=True)
async def get_regulatory_compliance_results(self, subscription_id: str): try: client = self.get_client(subscription_id) results = [] try: compliance_standards = await run_concurrently(lambda: list( client.regulatory_compliance_standards.list())) except Exception as e: if 'as it has no standard pricing bundle' in str(e): print_debug( 'Failed to retrieve regulatory compliance standards: {}' .format(e)) else: print_exception( 'Failed to retrieve regulatory compliance standards: {}' .format(e)) return {} else: for standard in compliance_standards: try: compliance_controls = await run_concurrently( lambda: list( client.regulatory_compliance_controls. list(regulatory_compliance_standard_name= standard.name))) for control in compliance_controls: control.standard_name = standard.name results.append(control) except Exception as e: print_exception( 'Failed to retrieve compliance controls: {}'. format(e)) finally: return results except Exception as e: print_exception( 'Failed to retrieve regulatory compliance results: {}'.format( e)) return []
def find_profiles_in_file(filename, names=None, quiet=True): if names is None: names = [] profiles = [] if type(names) != list: names = [names] if not quiet: print_debug('Searching for profiles matching %s in %s ... ' % (str(names), filename)) name_filters = [] for name in names: name_filters.append(re.compile('^%s$' % name)) if os.path.isfile(filename): with open(filename, 'rt') as f: aws_credentials = f.read() existing_profiles = re_profile_name.findall(aws_credentials) profile_count = len(existing_profiles) - 1 for i, profile in enumerate(existing_profiles): matching_profile = False raw_profile = None for name_filter in name_filters: if name_filter.match(profile[2]): matching_profile = True i1 = aws_credentials.index(profile[0]) if i < profile_count: i2 = aws_credentials.index( existing_profiles[i + 1][0]) raw_profile = aws_credentials[i1:i2] else: raw_profile = aws_credentials[i1:] if len(name_filters) == 0 or matching_profile: profiles.append( AWSProfile(filename=filename, raw_profile=raw_profile, name=profile[2])) return profiles
async def run_scan(args): # Configure the debug level set_config_debug_level(args.get('debug')) print_info('Launching Scout') credentials = None if not args.get('fetch_local'): auth_strategy = get_authentication_strategy(args.get('provider')) credentials = auth_strategy.authenticate( profile=args.get('profile'), user_account=args.get('user_account'), service_account=args.get('service_account'), cli=args.get('cli'), msi=args.get('msi'), service_principal=args.get('service_principal'), file_auth=args.get('file_auth'), tenant_id=args.get('tenant_id'), subscription_id=args.get('subscription_id'), client_id=args.get('client_id'), client_secret=args.get('client_secret'), username=args.get('username'), password=args.get('password')) if not credentials: return 401 # Create a cloud provider object cloud_provider = get_provider( provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), folder_id=args.get('folder_id'), organization_id=args.get('organization_id'), all_projects=args.get('all_projects'), report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), skipped_services=args.get('skipped_services'), thread_config=args.get('thread_config'), credentials=credentials) report_file_name = generate_report_name(cloud_provider.provider_code, args) # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): # Fetch data from provider APIs try: print_info('Gathering data from APIs') await cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: print_info('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it if args.get('update'): print_info('Updating existing data') current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE) cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[ service] # Partial run, using pre-pulled data else: print_info('Using local data') # Reload to flatten everything into a python dictionary last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE) for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) # Pre processing cloud_provider.preprocessing(args.get('ip_ranges'), args.get('ip_ranges_name_key')) # Analyze config print_info('Running rule engine') finding_rules = Ruleset(environment_name=args.get('profile'), cloud_provider=args.get('provider'), filename=args.get('ruleset'), ip_ranges=args.get('ip_ranges'), aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(finding_rules) processing_engine.run(cloud_provider) # Create display filters print_info('Applying display filters') filter_rules = Ruleset(cloud_provider=args.get('provider'), filename='filters.json', rule_type='filters', aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) if args.get('exceptions')[0]: print_info('Applying exceptions') try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: print_debug( 'Failed to load exceptions. The file may not exist or may have an invalid format.' ) exceptions = {} else: exceptions = {} # Handle exceptions try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: print_debug( 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.' ) exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) # Save config and create HTML report html_report_path = report.save(cloud_provider, exceptions, args.get('force_write'), args.get('debug')) # Open the report by default if not args.get('no_browser'): print_info('Opening the HTML report') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) return 0
def main(args=None): """ Main method that runs a scan :return: """ if not args: parser = ScoutSuiteArgumentParser() args = parser.parse_args() # Get the dictionnary to get None instead of a crash args = args.__dict__ # Configure the debug level config_debug_level(args.get('debug')) # Create a cloud provider object cloud_provider = get_provider( provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), folder_id=args.get('folder_id'), organization_id=args.get('organization_id'), all_projects=args.get('all_projects'), report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), skipped_services=args.get('skipped_services'), thread_config=args.get('thread_config')) report_file_name = generate_report_name(cloud_provider.provider_code, args) # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): # Authenticate to the cloud provider authenticated = cloud_provider.authenticate( profile=args.get('profile'), user_account=args.get('user_account'), service_account=args.get('service_account'), cli=args.get('cli'), msi=args.get('msi'), service_principal=args.get('service_principal'), file_auth=args.get('file_auth'), tenant_id=args.get('tenant_id'), subscription_id=args.get('subscription_id'), client_id=args.get('client_id'), client_secret=args.get('client_secret'), username=args.get('username'), password=args.get('password')) if not authenticated: return 401 # Fetch data from provider APIs try: cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: print_info('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it if args.get('update'): current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.jsrw.load_from_file(AWSCONFIG) cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[ service] # Partial run, using pre-pulled data else: # Reload to flatten everything into a python dictionary last_run_dict = report.jsrw.load_from_file(AWSCONFIG) for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) # Pre processing cloud_provider.preprocessing(args.get('ip_ranges'), args.get('ip_ranges_name_key')) # Analyze config finding_rules = Ruleset(environment_name=args.get('profile'), cloud_provider=args.get('provider'), filename=args.get('ruleset'), ip_ranges=args.get('ip_ranges'), aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(finding_rules) processing_engine.run(cloud_provider) # Create display filters filter_rules = Ruleset(cloud_provider=args.get('provider'), filename='filters.json', rule_type='filters', aws_account_id=cloud_provider.aws_account_id) processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) # Handle exceptions try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: print_debug( 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.' ) exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) # TODO: this is AWS-specific - move to postprocessing? # This is partially implemented # Get organization data if it exists try: profile = AWSProfiles.get(args.get('profile'))[0] if 'source_profile' in profile.attributes: organization_info_file = os.path.join( os.path.expanduser('~/.aws/recipes/%s/organization.json' % profile.attributes['source_profile'])) if os.path.isfile(organization_info_file): with open(organization_info_file, 'rt') as f: org = {} accounts = json.load(f) for account in accounts: account_id = account.pop('Id') org[account_id] = account setattr(cloud_provider, 'organization', org) except Exception as e: pass # Save config and create HTML report html_report_path = report.save(cloud_provider, exceptions, args.get('force_write'), args.get('debug')) # Open the report by default if not args.get('no_browser'): print_info('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) return 0