async def _fetch(self, service, regions=None, excluded_regions=None): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = {} if regions: method_args['regions'] = regions if excluded_regions: method_args['excluded_regions'] = excluded_regions if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials.session) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_exception(f'Could not fetch {service} configuration: {e}')
def connect_service(service, credentials, region_name=None, config=None, silent=False): """ Instantiates an AWS API client :param service: Service targeted, e.g. ec2 :param credentials: Id, secret, token :param region_name: Region desired, e.g. us-east-2 :param config: Configuration (optional) :param silent: Whether or not to print messages :return: """ api_client = None try: client_params = {'service_name': service.lower()} session_params = {'aws_access_key_id': credentials.get('access_key'), 'aws_secret_access_key': credentials.get('secret_key'), 'aws_session_token': credentials.get('token')} if region_name: client_params['region_name'] = region_name session_params['region_name'] = region_name if config: client_params['config'] = config aws_session = boto3.session.Session(**session_params) if not silent: info_message = 'Connecting to AWS %s' % service if region_name: info_message = info_message + ' in %s' % region_name print_info('%s...' % info_message) api_client = aws_session.client(**client_params) except Exception as e: print_exception(e) return api_client
async def _fetch(self, service, regions): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = { 'credentials': self.credentials, 'regions': regions } if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_error('Error: could not fetch %s configuration.' % service) print_exception(e)
async def get_projects(self): try: # All projects to which the user / Service Account has access to if self.all_projects: return await self._get_projects_recursively( parent_type='all', parent_id=None) # Project passed through the CLI elif self.project_id: return await self._get_projects_recursively( parent_type='project', parent_id=self.project_id) # Folder passed through the CLI elif self.folder_id: return await self._get_projects_recursively( parent_type='folder', parent_id=self.folder_id) # Organization passed through the CLI elif self.organization_id: return await self._get_projects_recursively( parent_type='organization', parent_id=self.organization_id) # Project inferred from default configuration elif self.default_project_id: return await self._get_projects_recursively( parent_type='project', parent_id=self.default_project_id) # Raise exception if none of the above else: print_info( "Could not infer the Projects to scan and no default Project ID was found.") return [] except Exception as e: print_exception(f'Failed to retrieve projects: {e}') return []
def _match_instances_and_roles(self): print_info('Matching EC2 instances and IAM roles') ec2_config = self.services['ec2'] iam_config = self.services['iam'] role_instances = {} for r in ec2_config['regions']: for v in ec2_config['regions'][r]['vpcs']: if 'instances' in ec2_config['regions'][r]['vpcs'][v]: for i in ec2_config['regions'][r]['vpcs'][v]['instances']: instance_profile = ec2_config['regions'][r]['vpcs'][v][ 'instances'][i]['IamInstanceProfile'] instance_profile_id = instance_profile[ 'Id'] if instance_profile else None if instance_profile_id: manage_dictionary(role_instances, instance_profile_id, []) role_instances[instance_profile_id].append(i) for role_id in iam_config['roles']: iam_config['roles'][role_id]['instances_count'] = 0 for instance_profile_id in iam_config['roles'][role_id][ 'instance_profiles']: if instance_profile_id in role_instances: iam_config['roles'][role_id]['instance_profiles'][instance_profile_id]['instances'] = \ role_instances[instance_profile_id] iam_config['roles'][role_id]['instances_count'] += len( role_instances[instance_profile_id])
def save_to_file(self, content, file_type, force_write, debug): config_path, first_line = get_filename(file_type, self.report_name, self.report_dir) print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write) as f: if first_line: print('%s' % first_line, file=f) results = json.dumps(content, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=ScoutJsonEncoder) print('%s' % results, file=f) if file_type == 'RESULTS': timestamp = datetime.datetime.now().strftime( "%d-%m-%y %H:%M:%S") store_custom_format(json.loads(results), config_path, self.report_name, force_write, content.account_id, timestamp) except AttributeError as e: # __open_file returned None pass except Exception as e: print_exception(e)
def _get_projects(self): # All projects to which the user / Service Account has access to if self.all_projects: self.projects = self._gcp_facade_get_projects( parent_type='all', parent_id=None) # Project passed through the CLI elif self.project_id: self.projects = self._gcp_facade_get_projects( parent_type='project', parent_id=self.project_id) # Folder passed through the CLI elif self.folder_id: self.projects = self._gcp_facade_get_projects( parent_type='folder', parent_id=self.folder_id) # Organization passed through the CLI elif self.organization_id: self.projects = self._gcp_facade_get_projects( parent_type='organization', parent_id=self.organization_id) # Project inferred from default configuration elif self.credentials.default_project_id: self.projects = self._gcp_facade_get_projects( parent_type='project', parent_id=self.credentials.default_project_id) # Raise exception if none of the above else: print_info( "Could not infer the Projects to scan and no default Project ID was found.")
def create_html_report(self, force_write): contents = '' # Use the script corresponding to the result format contents += self.get_content_from_file('/%s_format.html' % self.result_format) # Use all scripts under html/partials/ contents += self.get_content_from_folder('partials') contents += self.get_content_from_folder('partials/%s' % self.provider) # Use all scripts under html/summaries/ contents += self.get_content_from_folder('summaries') contents += self.get_content_from_folder('summaries/%s' % self.provider) new_file, first_line = get_filename('REPORT', self.report_name, self.report_dir) print_info('Creating %s' % new_file) if prompt_for_overwrite(new_file, force_write): if os.path.exists(new_file): os.remove(new_file) with open(os.path.join(self.html_data_path, 'report.html')) as f: with open(new_file, 'wt') as nf: for line in f: newline = line newline = newline.replace('<!-- CONTENTS PLACEHOLDER -->', contents) newline = newline.replace('<!-- RESULTS PLACEHOLDER -->', get_filename('RESULTS', self.report_name, self.report_dir, relative_path=True)[0]) newline = newline.replace('<!-- EXCEPTIONS PLACEHOLDER -->', get_filename('EXCEPTIONS', self.report_name, self.report_dir, relative_path=True)[0]) newline = newline.replace('<!-- SQLITE JS PLACEHOLDER -->', '{}/sqlite.js'.format(DEFAULT_INCLUDES_DIRECTORY)) nf.write(newline) return new_file
def create_html_report(self, force_write): contents = '' # Use all scripts under html/partials/ contents += self.get_content_from('partials') contents += self.get_content_from('partials/%s' % self.provider) # Use all scripts under html/summaries/ contents += self.get_content_from('summaries') contents += self.get_content_from('summaries/%s' % self.provider) new_file, first_line = get_filename('HTMLREPORT', self.profile, self.report_dir) print_info('Creating %s' % new_file) if prompt_4_overwrite(new_file, force_write): if os.path.exists(new_file): os.remove(new_file) with open(os.path.join(self.html_data_path, self.html_root)) as f: with open(new_file, 'wt') as nf: for line in f: newline = line if self.profile != 'default': newline = newline.replace( DEFAULT_RESULT_FILE, DEFAULT_RESULT_FILE.replace( '.js', '-%s.js' % self.profile)) newline = newline.replace( DEFAULT_EXCEPTIONS_FILE, DEFAULT_EXCEPTIONS_FILE.replace( '.js', '-%s.js' % self.profile)) newline = newline.replace('<!-- PLACEHOLDER -->', contents) nf.write(newline) return new_file
def _process_cloudtrail_trails(cloudtrail_config): print_info('Processing CloudTrail config') global_events_logging = [] data_logging_trails_count = 0 for region in cloudtrail_config['regions']: for trail_id in cloudtrail_config['regions'][region]['trails']: trail = cloudtrail_config['regions'][region]['trails'][ trail_id] if 'HomeRegion' in trail and trail['HomeRegion'] != region: # Part of a multi-region trail, skip until we find the whole object continue if trail['IncludeGlobalServiceEvents'] and trail['IsLogging']: global_events_logging.append(( region, trail_id, )) # Any wildcard logging? if trail.get('wildcard_data_logging', False): data_logging_trails_count += 1 cloudtrail_config[ 'data_logging_trails_count'] = data_logging_trails_count cloudtrail_config['IncludeGlobalServiceEvents'] = len( global_events_logging) > 0 cloudtrail_config['DuplicatedGlobalServiceEvents'] = len( global_events_logging) > 1
async def is_api_enabled(self, project_id, service): """ Given a project ID and service name, this method tries to determine if the service's API is enabled """ serviceusage_client = self._build_arbitrary_client('serviceusage', 'v1', force_new=True) services = serviceusage_client.services() try: request = services.list(parent=f'projects/{project_id}') services_response = await GCPFacadeUtils.get_all( 'services', request, services) except Exception as e: print_exception( f'Could not fetch the state of services for project \"{project_id}\", ' f'including {format_service_name(service.lower())} in the execution', {'exception': e}) return True # These are hardcoded endpoint correspondences as there's no easy way to do this. if service == 'IAM': endpoint = 'iam' elif service == 'KMS': endpoint = 'cloudkms' elif service == 'CloudStorage': endpoint = 'storage-component' elif service == 'CloudSQL': endpoint = 'sql-component' elif service == 'ComputeEngine': endpoint = 'compute' elif service == 'KubernetesEngine': endpoint = 'container' elif service == 'StackdriverLogging': endpoint = 'logging' elif service == 'StackdriverMonitoring': endpoint = 'monitoring' else: print_debug( 'Could not validate the state of the {} API for project \"{}\", ' 'including it in the execution'.format( format_service_name(service.lower()), project_id)) return True for s in services_response: if endpoint in s.get('name'): if s.get('state') == 'ENABLED': return True else: print_info( '{} API not enabled for project \"{}\", skipping'. format(format_service_name(service.lower()), project_id)) return False print_error( f'Could not validate the state of the {format_service_name(service.lower())} API ' f'for project \"{project_id}\", including it in the execution') return True
def run_from_cli(): parser = ScoutSuiteArgumentParser() args = parser.parse_args() # Get the dictionary to get None instead of a crash args = args.__dict__ try: return run( provider=args.get('provider'), # AWS profile=args.get('profile'), # Azure user_account=args.get('user_account'), service_account=args.get('service_account'), cli=args.get('cli'), msi=args.get('msi'), service_principal=args.get('service_principal'), file_auth=args.get('file_auth'), tenant_id=args.get('tenant_id'), subscription_id=args.get('subscription_id'), client_id=args.get('client_id'), client_secret=args.get('client_secret'), username=args.get('username'), password=args.get('password'), # GCP project_id=args.get('project_id'), folder_id=args.get('folder_id'), organization_id=args.get('organization_id'), all_projects=args.get('all_projects'), # General report_name=args.get('report_name'), report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), skipped_services=args.get('skipped_services'), result_format=args.get('result_format'), database_name=args.get('database_name'), host_ip=args.get('host_ip'), host_port=args.get('host_port'), max_workers=args.get('max_workers'), regions=args.get('regions'), fetch_local=args.get('fetch_local'), update=args.get('update'), ip_ranges=args.get('ip_ranges'), ip_ranges_name_key=args.get('ip_ranges_name_key'), ruleset=args.get('ruleset'), exceptions=args.get('exceptions'), force_write=args.get('force_write'), debug=args.get('debug'), quiet=args.get('quiet'), log_file=args.get('log_file'), no_browser=args.get('no_browser'), programmatic_execution=False) except (KeyboardInterrupt, SystemExit): print_info('Exiting')
def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, instance_id, callback_args): if instance_id == callback_args['instance_id']: if 'PublicIpAddress' not in current_config: current_config['PublicIpAddress'] = callback_args['elastic_ip'] elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: print_info( 'Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % (current_config['PublicIpAddress'], callback_args['elastic_ip']))
def upload_findigs_to_securityhub(session, formatted_findings_list): try: if formatted_findings_list: print_info('Batch uploading {} findings'.format(len(formatted_findings_list))) securityhub = session.client('securityhub') response = securityhub.batch_import_findings(Findings=formatted_findings_list) print_info('Upload completed, {} succeeded, {} failed'.format(response.get('SuccessCount'), response.get('FailedCount'))) return response except Exception as e: print_exception('Unable to upload findings to Security Hub: {}'.format(e))
def save_to_file(self, config, config_type, force_write, _debug): config_path, first_line = get_filename(config_type, self.report_name, self.report_dir, file_extension="db") print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write) as database: result_dict = self.to_dict(config) for k, v in result_dict.items(): database[k] = v database.commit() except Exception as e: print_exception(e)
def _build_services_list(supported_services, services, skipped_services): # Ensure services and skipped services exist, otherwise log exception error = False for service in services + skipped_services: if service not in supported_services: print_error('Service \"{}\" does not exist, skipping'.format(service)) error = True if error: print_info('Available services are: {}'.format(str(list(supported_services)).strip('[]'))) return [s for s in supported_services if (services == [] or s in services) and s not in skipped_services]
async def run_concurrently(function, backoff_seconds=15): try: async with asyncio.get_event_loop().throttler: return await run_function_concurrently(function) except Exception as e: # Determine whether the exception is due to API throttling if is_throttled(e): print_info('Hitting API rate limiting, will retry in {}s'.format(backoff_seconds)) await asyncio.sleep(backoff_seconds) return await run_concurrently(function, backoff_seconds + 15) else: raise
def save_to_file(self, config, file_type, force_write, debug): config_path, first_line = get_filename(file_type, self.profile, self.report_dir) print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write) as f: if first_line: print('%s' % first_line, file=f) print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) except AttributeError as e: # __open_file returned None pass except Exception as e: print_exception(e)
async def run_concurrently(function, backoff_seconds=15): try: async with asyncio.get_event_loop().throttler: return await run_function_concurrently(function) except Exception as e: # Determine whether the exception is due to API throttling if is_throttled(e): source_file = inspect.getsourcefile(function) source_file_line = inspect.getsourcelines(function)[1] print_info(f'Hitting API rate limiting ({"/".join(source_file.split("/")[-2:])} L{source_file_line}), will retry in {backoff_seconds}s') await asyncio.sleep(backoff_seconds) return await run_concurrently(function, backoff_seconds + 15) else: raise
def run(profile, file): session = boto3.Session(profile_name=profile) # Test querying for current user get_caller_identity(session) print_info('Authenticated with profile {}'.format(profile)) try: with open(file) as f: formatted_findings_list = process_results_file(f, session.region_name) except Exception as e: print_exception('Unable to open file {}: {}'.format(file, e)) upload_findigs_to_securityhub(session, formatted_findings_list)
def _new_go_to_and_do(self, current_config, path, current_path, callbacks): """ Recursively go to a target and execute a callback """ try: key = path.pop(0) if not current_config: current_config = self.config if not current_path: current_path = [] keys = key.split('.') if len(keys) > 1: while True: key = keys.pop(0) if not len(keys): break current_path.append(key) current_config = current_config[key] if key in current_config: current_path.append(key) for (i, value) in enumerate(list(current_config[key])): if len(path) == 0: for callback_info in callbacks: callback_name = callback_info[0] # callback = globals()[callback_name] callback = getattr(self, callback_name) callback_args = callback_info[1] if type(current_config[key] == dict) and type( value) != dict and type(value) != list: callback(current_config[key][value], path, current_path, value, callback_args) else: callback(current_config, path, current_path, value, callback_args) else: tmp = copy.deepcopy(current_path) try: tmp.append(value) self._new_go_to_and_do(current_config[key][value], copy.deepcopy(path), tmp, callbacks) except: tmp.pop() tmp.append(i) self._new_go_to_and_do(current_config[key][i], copy.deepcopy(path), tmp, callbacks) except Exception as e: print_exception(e) print_info('Path: %s' % str(current_path)) print_info('Key = %s' % str(key) if 'key' in locals() else 'not defined') print_info('Value = %s' % str(value) if 'value' in locals() else 'not defined') print_info('Path = %s' % path)
def __open_file(self, config_filename, force_write, quiet=False): """ :param config_filename: :param force_write: :param quiet: :return: """ if not quiet: print_info('Saving config...') if prompt_4_overwrite(config_filename, force_write): try: config_dirname = os.path.dirname(config_filename) if not os.path.isdir(config_dirname): os.makedirs(config_dirname) return open(config_filename, 'wt') except Exception as e: print_exception(e) else: return None
def save_blob_as_json(filename, blob, force_write): """ Creates/Modifies file and saves python object as JSON :param filename: :param blob: :param force_write: :return: """ try: if prompt_overwrite(filename, force_write): with open(filename, 'wt') as f: print_info('%s' % json.dumps(blob, indent=4, separators=(',', ': '), sort_keys=True, cls=CustomJSONEncoder)) except Exception as e: print_exception(e)
def parse_buckets(self, bucket, params): """ Parse a single S3 bucket TODO: - CORS - Lifecycle - Notification ? - Get bucket's policy :param bucket: :param params: :return: """ bucket['name'] = bucket.pop('Name') api_client = params['api_clients'][get_s3_list_region(list(params['api_clients'].keys())[0])] bucket['CreationDate'] = str(bucket['CreationDate']) bucket['region'] = get_s3_bucket_location(api_client, bucket['name']) # h4ck :: fix issue #59, location constraint can be EU or eu-west-1 for Ireland... if bucket['region'] == 'EU': bucket['region'] = 'eu-west-1' # h4ck :: S3 is global but region-aware... if bucket['region'] not in params['api_clients']: print_info('Skipping bucket %s (region %s outside of scope)' % (bucket['name'], bucket['region'])) self.buckets_count -= 1 return api_client = params['api_clients'][bucket['region']] get_s3_bucket_logging(api_client, bucket['name'], bucket) get_s3_bucket_versioning(api_client, bucket['name'], bucket) get_s3_bucket_webhosting(api_client, bucket['name'], bucket) get_s3_bucket_default_encryption(api_client, bucket['name'], bucket) bucket['grantees'] = get_s3_acls(api_client, bucket['name'], bucket) get_s3_bucket_policy(api_client, bucket['name'], bucket) get_s3_bucket_secure_transport(api_client, bucket['name'], bucket) # If requested, get key properties bucket['id'] = self.get_non_provider_id(bucket['name']) self.buckets[bucket['id']] = bucket
async def get_projects(self): try: # resourcemanager_client = self._get_client() # request = resourcemanager_client.projects().list() # projects_group = resourcemanager_client.projects() # return await GCPFacadeUtils.get_all('projects', request, projects_group) # All projects to which the user / Service Account has access to if self.all_projects: return await self._get_projects_recursively(parent_type='all', parent_id=None) # Project passed through the CLI elif self.project_id: return await self._get_projects_recursively( parent_type='project', parent_id=self.project_id) # Folder passed through the CLI elif self.folder_id: return await self._get_projects_recursively( parent_type='folder', parent_id=self.folder_id) # Organization passed through the CLI elif self.organization_id: return await self._get_projects_recursively( parent_type='organization', parent_id=self.organization_id) # Project inferred from default configuration elif self.default_project_id: return await self._get_projects_recursively( parent_type='project', parent_id=self.default_project_id) # Raise exception if none of the above else: print_info( "Could not infer the Projects to scan and no default Project ID was found." ) return [] except Exception as e: print_exception('Failed to retrieve projects: {}'.format(e)) return []
def analyze_ec2_config(ec2_info, aws_account_id, force_write): try: print_info('Analyzing EC2 config... ', new_line=False) # Custom EC2 analysis # check_for_elastic_ip(ec2_info) # FIXME - commented for now as this method doesn't seem to be defined anywhere' # list_network_attack_surface(ec2_info, 'attack_surface', 'PublicIpAddress') # TODO: make this optional, commented out for now # list_network_attack_surface(ec2_info, 'private_attack_surface', 'PrivateIpAddress') print_info('Success') except Exception as e: print_info('Error') print_exception(e)
def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. :return: """ regions = [] if regions is None else regions global status, formatted_string # Initialize targets if not targets: targets = type(self).targets print_info('Fetching %s config...' % format_service_name(self.service)) formatted_string = None # FIXME the below should be in moved to each provider's code # Connect to the service if self._is_provider('aws'): if self.service in ['s3']: # S3 namespace is global but APIs aren't.... api_clients = {} for region in build_region_list(self.service, regions, partition_name): api_clients[region] = connect_service('s3', credentials, region, silent=True) api_client = api_clients[list(api_clients.keys())[0]] elif self.service == 'route53domains': api_client = connect_service(self.service, credentials, 'us-east-1', silent=True) # TODO: use partition's default region else: api_client = connect_service(self.service, credentials, silent=True) elif self._is_provider('gcp'): api_client = gcp_connect_service(service=self.service, credentials=credentials) elif self._is_provider('azure'): api_client = azure_connect_service(service=self.service, credentials=credentials) # Threading to fetch & parse resources (queue consumer) params = {'api_client': api_client} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients # Threading to parse resources (queue feeder) target_queue = self._init_threading(self.__fetch_target, params, self.thread_config['parse']) # Threading to list resources (queue feeder) params = {'api_client': api_client, 'q': target_queue} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients service_queue = self._init_threading(self.__fetch_service, params, self.thread_config['list']) # Init display self.fetchstatuslogger = FetchStatusLogger(targets) # Go for target in targets: service_queue.put(target) # Blocks until all items in the queue have been gotten and processed. service_queue.join() target_queue.join() # Show completion and force newline if self._is_provider('aws'): # Show completion and force newline if self.service != 'iam': self.fetchstatuslogger.show(True) else: self.fetchstatuslogger.show(True) # Threads should stop running as queues are empty self.run_target_threads = False self.run_service_threads = False # Put x items in the queues to ensure threads run one last time (and exit) for i in range(self.thread_config['parse']): target_queue.put(None) for j in range(self.thread_config['list']): service_queue.put(None)
def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. """ # Initialize targets # Tweak params regions = [] if regions is None else regions realtargets = () if not targets: targets = self.targets for i, target in enumerate(targets['first_region']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ( (target[0], target[1], target[2], params, target[4]), ) targets['first_region'] = realtargets realtargets = () for i, target in enumerate(targets['other_regions']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ( (target[0], target[1], target[2], params, target[4]), ) targets['other_regions'] = realtargets print_info('Fetching %s config...' % format_service_name(self.service)) self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True) api_service = 'ec2' if self.service.lower( ) == 'vpc' else self.service.lower() # Init regions regions = build_region_list( api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) q = self._init_threading(self._fetch_target, {}, self.thread_config['parse']) # Threading to list resources (queue feeder) qr = self._init_threading( self._fetch_region, { 'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': () }, self.thread_config['list']) # Go for i, region in enumerate(regions): qr.put((region, targets['first_region'] if i == 0 else targets['other_regions'])) # Blocks until all items in the queue have been gotten and processed. qr.join() q.join() # Show completion and force newline self.fetchstatuslogger.show(True) # Threads should stop running as queues are empty self.run_qr_threads = False self.run_q_threads = False # Put x items in the queues to ensure threads run one last time (and exit) for i in range(self.thread_config['parse']): q.put(None) for j in range(self.thread_config['list']): qr.put(None)
def main(args): # Configure the debug level config_debug_level(args.debug) # FIXME check that all requirements are installed # # Check version of opinel # if not check_requirements(os.path.realpath(__file__)): # return 42 # Support multiple environments for profile_name in args.profile: # Load the config try: # FIXME this is specific to AWS report_file_name = 'aws-%s' % profile_name report = Scout2Report('aws', report_file_name, args.report_dir, args.timestamp) aws_config = report.jsrw.load_from_file(AWSCONFIG) services = aws_config['service_list'] except Exception as e: print_exception(e) print_error('Error, failed to load the configuration for profile %s' % profile_name) continue # Create a ruleset with only whatever rules were specified... if args.config: rule_filename = args.config ruleset = TmpRuleset(environment_name=args.profile[0], cloud_provider='aws', rule_dirs=[os.getcwd()], rule_filename=args.config, rule_args=args.config_args) elif len(args.path) > 0: # Create a local tmp rule rule_dict = {'description': 'artifact'} rule_dict['path'] = args.path[0] rule_dict['conditions'] = [] rule_filename = 'listall-artifact.json' with open(os.path.join(os.getcwd(), rule_filename), 'wt') as f: f.write(json.dumps(rule_dict)) ruleset = TmpRuleset(rule_dirs=[os.getcwd()], rule_filename=rule_filename, rule_args=[]) else: print_error( 'Error, you must provide either a rule configuration file or the path to the resources targeted.') continue # FIXME is broken in Scout Suite, only handles AWS cloud_provider = get_provider(provider='aws', profile=args.profile[0]) # Process the rule pe = ProcessingEngine(ruleset) pe.run(cloud_provider, skip_dashboard=True) # Retrieve items rule = ruleset.rules[rule_filename][0] rule_service = rule.service.lower() rule_key = rule.key rule_type = rule.rule_type resources = aws_config['services'][rule_service][rule_type][rule_key]['items'] # Set the keys to output if len(args.keys): # 1. Explicitly provided on the CLI rule.keys = args.keys elif len(args.keys_file): # 2. Explicitly provided files that contain the list of keys rule.keys = [] for filename in args.keys_file: with open(filename, 'rt') as f: rule.keys += json.load(f)['keys'] else: try: # 3. Load default set of keys based on path target_path = rule.display_path if hasattr(rule, 'display_path') else rule.path listall_configs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'output/data/listall-configs') target_file = os.path.join(listall_configs_dir, '%s.json' % target_path) if os.path.isfile(target_file): with open(target_file, 'rt') as f: rule.keys = json.load(f)['keys'] except: # 4. Print the object name rule.keys = ['name'] # Prepare the output format (lines, template) = format_listall_output(args.format_file[0], None, args.format, rule) # Print the output print_info(generate_listall_output(lines, resources, aws_config, template, []))
async def _run(provider, # AWS profile, aws_access_key_id, aws_secret_access_key, aws_session_token, # Azure user_account, service_account, cli, msi, service_principal, file_auth, tenant_id, subscription_id, client_id, client_secret, username, password, # GCP project_id, folder_id, organization_id, all_projects, # Aliyun access_key_id, access_key_secret, # General report_name, report_dir, timestamp, services, skipped_services, result_format, database_name, host_ip, host_port, regions, excluded_regions, fetch_local, update, ip_ranges, ip_ranges_name_key, ruleset, exceptions, force_write, debug, quiet, log_file, no_browser, programmatic_execution, **kwargs): """ Run a scout job. """ # Configure the debug level set_logger_configuration(debug, quiet, log_file) print_info('Launching Scout') print_info('Authenticating to cloud provider') auth_strategy = get_authentication_strategy(provider) try: credentials = auth_strategy.authenticate(profile=profile, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, user_account=user_account, service_account=service_account, cli=cli, msi=msi, service_principal=service_principal, file_auth=file_auth, tenant_id=tenant_id, subscription_id=subscription_id, client_id=client_id, client_secret=client_secret, username=username, password=password, access_key_id=access_key_id, access_key_secret=access_key_secret, programmatic_execution=programmatic_execution) if not credentials: return 101 except Exception as e: print_exception('Authentication failure: {}'.format(e)) return 101 # Create a cloud provider object cloud_provider = get_provider(provider=provider, profile=profile, project_id=project_id, folder_id=folder_id, organization_id=organization_id, all_projects=all_projects, report_dir=report_dir, timestamp=timestamp, services=services, skipped_services=skipped_services, credentials=credentials) # Create a new report report_name = report_name if report_name else cloud_provider.get_report_name() report = ScoutReport(cloud_provider.provider_code, report_name, report_dir, timestamp, result_format=result_format) if database_name: database_file, _ = get_filename('RESULTS', report_name, report_dir, file_extension="db") Server.init(database_file, host_ip, host_port) return # Complete run, including pulling data from provider if not fetch_local: # Fetch data from provider APIs try: print_info('Gathering data from APIs') await cloud_provider.fetch(regions=regions, excluded_regions=excluded_regions) except KeyboardInterrupt: print_info('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it if update: print_info('Updating existing data') current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.encoder.load_from_file('RESULTS') cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[service] # Partial run, using pre-pulled data else: print_info('Using local data') # Reload to flatten everything into a python dictionary last_run_dict = report.encoder.load_from_file('RESULTS') for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) # Pre processing cloud_provider.preprocessing( ip_ranges, ip_ranges_name_key) # Analyze config print_info('Running rule engine') finding_rules = Ruleset(cloud_provider=cloud_provider.provider_code, environment_name=cloud_provider.environment, filename=ruleset, ip_ranges=ip_ranges, account_id=cloud_provider.account_id) processing_engine = ProcessingEngine(finding_rules) processing_engine.run(cloud_provider) # Create display filters print_info('Applying display filters') filter_rules = Ruleset(cloud_provider=cloud_provider.provider_code, environment_name=cloud_provider.environment, rule_type='filters', account_id=cloud_provider.account_id) processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) # Handle exceptions if exceptions: print_info('Applying exceptions') try: exceptions = RuleExceptions(exceptions) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: print_exception('Failed to load exceptions: {}'.format(e)) exceptions = {} else: exceptions = {} run_parameters = { 'services': services, 'skipped_services': skipped_services, 'regions': regions, 'excluded_regions': excluded_regions, } # Finalize cloud_provider.postprocessing(report.current_time, finding_rules, run_parameters) # Save config and create HTML report html_report_path = report.save( cloud_provider, exceptions, force_write, debug) # Open the report by default if not no_browser: print_info('Opening the HTML report') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) if ERRORS_LIST: # errors were handled during execution return 200 else: return 0