async def is_api_enabled(self, project_id, service): """ Given a project ID and service name, this method tries to determine if the service's API is enabled """ serviceusage_client = self._build_arbitrary_client('serviceusage', 'v1', force_new=True) services = serviceusage_client.services() try: request = services.list(parent=f'projects/{project_id}') services_response = await GCPFacadeUtils.get_all( 'services', request, services) except Exception as e: print_exception( f'Could not fetch the state of services for project \"{project_id}\", ' f'including {format_service_name(service.lower())} in the execution', {'exception': e}) return True # These are hardcoded endpoint correspondences as there's no easy way to do this. if service == 'IAM': endpoint = 'iam' elif service == 'KMS': endpoint = 'cloudkms' elif service == 'CloudStorage': endpoint = 'storage-component' elif service == 'CloudSQL': endpoint = 'sql-component' elif service == 'ComputeEngine': endpoint = 'compute' elif service == 'KubernetesEngine': endpoint = 'container' elif service == 'StackdriverLogging': endpoint = 'logging' elif service == 'StackdriverMonitoring': endpoint = 'monitoring' else: print_debug( 'Could not validate the state of the {} API for project \"{}\", ' 'including it in the execution'.format( format_service_name(service.lower()), project_id)) return True for s in services_response: if endpoint in s.get('name'): if s.get('state') == 'ENABLED': return True else: print_info( '{} API not enabled for project \"{}\", skipping'. format(format_service_name(service.lower()), project_id)) return False print_error( f'Could not validate the state of the {format_service_name(service.lower())} API ' f'for project \"{project_id}\", including it in the execution') return True
async def _fetch(self, service, regions=None, excluded_regions=None): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = {} if regions: method_args['regions'] = regions if excluded_regions: method_args['excluded_regions'] = excluded_regions if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials.session) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_exception(f'Could not fetch {service} configuration: {e}')
async def _fetch(self, service, regions): try: print_info('Fetching resources for the {} service'.format( format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): method_args = { 'credentials': self.credentials, 'regions': regions } if self._is_provider('aws'): if service != 'iam': method_args['partition_name'] = get_partition_name( self.credentials) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): await service_config.finalize() else: print_debug('No method to fetch service %s.' % service) except Exception as e: print_error('Error: could not fetch %s configuration.' % service) print_exception(e)
def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. """ # Initialize targets # Tweak params regions = [] if regions is None else regions realtargets = () if not targets: targets = self.targets for i, target in enumerate(targets['first_region']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) targets['first_region'] = realtargets realtargets = () for i, target in enumerate(targets['other_regions']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) targets['other_regions'] = realtargets printInfo('Fetching %s config...' % format_service_name(self.service)) self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True) api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) q = self._init_threading(self._fetch_target, {}, self.thread_config['parse']) # Threading to list resources (queue feeder) qr = self._init_threading(self._fetch_region, {'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()}, self.thread_config['list']) # Go for i, region in enumerate(regions): qr.put((region, targets['first_region'] if i == 0 else targets['other_regions'])) # Join qr.join() q.join() # Show completion and force newline self.fetchstatuslogger.show(True)
async def fetch(self, services=None, regions=None): # If services is set to None, fetch all services: services = vars(self) if services is None else services regions = [] if regions is None else regions # First, print services that are going to get skipped: for service in vars(self): if service not in services: print_debug('Skipping the {} service'.format( format_service_name(service))) # Then, fetch concurrently all services: tasks = { asyncio.ensure_future(self._fetch(service, regions)) for service in services } await asyncio.wait(tasks)
async def fetch(self, services: list, regions: list): if not services: print_debug('No services to scan') else: # Print services that are going to get skipped: for service in vars(self): if service not in services: print_debug('Skipping the {} service'.format( format_service_name(service))) # Remove "credentials" as it isn't a service if 'credentials' in services: services.remove('credentials') # Then, fetch concurrently all services: if services: tasks = { asyncio.ensure_future(self._fetch(service, regions)) for service in services } await asyncio.wait(tasks)
def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. :return: """ regions = [] if regions is None else regions global status, formatted_string # Initialize targets if not targets: targets = type(self).targets print_info('Fetching %s config...' % format_service_name(self.service)) formatted_string = None # FIXME the below should be in moved to each provider's code # Connect to the service if self._is_provider('aws'): if self.service in ['s3']: # S3 namespace is global but APIs aren't.... api_clients = {} for region in build_region_list(self.service, regions, partition_name): api_clients[region] = connect_service('s3', credentials, region, silent=True) api_client = api_clients[list(api_clients.keys())[0]] elif self.service == 'route53domains': api_client = connect_service(self.service, credentials, 'us-east-1', silent=True) # TODO: use partition's default region else: api_client = connect_service(self.service, credentials, silent=True) elif self._is_provider('gcp'): api_client = gcp_connect_service(service=self.service, credentials=credentials) elif self._is_provider('azure'): api_client = azure_connect_service(service=self.service, credentials=credentials) # Threading to fetch & parse resources (queue consumer) params = {'api_client': api_client} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients # Threading to parse resources (queue feeder) target_queue = self._init_threading(self.__fetch_target, params, self.thread_config['parse']) # Threading to list resources (queue feeder) params = {'api_client': api_client, 'q': target_queue} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients service_queue = self._init_threading(self.__fetch_service, params, self.thread_config['list']) # Init display self.fetchstatuslogger = FetchStatusLogger(targets) # Go for target in targets: service_queue.put(target) # Blocks until all items in the queue have been gotten and processed. service_queue.join() target_queue.join() # Show completion and force newline if self._is_provider('aws'): # Show completion and force newline if self.service != 'iam': self.fetchstatuslogger.show(True) else: self.fetchstatuslogger.show(True) # Threads should stop running as queues are empty self.run_target_threads = False self.run_service_threads = False # Put x items in the queues to ensure threads run one last time (and exit) for i in range(self.thread_config['parse']): target_queue.put(None) for j in range(self.thread_config['list']): service_queue.put(None)
def fetch_all(self, credentials, regions=[], partition_name='aws', targets=None): """ :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. :return: """ global status, formatted_string # Initialize targets if not targets: targets = type(self).targets printInfo('Fetching %s config...' % format_service_name(self.service)) formatted_string = None # Connect to the service if self._is_provider('aws'): if self.service in [ 's3' ]: # S3 namespace is global but APIs aren't.... api_clients = {} for region in build_region_list(self.service, regions, partition_name): api_clients[region] = connect_service('s3', credentials, region, silent=True) api_client = api_clients[list(api_clients.keys())[0]] elif self.service == 'route53domains': api_client = connect_service( self.service, credentials, 'us-east-1', silent=True) # TODO: use partition's default region else: api_client = connect_service(self.service, credentials, silent=True) elif self._is_provider('gcp'): api_client = gcp_connect_service(service=self.service, credentials=credentials) elif self._is_provider('azure'): api_client = azure_connect_service(service=self.service, credentials=credentials) # Threading to fetch & parse resources (queue consumer) params = {'api_client': api_client} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients # Threading to parse resources (queue feeder) target_queue = self._init_threading(self.__fetch_target, params, self.thread_config['parse']) # Threading to list resources (queue feeder) params = {'api_client': api_client, 'q': target_queue} if self._is_provider('aws'): if self.service in ['s3']: params['api_clients'] = api_clients service_queue = self._init_threading(self.__fetch_service, params, self.thread_config['list']) # Init display self.fetchstatuslogger = FetchStatusLogger(targets) # Go for target in targets: service_queue.put(target) # Join service_queue.join() target_queue.join() if self._is_provider('aws'): # Show completion and force newline if self.service != 'iam': self.fetchstatuslogger.show(True) else: self.fetchstatuslogger.show(True)
def set_definition(self, rule_definitions, attributes=None, ip_ranges=None, params=None): """ Update every attribute of the rule by setting the argument values as necessary :param rule_definitions: TODO :param attributes: TODO :param ip_ranges: TODO :param params: TODO :return: """ attributes = [] if attributes is None else attributes ip_ranges = [] if ip_ranges is None else ip_ranges params = {} if params is None else params try: string_definition = rule_definitions[ self.filename].string_definition # Load condition dependencies definition = json.loads(string_definition) definition['conditions'] += self.conditions loaded_conditions = [] for condition in definition['conditions']: if condition[0].startswith('_INCLUDE_('): include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0] # new_conditions = load_data(include, key_name = 'conditions') rules_path = f'{self.data_path}/{include}' with open(rules_path) as f: new_conditions = f.read() for (i, value) in enumerate(condition[1]): new_conditions = re.sub(condition[1][i], condition[2][i], new_conditions) new_conditions = json.loads( new_conditions)['conditions'] loaded_conditions.append(new_conditions) else: loaded_conditions.append(condition) definition['conditions'] = loaded_conditions string_definition = json.dumps(definition) # Set parameters parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)', string_definition) for param in parameters: index = int(param[1]) if len(self.args) <= index: string_definition = string_definition.replace(param[0], '') elif type(self.args[index]) == list: value = '[ %s ]' % ', '.join('"%s"' % v for v in self.args[index]) string_definition = string_definition.replace( '"%s"' % param[0], value) else: string_definition = string_definition.replace( param[0], self.args[index]) # Strip dots if necessary stripdots = re_strip_dots.findall(string_definition) for value in stripdots: string_definition = string_definition.replace( value[0], value[1].replace('.', '')) definition = json.loads(string_definition) # Set special values (IP ranges, AWS account ID, ...) for condition in definition['conditions']: if type(condition) != list \ or len(condition) == 1 \ or type(condition[2]) == list \ or type(condition[2]) == dict: continue for testcase in testcases: result = testcase['regex'].match(condition[2]) if result and (testcase['name'] == 'ip_ranges_from_file' or testcase['name'] == 'ip_ranges_from_local_file'): filename = result.groups()[0] conditions = result.groups()[1] if len( result.groups()) > 1 else [] # TODO :: handle comma here... if filename == ip_ranges_from_args: prefixes = [] for filename in ip_ranges: prefixes += read_ip_ranges( filename, local_file=True, ip_only=True, conditions=conditions) condition[2] = prefixes break else: local_file = True if testcase[ 'name'] == 'ip_ranges_from_local_file' else False condition[2] = read_ip_ranges( filename, local_file=local_file, ip_only=True, conditions=conditions) break elif result: condition[2] = params[testcase['name']] break if len(attributes) == 0: attributes = [attr for attr in definition] for attr in attributes: if attr in definition: setattr(self, attr, definition[attr]) if hasattr(self, 'path'): self.service = format_service_name(self.path.split('.')[0]) if not hasattr(self, 'key'): setattr(self, 'key', self.filename) setattr(self, 'key', self.key.replace('.json', '')) if self.key_suffix: setattr(self, 'key', f'{self.key}-{self.key_suffix}') except Exception as e: print_exception(f'Failed to set definition {self.filename}: {e}')