def fetch_all(self, credentials, regions=[], partition_name='aws', targets=None): """ Fetch all the SNS configuration supported by Scout2 :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. """ # Initialize targets if not targets: try: targets = type( self).targets # TODO: remove this case eventually except: targets = self.targets # Tweak params realtargets = () for i, target in enumerate(targets): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ( (target[0], target[1], target[2], params, target[4]), ) targets = realtargets printInfo('Fetching %s config...' % format_service_name(self.service)) self.fetchstatuslogger = FetchStatusLogger(targets, True) api_service = 'ec2' if self.service.lower( ) == 'vpc' else self.service.lower() # Init regions regions = build_region_list( api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) q = self._init_threading(self._fetch_target, {}, 20) # Threading to list resources (queue feeder) qr = self._init_threading( self._fetch_region, { 'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': targets }, 10) # Go for region in regions: qr.put(region) # Join qr.join() q.join() # Show completion and force newline self.fetchstatuslogger.show(True)
def postprocessing(aws_config): for service in aws_config['services']: method_name = '%s_postprocessing' % service if method_name in globals(): try: printInfo('Post-processing %s config...' % format_service_name(service)) method = globals()[method_name] method(aws_config) except Exception as e: printException(e) pass
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None): """ Generic fetching function that iterates through all of the service's targets :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. """ global status, formatted_string # Initialize targets if not targets: targets = type(self).targets printInfo('Fetching %s config...' % format_service_name(self.service)) formatted_string = None api_service = self.service.lower() # Connect to the service if self.service in [ 's3' ]: # S3 namespace is global but APIs aren't.... api_clients = {} for region in build_region_list(self.service, regions, partition_name): api_clients[region] = connect_service('s3', credentials, region, silent = True) api_client = api_clients[list(api_clients.keys())[0]] elif self.service == 'route53domains': api_client = connect_service(self.service, credentials, 'us-east-1', silent = True) # TODO: use partition's default region else: api_client = connect_service(self.service, credentials, silent = True) # Threading to fetch & parse resources (queue consumer) params = {'api_client': api_client} if self.service in ['s3']: params['api_clients'] = api_clients q = self._init_threading(self.__fetch_target, params, 1) # Threading to list resources (queue feeder) params = {'api_client': api_client, 'q': q} if self.service in ['s3']: params['api_clients'] = api_clients qt = self._init_threading(self.__fetch_service, params, 1) # Init display self.fetchstatuslogger = FetchStatusLogger(targets) # Go for target in targets: qt.put(target) # Join qt.join() q.join() # Show completion and force newline if self.service != 'iam': self.fetchstatuslogger.show(True)
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. """ # Initialize targets # Tweak params realtargets = () if not targets: targets = self.targets for i, target in enumerate(targets['first_region']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) targets['first_region'] = realtargets realtargets = () for i, target in enumerate(targets['other_regions']): params = self.tweak_params(target[3], credentials) realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) targets['other_regions'] = realtargets printInfo('Fetching %s config...' % format_service_name(self.service)) self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True) api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) q = self._init_threading(self._fetch_target, {}, self.thread_config['parse']) # Threading to list resources (queue feeder) qr = self._init_threading(self._fetch_region, {'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()}, self.thread_config['list']) # Go for i, region in enumerate(regions): qr.put((region, targets['first_region'] if i == 0 else targets['other_regions'])) # Join qr.join() q.join() # Show completion and force newline self.fetchstatuslogger.show(True)
def set_definition(self, rule_definitions, attributes = [], ip_ranges = [], params = {}): """ Update every attribute of the rule by setting the argument values as necessary :param parameterized_input: :param arg_values: :param convert: :return: """ string_definition = rule_definitions[self.filename].string_definition # Load condition dependencies definition = json.loads(string_definition) definition['conditions'] += self.conditions loaded_conditions = [] for condition in definition['conditions']: if condition[0].startswith('_INCLUDE_('): include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0] #new_conditions = load_data(include, key_name = 'conditions') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/%s' % include), 'rt') as f: new_conditions = f.read() for (i, value) in enumerate(condition[1]): new_conditions = re.sub(condition[1][i], condition[2][i], new_conditions) new_conditions = json.loads(new_conditions)['conditions'] loaded_conditions.append(new_conditions) else: loaded_conditions.append(condition) definition['conditions'] = loaded_conditions string_definition = json.dumps(definition) # Set parameters parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)', string_definition) for param in parameters: index = int(param[1]) if len(self.args) <= index: string_definition = string_definition.replace(param[0], '') elif type(self.args[index]) == list: value = '[ %s ]' % ', '.join('"%s"' % v for v in self.args[index]) string_definition = string_definition.replace('"%s"' % param[0], value) else: string_definition = string_definition.replace(param[0], self.args[index]) # Strip dots if necessary stripdots = re_strip_dots.findall(string_definition) for value in stripdots: string_definition = string_definition.replace(value[0], value[1].replace('.', '')) definition = json.loads(string_definition) # Set special values (IP ranges, AWS account ID, ...) for condition in definition['conditions']: if type(condition) != list or len(condition) == 1 or type(condition[2]) == list: continue for testcase in testcases: result = testcase['regex'].match(condition[2]) if result and (testcase['name'] == 'ip_ranges_from_file' or testcase['name'] == 'ip_ranges_from_local_file'): filename = result.groups()[0] conditions = result.groups()[1] if len(result.groups()) > 1 else [] # TODO :: handle comma here... if filename == ip_ranges_from_args: prefixes = [] for filename in ip_ranges: prefixes += read_ip_ranges(filename, local_file = True, ip_only = True, conditions = conditions) condition[2] = prefixes break else: local_file = True if testcase['name'] == 'ip_ranges_from_local_file' else False condition[2] = read_ip_ranges(filename, local_file = local_file, ip_only = True, conditions = conditions) break break elif result: condition[2] = params[testcase['name']] break if len(attributes) == 0: attributes = [attr for attr in definition] for attr in attributes: if attr in definition: setattr(self, attr, definition[attr]) if hasattr(self, 'path'): self.service = format_service_name(self.path.split('.')[0]) if not hasattr(self, 'key'): setattr(self, 'key', self.filename) setattr(self, 'key', self.key.replace('.json', '')) if self.key_suffix: setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix))
def set_definition(self, rule_definitions, attributes=[], ip_ranges=[], params={}): """ Update every attribute of the rule by setting the argument values as necessary :param parameterized_input: :param arg_values: :param convert: :return: """ string_definition = rule_definitions[self.filename].string_definition # Load condition dependencies definition = json.loads(string_definition) loaded_conditions = [] for condition in definition['conditions']: if condition[0].startswith('_INCLUDE_('): include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0] #new_conditions = load_data(include, key_name = 'conditions') with open( os.path.join( os.path.dirname(os.path.realpath(__file__)), 'data/%s' % include), 'rt') as f: new_conditions = f.read() for (i, value) in enumerate(condition[1]): new_conditions = re.sub(condition[1][i], condition[2][i], new_conditions) new_conditions = json.loads(new_conditions)['conditions'] loaded_conditions.append(new_conditions) else: loaded_conditions.append(condition) definition['conditions'] = loaded_conditions string_definition = json.dumps(definition) # Set parameters parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)', string_definition) for param in parameters: index = int(param[1]) if type(self.args[index]) == list: value = '[ %s ]' % ', '.join('"%s"' % v for v in self.args[index]) string_definition = string_definition.replace( '"%s"' % param[0], value) else: string_definition = string_definition.replace( param[0], self.args[index]) # Strip dots if necessary stripdots = re_strip_dots.findall(string_definition) for value in stripdots: string_definition = string_definition.replace( value[0], value[1].replace('.', '')) definition = json.loads(string_definition) # Set special values (IP ranges, AWS account ID, ...) for condition in definition['conditions']: if type(condition) != list or len(condition) == 1 or type( condition[2]) == list: continue for testcase in testcases: result = testcase['regex'].match(condition[2]) if result and (testcase['name'] == 'ip_ranges_from_file' or testcase['name'] == 'ip_ranges_from_local_file'): filename = result.groups()[0] conditions = result.groups()[1] if len( result.groups()) > 1 else [] # TODO :: handle comma here... if filename == ip_ranges_from_args: prefixes = [] for filename in ip_ranges: prefixes += read_ip_ranges(filename, local_file=True, ip_only=True, conditions=conditions) condition[2] = prefixes break else: local_file = True if testcase[ 'name'] == 'ip_ranges_from_local_file' else False condition[2] = read_ip_ranges(filename, local_file=local_file, ip_only=True, conditions=conditions) break break elif result: condition[2] = params[testcase['name']] break if len(attributes) == 0: attributes = [attr for attr in definition] for attr in attributes: if attr in definition: setattr(self, attr, definition[attr]) if hasattr(self, 'path'): self.service = format_service_name(self.path.split('.')[0]) if not hasattr(self, 'key'): setattr(self, 'key', self.filename) setattr(self, 'key', self.key.replace('.json', ''))