def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Fetch all the SNS configuration supported by Scout2
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
if not targets:
try:
targets = type(
self).targets # TODO: remove this case eventually
except:
targets = self.targets
# Tweak params
realtargets = ()
for i, target in enumerate(targets):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + (
(target[0], target[1], target[2], params, target[4]), )
targets = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets, True)
api_service = 'ec2' if self.service.lower(
) == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(
api_service, regions,
partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, 20)
# Threading to list resources (queue feeder)
qr = self._init_threading(
self._fetch_region, {
'api_service': api_service,
'credentials': credentials,
'q': q,
'targets': targets
}, 10)
# Go
for region in regions:
qr.put(region)
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def postprocessing(aws_config):
for service in aws_config['services']:
method_name = '%s_postprocessing' % service
if method_name in globals():
try:
printInfo('Post-processing %s config...' % format_service_name(service))
method = globals()[method_name]
method(aws_config)
except Exception as e:
printException(e)
pass
def postprocessing(aws_config):
for service in aws_config['services']:
method_name = '%s_postprocessing' % service
if method_name in globals():
try:
printInfo('Post-processing %s config...' % format_service_name(service))
method = globals()[method_name]
method(aws_config)
except Exception as e:
printException(e)
pass
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None):
"""
Generic fetching function that iterates through all of the service's targets
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
global status, formatted_string
# Initialize targets
if not targets:
targets = type(self).targets
printInfo('Fetching %s config...' % format_service_name(self.service))
formatted_string = None
api_service = self.service.lower()
# Connect to the service
if self.service in [ 's3' ]: # S3 namespace is global but APIs aren't....
api_clients = {}
for region in build_region_list(self.service, regions, partition_name):
api_clients[region] = connect_service('s3', credentials, region, silent = True)
api_client = api_clients[list(api_clients.keys())[0]]
elif self.service == 'route53domains':
api_client = connect_service(self.service, credentials, 'us-east-1', silent = True) # TODO: use partition's default region
else:
api_client = connect_service(self.service, credentials, silent = True)
# Threading to fetch & parse resources (queue consumer)
params = {'api_client': api_client}
if self.service in ['s3']:
params['api_clients'] = api_clients
q = self._init_threading(self.__fetch_target, params, 1)
# Threading to list resources (queue feeder)
params = {'api_client': api_client, 'q': q}
if self.service in ['s3']:
params['api_clients'] = api_clients
qt = self._init_threading(self.__fetch_service, params, 1)
# Init display
self.fetchstatuslogger = FetchStatusLogger(targets)
# Go
for target in targets:
qt.put(target)
# Join
qt.join()
q.join()
# Show completion and force newline
if self.service != 'iam':
self.fetchstatuslogger.show(True)
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None):
"""
Fetch all the configuration supported by Scout2 for a given service
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
# Tweak params
realtargets = ()
if not targets:
targets = self.targets
for i, target in enumerate(targets['first_region']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['first_region'] = realtargets
realtargets = ()
for i, target in enumerate(targets['other_regions']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['other_regions'] = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True)
api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, self.thread_config['parse'])
# Threading to list resources (queue feeder)
qr = self._init_threading(self._fetch_region,
{'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()},
self.thread_config['list'])
# Go
for i, region in enumerate(regions):
qr.put((region, targets['first_region'] if i == 0 else targets['other_regions']))
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def set_definition(self, rule_definitions, attributes = [], ip_ranges = [], params = {}):
"""
Update every attribute of the rule by setting the argument values as necessary
:param parameterized_input:
:param arg_values:
:param convert:
:return:
"""
string_definition = rule_definitions[self.filename].string_definition
# Load condition dependencies
definition = json.loads(string_definition)
definition['conditions'] += self.conditions
loaded_conditions = []
for condition in definition['conditions']:
if condition[0].startswith('_INCLUDE_('):
include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0]
#new_conditions = load_data(include, key_name = 'conditions')
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/%s' % include), 'rt') as f:
new_conditions = f.read()
for (i, value) in enumerate(condition[1]):
new_conditions = re.sub(condition[1][i], condition[2][i], new_conditions)
new_conditions = json.loads(new_conditions)['conditions']
loaded_conditions.append(new_conditions)
else:
loaded_conditions.append(condition)
definition['conditions'] = loaded_conditions
string_definition = json.dumps(definition)
# Set parameters
parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)', string_definition)
for param in parameters:
index = int(param[1])
if len(self.args) <= index:
string_definition = string_definition.replace(param[0], '')
elif type(self.args[index]) == list:
value = '[ %s ]' % ', '.join('"%s"' % v for v in self.args[index])
string_definition = string_definition.replace('"%s"' % param[0], value)
else:
string_definition = string_definition.replace(param[0], self.args[index])
# Strip dots if necessary
stripdots = re_strip_dots.findall(string_definition)
for value in stripdots:
string_definition = string_definition.replace(value[0], value[1].replace('.', ''))
definition = json.loads(string_definition)
# Set special values (IP ranges, AWS account ID, ...)
for condition in definition['conditions']:
if type(condition) != list or len(condition) == 1 or type(condition[2]) == list:
continue
for testcase in testcases:
result = testcase['regex'].match(condition[2])
if result and (testcase['name'] == 'ip_ranges_from_file' or testcase['name'] == 'ip_ranges_from_local_file'):
filename = result.groups()[0]
conditions = result.groups()[1] if len(result.groups()) > 1 else []
# TODO :: handle comma here...
if filename == ip_ranges_from_args:
prefixes = []
for filename in ip_ranges:
prefixes += read_ip_ranges(filename, local_file = True, ip_only = True, conditions = conditions)
condition[2] = prefixes
break
else:
local_file = True if testcase['name'] == 'ip_ranges_from_local_file' else False
condition[2] = read_ip_ranges(filename, local_file = local_file, ip_only = True, conditions = conditions)
break
break
elif result:
condition[2] = params[testcase['name']]
break
if len(attributes) == 0:
attributes = [attr for attr in definition]
for attr in attributes:
if attr in definition:
setattr(self, attr, definition[attr])
if hasattr(self, 'path'):
self.service = format_service_name(self.path.split('.')[0])
if not hasattr(self, 'key'):
setattr(self, 'key', self.filename)
setattr(self, 'key', self.key.replace('.json', ''))
if self.key_suffix:
setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix))
def set_definition(self,
rule_definitions,
attributes=[],
ip_ranges=[],
params={}):
"""
Update every attribute of the rule by setting the argument values as necessary
:param parameterized_input:
:param arg_values:
:param convert:
:return:
"""
string_definition = rule_definitions[self.filename].string_definition
# Load condition dependencies
definition = json.loads(string_definition)
loaded_conditions = []
for condition in definition['conditions']:
if condition[0].startswith('_INCLUDE_('):
include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0]
#new_conditions = load_data(include, key_name = 'conditions')
with open(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'data/%s' % include), 'rt') as f:
new_conditions = f.read()
for (i, value) in enumerate(condition[1]):
new_conditions = re.sub(condition[1][i],
condition[2][i],
new_conditions)
new_conditions = json.loads(new_conditions)['conditions']
loaded_conditions.append(new_conditions)
else:
loaded_conditions.append(condition)
definition['conditions'] = loaded_conditions
string_definition = json.dumps(definition)
# Set parameters
parameters = re.findall(r'(_ARG_([a-zA-Z0-9]+)_)', string_definition)
for param in parameters:
index = int(param[1])
if type(self.args[index]) == list:
value = '[ %s ]' % ', '.join('"%s"' % v
for v in self.args[index])
string_definition = string_definition.replace(
'"%s"' % param[0], value)
else:
string_definition = string_definition.replace(
param[0], self.args[index])
# Strip dots if necessary
stripdots = re_strip_dots.findall(string_definition)
for value in stripdots:
string_definition = string_definition.replace(
value[0], value[1].replace('.', ''))
definition = json.loads(string_definition)
# Set special values (IP ranges, AWS account ID, ...)
for condition in definition['conditions']:
if type(condition) != list or len(condition) == 1 or type(
condition[2]) == list:
continue
for testcase in testcases:
result = testcase['regex'].match(condition[2])
if result and (testcase['name'] == 'ip_ranges_from_file'
or testcase['name']
== 'ip_ranges_from_local_file'):
filename = result.groups()[0]
conditions = result.groups()[1] if len(
result.groups()) > 1 else []
# TODO :: handle comma here...
if filename == ip_ranges_from_args:
prefixes = []
for filename in ip_ranges:
prefixes += read_ip_ranges(filename,
local_file=True,
ip_only=True,
conditions=conditions)
condition[2] = prefixes
break
else:
local_file = True if testcase[
'name'] == 'ip_ranges_from_local_file' else False
condition[2] = read_ip_ranges(filename,
local_file=local_file,
ip_only=True,
conditions=conditions)
break
break
elif result:
condition[2] = params[testcase['name']]
break
if len(attributes) == 0:
attributes = [attr for attr in definition]
for attr in attributes:
if attr in definition:
setattr(self, attr, definition[attr])
if hasattr(self, 'path'):
self.service = format_service_name(self.path.split('.')[0])
if not hasattr(self, 'key'):
setattr(self, 'key', self.filename)
setattr(self, 'key', self.key.replace('.json', ''))