def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Fetch all the SNS configuration supported by Scout2
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
global status, formatted_string
# Initialize targets
if not targets:
targets = type(self).targets
printInfo('Fetching %s config...' % format_service_name(self.service))
formatted_string = None
api_service = self.service.lower()
# Connect to the service
if self.service in ['s3'
]: # S3 namespace is global but APIs aren't....
api_clients = {}
for region in build_region_list(self.service, regions,
partition_name):
api_clients[region] = connect_service('s3', credentials,
region)
api_client = api_clients[list(api_clients.keys())[0]]
elif self.service == 'route53domains':
api_client = connect_service(
self.service, credentials,
'us-east-1') # TODO: use partition's default region
else:
api_client = connect_service(self.service, credentials)
# Threading to fetch & parse resources (queue consumer)
params = {'api_client': api_client}
if self.service in ['s3']:
params['api_clients'] = api_clients
q = self._init_threading(self.__fetch_target, params, 20)
# Threading to list resources (queue feeder)
params = {'api_client': api_client, 'q': q}
if self.service in ['s3']:
params['api_clients'] = api_clients
qt = self._init_threading(self.__fetch_service, params, 10)
# Init display
self.fetchstatuslogger = FetchStatusLogger(targets)
# Go
for target in targets:
qt.put(target)
# Join
qt.join()
q.join()
# Show completion and force newline
if self.service != 'iam':
self.fetchstatuslogger.show(True)
def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Fetch all the SNS configuration supported by Scout2
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
if not targets:
try:
targets = type(
self).targets # TODO: remove this case eventually
except:
targets = self.targets
# Tweak params
realtargets = ()
for i, target in enumerate(targets):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + (
(target[0], target[1], target[2], params, target[4]), )
targets = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets, True)
api_service = 'ec2' if self.service.lower(
) == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(
api_service, regions,
partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, 20)
# Threading to list resources (queue feeder)
qr = self._init_threading(
self._fetch_region, {
'api_service': api_service,
'credentials': credentials,
'q': q,
'targets': targets
}, 10)
# Go
for region in regions:
qr.put(region)
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None):
"""
Fetch all the configuration supported by Scout2 for a given service
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
# Tweak params
realtargets = ()
if not targets:
targets = self.targets
for i, target in enumerate(targets['first_region']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['first_region'] = realtargets
realtargets = ()
for i, target in enumerate(targets['other_regions']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['other_regions'] = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True)
api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, self.thread_config['parse'])
# Threading to list resources (queue feeder)
qr = self._init_threading(self._fetch_region,
{'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()},
self.thread_config['list'])
# Go
for i, region in enumerate(regions):
qr.put((region, targets['first_region'] if i == 0 else targets['other_regions']))
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
class RegionalServiceConfig(object):
"""
Single service configuration for non-global services
:ivar regions: Dictionary of regions
:ivar service: Name of the service
"""
def __init__(self, service_metadata={}, thread_config=4):
self.regions = {}
self.thread_config = thread_configs[thread_config]
self.service = \
type(self).__name__.replace('Config', '').lower() # TODO: use regex with EOS instead of plain replace
if service_metadata != {}:
self.resource_types = {'global': [], 'region': [], 'vpc': []}
self.targets = {'first_region': (), 'other_regions': ()}
for resource in service_metadata['resources']:
only_first_region = False
if re.match(r'.*?\.vpcs\.id\..*?',
service_metadata['resources'][resource]['path']):
self.resource_types['vpc'].append(resource)
elif re.match(r'.*?\.regions\.id\..*?',
service_metadata['resources'][resource]['path']):
self.resource_types['region'].append(resource)
else:
only_first_region = True
self.resource_types['global'].append(resource)
resource_metadata = service_metadata['resources'][resource]
if 'api_call' not in resource_metadata:
continue
params = resource_metadata[
'params'] if 'params' in resource_metadata else {}
ignore_exceptions = True if 'no_exceptions' in resource_metadata and \
resource_metadata['no_exceptions'] == True else False
if not only_first_region:
self.targets['other_regions'] += (
(resource, resource_metadata['response'],
resource_metadata['api_call'], params,
ignore_exceptions), )
self.targets['first_region'] += (
(resource, resource_metadata['response'],
resource_metadata['api_call'], params,
ignore_exceptions), )
def init_region_config(self, region):
"""
Initialize the region's configuration
:param region: Name of the region
"""
self.regions[region] = self.region_config_class(
region_name=region, resource_types=self.resource_types)
def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Fetch all the configuration supported by Scout2 for a given service
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
# Tweak params
realtargets = ()
if not targets:
targets = self.targets
for i, target in enumerate(targets['first_region']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + (
(target[0], target[1], target[2], params, target[4]), )
targets['first_region'] = realtargets
realtargets = ()
for i, target in enumerate(targets['other_regions']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + (
(target[0], target[1], target[2], params, target[4]), )
targets['other_regions'] = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets['first_region'],
True)
api_service = 'ec2' if self.service.lower(
) == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(
api_service, regions,
partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {},
self.thread_config['parse'])
# Threading to list resources (queue feeder)
qr = self._init_threading(
self._fetch_region, {
'api_service': api_service,
'credentials': credentials,
'q': q,
'targets': ()
}, self.thread_config['list'])
# Go
for i, region in enumerate(regions):
qr.put((region, targets['first_region']
if i == 0 else targets['other_regions']))
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def _init_threading(self, function, params={}, num_threads=10):
"""
Initialize queue and threads
:param function:
:param params:
:param num_threads:
:return:
"""
q = Queue(maxsize=0) # TODO: find something appropriate
for i in range(num_threads):
worker = Thread(target=function, args=(q, params))
worker.setDaemon(True)
worker.start()
return q
def _fetch_region(self, q, params):
global api_clients
try:
while True:
try:
region, targets = q.get()
#print('Targets for region %s : %s' % (region, str(targets)))
self.init_region_config(region)
api_client = connect_service(params['api_service'],
params['credentials'],
region,
silent=True)
api_clients[region] = api_client
# TODO : something here for single_region stuff
self.regions[region].fetch_all(
api_client, self.fetchstatuslogger, params['q'],
targets) # params['targets'])
self.fetchstatuslogger.counts['regions']['fetched'] += 1
except Exception as e:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def _fetch_target(self, q, params):
try:
while True:
try:
method, region, target = q.get()
backup = copy.deepcopy(target)
if method.__name__ == 'store_target':
target_type = target['scout2_target_type']
else:
target_type = method.__name__.replace('parse_',
'') + 's'
method(params, region, target)
self.fetchstatuslogger.counts[target_type]['fetched'] += 1
self.fetchstatuslogger.show()
except Exception as e:
if is_throttled(e):
q.put((method, region, backup))
else:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def finalize(self):
for t in self.fetchstatuslogger.counts:
setattr(self, '%s_count' % t,
self.fetchstatuslogger.counts[t]['fetched'])
delattr(self, 'fetchstatuslogger')
for r in self.regions:
if hasattr(self.regions[r], 'fetchstatuslogger'):
delattr(self.regions[r], 'fetchstatuslogger')
def tweak_params(self, params, credentials):
if type(params) == dict:
for k in params:
params[k] = self.tweak_params(params[k], credentials)
elif type(params) == list:
newparams = []
for v in params:
newparams.append(self.tweak_params(v, credentials))
params = newparams
else:
if params == '_AWS_ACCOUNT_ID_':
params = get_aws_account_id(credentials)
return params
class RegionalServiceConfig(object):
"""
Single service configuration for non-global services
:ivar regions: Dictionary of regions
:ivar service: Name of the service
"""
def __init__(self, service_metadata={}, thread_config=4):
self.regions = {}
self.thread_config = thread_configs[thread_config]
self.service = \
type(self).__name__.replace('Config', '').lower() # TODO: use regex with EOS instead of plain replace
if service_metadata != {}:
self.resource_types = {'global': [], 'region': [], 'vpc': []}
self.targets = {'first_region': (), 'other_regions': ()}
for resource in service_metadata['resources']:
only_first_region = False
if re.match(r'.*?\.vpcs\.id\..*?', service_metadata['resources'][resource]['path']):
self.resource_types['vpc'].append(resource)
elif re.match(r'.*?\.regions\.id\..*?', service_metadata['resources'][resource]['path']):
self.resource_types['region'].append(resource)
else:
only_first_region = True
self.resource_types['global'].append(resource)
resource_metadata = service_metadata['resources'][resource]
if 'api_call' not in resource_metadata:
continue
params = resource_metadata['params'] if 'params' in resource_metadata else {}
ignore_exceptions = True if 'no_exceptions' in resource_metadata and \
resource_metadata['no_exceptions'] == True else False
if not only_first_region:
self.targets['other_regions'] += ((resource,
resource_metadata['response'],
resource_metadata['api_call'],
params,
ignore_exceptions),)
self.targets['first_region'] += ((resource,
resource_metadata['response'],
resource_metadata['api_call'],
params,
ignore_exceptions),)
def init_region_config(self, region):
"""
Initialize the region's configuration
:param region: Name of the region
"""
self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types)
def fetch_all(self, credentials, regions = [], partition_name = 'aws', targets = None):
"""
Fetch all the configuration supported by Scout2 for a given service
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
# Tweak params
realtargets = ()
if not targets:
targets = self.targets
for i, target in enumerate(targets['first_region']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['first_region'] = realtargets
realtargets = ()
for i, target in enumerate(targets['other_regions']):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),)
targets['other_regions'] = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True)
api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, self.thread_config['parse'])
# Threading to list resources (queue feeder)
qr = self._init_threading(self._fetch_region,
{'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()},
self.thread_config['list'])
# Go
for i, region in enumerate(regions):
qr.put((region, targets['first_region'] if i == 0 else targets['other_regions']))
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def _init_threading(self, function, params={}, num_threads=10):
"""
Initialize queue and threads
:param function:
:param params:
:param num_threads:
:return:
"""
q = Queue(maxsize=0) # TODO: find something appropriate
for i in range(num_threads):
worker = Thread(target=function, args=(q, params))
worker.setDaemon(True)
worker.start()
return q
def _fetch_region(self, q, params):
global api_clients
try:
while True:
try:
region, targets = q.get()
#print('Targets for region %s : %s' % (region, str(targets)))
self.init_region_config(region)
api_client = connect_service(params['api_service'], params['credentials'], region, silent = True)
api_clients[region] = api_client
# TODO : something here for single_region stuff
self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets'])
self.fetchstatuslogger.counts['regions']['fetched'] += 1
except Exception as e:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def _fetch_target(self, q, params):
try:
while True:
try:
method, region, target = q.get()
backup = copy.deepcopy(target)
if method.__name__ == 'store_target':
target_type = target['scout2_target_type']
else:
target_type = method.__name__.replace('parse_', '') + 's'
method(params, region, target)
self.fetchstatuslogger.counts[target_type]['fetched'] += 1
self.fetchstatuslogger.show()
except Exception as e:
if is_throttled(e):
q.put((method, region, backup))
else:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def finalize(self):
for t in self.fetchstatuslogger.counts:
setattr(self, '%s_count' % t, self.fetchstatuslogger.counts[t]['fetched'])
delattr(self, 'fetchstatuslogger')
for r in self.regions:
if hasattr(self.regions[r], 'fetchstatuslogger'):
delattr(self.regions[r], 'fetchstatuslogger')
def tweak_params(self, params, credentials):
if type(params) == dict:
for k in params:
params[k] = self.tweak_params(params[k], credentials)
elif type(params) == list:
newparams = []
for v in params:
newparams.append(self.tweak_params(v, credentials))
params = newparams
else:
if params == '_AWS_ACCOUNT_ID_':
params = get_aws_account_id(credentials)
return params
class RegionalServiceConfig(object):
"""
Single service configuration for non-global services
:ivar regions: Dictionary of regions
:ivar service: Name of the service
"""
def __init__(self):
self.regions = {}
self.service = type(self).__name__.replace(
'Config',
'').lower() # TODO: use regex with EOS instead of plain replace
def init_region_config(self, region):
"""
Initialize the region's configuration
:param region: Name of the region
"""
self.regions[region] = self.region_config_class()
def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Fetch all the SNS configuration supported by Scout2
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
# Initialize targets
if not targets:
targets = type(self).targets
# Tweak params
realtargets = ()
for i, target in enumerate(targets):
params = self.tweak_params(target[3], credentials)
realtargets = realtargets + (
(target[0], target[1], target[2], params, target[4]), )
targets = realtargets
printInfo('Fetching %s config...' % format_service_name(self.service))
self.fetchstatuslogger = FetchStatusLogger(targets, True)
api_service = 'ec2' if self.service.lower(
) == 'vpc' else self.service.lower()
# Init regions
regions = build_region_list(
api_service, regions,
partition_name) # TODO: move this code within this class
self.fetchstatuslogger.counts['regions']['discovered'] = len(regions)
# Threading to fetch & parse resources (queue consumer)
q = self._init_threading(self._fetch_target, {}, 1)
# Threading to list resources (queue feeder)
qr = self._init_threading(
self._fetch_region, {
'api_service': api_service,
'credentials': credentials,
'q': q,
'targets': targets
}, 1)
# Go
for region in regions:
qr.put(region)
# Join
qr.join()
q.join()
# Show completion and force newline
self.fetchstatuslogger.show(True)
def _init_threading(self, function, params={}, num_threads=1):
# Init queue and threads
q = Queue(maxsize=0) # TODO: find something appropriate
if not num_threads:
num_threads = len(targets)
for i in range(num_threads):
worker = Thread(target=function, args=(q, params))
worker.setDaemon(True)
worker.start()
return q
def _fetch_region(self, q, params):
global api_clients
try:
while True:
try:
region = q.get()
self.init_region_config(region)
api_client = connect_service(params['api_service'],
params['credentials'],
region,
silent=True)
api_clients[region] = api_client
self.regions[region].fetch_all(api_client,
self.fetchstatuslogger,
params['q'],
params['targets'])
self.fetchstatuslogger.counts['regions']['fetched'] += 1
except Exception as e:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def _fetch_target(self, q, params):
try:
while True:
try:
method, region, target = q.get()
method(params, region, target)
target = method.__name__.replace('parse_', '') + 's'
self.fetchstatuslogger.counts[target]['fetched'] += 1
self.fetchstatuslogger.show()
except Exception as e:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def finalize(self):
for t in self.fetchstatuslogger.counts:
setattr(self, '%s_count' % t,
self.fetchstatuslogger.counts[t]['fetched'])
delattr(self, 'fetchstatuslogger')
for r in self.regions:
if hasattr(self.regions[r], 'fetchstatuslogger'):
delattr(self.regions[r], 'fetchstatuslogger')
def tweak_params(self, params, credentials):
if type(params) == dict:
for k in params:
params[k] = self.tweak_params(params[k], credentials)
elif type(params) == list:
newparams = []
for v in params:
newparams.append(self.tweak_params(v, credentials))
params = newparams
else:
if params == '_AWS_ACCOUNT_ID_':
params = get_aws_account_id(credentials)
return params
class BaseConfig(GlobalConfig):
"""
FooBar
"""
def __init__(self, thread_config=4):
self.service = type(self).__name__.replace(
'Config',
'').lower() # TODO: use regex with EOS instead of plain replace
self.thread_config = thread_configs[thread_config]
def fetch_all(self,
credentials,
regions=[],
partition_name='aws',
targets=None):
"""
Generic fetching function that iterates through all of the service's targets
:param credentials: F
:param service: Name of the service
:param regions: Name of regions to fetch data from
:param partition_name: AWS partition to connect to
:param targets: Type of resources to be fetched; defaults to all.
"""
global status, formatted_string
# Initialize targets
if not targets:
targets = type(self).targets
printInfo('Fetching %s config...' % format_service_name(self.service))
formatted_string = None
api_service = self.service.lower()
# Connect to the service
if self.service in ['s3'
]: # S3 namespace is global but APIs aren't....
api_clients = {}
for region in build_region_list(self.service, regions,
partition_name):
api_clients[region] = connect_service('s3',
credentials,
region,
silent=True)
api_client = api_clients[list(api_clients.keys())[0]]
elif self.service == 'route53domains':
api_client = connect_service(
self.service, credentials, 'us-east-1',
silent=True) # TODO: use partition's default region
else:
api_client = connect_service(self.service,
credentials,
silent=True)
# Threading to fetch & parse resources (queue consumer)
params = {'api_client': api_client}
if self.service in ['s3']:
params['api_clients'] = api_clients
q = self._init_threading(self.__fetch_target, params,
self.thread_config['parse'])
# Threading to list resources (queue feeder)
params = {'api_client': api_client, 'q': q}
if self.service in ['s3']:
params['api_clients'] = api_clients
qt = self._init_threading(self.__fetch_service, params,
self.thread_config['list'])
# Init display
self.fetchstatuslogger = FetchStatusLogger(targets)
# Go
for target in targets:
qt.put(target)
# Join
qt.join()
q.join()
# Show completion and force newline
if self.service != 'iam':
self.fetchstatuslogger.show(True)
def finalize(self):
for t in self.fetchstatuslogger.counts:
setattr(self, '%s_count' % t,
self.fetchstatuslogger.counts[t]['fetched'])
self.__delattr__('fetchstatuslogger')
def _init_threading(self, function, params={}, num_threads=10):
# Init queue and threads
q = Queue(maxsize=0) # TODO: find something appropriate
if not num_threads:
num_threads = len(targets)
for i in range(num_threads):
worker = Thread(target=function, args=(q, params))
worker.setDaemon(True)
worker.start()
return q
def __fetch_service(self, q, params):
api_client = params['api_client']
try:
while True:
try:
target_type, response_attribute, list_method_name, list_params, ignore_list_error = q.get(
)
if not list_method_name:
continue
try:
method = getattr(api_client, list_method_name)
except Exception as e:
printException(e)
continue
try:
if type(list_params) != list:
list_params = [list_params]
targets = []
for lp in list_params:
targets += handle_truncated_response(
method, lp,
[response_attribute])[response_attribute]
except Exception as e:
if not ignore_list_error:
printException(e)
targets = []
self.fetchstatuslogger.counts[target_type][
'discovered'] += len(targets)
for target in targets:
params['q'].put((target_type, target), )
except Exception as e:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
def __fetch_target(self, q, params):
global status
try:
while True:
try:
target_type, target = q.get()
# Make a full copy of the target in case we need to re-queue it
backup = copy.deepcopy(target)
method = getattr(self, 'parse_%s' % target_type)
method(target, params)
self.fetchstatuslogger.counts[target_type]['fetched'] += 1
self.fetchstatuslogger.show()
except Exception as e:
if hasattr(e, 'response'
) and 'Error' in e.response and e.response[
'Error']['Code'] in ['Throttling']:
q.put((target_type, backup), )
else:
printException(e)
finally:
q.task_done()
except Exception as e:
printException(e)
pass
