def get_sns_regions(name=None):
res = sns.regions()
if name:
res = [
ri for ri in res
if ri.name == name]
return res
def connect_sns(self):
'''
Connect to SNS
'''
try:
for reg in sns.regions():
if(reg.name == os.environ['AWS_DEFAULT_REGION']):
return sns.connect_to_region(reg.name)
return None
except Exception as e:
print "%s" % e
exit(-1)
def connect_sns(self):
'''
Connect to SNS
'''
try:
for reg in sns.regions():
if (reg.name == os.environ['AWS_DEFAULT_REGION']):
return sns.connect_to_region(reg.name)
return None
except Exception as e:
print "%s" % e
exit(-1)
def sns_connection(region_name=settings.AWS_SNS_REGION_NAME):
"""Returns an SNSConnection that is already authenticated.
us-west-2 is oregon, the region we use by default.
@raises IndexError
"""
region = filter(lambda x: x.name == region_name, sns.regions())[0]
return sns.SNSConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region=region)
def slurp(self):
"""
:returns: item_list - list of SNSItem's.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
try:
(sns,
topics) = self.get_all_topics_in_region(account, region)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sns', account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(topics),
SNS.i_am_plural))
for topic in topics:
arn = topic['TopicArn']
### Check if this SNS Topic is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if arn.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
attrs = self.wrap_aws_rate_limited_call(
sns.get_topic_attributes, arn)
item = self.build_item(arn=arn,
attrs=attrs,
region=region.name,
account=account,
exception_map=exception_map)
if item:
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of SNSItem's.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
try:
(sns,
topics) = self.get_all_topics_in_region(account, region)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sns', account,
region.name)
self.slurp_exception(
(self.index, account, region.name),
exc,
exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(topics),
SNS.i_am_plural))
for topic in topics:
arn = topic['TopicArn']
if self.check_ignore_list(arn.split(':')[5]):
continue
item = self.build_item(arn=arn,
conn=sns,
region=region.name,
account=account,
exception_map=exception_map)
if item:
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of SNSItem's.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
try:
(sns, topics) = self.get_all_topics_in_region(account, region)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sns', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(topics), SNS.i_am_plural))
for topic in topics:
arn = topic['TopicArn']
### Check if this SNS Topic is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if arn.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
attrs = sns.get_topic_attributes(arn)
item = self.build_item(arn=arn,
attrs=attrs,
region=region.name,
account=account,
exception_map=exception_map)
if item:
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of SNSItem's.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
try:
(sns, topics) = self.get_all_topics_in_region(account, region)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sns', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(topics), SNS.i_am_plural))
for topic in topics:
arn = topic['TopicArn']
if self.check_ignore_list(arn):
continue
attrs = self.wrap_aws_rate_limited_call(
sns.get_topic_attributes,
arn
)
item = self.build_item(arn=arn,
attrs=attrs,
region=region.name,
account=account,
exception_map=exception_map)
if item:
item_list.append(item)
return item_list, exception_map
def compose(*functions):
return functools.reduce(lambda f, g: lambda x: f(g(x)), functions,
lambda x: x)
load_dotenv(find_dotenv())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
sNSConnection = sns.SNSConnection(
aws_access_key_id=os.environ['AWS_KEY'],
aws_secret_access_key=os.environ['AWS_SECRET'],
region=[r for r in sns.regions() if r.name == os.environ['AWS_REGION']][0])
s3Connection = S3Connection(aws_access_key_id=os.environ['AWS_KEY'],
aws_secret_access_key=os.environ['AWS_SECRET'])
def elapsed_lambda(d, percentiles=[50, 90, 95, 99]):
series = pd.Series({
'min': np.min(d['elapsed']),
'max': np.max(d['elapsed']),
'mean': np.mean(d['elapsed']),
'std': np.std(d['elapsed'])
})
perc = lmap(lambda x: pd.Series({str(x): np.percentile(d['elapsed'], x)}),
percentiles)