def test_aws_error(self):
aws_exception = Exception("Some AWS Error")
aws = AWSError("Cannot do XYZ", aws_exception)
# mac os and linux produce slightly different results
self.assertFalse(aws.resource_not_found())
def test_aws_resource_not_found(self):
aws_exception = Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found"
)
aws = AWSError("Cannot do XYZ", aws_exception)
# mac os and linux produce slightly different results
self.assertTrue(aws.resource_not_found())
def test_test_aws_connectivity_with_exception(self):
with patch.object(aws_service_wrapper,
'cf_list_exports',
side_effect=AWSError("Some Error", None)):
with self.assertRaises(AWSError):
connector_test.test_aws_connectivity()
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
expected_return = ['TICKER-A', 'TICKER-B']
with patch.object(TickerFile, 'from_local_file',
return_value=TickerFile(expected_return)), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None),\
patch.object(os.path, 'isfile',
return_value=True):
ticker_file = TickerFile.from_s3_bucket('ticker-file', 'sa')
actual_return = ticker_file.ticker_list
self.assertListEqual(expected_return, actual_return)
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
configuration = Configuration.from_local_config(
constants.STRATEGY_CONFIG_FILE_NAME)
with patch.object(Configuration, 'from_local_config',
return_value=configuration), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=True):
Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME,
'sa')
# assert that s3_upload_object method was called once
self.assertEqual(mock_s3_upload_object.call_count, 1)
def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
a local no alternative is found, an exception will be
thrown
'''
configuration = Configuration.from_local_config(
constants.STRATEGY_CONFIG_FILE_NAME)
with patch.object(Configuration, 'from_local_config',
return_value=configuration), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(os.path, 'isfile',
return_value=False):
with self.assertRaises(AWSError):
Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME,
'sa')
def test_from_s3_with_boto_error_2(self):
with patch.object(aws_service_wrapper, 'cf_read_export_value',
side_effect=AWSError("test exception", None)), \
patch.object(aws_service_wrapper, 's3_download_object',
return_value=None):
with self.assertRaises(AWSError):
TestModel.from_s3("sa", 'test-object')
def test_notify_error_boto_error(self):
with patch.object(aws_service_wrapper, 'cf_read_export_value',
return_value="some_sns_arn"), \
patch.object(aws_service_wrapper, 'sns_publish_notification',
side_effect=AWSError("test exception", None)):
with self.assertRaises(AWSError):
aws_service_wrapper.notify_error(
"Security Recommendation Service", Exception("None"), 'stack trace', 'sa')
def test_save_to_s3_with_boto_error(self):
with patch.object(aws_service_wrapper, 'cf_read_export_value',
return_value="some_s3_bucket"), \
patch.object(aws_service_wrapper, 's3_upload_ascii_string',
side_effect=AWSError("test exception", None)):
with self.assertRaises(AWSError):
test_model = TestModel({})
test_model.save_to_s3("sa", 'test-object')
def test_from_s3_bucket_aws_error(self):
with patch.object(aws_service_wrapper, 's3_download_object',
return_value=None), \
patch.object(aws_service_wrapper, 'cf_list_exports',
side_effect=AWSError("test", None)):
with self.assertRaises(AWSError):
TickerFile.from_s3_bucket('ticker-file', 'sa')
with patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError("test", None)), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}):
with self.assertRaises(AWSError):
TickerFile.from_s3_bucket('ticker-file', 'sa')
def test_get_service_inputs_portfolio_not_found(self):
with patch.object(SecurityRecommendationSet, 'from_s3',
return_value=SecurityRecommendationSet), \
patch.object(SecurityRecommendationSet, 'is_current',
return_value=True), \
patch.object(Portfolio, 'from_s3',
side_effect=AWSError("", Exception("(404) Not found"))):
portfolio = portfolio_mgr_svc.get_service_inputs('sa')[0]
self.assertEqual(portfolio, None)
def s3_upload_object(source_path: str, bucket_name: str, object_name: str):
'''
Uploads a file from the source_path (path + file) to the destination bucket
'''
try:
S3_CLIENT.upload_file(source_path, bucket_name, object_name)
except Exception as e:
raise AWSError(
"Could not upload %s --> s3://%s/%s" %
(source_path, bucket_name, object_name), e)
def test_get_service_inputs_portfolio_error(self):
with patch.object(SecurityRecommendationSet, 'from_s3',
return_value=SecurityRecommendationSet), \
patch.object(SecurityRecommendationSet, 'is_current',
return_value=True), \
patch.object(Portfolio, 'from_s3',
side_effect=AWSError("", None)):
with self.assertRaises(AWSError):
portfolio_mgr_svc.get_service_inputs('sa')
def test_notify_no_new_recommendations(self):
with patch.object(aws_service_wrapper, 'cf_read_export_value',
return_value="some_sns_arn"), \
patch.object(aws_service_wrapper, 'sns_publish_notification',
side_effect=AWSError("test exception", None)):
notification_list = []
recommendation_svc.notify_new_recommendation(
notification_list, 'sa')
def test_aws_connectivity():
'''
Tests the connection to AWS by reading a cloudformation export.
Raises an AWSError if any errors are found
'''
log.info("Testing AWS connectivity")
try:
aws_service_wrapper.cf_list_exports(constants.APP_CF_STACK_NAMES)
log.info("AWS connectivity test successful")
except AWSError as awe:
raise AWSError("AWS connectivity test failed", awe.cause)
def s3_download_object(bucket_name: str, object_name: str, dest_path: str):
'''
Downloads an s3 object to the local filesystem and saves it to
the destination path (path + filename)
'''
try:
S3_CLIENT.download_file(bucket_name, object_name, dest_path)
except Exception as e:
raise AWSError(
"Could not download s3://%s/%s --> %s" %
(bucket_name, object_name, dest_path), e)
def sns_publish_notification(topic_arn: str, subject: str, message: str):
'''
Publishes a simple SNS message
'''
try:
SNS_CLIENT.publish(TopicArn=topic_arn,
Message=message,
Subject=subject)
except Exception as e:
raise AWSError("Cannot publish message to SNS topic: %s" % topic_arn,
e)
def test_simple_exception_nocause(self):
self.assertEqual(str(ValidationError("Cannot do XYZ", None)),
"Validation Error: Cannot do XYZ")
self.assertEqual(str(CalculationError("Cannot do XYZ", None)),
"Calculation Error: Cannot do XYZ")
self.assertEqual(str(DataError("Cannot do XYZ", None)),
"Data Error: Cannot do XYZ")
self.assertEqual(str(ReportError("Cannot do XYZ", None)),
"Report Error: Cannot do XYZ")
self.assertEqual(repr(AWSError("Cannot do XYZ", None)),
"AWS Error: Cannot do XYZ")
def test_notify_new_recommendation_with_boto_error(self):
with patch.object(aws_service_wrapper, 'cf_read_export_value',
return_value="some_sns_arn"), \
patch.object(aws_service_wrapper, 'sns_publish_notification',
side_effect=AWSError("test exception", None)):
notification_list = []
notification_list.append(SecurityRecommendationSet.from_parameters(datetime.now(), datetime.now(
), datetime.now(), datetime.now(), 'STRATEGY_NAME', 'US Equities', {'AAPL': 100}))
with self.assertRaises(AWSError):
recommendation_svc.notify_new_recommendation(
notification_list, 'sa')
def s3_upload_ascii_string(object_contents: str, s3_bucket_name: str,
s3_object_name: str):
'''
Uploads an ASCII string directly to S3, bypassing a local file.
'''
try:
S3_CLIENT.put_object(Body=bytes(object_contents, 'ascii'),
Bucket=s3_bucket_name,
Key=s3_object_name)
except Exception as e:
raise AWSError(
"Could not upload String to s3://%s/%s" %
(s3_bucket_name, s3_object_name), e)
def test_publish_current_returns(self):
security_recommendation = SecurityRecommendationSet.from_dict(
self.sr_dict)
portfolio = Portfolio()
portfolio.create_empty_portfolio(security_recommendation)
(new_p, updated) = portfolio_mgr_svc.update_portfolio(
portfolio, security_recommendation, 1)
with patch.object(aws_service_wrapper, 'sns_publish_notification',
side_effect=AWSError("", None)), \
patch.object(aws_service_wrapper, 'cf_read_export_value',
return_value="some_value"):
with self.assertRaises(AWSError):
portfolio_mgr_svc.publish_current_returns(new_p, updated, 'sa')
def test_from_s3_bucket_exception(self):
'''
Tests that if there was an error downloading the ticker file
from S3, it will thrown an exception
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception("Download Exception")
)
):
try:
TickerList.try_from_s3('sa', 'ticker-file')
except AWSError as awe:
self.assertTrue("Download Exception" in str(awe))
def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
no local alternative is found, it throws an exception.
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception("Download Exception")
)
),\
patch.object(os.path, 'isfile',
return_value=False):
try:
TickerFile.from_s3_bucket('ticker-file', 'sa')
except AWSError as awe:
self.assertTrue("Download Exception" in str(awe))
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
ticker_list = TickerList.from_dict({
"list_name":
"DOW30",
"list_type":
"US_EQUITIES",
"comparison_symbol":
"DIA",
"ticker_symbols": ["AAPL", "AXP", "BA"]
})
with patch.object(TickerList, 'from_local_file',
return_value=ticker_list), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(TickerList, 'from_s3',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=True):
ticker_file = TickerList.try_from_s3('sa', 'ticker-file')
# assert that s3_upload_object method was called once
self.assertEqual(mock_s3_upload_object.call_count, 1)
def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(TickerList, 'from_s3',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=False):
with self.assertRaises(AWSError):
TickerList.try_from_s3('sa', 'ticker-file')
def cf_list_exports(stack_name_filter: list):
'''
Reads all ClouFormation exports and returns only the ones
included in the stack_name_filter list
Parmeters
---------
stack_name_filter : list
A list of strings representing the names of the stacks used as a filter
Returns
---------
A dictionary {export_name, export_value} with the filtered values. e.g.
{
'export-name-1': 'value1',
'export-name-2': 'value2',
}
'''
def get_stackname_from_stackarn(arn: str):
# arn:aws:cloudformation:region:acct:stack/app-infra-base/c9481160-6df5-11ea-ac9f-121b58656156
try:
arn_elements = arn.split(':')
stack_id = arn_elements[5]
stack_elements = stack_id.split("/")
return stack_elements[1]
except Exception as e:
raise ValidationError("Could not parse stack ID from arn", e)
if stack_name_filter is None:
stack_name_filter = []
return_dict = {}
key_name = 'get_stackname_from_stackarn'
try:
log.debug("looking for cached cloudformation exports")
return aws_response_cache[key_name]
except KeyError:
log.debug("Exports not found. Looking them up")
try:
paginator = CF_CLIENT.get_paginator('list_exports')
response_iterator = paginator.paginate()
for page in response_iterator:
for export in page['Exports']:
stack_name = get_stackname_from_stackarn(
export['ExportingStackId'])
if (stack_name in stack_name_filter):
return_dict[export['Name']] = export['Value']
aws_response_cache[key_name] = return_dict
return return_dict
except Exception as e:
raise AWSError("Could not list Cloudformation exports", e)
"""
import boto3
from exception.exceptions import ValidationError, AWSError
from support import util, constants
import logging
log = logging.getLogger()
# Global clients available to this module]
try:
CF_CLIENT = boto3.client('cloudformation')
S3_CLIENT = boto3.client('s3')
SNS_CLIENT = boto3.client('sns')
except Exception as e:
raise AWSError("Could not connect to AWS", e)
# A simple in memory cached used to reduce roundtrips to AWS
# pylint: disable=invalid-name
aws_response_cache = {}
def cf_list_exports(stack_name_filter: list):
'''
Reads all ClouFormation exports and returns only the ones
included in the stack_name_filter list
Parmeters
---------
stack_name_filter : list
A list of strings representing the names of the stacks used as a filter
def test_get_service_inputs_with_s3_exception(self):
with patch.object(SecurityRecommendationSet, 'from_s3',
side_effect=AWSError("test error", None)):
with self.assertRaises(AWSError):
portfolio_mgr_svc.get_service_inputs('sa')