def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
a local no alternative is found, an exception will be
thrown
'''
configuration = Configuration.from_local_config(
constants.STRATEGY_CONFIG_FILE_NAME)
with patch.object(Configuration, 'from_local_config',
return_value=configuration), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(os.path, 'isfile',
return_value=False):
with self.assertRaises(AWSError):
Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME,
'sa')
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
expected_return = ['TICKER-A', 'TICKER-B']
with patch.object(TickerFile, 'from_local_file',
return_value=TickerFile(expected_return)), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None),\
patch.object(os.path, 'isfile',
return_value=True):
ticker_file = TickerFile.from_s3_bucket('ticker-file', 'sa')
actual_return = ticker_file.ticker_list
self.assertListEqual(expected_return, actual_return)
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
configuration = Configuration.from_local_config(
constants.STRATEGY_CONFIG_FILE_NAME)
with patch.object(Configuration, 'from_local_config',
return_value=configuration), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=True):
Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME,
'sa')
# assert that s3_upload_object method was called once
self.assertEqual(mock_s3_upload_object.call_count, 1)
def from_s3_bucket(cls, ticker_object_name: str, app_ns: str):
'''
Creates a TickerFile object instane based on an S3 bucket.
The bucket is detrmined by looking at the system's ClouFormation
exports.
Parameters
----------
ticker_object_name : str
S3 object name
app_ns : str
Application namespace used to identify the appropriate
CloudFormation exports
'''
s3_object_path = "%s/%s" % (constants.S3_TICKER_FILE_FOLDER_PREFIX,
ticker_object_name)
destination_path = "%s/%s" % (constants.APP_DATA_DIR,
ticker_object_name)
log.debug(
"Reading S3 Data Bucket location from CloudFormation Exports")
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
util.create_dir(constants.APP_DATA_DIR)
log.debug("Downloading s3://%s --> %s" %
(s3_object_path, destination_path))
try:
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_path,
destination_path)
except AWSError as awe:
if "(404)" in str(awe) and "Not Found" in str(awe):
log.debug(
"File not found in S3. Looking for local alternatives")
# Attempt to upload a local copy of the file if it exists
local_ticker_path = '%s/%s' % (constants.TICKER_DATA_DIR,
ticker_object_name)
if os.path.isfile(local_ticker_path):
log.debug("Attempting to upload %s --> s3://%s/%s" %
(local_ticker_path, s3_data_bucket_name,
s3_object_path))
aws_service_wrapper.s3_upload_object(
local_ticker_path, s3_data_bucket_name, s3_object_path)
return cls.from_local_file(constants.TICKER_DATA_DIR,
ticker_object_name)
else:
log.debug("No local alternatives found")
raise awe
else:
raise awe
return cls.from_local_file(constants.APP_DATA_DIR, ticker_object_name)
def try_from_s3(cls, config_filename: str, app_ns: str):
'''
Downloads a configuration file from S3 given the supplied file (object) name
and application namespace used to read cloudformation exports.
If the file does not exist use the local one and upload it to S3
'''
try:
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
s3_object_name = "%s/%s" % (constants.S3_CONFIG_OLDER_PREFIX,
config_filename)
dest_filename = "%s.s3download" % config_filename
dest_path = "%s%s" % (constants.CONFIG_FILE_PATH, dest_filename)
log.info("Downloading Configuration File: s3://%s/%s --> %s" %
(s3_data_bucket_name, s3_object_name, dest_path))
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_name, dest_path)
return cls.from_local_config(dest_filename)
except AWSError as awe:
if awe.resource_not_found():
log.debug(
"Configuration not found in S3. Looking for local alternatives"
)
# Attempt to upload a local copy of the configuration if it
# exists
local_configuration_path = "%s%s" % (
constants.CONFIG_FILE_PATH, config_filename)
if os.path.isfile(local_configuration_path):
log.debug("Attempting to upload %s --> s3://%s/%s" %
(local_configuration_path, s3_data_bucket_name,
s3_object_name))
aws_service_wrapper.s3_upload_object(
local_configuration_path, s3_data_bucket_name,
s3_object_name)
return cls.from_local_config(config_filename)
else:
log.debug("No local alternatives found")
raise awe
else:
raise awe
def test_from_s3_bucket_valid(self):
expected_return = ['TICKER-A', 'TICKER-B']
with patch.object(TickerFile, 'from_local_file',
return_value=TickerFile(expected_return)), \
patch.object(aws_service_wrapper, 's3_download_object',
return_value=None), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}):
ticker_file = TickerFile.from_s3_bucket('ticker-file', 'sa')
actual_return = ticker_file.ticker_list
self.assertListEqual(expected_return, actual_return)
def test_from_s3_bucket_aws_error(self):
with patch.object(aws_service_wrapper, 's3_download_object',
return_value=None), \
patch.object(aws_service_wrapper, 'cf_list_exports',
side_effect=AWSError("test", None)):
with self.assertRaises(AWSError):
TickerFile.from_s3_bucket('ticker-file', 'sa')
with patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError("test", None)), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}):
with self.assertRaises(AWSError):
TickerFile.from_s3_bucket('ticker-file', 'sa')
def from_s3(cls, app_ns: str, s3_object_name: str):
'''
loads the model from S3 using preconfigured object names
'''
util.create_dir(constants.APP_DATA_DIR)
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
s3_object_path = "%s/%s" % (cls.model_s3_folder_prefix, s3_object_name)
dest_path = "%s/%s" % (constants.APP_DATA_DIR, s3_object_name)
log.info(
"Downloading %s: s3://%s/%s --> %s" %
(cls.model_name, s3_data_bucket_name, s3_object_path, dest_path))
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_path, dest_path)
return cls.from_local_file(dest_path)
def try_from_s3(cls, app_ns: str, ticker_file_name: str):
'''
Override for BaseModel.from_s3
loads the model from S3. If one is not found look for a local alternative
and upload it to the same bucket. This is done to eliminate the need to
pre-populate S3 with any data when the application is first installed.
'''
try:
return cls.from_s3(app_ns, ticker_file_name)
except AWSError as awe:
if awe.resource_not_found():
log.debug(
"File not found in S3. Looking for local alternatives")
s3_object_path = "%s/%s" % (cls.model_s3_folder_prefix,
ticker_file_name)
log.debug(
"Reading S3 Data Bucket location from CloudFormation Exports"
)
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
# Attempt to upload a local copy of the file if it exists
local_ticker_path = '%s/%s' % (constants.TICKER_DATA_DIR,
ticker_file_name)
if os.path.isfile(local_ticker_path):
log.debug("Attempting to upload %s --> s3://%s/%s" %
(local_ticker_path, s3_data_bucket_name,
s3_object_path))
aws_service_wrapper.s3_upload_object(
local_ticker_path, s3_data_bucket_name, s3_object_path)
return cls.from_local_file(local_ticker_path)
else:
log.debug("No local alternatives found")
raise awe
else:
raise awe
def test_from_s3_bucket_exception(self):
'''
Tests that if there was an error downloading the ticker file
from S3, it will thrown an exception
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception("Download Exception")
)
):
try:
TickerList.try_from_s3('sa', 'ticker-file')
except AWSError as awe:
self.assertTrue("Download Exception" in str(awe))
def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
no local alternative is found, it throws an exception.
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(aws_service_wrapper, 's3_download_object',
side_effect=AWSError(
"test", Exception("Download Exception")
)
),\
patch.object(os.path, 'isfile',
return_value=False):
try:
TickerFile.from_s3_bucket('ticker-file', 'sa')
except AWSError as awe:
self.assertTrue("Download Exception" in str(awe))
def save_to_s3(self, app_ns: str):
'''
Uploads the model to S3
Parameters
----------
app_ns : str
The application namespace supplied to the command line
used to identify the appropriate CloudFormation exports
'''
self.validate_model()
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
object_name = "%s/%s" % (self.model_s3_folder_prefix,
self.model_s3_object_name)
log.info("Uploading %s to S3: s3://%s/%s" %
(self.model_name, s3_data_bucket_name, object_name))
aws_service_wrapper.s3_upload_ascii_string(
util.format_dict(self.model), s3_data_bucket_name, object_name)
def test_from_s3_bucket_exception_upload_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
ticker_list = TickerList.from_dict({
"list_name":
"DOW30",
"list_type":
"US_EQUITIES",
"comparison_symbol":
"DIA",
"ticker_symbols": ["AAPL", "AXP", "BA"]
})
with patch.object(TickerList, 'from_local_file',
return_value=ticker_list), \
patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(TickerList, 'from_s3',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=True):
ticker_file = TickerList.try_from_s3('sa', 'ticker-file')
# assert that s3_upload_object method was called once
self.assertEqual(mock_s3_upload_object.call_count, 1)
def test_from_s3_bucket_exception_no_local_file(self):
'''
Tests that if the file was not found in s3, and
a local alternative is found, it will self heal by
restoring the s3 file
'''
with patch.object(aws_service_wrapper, 'cf_list_exports',
return_value={
constants.s3_data_bucket_export_name('sa'): "test-bucket"
}),\
patch.object(TickerList, 'from_s3',
side_effect=AWSError(
"test", Exception(
"An error occurred (404) when calling the HeadObject operation: Not Found")
)
),\
patch.object(aws_service_wrapper, 's3_upload_object',
return_value=None) as mock_s3_upload_object,\
patch.object(os.path, 'isfile',
return_value=False):
with self.assertRaises(AWSError):
TickerList.try_from_s3('sa', 'ticker-file')