def test_s3_download_object_with_boto_exception(self):
with patch.object(aws_service_wrapper.S3_CLIENT, 'download_file',
side_effect=botocore.exceptions.BotoCoreError()):
with self.assertRaises(AWSError):
aws_service_wrapper.s3_download_object(
"bucket_name", "object_name", "./dest_path")
def from_s3_bucket(cls, ticker_object_name: str, app_ns: str):
'''
Creates a TickerFile object instane based on an S3 bucket.
The bucket is detrmined by looking at the system's ClouFormation
exports.
Parameters
----------
ticker_object_name : str
S3 object name
app_ns : str
Application namespace used to identify the appropriate
CloudFormation exports
'''
s3_object_path = "%s/%s" % (constants.S3_TICKER_FILE_FOLDER_PREFIX,
ticker_object_name)
destination_path = "%s/%s" % (constants.APP_DATA_DIR,
ticker_object_name)
log.debug(
"Reading S3 Data Bucket location from CloudFormation Exports")
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
util.create_dir(constants.APP_DATA_DIR)
log.debug("Downloading s3://%s --> %s" %
(s3_object_path, destination_path))
try:
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_path,
destination_path)
except AWSError as awe:
if "(404)" in str(awe) and "Not Found" in str(awe):
log.debug(
"File not found in S3. Looking for local alternatives")
# Attempt to upload a local copy of the file if it exists
local_ticker_path = '%s/%s' % (constants.TICKER_DATA_DIR,
ticker_object_name)
if os.path.isfile(local_ticker_path):
log.debug("Attempting to upload %s --> s3://%s/%s" %
(local_ticker_path, s3_data_bucket_name,
s3_object_path))
aws_service_wrapper.s3_upload_object(
local_ticker_path, s3_data_bucket_name, s3_object_path)
return cls.from_local_file(constants.TICKER_DATA_DIR,
ticker_object_name)
else:
log.debug("No local alternatives found")
raise awe
else:
raise awe
return cls.from_local_file(constants.APP_DATA_DIR, ticker_object_name)
def try_from_s3(cls, config_filename: str, app_ns: str):
'''
Downloads a configuration file from S3 given the supplied file (object) name
and application namespace used to read cloudformation exports.
If the file does not exist use the local one and upload it to S3
'''
try:
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
s3_object_name = "%s/%s" % (constants.S3_CONFIG_OLDER_PREFIX,
config_filename)
dest_filename = "%s.s3download" % config_filename
dest_path = "%s%s" % (constants.CONFIG_FILE_PATH, dest_filename)
log.info("Downloading Configuration File: s3://%s/%s --> %s" %
(s3_data_bucket_name, s3_object_name, dest_path))
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_name, dest_path)
return cls.from_local_config(dest_filename)
except AWSError as awe:
if awe.resource_not_found():
log.debug(
"Configuration not found in S3. Looking for local alternatives"
)
# Attempt to upload a local copy of the configuration if it
# exists
local_configuration_path = "%s%s" % (
constants.CONFIG_FILE_PATH, config_filename)
if os.path.isfile(local_configuration_path):
log.debug("Attempting to upload %s --> s3://%s/%s" %
(local_configuration_path, s3_data_bucket_name,
s3_object_name))
aws_service_wrapper.s3_upload_object(
local_configuration_path, s3_data_bucket_name,
s3_object_name)
return cls.from_local_config(config_filename)
else:
log.debug("No local alternatives found")
raise awe
else:
raise awe
def from_s3(cls, app_ns: str, s3_object_name: str):
'''
loads the model from S3 using preconfigured object names
'''
util.create_dir(constants.APP_DATA_DIR)
s3_data_bucket_name = aws_service_wrapper.cf_read_export_value(
constants.s3_data_bucket_export_name(app_ns))
s3_object_path = "%s/%s" % (cls.model_s3_folder_prefix, s3_object_name)
dest_path = "%s/%s" % (constants.APP_DATA_DIR, s3_object_name)
log.info(
"Downloading %s: s3://%s/%s --> %s" %
(cls.model_name, s3_data_bucket_name, s3_object_path, dest_path))
aws_service_wrapper.s3_download_object(s3_data_bucket_name,
s3_object_path, dest_path)
return cls.from_local_file(dest_path)