def test_from_s3_bucket_exception_no_local_file(self): ''' Tests that if the file was not found in s3, and a local no alternative is found, an exception will be thrown ''' configuration = Configuration.from_local_config( constants.STRATEGY_CONFIG_FILE_NAME) with patch.object(Configuration, 'from_local_config', return_value=configuration), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError( "test", Exception( "An error occurred (404) when calling the HeadObject operation: Not Found") ) ),\ patch.object(os.path, 'isfile', return_value=False): with self.assertRaises(AWSError): Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME, 'sa')
def test_from_s3_bucket_exception_upload_local_file(self): ''' Tests that if the file was not found in s3, and a local alternative is found, it will self heal by restoring the s3 file ''' expected_return = ['TICKER-A', 'TICKER-B'] with patch.object(TickerFile, 'from_local_file', return_value=TickerFile(expected_return)), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError( "test", Exception( "An error occurred (404) when calling the HeadObject operation: Not Found") ) ),\ patch.object(aws_service_wrapper, 's3_upload_object', return_value=None),\ patch.object(os.path, 'isfile', return_value=True): ticker_file = TickerFile.from_s3_bucket('ticker-file', 'sa') actual_return = ticker_file.ticker_list self.assertListEqual(expected_return, actual_return)
def test_from_s3_bucket_exception_upload_local_file(self): ''' Tests that if the file was not found in s3, and a local alternative is found, it will self heal by restoring the s3 file ''' configuration = Configuration.from_local_config( constants.STRATEGY_CONFIG_FILE_NAME) with patch.object(Configuration, 'from_local_config', return_value=configuration), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError( "test", Exception( "An error occurred (404) when calling the HeadObject operation: Not Found") ) ),\ patch.object(aws_service_wrapper, 's3_upload_object', return_value=None) as mock_s3_upload_object,\ patch.object(os.path, 'isfile', return_value=True): Configuration.try_from_s3(constants.STRATEGY_CONFIG_FILE_NAME, 'sa') # assert that s3_upload_object method was called once self.assertEqual(mock_s3_upload_object.call_count, 1)
def from_s3_bucket(cls, ticker_object_name: str, app_ns: str): ''' Creates a TickerFile object instane based on an S3 bucket. The bucket is detrmined by looking at the system's ClouFormation exports. Parameters ---------- ticker_object_name : str S3 object name app_ns : str Application namespace used to identify the appropriate CloudFormation exports ''' s3_object_path = "%s/%s" % (constants.S3_TICKER_FILE_FOLDER_PREFIX, ticker_object_name) destination_path = "%s/%s" % (constants.APP_DATA_DIR, ticker_object_name) log.debug( "Reading S3 Data Bucket location from CloudFormation Exports") s3_data_bucket_name = aws_service_wrapper.cf_read_export_value( constants.s3_data_bucket_export_name(app_ns)) util.create_dir(constants.APP_DATA_DIR) log.debug("Downloading s3://%s --> %s" % (s3_object_path, destination_path)) try: aws_service_wrapper.s3_download_object(s3_data_bucket_name, s3_object_path, destination_path) except AWSError as awe: if "(404)" in str(awe) and "Not Found" in str(awe): log.debug( "File not found in S3. Looking for local alternatives") # Attempt to upload a local copy of the file if it exists local_ticker_path = '%s/%s' % (constants.TICKER_DATA_DIR, ticker_object_name) if os.path.isfile(local_ticker_path): log.debug("Attempting to upload %s --> s3://%s/%s" % (local_ticker_path, s3_data_bucket_name, s3_object_path)) aws_service_wrapper.s3_upload_object( local_ticker_path, s3_data_bucket_name, s3_object_path) return cls.from_local_file(constants.TICKER_DATA_DIR, ticker_object_name) else: log.debug("No local alternatives found") raise awe else: raise awe return cls.from_local_file(constants.APP_DATA_DIR, ticker_object_name)
def try_from_s3(cls, config_filename: str, app_ns: str): ''' Downloads a configuration file from S3 given the supplied file (object) name and application namespace used to read cloudformation exports. If the file does not exist use the local one and upload it to S3 ''' try: s3_data_bucket_name = aws_service_wrapper.cf_read_export_value( constants.s3_data_bucket_export_name(app_ns)) s3_object_name = "%s/%s" % (constants.S3_CONFIG_OLDER_PREFIX, config_filename) dest_filename = "%s.s3download" % config_filename dest_path = "%s%s" % (constants.CONFIG_FILE_PATH, dest_filename) log.info("Downloading Configuration File: s3://%s/%s --> %s" % (s3_data_bucket_name, s3_object_name, dest_path)) aws_service_wrapper.s3_download_object(s3_data_bucket_name, s3_object_name, dest_path) return cls.from_local_config(dest_filename) except AWSError as awe: if awe.resource_not_found(): log.debug( "Configuration not found in S3. Looking for local alternatives" ) # Attempt to upload a local copy of the configuration if it # exists local_configuration_path = "%s%s" % ( constants.CONFIG_FILE_PATH, config_filename) if os.path.isfile(local_configuration_path): log.debug("Attempting to upload %s --> s3://%s/%s" % (local_configuration_path, s3_data_bucket_name, s3_object_name)) aws_service_wrapper.s3_upload_object( local_configuration_path, s3_data_bucket_name, s3_object_name) return cls.from_local_config(config_filename) else: log.debug("No local alternatives found") raise awe else: raise awe
def test_from_s3_bucket_valid(self): expected_return = ['TICKER-A', 'TICKER-B'] with patch.object(TickerFile, 'from_local_file', return_value=TickerFile(expected_return)), \ patch.object(aws_service_wrapper, 's3_download_object', return_value=None), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }): ticker_file = TickerFile.from_s3_bucket('ticker-file', 'sa') actual_return = ticker_file.ticker_list self.assertListEqual(expected_return, actual_return)
def test_from_s3_bucket_aws_error(self): with patch.object(aws_service_wrapper, 's3_download_object', return_value=None), \ patch.object(aws_service_wrapper, 'cf_list_exports', side_effect=AWSError("test", None)): with self.assertRaises(AWSError): TickerFile.from_s3_bucket('ticker-file', 'sa') with patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError("test", None)), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }): with self.assertRaises(AWSError): TickerFile.from_s3_bucket('ticker-file', 'sa')
def from_s3(cls, app_ns: str, s3_object_name: str): ''' loads the model from S3 using preconfigured object names ''' util.create_dir(constants.APP_DATA_DIR) s3_data_bucket_name = aws_service_wrapper.cf_read_export_value( constants.s3_data_bucket_export_name(app_ns)) s3_object_path = "%s/%s" % (cls.model_s3_folder_prefix, s3_object_name) dest_path = "%s/%s" % (constants.APP_DATA_DIR, s3_object_name) log.info( "Downloading %s: s3://%s/%s --> %s" % (cls.model_name, s3_data_bucket_name, s3_object_path, dest_path)) aws_service_wrapper.s3_download_object(s3_data_bucket_name, s3_object_path, dest_path) return cls.from_local_file(dest_path)
def try_from_s3(cls, app_ns: str, ticker_file_name: str): ''' Override for BaseModel.from_s3 loads the model from S3. If one is not found look for a local alternative and upload it to the same bucket. This is done to eliminate the need to pre-populate S3 with any data when the application is first installed. ''' try: return cls.from_s3(app_ns, ticker_file_name) except AWSError as awe: if awe.resource_not_found(): log.debug( "File not found in S3. Looking for local alternatives") s3_object_path = "%s/%s" % (cls.model_s3_folder_prefix, ticker_file_name) log.debug( "Reading S3 Data Bucket location from CloudFormation Exports" ) s3_data_bucket_name = aws_service_wrapper.cf_read_export_value( constants.s3_data_bucket_export_name(app_ns)) # Attempt to upload a local copy of the file if it exists local_ticker_path = '%s/%s' % (constants.TICKER_DATA_DIR, ticker_file_name) if os.path.isfile(local_ticker_path): log.debug("Attempting to upload %s --> s3://%s/%s" % (local_ticker_path, s3_data_bucket_name, s3_object_path)) aws_service_wrapper.s3_upload_object( local_ticker_path, s3_data_bucket_name, s3_object_path) return cls.from_local_file(local_ticker_path) else: log.debug("No local alternatives found") raise awe else: raise awe
def test_from_s3_bucket_exception(self): ''' Tests that if there was an error downloading the ticker file from S3, it will thrown an exception ''' with patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError( "test", Exception("Download Exception") ) ): try: TickerList.try_from_s3('sa', 'ticker-file') except AWSError as awe: self.assertTrue("Download Exception" in str(awe))
def test_from_s3_bucket_exception_no_local_file(self): ''' Tests that if the file was not found in s3, and no local alternative is found, it throws an exception. ''' with patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(aws_service_wrapper, 's3_download_object', side_effect=AWSError( "test", Exception("Download Exception") ) ),\ patch.object(os.path, 'isfile', return_value=False): try: TickerFile.from_s3_bucket('ticker-file', 'sa') except AWSError as awe: self.assertTrue("Download Exception" in str(awe))
def save_to_s3(self, app_ns: str): ''' Uploads the model to S3 Parameters ---------- app_ns : str The application namespace supplied to the command line used to identify the appropriate CloudFormation exports ''' self.validate_model() s3_data_bucket_name = aws_service_wrapper.cf_read_export_value( constants.s3_data_bucket_export_name(app_ns)) object_name = "%s/%s" % (self.model_s3_folder_prefix, self.model_s3_object_name) log.info("Uploading %s to S3: s3://%s/%s" % (self.model_name, s3_data_bucket_name, object_name)) aws_service_wrapper.s3_upload_ascii_string( util.format_dict(self.model), s3_data_bucket_name, object_name)
def test_from_s3_bucket_exception_upload_local_file(self): ''' Tests that if the file was not found in s3, and a local alternative is found, it will self heal by restoring the s3 file ''' ticker_list = TickerList.from_dict({ "list_name": "DOW30", "list_type": "US_EQUITIES", "comparison_symbol": "DIA", "ticker_symbols": ["AAPL", "AXP", "BA"] }) with patch.object(TickerList, 'from_local_file', return_value=ticker_list), \ patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(TickerList, 'from_s3', side_effect=AWSError( "test", Exception( "An error occurred (404) when calling the HeadObject operation: Not Found") ) ),\ patch.object(aws_service_wrapper, 's3_upload_object', return_value=None) as mock_s3_upload_object,\ patch.object(os.path, 'isfile', return_value=True): ticker_file = TickerList.try_from_s3('sa', 'ticker-file') # assert that s3_upload_object method was called once self.assertEqual(mock_s3_upload_object.call_count, 1)
def test_from_s3_bucket_exception_no_local_file(self): ''' Tests that if the file was not found in s3, and a local alternative is found, it will self heal by restoring the s3 file ''' with patch.object(aws_service_wrapper, 'cf_list_exports', return_value={ constants.s3_data_bucket_export_name('sa'): "test-bucket" }),\ patch.object(TickerList, 'from_s3', side_effect=AWSError( "test", Exception( "An error occurred (404) when calling the HeadObject operation: Not Found") ) ),\ patch.object(aws_service_wrapper, 's3_upload_object', return_value=None) as mock_s3_upload_object,\ patch.object(os.path, 'isfile', return_value=False): with self.assertRaises(AWSError): TickerList.try_from_s3('sa', 'ticker-file')