def _get_amazon_s3_store() -> KeyValueStore:
"""Get lazy initialized amazon s3 store, with credentials from mediawords.yml."""
global _amazon_s3_store
if _amazon_s3_store:
return _amazon_s3_store
from mediawords.key_value_store.amazon_s3 import AmazonS3Store
from mediawords.key_value_store.cached_amazon_s3 import CachedAmazonS3Store
config = mediawords.util.config.get_config()
if 'amazon_s3' not in config:
raise McDBIDownloadsException("Amazon S3 download store is not configured.")
store_params = {
'access_key_id': config['amazon_s3']['downloads']['access_key_id'],
'secret_access_key': config['amazon_s3']['downloads']['secret_access_key'],
'bucket_name': config['amazon_s3']['downloads']['bucket_name'],
'directory_name': config['amazon_s3']['downloads']['directory_name'],
}
if config['mediawords'].get('cache_s3_downloads', False):
store_params['cache_table'] = 'cache.s3_raw_downloads_cache'
_amazon_s3_store = CachedAmazonS3Store(**store_params)
else:
_amazon_s3_store = AmazonS3Store(**store_params)
return _amazon_s3_store
def _get_amazon_s3_store() -> KeyValueStore:
"""Get lazy initialized amazon s3 store, with credentials from mediawords.yml."""
global _amazon_s3_store
if _amazon_s3_store:
return _amazon_s3_store
config = get_config()
if 'amazon_s3' not in config:
raise McDBIDownloadsException(
"Amazon S3 download store is not configured.")
store_params = {
'access_key_id': config['amazon_s3']['downloads']['access_key_id'],
'secret_access_key':
config['amazon_s3']['downloads']['secret_access_key'],
'bucket_name': config['amazon_s3']['downloads']['bucket_name'],
'directory_name': config['amazon_s3']['downloads']['directory_name'],
}
if config['mediawords'].get('cache_s3_downloads', False):
store_params['cache_table'] = S3_RAW_DOWNLOADS_CACHE_TABLE_NAME
_amazon_s3_store = CachedAmazonS3Store(**store_params)
else:
_amazon_s3_store = AmazonS3Store(**store_params)
return _amazon_s3_store
def _initialize_store(self) -> CachedAmazonS3Store:
return CachedAmazonS3Store(
access_key_id=test_credentials['access_key_id'],
secret_access_key=test_credentials['secret_access_key'],
bucket_name=test_credentials['bucket_name'],
directory_name=test_credentials['directory_name'],
cache_table='cache.s3_raw_downloads_cache')
def _initialize_store(self) -> CachedAmazonS3Store:
return CachedAmazonS3Store(
access_key_id=test_credentials.access_key_id(),
secret_access_key=test_credentials.secret_access_key(),
bucket_name=test_credentials.bucket_name(),
directory_name=test_credentials.directory_name() + '/' +
random_string(16),
cache_table='cache.s3_raw_downloads_cache',
)
def _get_amazon_s3_store(
amazon_s3_downloads_config: AmazonS3DownloadsConfig,
download_storage_config: DownloadStorageConfig,
) -> KeyValueStore:
"""Get lazy initialized amazon s3 store, with credentials from configuration."""
if not amazon_s3_downloads_config.access_key_id():
raise McDBIDownloadsException(
"Amazon S3 download store is not configured.")
store_params = {
'access_key_id': amazon_s3_downloads_config.access_key_id(),
'secret_access_key': amazon_s3_downloads_config.secret_access_key(),
'bucket_name': amazon_s3_downloads_config.bucket_name(),
'directory_name': amazon_s3_downloads_config.directory_name(),
}
if download_storage_config.cache_s3():
store_params['cache_table'] = S3_RAW_DOWNLOADS_CACHE_TABLE_NAME
amazon_s3_store = CachedAmazonS3Store(**store_params)
else:
amazon_s3_store = AmazonS3Store(**store_params)
return amazon_s3_store