def put_recipe(bucket, keyname):
s3 = Session().resource('s3')
bucket = s3.Bucket(bucket)
obj = bucket.Object(keyname)
body = keyname
response = obj.put(Body=body.encode('utf-8'),
ContentEncoding='utf-8',
ContentType='text/plane')
def put_img_list(bucket, img_list):
s3 = Session().resource('s3')
bucket = s3.Bucket(bucket)
keyname = cfg['KEY']
# keyname = "{0:%Y-%m-%d}".format(datetime.today())
obj = bucket.Object(keyname)
img_list = ','.join(img_list)
# print(img_list)
body = img_list
response = obj.put(
Body=body.encode('utf-8'),
ContentEncoding='utf-8',
ContentType='text/plane'
)
class S3Driver(Driver):
def _setup(self, driver_conf):
self._client = Session(
aws_access_key_id=driver_conf.access_key,
aws_secret_access_key=driver_conf.secret_key,
).resource('s3', endpoint_url=driver_conf.endpoint)
try:
self._client.Bucket(self._bucket).create()
except Exception:
pass
def _put(self, bucket, key, data):
s3_client = self._client.Object(bucket, key)
s3_client.upload_fileobj(data)
return True
def _get(self, bucket, key, output):
s3_client = self._client.Object(bucket, key)
s3_client.download_fileobj(output)
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--version', '-v', required=True,
help='Version to deploy')
args = parser.parse_args()
s3 = Session(
aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'),
region_name=os.environ.get('AWS_REGION'),
).resource('s3')
bucket = s3.Bucket(os.environ.get('S3_BUCKET_NAME'))
files = [
f'amplitude-{args.version}.js',
f'amplitude-{args.version}-min.js',
f'amplitude-{args.version}.umd.js',
f'amplitude-{args.version}-min.umd.js'
]
for file in files:
if check_exists(s3.Object(os.environ.get('S3_BUCKET_NAME'), os.path.join('libs', file))):
sys.exit(f'ERROR: {file} already exists and shouldn\'t be republished. Consider releasing a new version')
print(f'Uploading {file}')
upload(bucket, file, unzipped_args)
gz_files = [
f'amplitude-{args.version}-min.gz.js',
f'amplitude-{args.version}-min.umd.gz.js'
]
for file in gz_files:
if check_exists(s3.Object(os.environ.get('S3_BUCKET_NAME'), file)):
sys.exit(f'{file} already exists!')
print(f'Uploading {file}')
upload(bucket, file, zipped_args)
print(f'Success: S3 upload completed. Example: https://cdn.amplitude.com/libs/amplitude-{args.version}.js')
return 0
def get_recipe_list(bucket):
s3 = Session().resource('s3')
bucket = s3.Bucket(bucket)
return [obj.key for obj in bucket.objects.all()]
class S3ArchiveDiffer(ArchiveDiffer):
"""
AWS S3 backend for archiving.
Archives CSV files into a S3 bucket, with keys "{indicator_prefix}/{csv_file_name}".
Ideally, versioning should be enabled in this bucket to track versions of each CSV file.
"""
def __init__(
self,
cache_dir: str,
export_dir: str,
bucket_name: str,
indicator_prefix: str,
aws_credentials: Dict[str, str],
):
"""
Initialize a S3ArchiveDiffer.
See this link for possible aws_credentials kwargs:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session
Parameters
----------
cache_dir: str
The directory for storing most recent archived/uploaded CSVs to do start diffing from.
Usually 'cache'.
export_dir: str
The directory with most recent exported CSVs to diff to.
Usually 'receiving'.
bucket_name: str
The S3 bucket to upload files to.
indicator_prefix: str
The prefix for S3 keys related to this indicator.
aws_credentials: Dict[str, str]
kwargs to create a boto3.Session, containing AWS credentials/profile to use.
"""
super().__init__(cache_dir, export_dir)
self.s3 = Session(**aws_credentials).resource("s3")
self.bucket = self.s3.Bucket(bucket_name)
self.indicator_prefix = indicator_prefix
def update_cache(self):
"""Make sure cache_dir is updated with all latest files from the S3 bucket."""
# List all indicator-related objects from S3
archive_objects = self.bucket.objects.filter(
Prefix=self.indicator_prefix).all()
archive_objects = [
obj for obj in archive_objects if obj.key.endswith(".csv")
]
# Check against what we have locally and download missing ones
cached_files = set(
basename(f) for f in glob(join(self.cache_dir, "*.csv")))
for obj in archive_objects:
archive_file = basename(obj.key)
cached_file = join(self.cache_dir, archive_file)
if archive_file not in cached_files:
print(f"Updating cache with {cached_file}")
obj.Object().download_file(cached_file)
self._cache_updated = True
def archive_exports(
self, # pylint: disable=arguments-differ
exported_files: Files,
update_cache: bool = True,
update_s3: bool = True) -> Tuple[Files, Files]:
"""
Handle actual archiving of files to the S3 bucket.
Parameters
----------
exported_files: Files
List of files to be archived. Usually new and changed files.
Returns
-------
(successes, fails): Tuple[Files, Files]
successes: List of successfully archived files
fails: List of unsuccessfully archived files
"""
archive_success = []
archive_fail = []
for exported_file in exported_files:
cached_file = abspath(join(self.cache_dir,
basename(exported_file)))
archive_key = join(self.indicator_prefix, basename(exported_file))
try:
if update_cache:
# Update local cache
shutil.copyfile(exported_file, cached_file)
if update_s3:
self.bucket.Object(archive_key).upload_file(exported_file)
archive_success.append(exported_file)
except FileNotFoundError:
archive_fail.append(exported_file)
self._exports_archived = True
return archive_success, archive_fail
