def test_add_timedelta():
mytime = MyTime(
year=2018, month=4, day=5, hour=8, minute=0, tzinfo=timezone("US/Pacific")
)
mytime_plus_65_mins = mytime.add_timedelta(minutes=65)
assert (mytime.epoch + 60 * 65) == mytime_plus_65_mins.epoch
assert str(mytime_plus_65_mins.tzinfo) == "US/Pacific"
def fetch_file_names_on_s3(bucket: str, key_prefix: str, mytime: MyTime,
time_offset: int):
files = _list_folder_contents(bucket, key_prefix)
print('# files pre filter:', len(files))
start_time = mytime.add_timedelta(minutes=-time_offset)
end_time = mytime
files_filtered = _filter_files_on_time(files, start_time, end_time)
return files_filtered, start_time
def put_payload_on_s3(*, payload: dict, bucket: str, mytime: MyTime):
mytime = mytime.add_timedelta(minutes=-1)
ddb_items = []
for app, data in payload.items():
s3_key = put_on_s3(data, mytime, app, bucket)
ddb_item = {
'timestamp': mytime.epoch,
's3_key': s3_key,
'app': app,
'num_uris': len(data)
}
ddb_items.append(ddb_item)
return ddb_items
def put_payload_on_s3(*, payload: dict, bucket: str, mytime: MyTime):
mytime = mytime.add_timedelta(minutes=-1)
ddb_items = []
for app, data in payload.items():
s3_key = put_on_s3(data, mytime, app, bucket)
ddb_item = {
"timestamp": mytime.epoch,
"s3_key": s3_key,
"app": app,
"num_uris": len(data),
}
ddb_items.append(ddb_item)
return ddb_items