def test_download_log_files_and_skip_existing_files():
with tempfile.TemporaryDirectory() as dirpath:
given_a_bucket("some-bucket")
given_an_object("some-bucket", TEST_LOG_KEY, "some-file-content")
given_an_object("some-bucket", TEST_LOG_KEY_EXISTING, "some-file-content")
given_a_file(dirpath, TEST_LOG_KEY_EXISTING, "some-content-already-existing")
download_cloudtrail_logs(
target_dir=dirpath,
bucket="some-bucket",
cloudtrail_prefix="some-prefix/",
from_date=datetime.datetime(2017, 1, 1, tzinfo=pytz.utc),
to_date=datetime.datetime(2017, 1, 1, tzinfo=pytz.utc),
account_ids=["000"],
regions=["some-region-1"])
runner = CliRunner()
result = runner.invoke(cli.root_group, args=[
"download",
"--bucket", "some-bucket",
"--region", "some-region-1",
"--account-id", "000",
"--prefix", "some-prefix/",
"--from", "2017-01-01",
"--to", "2017-01-01"
])
assert result.exit_code == 0
assert file_content(dirpath, TEST_LOG_KEY) == "some-file-content"
assert file_content(dirpath, TEST_LOG_KEY_EXISTING) == "some-content-already-existing"
def download(bucket, prefix, org_id, account_id, region, log_dir, from_s, to_s,
wait, parallelism):
"""Downloads CloudTrail Logs from S3."""
log_dir = os.path.expanduser(log_dir)
from_date = time_utils.parse_human_readable_time(from_s)
to_date = time_utils.parse_human_readable_time(to_s)
download_cloudtrail_logs(log_dir, bucket, prefix, org_id, account_id,
region, from_date, to_date, parallelism)
if wait:
last_timestamp = last_event_timestamp_in_dir(log_dir)
while last_timestamp <= to_date:
click.echo("CloudTrail logs haven't caught up to " + str(to_date) +
" yet. " + "Most recent timestamp: " +
str(last_timestamp.astimezone(to_date.tzinfo)) + ". " +
"Trying again in 60sec.")
time.sleep(60 * 1)
download_cloudtrail_logs(log_dir, bucket, prefix, org_id,
account_id, region, from_date, to_date,
parallelism)
last_timestamp = last_event_timestamp_in_dir(log_dir)
def test_download_log_files_and_skip_existing_files():
with tempfile.TemporaryDirectory() as dirpath:
given_a_bucket("some-bucket")
given_an_object("some-bucket", TEST_LOG_KEY, "some-file-content")
given_an_object("some-bucket", TEST_LOG_KEY_EXISTING,
"some-file-content")
given_a_file(dirpath, TEST_LOG_KEY_EXISTING,
"some-content-already-existing")
download_cloudtrail_logs(
target_dir=dirpath,
bucket="some-bucket",
cloudtrail_prefix="some-prefix/",
from_date=datetime.datetime(2017, 1, 1, tzinfo=pytz.utc),
to_date=datetime.datetime(2018, 1, 1, tzinfo=pytz.utc),
account_ids=["000"],
org_ids=[],
regions=["some-region-1"],
parallelism=10,
)
assert file_content(dirpath, TEST_LOG_KEY) == "some-file-content"
assert file_content(
dirpath, TEST_LOG_KEY_EXISTING) == "some-content-already-existing"