def test_wait_for_log(mock_s3_bucket):
def create_log():
time.sleep(0.5)
out = io.BytesIO()
with gzip.GzipFile(fileobj=out, mode="w") as fo:
fo.write("foo bar".encode())
mock_s3_bucket.Object("some_log_file").put(Body=out.getvalue()) # pylint: disable=no-member
thread = threading.Thread(target=create_log, args=())
thread.daemon = True
thread.start()
context = create_test_pipeline_execution_context()
emr = EmrJobRunner(region=REGION)
res = emr.wait_for_log(
context.log,
log_bucket=mock_s3_bucket.name,
log_key="some_log_file",
waiter_delay=1,
waiter_max_attempts=2,
)
assert res == "foo bar"
with pytest.raises(EmrError) as exc_info:
emr.wait_for_log(
context.log,
log_bucket=mock_s3_bucket.name,
log_key="does_not_exist",
waiter_delay=1,
waiter_max_attempts=1,
)
assert "EMR log file did not appear on S3 after waiting" in str(
exc_info.value)
def test_wait_for_log():
s3 = boto3.resource('s3', region_name=REGION)
s3.create_bucket(Bucket='log_bucket') # pylint: disable=no-member
def create_log():
time.sleep(0.5)
out = io.BytesIO()
with gzip.GzipFile(fileobj=out, mode='w') as fo:
fo.write('foo bar'.encode())
s3.Object('log_bucket', 'some_log_file').put( # pylint: disable=no-member
Body=out.getvalue()
)
thread = threading.Thread(target=create_log, args=())
thread.daemon = True
thread.start()
context = create_test_pipeline_execution_context()
emr = EmrJobRunner(region=REGION)
res = emr.wait_for_log(
context.log,
log_bucket='log_bucket',
log_key='some_log_file',
waiter_delay=1,
waiter_max_attempts=2,
)
assert res == 'foo bar'
with pytest.raises(EmrError) as exc_info:
emr.wait_for_log(
context.log,
log_bucket='log_bucket',
log_key='does_not_exist',
waiter_delay=1,
waiter_max_attempts=1,
)
assert 'EMR log file did not appear on S3 after waiting' in str(exc_info.value)