def test_cloudwatch_logging(cloudwatch_client):
res = execute_pipeline(
define_hello_cloudwatch_pipeline(),
{
'loggers': {
'cloudwatch': {
'config': {
'log_group_name': TEST_CLOUDWATCH_LOG_GROUP_NAME,
'log_stream_name': TEST_CLOUDWATCH_LOG_STREAM_NAME,
'aws_region': AWS_REGION,
}
}
}
},
)
now = millisecond_timestamp(datetime.datetime.utcnow())
attempt_num = 0
found_orig_message = False
while not found_orig_message and attempt_num < NUM_POLL_ATTEMPTS:
# Hack: the get_log_events call below won't include events logged in the pipeline execution
# above if we query too soon after completion.
time.sleep(1)
# This is implicitly assuming that we're not running these tests with too much concurrency, etc.
events = cloudwatch_client.get_log_events(
startTime=now - TEN_MINUTES_MS,
logGroupName=TEST_CLOUDWATCH_LOG_GROUP_NAME,
logStreamName=TEST_CLOUDWATCH_LOG_STREAM_NAME,
limit=100,
)['events']
for parsed_msg in (json.loads(event['message']) for event in events):
if parsed_msg['dagster_meta']['run_id'] == res.run_id:
if parsed_msg['dagster_meta'][
'orig_message'] == 'Hello, Cloudwatch!':
found_orig_message = True
break
attempt_num += 1
assert found_orig_message
def test_cloudwatch_logging():
res = execute_pipeline(
define_hello_cloudwatch_pipeline(),
{
'loggers': {
'cloudwatch': {
'config': {
'log_group_name': TEST_CLOUDWATCH_LOG_GROUP_NAME,
'log_stream_name': TEST_CLOUDWATCH_LOG_STREAM_NAME,
'aws_region': 'us-west-1',
}
}
}
},
)
client = boto3.client('logs', 'us-west-1')
now = millisecond_timestamp(datetime.datetime.utcnow())
# This is implicitly assuming that we're not running these tests with too much concurrency, etc.
events = client.get_log_events(
endTime=now,
logGroupName=TEST_CLOUDWATCH_LOG_GROUP_NAME,
logStreamName=TEST_CLOUDWATCH_LOG_STREAM_NAME,
limit=100,
)['events']
found_orig_message = False
for parsed_msg in (json.loads(event['message']) for event in events):
if parsed_msg['dagster_meta']['run_id'] == res.run_id:
if parsed_msg['dagster_meta'][
'orig_message'] == 'Hello, Cloudwatch!':
found_orig_message = True
break
assert found_orig_message