def _stream_single_stream(log_group_name, stream_name, streamer, sleep_time=4):
next_token = None
while True:
try:
response = cloudwatch.get_log_events(log_group_name,
stream_name,
next_token=next_token)
if response and response.get('events'):
for event in response.get('events'):
message = event.get('message')
streamer.stream_event('[{}] {}'.format(
stream_name, message))
# Set the next token
next_token = response.get('nextForwardToken')
else:
time.sleep(sleep_time)
except MaxRetriesError:
LOG.debug('Received max retries')
io.echo('Received Max retries. You are being heavily throttled.')
except ServiceError as e:
# Something went wrong getting the stream
# It probably doesnt exist anymore.
LOG.debug('Received service error {}'.format(e))
return
except Exception as e:
# We want to swallow all exceptions or else they will be
# printed as a stack trace to the Console
# Exceptions are typically connections reset and
# Various things
LOG.debug('Exception raised: ' + str(e))
def get_cloudwatch_log_stream_events(log_group_name, stream_name, num_log_events=None):
"""
Gets log events from CloudWatch and appends them to a single string to output with each line prefixed with
the stream name.
:param log_group_name: cloudwatch logGroup
:param stream_name: cloudwatch stream name
:param num_log_events: number of log events to retrieve; default is cloudwatch's max: 10k or 1MB of messages
:return: single string will all log events concatenated together
"""
full_log = []
full_log_blob = ''
try:
response = cloudwatch.get_log_events(log_group_name, stream_name, limit=num_log_events)
for event in response.get('events', []):
message = event.get('message')
full_log.append('[{stream_name}] {message}'.format(stream_name=stream_name, message=message))
full_log_blob = os.linesep.join(full_log)
except ServiceError as e:
LOG.debug('Received service error {}'.format(e))
except Exception as e:
LOG.debug('Exception raised: ' + str(e))
LOG.debug(traceback.format_exc())
return full_log_blob
def get_cloudwatch_log_stream_events(log_group_name, stream_name, num_log_events=None):
"""
Gets log events from CloudWatch and appends them to a single string to output with each line prefixed with
the stream name.
:param log_group_name: cloudwatch logGroup
:param stream_name: cloudwatch stream name
:param num_log_events: number of log events to retrieve; default is cloudwatch's max: 10k or 1MB of messages
:return: single string will all log events concatenated together
"""
full_log = []
full_log_blob = ''
try:
response = cloudwatch.get_log_events(log_group_name, stream_name, limit=num_log_events)
for event in response.get('events', []):
message = event.get('message')
full_log.append('[{stream_name}] {message}'.format(stream_name=stream_name, message=message))
full_log_blob = os.linesep.join(full_log)
except ServiceError as e:
LOG.debug('Received service error {}'.format(e))
except Exception as e:
LOG.debug('Exception raised: ' + str(e))
LOG.debug(traceback.format_exc())
return full_log_blob
def get_cloudwatch_stream_logs(log_group_name, stream_name, num_log_events=None):
"""
Will get logs events from cloudwatch and append them to a single string to output with each line prefixed with
the stream name.
:param log_group_name: cloudwatch log group
:param stream_name: cloudwatch stream name
:param num_log_events: number of log events to retrieve; default is cloudwatch's max: 10k or 1MB of messages
:return: single string will all log events concatenated together
"""
full_log = ''
try:
response = cloudwatch.get_log_events(log_group_name, stream_name, limit=num_log_events)
if response and response.get('events'):
for event in response.get('events'):
message = event.get('message')
full_log += '[{}] {}\n'.format(stream_name, message)
except ServiceError as e:
# Something went wrong getting the stream
# It probably doesnt exist anymore.
LOG.debug('Received service error {}'.format(e))
return
except Exception as e:
# We want to swallow all exceptions or else they will be
# printed as a stack trace to the Console
# Exceptions are typically connections reset and
# Various things
LOG.debug('Exception raised: ' + str(e))
# Loop will cause a retry
return full_log
def get_cloudwatch_messages(log_group_name, stream_name, formatter=None, next_token=None):
messages = []
response = None
try:
response = cloudwatch.get_log_events(log_group_name, stream_name, next_token=next_token)
except MaxRetriesError as e:
LOG.debug('Received max retries')
io.echo(e.message())
time.sleep(1)
if response and response.get('events'):
for event in response.get('events'):
message = event.get('message').encode('utf8', 'replace')
if formatter:
timestamp = event.get('timestamp')
formatted_message = formatter.format(stream_name, message, timestamp)
else:
formatted_message = '[{}] {}'.format(stream_name, message)
messages.append(formatted_message)
# Set the next token
next_token = response.get('nextForwardToken')
return messages, next_token
def test_get_log_events(self, make_api_call_mock):
cloudwatch.get_log_events('environment-health.log',
'archive-health-2018-03-26',
next_token='1234123412341234',
start_time='4567456745674567',
end_time='7890789078907890',
limit=10)
make_api_call_mock.assert_called_once_with(
'logs',
'get_log_events',
endTime='7890789078907890',
limit=10,
logGroupName='environment-health.log',
logStreamName='archive-health-2018-03-26',
nextToken='1234123412341234',
startFromHead=False,
startTime='4567456745674567')
def _get_cloudwatch_messages(
log_group_name,
stream_name,
formatter=None,
next_token=None,
start_time=None
):
messages = []
response = None
latest_event_timestamp = start_time
try:
response = cloudwatch.get_log_events(
log_group_name,
stream_name,
next_token=next_token,
start_time=start_time
)
except MaxRetriesError as e:
LOG.debug('Received max retries')
io.echo(e.message())
time.sleep(1)
if response and response.get('events'):
for event in response.get('events'):
message = event.get('message').encode('utf8', 'replace')
if formatter:
timestamp = event.get('timestamp')
if timestamp:
latest_event_timestamp = timestamp
formatted_message = formatter.format(message, stream_name)
else:
formatted_message = '[{1}] {0}'.format(message, stream_name)
messages.append(formatted_message)
# Set the next token
next_token = response.get('nextForwardToken')
return messages, next_token, latest_event_timestamp
