def _stream_single_stream(log_group_name, stream_name, streamer, sleep_time=4): next_token = None while True: try: response = cloudwatch.get_log_events(log_group_name, stream_name, next_token=next_token) if response and response.get('events'): for event in response.get('events'): message = event.get('message') streamer.stream_event('[{}] {}'.format( stream_name, message)) # Set the next token next_token = response.get('nextForwardToken') else: time.sleep(sleep_time) except MaxRetriesError: LOG.debug('Received max retries') io.echo('Received Max retries. You are being heavily throttled.') except ServiceError as e: # Something went wrong getting the stream # It probably doesnt exist anymore. LOG.debug('Received service error {}'.format(e)) return except Exception as e: # We want to swallow all exceptions or else they will be # printed as a stack trace to the Console # Exceptions are typically connections reset and # Various things LOG.debug('Exception raised: ' + str(e))
def get_cloudwatch_log_stream_events(log_group_name, stream_name, num_log_events=None): """ Gets log events from CloudWatch and appends them to a single string to output with each line prefixed with the stream name. :param log_group_name: cloudwatch logGroup :param stream_name: cloudwatch stream name :param num_log_events: number of log events to retrieve; default is cloudwatch's max: 10k or 1MB of messages :return: single string will all log events concatenated together """ full_log = [] full_log_blob = '' try: response = cloudwatch.get_log_events(log_group_name, stream_name, limit=num_log_events) for event in response.get('events', []): message = event.get('message') full_log.append('[{stream_name}] {message}'.format(stream_name=stream_name, message=message)) full_log_blob = os.linesep.join(full_log) except ServiceError as e: LOG.debug('Received service error {}'.format(e)) except Exception as e: LOG.debug('Exception raised: ' + str(e)) LOG.debug(traceback.format_exc()) return full_log_blob
def get_cloudwatch_stream_logs(log_group_name, stream_name, num_log_events=None): """ Will get logs events from cloudwatch and append them to a single string to output with each line prefixed with the stream name. :param log_group_name: cloudwatch log group :param stream_name: cloudwatch stream name :param num_log_events: number of log events to retrieve; default is cloudwatch's max: 10k or 1MB of messages :return: single string will all log events concatenated together """ full_log = '' try: response = cloudwatch.get_log_events(log_group_name, stream_name, limit=num_log_events) if response and response.get('events'): for event in response.get('events'): message = event.get('message') full_log += '[{}] {}\n'.format(stream_name, message) except ServiceError as e: # Something went wrong getting the stream # It probably doesnt exist anymore. LOG.debug('Received service error {}'.format(e)) return except Exception as e: # We want to swallow all exceptions or else they will be # printed as a stack trace to the Console # Exceptions are typically connections reset and # Various things LOG.debug('Exception raised: ' + str(e)) # Loop will cause a retry return full_log
def get_cloudwatch_messages(log_group_name, stream_name, formatter=None, next_token=None): messages = [] response = None try: response = cloudwatch.get_log_events(log_group_name, stream_name, next_token=next_token) except MaxRetriesError as e: LOG.debug('Received max retries') io.echo(e.message()) time.sleep(1) if response and response.get('events'): for event in response.get('events'): message = event.get('message').encode('utf8', 'replace') if formatter: timestamp = event.get('timestamp') formatted_message = formatter.format(stream_name, message, timestamp) else: formatted_message = '[{}] {}'.format(stream_name, message) messages.append(formatted_message) # Set the next token next_token = response.get('nextForwardToken') return messages, next_token
def test_get_log_events(self, make_api_call_mock): cloudwatch.get_log_events('environment-health.log', 'archive-health-2018-03-26', next_token='1234123412341234', start_time='4567456745674567', end_time='7890789078907890', limit=10) make_api_call_mock.assert_called_once_with( 'logs', 'get_log_events', endTime='7890789078907890', limit=10, logGroupName='environment-health.log', logStreamName='archive-health-2018-03-26', nextToken='1234123412341234', startFromHead=False, startTime='4567456745674567')
def _get_cloudwatch_messages( log_group_name, stream_name, formatter=None, next_token=None, start_time=None ): messages = [] response = None latest_event_timestamp = start_time try: response = cloudwatch.get_log_events( log_group_name, stream_name, next_token=next_token, start_time=start_time ) except MaxRetriesError as e: LOG.debug('Received max retries') io.echo(e.message()) time.sleep(1) if response and response.get('events'): for event in response.get('events'): message = event.get('message').encode('utf8', 'replace') if formatter: timestamp = event.get('timestamp') if timestamp: latest_event_timestamp = timestamp formatted_message = formatter.format(message, stream_name) else: formatted_message = '[{1}] {0}'.format(message, stream_name) messages.append(formatted_message) # Set the next token next_token = response.get('nextForwardToken') return messages, next_token, latest_event_timestamp