def get_logs(self, start, end): log_source = self.policy.ctx.output log_gen = () if self.policy.options.log_group is not None: session = utils.local_session(self.policy.session_factory) log_gen = log_entries_from_group( session, self.policy.options.log_group, start, end, ) elif log_source.use_s3(): raw_entries = log_entries_from_s3( self.policy.session_factory, log_source, start, end, ) # log files can be downloaded out of order, so sort on timestamp # log_gen isn't really a generator once we do this, but oh well log_gen = sorted( normalized_log_entries(raw_entries), key=lambda e: e.get('timestamp', 0), ) else: log_path = os.path.join(log_source.root_dir, 'custodian-run.log') with open(log_path) as log_fh: raw_entries = log_fh.readlines() log_gen = normalized_log_entries(raw_entries) return log_entries_in_range( log_gen, start, end, )
def get_logs(self, start, end): from c7n import logs_support log_source = self.policy.ctx.output log_gen = () if self.policy.options.log_group is not None: session = utils.local_session(self.policy.session_factory) log_gen = logs_support.log_entries_from_group( session, self.policy.options.log_group, start, end, ) elif log_source.type == 's3': raw_entries = logs_support.log_entries_from_s3( self.policy.session_factory, log_source, start, end, ) # log files can be downloaded out of order, so sort on timestamp # log_gen isn't really a generator once we do this, but oh well log_gen = sorted( logs_support.normalized_log_entries(raw_entries), key=lambda e: e.get('timestamp', 0), ) else: log_path = os.path.join(log_source.root_dir, 'custodian-run.log') with open(log_path) as log_fh: raw_entries = log_fh.readlines() log_gen = logs_support.normalized_log_entries(raw_entries) return logs_support.log_entries_in_range( log_gen, start, end, )
def test_entries_in_range(self): raw_entries = log_lines() log_gen = normalized_log_entries(raw_entries) nrm_entries = list(log_gen) range_gen = log_entries_in_range(nrm_entries, "2016-11-21 12:40:00", "2016-11-21 12:45:00") in_range = list(range_gen) # fewer entries than we started with self.assertLess(len(in_range), len(nrm_entries)) # entries are within 5 minutes of each other span = (in_range[-1]["timestamp"] - in_range[0]["timestamp"]) / 1000 self.assertLess(span, 300)
def test_normalization(self): raw_entries = log_lines() log_gen = normalized_log_entries(raw_entries) nrm_entries = list(log_gen) # multi-line entries are being combined self.assertEqual(len(raw_entries), 144) self.assertEqual(len(nrm_entries), 55) # entries look reasonable entry = nrm_entries[1] self.assertIn('timestamp', entry) self.assertIn('message', entry) self.assertIsInstance(entry['timestamp'], six.integer_types) self.assertIsInstance(entry['message'], six.text_type)
def test_entries_in_range(self): raw_entries = log_lines() log_gen = normalized_log_entries(raw_entries) nrm_entries = list(log_gen) range_gen = log_entries_in_range( nrm_entries, "2016-11-21 12:40:00", "2016-11-21 12:45:00" ) in_range = list(range_gen) # fewer entries than we started with self.assertLess(len(in_range), len(nrm_entries)) # entries are within 5 minutes of each other span = (in_range[-1]["timestamp"] - in_range[0]["timestamp"]) / 1000 self.assertLess(span, 300)
def test_normalization(self): raw_entries = log_lines() log_gen = normalized_log_entries(raw_entries) nrm_entries = list(log_gen) # multi-line entries are being combined self.assertEqual(len(raw_entries), 144) self.assertEqual(len(nrm_entries), 55) # entries look reasonable entry = nrm_entries[1] self.assertIn("timestamp", entry) self.assertIn("message", entry) self.assertIsInstance(entry["timestamp"], six.integer_types) self.assertIsInstance(entry["message"], six.text_type)