def get_logs(self, start, end):
log_source = self.policy.ctx.output
log_gen = ()
if self.policy.options.log_group is not None:
session = utils.local_session(self.policy.session_factory)
log_gen = log_entries_from_group(
session,
self.policy.options.log_group,
start,
end,
)
elif log_source.use_s3():
raw_entries = log_entries_from_s3(
self.policy.session_factory,
log_source,
start,
end,
)
# log files can be downloaded out of order, so sort on timestamp
# log_gen isn't really a generator once we do this, but oh well
log_gen = sorted(
normalized_log_entries(raw_entries),
key=lambda e: e.get('timestamp', 0),
)
else:
log_path = os.path.join(log_source.root_dir, 'custodian-run.log')
with open(log_path) as log_fh:
raw_entries = log_fh.readlines()
log_gen = normalized_log_entries(raw_entries)
return log_entries_in_range(
log_gen,
start,
end,
)
def get_logs(self, start, end):
from c7n import logs_support
log_source = self.policy.ctx.output
log_gen = ()
if self.policy.options.log_group is not None:
session = utils.local_session(self.policy.session_factory)
log_gen = logs_support.log_entries_from_group(
session,
self.policy.options.log_group,
start,
end,
)
elif log_source.type == 's3':
raw_entries = logs_support.log_entries_from_s3(
self.policy.session_factory,
log_source,
start,
end,
)
# log files can be downloaded out of order, so sort on timestamp
# log_gen isn't really a generator once we do this, but oh well
log_gen = sorted(
logs_support.normalized_log_entries(raw_entries),
key=lambda e: e.get('timestamp', 0),
)
else:
log_path = os.path.join(log_source.root_dir, 'custodian-run.log')
with open(log_path) as log_fh:
raw_entries = log_fh.readlines()
log_gen = logs_support.normalized_log_entries(raw_entries)
return logs_support.log_entries_in_range(
log_gen,
start,
end,
)
def test_entries_in_range(self):
raw_entries = log_lines()
log_gen = normalized_log_entries(raw_entries)
nrm_entries = list(log_gen)
range_gen = log_entries_in_range(nrm_entries, "2016-11-21 12:40:00",
"2016-11-21 12:45:00")
in_range = list(range_gen)
# fewer entries than we started with
self.assertLess(len(in_range), len(nrm_entries))
# entries are within 5 minutes of each other
span = (in_range[-1]["timestamp"] - in_range[0]["timestamp"]) / 1000
self.assertLess(span, 300)
def test_normalization(self):
raw_entries = log_lines()
log_gen = normalized_log_entries(raw_entries)
nrm_entries = list(log_gen)
# multi-line entries are being combined
self.assertEqual(len(raw_entries), 144)
self.assertEqual(len(nrm_entries), 55)
# entries look reasonable
entry = nrm_entries[1]
self.assertIn('timestamp', entry)
self.assertIn('message', entry)
self.assertIsInstance(entry['timestamp'], six.integer_types)
self.assertIsInstance(entry['message'], six.text_type)
def test_entries_in_range(self):
raw_entries = log_lines()
log_gen = normalized_log_entries(raw_entries)
nrm_entries = list(log_gen)
range_gen = log_entries_in_range(
nrm_entries, "2016-11-21 12:40:00", "2016-11-21 12:45:00"
)
in_range = list(range_gen)
# fewer entries than we started with
self.assertLess(len(in_range), len(nrm_entries))
# entries are within 5 minutes of each other
span = (in_range[-1]["timestamp"] - in_range[0]["timestamp"]) / 1000
self.assertLess(span, 300)
def test_normalization(self):
raw_entries = log_lines()
log_gen = normalized_log_entries(raw_entries)
nrm_entries = list(log_gen)
# multi-line entries are being combined
self.assertEqual(len(raw_entries), 144)
self.assertEqual(len(nrm_entries), 55)
# entries look reasonable
entry = nrm_entries[1]
self.assertIn("timestamp", entry)
self.assertIn("message", entry)
self.assertIsInstance(entry["timestamp"], six.integer_types)
self.assertIsInstance(entry["message"], six.text_type)
