def parse_date(timestamp):
try:
return common.parse_date(timestamp, DATE_FORMAT)
except ValueError:
# Only Python >= 2.6 supports %f in the date string
timestamp, _ = timestamp.split(',')
common.parse_date(timestamp, LEGACY_DATE_FORMAT)
def parse_date(timestamp):
try:
return common.parse_date(timestamp, DATE_FORMAT)
except ValueError:
# Only Python >= 2.6 supports %f in the date string
timestamp, _ = timestamp.split(',')
return common.parse_date(timestamp, LEGACY_DATE_FORMAT)
def parse_cassandra(log, line):
matched = LOG_PATTERN.match(line)
if matched:
event = matched.groupdict()
# Convert the timestamp string into an epoch timestamp
time_val = event.get('time', None)
if time_val:
event['timestamp'] = common.parse_date("%s %s" % (datetime.utcnow().strftime("%Y-%m-%d"), time_val), DATE_FORMAT)
else:
event['timestamp'] = common.parse_date(event['timestamp'], DATE_FORMAT)
del event['time']
# Process the log priority
event['alert_type'] = ALERT_TYPES.get(event['priority'], "info")
if event['alert_type'] in ('error', 'warning'):
event['auto_priority'] = 1
else:
event['auto_priority'] = 0
del event['priority']
# Process the aggregation metadata
event['event_type'] = EVENT_TYPE
# Process the message
msg = event['msg']
if len(msg) > common.MAX_TITLE_LEN:
event['msg_title'] = msg[0:common.MAX_TITLE_LEN]
event['msg_text'] = msg
else:
event['msg_title'] = msg
del event['msg']
return [event]
else:
return None
def parse_date(timestamp):
return common.parse_date(timestamp, DATE_FORMAT)
def test_cassandra_parser(self):
from dogstream import cassandra, common
log_data = """ INFO [CompactionExecutor:1594] 2012-05-12 21:05:12,924 Saved test_data-Encodings-KeyCache (86400 items) in 85 ms
INFO [CompactionExecutor:1595] 2012-05-12 21:05:15,144 Saved test_data-Metrics-KeyCache (86400 items) in 96 ms
INFO [CompactionExecutor:1596] 2012-05-12 21:10:48,058 Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6528-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6531-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6529-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6530-Data.db')]
INFO [CompactionExecutor:1596] 2012-05-12 21:10:54,851 Compacted to [/var/cassandra/a-hc-65-Data.db,]. 102,079,134 to 101,546,397
INFO [CompactionExecutor:1598] 2012-05-12 22:05:04,313 Saved test_data-ResourcesMetadata-KeyCache (1 items) in 10 ms
INFO [CompactionExecutor:1599] 2012-05-12 22:05:14,813 Saved test_data-Encodings-KeyCache (86400 items) in 83 ms
INFO [CompactionExecutor:1630] 2012-05-13 13:05:44,963 Saved test_data-Metrics-KeyCache (86400 items) in 77 ms
INFO [CompactionExecutor:1631] 2012-05-13 13:15:01,923 Nothing to compact in data_log. Use forceUserDefinedCompaction if you wish to force compaction of single sstables (e.g. for tombstone collection)
INFO [CompactionExecutor:1632] 2012-05-13 13:15:01,927 Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6527-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6522-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6532-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6517-Data.db')]
INFO [CompactionExecutor:1632] 2012-05-13 13:27:17,685 Compacting large row test_data/series:6c6f677c32 (782001077 bytes) incrementally
INFO [CompactionExecutor:34] 2012-05-14 18:00:41,281 Saved test_data-Encodings-KeyCache (86400 items) in 78 ms
INFO 13:27:17,685 Compacting large row test_data/series:6c6f677c32 (782001077 bytes) incrementally
"""
alert_type = cassandra.ALERT_TYPES["INFO"]
event_type = cassandra.EVENT_TYPE
event_object = EventDefaults.EVENT_OBJECT
expected_output = {
"dogstreamEvents":[
{
"timestamp": common.parse_date("2012-05-12 21:10:48,058", cassandra.DATE_FORMAT),
"msg_title": "Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6528-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6531-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6529-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6530-Data.db')]"[0:common.MAX_TITLE_LEN],
"msg_text": "Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6528-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6531-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6529-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6530-Data.db')]",
"alert_type": alert_type,
"auto_priority": 0,
"event_type": event_type,
"event_object": event_object,
}, {
"timestamp": common.parse_date("2012-05-12 21:10:54,851", cassandra.DATE_FORMAT),
"msg_title": "Compacted to [/var/cassandra/a-hc-65-Data.db,]. 102,079,134 to 101,546,397",
"alert_type": alert_type,
"auto_priority": 0,
"event_type": event_type,
"event_object": event_object,
}, {
"timestamp": common.parse_date("2012-05-13 13:15:01,927", cassandra.DATE_FORMAT),
"msg_title": "Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6527-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6522-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6532-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6517-Data.db')]"[0:common.MAX_TITLE_LEN],
"msg_text": "Compacting [SSTableReader(path='/var/cassandra/data/test_data/series-hc-6527-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6522-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6532-Data.db'), SSTableReader(path='/var/cassandra/data/test_data/series-hc-6517-Data.db')]",
"alert_type": alert_type,
"event_type": event_type,
"auto_priority": 0,
"event_object": event_object,
}, {
"timestamp": common.parse_date("2012-05-13 13:27:17,685", cassandra.DATE_FORMAT),
"msg_title": "Compacting large row test_data/series:6c6f677c32 (782001077 bytes) incrementally",
"alert_type": alert_type,
"event_type": event_type,
"auto_priority": 0,
"event_object": event_object,
}, {
"timestamp": common.parse_date(datetime.utcnow().strftime("%Y-%m-%d") + " 13:27:17,685", cassandra.DATE_FORMAT),
"msg_title": "Compacting large row test_data/series:6c6f677c32 (782001077 bytes) incrementally",
"alert_type": alert_type,
"event_type": event_type,
"auto_priority": 0,
"event_object": event_object,
},
]}
self._write_log(log_data.split("\n"))
dogstream = Dogstreams.init(self.logger, {'dogstreams': '%s:dogstream.cassandra:parse_cassandra' % self.log_file.name})
actual_output = dogstream.check(self.config, move_end=False)
self.assertEquals(expected_output, actual_output)
def parse_date(timestamp):
return common.parse_date(timestamp, DATE_FORMAT)