def test_change_not_ignore_null(self):
action = FilterChange("*", "alert.change", {"title":"something changed !"}, "c_key", "q_key", ignore_null=False)
event = Event("my_tag", 123456789, {"q_key": "XXXX", "c_key":True})
action.do(event)
event = Event("my_tag", 123456790, {"q_key": "XXXX"})
result = action.do(event)
self.assertEqual(result[1].tag, "alert.change")
def test_force_flush(self):
action = OutputArchive("*", "timestamp", "/tmp/archive/{source}/sys-%Y%m%d", buffer_size=1)
event_1 = Event("my_tag", 1603373122, {"timestamp":1603373122, "data":"ddd", "source":"j2"})
event_2 = Event("my_tag", 1603373123, {"timestamp":1603373123, "data":"eee", "source":"j2"})
action.do(event_1)
action.do(event_2) # pushed because buffer is full
with open("/tmp/archive/j2/sys-20201022") as f:
data = f.read()
def test_change_ignore_query_key(self):
action = FilterChange("*", "alert.change", {"title":"something changed !"}, "c_key", "q_key")
event = Event("my_tag", 123456789, {"c_key":True}) # missing q_key, ignored
action.do(event)
event = Event("my_tag", 123456790, {"q_key": "XXXX", "c_key":False})
result = action.do(event)
self.assertEqual(result, event)
def test_no_change(self):
action = FilterChange("*", "alert.change", {"title":"something changed !"}, "c_key", "q_key")
event = Event("my_tag", 123456789, {"q_key": "XXXX", "c_key":True})
action.do(event)
event = Event("my_tag", 123456790, {"q_key": "XXXX", "c_key":True})
result = action.do(event)
self.assertEqual(result, event)
def test_2(self):
action = OutputArchive("*", "timestamp", "/tmp/archive/{source}/sys-%Y%m%d")
event_1 = Event("my_tag", 1603373122, {"timestamp":1603373122, "data":"BBBB", "source":"j2"})
event_2 = Event("my_tag", 1603373123, {"timestamp":1603373123, "data":"ccccc", "source":"j2"})
action.do(event_1)
action.do(event_2)
action.finish()
with open("/tmp/archive/j2/sys-20201022") as f:
data = f.read()
def test_existing_dir(self):
action = OutputArchive("*", "timestamp", "/tmp/archive/{source}/sys-%Y%m%d", buffer_size=1)
event_1 = Event("my_tag", 1603373122, {"timestamp":1603373122, "data":"ddd", "source":"j2"})
event_2 = Event("my_tag", 1603373123, {"timestamp":1603373123, "data":"eee", "source":"j2"})
event_3 = Event("my_tag", 1603373122, {"timestamp":1603373124, "data":"ff", "source":"j2"})
event_4 = Event("my_tag", 1603373123, {"timestamp":1603373125, "data":"gg", "source":"j2"})
action.do(event_1)
action.do(event_2)
action.do(event_3)
action.do(event_4)
def test_store_and_load(self):
old_action = FilterChange("*", "alert.change", {"title":"something changed !"}, "c_key", "q_key", db_path=TEST_DB)
event = Event("my_tag", 123456789, {"q_key": "XXXX", "c_key":True})
old_action.do(event)
old_action.update()
action = FilterChange("*", "alert.change", {"title":"something changed !"}, "c_key", "q_key", db_path=TEST_DB)
event = Event("my_tag", 123456790, {"q_key": "XXXX", "c_key":False})
result = action.do(event)
self.assertEqual(result[1].tag, "alert.change")
def update(self):
if self._data is None:
return
if isinstance(self._data, dict):
event = Event(self._tag, int(time.time()), self._data)
self.call_next(event)
self._data = None
return
if isinstance(self._data, list):
for data in self._data:
event = Event(self._tag, int(time.time()), data)
self.call_next(event)
self._data = None
def test_1(self):
action = FilterRemoveKeys("*", "a")
event = Event("my_tag", 123456789, {"a": "b", "c": "d"})
result = action.do(event)
expected = {"c": "d"}
self.assertEqual(result.record, expected)
def test_1(self):
action = FilterDate("*", "t")
event = Event("my_tag", 123456789, {"t": "2020-10-22T08:50:20+00:00"})
result = action.do(event)
expected = {"t": 1603356620.0}
self.assertEqual(result.record, expected)
def test_2(self):
action = FilterDate("*", "t", "%Y-%m-%d")
event = Event("my_tag", 123456789, {"t": 1603356620.0})
result = action.do(event)
expected = {"t": "2020-10-22"}
self.assertEqual(result.record, expected)
def test_dict_tree(self):
action = ParserRegex("*", r"(?P<x>a+) (?P<y>b+)", "value", "tree")
event = Event("tag", 123456789, {"key":"Z", "value":"aaa bb"})
result = action.do(event)
expected = {"key":"Z", "value":{"x":"aaa", "y":"bb"}}
self.assertEqual(result.record, expected)
def do(self, event):
record = dict(event.record)
to_parse = record.get(self._key)
if to_parse is None:
self._log.debug(
"Failed to parse '{}', key doesn't exist".format(to_parse))
return
result = self.match(to_parse)
if result is None:
return
if self._mode == "tree":
record[self._key] = result
elif self._mode == "merge":
del record[self._key]
record.update(result)
elif self._mode == "add":
record.update(result)
else:
self._log.error("Bad mode '{}' ! (check the configuration)".format(
self._mode))
raise Exception("Bad mode '{}'".format(self._mode))
return Event(event.tag, event.timestamp, record)
def test_dict_tree(self):
action = ParserJson("*", "value", "tree")
event = Event("tag", 123456789, {"key": "Z", "value": '{"aaa": "bb"}'})
result = action.do(event)
expected = {"key": "Z", "value": {"aaa": "bb"}}
self.assertEqual(result.record, expected)
def update(self):
while (1):
try:
event = self._queue.get_nowait()
new_event = Event(event[0].decode("ascii"), event[1], event[2])
self.call_next(new_event)
except queue.Empty:
return
def test_dict_merge_2(self):
parameters = {"key": "value", "mode": "merge"}
action = ParserJson("*", **parameters)
event = Event("tag", 123456789, {"key": "Z", "value": '{"aaa": "bb"}'})
result = action.do(event)
expected = {"key": "Z", "aaa": "bb"}
self.assertEqual(result.record, expected)
def test_dict_merge_2(self):
parameters = {"key":"value", "mode":"merge"}
action = ParserRegex("*", r"(?P<x>a+) (?P<y>b+)", **parameters)
event = Event("tag", 123456789, {"key":"Z", "value":"aaa bb"})
result = action.do(event)
expected = {"key":"Z", "x":"aaa", "y":"bb"}
self.assertEqual(result.record, expected)
def test_from_log_ram(self):
parameters = {"key":"log", "mode":"merge"}
action = ParserRegex("*", r"(?P<date>.+?)\s+sys.ram\s+(?P<json>.+)", **parameters)
event = Event("tag", 123456789, {"log":'2020-11-02T07:50:04+00:00 sys.ram {"tag":"sys.ram","Mem.free":774748,"Swap.free":102396,"Mem.total":3919812,"source":"jarvis-2","Swap.total":102396,"Swap.used":0,"Mem.used":3145064}'})
result = action.do(event)
expected = {'date': '2020-11-02T07:50:04+00:00',
'json': '{"tag":"sys.ram","Mem.free":774748,"Swap.free":102396,"Mem.total":3919812,"source":"jarvis-2","Swap.total":102396,"Swap.used":0,"Mem.used":3145064}'}
self.assertEqual(result.record, expected)
def test_2(self):
action = FilterRetag("*", "t", "g")
event = Event("my_tag", 123456789, {
"t": "2020-10-22T08:50:20+00:00",
"g": "new_tag"
})
result = action.do(event)
self.assertEqual(result.tag, "new_tag")
def test_timeout(self):
action = OutputArchive("*", "timestamp", "/tmp/archive/{source}/sys-%Y%m%d", buffer_size=1)
event_1 = Event("my_tag", 1603373122, {"timestamp":1603373122, "data":"ddd", "source":"j2"})
action.do(event_1)
def my_time():
return time() + 3600
action.update(my_time)
with open("/tmp/archive/j2/sys-20201022") as f:
data = f.read()
def do(self, event):
record = dict(event.record)
time = Chronyk(record[self._key])
if self._format is None:
record[self._key] = time.timestamp()
else:
record[self._key] = time.timestring(self._format)
return Event(event.tag, event.timestamp, record)
def do(self, event):
record = dict(event.record)
# if string
if isinstance(self._keys, str):
if self._keys in record:
del record[self._keys]
# if list/tuple
else:
for k in self._keys:
if k in record:
del record[k]
return Event(event.tag, event.timestamp, record)
def process_file(self, directory, filename):
path = os.path.join(directory, filename)
match = fnmatch.fnmatch(path, self._path_pattern)
if match:
self._log.info ("Process file '{}'".format(path))
with open(path) as f:
line_number = 0
for line in f:
data = line.rstrip()
event = Event(self._tag, int(time()), {"log":data})
self.call_next(event)
line_number += 1
self._log.info("Processing file '{}' produce {} events".format(filename, line_number))
if self._remove:
self._log.debug ("Remove file '{}'".format(path))
os.remove(path)
def do(self, event):
record = dict(event.record)
to_parse = record.get(self._key)
if to_parse is None:
self._log.debug(
"Failed to parse '{}', key doesn't exist".format(to_parse))
return
try:
result = json.loads(to_parse)
except JSONDecodeError:
self._log.debug("Failed to parse '{}'".format(to_parse))
return
if self._mode == "tree":
record[self._key] = result
elif self._mode == "merge":
del record[self._key]
record.update(result)
elif self._mode == "add":
record.update(result)
else:
raise Exception("Bad mode '{}'".format(self._mode))
return Event(event.tag, event.timestamp, record)
def test_init(self):
forward = InputForward()
e = Event("my.tag", 123456789, {"log": "test"})
forward.callback(e)
forward.finish()
def test_1(self):
e = Event("my_tag", 123456789, '{"field":"test_1"}')
action = OutputStdout("*")
action.receive(e)
def do(self, event):
record = event.record
new_tag = record.get(self._key, self._value)
return Event(new_tag, event.timestamp, event.record)