def _set_agents_summary(self):
results = env.dataprovider.query(["max(heartbeat.create_time)", "heartbeat.analyzer(-1).analyzerid/group_by"])
if not results:
return
c = Criterion()
for create_time, analyzerid in results:
c |= Criterion("heartbeat.create_time", "==", create_time) & Criterion("heartbeat.analyzer(-1).analyzerid", "==", analyzerid)
agents = {
"up": utils.AttrObj(count=0, title=_("Online"), label="label-success", status=["online"]),
"down": utils.AttrObj(count=0, title=_("Offline"), label="label-danger", status=["offline", "missing", "unknown"])
}
heartbeat_error_margin = env.config.general.get_int("heartbeat_error_margin", 3)
for heartbeat in env.dataprovider.get(c):
heartbeat = heartbeat["heartbeat"]
analyzer = heartbeat["analyzer"][-1]
analyzer.status = utils.get_analyzer_status_from_latest_heartbeat(heartbeat, heartbeat_error_margin)[0]
for key, values in agents.items():
if analyzer.status in values.status:
values.count += 1
parameters = env.request.menu_parameters
val = agents["down"] if agents["down"].count else agents["up"]
data = resource.HTMLNode("a", localization.format_number(val.count), title=val.title, _class="label " + val.label, href=url_for("Agents.agents", status=val.status, **parameters))
return utils.AttrObj(
name="agents",
title=resource.HTMLNode("a", _("Agents"), href=url_for("Agents.agents", **parameters)),
data=[data]
)
def _get_series(self, query, selection, date_precision):
selection_index = len(query.paths) + 1
all_paths, all_criteria = self._prepare_query(query)
series_order = []
crit = Criterion()
if query.limit > 0 and query.paths:
for values in self._query(all_paths, all_criteria, limit=query.limit, type=query.datatype):
crit |= functools.reduce(lambda x, y: x & y, (Criterion(query.paths[i], "=", value) for i, value in enumerate(values[1:])))
series_order.append(tuple(values[1:]))
res = []
if query.limit != 0:
res = self._query(all_paths + selection, all_criteria + crit, type=query.datatype)
out = {}
for i in res:
key = tuple(i[1:selection_index]) or (self.title,)
tval = tuple((int(x) for x in i[selection_index:]))
out.setdefault(key, {})[tval[:date_precision]] = i[0]
if not series_order:
return out
out2 = collections.OrderedDict()
for i in series_order:
out2[i] = out[i]
return out2
def getUriCriteria(ptype, analyzerid, messageid):
criteria = Criterion()
if analyzerid:
criteria += Criterion("%s.analyzer.analyzerid" % (ptype), "=",
analyzerid)
return criteria + Criterion("%s.messageid" % (ptype), "=", messageid)
def _tree_to_criteria(self, t, parent_field=[], has_required=False):
tag = getattr(t, "data", None)
if not tag:
return text_type(t.value) if t else None
if tag == "or_":
return self._bool(t.children[0], "||", t.children[2], parent_field)
elif tag == "and_":
return self._bool(t.children[0], "&&", t.children[2], parent_field)
elif tag == "field" and t.children:
return t.children[0].replace(" ", "")[:-1]
elif tag == "parenthesis":
_, data, _ = t.children
return self._tree_to_criteria(data, parent_field).criterion
elif tag == "value_string":
op, value = t.children[0]
ret = Criterion()
for i in self._get_fields(parent_field):
if i.endswith(".exact"):
i = i[:-6:]
op = "=="
ret |= self._compile(i, op, value.value)
return ret
elif tag == "inclusive_range":
from_, to = filter(lambda x: isinstance(x, tuple), t.children)
return self._range(parent_field, from_[1], to[1], ">=", "<=")
elif tag == "exclusive_range":
from_, to = filter(lambda x: isinstance(x, tuple), t.children)
return self._range(parent_field, from_[1], to[1], ">", "<")
elif tag in "required":
if len(t.children) > 1:
_, field, data = t.children
return _Required(self._get_criterion(data, field, parent_field))
elif tag == "excluded":
if len(t.children) > 1:
_, field, data = t.children
return _Required(Criterion(operator="!", right=self._get_criterion(data, field, parent_field)))
elif tag == "optional":
field, data = t.children
return _Optional(self._get_criterion(data, field, parent_field))
elif tag == "input":
return self._tree_to_criteria(t.children[1], parent_field)
elif t.children:
return self._tree_to_criteria(t.children[0], parent_field)
def filter_fixtures(request):
"""
Fixture for filter tests.
:return: view for filter.
:rtype: prewikka.view.View
"""
from prewikka.plugins.filter.filter import FilterDatabase, Filter # prevent import error
view = load_view_for_fixtures(request.param)
view.process_parameters()
database = FilterDatabase()
criterion_1 = Criterion('alert.messageid', '=', 'fakemessageid1')
criteria_1 = {'alert': criterion_1}
filter_obj_1 = Filter(None, 'Test filter 1', 'Filter category',
'Filter description', criteria_1)
database.upsert_filter(env.request.user, filter_obj_1)
criterion_2 = Criterion('heartbeat.messageid', '=', 'fakemessageid1')
criteria_2 = {'heartbeat': criterion_2}
filter_obj_2 = Filter(None, 'Test filter 2', 'Filter category',
'Filter description', criteria_2)
database.upsert_filter(env.request.user, filter_obj_2)
criterion_3 = Criterion(criterion_1, '||', criterion_2)
criteria_3 = {'alert': criterion_3}
filter_obj_3 = Filter(None, 'Test filter 3', 'Filter category',
'Filter description', criteria_3)
database.upsert_filter(env.request.user, filter_obj_3)
# complex criterion
criterion_4 = Criterion(criterion_3, '||', criterion_2)
criteria_4 = {'alert': criterion_4}
filter_obj_4 = Filter(None, 'Test filter 4', 'Filter category',
'Filter description', criteria_4)
database.upsert_filter(env.request.user, filter_obj_4)
def tear_down():
"""
TearDown
"""
env.db.query('DELETE FROM Prewikka_Filter')
request.addfinalizer(tear_down)
return {
'view': view,
'database': database,
'filter_obj_1': filter_obj_1,
'filter_obj_2': filter_obj_2,
'filter_obj_3': filter_obj_3,
'filter_obj_4': filter_obj_4,
'last_insert_id': env.db.get_last_insert_ident(),
'criterion_list': [criterion_1, criterion_2, criterion_3, criterion_4]
}
def buildAlertIdent(self, alert, parent):
calist = {}
for alertident in parent["alertident"]:
# IDMEF draft 14 page 27
# If the "analyzerid" is not provided, the alert is assumed to have come
# from the same analyzer that is sending the Alert.
analyzerid = alertident["analyzerid"]
if not analyzerid:
for a in alert["analyzer"]:
if a["analyzerid"]:
analyzerid = a["analyzerid"]
break
if not analyzerid in calist:
calist[analyzerid] = []
calist[analyzerid].append(alertident["alertident"])
idx = 1
for analyzerid in calist.keys():
content = ""
missing = 0
for ident in calist[analyzerid]:
criteria = Criterion("alert.analyzer.analyzerid", "=",
analyzerid) & Criterion(
"alert.messageid", "=", ident)
results = env.dataprovider.get(criteria)
if len(results) == 0:
missing += 1
#content += "<li>" + _("Invalid 'analyzerid:messageid' pair, '%(analyzerid):%(messageid)'") % { "analyzerid": analyzerid, "messageid": ident } + "</li>"
else:
alert = results[0]["alert"]
link = url_for(".", analyzerid=analyzerid, messageid=ident)
content += '<li><a class="widget-link" title="%s" href="%s">%s</a></li>' % (
_("Alert details"), link,
html.escape(alert["classification.text"]))
if missing > 0:
content += "<li>" + (
_("%d linked alerts missing (probably deleted)") %
missing) + "</li>"
self.newTableCol(
idx,
resource.HTMLSource(
"<ul style='padding: 0px; margin: 0px 0px 0px 10px;'>%s</ul>"
% content))
self.buildAnalyzer(alert["analyzer(-1)"])
self.newTableRow()
idx += 1
def test_criterion_flatten():
"""
Test `prewikka.dataprovider.Criterion.flatten()` method.
"""
criterion_1 = Criterion('alert.messageid', '=', 'fakemessageid1')
criterion_2 = Criterion('alert.messageid', '=', 'fakemessageid2')
criterion_3 = Criterion('alert.messageid', '=', 'fakemessageid3')
criterion_4 = Criterion('alert.messageid', '=', 'fakemessageid4')
criterion = ((criterion_1 & criterion_2) & criterion_3) | criterion_4
flattened = criterion.flatten()
assert flattened.operator == CriterionOperator.OR
assert len(flattened.operands) == 2
assert flattened.operands[0].operator == CriterionOperator.AND
assert flattened.operands[0].operands == [
criterion_1, criterion_2, criterion_3
]
assert flattened.operands[1] == criterion_4
criterion = Criterion(
None, '!', Criterion(None, '!', Criterion(None, '!', criterion_1)))
flattened = criterion.flatten()
assert flattened.operator == CriterionOperator.NOT
assert len(flattened.operands) == 1
assert flattened.operands[0] == criterion_1
def delete(self):
c = Criterion()
for analyzerid in env.request.parameters.getlist("id[]"):
for i in env.request.parameters.getlist("types[]"):
if i in ("alert", "heartbeat"):
c |= Criterion("%s.analyzer.analyzerid" % i, "=",
analyzerid)
env.dataprovider.delete(c)
return response.PrewikkaRedirectResponse(url_for(".agents"))
def get_criteria(self):
criteria = Criterion()
if self.start:
start = self.start.astimezone(utils.timeutil.timezone("UTC"))
criteria += Criterion("{backend}.{end_time_field}", ">=", start)
if self.end:
end = self.end.astimezone(utils.timeutil.timezone("UTC"))
criteria += Criterion("{backend}.{start_time_field}", "<=", end)
return criteria
def compile_criterion(self, criterion):
if criterion.right:
criterion.right = self._value_adjust(criterion.operator, criterion.right)
elif criterion.operator == CriterionOperator.EQUAL:
criterion = Criterion(criterion.left, "==", None)
if _IDMEFPath(criterion.left).getValueType() == prelude.IDMEFValue.TYPE_STRING:
criterion |= Criterion(criterion.left, "==", "''")
if criterion.operator.negated and _IDMEFPath(criterion.left).isAmbiguous():
criterion.left = self._path_adjust(criterion.left)
return DataProviderBase.compile_criterion(self, criterion)
def _get_analyzers(self, reqstatus):
# Do not take the control menu into account.
# The expected behavior is yet to be determined.
results = env.dataprovider.query(["max(heartbeat.create_time)", "heartbeat.analyzer(-1).analyzerid/group_by"])
if not results:
return
c = Criterion()
for create_time, analyzerid in results:
c |= Criterion("heartbeat.create_time", "==", create_time) & Criterion("heartbeat.analyzer(-1).analyzerid", "==", analyzerid)
for heartbeat in env.dataprovider.get(c):
heartbeat = heartbeat["heartbeat"]
status, status_text = utils.get_analyzer_status_from_latest_heartbeat(
heartbeat, self._heartbeat_error_margin
)
if reqstatus and status not in reqstatus:
continue
delta = heartbeat.get("create_time") - utils.timeutil.now()
analyzerid = heartbeat["analyzer(-1).analyzerid"]
heartbeat_listing = url_for("HeartbeatDataSearch.forensic", criteria=Criterion("heartbeat.analyzer(-1).analyzerid", "==", analyzerid), _default=None)
alert_listing = url_for("AlertDataSearch.forensic", criteria=Criterion("alert.analyzer.analyzerid", "==", analyzerid), _default=None)
heartbeat_analyze = url_for(".analyze", analyzerid=analyzerid)
analyzer = heartbeat["analyzer(-1)"]
node_name = analyzer["node.name"] or _("Node name n/a")
osversion = analyzer["osversion"] or _("OS version n/a")
ostype = analyzer["ostype"] or _("OS type n/a")
yield {
"id": analyzerid,
"label": "%s - %s %s" % (node_name, ostype, osversion),
"location": analyzer["node.location"] or _("Node location n/a"),
"node": node_name,
"name": analyzer["name"],
"model": analyzer["model"],
"class": analyzer["class"],
"version": analyzer["version"],
"latest_heartbeat": localization.format_timedelta(delta, add_direction=True),
"status": status,
"status_text": status_text,
"links": [
resource.HTMLNode("a", _("Alert listing"), href=alert_listing),
resource.HTMLNode("a", _("Heartbeat listing"), href=heartbeat_listing),
resource.HTMLNode("a", _("Heartbeat analysis"), href=heartbeat_analyze)
]
}
class CriteriaTransformer(CommonTransformer):
def __init__(self, compile=Criterion):
self._compile = compile
parenthesis = v_args(inline=True)(lambda self, criterion: criterion)
or_ = v_args(
inline=True)(lambda self, left, right: Criterion(left, "||", right))
and_ = v_args(
inline=True)(lambda self, left, right: Criterion(left, "&&", right))
not_ = v_args(
inline=True)(lambda self, right: Criterion(operator="!", right=right))
criterion = v_args(inline=True)(
lambda self, left, op, right: self._compile(left, op, right))
not_null = v_args(
inline=True)(lambda self, path: self._compile(path, "!=", None))
def test_dumps():
"""
Test prewikka.utils.json.dump().
"""
assert json.dumps(['foo', {'bar': ['baz', None, 1.0, 2]}]) == '["foo", {"bar": ["baz", null, 1.0, 2]}]'
assert json.dumps('"foo\x08ar') == '"\\"foo\\bar"'
# Prewikka objects
criterion = Criterion('alert.messageid', '=', 'fakemessageid')
criterion_dumps = json.dumps(criterion)
assert '{"__prewikka_class__": ["Criterion"' in criterion_dumps
assert '"operator": "="' in criterion_dumps
assert '"right": "fakemessageid"' in criterion_dumps
assert '"left": "alert.messageid"' in criterion_dumps
# datetime object
assert json.dumps(datetime(year=2012, month=10, day=12, hour=0, minute=0, second=0)) == \
'"2012-10-12 00:00:00"'
# object not JSON serializable
fake_obj = FakeClass()
with pytest.raises(TypeError):
json.dumps(fake_obj)
def get_heartbeat(heartbeat_id):
"""
Delete a Heartbeat in database after tests.
:param str heartbeat_id: Heartbeat ID.
"""
return env.dataprovider.get(Criterion('heartbeat.messageid', '=', heartbeat_id))
def get_criteria(self):
criteria = Criterion()
if self.start:
start = self.start.astimezone(utils.timeutil.timezone("UTC"))
criteria += Criterion("{backend}.{time_field}", ">=", start)
if self.end:
end = self.end
if not env.request.parameters["timeline_absolute"]:
end = self.end + relativedelta(minutes=1)
end = self.end.astimezone(utils.timeutil.timezone("UTC"))
criteria += Criterion("{backend}.{time_field}", "<", end)
return criteria
def register(self):
mainmenu.MainMenuParameters.register(self)
self.optional("offset", int, default=0)
self.optional("limit", int, default=50, save=True)
self.optional("selection", [json.loads], Criterion())
self.optional("listing_apply", text_type)
self.optional("action", text_type)
def _get_analyzer(self, analyzerid):
res = env.dataprovider.get(Criterion(
"heartbeat.analyzer(-1).analyzerid", "=", analyzerid),
limit=1)
heartbeat = res[0]["heartbeat"]
analyzer = heartbeat["analyzer"][-1]
return analyzer, heartbeat
def _get_categories(self, query):
all_paths, all_criteria = self._prepare_query(query)
for row in self._query(all_paths, all_criteria, limit=query.limit, type=query.datatype):
count = row[0]
category = tuple(row[1:])
crit = functools.reduce(lambda x, y: x & y, (Criterion(path, '=', row[i + 1])
for i, path in enumerate(query.paths)))
yield count, category, crit
def get_alert(alert_id):
"""
Get an alert for test suite.
:param str alert_id: Alert ID.
:return: alert if exists.
:rtype: prewikka.utils.misc.CachingIterator
"""
return env.dataprovider.get(Criterion('alert.messageid', '=', alert_id))
def delete_heartbeat(heartbeat_id):
"""
Delete a Heartbeat in database after tests.
:param str heartbeat_id: Heartbeat ID.
:type heartbeat_id: str
:return: None.
"""
env.dataprovider.delete(Criterion('heartbeat.messageid', '=', heartbeat_id))
def buildAlertIdent(self, alert, parent):
calist = {}
for alertident in parent["alertident"]:
# IDMEF draft 14 page 27
# If the "analyzerid" is not provided, the alert is assumed to have come
# from the same analyzer that is sending the Alert.
analyzerid = alertident["analyzerid"]
if not analyzerid:
for a in alert["analyzer"]:
if a["analyzerid"]:
analyzerid = a["analyzerid"]
break
calist.setdefault(analyzerid, []).append(alertident["alertident"])
for idx, (analyzerid, idents) in enumerate(calist.items()):
content = ""
results = []
total = 0
step = 50
limit = min(len(idents), 500)
for i in range(0, limit, step):
# FIXME #3250, #3251
# We execute several queries to avoid recursion errors
criteria = Criterion()
for ident in idents[i:i+step]:
criteria |= self._get_alert_ident_criterion(analyzerid, ident)
results.append(env.dataprovider.query(["alert.messageid", "alert.classification.text"], criteria))
for ident, classif in itertools.chain(*results):
link = url_for(".render", analyzerid=analyzerid, messageid=ident)
content += '<li><a title="%s" href="%s">%s</a></li>' % (_("Alert details"), link, html.escape(classif))
total += 1
missing = limit - total
if missing > 0:
content += "<li>" + (_("%d linked alerts missing (probably deleted)") % missing) + "</li>"
omitted = len(idents) - limit
if omitted > 0:
content += "<li>" + (_("%d linked alerts omitted") % omitted) + "</li>"
self.newTableCol(idx + 1, resource.HTMLSource("<ul style='padding: 0px; margin: 0px 0px 0px 10px;'>%s</ul>" % content))
linked_alerts = env.dataprovider.get(self._get_alert_ident_criterion(analyzerid, idents[0]))
if linked_alerts:
self.buildAnalyzer(linked_alerts[0]["alert.analyzer(-1)"])
else:
self.newTableCol(1, None)
self.newTableRow()
def _applyInlineFilters(self, criteria):
for column, path in (("analyzerid", "heartbeat.analyzer(-1).analyzerid"),
("name", "heartbeat.analyzer(-1).name"),
("model", "heartbeat.analyzer(-1).model"),
("address", "heartbeat.analyzer(-1).node.address.address"),
("node_name", "heartbeat.analyzer(-1).node.name")):
env.request.dataset[column + "_filtered"] = False
if path in env.request.parameters:
criteria += Criterion(path, "=", env.request.parameters[path])
env.request.dataset[column + "_filtered"] = True
def get_criteria(self, query):
if not query:
return Criterion()
qmode = env.request.parameters.get(
"query_mode", self._parent.criterion_config_default)
if qmode == "criterion":
return criteria.parse(query,
transformer=criteria.CriteriaTransformer(
compile=self._criterion_compile))
elif qmode == "lucene":
if self._parent.criterion_config_default != "lucene":
tr = lucene.CriteriaTransformer(
compile=self._criterion_compile,
default_paths=self._parent.lucene_search_fields)
return lucene.parse(query, transformer=tr)
else:
return Criterion("{backend}._raw_query", "==", query)
def _bool(self, a, op, b, parent_field=[]):
a = self._tree_to_criteria(a, parent_field)
b = self._tree_to_criteria(b, parent_field)
if b is None or (isinstance(a, _Required) and isinstance(b, _Optional)):
return a
elif a is None or (isinstance(b, _Required) and isinstance(a, _Optional)):
return b
else:
return a.__class__(Criterion(a.criterion, "&&" if isinstance(a, _Required) else op, b.criterion))
def format_criterion(cls, path, value, mode):
if mode == "lucene":
if isinstance(value, (int, float, datetime.datetime)):
return "%s:%s" % (path, value)
if not value:
return "-%s:[* TO *]" % path
return "%s.exact:%s" % (path, cls._lucene_escape(value))
return text_type(Criterion(path, "==", value))
def _alert_cron(self, job):
config = env.config.cron.get_instance_by_name("alert")
if config is None:
return
criteria = Criterion()
age = int(config.get("age", 0))
now = utils.timeutil.utcnow()
for severity in (None, "info", "low", "medium", "high"):
days = int(config.get(severity, age))
if days < 1:
continue
criteria |= (Criterion("alert.assessment.impact.severity", "==", severity) &
Criterion("alert.create_time", "<", now - datetime.timedelta(days=days)))
if not criteria:
return
if not list(hookmanager.trigger("HOOK_CRON_DELETE", criteria, "alert")):
env.dataprovider.delete(criteria, type="alert")
def _handle_indexation_by_string(self, criteria, query, with_aliases):
matches = re.findall(STRING_INDEX_REGEX, criteria.left)
if matches:
path = re.sub(STRING_INDEX_REGEX, "", criteria.left)
string_index = (env.dataprovider.get_indexation_path(
criteria.left), matches[0][1])
self._add_join(self._paths_map[path][0],
query,
string_index=string_index)
return self._process_criteria(
Criterion(path, criteria.operator, criteria.right), query,
with_aliases)
def __init__(self, **kwargs):
datatype, path, aggregate, limit, order, criteria = [kwargs.get(i) for i in self.KEYS]
if isinstance(path, list):
self.paths = path
else:
self.paths = [path] if path else []
self.datatype = datatype
self.aggregation = aggregate
self.limit = int(limit or env.request.parameters["limit"])
self.order = order or "desc"
self.criteria = criteria or Criterion()
def _heartbeat_cron(self, job):
config = env.config.cron.get_instance_by_name("heartbeat")
if config is None:
return
days = int(config.get("age", 0))
if days < 1:
return
criteria = Criterion("heartbeat.create_time", "<", utils.timeutil.utcnow() - datetime.timedelta(days=days))
if not list(hookmanager.trigger("HOOK_CRON_DELETE", criteria, "heartbeat")):
env.dataprovider.delete(criteria, type="heartbeat")
def _criterion_compile(self, left, op, right):
if left not in self._parent.path_translate:
left = "%s.%s" % (self.type, left)
return Criterion(left, self._fix_operator(left, op), right)
paths, valuefunc = self._parent.path_translate[left]
if valuefunc:
right = valuefunc(right)
# Translation (for source [A, B]):
# source : (A != None || B != None)
# !source : !(A != None || B != None) => !(A || B)
# source != test: (A != test && B != test)
# source == test: (A == test || B == test)
if op[0] == "!" and right is not None:
f = operator.and_
else:
f = operator.or_
return functools.reduce(lambda x, y: f(x, y),
(Criterion(i, self._fix_operator(i, op), right)
for i in paths))
