def inserter(bulk, rec):
keys = desc["keys"]
link_type = desc.get("link", "INTEL")
counters = desc.get("counters", [])
accumulators = desc.get("accumulators", {})
keys = utils.normalize_props(keys)
counters = utils.normalize_props(counters)
for props in (keys, counters, accumulators):
for k, v in list(viewitems(props)):
if v[0] == '{' and v[-1] == '}':
prop = v[1:-1]
else:
prop = k
if (prop not in rec or rec[prop] is None) and k in props:
del(props[k])
if kind == "flow":
flow_keys = desc.get("flow_keys", DEFAULT_FLOW_KEYS)
bulk.append(
db.flow.add_flow_metadata(
desc["labels"], link_type, keys, flow_keys,
counters=counters, accumulators=accumulators),
rec
)
elif kind == "host":
host_keys = desc.get("host_keys", DEFAULT_HOST_KEYS)
bulk.append(
db.flow.add_host_metadata(
desc["labels"], link_type, keys, host_keys=host_keys,
counters=counters, accumulators=accumulators),
rec
)
else:
raise ValueError("Unrecognized kind")
def inserter(bulk, rec):
keys = desc["keys"]
link_type = desc.get("link", "INTEL")
counters = desc.get("counters", [])
accumulators = desc.get("accumulators", {})
keys = utils.normalize_props(keys)
counters = utils.normalize_props(counters)
for props in (keys, counters, accumulators):
for k, v in props.items():
if v[0] == '{' and v[-1] == '}':
prop = v[1:-1]
else:
prop = k
if (prop not in rec or rec[prop] is None) and k in props:
del(props[k])
if kind == "flow":
flow_keys = desc.get("flow_keys", DEFAULT_FLOW_KEYS)
bulk.append(
db.flow.add_flow_metadata(
desc["labels"], link_type, keys, flow_keys,
counters=counters, accumulators=accumulators),
rec
)
elif kind == "host":
host_keys = desc.get("host_keys", DEFAULT_HOST_KEYS)
bulk.append(
db.flow.add_host_metadata(
desc["labels"], link_type, keys, host_keys=host_keys,
counters=counters, accumulators=accumulators),
rec
)
else:
raise ValueError("Unrecognized kind")
def add_host_metadata(self,
labels,
linktype,
keys,
host_keys=None,
counters=None,
accumulators=None,
time=True):
counters = {} if counters is None else counters
query = [self.add_host("h", keys=host_keys)]
keys = utils.normalize_props(keys)
key = self._key_from_attrs(keys, src=None, dst=None)
query.extend([
"MERGE (%s)" %
(self._gen_merge_elt("meta", labels, {"__key__": key})),
"MERGE (h)-[:%s]->(meta)" % (linktype, ),
])
query.append(
self._prop_update("meta",
props=keys,
counters=counters,
create_clauses=["meta:Intel"],
accumulators=accumulators,
time=time))
return "\n".join(query)
def _add_flow(self,
labels,
keys,
elt="link",
counters=None,
accumulators=None,
srcnode=None,
dstnode=None,
time=True):
keys = utils.normalize_props(keys)
if srcnode is None:
srcnode = (["Host"], {"addr": "{src}"})
if dstnode is None:
dstnode = (["Host"], {"addr": "{dst}"})
key = self._key_from_attrs(keys)
query = [
self.add_host("src", srcnode[0], srcnode[1], time=time),
self.add_host("dst", dstnode[0], dstnode[1], time=time),
"MERGE (%s)" %
(self._gen_merge_elt(elt, labels, {"__key__": key})),
"MERGE (%s)<-[:SEND]-(src)" % elt,
"MERGE (%s)-[:TO]->(dst)" % elt,
]
query.append(
self._prop_update(elt,
props=keys,
counters=counters,
accumulators=accumulators,
time=time))
query.append(self._update_time_seen(elt))
return "\n".join(query)
def _gen_merge_elt(cls, elt, labels, attrs):
attrs = utils.normalize_props(attrs)
return "%s:%s {%s}" % (
elt,
":".join(labels),
", ".join("%s: %s" % (key, value)
for key, value in attrs.iteritems())
)
def _update_counters(cls, elt, counters, on_create_set, on_match_set):
counters = utils.normalize_props(counters)
counters["count"] = 1
cls._set_props(elt, counters, on_create_set)
on_match_set.extend(
["%(elt)s.%(key)s = COALESCE(%(elt)s.%(key)s, 0) + %(value)s" % (
{"elt": elt,
"key": key,
"value": value}
) for key, value in counters.iteritems()])
def add_host_metadata(self, labels, linktype, keys, host_keys=None,
counters=None, accumulators=None, time=True):
counters = {} if counters is None else counters
query = [self.add_host("h", keys=host_keys)]
keys = utils.normalize_props(keys)
key = self._key_from_attrs(keys, src=None, dst=None)
query.extend([
"MERGE (%s)" % (
self._gen_merge_elt("meta", labels, {"__key__": key})),
"MERGE (h)-[:%s]->(meta)" % (linktype,),
])
query.append(self._prop_update("meta", props=keys, counters=counters,
create_clauses=["meta:Intel"],
accumulators=accumulators, time=time))
return "\n".join(query)
def _add_flow(self, labels, keys, elt="link", counters=None,
accumulators=None, srcnode=None, dstnode=None, time=True):
keys = utils.normalize_props(keys)
if srcnode is None:
srcnode = (["Host"], {"addr": "{src}"})
if dstnode is None:
dstnode = (["Host"], {"addr": "{dst}"})
key = self._key_from_attrs(keys)
query = [
self.add_host("src", srcnode[0], srcnode[1], time=time),
self.add_host("dst", dstnode[0], dstnode[1], time=time),
"MERGE (%s)" % (
self._gen_merge_elt(elt, labels, {"__key__": key})),
"MERGE (%s)<-[:SEND]-(src)" % elt,
"MERGE (%s)-[:TO]->(dst)" % elt,
]
query.append(self._prop_update(elt, props=keys, counters=counters,
accumulators=accumulators, time=time))
return "\n".join(query)
def _set_props(cls, elt, props, set_list):
props = utils.normalize_props(props)
set_list.extend("%s.%s = %s" % (elt, attr, cnt)
for attr, cnt in viewitems(props))
def _set_props(cls, elt, props, set_list):
props = utils.normalize_props(props)
set_list.extend(["%s.%s = %s" % (elt, attr, cnt)
for attr, cnt in props.iteritems()])