def inserter(bulk, rec): keys = desc["keys"] link_type = desc.get("link", "INTEL") counters = desc.get("counters", []) accumulators = desc.get("accumulators", {}) keys = utils.normalize_props(keys) counters = utils.normalize_props(counters) for props in (keys, counters, accumulators): for k, v in list(viewitems(props)): if v[0] == '{' and v[-1] == '}': prop = v[1:-1] else: prop = k if (prop not in rec or rec[prop] is None) and k in props: del(props[k]) if kind == "flow": flow_keys = desc.get("flow_keys", DEFAULT_FLOW_KEYS) bulk.append( db.flow.add_flow_metadata( desc["labels"], link_type, keys, flow_keys, counters=counters, accumulators=accumulators), rec ) elif kind == "host": host_keys = desc.get("host_keys", DEFAULT_HOST_KEYS) bulk.append( db.flow.add_host_metadata( desc["labels"], link_type, keys, host_keys=host_keys, counters=counters, accumulators=accumulators), rec ) else: raise ValueError("Unrecognized kind")
def inserter(bulk, rec): keys = desc["keys"] link_type = desc.get("link", "INTEL") counters = desc.get("counters", []) accumulators = desc.get("accumulators", {}) keys = utils.normalize_props(keys) counters = utils.normalize_props(counters) for props in (keys, counters, accumulators): for k, v in props.items(): if v[0] == '{' and v[-1] == '}': prop = v[1:-1] else: prop = k if (prop not in rec or rec[prop] is None) and k in props: del(props[k]) if kind == "flow": flow_keys = desc.get("flow_keys", DEFAULT_FLOW_KEYS) bulk.append( db.flow.add_flow_metadata( desc["labels"], link_type, keys, flow_keys, counters=counters, accumulators=accumulators), rec ) elif kind == "host": host_keys = desc.get("host_keys", DEFAULT_HOST_KEYS) bulk.append( db.flow.add_host_metadata( desc["labels"], link_type, keys, host_keys=host_keys, counters=counters, accumulators=accumulators), rec ) else: raise ValueError("Unrecognized kind")
def add_host_metadata(self, labels, linktype, keys, host_keys=None, counters=None, accumulators=None, time=True): counters = {} if counters is None else counters query = [self.add_host("h", keys=host_keys)] keys = utils.normalize_props(keys) key = self._key_from_attrs(keys, src=None, dst=None) query.extend([ "MERGE (%s)" % (self._gen_merge_elt("meta", labels, {"__key__": key})), "MERGE (h)-[:%s]->(meta)" % (linktype, ), ]) query.append( self._prop_update("meta", props=keys, counters=counters, create_clauses=["meta:Intel"], accumulators=accumulators, time=time)) return "\n".join(query)
def _add_flow(self, labels, keys, elt="link", counters=None, accumulators=None, srcnode=None, dstnode=None, time=True): keys = utils.normalize_props(keys) if srcnode is None: srcnode = (["Host"], {"addr": "{src}"}) if dstnode is None: dstnode = (["Host"], {"addr": "{dst}"}) key = self._key_from_attrs(keys) query = [ self.add_host("src", srcnode[0], srcnode[1], time=time), self.add_host("dst", dstnode[0], dstnode[1], time=time), "MERGE (%s)" % (self._gen_merge_elt(elt, labels, {"__key__": key})), "MERGE (%s)<-[:SEND]-(src)" % elt, "MERGE (%s)-[:TO]->(dst)" % elt, ] query.append( self._prop_update(elt, props=keys, counters=counters, accumulators=accumulators, time=time)) query.append(self._update_time_seen(elt)) return "\n".join(query)
def _gen_merge_elt(cls, elt, labels, attrs): attrs = utils.normalize_props(attrs) return "%s:%s {%s}" % ( elt, ":".join(labels), ", ".join("%s: %s" % (key, value) for key, value in attrs.iteritems()) )
def _update_counters(cls, elt, counters, on_create_set, on_match_set): counters = utils.normalize_props(counters) counters["count"] = 1 cls._set_props(elt, counters, on_create_set) on_match_set.extend( ["%(elt)s.%(key)s = COALESCE(%(elt)s.%(key)s, 0) + %(value)s" % ( {"elt": elt, "key": key, "value": value} ) for key, value in counters.iteritems()])
def add_host_metadata(self, labels, linktype, keys, host_keys=None, counters=None, accumulators=None, time=True): counters = {} if counters is None else counters query = [self.add_host("h", keys=host_keys)] keys = utils.normalize_props(keys) key = self._key_from_attrs(keys, src=None, dst=None) query.extend([ "MERGE (%s)" % ( self._gen_merge_elt("meta", labels, {"__key__": key})), "MERGE (h)-[:%s]->(meta)" % (linktype,), ]) query.append(self._prop_update("meta", props=keys, counters=counters, create_clauses=["meta:Intel"], accumulators=accumulators, time=time)) return "\n".join(query)
def _add_flow(self, labels, keys, elt="link", counters=None, accumulators=None, srcnode=None, dstnode=None, time=True): keys = utils.normalize_props(keys) if srcnode is None: srcnode = (["Host"], {"addr": "{src}"}) if dstnode is None: dstnode = (["Host"], {"addr": "{dst}"}) key = self._key_from_attrs(keys) query = [ self.add_host("src", srcnode[0], srcnode[1], time=time), self.add_host("dst", dstnode[0], dstnode[1], time=time), "MERGE (%s)" % ( self._gen_merge_elt(elt, labels, {"__key__": key})), "MERGE (%s)<-[:SEND]-(src)" % elt, "MERGE (%s)-[:TO]->(dst)" % elt, ] query.append(self._prop_update(elt, props=keys, counters=counters, accumulators=accumulators, time=time)) return "\n".join(query)
def _set_props(cls, elt, props, set_list): props = utils.normalize_props(props) set_list.extend("%s.%s = %s" % (elt, attr, cnt) for attr, cnt in viewitems(props))
def _set_props(cls, elt, props, set_list): props = utils.normalize_props(props) set_list.extend(["%s.%s = %s" % (elt, attr, cnt) for attr, cnt in props.iteritems()])