def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
input_idents = proto.input_idents
query_memento = proto.query_memento
query_settings = json.loads(query_memento.memento.decode("utf-8"))
#for k,v in query_settings.items():
#self.log.info(" execute settings {} = {}".format(k,v))
try:
compare_func = self.operatorFactory.lookup(
query_settings['target_field_type'],
query_settings['operator'], query_settings['query_field_type'])
#self.info("Function obtained")
func = lambda row: InMemoryDap.runCompareFunc(
row, compare_func, query_settings['target_field_name'],
query_settings['query_field_value'], self.log)
r = self.processRows(func, query_settings['target_table_name'],
input_idents)
except Exception as ex:
for k, v in query_settings.items():
self.log.error(" execute settings {} = {}".format(k, v))
self.log.error(ex)
r = dap_interface_pb2.IdentifierSequence()
r.status.success = False
return r
def execute(self, dapQueryRepn) -> dap_interface_pb2.IdentifierSequence:
v1 = DapManager.PopulateActionsVisitorDescentPass(self)
try:
dapQueryRepn.visitDescending(v1)
except DapManager.NoConstraintCompilerException as ex:
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
failure = r.status
failure.success = False
failure.narrative.append(str(ex))
return r
self.printQuery("GEN_ACTIONS_PASS", dapQueryRepn)
start = dap_interface_pb2.IdentifierSequence()
start.originator = True
return self._execute(dapQueryRepn.root, start)
def processRows(
self, rowProcessor, target_table_name,
cores: dap_interface_pb2.IdentifierSequence
) -> dap_interface_pb2.IdentifierSequence:
#self.log.info("processRows")
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
table = self.store.get(target_table_name, None)
if table == None:
self.log.error("No table {} in {}".format(target_table_name,
self.store.keys()))
else:
if cores.originator:
#self.log.info("ORIGINATING")
for (core_ident, agent_ident), row in table.items():
#self.log.info("TESTING ORIGINATED: core={}, agent={}".format(core_ident, agent_ident))
self.processRow(rowProcessor, (core_ident, agent_ident),
row, r)
else:
#self.log.info("SUPPLIED WITH {}".format(len(cores.identifiers)))
for key in cores.identifiers:
core_ident, agent_ident = key.core, key.agent
#self.log.info("TESTING SUPPLIED: core={}, agent={}".format(core_ident, agent_ident))
row = table.get((core_ident, agent_ident), None)
if row == None:
self.log.error("{} not found".format(
(core_ident, agent_ident)))
self.log.error("table keys = {}".format(table.keys()))
else:
self.processRow(rowProcessor,
(core_ident, agent_ident), row, r, key)
return r
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
input_idents = proto.input_idents
query_memento = proto.query_memento
graphQuery = SearchEngine.SubQuery().setSearchSystem(self).fromJSON(
query_memento.memento.decode("utf-8"))
self.stats["execute_count"] += 1
if input_idents.originator:
idents = None
else:
idents = [DapQueryResult(pb=x) for x in input_idents.identifiers]
reply = dap_interface_pb2.IdentifierSequence()
reply.originator = False
execute_results = graphQuery.execute(idents)
for ident in execute_results:
c = reply.identifiers.add()
c.core = ident.core_id
c.agent = ident.agent_id
c.score = ident.score
self.info("SUCCESS: ", c)
nr = len(reply.identifiers)
if len(reply.identifiers) == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
if not input_idents.originator:
self.stats["filter_inputs"] += len(input_idents.identifiers) - nr
else:
self.stats["originated_queries"] += 1
self.stats["num_originated_rows"] += nr
return reply
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
input_idents = proto.input_idents
query_memento = proto.query_memento
query_settings = json.loads(query_memento.memento.decode("utf-8"))
for k, v in query_settings.items():
self.log.info(" execute settings {} = {}".format(k, v))
result = dap_interface_pb2.IdentifierSequence()
result.originator = False
if input_idents.originator:
for row_key, row in self.table.items():
core_ident, agent_ident = row_key
#self.log.info("TESTING ORIGINATED: core={}, agent={}".format(core_ident, agent_ident))
if self.test(row, query_settings):
#self.log.info("PASSING: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.core = core_ident
i.agent = agent_ident
else:
for key in input_idents.identifiers:
core_ident, agent_ident = key.core, key.agent
row = self.table.get((core_ident, agent_ident), None)
#self.log.info("TESTING SUPPLIED: core={}, agent={}".format(core_ident, agent_ident))
if row != None and self.test(row, query_settings):
#self.log.info("PASSING: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.CopyFrom(key)
return result
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
for key in proto.input_idents.identifiers:
new_result = r.identifiers.add()
new_result.CopyFrom(key)
addr = self.resolveCore(key.core)
if addr:
new_result.uri = addr
return r
def coresToIdentifierSequence(
cores: List[DapQueryResult.DapQueryResult]
) -> dap_interface_pb2.IdentifierSequence:
m = dap_interface_pb2.IdentifierSequence()
m.originator = False
if cores != None:
m.originator = False
for c in cores:
ident = m.identifiers.add()
ident.CopyFrom(c.asIdentifierProto())
else:
m.originator = True
return m
def _executeAnd(
self, node, cores: dap_interface_pb2.IdentifierSequence
) -> dap_interface_pb2.IdentifierSequence:
self.warning("_executeAnd ", node.printable())
leafstart = 0
nodestart = 0
for n in node.leaves[leafstart:]:
cores = self._executeLeaf(n, cores)
if len(cores.identifiers) == 0:
result = dap_interface_pb2.IdentifierSequence()
result.originator = False
return result
for n in node.subnodes[nodestart:]:
cores = self._execute(n, cores)
if len(cores.identifiers) == 0:
result = dap_interface_pb2.IdentifierSequence()
result.originator = False
return result
return cores
def execute(self, proto: dap_interface_pb2.DapExecute) -> dap_interface_pb2.IdentifierSequence:
self.stats["execute_count"] += 1
geoQuery = DapGeo.DapGeoQuery(self)
input_idents = proto.input_idents
query_memento = proto.query_memento
j = query_memento.memento.decode("utf-8")
geoQuery.fromJSON(j)
geoQuery.setGeo(self.getGeoByTableName(geoQuery.tablename))
coreagent_to_identifier = {}
originator = False
if hasattr(input_idents, 'originator') and input_idents.originator:
originator = True
self.warning("geoQuery.tablename=", geoQuery.tablename)
idents = list(self.geos[geoQuery.tablename].getAllKeys())
self.warning("idents=", idents)
coreagent_to_identifier = {}
else:
coreagent_to_identifier.update({
(identifier.core, identifier.agent): identifier
for identifier
in input_idents.identifiers
})
idents = coreagent_to_identifier.keys()
self.warning("idents=", idents)
reply = dap_interface_pb2.IdentifierSequence()
reply.originator = False
for r in geoQuery.execute(set(idents)):
c = reply.identifiers.add()
if r in coreagent_to_identifier:
c.CopyFrom(coreagent_to_identifier[r])
else:
c.core = r[0]
c.agent = r[1]
nr = len(reply.identifiers)
if nr == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
if not originator:
self.stats["filter_inputs"] += len(input_idents.identifiers) - nr
else:
self.stats["num_originated_rows"] += nr
self.stats["originated_queries"] += 1
return reply
def execute(self, proto: dap_interface_pb2.DapExecute) -> dap_interface_pb2.IdentifierSequence:
self.stats["execute_count"] += 1
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
for key in proto.input_idents.identifiers:
new_result = r.identifiers.add()
new_result.CopyFrom(key)
addr = self.resolveCore(key.core)
if addr:
new_result.uri = addr
nr = len(r.identifiers)
if nr == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
self.stats["filter_inputs"] += len(proto.input_idents.identifiers) - nr
return r
def _executeOr(
self, node, cores: dap_interface_pb2.IdentifierSequence
) -> dap_interface_pb2.IdentifierSequence:
self.warning("_executeOr ", node.printable())
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
for n in node.subnodes:
res = self._execute(n, cores)
for ident in res.identifiers:
newid = r.identifiers.add()
newid.CopyFrom(ident)
for n in node.leaves:
res = self._executeLeaf(n, cores)
for ident in res.identifiers:
newid = r.identifiers.add()
newid.CopyFrom(ident)
return r
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
r = dap_interface_pb2.IdentifierSequence()
r.originator = False
done = {}
self.stats["execute_count"] += 1
for key in proto.input_idents.identifiers:
handled = done.setdefault(key.core,
{}).setdefault(key.agent, False)
if not handled:
done[key.core][key.agent] = True
new_result = r.identifiers.add()
new_result.CopyFrom(key)
nr = len(r.identifiers)
if nr == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
self.stats["filter_inputs"] = len(proto.input_idents.identifiers) - nr
return r
def execute(self, proto: dap_interface_pb2.DapExecute) -> dap_interface_pb2.IdentifierSequence:
graphQuery = DapERNetwork.DapGraphQuery()
input_idents = proto.input_idents
query_memento = proto.query_memento
j = query_memento.memento.decode("utf-8")
graphQuery.fromJSON(j)
graphQuery.setGraph(self.graphs[graphQuery.tablename])
if input_idents.HasField('originator') and input_idents.originator:
idents = None
else:
idents = [ DapQueryResult(x) for x in input_idents.identifiers ]
reply = dap_interface_pb2.IdentifierSequence()
reply.originator = False
#BUG(KLL): missing score out of the copy
for core in graphQuery.execute(idents):
c = reply.identifiers.add()
c.core = core[0].encode("utf-8")
c.agent = core[1].encode("utf-8")
return reply
def execute(self, proto: dap_interface_pb2.DapExecute) -> dap_interface_pb2.IdentifierSequence:
result = dap_interface_pb2.IdentifierSequence()
result.originator = False
cores = proto.input_idents
query_memento = proto.query_memento
j = json.loads(query_memento.memento.decode("utf-8"))
if cores.originator:
for row_key in self.all_my_keys:
core_ident, agent_ident = row_key
self.log.info("RETURNING ORIGINATED: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.core = core_ident
i.agent = agent_ident
else:
for key in cores.identifiers:
core_ident, agent_ident = key.core, key.agent
self.log.info("RETURNING SUPPLIED: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.CopyFrom(key)
return result
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
input_idents = proto.input_idents
query_memento = proto.query_memento
query_settings = json.loads(query_memento.memento.decode("utf-8"))
self.stats["execute_count"] += 1
#for k,v in query_settings.items():
# self.log.info(" execute settings {} = {}".format(k,v))
result = dap_interface_pb2.IdentifierSequence()
result.originator = False
if input_idents.originator:
for row_key, row in self.table.items():
core_ident, agent_ident = row_key
#self.log.info("TESTING ORIGINATED: core={}, agent={}".format(core_ident, agent_ident))
if self.test(row, query_settings):
#self.log.info("PASSING: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.core = core_ident
i.agent = agent_ident
self.stats["originated_queries"] += 1
self.stats["num_originated_rows"] += len(result.identifiers)
else:
for key in input_idents.identifiers:
core_ident, agent_ident = key.core, key.agent
row = self.table.get((core_ident, agent_ident), None)
#self.log.info("TESTING SUPPLIED: core={}, agent={}".format(core_ident, agent_ident))
if row != None and self.test(row, query_settings):
#self.log.info("PASSING: core={}, agent={}".format(core_ident, agent_ident))
i = result.identifiers.add()
i.CopyFrom(key)
nr = len(result.identifiers)
if nr == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
if not proto.input_idents.originator:
self.stats["filter_inputs"] += len(
proto.input_idents.identifiers) - nr
return result
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
input_idents = proto.input_idents
query_memento = proto.query_memento
query_settings = json.loads(query_memento.memento.decode("utf-8"))
self.stats["execute_count"] += 1
#for k,v in query_settings.items():
#self.log.info(" execute settings {} = {}".format(k,v))
try:
compare_func = self.operatorFactory.lookup(
query_settings['target_field_type'],
query_settings['operator'], query_settings['query_field_type'])
#self.info("Function obtained")
func = lambda row: InMemoryDap.runCompareFunc(
row, compare_func, query_settings['target_field_name'],
query_settings['query_field_value'], self.log)
r = self.processRows(func, query_settings['target_table_name'],
input_idents)
except Exception as ex:
for k, v in query_settings.items():
self.log.error(" execute settings {} = {}".format(k, v))
self.log.error(ex)
r = dap_interface_pb2.IdentifierSequence()
r.status.success = False
nr = len(r.identifiers)
if nr == 0:
self.stats["empty_execute_response"] += 1
self.stats["total_result_count"] += nr
if not input_idents.originator:
self.stats["filter_inputs"] += len(input_idents.identifiers) - nr
else:
self.stats["originated_queries"] += 1
self.stats["num_originated_rows"] += nr
return r
def execute(
self, proto: dap_interface_pb2.DapExecute
) -> dap_interface_pb2.IdentifierSequence:
geoQuery = DapGeo.DapGeoQuery(self)
input_idents = proto.input_idents
query_memento = proto.query_memento
j = query_memento.memento.decode("utf-8")
geoQuery.fromJSON(j)
geoQuery.setGeo(self.getGeoByTableName(geoQuery.tablename))
coreagent_to_identifier = {}
if input_idents.HasField('originator') and input_idents.originator:
self.warning("geoQuery.tablename=", geoQuery.tablename)
idents = list(self.geos[geoQuery.tablename].getAllKeys())
self.warning("idents=", idents)
coreagent_to_identifier = {}
else:
coreagent_to_identifier.update({
(identifier.core, identifier.agent): identifier
for identifier in input_idents.identifiers
})
idents = coreagent_to_identifier.keys()
self.warning("idents=", idents)
reply = dap_interface_pb2.IdentifierSequence()
reply.originator = False
for r in geoQuery.execute(set(idents)):
c = reply.identifiers.add()
if r in coreagent_to_identifier:
c.CopyFrom(coreagent_to_identifier[r])
else:
c.core = r[0]
c.agent = r[1]
return reply