def update(self, tfv: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness:
self.warning(tfv)
typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
r = dap_interface_pb2.Successfulness()
r.success = True
if typecode != 'location':
r.success = False
r.narrative = "DapGeo won't do " + typecode + " updates."
self.stats["row_add_fails"] += 1
return r
entity = (tfv.key.core, tfv.key.agent)
locn = ProtoHelpers._get_location(val)
self.warning("storing ", entity, locn)
new = self.geos[tfv.tablename].place( entity, locn)
if new:
self.stats["rows_added"] += 1
else:
self.stats["row_add_duplicate"] += 1
agent_parts = tfv.key.agent.decode("UTF-8").split("/")
if len(agent_parts) == 2 and len(agent_parts[1]) > 0:
self._service_alias_store.setdefault(agent_parts[0], []).append(tfv.key.agent)
return r
def remove(
self, remove_data: DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
upd = remove_data
if upd:
k, v = "dm", upd.value.dm
tbname = self.tablenames[0]
if upd.fieldname not in self.structure[tbname]:
r.narrative.append("No such field key={} fname={}".format(
upd.key, upd.fieldname))
r.success = False
self.stats["row_remove_fails"] += 1
field_type = self.structure[tbname][upd.fieldname]['type']
if field_type != 'embedding':
r.narrative.append(
"Bad Type tname={} key={} fname={} ftype={} vtype={}".
format(tbname, upd.key, upd.fieldname, ftype, k))
r.success = False
self.stats["row_remove_fails"] += 1
try:
row = self.store[tbname][upd.key.core][upd.key.agent]
s = row.pop(v.name, None) is not None
r.success &= s
if s:
self.stats["rows_removed"] += 1
row[upd.fieldname] = self._get_avg_oef_vec(row, upd.fieldname)
if np.sum(row[upd.fieldname]) == 0:
self.store[tbname][upd.key.core].pop(upd.key.agent, None)
except KeyError:
self.stats["row_remove_not_found"] += 1
return r
def removeRow(
self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0]
try:
row_key = (remove_data.key.core, remove_data.key.agent)
num = len(self.table[row_key])
self.table.pop(row_key)
self.stats["rows_removed"] += 1
self.stats["fields_removed"] += num
if agent_pubkey in self._service_alias_store:
for services in self._service_alias_store[agent_pubkey]:
try:
row_key = (remove_data.key.core, services)
num = len(self.table[row_key])
self.table.pop(row_key)
self.stats["rows_removed"] += 1
self.stats["fields_removed"] += num
except KeyError:
self.stats["row_remove_not_found"] += 1
except KeyError:
self.stats["row_remove_not_found"] += 1
try:
self._service_alias_store.pop(agent_pubkey)
except KeyError:
pass
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
if tfv:
row_key = (tfv.key.core, tfv.key.agent)
core_ident, agent_ident = row_key
field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType(
tfv.value)
if field_type == "key-type-value_list":
for k, t, v in field_value:
self.table.setdefault(
row_key,
{})[k] = DapAttributeStore.StoredFieldValue(ft=t, fv=v)
else:
target_field_name = tfv.fieldname
if target_field_name[0:5] != 'them.':
target_field_name = 'them.' + target_field_name
#self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident))
self.table.setdefault(
row_key, {}
)[target_field_name] = DapAttributeStore.StoredFieldValue(
ft=field_type, fv=field_value)
return r
def removeRow(
self, remove_data: DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0]
for tbname in self.store:
try:
self.store[tbname][remove_data.key.core].pop(
remove_data.key.agent)
self.stats["rows_removed"] += 1
if agent_pubkey not in self._service_alias_store:
continue
for service in self._service_alias_store[agent_pubkey]:
try:
self.store[tbname][remove_data.key.core].pop(service)
self.stats["rows_removed"] += 1
except KeyError:
if service != remove_data.key.agent:
self.stats["row_remove_not_found"] += 1
except KeyError as e:
self.stats["row_remove_not_found"] += 1
try:
self._service_alias_store.pop(agent_pubkey)
except KeyError:
pass
return r
def remove(self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
success = False
for commit in [ False, True ]:
upd = remove_data
row_key = (upd.key.core, upd.key.agent)
self.all_my_keys.pop(row_key)
self.log.info("REMOVE: core={}, agent={}".format(core_ident, agent_ident))
return r
def update(self, update_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
upd = update_data
if upd:
row_key = (upd.key.core, upd.key.agent)
core_ident, agent_ident = row_key
self.all_my_keys.add(row_key)
self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident))
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
self.info("Got update: ", tfv)
for commit in [False, True]:
k, v = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
key = tfv.key
core_ident, agent_ident = key.core, key.agent
if tfv.fieldname not in self.fields:
r.narrative.append(
"ImMemoryDap:update No such field key={},{} fname={}".
format(core_ident, agent_ident, tfv.fieldname))
r.success = False
self.stats["row_add_fails"] += 1
break
else:
tbname = self.fields[tfv.fieldname]["tablename"]
ftype = self.fields[tfv.fieldname]["type"]
if tbname not in self.tablenames:
r.narrative.append(
"ImMemoryDap:update Bad tablename tname={} not in {}".
format(tbname, ','.join(self.tablenames)))
r.success = False
self.stats["row_add_fails"] += 1
break
if ftype != k:
r.narrative.append(
"ImMemoryDap:update Bad Type tname={} key={} fname={} ftype={} vtype={}"
.format(tbname, tfv.key.core, tfv.fieldname, ftype, k))
r.success = False
self.stats["row_add_fails"] += 1
break
if commit:
target = self.store.setdefault(tbname, {}).setdefault(
(core_ident, agent_ident), {})
if tfv.fieldname in target and target[tfv.fieldname] == v:
self.stats["row_add_duplicate"] += 1
else:
self.stats["rows_added"] += 1
target[tfv.fieldname] = v
# self.log.info("Stored {} into {} for {},{}".format(
# tfv.fieldname, tbname, core_ident, agent_ident
# ))
if not r.success:
break
return r
def remove(
self, remove_data: dap_update_pb2.DapUpdate
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
success = False
for commit in [False, True]:
row_key = (remove_data.key.core, remove_data.key.agent)
core_ident, agent_ident = row_key
for tbname in self.store.keys():
if commit:
self.store[tbname].pop(row_key)
if not r.success:
break
return r
async def handle_message(self, msg: Any) -> DataWrapper[Any]:
response = DataWrapper(False, "", None)
try:
if isinstance(msg, dap_interface_pb2.NoInputParameter):
proto = self._dap_call()
else:
proto = self._dap_call(msg)
response.success = True
response.data = proto
except Exception as e:
error = dap_interface_pb2.Successfulness()
error.success = False
error.errorcode = 503
error.narrative.append(str(e))
self.exception("Exception during DAP call: ", e)
response.error_code = 503
response.add_narrative(str(e))
response.data = error
return response
def remove(
self, remove_data: dap_update_pb2.DapUpdate
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
for commit in [True]:
row_key = (remove_data.key.core, remove_data.key.agent)
if commit:
try:
num = len(self.table[row_key])
self.table.pop(row_key)
self.stats["rows_removed"] += 1
self.stats["fields_removed"] += num
except KeyError:
self.info("RowID {} not in the table".format(row_key))
self.stats["row_remove_not_found"] += 1
if not r.success:
break
return r
def update(
self, update_data: DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
try:
upd = update_data
if upd:
k, v = "dm", upd.value.dm
tbname = self.tablenames[0]
if upd.fieldname not in self.structure[tbname]:
raise DapBadUpdateRow("No such field", tbname,
upd.key.core, upd.fieldname, k)
field_type = self.structure[tbname][upd.fieldname]['type']
if field_type != 'embedding':
r.narrative.append(
"Bad Type tname={} key={} fname={} ftype={} vtype={}".
format(tbname, upd.key, upd.fieldname, ftype, k))
r.success = False
row = self.store.setdefault(tbname, {}).setdefault(
upd.key.core, {}).setdefault(upd.key.agent, {})
data_present = False
if v.name in row and row[v.name] == v:
data_present = True
row[v.name] = v
cache = row.get(upd.fieldname, None)
row[upd.fieldname] = self._get_avg_oef_vec(row, upd.fieldname)
if not data_present or cache is None or (row[upd.fieldname] !=
cache).all():
self.stats["rows_added"] += 1
else:
self.stats["row_add_duplicate"] += 1
agent_parts = upd.key.agent.decode("UTF-8").split("/")
if len(agent_parts) == 2 and len(agent_parts[1]) > 0:
self._service_alias_store.setdefault(
agent_parts[0], []).append(upd.key.agent)
except Exception as e:
r.success = False
self.stats["row_add_fails"] += 1
return r
def remove(
self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
success = False
for commit in [False, True]:
row_key = (remove_data.key.core, remove_data.key.agent)
core_ident, agent_ident = row_key
for tbname in self.store.keys():
if commit:
try:
self.store[tbname].pop(row_key)
self.stats["rows_removed"] += 1
except KeyError:
self.stats["row_remove_not_found"] += 1
if not r.success:
break
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
self.warning(tfv)
typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
r = dap_interface_pb2.Successfulness()
r.success = True
if typecode != 'location':
r.success = False
r.narrative = "DapGeo won't do " + typecode + " updates."
return r
entity = (tfv.key.core, tfv.key.agent)
locn = (val.lat, val.lon)
self.warning("storing ", entity, locn)
self.geos[tfv.tablename].place(entity, locn)
return r
def removeRow(self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0]
for tbname in self.geos:
try:
self.geos[tbname].remove((remove_data.key.core, remove_data.key.agent))
self.stats["rows_removed"] += 1
except KeyError:
self.stats["row_remove_not_found"] += 1
if agent_pubkey in self._service_alias_store:
for services in self._service_alias_store[agent_pubkey]:
try:
self.geos[tbname].remove((remove_data.key.core, services))
self.stats["rows_removed"] += 1
except KeyError:
self.stats["row_remove_not_found"] += 1
try:
self._service_alias_store.pop(agent_pubkey)
except KeyError:
pass
return r
def _call(self, path, data_in, output_type):
request = data_in.SerializeToString()
response = self.client.call(path, request)
proto = output_type()
if not response.success:
self.error("Error response for uri %s, code: %d, reason: %s",
response.uri, response.error_code, response.msg())
return proto
try:
proto.ParseFromString(response.data)
except:
try:
sproto = dap_interface_pb2.Successfulness()
sproto.ParseFromString(response.data)
if not sproto.success:
self.error("Dap failure: code %d, message: %s",
sproto.errorcode, sproto.narrative)
else:
self.error("Unknown failure! Message: %s", sproto.message)
except:
self.exception("Unknown error!")
return proto
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
if tfv:
row_key = (tfv.key.core, tfv.key.agent)
core_ident, agent_ident = row_key
field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType(
tfv.value)
if field_type == "key-type-value_list":
row_present = row_key in self.table
duplicate_inserted = False
new_inserted = False
for k, t, v in field_value:
target = self.table.setdefault(row_key, {})
new_value = DapAttributeStore.StoredFieldValue(ft=t, fv=v)
if k in target and target[k] == new_value:
self.stats["fields_add_duplicate"] += 1
duplicate_inserted = True
else:
self.stats["fields_added"] += 1
new_inserted = True
target[k] = new_value
if not row_present and (not duplicate_inserted
and new_inserted):
self.stats["rows_added"] += 1
elif row_present and (duplicate_inserted and not new_inserted):
self.stats["row_add_duplicate"] += 1
else:
target_field_name = tfv.fieldname
if target_field_name[0:5] != 'them.':
target_field_name = 'them.' + target_field_name
#self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident))
new_value = DapAttributeStore.StoredFieldValue(ft=field_type,
fv=field_value)
row_present = row_key in self.table
duplicate_inserted = False
new_inserted = False
target = self.table.setdefault(row_key, {})
if target_field_name in target and target[
target_field_name] == new_value:
self.stats["fields_add_duplicate"] += 1
duplicate_inserted = True
else:
self.stats["fields_added"] += 1
new_inserted = True
target[target_field_name] = new_value
if not row_present and (not duplicate_inserted
and new_inserted):
self.stats["rows_added"] += 1
elif row_present and (duplicate_inserted and not new_inserted):
self.stats["row_add_duplicate"] += 1
agent_parts = tfv.key.agent.decode("UTF-8").split("/")
if len(agent_parts) == 2 and len(agent_parts[1]) > 0:
self._service_alias_store.setdefault(agent_parts[0],
[]).append(tfv.key.agent)
return r
