def update(self, tfv: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness: self.warning(tfv) typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value) r = dap_interface_pb2.Successfulness() r.success = True if typecode != 'location': r.success = False r.narrative = "DapGeo won't do " + typecode + " updates." self.stats["row_add_fails"] += 1 return r entity = (tfv.key.core, tfv.key.agent) locn = ProtoHelpers._get_location(val) self.warning("storing ", entity, locn) new = self.geos[tfv.tablename].place( entity, locn) if new: self.stats["rows_added"] += 1 else: self.stats["row_add_duplicate"] += 1 agent_parts = tfv.key.agent.decode("UTF-8").split("/") if len(agent_parts) == 2 and len(agent_parts[1]) > 0: self._service_alias_store.setdefault(agent_parts[0], []).append(tfv.key.agent) return r
def remove( self, remove_data: DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True upd = remove_data if upd: k, v = "dm", upd.value.dm tbname = self.tablenames[0] if upd.fieldname not in self.structure[tbname]: r.narrative.append("No such field key={} fname={}".format( upd.key, upd.fieldname)) r.success = False self.stats["row_remove_fails"] += 1 field_type = self.structure[tbname][upd.fieldname]['type'] if field_type != 'embedding': r.narrative.append( "Bad Type tname={} key={} fname={} ftype={} vtype={}". format(tbname, upd.key, upd.fieldname, ftype, k)) r.success = False self.stats["row_remove_fails"] += 1 try: row = self.store[tbname][upd.key.core][upd.key.agent] s = row.pop(v.name, None) is not None r.success &= s if s: self.stats["rows_removed"] += 1 row[upd.fieldname] = self._get_avg_oef_vec(row, upd.fieldname) if np.sum(row[upd.fieldname]) == 0: self.store[tbname][upd.key.core].pop(upd.key.agent, None) except KeyError: self.stats["row_remove_not_found"] += 1 return r
def removeRow( self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0] try: row_key = (remove_data.key.core, remove_data.key.agent) num = len(self.table[row_key]) self.table.pop(row_key) self.stats["rows_removed"] += 1 self.stats["fields_removed"] += num if agent_pubkey in self._service_alias_store: for services in self._service_alias_store[agent_pubkey]: try: row_key = (remove_data.key.core, services) num = len(self.table[row_key]) self.table.pop(row_key) self.stats["rows_removed"] += 1 self.stats["fields_removed"] += num except KeyError: self.stats["row_remove_not_found"] += 1 except KeyError: self.stats["row_remove_not_found"] += 1 try: self._service_alias_store.pop(agent_pubkey) except KeyError: pass return r
def update( self, tfv: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True if tfv: row_key = (tfv.key.core, tfv.key.agent) core_ident, agent_ident = row_key field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType( tfv.value) if field_type == "key-type-value_list": for k, t, v in field_value: self.table.setdefault( row_key, {})[k] = DapAttributeStore.StoredFieldValue(ft=t, fv=v) else: target_field_name = tfv.fieldname if target_field_name[0:5] != 'them.': target_field_name = 'them.' + target_field_name #self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident)) self.table.setdefault( row_key, {} )[target_field_name] = DapAttributeStore.StoredFieldValue( ft=field_type, fv=field_value) return r
def removeRow( self, remove_data: DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0] for tbname in self.store: try: self.store[tbname][remove_data.key.core].pop( remove_data.key.agent) self.stats["rows_removed"] += 1 if agent_pubkey not in self._service_alias_store: continue for service in self._service_alias_store[agent_pubkey]: try: self.store[tbname][remove_data.key.core].pop(service) self.stats["rows_removed"] += 1 except KeyError: if service != remove_data.key.agent: self.stats["row_remove_not_found"] += 1 except KeyError as e: self.stats["row_remove_not_found"] += 1 try: self._service_alias_store.pop(agent_pubkey) except KeyError: pass return r
def remove(self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True success = False for commit in [ False, True ]: upd = remove_data row_key = (upd.key.core, upd.key.agent) self.all_my_keys.pop(row_key) self.log.info("REMOVE: core={}, agent={}".format(core_ident, agent_ident)) return r
def update(self, update_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True upd = update_data if upd: row_key = (upd.key.core, upd.key.agent) core_ident, agent_ident = row_key self.all_my_keys.add(row_key) self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident)) return r
def update( self, tfv: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True self.info("Got update: ", tfv) for commit in [False, True]: k, v = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value) key = tfv.key core_ident, agent_ident = key.core, key.agent if tfv.fieldname not in self.fields: r.narrative.append( "ImMemoryDap:update No such field key={},{} fname={}". format(core_ident, agent_ident, tfv.fieldname)) r.success = False self.stats["row_add_fails"] += 1 break else: tbname = self.fields[tfv.fieldname]["tablename"] ftype = self.fields[tfv.fieldname]["type"] if tbname not in self.tablenames: r.narrative.append( "ImMemoryDap:update Bad tablename tname={} not in {}". format(tbname, ','.join(self.tablenames))) r.success = False self.stats["row_add_fails"] += 1 break if ftype != k: r.narrative.append( "ImMemoryDap:update Bad Type tname={} key={} fname={} ftype={} vtype={}" .format(tbname, tfv.key.core, tfv.fieldname, ftype, k)) r.success = False self.stats["row_add_fails"] += 1 break if commit: target = self.store.setdefault(tbname, {}).setdefault( (core_ident, agent_ident), {}) if tfv.fieldname in target and target[tfv.fieldname] == v: self.stats["row_add_duplicate"] += 1 else: self.stats["rows_added"] += 1 target[tfv.fieldname] = v # self.log.info("Stored {} into {} for {},{}".format( # tfv.fieldname, tbname, core_ident, agent_ident # )) if not r.success: break return r
def remove( self, remove_data: dap_update_pb2.DapUpdate ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True success = False for commit in [False, True]: row_key = (remove_data.key.core, remove_data.key.agent) core_ident, agent_ident = row_key for tbname in self.store.keys(): if commit: self.store[tbname].pop(row_key) if not r.success: break return r
async def handle_message(self, msg: Any) -> DataWrapper[Any]: response = DataWrapper(False, "", None) try: if isinstance(msg, dap_interface_pb2.NoInputParameter): proto = self._dap_call() else: proto = self._dap_call(msg) response.success = True response.data = proto except Exception as e: error = dap_interface_pb2.Successfulness() error.success = False error.errorcode = 503 error.narrative.append(str(e)) self.exception("Exception during DAP call: ", e) response.error_code = 503 response.add_narrative(str(e)) response.data = error return response
def remove( self, remove_data: dap_update_pb2.DapUpdate ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True for commit in [True]: row_key = (remove_data.key.core, remove_data.key.agent) if commit: try: num = len(self.table[row_key]) self.table.pop(row_key) self.stats["rows_removed"] += 1 self.stats["fields_removed"] += num except KeyError: self.info("RowID {} not in the table".format(row_key)) self.stats["row_remove_not_found"] += 1 if not r.success: break return r
def update( self, update_data: DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True try: upd = update_data if upd: k, v = "dm", upd.value.dm tbname = self.tablenames[0] if upd.fieldname not in self.structure[tbname]: raise DapBadUpdateRow("No such field", tbname, upd.key.core, upd.fieldname, k) field_type = self.structure[tbname][upd.fieldname]['type'] if field_type != 'embedding': r.narrative.append( "Bad Type tname={} key={} fname={} ftype={} vtype={}". format(tbname, upd.key, upd.fieldname, ftype, k)) r.success = False row = self.store.setdefault(tbname, {}).setdefault( upd.key.core, {}).setdefault(upd.key.agent, {}) data_present = False if v.name in row and row[v.name] == v: data_present = True row[v.name] = v cache = row.get(upd.fieldname, None) row[upd.fieldname] = self._get_avg_oef_vec(row, upd.fieldname) if not data_present or cache is None or (row[upd.fieldname] != cache).all(): self.stats["rows_added"] += 1 else: self.stats["row_add_duplicate"] += 1 agent_parts = upd.key.agent.decode("UTF-8").split("/") if len(agent_parts) == 2 and len(agent_parts[1]) > 0: self._service_alias_store.setdefault( agent_parts[0], []).append(upd.key.agent) except Exception as e: r.success = False self.stats["row_add_fails"] += 1 return r
def remove( self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True success = False for commit in [False, True]: row_key = (remove_data.key.core, remove_data.key.agent) core_ident, agent_ident = row_key for tbname in self.store.keys(): if commit: try: self.store[tbname].pop(row_key) self.stats["rows_removed"] += 1 except KeyError: self.stats["row_remove_not_found"] += 1 if not r.success: break return r
def update( self, tfv: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: self.warning(tfv) typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value) r = dap_interface_pb2.Successfulness() r.success = True if typecode != 'location': r.success = False r.narrative = "DapGeo won't do " + typecode + " updates." return r entity = (tfv.key.core, tfv.key.agent) locn = (val.lat, val.lon) self.warning("storing ", entity, locn) self.geos[tfv.tablename].place(entity, locn) return r
def removeRow(self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True agent_pubkey = remove_data.key.agent.decode("UTF-8").split("/")[0] for tbname in self.geos: try: self.geos[tbname].remove((remove_data.key.core, remove_data.key.agent)) self.stats["rows_removed"] += 1 except KeyError: self.stats["row_remove_not_found"] += 1 if agent_pubkey in self._service_alias_store: for services in self._service_alias_store[agent_pubkey]: try: self.geos[tbname].remove((remove_data.key.core, services)) self.stats["rows_removed"] += 1 except KeyError: self.stats["row_remove_not_found"] += 1 try: self._service_alias_store.pop(agent_pubkey) except KeyError: pass return r
def _call(self, path, data_in, output_type): request = data_in.SerializeToString() response = self.client.call(path, request) proto = output_type() if not response.success: self.error("Error response for uri %s, code: %d, reason: %s", response.uri, response.error_code, response.msg()) return proto try: proto.ParseFromString(response.data) except: try: sproto = dap_interface_pb2.Successfulness() sproto.ParseFromString(response.data) if not sproto.success: self.error("Dap failure: code %d, message: %s", sproto.errorcode, sproto.narrative) else: self.error("Unknown failure! Message: %s", sproto.message) except: self.exception("Unknown error!") return proto
def update( self, tfv: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True return r
def update( self, tfv: dap_update_pb2.DapUpdate.TableFieldValue ) -> dap_interface_pb2.Successfulness: r = dap_interface_pb2.Successfulness() r.success = True if tfv: row_key = (tfv.key.core, tfv.key.agent) core_ident, agent_ident = row_key field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType( tfv.value) if field_type == "key-type-value_list": row_present = row_key in self.table duplicate_inserted = False new_inserted = False for k, t, v in field_value: target = self.table.setdefault(row_key, {}) new_value = DapAttributeStore.StoredFieldValue(ft=t, fv=v) if k in target and target[k] == new_value: self.stats["fields_add_duplicate"] += 1 duplicate_inserted = True else: self.stats["fields_added"] += 1 new_inserted = True target[k] = new_value if not row_present and (not duplicate_inserted and new_inserted): self.stats["rows_added"] += 1 elif row_present and (duplicate_inserted and not new_inserted): self.stats["row_add_duplicate"] += 1 else: target_field_name = tfv.fieldname if target_field_name[0:5] != 'them.': target_field_name = 'them.' + target_field_name #self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident)) new_value = DapAttributeStore.StoredFieldValue(ft=field_type, fv=field_value) row_present = row_key in self.table duplicate_inserted = False new_inserted = False target = self.table.setdefault(row_key, {}) if target_field_name in target and target[ target_field_name] == new_value: self.stats["fields_add_duplicate"] += 1 duplicate_inserted = True else: self.stats["fields_added"] += 1 new_inserted = True target[target_field_name] = new_value if not row_present and (not duplicate_inserted and new_inserted): self.stats["rows_added"] += 1 elif row_present and (duplicate_inserted and not new_inserted): self.stats["row_add_duplicate"] += 1 agent_parts = tfv.key.agent.decode("UTF-8").split("/") if len(agent_parts) == 2 and len(agent_parts[1]) > 0: self._service_alias_store.setdefault(agent_parts[0], []).append(tfv.key.agent) return r