def update(self, tfv: dap_update_pb2.DapUpdate.TableFieldValue) -> dap_interface_pb2.Successfulness:
self.warning(tfv)
typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
r = dap_interface_pb2.Successfulness()
r.success = True
if typecode != 'location':
r.success = False
r.narrative = "DapGeo won't do " + typecode + " updates."
self.stats["row_add_fails"] += 1
return r
entity = (tfv.key.core, tfv.key.agent)
locn = ProtoHelpers._get_location(val)
self.warning("storing ", entity, locn)
new = self.geos[tfv.tablename].place( entity, locn)
if new:
self.stats["rows_added"] += 1
else:
self.stats["row_add_duplicate"] += 1
agent_parts = tfv.key.agent.decode("UTF-8").split("/")
if len(agent_parts) == 2 and len(agent_parts[1]) > 0:
self._service_alias_store.setdefault(agent_parts[0], []).append(tfv.key.agent)
return r
def DoFieldsUpdateForAgent(self, agent, list_of_field_val_tuples):
for field_val_tuple in list_of_field_val_tuples:
update = dap_update_pb2.DapUpdate()
update.update.add()
update.update[0].tablename = "ratings"
ProtoHelpers.populateUpdateTFV(update.update[0],
field_val_tuple[0],
field_val_tuple[1])
update.update[0].key.agent = agent.encode("utf-8")
update.update[0].key.core = b'localhost'
r = self.dapManager.update(update)
self.warning("doing update", list_of_field_val_tuples, agent, r)
def listCores(self, tablename, fieldname):
r = []
for k in self.geos[tablename].getAllKeys():
if k[1] == b'':
latlon = self.geos[tablename].get(k)
result = dap_update_pb2.DapUpdate.TableFieldValue()
ProtoHelpers.populateUpdateTFV(result, fieldname, latlon, typename='location')
result.key.core = k[0]
result.tablename = tablename
r.append(result)
return r
def remove(self, remove_data: dap_update_pb2.DapUpdate.TableFieldValue):
success = False
for commit in [ False, True ]:
upd = remove_data
if upd:
k, v = ProtoHelpers.decodeAttributeValueToTypeValue(upd.value)
key = (upd.key.core, b'')
if upd.fieldname not in self.fields:
self.stats["row_remove_fails"] += 1
raise DapBadUpdateRow("No such field", None, upd.key.core, upd.fieldname, k)
else:
tbname = self.fields[upd.fieldname]["tablename"]
ftype = self.fields[upd.fieldname]["type"]
if ftype != k:
self.stats["row_remove_fails"] += 1
raise DapBadUpdateRow("Bad type", tbname, upd.key.core, upd.fieldname, k)
if commit:
s = self.store[tbname][key].pop(upd.fieldname, None) is not None
success |= s
if s:
self.stats["rows_removed"] += 1
else:
self.stats["row_remove_not_found"] += 1
return success
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
if tfv:
row_key = (tfv.key.core, tfv.key.agent)
core_ident, agent_ident = row_key
field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType(
tfv.value)
if field_type == "key-type-value_list":
for k, t, v in field_value:
self.table.setdefault(
row_key,
{})[k] = DapAttributeStore.StoredFieldValue(ft=t, fv=v)
else:
target_field_name = tfv.fieldname
if target_field_name[0:5] != 'them.':
target_field_name = 'them.' + target_field_name
#self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident))
self.table.setdefault(
row_key, {}
)[target_field_name] = DapAttributeStore.StoredFieldValue(
ft=field_type, fv=field_value)
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
self.info("Got update: ", tfv)
for commit in [False, True]:
k, v = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
key = tfv.key
core_ident, agent_ident = key.core, key.agent
if tfv.fieldname not in self.fields:
r.narrative.append(
"ImMemoryDap:update No such field key={},{} fname={}".
format(core_ident, agent_ident, tfv.fieldname))
r.success = False
self.stats["row_add_fails"] += 1
break
else:
tbname = self.fields[tfv.fieldname]["tablename"]
ftype = self.fields[tfv.fieldname]["type"]
if tbname not in self.tablenames:
r.narrative.append(
"ImMemoryDap:update Bad tablename tname={} not in {}".
format(tbname, ','.join(self.tablenames)))
r.success = False
self.stats["row_add_fails"] += 1
break
if ftype != k:
r.narrative.append(
"ImMemoryDap:update Bad Type tname={} key={} fname={} ftype={} vtype={}"
.format(tbname, tfv.key.core, tfv.fieldname, ftype, k))
r.success = False
self.stats["row_add_fails"] += 1
break
if commit:
target = self.store.setdefault(tbname, {}).setdefault(
(core_ident, agent_ident), {})
if tfv.fieldname in target and target[tfv.fieldname] == v:
self.stats["row_add_duplicate"] += 1
else:
self.stats["rows_added"] += 1
target[tfv.fieldname] = v
# self.log.info("Stored {} into {} for {},{}".format(
# tfv.fieldname, tbname, core_ident, agent_ident
# ))
if not r.success:
break
return r
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
self.warning(tfv)
typecode, val = ProtoHelpers.decodeAttributeValueToTypeValue(tfv.value)
r = dap_interface_pb2.Successfulness()
r.success = True
if typecode != 'location':
r.success = False
r.narrative = "DapGeo won't do " + typecode + " updates."
return r
entity = (tfv.key.core, tfv.key.agent)
locn = (val.lat, val.lon)
self.warning("storing ", entity, locn)
self.geos[tfv.tablename].place(entity, locn)
return r
def remove(self, remove_data):
success = False
for commit in [False, True]:
upd = remove_data
if upd:
k, v = ProtoHelpers.decodeAttributeValueToTypeValue(upd.value)
if upd.fieldname not in self.fields:
raise DapBadUpdateRow("No such field", None, upd.key.core,
upd.fieldname, k)
else:
tbname = self.fields[upd.fieldname]["tablename"]
ftype = self.fields[upd.fieldname]["type"]
if ftype != k:
raise DapBadUpdateRow("Bad type", tbname, upd.key.core,
upd.fieldname, k)
if commit:
success |= self.store[tbname][upd.key.core].pop(
upd.fieldname, None) is not None
return success
def update(self, update_data: dap_update_pb2.DapUpdate.TableFieldValue):
print("GOT UPDATE", update_data)
for commit in [False, True]:
upd = update_data
if upd:
k, v = ProtoHelpers.decodeAttributeValueToTypeValue(upd.value)
if upd.fieldname not in self.fields:
raise DapBadUpdateRow("No such field", None, upd.key.core,
upd.fieldname, k)
else:
tbname = self.fields[upd.fieldname]["tablename"]
ftype = self.fields[upd.fieldname]["type"]
if ftype != k:
raise DapBadUpdateRow("Bad type", tbname, upd.key.core,
upd.fieldname, k)
if commit:
self.store.setdefault(tbname,
{}).setdefault(upd.key.core,
{})[upd.fieldname] = v
def update(
self, tfv: dap_update_pb2.DapUpdate.TableFieldValue
) -> dap_interface_pb2.Successfulness:
r = dap_interface_pb2.Successfulness()
r.success = True
if tfv:
row_key = (tfv.key.core, tfv.key.agent)
core_ident, agent_ident = row_key
field_type, field_value = ProtoHelpers.decodeAttributeValueInfoToPythonType(
tfv.value)
if field_type == "key-type-value_list":
row_present = row_key in self.table
duplicate_inserted = False
new_inserted = False
for k, t, v in field_value:
target = self.table.setdefault(row_key, {})
new_value = DapAttributeStore.StoredFieldValue(ft=t, fv=v)
if k in target and target[k] == new_value:
self.stats["fields_add_duplicate"] += 1
duplicate_inserted = True
else:
self.stats["fields_added"] += 1
new_inserted = True
target[k] = new_value
if not row_present and (not duplicate_inserted
and new_inserted):
self.stats["rows_added"] += 1
elif row_present and (duplicate_inserted and not new_inserted):
self.stats["row_add_duplicate"] += 1
else:
target_field_name = tfv.fieldname
if target_field_name[0:5] != 'them.':
target_field_name = 'them.' + target_field_name
#self.log.info("INSERT: core={}, agent={}".format(core_ident, agent_ident))
new_value = DapAttributeStore.StoredFieldValue(ft=field_type,
fv=field_value)
row_present = row_key in self.table
duplicate_inserted = False
new_inserted = False
target = self.table.setdefault(row_key, {})
if target_field_name in target and target[
target_field_name] == new_value:
self.stats["fields_add_duplicate"] += 1
duplicate_inserted = True
else:
self.stats["fields_added"] += 1
new_inserted = True
target[target_field_name] = new_value
if not row_present and (not duplicate_inserted
and new_inserted):
self.stats["rows_added"] += 1
elif row_present and (duplicate_inserted and not new_inserted):
self.stats["row_add_duplicate"] += 1
agent_parts = tfv.key.agent.decode("UTF-8").split("/")
if len(agent_parts) == 2 and len(agent_parts[1]) > 0:
self._service_alias_store.setdefault(agent_parts[0],
[]).append(tfv.key.agent)
return r
def testFieldOptionsWork(self):
dapManager = DapManager.DapManager()
dapManagerConfig = {
"geo": {
"class": "DapGeo",
"config": {
"structure": {
"wobbles": {
"location": {
"type": "location",
"options": [
"plane",
]
}
},
},
},
},
"attrs": {
"class": "DapAttributeStore",
"config": {
"structure": {},
},
"options": [
"lazy",
],
},
"annotations": {
"class": "InMemoryDap",
"config": {
"structure": {
"annotations": {
"name": "string",
"id": "int64",
},
},
},
},
}
dapManager.setup(sys.modules[__name__], dapManagerConfig)
DATA = [
(
"007/James/Bond",
(51.477, -0.461),
[('id', 1), ('foo', 1)],
), # LHR
(
"White/Spy",
(53.354, -2.275),
[('id', 2), ('foo', 2)],
), # MANCHESTER
(
"Black/Spy",
(50.734, -3.414),
[('id', 3), ('foo', 3)],
), #EXETER
(
"86/Maxwell/Smart",
(40.640, -73.779),
[('id', 4), ('foo', 4)],
), # JFK
]
lat = 0
lon = 0
count = 1
#update = dap_update_pb2.DapUpdate()
#update.update.add()
#update.update[0].fieldname = "location"
#update.update[0].key.core = b'localhost'
#update.update[0].key.agent = agent.encode("utf-8")
#update.update[0].value.l.lat = loc[0]
#update.update[0].value.l.lon = loc[1]
#update.update[0].value.type = 9
#dapManager.update(update)
for agent, loc, attrs in DATA:
update = dap_update_pb2.DapUpdate()
for (k, v) in attrs[0:1]:
u = update.update.add()
u.key.core = b'localhost'
u.key.agent = agent.encode("utf-8")
ProtoHelpers.populateUpdateTFV(u, k, v)
dapManager.update(update)
break
# we're going to check that when the assignments to the "id"
# field were routed, they did not go to the attribute store.
assert dapManager.getInstance("attrs").table == {}
for agent, loc, attrs in DATA:
update = dap_update_pb2.DapUpdate()
for (k, v) in attrs[1:]:
u = update.update.add()
u.key.core = b'localhost'
u.key.agent = agent.encode("utf-8")
ProtoHelpers.populateUpdateTFV(u, k, v)
dapManager.update(update)
break
# But "foo" should have been...
assert dapManager.getInstance("attrs").table != {}