def test_structured_value_to_array(translator):
entity = {
'id': '8906',
'type': 'AirQualityObserved',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'aqi': {'type': 'Number', 'value': 43},
'city': {'type': 'Text', 'value': 'Antwerpen'},
'h': {'type': 'Number', 'value': 93},
'location': {
'type': 'geo:point',
'value': '51.2056589, 4.4180728',
},
'measurand': {
'type': 'StructuredValue',
'value': ['pm25, 43, ugm3, PM25', 'pm10, 30, ugm3, PM10',
'p, 1012, hPa, Pressure']
},
'p': {'type': 'Number', 'value': 1012},
'pm10': {'type': 'Number', 'value': 30},
'pm25': {'type': 'Number', 'value': 43},
't': {'type': 'Number', 'value': 8.33}
}
translator.insert([entity])
translator._refresh([entity['type']])
r = translator.query()
assert len(r) == 1
def test_geo_point(translator):
# Github issue #35: Support geo:point
entity = {
'id': 'Room1',
'type': 'Room',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'location': {
'type': 'geo:point',
'value': "19.6389474, -98.9109537" # lat, long
}
}
translator.insert([entity])
translator._refresh([entity['type']])
# Check location is saved as a geo_point column in crate
op = 'select latitude(location), longitude(location) from etroom'
translator.cursor.execute(op)
res = translator.cursor.fetchall()
assert len(res) == 1
assert res[0] == [19.6389474, -98.9109537]
entities = translator.query()
assert len(entities) == 1
# Check entity is retrieved as it was inserted
assert_ngsi_entity_equals(entity, entities[0])
def test_delete_entity(translator):
"""
By default, delete all records of some entity.
"""
insert_test_data()
entity_type = "AirQualityObserved"
params = {
'type': entity_type,
}
url = '{}/entities/{}'.format(QL_URL, entity_type + '0')
# Values are there
r = requests.get(url, params=params)
assert r.status_code == 200, r.text
assert r.text != ''
# Delete them
r = requests.delete(url, params=params)
assert r.status_code == 204, r.text
translator._refresh([entity_type])
# Values are gone
r = requests.get(url, params=params)
assert r.status_code == 404, r.text
# But not for other entities of same type
url = '{}/entities/{}'.format(QL_URL, entity_type + '1')
r = requests.get(url, params=params)
assert r.status_code == 200, r.text
assert r.text != ''
def test_delete_entities(translator):
"""
By default, delete all historical records of all entities of some type.
"""
entity_type = "TrafficFlowObserved"
params = {
'type': entity_type,
}
insert_test_data()
translator._refresh([entity_type, "Room"])
# Values are there for both entities
for e in range(2):
url = '{}/entities/{}'.format(QL_URL, '{}{}'.format(entity_type, e))
r = requests.get(url, params=params)
assert r.status_code == 200, r.text
assert r.text != ''
# 1 Delete call
url = '{}/types/{}'.format(QL_URL, entity_type)
r = requests.delete(url, params=params)
assert r.status_code == 204, r.text
# Values are gone for both entities
for e in range(2):
url = '{}/entities/{}'.format(QL_URL, '{}{}'.format(entity_type, e))
r = requests.get(url, params=params)
assert r.status_code == 404, r.text
# But not for entities of other types
url = '{}/entities/{}'.format(QL_URL, 'Room1')
r = requests.get(url, params={'type': 'Room'})
assert r.status_code == 200, r.text
assert r.text != ''
def test_query_per_attribute(translator, attr_name, clause, tester):
num_types = 1
num_ids_per_type = 2
num_updates = 10
entities = create_random_entities(num_types,
num_ids_per_type,
num_updates,
use_time=True,
use_geo=True)
translator.insert(entities)
translator._refresh(['0'])
entities = translator.query(entity_type='0',
where_clause="where {} {}".format(
attr_name, clause))
total = num_types * num_ids_per_type * num_updates
assert len(
entities) > 0, "No entities where found with the clause: {}{}".format(
attr_name, clause)
assert len(
entities
) < total, "All entities matched the clause. Not expected from an uniform random distribution"
assert all(map(tester, entities))
def test_fiware_tenant_reserved_word(translator):
e = entity("Room1")
fs = "default"
fsp = "/"
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
entities = translator.query(fiware_service=fs, fiware_servicepath=fsp)
assert len(entities) == 1
def test_traffic_flow_observed(translator, traffic_flow_observed):
# Add TIME_INDEX as Reporter would
traffic_flow_observed[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
result = translator.insert([traffic_flow_observed])
assert result.rowcount > 0
translator._refresh([traffic_flow_observed['type']])
loaded = translator.query()
assert len(loaded) > 0
assert_ngsi_entity_equals(traffic_flow_observed, loaded[0])
def test_attrs_by_entity_id(translator):
# First insert some data
num_updates = 10
entities = create_random_entities(1, 2, num_updates, use_time=True, use_geo=True)
translator.insert(entities)
translator._refresh(['0'])
# Now query by entity id
entity_id = '0-1'
loaded_entities = translator.query(entity_type='0', entity_id=entity_id)
assert len(loaded_entities) == num_updates
assert all(map(lambda e: e['id'] == entity_id, loaded_entities))
def test_insert_entity(translator, entity):
entity[BaseTranslator.TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
result = translator.insert([entity])
assert result.rowcount == 1
translator._refresh([entity['type']])
loaded_entity = translator.query()
# These 2 can be ignored when empty. TODO: #12 Support attribute metadata
entity['temperature'].pop('metadata')
entity['pressure'].pop('metadata')
assert_ngsi_entity_equals(entity, loaded_entity[0])
def test_no_time_index(translator):
"""
The Reporter is responsible for injecting the 'time_index' attribute to the
entity, but even if for some reason the attribute is not there, there
should be no problem with the insertion.
"""
e = {
'id': 'entityId1',
'type': 'type1',
'foo': {'type': 'Text', 'value': "SomeText"}
}
translator.insert([e])
translator._refresh([e['type']])
assert len(translator.query()) == 1
def test_unsupported_ngsi_type(translator):
e = {
"type": "SoMeWeIrDtYpE",
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"foo": {
"type": "DefinitivelyNotAValidNGSIType",
"value": "BaR",
},
}
translator.insert([e])
translator._refresh([e['type']])
entities = translator.query()
assert len(entities) == 1
assert_ngsi_entity_equals(e, entities[0])
def test_query_all(translator):
entities = create_random_entities(2, 2, 2, use_time=True, use_geo=True)
result = translator.insert(entities)
assert result.rowcount > 0
translator._refresh(['0', '1'])
loaded_entities = translator.query()
assert len(loaded_entities) == len(entities)
key = lambda e: e[BaseTranslator.TIME_INDEX_NAME]
a = sorted(entities, key=key)
b = sorted(loaded_entities, key=key)
for e, le in zip(a, b):
assert_ngsi_entity_equals(e, le)
def test_fiware_tenant(translator):
# Insert WITH tenant
e = entity("Room1")
fs = "tenant"
fsp = "/"
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
# Query NO tenant -> No results
entities = translator.query()
assert len(entities) == 0
# Query WITH tenant -> Result
entities = translator.query(fiware_service=fs, fiware_servicepath=fsp)
assert len(entities) == 1
def test_average(translator):
num_updates = 10
entities = create_random_entities(2, 2, num_updates, use_time=True, use_geo=True)
translator.insert(entities)
translator._refresh(['0', '1'])
# Per entity_id
eid = '0-1'
entity_mean = statistics.mean(e['attr_float']['value'] for e in entities if e['id'] == eid)
entity_mean_read = translator.average(attr_name='attr_float', entity_type='0', entity_id=eid)
assert pytest.approx(entity_mean_read) == entity_mean
# Total
total_mean = statistics.mean(e['attr_float']['value'] for e in entities)
total_mean_read = translator.average(attr_name='attr_float')
assert pytest.approx(total_mean_read) == total_mean
def test_missing_type_defaults_string(translator):
e = {
"type": "SoMeWeIrDtYpE",
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"foo": {
"value": "BaR",
},
}
translator.insert([e])
translator._refresh([e['type']])
entities = translator.query()
assert len(entities) == 1
# Response will include the type
e["foo"]["type"] = NGSI_TEXT
assert_ngsi_entity_equals(e, entities[0])
def test_long_json(translator):
# Github issue 44
big_entity = {
'id': 'entityId1',
'type': 'type1',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'foo': {
'type': 'Text',
'value': "SomeTextThatWillGetLong" * 2000
}
}
translator.insert([big_entity])
translator._refresh([big_entity['type']])
r = translator.query()
assert len(r) == 1
assert_ngsi_entity_equals(big_entity, r[0])
def test_ISO8601(translator):
"""
ISO8601 should be a valid type, equivalent to DateTime.
"""
e = {
"type": "MyType",
"id": "MyId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"iso_attr": {
"type": "ISO8601",
"value": "2018-03-20T13:26:38.722000",
},
}
result = translator.insert([e])
assert result.rowcount > 0
translator._refresh([e['type']])
loaded = translator.query()
assert len(loaded) > 0
assert_ngsi_entity_equals(e, loaded[0])
def test_fiware_empty_tenant_is_no_tenant(translator):
# Insert with EMPTY tenant
e = entity("Room1")
fs = ""
fsp = ""
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
# Query WITHOUT tenant -> get results
entities = translator.query()
assert len(entities) == 1
# Insert WITHOUT tenant
e = entity("Room2")
translator.insert([e])
translator._refresh([e['type']])
# Query with EMPTY tenant -> get results
entities = translator.query()
assert len(entities) == 2
def test_fiware_tenant_services(translator):
# Insert in tenant A
e = entity("X")
translator.insert([e], fiware_service="A", fiware_servicepath="/")
translator._refresh([e['type']], fiware_service="A")
# Insert in tenant B
e = entity("Y")
translator.insert([e], fiware_service="B", fiware_servicepath="/")
translator._refresh([e['type']], fiware_service="B")
# Query tenant A
entities = translator.query(fiware_service="A", fiware_servicepath="/")
assert len(entities) == 1
assert entities[0]['id'] == "X"
# Query tenant B
entities = translator.query(fiware_service="B", fiware_servicepath="/")
assert len(entities) == 1
assert entities[0]['id'] == "Y"
def test_default_replication(translator):
"""
By default there should be 2-all replicas
https://crate.io/docs/crate/reference/en/latest/general/ddl/replication.html
"""
entities = create_random_entities(1, 2, 10)
entity = entities[0]
e_type = entity['type']
translator.insert(entities)
translator._refresh([e_type])
op = "select number_of_replicas from information_schema.tables where " \
"table_name = '{}'"
translator.cursor.execute(op.format(translator._et2tn(e_type)))
res = translator.cursor.fetchall()
assert res[0] == ['2-all']
# Metadata table should also be replicated
translator.cursor.execute(op.format(METADATA_TABLE_NAME))
res = translator.cursor.fetchall()
assert res[0] == ['2-all']
def test_capitals(translator):
entity_type = "SoMeWeIrDtYpE"
e = {
"type": entity_type,
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"Foo": {
"type": "Text",
"value": "FoO",
},
"bAr": {
"type": "Text",
"value": "bAr",
},
}
translator.insert([e])
translator._refresh([entity_type])
entities = translator.query()
assert len(entities) == 1
assert_ngsi_entity_equals(e, entities[0])
# If a new attribute comes later, I want it translated as well.
e2 = e.copy()
e2['id'] = 'SOmEwEiRdId2'
e2['NewAttr'] = {"type": "Text", "value": "NewAttrValue!"}
e2[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
translator.insert([e2])
translator._refresh([entity_type])
entities = translator.query()
assert len(entities) == 2
assert_ngsi_entity_equals(e2, entities[1])
# Note that old entity gets None for the new attribute
e['NewAttr'] = {'type': 'Text', 'value': None}
assert_ngsi_entity_equals(e, entities[0])
def _refresh_all(translator, entities, fiware_service=None):
types = set([e['type'] for e in entities])
translator._refresh(types, fiware_service)
def insert_with_tenant(e, path):
translator.insert([e], fiware_service="EU", fiware_servicepath=path)
translator._refresh([e['type']], fiware_service="EU")
