def test_fiware_tenant_servicepath(translator):
def insert_with_tenant(e, path):
translator.insert([e], fiware_service="EU", fiware_servicepath=path)
translator._refresh([e['type']], fiware_service="EU")
# Insert entities with different tenant paths
insert_with_tenant(entity("Rome"), "/eu/italy")
insert_with_tenant(entity("Berlin"), "/eu/germany")
insert_with_tenant(entity("Athens"), "/eu/greece/athens")
insert_with_tenant(entity("Patras"), "/eu/greece/patras")
entities = translator.query(fiware_service="EU", fiware_servicepath="/eu")
assert len(entities) == 4
entities = translator.query(fiware_service="EU",
fiware_servicepath="/eu/germany")
assert len(entities) == 1
entities = translator.query(fiware_service="EU",
fiware_servicepath="/eu/greece")
assert len(entities) == 2
assert set([e['id'] for e in entities]) == set(["Athens", "Patras"])
entities = translator.query(fiware_service="EU",
fiware_servicepath="/eu/g")
assert len(entities) == 0
entities = translator.query(fiware_service="EU",
fiware_servicepath="/eu/greece/athens")
assert len(entities) == 1
def test_delete_entity_defaults(translator):
entities = create_random_entities(num_types=2,
num_ids_per_type=2,
num_updates=5)
translator.insert(entities)
_refresh_all(translator, entities)
to_delete = entities[0]
deleted_type = to_delete['type']
deleted_id = to_delete['id']
n_total = len(translator.query())
assert n_total == 2 * 2 * 5
n_selected = len(translator.query(entity_type=deleted_type,
entity_id=deleted_id))
assert n_selected == 5
n_deleted = translator.delete_entity(deleted_id, entity_type=deleted_type)
assert n_deleted == 5
_refresh_all(translator, entities)
remaining = translator.query()
assert len(remaining) == (n_total - n_deleted)
survivors = translator.query(entity_type=deleted_type, entity_id=deleted_id)
assert len(survivors) == 0
def test_fiware_tenant(translator):
# Insert WITH tenant
e = entity("Room1")
fs = "tenant"
fsp = "/"
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
# Query NO tenant -> No results
entities = translator.query()
assert len(entities) == 0
# Query WITH tenant -> Result
entities = translator.query(fiware_service=fs, fiware_servicepath=fsp)
assert len(entities) == 1
def test_query_per_attribute(translator, attr_name, clause, tester):
num_types = 1
num_ids_per_type = 2
num_updates = 10
entities = create_random_entities(num_types,
num_ids_per_type,
num_updates,
use_time=True,
use_geo=True)
translator.insert(entities)
translator._refresh(['0'])
entities = translator.query(entity_type='0',
where_clause="where {} {}".format(
attr_name, clause))
total = num_types * num_ids_per_type * num_updates
assert len(
entities) > 0, "No entities where found with the clause: {}{}".format(
attr_name, clause)
assert len(
entities
) < total, "All entities matched the clause. Not expected from an uniform random distribution"
assert all(map(tester, entities))
def test_structured_value_to_array(translator):
entity = {
'id': '8906',
'type': 'AirQualityObserved',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'aqi': {'type': 'Number', 'value': 43},
'city': {'type': 'Text', 'value': 'Antwerpen'},
'h': {'type': 'Number', 'value': 93},
'location': {
'type': 'geo:point',
'value': '51.2056589, 4.4180728',
},
'measurand': {
'type': 'StructuredValue',
'value': ['pm25, 43, ugm3, PM25', 'pm10, 30, ugm3, PM10',
'p, 1012, hPa, Pressure']
},
'p': {'type': 'Number', 'value': 1012},
'pm10': {'type': 'Number', 'value': 30},
'pm25': {'type': 'Number', 'value': 43},
't': {'type': 'Number', 'value': 8.33}
}
translator.insert([entity])
translator._refresh([entity['type']])
r = translator.query()
assert len(r) == 1
def test_geocoding(notification, clean_mongo, crate_translator):
# Add an address attribute to the entity
notification['data'][0]['address'] = {
'type': 'StructuredValue',
'value': {
"streetAddress": "Kaivokatu",
"postOfficeBoxNumber": "1",
"addressLocality": "Helsinki",
"addressCountry": "FI",
},
'metadata': {
'dateModified': {
'type': 'DateTime',
'value': '2017-06-19T11:46:45.00Z'
}
}
}
r = requests.post('{}'.format(notify_url),
data=json.dumps(notification),
headers=HEADERS_PUT)
assert r.status_code == 200
assert r.json() == 'Notification successfully processed'
time.sleep(3) # Give time for notification to be processed.
entities = crate_translator.query()
assert len(entities) == 1
assert 'location' in entities[0]
assert entities[0]['location']['type'] == 'geo:point'
lon, lat = entities[0]['location']['value'].split(',')
assert float(lon) == pytest.approx(60.1707129, abs=1e-2)
assert float(lat) == pytest.approx(24.9412167, abs=1e-2)
def test_air_quality_observed(air_quality_observed, orion_client, clean_mongo,
crate_translator):
entity = air_quality_observed
subscription = {
"description": "Test subscription",
"subject": {
"entities": [{
"id": entity['id'],
"type": entity['type']
}],
"condition": {
"attrs": [] # all attributes
}
},
"notification": {
"http": {
"url": notify_url
},
"attrs": [], # all attributes
"metadata": ["dateCreated", "dateModified"]
}
}
orion_client.subscribe(subscription)
orion_client.insert(entity)
import time
time.sleep(3) # Give time for notification to be processed.
entities = crate_translator.query()
# Not exactly one because first insert generates 2 notifications, see...
# https://fiware-orion.readthedocs.io/en/master/user/walkthrough_apiv2/index.html#subscriptions
assert len(entities) > 0
def test_geo_point(translator):
# Github issue #35: Support geo:point
entity = {
'id': 'Room1',
'type': 'Room',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'location': {
'type': 'geo:point',
'value': "19.6389474, -98.9109537" # lat, long
}
}
translator.insert([entity])
translator._refresh([entity['type']])
# Check location is saved as a geo_point column in crate
op = 'select latitude(location), longitude(location) from etroom'
translator.cursor.execute(op)
res = translator.cursor.fetchall()
assert len(res) == 1
assert res[0] == [19.6389474, -98.9109537]
entities = translator.query()
assert len(entities) == 1
# Check entity is retrieved as it was inserted
assert_ngsi_entity_equals(entity, entities[0])
def test_fiware_tenant_reserved_word(translator):
e = entity("Room1")
fs = "default"
fsp = "/"
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
entities = translator.query(fiware_service=fs, fiware_servicepath=fsp)
assert len(entities) == 1
def test_traffic_flow_observed(translator, traffic_flow_observed):
# Add TIME_INDEX as Reporter would
traffic_flow_observed[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
result = translator.insert([traffic_flow_observed])
assert result.rowcount > 0
translator._refresh([traffic_flow_observed['type']])
loaded = translator.query()
assert len(loaded) > 0
assert_ngsi_entity_equals(traffic_flow_observed, loaded[0])
def test_fiware_tenant_services(translator):
# Insert in tenant A
e = entity("X")
translator.insert([e], fiware_service="A", fiware_servicepath="/")
translator._refresh([e['type']], fiware_service="A")
# Insert in tenant B
e = entity("Y")
translator.insert([e], fiware_service="B", fiware_servicepath="/")
translator._refresh([e['type']], fiware_service="B")
# Query tenant A
entities = translator.query(fiware_service="A", fiware_servicepath="/")
assert len(entities) == 1
assert entities[0]['id'] == "X"
# Query tenant B
entities = translator.query(fiware_service="B", fiware_servicepath="/")
assert len(entities) == 1
assert entities[0]['id'] == "Y"
def test_fiware_empty_tenant_is_no_tenant(translator):
# Insert with EMPTY tenant
e = entity("Room1")
fs = ""
fsp = ""
translator.insert([e], fiware_service=fs, fiware_servicepath=fsp)
translator._refresh([e['type']], fiware_service=fs)
# Query WITHOUT tenant -> get results
entities = translator.query()
assert len(entities) == 1
# Insert WITHOUT tenant
e = entity("Room2")
translator.insert([e])
translator._refresh([e['type']])
# Query with EMPTY tenant -> get results
entities = translator.query()
assert len(entities) == 2
def test_attrs_by_entity_id(translator):
# First insert some data
num_updates = 10
entities = create_random_entities(1, 2, num_updates, use_time=True, use_geo=True)
translator.insert(entities)
translator._refresh(['0'])
# Now query by entity id
entity_id = '0-1'
loaded_entities = translator.query(entity_type='0', entity_id=entity_id)
assert len(loaded_entities) == num_updates
assert all(map(lambda e: e['id'] == entity_id, loaded_entities))
def test_no_time_index(translator):
"""
The Reporter is responsible for injecting the 'time_index' attribute to the
entity, but even if for some reason the attribute is not there, there
should be no problem with the insertion.
"""
e = {
'id': 'entityId1',
'type': 'type1',
'foo': {'type': 'Text', 'value': "SomeText"}
}
translator.insert([e])
translator._refresh([e['type']])
assert len(translator.query()) == 1
def test_insert_entity(translator, entity):
entity[BaseTranslator.TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
result = translator.insert([entity])
assert result.rowcount == 1
translator._refresh([entity['type']])
loaded_entity = translator.query()
# These 2 can be ignored when empty. TODO: #12 Support attribute metadata
entity['temperature'].pop('metadata')
entity['pressure'].pop('metadata')
assert_ngsi_entity_equals(entity, loaded_entity[0])
def test_delete_entities_defaults(translator):
entities = create_random_entities(num_types=3,
num_ids_per_type=2,
num_updates=20)
translator.insert(entities)
_refresh_all(translator, entities)
type_to_delete = entities[0]['type']
res = translator.delete_entities(type_to_delete)
assert res == 20 * 2
remaining = translator.query()
assert len(remaining) == 20 * 2 * (3-1)
assert all([r['type'] != type_to_delete for r in remaining])
def test_query_all(translator):
entities = create_random_entities(2, 2, 2, use_time=True, use_geo=True)
result = translator.insert(entities)
assert result.rowcount > 0
translator._refresh(['0', '1'])
loaded_entities = translator.query()
assert len(loaded_entities) == len(entities)
key = lambda e: e[BaseTranslator.TIME_INDEX_NAME]
a = sorted(entities, key=key)
b = sorted(loaded_entities, key=key)
for e, le in zip(a, b):
assert_ngsi_entity_equals(e, le)
def test_unsupported_ngsi_type(translator):
e = {
"type": "SoMeWeIrDtYpE",
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"foo": {
"type": "DefinitivelyNotAValidNGSIType",
"value": "BaR",
},
}
translator.insert([e])
translator._refresh([e['type']])
entities = translator.query()
assert len(entities) == 1
assert_ngsi_entity_equals(e, entities[0])
def test_capitals(translator):
entity_type = "SoMeWeIrDtYpE"
e = {
"type": entity_type,
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"Foo": {
"type": "Text",
"value": "FoO",
},
"bAr": {
"type": "Text",
"value": "bAr",
},
}
translator.insert([e])
translator._refresh([entity_type])
entities = translator.query()
assert len(entities) == 1
assert_ngsi_entity_equals(e, entities[0])
# If a new attribute comes later, I want it translated as well.
e2 = e.copy()
e2['id'] = 'SOmEwEiRdId2'
e2['NewAttr'] = {"type": "Text", "value": "NewAttrValue!"}
e2[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3]
translator.insert([e2])
translator._refresh([entity_type])
entities = translator.query()
assert len(entities) == 2
assert_ngsi_entity_equals(e2, entities[1])
# Note that old entity gets None for the new attribute
e['NewAttr'] = {'type': 'Text', 'value': None}
assert_ngsi_entity_equals(e, entities[0])
def test_missing_type_defaults_string(translator):
e = {
"type": "SoMeWeIrDtYpE",
"id": "sOmEwEiRdId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"foo": {
"value": "BaR",
},
}
translator.insert([e])
translator._refresh([e['type']])
entities = translator.query()
assert len(entities) == 1
# Response will include the type
e["foo"]["type"] = NGSI_TEXT
assert_ngsi_entity_equals(e, entities[0])
def test_long_json(translator):
# Github issue 44
big_entity = {
'id': 'entityId1',
'type': 'type1',
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
'foo': {
'type': 'Text',
'value': "SomeTextThatWillGetLong" * 2000
}
}
translator.insert([big_entity])
translator._refresh([big_entity['type']])
r = translator.query()
assert len(r) == 1
assert_ngsi_entity_equals(big_entity, r[0])
def do_integration(entity, notify_url, orion_client, crate_translator):
entity_id = entity['id']
subscription = {
"description": "Test subscription",
"subject": {
"entities": [{
"id": entity_id,
"type": "Room"
}],
"condition": {
"attrs": [
"temperature",
]
}
},
"notification": {
"http": {
"url": notify_url
},
"attrs": [
"temperature",
],
"metadata": ["dateCreated", "dateModified"]
},
"throttling": 5
}
orion_client.subscribe(subscription)
orion_client.insert(entity)
import time
time.sleep(3) # Give time for notification to be processed.
entities = crate_translator.query()
# Not exactly one because first insert generates 2 notifications, see...
# https://fiware-orion.readthedocs.io/en/master/user/walkthrough_apiv2/index.html#subscriptions
assert len(entities) > 0
# Note: How Quantumleap will return entities is still not well defined.
entities[0].pop(CrateTranslator.TIME_INDEX_NAME)
entity.pop('pressure')
entity['temperature'].pop('metadata')
assert_ngsi_entity_equals(entities[0], entity)
def test_ISO8601(translator):
"""
ISO8601 should be a valid type, equivalent to DateTime.
"""
e = {
"type": "MyType",
"id": "MyId",
TIME_INDEX_NAME: datetime.now().isoformat()[:-3],
"iso_attr": {
"type": "ISO8601",
"value": "2018-03-20T13:26:38.722000",
},
}
result = translator.insert([e])
assert result.rowcount > 0
translator._refresh([e['type']])
loaded = translator.query()
assert len(loaded) > 0
assert_ngsi_entity_equals(e, loaded[0])
def do_integration(entity, notify_url, orion_client, crate_translator):
entity_id = entity['id']
subscription = {
"description": "Integration Test subscription",
"subject": {
"entities": [{
"id": entity_id,
"type": "Room"
}],
"condition": {
"attrs": [
"temperature",
]
}
},
"notification": {
"http": {
"url": notify_url
},
"attrs": [
"temperature",
],
"metadata": ["dateCreated", "dateModified"]
},
"throttling": 1,
}
orion_client.subscribe(subscription)
orion_client.insert(entity)
time.sleep(3) # Give time for notification to be processed.
entities = crate_translator.query()
# Not exactly one because first insert generates 2 notifications, see...
# https://fiware-orion.readthedocs.io/en/master/user/walkthrough_apiv2/index.html#subscriptions
assert len(entities) > 0
# Metadata still not supported
entity['temperature'].pop('metadata')
assert entities[0]['id'] == entity['id']
assert entities[0]['type'] == entity['type']
assert entities[0]['temperature'] == entity['temperature']
def test_delete_entities_customs(translator):
entities = create_random_entities(num_types=4,
num_ids_per_type=1,
num_updates=4)
for i, e in enumerate(entities):
time_index = datetime(2018, 1, 1 + i).isoformat()[:-3]
e[TIME_INDEX_NAME] = time_index
translator.insert(entities)
_refresh_all(translator, entities)
type_to_delete = entities[-1]['type']
res = translator.delete_entities(type_to_delete,
from_date=datetime(2018, 1, 4).isoformat(),
to_date=datetime(2018, 1, 12).isoformat())
assert res == 3
_refresh_all(translator, entities)
remaining = translator.query()
assert len(remaining) == 4 * 4 - 3
def test_delete_entity_customs(translator):
entities = create_random_entities(num_types=1,
num_ids_per_type=2,
num_updates=10)
for i, e in enumerate(entities):
time_index = datetime(2018, 1, 1 + i).isoformat()[:-3]
e[TIME_INDEX_NAME] = time_index
translator.insert(entities)
_refresh_all(translator, entities)
to_delete = entities[-1]
deleted_type = to_delete['type']
deleted_id = to_delete['id']
res = translator.delete_entity(deleted_id, entity_type=deleted_type,
from_date=datetime(2018, 1, 8).isoformat(),
to_date=datetime(2018, 1, 16).isoformat())
assert res == 5
_refresh_all(translator, entities)
remaining = translator.query()
assert len(remaining) == 15
def test_query_all_before_insert(translator):
loaded_entities = translator.query()
assert len(loaded_entities) == 0
