def test_delete_entity_defaults(translator): entities = create_random_entities(num_types=2, num_ids_per_type=2, num_updates=5) translator.insert(entities) _refresh_all(translator, entities) to_delete = entities[0] deleted_type = to_delete['type'] deleted_id = to_delete['id'] n_total = len(translator.query()) assert n_total == 2 * 2 * 5 n_selected = len(translator.query(entity_type=deleted_type, entity_id=deleted_id)) assert n_selected == 5 n_deleted = translator.delete_entity(deleted_id, entity_type=deleted_type) assert n_deleted == 5 _refresh_all(translator, entities) remaining = translator.query() assert len(remaining) == (n_total - n_deleted) survivors = translator.query(entity_type=deleted_type, entity_id=deleted_id) assert len(survivors) == 0
def test_structured_value_to_array(translator): entity = { 'id': '8906', 'type': 'AirQualityObserved', TIME_INDEX_NAME: datetime.now().isoformat()[:-3], 'aqi': {'type': 'Number', 'value': 43}, 'city': {'type': 'Text', 'value': 'Antwerpen'}, 'h': {'type': 'Number', 'value': 93}, 'location': { 'type': 'geo:point', 'value': '51.2056589, 4.4180728', }, 'measurand': { 'type': 'StructuredValue', 'value': ['pm25, 43, ugm3, PM25', 'pm10, 30, ugm3, PM10', 'p, 1012, hPa, Pressure'] }, 'p': {'type': 'Number', 'value': 1012}, 'pm10': {'type': 'Number', 'value': 30}, 'pm25': {'type': 'Number', 'value': 43}, 't': {'type': 'Number', 'value': 8.33} } translator.insert([entity]) translator._refresh([entity['type']]) r = translator.query() assert len(r) == 1
def test_geo_point(translator): # Github issue #35: Support geo:point entity = { 'id': 'Room1', 'type': 'Room', TIME_INDEX_NAME: datetime.now().isoformat()[:-3], 'location': { 'type': 'geo:point', 'value': "19.6389474, -98.9109537" # lat, long } } translator.insert([entity]) translator._refresh([entity['type']]) # Check location is saved as a geo_point column in crate op = 'select latitude(location), longitude(location) from etroom' translator.cursor.execute(op) res = translator.cursor.fetchall() assert len(res) == 1 assert res[0] == [19.6389474, -98.9109537] entities = translator.query() assert len(entities) == 1 # Check entity is retrieved as it was inserted assert_ngsi_entity_equals(entity, entities[0])
def test_query_per_attribute(translator, attr_name, clause, tester): num_types = 1 num_ids_per_type = 2 num_updates = 10 entities = create_random_entities(num_types, num_ids_per_type, num_updates, use_time=True, use_geo=True) translator.insert(entities) translator._refresh(['0']) entities = translator.query(entity_type='0', where_clause="where {} {}".format( attr_name, clause)) total = num_types * num_ids_per_type * num_updates assert len( entities) > 0, "No entities where found with the clause: {}{}".format( attr_name, clause) assert len( entities ) < total, "All entities matched the clause. Not expected from an uniform random distribution" assert all(map(tester, entities))
def test_insert_multiple_types(translator): entities = create_random_entities(num_types=3, num_ids_per_type=2, num_updates=1, use_time=True, use_geo=True) result = translator.insert(entities) assert result.rowcount > 0 # Again to check metadata handling works fine entities = create_random_entities(num_types=3, num_ids_per_type=2, num_updates=1, use_time=True, use_geo=True) result = translator.insert(entities) assert result.rowcount > 0
def test_fiware_tenant_reserved_word(translator): e = entity("Room1") fs = "default" fsp = "/" translator.insert([e], fiware_service=fs, fiware_servicepath=fsp) translator._refresh([e['type']], fiware_service=fs) entities = translator.query(fiware_service=fs, fiware_servicepath=fsp) assert len(entities) == 1
def test_attrs_by_entity_id(translator): # First insert some data num_updates = 10 entities = create_random_entities(1, 2, num_updates, use_time=True, use_geo=True) translator.insert(entities) translator._refresh(['0']) # Now query by entity id entity_id = '0-1' loaded_entities = translator.query(entity_type='0', entity_id=entity_id) assert len(loaded_entities) == num_updates assert all(map(lambda e: e['id'] == entity_id, loaded_entities))
def test_delete_entities_defaults(translator): entities = create_random_entities(num_types=3, num_ids_per_type=2, num_updates=20) translator.insert(entities) _refresh_all(translator, entities) type_to_delete = entities[0]['type'] res = translator.delete_entities(type_to_delete) assert res == 20 * 2 remaining = translator.query() assert len(remaining) == 20 * 2 * (3-1) assert all([r['type'] != type_to_delete for r in remaining])
def test_no_time_index(translator): """ The Reporter is responsible for injecting the 'time_index' attribute to the entity, but even if for some reason the attribute is not there, there should be no problem with the insertion. """ e = { 'id': 'entityId1', 'type': 'type1', 'foo': {'type': 'Text', 'value': "SomeText"} } translator.insert([e]) translator._refresh([e['type']]) assert len(translator.query()) == 1
def test_unsupported_ngsi_type(translator): e = { "type": "SoMeWeIrDtYpE", "id": "sOmEwEiRdId", TIME_INDEX_NAME: datetime.now().isoformat()[:-3], "foo": { "type": "DefinitivelyNotAValidNGSIType", "value": "BaR", }, } translator.insert([e]) translator._refresh([e['type']]) entities = translator.query() assert len(entities) == 1 assert_ngsi_entity_equals(e, entities[0])
def test_fiware_tenant(translator): # Insert WITH tenant e = entity("Room1") fs = "tenant" fsp = "/" translator.insert([e], fiware_service=fs, fiware_servicepath=fsp) translator._refresh([e['type']], fiware_service=fs) # Query NO tenant -> No results entities = translator.query() assert len(entities) == 0 # Query WITH tenant -> Result entities = translator.query(fiware_service=fs, fiware_servicepath=fsp) assert len(entities) == 1
def test_missing_type_defaults_string(translator): e = { "type": "SoMeWeIrDtYpE", "id": "sOmEwEiRdId", TIME_INDEX_NAME: datetime.now().isoformat()[:-3], "foo": { "value": "BaR", }, } translator.insert([e]) translator._refresh([e['type']]) entities = translator.query() assert len(entities) == 1 # Response will include the type e["foo"]["type"] = NGSI_TEXT assert_ngsi_entity_equals(e, entities[0])
def test_average(translator): num_updates = 10 entities = create_random_entities(2, 2, num_updates, use_time=True, use_geo=True) translator.insert(entities) translator._refresh(['0', '1']) # Per entity_id eid = '0-1' entity_mean = statistics.mean(e['attr_float']['value'] for e in entities if e['id'] == eid) entity_mean_read = translator.average(attr_name='attr_float', entity_type='0', entity_id=eid) assert pytest.approx(entity_mean_read) == entity_mean # Total total_mean = statistics.mean(e['attr_float']['value'] for e in entities) total_mean_read = translator.average(attr_name='attr_float') assert pytest.approx(total_mean_read) == total_mean
def test_long_json(translator): # Github issue 44 big_entity = { 'id': 'entityId1', 'type': 'type1', TIME_INDEX_NAME: datetime.now().isoformat()[:-3], 'foo': { 'type': 'Text', 'value': "SomeTextThatWillGetLong" * 2000 } } translator.insert([big_entity]) translator._refresh([big_entity['type']]) r = translator.query() assert len(r) == 1 assert_ngsi_entity_equals(big_entity, r[0])
def test_fiware_tenant_services(translator): # Insert in tenant A e = entity("X") translator.insert([e], fiware_service="A", fiware_servicepath="/") translator._refresh([e['type']], fiware_service="A") # Insert in tenant B e = entity("Y") translator.insert([e], fiware_service="B", fiware_servicepath="/") translator._refresh([e['type']], fiware_service="B") # Query tenant A entities = translator.query(fiware_service="A", fiware_servicepath="/") assert len(entities) == 1 assert entities[0]['id'] == "X" # Query tenant B entities = translator.query(fiware_service="B", fiware_servicepath="/") assert len(entities) == 1 assert entities[0]['id'] == "Y"
def test_fiware_empty_tenant_is_no_tenant(translator): # Insert with EMPTY tenant e = entity("Room1") fs = "" fsp = "" translator.insert([e], fiware_service=fs, fiware_servicepath=fsp) translator._refresh([e['type']], fiware_service=fs) # Query WITHOUT tenant -> get results entities = translator.query() assert len(entities) == 1 # Insert WITHOUT tenant e = entity("Room2") translator.insert([e]) translator._refresh([e['type']]) # Query with EMPTY tenant -> get results entities = translator.query() assert len(entities) == 2
def test_delete_entities_customs(translator): entities = create_random_entities(num_types=4, num_ids_per_type=1, num_updates=4) for i, e in enumerate(entities): time_index = datetime(2018, 1, 1 + i).isoformat()[:-3] e[TIME_INDEX_NAME] = time_index translator.insert(entities) _refresh_all(translator, entities) type_to_delete = entities[-1]['type'] res = translator.delete_entities(type_to_delete, from_date=datetime(2018, 1, 4).isoformat(), to_date=datetime(2018, 1, 12).isoformat()) assert res == 3 _refresh_all(translator, entities) remaining = translator.query() assert len(remaining) == 4 * 4 - 3
def test_traffic_flow_observed(translator, traffic_flow_observed): # Add TIME_INDEX as Reporter would traffic_flow_observed[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3] result = translator.insert([traffic_flow_observed]) assert result.rowcount > 0 translator._refresh([traffic_flow_observed['type']]) loaded = translator.query() assert len(loaded) > 0 assert_ngsi_entity_equals(traffic_flow_observed, loaded[0])
def test_delete_entities_with_tenancy(translator): fs = 'fs' fsp = 'fsp' entities = create_random_entities(num_types=3, num_ids_per_type=1, num_updates=10) translator.insert(entities, fiware_service=fs, fiware_servicepath=fsp) _refresh_all(translator, entities, fiware_service=fs) type_to_delete = entities[0]['type'] res = translator.delete_entities(type_to_delete) assert res == 0 res = translator.delete_entities(type_to_delete, fiware_service=fs, fiware_servicepath='another/path') assert res == 0 res = translator.delete_entities(type_to_delete, fiware_service=fs, fiware_servicepath=fsp) assert res == 10
def test_insert_entity(translator, entity): entity[BaseTranslator.TIME_INDEX_NAME] = datetime.now().isoformat()[:-3] result = translator.insert([entity]) assert result.rowcount == 1 translator._refresh([entity['type']]) loaded_entity = translator.query() # These 2 can be ignored when empty. TODO: #12 Support attribute metadata entity['temperature'].pop('metadata') entity['pressure'].pop('metadata') assert_ngsi_entity_equals(entity, loaded_entity[0])
def test_default_replication(translator): """ By default there should be 2-all replicas https://crate.io/docs/crate/reference/en/latest/general/ddl/replication.html """ entities = create_random_entities(1, 2, 10) entity = entities[0] e_type = entity['type'] translator.insert(entities) translator._refresh([e_type]) op = "select number_of_replicas from information_schema.tables where " \ "table_name = '{}'" translator.cursor.execute(op.format(translator._et2tn(e_type))) res = translator.cursor.fetchall() assert res[0] == ['2-all'] # Metadata table should also be replicated translator.cursor.execute(op.format(METADATA_TABLE_NAME)) res = translator.cursor.fetchall() assert res[0] == ['2-all']
def test_delete_entity_with_tenancy(translator): entities = create_random_entities(num_types=2, num_ids_per_type=2, num_updates=5) fs = 'fs' fsp = 'fsp' translator.insert(entities, fiware_service=fs, fiware_servicepath=fsp) _refresh_all(translator, entities, fiware_service=fs) to_delete = entities[0] deleted_type = to_delete['type'] deleted_id = to_delete['id'] res = translator.delete_entity(deleted_id, entity_type=deleted_type) assert res == 0 res = translator.delete_entity(deleted_id, entity_type=deleted_type, fiware_service=fs) assert res == 0 res = translator.delete_entity(deleted_id, entity_type=deleted_type, fiware_service=fs, fiware_servicepath=fsp) assert res == 5
def test_delete_entity_customs(translator): entities = create_random_entities(num_types=1, num_ids_per_type=2, num_updates=10) for i, e in enumerate(entities): time_index = datetime(2018, 1, 1 + i).isoformat()[:-3] e[TIME_INDEX_NAME] = time_index translator.insert(entities) _refresh_all(translator, entities) to_delete = entities[-1] deleted_type = to_delete['type'] deleted_id = to_delete['id'] res = translator.delete_entity(deleted_id, entity_type=deleted_type, from_date=datetime(2018, 1, 8).isoformat(), to_date=datetime(2018, 1, 16).isoformat()) assert res == 5 _refresh_all(translator, entities) remaining = translator.query() assert len(remaining) == 15
def test_capitals(translator): entity_type = "SoMeWeIrDtYpE" e = { "type": entity_type, "id": "sOmEwEiRdId", TIME_INDEX_NAME: datetime.now().isoformat()[:-3], "Foo": { "type": "Text", "value": "FoO", }, "bAr": { "type": "Text", "value": "bAr", }, } translator.insert([e]) translator._refresh([entity_type]) entities = translator.query() assert len(entities) == 1 assert_ngsi_entity_equals(e, entities[0]) # If a new attribute comes later, I want it translated as well. e2 = e.copy() e2['id'] = 'SOmEwEiRdId2' e2['NewAttr'] = {"type": "Text", "value": "NewAttrValue!"} e2[TIME_INDEX_NAME] = datetime.now().isoformat()[:-3] translator.insert([e2]) translator._refresh([entity_type]) entities = translator.query() assert len(entities) == 2 assert_ngsi_entity_equals(e2, entities[1]) # Note that old entity gets None for the new attribute e['NewAttr'] = {'type': 'Text', 'value': None} assert_ngsi_entity_equals(e, entities[0])
def test_query_all(translator): entities = create_random_entities(2, 2, 2, use_time=True, use_geo=True) result = translator.insert(entities) assert result.rowcount > 0 translator._refresh(['0', '1']) loaded_entities = translator.query() assert len(loaded_entities) == len(entities) key = lambda e: e[BaseTranslator.TIME_INDEX_NAME] a = sorted(entities, key=key) b = sorted(loaded_entities, key=key) for e, le in zip(a, b): assert_ngsi_entity_equals(e, le)
def test_ISO8601(translator): """ ISO8601 should be a valid type, equivalent to DateTime. """ e = { "type": "MyType", "id": "MyId", TIME_INDEX_NAME: datetime.now().isoformat()[:-3], "iso_attr": { "type": "ISO8601", "value": "2018-03-20T13:26:38.722000", }, } result = translator.insert([e]) assert result.rowcount > 0 translator._refresh([e['type']]) loaded = translator.query() assert len(loaded) > 0 assert_ngsi_entity_equals(e, loaded[0])
def test_insert(translator): entities = create_random_entities(1, 2, 10, use_time=True, use_geo=True) result = translator.insert(entities) assert result.rowcount == len(entities)
def insert_with_tenant(e, path): translator.insert([e], fiware_service="EU", fiware_servicepath=path) translator._refresh([e['type']], fiware_service="EU")