def test_delete_entities_defaults(translator):
entities = create_random_entities(num_types=3,
num_ids_per_type=2,
num_updates=20)
translator.insert(entities)
_refresh_all(translator, entities)
type_to_delete = entities[0]['type']
res = translator.delete_entities(type_to_delete)
assert res == 20 * 2
remaining = translator.query()
assert len(remaining) == 20 * 2 * (3-1)
assert all([r['type'] != type_to_delete for r in remaining])
def test_query_multiple_ids_with_invalids(translator):
# Nonexistent ids should be ignored
num_updates = 3
entities = create_random_entities(num_types=2,
num_ids_per_type=4,
num_updates=num_updates)
translator.insert(entities)
loaded_entities = translator.query(entity_type='0',
entity_ids=['nonexistent'])
assert len(loaded_entities) == 0
loaded_entities = translator.query(entity_type='0',
entity_ids=['0-1', 'nonexistent'])
assert len(loaded_entities) == 1
def test_attrs_by_entity_id(translator):
# First insert some data
num_updates = 10
entities = create_random_entities(2,
2,
num_updates,
use_time=True,
use_geo=True)
translator.insert(entities)
# Now query by entity id
entity_id = '1-1'
loaded_entities = list(translator.query(entity_id=entity_id))
assert len(loaded_entities) == num_updates
assert all(map(lambda e: e['id'] == entity_id, loaded_entities))
def test_query_multiple_ids(translator):
# First insert some data
num_updates = 3
entities = create_random_entities(num_types=2,
num_ids_per_type=4,
num_updates=num_updates)
translator.insert(entities)
loaded_entities = translator.query(entity_type='0',
entity_ids=['0-0', '0-2'])
assert len(loaded_entities) == 2
# All results are of type 0 and cannot be of a non-requested id.
assert loaded_entities[0]['id'] == '0-0'
assert loaded_entities[0]['type'] == '0'
assert loaded_entities[1]['id'] == '0-2'
assert loaded_entities[1]['type'] == '0'
def test_aggr_per_second(translator):
entities = create_random_entities(num_ids_per_type=2, num_updates=17)
assert len(entities) == 34
# One update every 100 millis -> 10 updates per second.
base_index = datetime.datetime(2010, 1, 1, 8, 0, 0, 0,
datetime.timezone.utc)
delta = datetime.timedelta(milliseconds=100)
for i, e in enumerate(entities):
t = base_index + i * delta
e[TIME_INDEX_NAME] = t.isoformat(timespec='milliseconds')
add_attr(e, 'attr_float', i)
translator.insert(entities)
# Query avg attr_float per second.
res = translator.query(attr_names=['attr_float'],
aggr_method='avg',
aggr_period='second')
assert len(res) == 2
# 34 values span across 4 seconds
expected_index = []
for i in range(4):
d = datetime.datetime(2010, 1, 1, 8, 0, i, 0, datetime.timezone.utc)
expected_index.append(d.isoformat(timespec='milliseconds'))
assert res[0] == {
'type': '0',
'id': '0-0',
'index': expected_index,
'attr_float': {
'type': 'Number',
'values': [4, 14, 24, 31],
}
}
assert res[1] == {
'type': '0',
'id': '0-1',
'index': expected_index,
'attr_float': {
'type': 'Number',
'values': [5, 15, 25, 32],
}
}
def test_delete_entities_customs(translator):
entities = create_random_entities(num_types=4,
num_ids_per_type=1,
num_updates=4)
for i, e in enumerate(entities):
time_index = datetime(2018, 1, 1 + i).isoformat()[:-3]
e[TIME_INDEX_NAME] = time_index
translator.insert(entities)
type_to_delete = entities[-1]['type']
res = translator.delete_entities(type_to_delete,
from_date=datetime(2018, 1,
4).isoformat(),
to_date=datetime(2018, 1, 12).isoformat())
assert res == 3
remaining = translator.query()
assert sum([len(r['index']) for r in remaining]) == ((4 * 4) - 3)
def test_query_multiple_ids(translator):
# First insert some data
num_updates = 3
entities = create_random_entities(num_types=2,
num_ids_per_type=4,
num_updates=num_updates)
translator.insert(entities)
translator._refresh(['0', '1'])
loaded_entities = translator.query(entity_type='0',
entity_ids=['0-0', '0-2'])
assert len(loaded_entities) == 2 * num_updates
assert any([e['id'] == '0-0' for e in loaded_entities])
assert any([e['id'] == '0-2' for e in loaded_entities])
# All results are of type 0 and cannot be of a non-requested id.
assert all(
map(lambda e: e['id'] in ('0-0', '0-2') and e['type'] == '0',
loaded_entities))
def test_average(translator):
num_updates = 10
entities = create_random_entities(2,
2,
num_updates,
use_time=True,
use_geo=True)
translator.insert(entities)
# Per entity_id
eid = '0-1'
attr_name = 'attr_float'
entity_mean = statistics.mean(e[attr_name]['value'] for e in entities
if e['id'] == eid)
entity_mean_read = translator.average(attr_name=attr_name, entity_id=eid)
assert pytest.approx(entity_mean_read) == entity_mean
# Total
total_mean = statistics.mean(e[attr_name]['value'] for e in entities)
total_mean_read = translator.average(attr_name=attr_name)
assert pytest.approx(total_mean_read) == total_mean
def test_delete_entities_with_tenancy(translator):
fs = 'fs'
fsp = 'fsp'
entities = create_random_entities(num_types=3,
num_ids_per_type=1,
num_updates=10)
translator.insert(entities, fiware_service=fs, fiware_servicepath=fsp)
type_to_delete = entities[0]['type']
res = translator.delete_entities(type_to_delete)
assert res == 0
res = translator.delete_entities(type_to_delete,
fiware_service=fs,
fiware_servicepath='another/path')
assert res == 0
res = translator.delete_entities(type_to_delete,
fiware_service=fs,
fiware_servicepath=fsp)
assert res == 10
def test_default_replication(translator):
"""
By default there should be 2-all replicas
https://crate.io/docs/crate/reference/en/latest/general/ddl/replication.html
"""
entities = create_random_entities(1, 2, 10)
entity = entities[0]
e_type = entity['type']
translator.insert(entities)
op = "select number_of_replicas from information_schema.tables where " \
"table_name = '{}'"
translator.cursor.execute(op.format(translator._et2tn(e_type)))
res = translator.cursor.fetchall()
assert res[0] == ['2-all']
# Metadata table should also be replicated
translator.cursor.execute(op.format(METADATA_TABLE_NAME))
res = translator.cursor.fetchall()
assert res[0] == ['2-all']
def test_delete_entity_with_tenancy(translator):
entities = create_random_entities(num_types=2,
num_ids_per_type=2,
num_updates=5)
fs = 'fs'
fsp = 'fsp'
translator.insert(entities, fiware_service=fs, fiware_servicepath=fsp)
_refresh_all(translator, entities, fiware_service=fs)
to_delete = entities[0]
deleted_type = to_delete['type']
deleted_id = to_delete['id']
res = translator.delete_entity(deleted_id, entity_type=deleted_type)
assert res == 0
res = translator.delete_entity(deleted_id,
entity_type=deleted_type,
fiware_service=fs)
assert res == 0
res = translator.delete_entity(deleted_id,
entity_type=deleted_type,
fiware_service=fs,
fiware_servicepath=fsp)
assert res == 5
def test_delete_entity_customs(translator):
entities = create_random_entities(num_types=1,
num_ids_per_type=2,
num_updates=10)
for i, e in enumerate(entities):
time_index = datetime(2018, 1, 1 + i).isoformat()[:-3]
e[TIME_INDEX_NAME] = time_index
translator.insert(entities)
_refresh_all(translator, entities)
to_delete = entities[-1]
deleted_type = to_delete['type']
deleted_id = to_delete['id']
res = translator.delete_entity(deleted_id, entity_type=deleted_type,
from_date=datetime(2018, 1, 8).isoformat(),
to_date=datetime(2018, 1, 16).isoformat())
assert res == 5
_refresh_all(translator, entities)
remaining = translator.query()
assert len(remaining) == 15
def test_insert(translator):
entities = create_random_entities(2, 2, 10, use_time=True, use_geo=True)
res = translator.insert(entities)
assert res['inserted'] == 2 * 2 * 10
assert res['errors'] == 0
