def test_creation_dynamic_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_dynamic_tags/test_dynamic_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
ivA = IsacValue(nA,
uri,
dynamic_tags=dynamic_tags,
survey_last_value=False,
survey_static_tags=False)
assert ivA.tags == dynamic_tags
ivB = IsacValue(nB, uri, survey_static_tags=False)
# Did not get stored because the default timestamps (0) were equal in both cases
assert ivB.tags == {}
ivB = None
ivA.value = randint(0, 100)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.tags == dynamic_tags_full
assert ivB.value == ivA.value
assert ivB.timestamp == ivA.timestamp
assert ivB.tags == dynamic_tags_full
def test_creation_no_init(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_no_init/test_no_init'
ivA = IsacValue(nA, uri, survey_static_tags=False)
assert ivA.value is None
assert ivA.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivA.static_tags == {}
assert ivA.tags == {}
assert ivA.metadata is None
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.value is None
assert ivB.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivB.static_tags == {}
assert ivB.tags == {}
assert ivB.metadata is None
ivB = None
ivA.value = randint(0, 100)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.value == ivA.value
assert ivB.timestamp == ivA.timestamp
assert ivB.static_tags == {}
assert ivB.tags == nA.name_uuid()
assert ivB.metadata is None
def test_observer_metadata(two_nodes):
nA, nB = two_nodes
obs = Observer()
try:
nB.transport.join_event() # Necesarry, but not user friendly
uri = 'test://test_isac_value/test_observer_metadata/test_observer'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_last_value=False, survey_static_tags=False)
ivB.metadata_observers += obs.observer
ivA.metadata = {'this': 'is', 'meta': 'data'}
for i in range(10):
green.sleep(0.5)
if obs.args is not None:
break
assert obs.args, 'Callback not received'
iv_recv, metadata, source_peer = obs.args
assert iv_recv == ivB
assert metadata == ivA.metadata
assert source_peer['peer_name'] == nA.name
assert source_peer['peer_uuid'] == str(nA.transport.uuid())
finally:
nB.transport.leave_event()
def test_property_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_tags/test_property_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
dynamic_tags2 = {'this': 'is', 'dynamic': 'tags', 'additional': 'tag'}
dynamic_tags2_full = dynamic_tags2
dynamic_tags2_full.update(nA.name_uuid())
ivA = IsacValue(nA,
uri,
randint(0, 100),
dynamic_tags=dict(dynamic_tags),
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.tags == dynamic_tags_full
assert ivB.tags == dynamic_tags_full
ivA.tags['additional'] = 'tag'
ivA.value = randint(0, 100)
assert ivA.tags == dynamic_tags2_full
assert ivB.tags == dynamic_tags2_full
def test_precision_by_config_ms(config, clean_db, one_node):
t_start = datetime.now()
iv = IsacValue(one_node, 'test://test_precision/test_precision_by_config_ms/test', survey_last_value=False, survey_static_tags=False)
try:
config['config']['default_ts_precision'] = 'ms'
archiver = arch.InfluxDBArchiver(config)
expected_history = []
def _save_point():
value, ts, tags = iv.value_ts_tags
ts = degrade_time(ts, precision='ms')
expected_history.append((value, ts, tags))
base_ts = datetime.now()
while base_ts.microsecond > 500:
green.sleep(0.0001)
base_ts = datetime.now()
iv.value_ts = randint(0, 100), base_ts
iv.value_ts = randint(0, 100), base_ts + timedelta(microseconds=20)
_save_point()
iv.value_ts = randint(0, 100), base_ts + timedelta(microseconds=1000)
_save_point()
green.sleep(0.5)
t_end = datetime.now()
data = iv.get_history((t_start, t_end))
assert data == expected_history
finally:
archiver.shutdown()
config['config']['default_ts_precision'] = arch.DEFAULT_TS_PRECISION
def test_observer_metadata(two_nodes): # noqa: F811
nA, nB = two_nodes
obs = Observer()
try:
nB.transport.join_event() # Necesarry, but not user friendly
uri = 'test://test_isac_value/test_observer_metadata/test_observer'
ivA = IsacValue(nA,
uri,
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB,
uri,
survey_last_value=False,
survey_static_tags=False)
ivB.metadata_observers += obs.observer
green.sleep(0.25)
ivA.metadata = {'this': 'is', 'meta': 'data'}
for i in range(10):
green.sleep(0.5)
if obs.args is not None:
break
assert obs.args, 'Callback not received'
iv_recv, metadata, source_peer = obs.args
assert iv_recv == ivB
assert metadata == ivA.metadata
assert source_peer['peer_name'] == nA.name
assert source_peer['peer_uuid'] == str(nA.transport.uuid())
finally:
nB.transport.leave_event()
def test_update_value_tags(config, root_client, clean_db, one_node):
iv = IsacValue(one_node, 'test://test_update/test_update_value_tags/test', static_tags={'static': 'tag'}, survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
iv.value = randint(0, 100)
green.sleep(0.25)
stored_values = read_data(config, root_client, 'SELECT * FROM /.*/ GROUP BY authority, path ORDER BY time DESC LIMIT 1')
for uri, points in stored_values.items():
assert uri == iv.uri
assert points[0]['value_int'] == iv.value
compare_time(points[0]['time'], iv.timestamp)
assert points[0]['d_peer_name'] == iv.isac_node.transport.name()
assert points[0]['d_peer_uuid'] == str(iv.isac_node.transport.uuid())
assert points[0]['s_static'] == iv.static_tags['static']
iv.tags['test'] = str(randint(0, 100))
iv.value = randint(0, 100)
green.sleep(0.25)
stored_values = read_data(config, root_client, 'SELECT * FROM /.*/ GROUP BY authority, path ORDER BY time DESC LIMIT 1')
for uri, points in stored_values.items():
assert uri == iv.uri
assert points[0]['value_int'] == iv.value
compare_time(points[0]['time'], iv.timestamp)
assert points[0]['d_test'] == iv.tags['test']
assert points[0]['d_peer_name'] == iv.isac_node.transport.name()
assert points[0]['d_peer_uuid'] == str(iv.isac_node.transport.uuid())
assert points[0]['s_static'] == iv.static_tags['static']
finally:
archiver.shutdown()
def test_property_value_ts_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_value_ts_tags/test_property_value_ts_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
dynamic_tags2 = {'this': 'is', 'dynamic': 'tags', 'additional': 'tag'}
dynamic_tags2_full = dynamic_tags2
dynamic_tags2_full.update(nA.name_uuid())
ivA = IsacValue(nA,
uri,
randint(0, 100),
dynamic_tags=dynamic_tags,
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value_ts_tags == (ivA.value, ivA.timestamp, dynamic_tags_full)
assert ivB.value_ts_tags == (ivA.value, ivA.timestamp, dynamic_tags_full)
v = ivA.value + 10
ts = ivA.timestamp + timedelta(hours=1)
ivA.value_ts_tags = v, ts, dynamic_tags2
assert ivA.value_ts_tags == (v, ts, dynamic_tags2_full)
assert ivB.value_ts_tags == (v, ts, dynamic_tags2_full)
def test_survey_values_metadata(two_nodes): # noqa: F811
nA, nB = two_nodes
uri_nometa = 'test://test_survey_value/test_survey_values_metadata/iv_nometa'
iv_nometa = IsacValue(nA, uri_nometa, survey_last_value=False, survey_static_tags=False)
uri_meta1 = 'test://test_survey_value/test_survey_value_metadata/iv_meta1'
iv_meta1 = IsacValue(
nA, uri_meta1, metadata={'a': 1, 'b': 2}, survey_last_value=False, survey_static_tags=False)
uri_meta2 = 'test://test_survey_value/test_survey_value_metadata/iv_meta2'
iv_meta2 = IsacValue(
nA, uri_meta2, metadata={'c': 3, 'd': 4}, survey_last_value=False, survey_static_tags=False)
assert nB.survey_values_metadata([uri_meta1]) == {uri_meta1: iv_meta1.metadata}
assert nB.survey_values_metadata([uri_meta1, uri_meta2]) == {
uri_meta1: iv_meta1.metadata,
uri_meta2: iv_meta2.metadata,
}
assert nB.survey_values_metadata('.*meta.$', is_re=True) == {
uri_meta1: iv_meta1.metadata,
uri_meta2: iv_meta2.metadata,
}
assert nB.survey_values_metadata('.*nometa$', is_re=True, timeout=0.1) == {}
assert nB.survey_values_metadata(
'test://test_survey_value/test_survey_value_metadata/unknown', timeout=0.1) == {}
# Wrong RE should return an empty set AND not cause remote nodes to crash
assert nB.survey_values_metadata('*', is_re=True, timeout=0.1) == {}
assert nB.survey_values_metadata(
[uri_meta1]) == {uri_meta1: iv_meta1.metadata}, 'Remote node crashed'
def test_creation_no_init(two_nodes):
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_no_init/test_no_init'
ivA = IsacValue(nA, uri, survey_static_tags=False)
assert ivA.value is None
assert ivA.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivA.static_tags == {}
assert ivA.tags == {}
assert ivA.metadata is None
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.value is None
assert ivB.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivB.static_tags == {}
assert ivB.tags == {}
assert ivB.metadata is None
ivB = None
ivA.value = randint(0, 100)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.value == ivA.value
assert ivB.timestamp == ivA.timestamp
assert ivB.static_tags == {}
assert ivB.tags == nA.name_uuid()
assert ivB.metadata is None
def test_precision_by_config_u(config, clean_db, one_node):
t_start = datetime.now()
iv = IsacValue(one_node, 'test://test_precision/test_precision_by_config_u/test', survey_last_value=False, survey_static_tags=False)
try:
config['config']['default_ts_precision'] = 'u'
archiver = arch.InfluxDBArchiver(config)
expected_history = []
def _save_point():
expected_history.append(iv.value_ts_tags)
base_ts = datetime.now()
iv.value_ts = randint(0, 100), base_ts
_save_point() # When the us happen to be the same, it take the first value, not the last one like for other precisions
iv.value_ts = randint(0, 100), base_ts
iv.value_ts = randint(0, 100), base_ts + timedelta(microseconds=1)
_save_point()
green.sleep(0.5)
t_end = datetime.now()
data = iv.get_history((t_start, t_end))
assert data == expected_history
finally:
archiver.shutdown()
config['config']['default_ts_precision'] = arch.DEFAULT_TS_PRECISION
def test_publish_all_last_values_tags_metadata(config, one_node):
'''
Full alive->dead->alive cycle:
- Create a node
- Create Isac Values
- Create archiver (and archive current values)
- Stop node/values
- Stop archiver
- Recreate archiver (load from DB the last values + tags + metadata <<< What we want to test)
- Recreate node
- Recreate values
- Finally, assert values, tags and metadata
'''
ivs = {}
def _make_iv(uri):
iv = IsacValue(one_node, uri, static_tags={'nb': uri[-1]}, metadata={'leaf': uri[-3:], 'list': list('123456789'), 'dict': dict(zip(list('ABCDEFGHI'), list('123456789')))}, survey_last_value=False, survey_static_tags=False)
iv.value = randint(0, 100)
ivs[uri] = iv.value, iv.timestamp, iv.tags, iv.static_tags, iv.metadata
_make_iv('test://test_init/test_publish_all_last_values_tags_metadata/iv1')
_make_iv('test://test_init/test_publish_all_last_values_tags_metadata/iv2')
try:
archiver = arch.InfluxDBArchiver(config)
one_node.shutdown()
one_node = None
finally:
archiver.shutdown()
try:
archiver = arch.InfluxDBArchiver(config)
try:
one_node = IsacNode('test2')
assert one_node.transport.peers() == [archiver.isac_node.transport.uuid()], 'Seems that too much node are still on the network'
uris = one_node.survey_value_uri('.*')
assert sorted(uris) == sorted(ivs.keys())
for uri in ivs.keys():
iv = IsacValue(one_node, uri)
iv.survey_metadata()
assert iv.value == ivs[uri][0]
compare_time(iv.timestamp, ivs[uri][1])
# print '>>>>>>', uri, iv.value, iv.timestamp, iv.tags, iv.static_tags
# TODO: assert iv.tags == ivs[uri][2] # Original peer name/uuid get squashed by IsacValue because we give it an initial value...
assert iv.static_tags == ivs[uri][3]
assert iv.metadata == ivs[uri][4]
finally:
one_node.shutdown()
finally:
archiver.shutdown()
def test_property_value(two_nodes):
nA,nB = two_nodes
uri = 'test://test_isac_value/test_property_value/test_property_value'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value is None
assert ivB.value is None
v1 = randint(0, 100)
ivA.value = v1
ts1 = ivA.timestamp
assert ivA.value == v1
assert ivB.value == v1
def test_property_value(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_value/test_property_value'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value is None
assert ivB.value is None
v1 = randint(0, 100)
ivA.value = v1
ts1 = ivA.timestamp
assert ivA.value == v1
assert ivB.value == v1
def test_history(two_nodes):
nA, nB = two_nodes
uri = 'test://test_isac_value/test_history/test_history'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = FakeArchivedValue(nB, uri, survey_static_tags=False)
time_period = (0, 20)
data = ivA.get_history(time_period)
data_fixture_converted = [(point[0], datetime.fromtimestamp(point[1]), point[2]) for point in ivB._test_data]
assert ivB._test_time_period, 'History callback not called'
assert ivB._test_time_period == list(time_period)
assert data == data_fixture_converted
with pytest.raises(NoPeerWithHistoryException):
ivB.get_history(time_period)
def test_survey_value_metadata(two_nodes): # noqa: F811
nA, nB = two_nodes
uri_nometa = 'test://test_survey_value/test_survey_value_metadata/iv_nometa'
iv_nometa = IsacValue(nA, uri_nometa, survey_last_value=False, survey_static_tags=False)
uri_meta = 'test://test_survey_value/test_survey_value_metadata/iv_meta'
iv_meta = IsacValue(
nA, uri_meta, metadata={'a': 1, 'b': 2},
survey_last_value=False, survey_static_tags=False
)
assert nB.survey_value_metadata(uri_nometa, timeout=0.1) == (None, None)
assert nB.survey_value_metadata(
'test://test_survey_value/test_survey_value_metadata/unknown', timeout=0.1) == (None, None)
assert nB.survey_value_metadata(uri_meta) == (iv_meta.metadata, nA.name_uuid())
def test_survey_last_value(two_nodes):
nA, nB = two_nodes
uri = 'test://test_survey_value/test_survey_last_value/my_value'
iv = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
# Uninitialised/new value have a None value at time 0
assert nB.survey_last_value(uri, limit_peers=1) == (None, 0, {})
iv.value = randint(0, 100)
assert nB.survey_last_value(uri, limit_peers=1) == (iv.value, iv.timestamp_float, nA.name_uuid())
# Inexistent value are not an error, they simply return None at time 0
# (That is actually what give the default state of an uninitialised/new value behind the scene)
assert nB.survey_last_value('test://test_survey_value/test_survey_last_value/inexistent', limit_peers=1) == (None, 0, {})
def test_weakref(one_node): # noqa: F811
iv = IsacValue(one_node,
'test://test_isac_value/test_weakref/test_iv',
survey_last_value=False,
survey_static_tags=False)
del iv
assert one_node.isac_values.valuerefs() == []
def test_history(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_history/test_history'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = FakeArchivedValue(nB, uri, survey_static_tags=False)
time_period = (0, 20)
data = ivA.get_history(time_period)
data_fixture_converted = [(point[0], datetime.fromtimestamp(point[1]),
point[2]) for point in ivB._test_data]
assert ivB._test_time_period, 'History callback not called'
assert ivB._test_time_period == list(time_period)
assert data == data_fixture_converted
with pytest.raises(NoPeerWithHistoryException):
ivB.get_history(time_period)
def test_property_metadata(two_nodes):
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_metadata/test_property_metadata'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.metadata is None
assert ivB.metadata is None
metadata = {'this': 'is', 'meta': 'data'}
ivA.metadata = metadata
assert ivA.metadata == metadata
assert ivB.metadata is None
ivB.survey_metadata()
assert ivB.metadata == metadata
def test_update_metadata(config, root_client, clean_db, one_node):
iv = IsacValue(one_node, 'test://test_update/test_update_metadata/test', survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
iv.metadata = {'meta': 'data'}
green.sleep(0.25)
stored_values = read_data(config, root_client, 'SELECT * FROM /.*/ GROUP BY authority, path ORDER BY time DESC LIMIT 1')
for uri, points in stored_values.items():
uri = uri.replace('metadata', points[0]['scheme'], 1)
assert uri == iv.uri
assert points[0]['meta'] == iv.metadata['meta']
finally:
archiver.shutdown()
def test_survey_value_static_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri_nostatictags = 'test://test_survey_value/test_survey_value_static_tags/iv_nostatictags'
iv_nostatictags = IsacValue(
nA, uri_nostatictags, survey_last_value=False, survey_static_tags=False)
uri_statictags = 'test://test_survey_value/test_survey_value_static_tags/iv_statictags'
iv_statictags = IsacValue(
nA, uri_statictags, static_tags={'this': 'is', 'static': 'tags'},
survey_last_value=False, survey_static_tags=False
)
assert nB.survey_value_static_tags(uri_nostatictags, timeout=0.1) == {}
assert nB.survey_value_static_tags(
'test://test_survey_value/test_survey_value_static_tags/unknown', timeout=0.1) == {}
assert nB.survey_value_static_tags(uri_statictags) == iv_statictags.static_tags
def test_observer_after_creation(two_nodes):
nA, nB = two_nodes
obs = Observer()
uri = 'test://test_isac_value/test_observer/test_observer'
ivA = IsacValue(nA, uri, static_tags={'this': 'is', 'static': 'tags'}, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri)
ivB.observers += obs.observer
ivA.value = randint(0, 100)
green.sleep(0.5)
assert obs.args, 'Callback not received'
iv_recv, value, ts, tags = obs.args
assert iv_recv == ivB
assert value == ivA.value
assert ts == ivA.timestamp
assert tags == ivA.tags
assert obs.static_tags == ivA.static_tags
def test_creation_static_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_static_tags/test_static_tags'
static_tags = {'this': 'is', 'static': 'tags'}
ivA = IsacValue(nA,
uri,
static_tags=static_tags,
survey_last_value=False,
survey_static_tags=False)
assert ivA.static_tags == static_tags
ivB = IsacValue(nB, uri, survey_last_value=False)
assert ivB.static_tags == static_tags
iv2A = IsacValue(nA, uri + '_dont_exists', survey_last_value=False)
assert iv2A.static_tags == {}
def test_creation_double():
nA = IsacNode('A')
nB = IsacNode('B')
uri = 'test://test_isac_value/test_creation_double/my_value'
try:
ivA = IsacValue(nA,
uri,
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB,
uri,
survey_last_value=False,
survey_static_tags=False)
finally:
nA.shutdown()
nB.shutdown()
def test_property_timestamp(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_timestamp/test_property_timestamp'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivA.timestamp_float == 0
assert ivB.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivB.timestamp_float == 0
v1 = randint(0, 100)
ivA.value = v1
ts1 = ivA.timestamp
assert datetime.fromtimestamp(ivA.timestamp_float) == ts1
assert ivB.value == v1
assert ivB.timestamp == ts1
assert datetime.fromtimestamp(ivB.timestamp_float) == ts1
def test_property_timestamp(two_nodes):
nA,nB = two_nodes
uri = 'test://test_isac_value/test_property_timestamp/test_property_timestamp'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivA.timestamp_float == 0
assert ivB.timestamp == datetime(1970, 1, 1, 0, 0)
assert ivB.timestamp_float == 0
v1 = randint(0, 100)
ivA.value = v1
ts1 = ivA.timestamp
assert datetime.fromtimestamp(ivA.timestamp_float) == ts1
assert ivB.value == v1
assert ivB.timestamp == ts1
assert datetime.fromtimestamp(ivB.timestamp_float) == ts1
def test_property_ts_tags(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_ts_tags/test_property_ts_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
ivA = IsacValue(nA,
uri,
randint(0, 100),
dynamic_tags=dynamic_tags,
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.ts_tags == (ivA.timestamp, dynamic_tags_full)
assert ivB.ts_tags == (ivA.timestamp, dynamic_tags_full)
def test_creation_with_full_init(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_with_full_init/test_with_full_init'
v1 = randint(0, 100)
ts1 = datetime.now() - timedelta(hours=1)
ivA = IsacValue(nA,
uri, (v1, ts1),
survey_last_value=False,
survey_static_tags=False)
assert ivA.value == v1
assert ivA.timestamp == ts1
assert ivA.static_tags == {}
assert ivA.tags == nA.name_uuid()
assert ivA.metadata is None
v2 = v1 + 10
ts2 = datetime.now() - timedelta(hours=2)
ivB = IsacValue(nB, uri, (v2, ts2), survey_static_tags=False)
assert ivB.value == v1
assert ivB.timestamp == ts1
assert ivB.static_tags == {}
assert ivB.tags == nA.name_uuid()
assert ivB.metadata is None
assert ivA.value == v1
assert ivA.timestamp == ts1
assert ivA.static_tags == {}
assert ivA.tags == nA.name_uuid()
assert ivA.metadata is None
ivB = None
v3 = v2 + 10
ts3 = datetime.now()
ivB = IsacValue(nB, uri, (v3, ts3), survey_static_tags=False)
assert ivB.value == v3
assert ivB.timestamp == ts3
assert ivB.static_tags == {}
assert ivB.tags == nB.name_uuid()
assert ivB.metadata is None
assert ivA.value == v3
assert ivA.timestamp == ts3
assert ivA.static_tags == {}
assert ivA.tags == nB.name_uuid()
assert ivA.metadata is None
def test_survey_last_value(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_survey_value/test_survey_last_value/my_value'
iv = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
# Uninitialised/new value have a None value at time 0
assert nB.survey_last_value(uri, limit_peers=1) == (None, 0, {})
iv.value = randint(0, 100)
assert nB.survey_last_value(
uri, limit_peers=1) == (iv.value, iv.timestamp_float, nA.name_uuid())
# Inexistent value are not an error, they simply return None at time 0
# (That is actually what give the default state of an uninitialised/new value behind the scene)
assert nB.survey_last_value(
'test://test_survey_value/test_survey_last_value/inexistent',
limit_peers=1
) == (None, 0, {})
def test_property_value_ts(two_nodes):
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_value_ts/test_property_value_ts'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value_ts == (None, datetime(1970, 1, 1, 0, 0))
assert ivB.value_ts == (None, datetime(1970, 1, 1, 0, 0))
v1 = randint(0, 100)
ts1 = datetime.now() + timedelta(hours=1)
ivA.value_ts = v1, ts1
assert ivA.value_ts == (v1, ts1)
assert ivA.value == v1
assert datetime.fromtimestamp(ivA.timestamp_float) == ts1
assert ivB.value_ts == (v1, ts1)
assert ivB.value == v1
assert ivB.timestamp == ts1
assert datetime.fromtimestamp(ivB.timestamp_float) == ts1
def test_property_value_ts(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_value_ts/test_property_value_ts'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value_ts == (None, datetime(1970, 1, 1, 0, 0))
assert ivB.value_ts == (None, datetime(1970, 1, 1, 0, 0))
v1 = randint(0, 100)
ts1 = datetime.now() + timedelta(hours=1)
ivA.value_ts = v1, ts1
assert ivA.value_ts == (v1, ts1)
assert ivA.value == v1
assert datetime.fromtimestamp(ivA.timestamp_float) == ts1
assert ivB.value_ts == (v1, ts1)
assert ivB.value == v1
assert ivB.timestamp == ts1
assert datetime.fromtimestamp(ivB.timestamp_float) == ts1
def test_survey_value_uri(two_nodes): # noqa: F811
nA, nB = two_nodes
iv1 = IsacValue(nA, 'iv1', survey_last_value=False, survey_static_tags=False)
iv2 = IsacValue(nA, 'iv2', survey_last_value=False, survey_static_tags=False)
assert nB.survey_value_uri('', limit_peers=1) >= set(['iv1', 'iv2'])
assert nB.survey_value_uri('i', limit_peers=1) >= set(['iv1', 'iv2'])
assert nB.survey_value_uri('v', limit_peers=1) >= set(['iv1', 'iv2'])
assert nB.survey_value_uri('1', limit_peers=1) >= set(['iv1'])
assert nB.survey_value_uri('2', limit_peers=1) >= set(['iv2'])
assert nB.survey_value_uri('^i', limit_peers=1) >= set(['iv1', 'iv2'])
assert nB.survey_value_uri('[1|2]$', limit_peers=1) >= set(['iv1', 'iv2'])
assert nB.survey_value_uri('^.*1', limit_peers=1) >= set(['iv1'])
assert nB.survey_value_uri('^v', limit_peers=1, timeout=0.1) == set([])
assert nB.survey_value_uri('nothing', limit_peers=1, timeout=0.1) == set([])
# Wrong RE should return an empty set AND not cause remote nodes to crash
assert nB.survey_value_uri('*', limit_peers=1, timeout=0.1) == set([])
assert nB.survey_value_uri('', limit_peers=1) >= set(['iv1', 'iv2']), 'Remote node crashed'
def run(self):
self.running = True
while self.running:
try:
try:
tag, data = _read_trame(self.ser.readline())
except BadChecksum:
continue
logger.debug('Read %s: %s', tag, data)
if tag not in self.ALLOWED_TAGS:
logger.warning('Discarding %s: %s', tag, data)
continue
try:
signal = self.signals[tag]
except KeyError:
logger.info('Creating ISAC value for %s', tag)
metadata = {
'ts_precision': 's',
'smoothing': True
}
signal = IsacValue(
self.isac_node,
'tic://alidron-tic/%s' % tag,
static_tags={'location': 'entrance.switchboard'},
metadata=metadata,
survey_last_value=False,
survey_static_tags=False
)
self.signals[tag] = signal
try:
signal.value = int(data)
except ValueError:
signal.value = data
except Exception as ex:
logger.error('Hum, something weird: %s', ex)
def test_creation_with_init(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_with_init/test_with_init'
v = randint(0, 100)
ivA = IsacValue(nA,
uri,
v,
survey_last_value=False,
survey_static_tags=False)
assert ivA.value == v
assert ivA.static_tags == {}
assert ivA.tags == nA.name_uuid()
assert ivA.metadata is None
t = ivA.timestamp
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.value == v
assert ivB.timestamp == t
assert ivB.static_tags == {}
assert ivB.tags == nA.name_uuid()
assert ivB.metadata is None
def test_property_value_tags(two_nodes):
nA,nB = two_nodes
uri = 'test://test_isac_value/test_property_value_tags/test_property_value_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
dynamic_tags2 = {'this': 'is', 'dynamic': 'tags', 'additional': 'tag'}
dynamic_tags2_full = dynamic_tags2
dynamic_tags2_full.update(nA.name_uuid())
ivA = IsacValue(nA, uri, randint(0, 100), dynamic_tags=dynamic_tags, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.value_tags == (ivA.value, dynamic_tags_full)
assert ivB.value_tags == (ivA.value, dynamic_tags_full)
v = randint(0, 100)
ivA.value_tags = v, dynamic_tags2
assert ivA.value_tags == (v, dynamic_tags2_full)
assert ivB.value_tags == (v, dynamic_tags2_full)
def test_buffer_on_db_deleted(config, root_client, clean_db, one_node):
db = config['archiver-user']['db']
buffer_path = config['buffer']['path']
iv = IsacValue(one_node, 'test://test_buffer/test_buffer_on_db_deleted/test', survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
base = randint(0, 100)
iv.value = base
green.sleep(0.25)
root_client.drop_database(db)
iv.value += 10
iv.value += 10
green.sleep(0.25)
assert os.path.exists(buffer_path)
with open(buffer_path, 'r') as buffer_r:
assert len(pickle.load(buffer_r)) == 2
archiver._create_db()
iv.value += 10
green.sleep(0.5)
assert not os.path.exists(buffer_path)
stored_values = read_data(config, root_client, 'SELECT * FROM /.*/ GROUP BY authority, path ORDER BY time ASC')
got_values = []
for uri, points in stored_values.items():
assert uri == iv.uri
for point in points:
got_values.append(point['value_int'])
expected_values = [i for i in range(base+10, base+40, 10)]
assert got_values == expected_values
finally:
archiver.shutdown()
def test_creation_dynamic_tags(two_nodes):
nA, nB = two_nodes
uri = 'test://test_isac_value/test_creation_dynamic_tags/test_dynamic_tags'
dynamic_tags = {'this': 'is', 'dynamic': 'tags'}
dynamic_tags_full = dynamic_tags
dynamic_tags_full.update(nA.name_uuid())
ivA = IsacValue(nA, uri, dynamic_tags=dynamic_tags, survey_last_value=False, survey_static_tags=False)
assert ivA.tags == dynamic_tags
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivB.tags == {} # Did not get stored because the default timestamps (0) were equal in both cases
ivB = None
ivA.value = randint(0, 100)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.tags == dynamic_tags_full
assert ivB.value == ivA.value
assert ivB.timestamp == ivA.timestamp
assert ivB.tags == dynamic_tags_full
def test_smoothing_by_metadata(config, clean_db, one_node):
t_start = datetime.now()
ivS = IsacValue(one_node, 'test://test_smoothing/test_smoothing_by_metadata/test_smoothing', metadata={'smoothing': True}, survey_last_value=False, survey_static_tags=False)
ivNS = IsacValue(one_node, 'test://test_smoothing/test_smoothing_by_metadata/test_no_smoothing', survey_last_value=False, survey_static_tags=False)
ivSF = IsacValue(one_node, 'test://test_smoothing/test_smoothing_by_metadata/test_smoothing_false', metadata={'smoothing': False}, survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
ivS_expected_history = []
ivNS_expected_history = []
ivSF_expected_history = []
def _save_point(iv, record):
value, ts, tags = iv.value_ts_tags
ts = degrade_time(ts)
record.append((value, ts, tags))
def _save_all_points():
_save_point(ivS, ivS_expected_history)
_save_point(ivNS, ivNS_expected_history)
_save_point(ivSF, ivSF_expected_history)
def _save_not_smoothed_points():
_save_point(ivNS, ivNS_expected_history)
_save_point(ivSF, ivSF_expected_history)
base = randint(0, 100)
ivS.value = ivNS.value = ivSF.value = base
_save_all_points()
ivS.value = ivNS.value = ivSF.value = ivS.value + 10
_save_all_points()
ivS.value = ivNS.value = ivSF.value = ivS.value
_save_not_smoothed_points()
ivS.value = ivNS.value = ivSF.value = ivS.value
_save_not_smoothed_points()
ivS.value = ivNS.value = ivSF.value = ivS.value
_save_not_smoothed_points()
ivS.value = ivNS.value = ivSF.value = ivS.value
_save_all_points()
ivS.value = ivNS.value = ivSF.value = ivS.value + 10
_save_all_points()
green.sleep(0.5)
t_end = datetime.now()
data_ivS = ivS.get_history((t_start, t_end))
assert data_ivS == ivS_expected_history
data_ivNS = ivNS.get_history((t_start, t_end))
assert data_ivNS == ivNS_expected_history
data_ivSF = ivSF.get_history((t_start, t_end))
assert data_ivSF == ivSF_expected_history
finally:
archiver.shutdown()
def test_observer_at_creation(two_nodes): # noqa: F811
nA, nB = two_nodes
obs = Observer()
uri = 'test://test_isac_value/test_observer_at_creation/test_observer'
ivA = IsacValue(nA,
uri,
randint(0, 100),
static_tags={
'this': 'is',
'static': 'tags'
},
survey_last_value=False,
survey_static_tags=False)
ivB = IsacValue(nB, uri, observers=Observable([obs.observer]))
green.sleep(0.5)
assert obs.args, 'Callback not received'
iv_recv, value, ts, tags = obs.args
assert iv_recv == ivB
assert value == ivA.value
assert ts == ivA.timestamp
assert tags == ivA.tags
assert obs.static_tags == ivA.static_tags
def test_smoothing_by_config(config, clean_db, one_node):
t_start = datetime.now()
iv = IsacValue(one_node, 'test://test_smoothing/test_smoothing_by_config/test', survey_last_value=False, survey_static_tags=False)
try:
config['config']['default_smoothing'] = True
archiver = arch.InfluxDBArchiver(config)
expected_history = []
def _save_point():
value, ts, tags = iv.value_ts_tags
ts = degrade_time(ts)
expected_history.append((value, ts, tags))
base = randint(0, 100)
iv.value = base
_save_point()
iv.value += 10
_save_point()
iv.value = iv.value
iv.value = iv.value
iv.value = iv.value
iv.value = iv.value
_save_point()
iv.value += 10
_save_point()
green.sleep(0.5)
t_end = datetime.now()
data = iv.get_history((t_start, t_end))
assert data == expected_history
finally:
archiver.shutdown()
config['config']['default_smoothing'] = arch.DEFAULT_SMOOTHING
def test_survey_value_history(two_nodes): # noqa: F811
nA, nB = two_nodes
time_period = (time.time()-86400, time.time())
uri_arch = 'test://test_survey_value/test_survey_value_history/archived_iv'
archived_iv = ArchivedValue(nA, uri_arch, survey_last_value=False, survey_static_tags=False)
uri_simple = 'test://test_survey_value/test_survey_value_history/simple_iv'
simple_iv = IsacValue(nA, uri_simple, survey_last_value=False, survey_static_tags=False)
assert nB.survey_value_history(uri_simple, (0, 1000), timeout=0.1) is None
assert nB.survey_value_history(uri_arch, (0, 1000)) == b'testA'
assert nB.survey_value_history(
'test://test_survey_value/test_survey_value_history/unknown',
(0, 1000), timeout=0.1
) is None
def test_get_history(config, clean_db, two_nodes):
nA, nB = two_nodes
t_start = datetime.now()
uri = 'test://test_history/test_get_history/test'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
our_history = []
for i in range(10):
ivA.value = randint(0, 100)
value, ts, tags = ivA.value_ts_tags
ts = degrade_time(ts)
our_history.append((value, ts, tags))
green.sleep(0.5)
t_end = datetime.now()
ivB = IsacValue(nB, uri, survey_last_value=False, survey_static_tags=False)
data = ivB.get_history((t_start, t_end))
assert data == our_history
finally:
archiver.shutdown()
def test_property_metadata(two_nodes): # noqa: F811
nA, nB = two_nodes
uri = 'test://test_isac_value/test_property_metadata/test_property_metadata'
ivA = IsacValue(nA, uri, survey_last_value=False, survey_static_tags=False)
ivB = IsacValue(nB, uri, survey_static_tags=False)
assert ivA.metadata is None
assert ivB.metadata is None
metadata = {'this': 'is', 'meta': 'data'}
ivA.metadata = metadata
assert ivA.metadata == metadata
assert ivB.metadata is None
ivB.survey_metadata()
assert ivB.metadata == metadata
def test_types_all(config, clean_db, one_node):
t_start = datetime.now()
iv = IsacValue(one_node, "test://test_types/test_types_all/test", survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
expected_history = []
def _save_point():
value, ts, tags = iv.value_ts_tags
ts = degrade_time(ts)
expected_history.append((value, ts, tags))
iv.value = randint(0, 100)
_save_point()
iv.value = random() * 100
_save_point()
iv.value = False
_save_point()
iv.value = True
_save_point()
iv.value = "".join(choice(string.ascii_uppercase + string.digits) for _ in range(16))
_save_point()
iv.value = {"a": 1, "b": 2, "c": 3}
_save_point()
t_end = datetime.now()
data = iv.get_history((t_start, t_end))
assert len(data) == len(expected_history)
for got, expected in zip(data, expected_history):
if isinstance(expected[0], float):
assert abs(got[0] - expected[0]) < 1e-10
assert got[1:] == expected[1:]
else:
assert got == expected
finally:
archiver.shutdown()
def _make_iv(uri):
iv = IsacValue(one_node, uri, static_tags={'nb': uri[-1]}, metadata={'leaf': uri[-3:], 'list': list('123456789'), 'dict': dict(zip(list('ABCDEFGHI'), list('123456789')))}, survey_last_value=False, survey_static_tags=False)
iv.value = randint(0, 100)
ivs[uri] = iv.value, iv.timestamp, iv.tags, iv.static_tags, iv.metadata
def _make_iv(node, uri):
ivs[uri] = IsacValue(node, uri, static_tags={'nb': uri[-1]}, metadata={'leaf': uri[-4:]}, survey_last_value=False, survey_static_tags=False)
ivs[uri].value = randint(0, 100)
def test_precision_by_metadata(config, clean_db, one_node):
t_start = datetime.now()
iv_s = IsacValue(one_node, 'test://test_precision/test_precision_by_metadata/test_s', metadata={'ts_precision': 's'}, survey_last_value=False, survey_static_tags=False)
iv_ms = IsacValue(one_node, 'test://test_precision/test_precision_by_metadata/test_ms', metadata={'ts_precision': 'ms'}, survey_last_value=False, survey_static_tags=False)
iv_u = IsacValue(one_node, 'test://test_precision/test_precision_by_metadata/test_u', metadata={'ts_precision': 'u'}, survey_last_value=False, survey_static_tags=False)
try:
archiver = arch.InfluxDBArchiver(config)
iv_s_expected_history = []
iv_ms_expected_history = []
iv_u_expected_history = []
def _save_point(iv, expected_history, precision):
value, ts, tags = iv.value_ts_tags
if precision in ['s', 'ms']:
ts = degrade_time(ts, precision)
expected_history.append((value, ts, tags))
base_ts = datetime.now()
while base_ts.microsecond > 500:
green.sleep(0.0001)
base_ts = datetime.now()
# Second
iv_s.value_ts = randint(0, 100), base_ts
iv_s.value_ts = randint(0, 100), base_ts + timedelta(microseconds=20000)
_save_point(iv_s, iv_s_expected_history, 's')
iv_s.value_ts = randint(0, 100), base_ts + timedelta(seconds=1)
_save_point(iv_s, iv_s_expected_history, 's')
# Millisecond
iv_ms.value_ts = randint(0, 100), base_ts
iv_ms.value_ts = randint(0, 100), base_ts + timedelta(microseconds=20)
_save_point(iv_ms, iv_ms_expected_history, 'ms')
iv_ms.value_ts = randint(0, 100), base_ts + timedelta(microseconds=1000)
_save_point(iv_ms, iv_ms_expected_history, 'ms')
# Microsecond
iv_u.value_ts = randint(0, 100), base_ts
_save_point(iv_u, iv_u_expected_history, 'u') # When the us happen to be the same, it take the first value, not the last one like for other precisions
iv_u.value_ts = randint(0, 100), base_ts
iv_u.value_ts = randint(0, 100), base_ts + timedelta(microseconds=1)
_save_point(iv_u, iv_u_expected_history, 'u')
green.sleep(0.5)
t_end = datetime.now() + timedelta(seconds=10)
data_s = iv_s.get_history((t_start, t_end))
assert data_s == iv_s_expected_history
data_ms = iv_ms.get_history((t_start, t_end))
assert data_ms == iv_ms_expected_history
data_u = iv_u.get_history((t_start, t_end))
assert data_u == iv_u_expected_history
finally:
archiver.shutdown()
