def test_pull(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# first create 2 keys
kv1 = {"key": "test1", "value": "1"}
kv2 = {"key": "test11", "value": "2"}
kvs = [kv1, kv2]
assert engine.push(kvs)
# test basic pull - prefix = true by default
resp = engine.pull(key="test1")
assert len(resp) > 1
# test pull prefix = false -> only this key
resp0 = engine.pull(key="test1", prefix=False)
assert len(resp0) == 1
# test with not existing key
resp = engine.pull(key="test2", prefix=False)
assert len(resp) == 0
# test to pull only keys
resp = engine.pull(key="test1", prefix=False, key_only=True)
assert len(resp) == 1
assert "value" not in resp[0]
# test to retrieve creation rev
rev0 = int(resp0[0]["create_rev"])
resp = engine.pull(key="test1", prefix=False, rev=rev0)
assert len(resp) == 1
# test to retrieve non-existing rev
resp = engine.pull(key="test1", prefix=False, rev=rev0 - 1)
assert len(resp) == 0
def test_revisions(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# first create some keys
kvs = [{"key": "test0", "value": "0"}]
assert engine.push(kvs)
kvs = [{"key": "test1", "value": "1"}]
assert engine.push(kvs)
# on the last revision add one key and modify an old one
kvs = [{"key": "test2", "value": "2"}, {"key": "test1", "value": "2"}]
assert engine.push(kvs)
# retrieve this last revision
last_revision = engine._latest_revision("test/")
assert last_revision > 0
# add another key
kvs = [{"key": "test3", "value": "3"}]
assert engine.push(kvs)
# now check we get 3 keys: test1, test2, test3
resp = engine.pull(key="test", min_rev=last_revision)
assert len(resp) == 3
latest_revision = engine._latest_revision("test/")
# now check we get 2 keys: test1, test2
resp = engine.pull(key="test",
min_rev=last_revision,
max_rev=latest_revision - 1)
assert len(resp) == 2
def test_date_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"canonic": "%Y%m%d"}
validator = DateHandler(key="test", **schema)
try:
result = validator.process("aaa")
except ValueError as e:
assert e.args[
0] == "Date attribute is not complying with the format defined"
try:
result = validator.process("12")
except ValueError as e:
assert e.args[
0] == "Date attribute is not complying with the format defined"
try:
result = validator.process(12)
except ValueError as e:
assert e.args[
0] == "Date attribute is not complying with the format defined"
try:
result = validator.process("2020/02/15")
except ValueError as e:
assert e.args[
0] == "Date attribute is not complying with the format defined"
result = validator.process("20200215")
assert result == "20200215"
result = validator.process("202021")
assert result == "20200201"
def test_config_file_with_ev():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
os.environ["AVISO_CONFIG"] = test_config_folder + "config.yaml"
c = UserConfig()
assert c.debug
assert c.notification_engine.polling_interval == 1
assert c.notification_engine.type == EngineType.ETCD_GRPC
assert c.configuration_engine.type == EngineType.ETCD_GRPC
assert c.auth_type == AuthType.ETCD
assert c.username == "root"
assert c.username_file is None
assert c.notification_engine.port == 8080
assert c.notification_engine.host == "test"
assert c.notification_engine.timeout == 30
assert c.notification_engine.https
assert c.notification_engine.catchup
assert c.notification_engine.service == "aviso/v2"
assert c.configuration_engine.https
assert c.configuration_engine.port == 8080
assert c.configuration_engine.host == "test"
assert c.configuration_engine.max_file_size == 500
assert c.configuration_engine.timeout is None
assert c.quiet
assert c.no_fail
assert c.key_ttl == 10
assert c.remote_schema
assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_default():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
c = UserConfig._create_default_config()
assert not c["debug"]
assert c["notification_engine"]["timeout"] == 60
assert c["notification_engine"]["polling_interval"] == 30
assert c["notification_engine"]["type"] == "etcd_rest"
assert c["notification_engine"]["port"] == 2379
assert c["notification_engine"]["host"] == "localhost"
assert c["notification_engine"]["service"] == "aviso/v1"
assert not c["notification_engine"]["https"]
assert c["notification_engine"]["catchup"]
assert c["configuration_engine"]["timeout"] == 60
assert c["configuration_engine"]["port"] == 2379
assert c["configuration_engine"]["host"] == "localhost"
assert not c["configuration_engine"]["https"]
assert c["configuration_engine"]["max_file_size"] == 500
assert c["configuration_engine"]["type"] == "etcd_rest"
assert not c["quiet"]
assert not c["no_fail"]
assert c["key_file"] == os.path.join(SYSTEM_FOLDER, KEY_FILE)
assert c["username_file"] is None
assert c["username"] is None
assert c["auth_type"] == "none"
assert c["key_ttl"] == -1
assert c["schema_parser"] == "generic"
assert not c["remote_schema"]
def test_adding_listeners(conf, schema):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
aviso = listener_manager.ListenerManager()
authenticator = auth.Auth.get_auth(conf)
engine_factory: ef.EngineFactory = ef.EngineFactory(
conf.notification_engine, authenticator)
# create a listener that uses that trigger
eng = engine_factory.create_engine()
request = {"country": "italy", "date": 20210101}
listener1 = EventListener("flight", eng, request, [{
"type": "Log"
}], schema)
listener2 = EventListener("flight", eng, request, [{
"type": "Log"
}], schema)
listeners: list = [listener1, listener2]
aviso._add_listeners(listeners)
assert aviso.listeners.__len__() == 2
# now delete them
aviso.cancel_listeners()
assert aviso.listeners.__len__() == 0
def test_constructor():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create a config with passing the configuration file as well as the parameters. The parameters will take priority
notification_engine = {
"host": "localhost",
"port": 2379,
"type": "ETCD_REST",
"polling_interval": 60,
"timeout": 10,
"https": False,
"service": "aviso/v4",
"catchup": False
}
configuration_engine = {
"host": "localhost",
"port": 2379,
"type": "ETCD_REST",
"max_file_size": 200,
"timeout": 10,
"https": False
}
c = UserConfig(conf_path=test_config_folder + "config.yaml",
notification_engine=notification_engine,
configuration_engine=configuration_engine,
debug=False,
no_fail=False,
quiet=False,
username="******",
key_file="tests/unit/fixtures/bad/key",
auth_type="ecmwf",
key_ttl=30,
remote_schema=True,
schema_parser="ecmwf")
assert not c.debug
assert c.notification_engine.polling_interval == 60
assert c.notification_engine.type == EngineType.ETCD_REST
assert c.configuration_engine.type == EngineType.ETCD_REST
assert c.auth_type == AuthType.ECMWF
assert c.username == "test2"
assert c.username_file is None
assert c.notification_engine.port == 2379
assert c.notification_engine.host == "localhost"
assert c.notification_engine.timeout == 10
assert c.notification_engine.service == "aviso/v4"
assert not c.notification_engine.https
assert not c.notification_engine.catchup
assert not c.configuration_engine.https
assert c.configuration_engine.port == 2379
assert c.configuration_engine.host == "localhost"
assert c.configuration_engine.max_file_size == 200
assert c.configuration_engine.timeout == 10
assert not c.quiet
assert not c.no_fail
assert c.key_ttl == 30
assert c.remote_schema
assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_save_delete_state(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# first save state revision
last_rev = 10
assert engine._save_last_revision(last_rev)
assert engine._last_saved_revision() == last_rev
# delete it
engine._delete_saved_revision()
assert engine._last_saved_revision() == -1
def test_env_variables():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
os.environ["AVISO_NOTIFICATION_HOST"] = "test_env"
os.environ["AVISO_NOTIFICATION_PORT"] = "3"
os.environ["AVISO_NOTIFICATION_HTTPS"] = "True"
os.environ["AVISO_NOTIFICATION_CATCHUP"] = "False"
os.environ["AVISO_NOTIFICATION_ENGINE"] = "ETCD_GRPC"
os.environ["AVISO_NOTIFICATION_HTTPS"] = "True"
os.environ["AVISO_NOTIFICATION_SERVICE"] = "aviso/v3"
os.environ["AVISO_CONFIGURATION_HTTPS"] = "True"
os.environ["AVISO_CONFIGURATION_ENGINE"] = "ETCD_GRPC"
os.environ["AVISO_CONFIGURATION_HOST"] = "test_env"
os.environ["AVISO_CONFIGURATION_PORT"] = "3"
os.environ["AVISO_DEBUG"] = "True"
os.environ["AVISO_QUIET"] = "True"
os.environ["AVISO_NO_FAIL"] = "True"
os.environ["AVISO_USERNAME"] = "******"
os.environ["AVISO_KEY_FILE"] = "tests/unit/fixtures/bad/key"
os.environ["AVISO_POLLING_INTERVAL"] = "3"
os.environ["AVISO_MAX_FILE_SIZE"] = "300"
os.environ["AVISO_TIMEOUT"] = "null"
os.environ["AVISO_AUTH_TYPE"] = "etcd"
os.environ["AVISO_KEY_TTL"] = "20"
os.environ["AVISO_USERNAME_FILE"] = "tests/unit/fixtures/username"
os.environ["AVISO_REMOTE_SCHEMA"] = "true"
os.environ["AVISO_SCHEMA_PARSER"] = "ecmwf"
# create a config with the configuration file but the environment variables take priority
c = UserConfig()
assert c.debug
assert c.notification_engine.polling_interval == 3
assert c.notification_engine.type == EngineType.ETCD_GRPC
assert c.configuration_engine.type == EngineType.ETCD_GRPC
assert c.auth_type == AuthType.ETCD
assert c.username == "test_user"
assert c.username_file == "tests/unit/fixtures/username"
assert c.notification_engine.port == 3
assert c.notification_engine.host == "test_env"
assert c.notification_engine.timeout is None
assert c.notification_engine.https
assert not c.notification_engine.catchup
assert c.notification_engine.service == "aviso/v3"
assert c.configuration_engine.https
assert c.configuration_engine.port == 3
assert c.configuration_engine.host == "test_env"
assert c.configuration_engine.max_file_size == 300
assert c.configuration_engine.timeout is None
assert c.quiet
assert c.no_fail
assert c.key_ttl == 20
assert c.remote_schema
assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_bad_format(conf: user_config.UserConfig, schema):
logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create the notification listener factory
authenticator = auth.Auth.get_auth(conf)
engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator)
listener_factory = elf.EventListenerFactory(engine_factory, schema)
# open the listener yaml file
with open("tests/unit/fixtures/bad/badFormat.yaml", "r") as f:
listeners_dict = yaml.safe_load(f.read())
# parse it
try:
listeners: list = listener_factory.create_listeners(listeners_dict)
except ValueError as e:
assert e.args[0] == "Value 2021-01-01 is not valid for key date"
def test_regex_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"regex": "a"}
validator = RegexHandler(key="test", **schema)
try:
result = validator.process("bbb")
except ValueError as e:
assert e.args[0] == "Value bbb is not valid for key test"
result = validator.process("aaa")
assert result == 'aaa'
def test_no_listeners(conf: user_config.UserConfig, schema):
logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create the notification listener factory
authenticator = auth.Auth.get_auth(conf)
engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator)
listener_factory = elf.EventListenerFactory(engine_factory, schema)
# open the listener yaml file
with open("tests/unit/fixtures/bad/noListeners.yaml", "r") as f:
listeners_dict = yaml.safe_load(f.read())
# parse it
try:
listeners: list = listener_factory.create_listeners(listeners_dict)
except AssertionError as e:
assert e.args[0] == "Event listeners definition must start with the keyword 'listeners'"
def test_bad_trigger_type(conf: user_config.UserConfig, schema):
logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create the notification listener factory
authenticator = auth.Auth.get_auth(conf)
engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator)
listener_factory = elf.EventListenerFactory(engine_factory, schema)
# open the listener yaml file
with open("tests/unit/fixtures/bad/badTriggerType.yaml", "r") as f:
listeners_dict = yaml.safe_load(f.read())
# parse it
try:
listeners: list = listener_factory.create_listeners(listeners_dict)
except KeyError as e:
assert e.args[0] == "Trigger type logger not recognised"
def test_multiple_listener(conf: user_config.UserConfig, schema):
logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create the notification listener factory
authenticator = auth.Auth.get_auth(conf)
engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator)
listener_factory = elf.EventListenerFactory(engine_factory, schema)
# open the listener yaml file
with open("tests/unit/fixtures/multiple_flight_listeners.yaml", "r") as f:
listeners_dict = yaml.safe_load(f.read())
# parse it
listeners: list = listener_factory.create_listeners(listeners_dict)
assert listeners.__len__() == 3
for listener in listeners:
assert listener.keys is not None
assert listener.keys[0] # this will fail if the path was an empty string
def test_string_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"canonic": "upper"}
validator = StringHandler(key="test", **schema)
result = validator.process("aaa")
assert result == 'AAA'
schema = {"canonic": "uppercase"}
validator = StringHandler(key="test", **schema)
try:
result = validator.process("aaa")
except AttributeError as e:
assert e.args[0] == "Case uppercase not recognised"
def test_enum_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"values": ["1", "2", "3"]}
validator = EnumHandler(key="test", **schema)
try:
result = validator.process("11")
except ValueError as e:
assert e.args[0] == "Key test accepts only the following values: 1,2,3"
try:
result = validator.process(1)
except ValueError as e:
assert e.args[0] == "Key test is not of a valid type"
result = validator.process("1")
assert result == "1"
def test_float_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"canonic": "{:2.2f}"}
validator = FloatHandler(key="test", **schema)
try:
result = validator.process("aaa")
except ValueError as e:
assert e.args[0] == "Key test has to be a float"
try:
result = validator.process("12")
except ValueError as e:
assert e.args[0] == "Key test has to be a float"
result = validator.process(12.33243254323)
assert result == '12.33'
result = validator.process(12)
assert result == '12.00'
def test_listen_old_state(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
callback_list = []
def callback(key, value):
callback_list.append(1)
# listen to a test key from no state
assert engine.listen(["test"], callback)
# wait a fraction and check the function has been triggered
time.sleep(0.5)
assert len(callback_list) == 0
# create independent change to the test key to trigger the notification
kvs = [{"key": "test1", "value": "1"}]
assert engine.push(kvs)
# wait a fraction and check the function has been triggered
time.sleep(2)
assert len(callback_list) == 1
# stop listening
resp = engine.stop()
assert resp
# repeat the push operation
kvs = [{"key": "test1", "value": "2"}]
assert engine.push(kvs)
kvs = [{"key": "test2", "value": "2"}]
assert engine.push(kvs)
# wait a fraction and check the function has NOT been triggered
time.sleep(2)
assert len(callback_list) == 1
# listen again
assert engine.listen(["test"], callback)
# wait a fraction and check the function has been triggered
time.sleep(2)
# check the state has been used to retrieved the key updates while we were not listening
assert len(callback_list) == 3
def test_int_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"range": [0, 20], "canonic": "{0:0>4}", "error": "error"}
try:
validator = IntHandler(key="test", **schema)
except TypeError as e:
assert e.args[
0] == "__init__() got an unexpected keyword argument 'error'"
schema = {"range": [0, 20], "canonic": "{0:0>4}"}
validator = IntHandler(key="test", **schema)
try:
result = validator.process("aaa")
except ValueError as e:
assert e.args[0] == "Key test has to be an integer"
try:
result = validator.process("12")
except ValueError as e:
assert e.args[0] == "Key test has to be an integer"
result = validator.process(12.33)
assert result == '0012'
result = validator.process(12)
assert result == '0012'
try:
result = validator.process(25)
except ValueError as e:
assert e.args[
0] == "Value 25 for key test is outside the range defined"
try:
result = validator.process(-2)
except ValueError as e:
assert e.args[
0] == "Value -2 for key test is outside the range defined"
def test_time_handler():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
schema = {"values": ["0", "6", "12", "18"], "canonic": "{0:0>2}"}
validator = TimeHandler(key="test", **schema)
try:
result = validator.process("11")
except ValueError as e:
assert e.args[
0] == "Key test accepts only the following values: 0,6,12,18"
try:
result = validator.process(12)
except ValueError as e:
assert e.args[0] == "Key test is not of a valid type"
result = validator.process("12")
assert result == "12"
result = validator.process("0")
assert result == "00"
def test_listen(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
callback_list = []
def callback(key, value):
callback_list.append(1)
# listen to a test key
assert engine.listen(["test"], callback)
# wait a fraction and check the function has been triggered
time.sleep(0.5)
# create independent change to the test key to trigger the notification
kvs = [{"key": "test1", "value": "1"}]
assert engine.push(kvs)
# wait a fraction and check the function has been triggered
time.sleep(2)
assert len(callback_list) == 1
# repeat the push operation
kvs = [{"key": "test1", "value": "2"}]
assert engine.push(kvs)
# wait a fraction and check the function has been triggered
time.sleep(2)
assert len(callback_list) == 2
# stop listening
resp = engine.stop()
assert resp
# repeat the push operation
kvs = [{"key": "test1", "value": "2"}]
assert engine.push(kvs)
# wait a fraction and check the function has NOT been triggered
time.sleep(2)
assert len(callback_list) == 2
def test_push_delete(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
# create 2 kvs and pushed them
kv1 = {"key": "test1", "value": "1"}
kv2 = {"key": "test2", "value": "2"}
kvs = [kv1, kv2]
resp = engine.push(kvs)
assert resp
# verify they exist
resp = engine.pull(key="test")
assert len(resp) == 2
# modify one and delete the other
kv1 = {"key": "test1", "value": "3"}
kvs = [kv1]
kvs_delete = ["test2"]
resp = engine.push(kvs, kvs_delete)
assert resp
# verify there is only one now
resp = engine.pull(key="test")
assert len(resp) == 1
def test_constructor_with_env_var():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
os.environ["AVISO_NOTIFICATION_HOST"] = "test_env"
os.environ["AVISO_NOTIFICATION_PORT"] = "3"
os.environ["AVISO_NOTIFICATION_HTTPS"] = "True"
os.environ["AVISO_NOTIFICATION_CATCHUP"] = "True"
os.environ["AVISO_NOTIFICATION_SERVICE"] = "aviso/v3"
os.environ["AVISO_CONFIGURATION_HTTPS"] = "True"
os.environ["AVISO_CONFIGURATION_HOST"] = "test_env"
os.environ["AVISO_CONFIGURATION_PORT"] = "3"
os.environ["AVISO_DEBUG"] = "True"
os.environ["AVISO_QUIET"] = "True"
os.environ["AVISO_NO_FAIL"] = "True"
os.environ["AVISO_USERNAME"] = "******"
os.environ["AVISO_KEY_FILE"] = "tests/unit/fixtures/bad/key"
os.environ["AVISO_NOTIFICATION_ENGINE"] = "ETCD_GRPC"
os.environ["AVISO_CONFIGURATION_ENGINE"] = "ETCD_GRPC"
os.environ["AVISO_POLLING_INTERVAL"] = "3"
os.environ["AVISO_MAX_FILE_SIZE"] = "300"
os.environ["AVISO_TIMEOUT"] = "20"
os.environ["AVISO_AUTH_TYPE"] = "etcd"
os.environ["AVISO_KEY_TTL"] = "20"
os.environ["AVISO_USERNAME_FILE"] = "tests/unit/fixtures/username"
os.environ["AVISO_REMOTE_SCHEMA"] = "false"
os.environ["AVISO_SCHEMA_PARSER"] = "generic"
# create a config with passing the configuration file as well as the parameters. The parameters will take priority
notification_engine = {
"host": "localhost",
"port": 2379,
"type": "ETCD_REST",
"polling_interval": 60,
"timeout": 10,
"https": False,
"service": "aviso/v4",
"catchup": False
}
configuration_engine = {
"host": "localhost",
"port": 2379,
"type": "ETCD_REST",
"max_file_size": 200,
"timeout": 10,
"https": False
}
c = UserConfig(conf_path=test_config_folder + "config.yaml",
notification_engine=notification_engine,
configuration_engine=configuration_engine,
debug=False,
quiet=False,
no_fail=False,
username="******",
key_file="tests/unit/fixtures/key",
auth_type="ecmwf",
key_ttl=30,
remote_schema=True,
schema_parser="ecmwf")
assert not c.debug
assert c.notification_engine.polling_interval == 60
assert c.notification_engine.type == EngineType.ETCD_REST
assert c.configuration_engine.type == EngineType.ETCD_REST
assert c.auth_type == AuthType.ECMWF
assert c.username == "test_user"
assert c.username_file == "tests/unit/fixtures/username"
assert c.notification_engine.port == 2379
assert c.notification_engine.host == "localhost"
assert c.notification_engine.timeout == 10
assert c.notification_engine.service == "aviso/v4"
assert not c.notification_engine.https
assert not c.notification_engine.catchup
assert not c.configuration_engine.https
assert c.configuration_engine.port == 2379
assert c.configuration_engine.host == "localhost"
assert c.configuration_engine.max_file_size == 200
assert c.configuration_engine.timeout == 10
assert not c.quiet
assert not c.no_fail
assert c.key_ttl == 30
assert c.remote_schema
assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_authenticate(engine):
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
assert engine._authenticate()
