def test_pull(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # first create 2 keys kv1 = {"key": "test1", "value": "1"} kv2 = {"key": "test11", "value": "2"} kvs = [kv1, kv2] assert engine.push(kvs) # test basic pull - prefix = true by default resp = engine.pull(key="test1") assert len(resp) > 1 # test pull prefix = false -> only this key resp0 = engine.pull(key="test1", prefix=False) assert len(resp0) == 1 # test with not existing key resp = engine.pull(key="test2", prefix=False) assert len(resp) == 0 # test to pull only keys resp = engine.pull(key="test1", prefix=False, key_only=True) assert len(resp) == 1 assert "value" not in resp[0] # test to retrieve creation rev rev0 = int(resp0[0]["create_rev"]) resp = engine.pull(key="test1", prefix=False, rev=rev0) assert len(resp) == 1 # test to retrieve non-existing rev resp = engine.pull(key="test1", prefix=False, rev=rev0 - 1) assert len(resp) == 0
def test_revisions(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # first create some keys kvs = [{"key": "test0", "value": "0"}] assert engine.push(kvs) kvs = [{"key": "test1", "value": "1"}] assert engine.push(kvs) # on the last revision add one key and modify an old one kvs = [{"key": "test2", "value": "2"}, {"key": "test1", "value": "2"}] assert engine.push(kvs) # retrieve this last revision last_revision = engine._latest_revision("test/") assert last_revision > 0 # add another key kvs = [{"key": "test3", "value": "3"}] assert engine.push(kvs) # now check we get 3 keys: test1, test2, test3 resp = engine.pull(key="test", min_rev=last_revision) assert len(resp) == 3 latest_revision = engine._latest_revision("test/") # now check we get 2 keys: test1, test2 resp = engine.pull(key="test", min_rev=last_revision, max_rev=latest_revision - 1) assert len(resp) == 2
def test_date_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"canonic": "%Y%m%d"} validator = DateHandler(key="test", **schema) try: result = validator.process("aaa") except ValueError as e: assert e.args[ 0] == "Date attribute is not complying with the format defined" try: result = validator.process("12") except ValueError as e: assert e.args[ 0] == "Date attribute is not complying with the format defined" try: result = validator.process(12) except ValueError as e: assert e.args[ 0] == "Date attribute is not complying with the format defined" try: result = validator.process("2020/02/15") except ValueError as e: assert e.args[ 0] == "Date attribute is not complying with the format defined" result = validator.process("20200215") assert result == "20200215" result = validator.process("202021") assert result == "20200201"
def test_config_file_with_ev(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) os.environ["AVISO_CONFIG"] = test_config_folder + "config.yaml" c = UserConfig() assert c.debug assert c.notification_engine.polling_interval == 1 assert c.notification_engine.type == EngineType.ETCD_GRPC assert c.configuration_engine.type == EngineType.ETCD_GRPC assert c.auth_type == AuthType.ETCD assert c.username == "root" assert c.username_file is None assert c.notification_engine.port == 8080 assert c.notification_engine.host == "test" assert c.notification_engine.timeout == 30 assert c.notification_engine.https assert c.notification_engine.catchup assert c.notification_engine.service == "aviso/v2" assert c.configuration_engine.https assert c.configuration_engine.port == 8080 assert c.configuration_engine.host == "test" assert c.configuration_engine.max_file_size == 500 assert c.configuration_engine.timeout is None assert c.quiet assert c.no_fail assert c.key_ttl == 10 assert c.remote_schema assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_default(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) c = UserConfig._create_default_config() assert not c["debug"] assert c["notification_engine"]["timeout"] == 60 assert c["notification_engine"]["polling_interval"] == 30 assert c["notification_engine"]["type"] == "etcd_rest" assert c["notification_engine"]["port"] == 2379 assert c["notification_engine"]["host"] == "localhost" assert c["notification_engine"]["service"] == "aviso/v1" assert not c["notification_engine"]["https"] assert c["notification_engine"]["catchup"] assert c["configuration_engine"]["timeout"] == 60 assert c["configuration_engine"]["port"] == 2379 assert c["configuration_engine"]["host"] == "localhost" assert not c["configuration_engine"]["https"] assert c["configuration_engine"]["max_file_size"] == 500 assert c["configuration_engine"]["type"] == "etcd_rest" assert not c["quiet"] assert not c["no_fail"] assert c["key_file"] == os.path.join(SYSTEM_FOLDER, KEY_FILE) assert c["username_file"] is None assert c["username"] is None assert c["auth_type"] == "none" assert c["key_ttl"] == -1 assert c["schema_parser"] == "generic" assert not c["remote_schema"]
def test_adding_listeners(conf, schema): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) aviso = listener_manager.ListenerManager() authenticator = auth.Auth.get_auth(conf) engine_factory: ef.EngineFactory = ef.EngineFactory( conf.notification_engine, authenticator) # create a listener that uses that trigger eng = engine_factory.create_engine() request = {"country": "italy", "date": 20210101} listener1 = EventListener("flight", eng, request, [{ "type": "Log" }], schema) listener2 = EventListener("flight", eng, request, [{ "type": "Log" }], schema) listeners: list = [listener1, listener2] aviso._add_listeners(listeners) assert aviso.listeners.__len__() == 2 # now delete them aviso.cancel_listeners() assert aviso.listeners.__len__() == 0
def test_constructor(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create a config with passing the configuration file as well as the parameters. The parameters will take priority notification_engine = { "host": "localhost", "port": 2379, "type": "ETCD_REST", "polling_interval": 60, "timeout": 10, "https": False, "service": "aviso/v4", "catchup": False } configuration_engine = { "host": "localhost", "port": 2379, "type": "ETCD_REST", "max_file_size": 200, "timeout": 10, "https": False } c = UserConfig(conf_path=test_config_folder + "config.yaml", notification_engine=notification_engine, configuration_engine=configuration_engine, debug=False, no_fail=False, quiet=False, username="******", key_file="tests/unit/fixtures/bad/key", auth_type="ecmwf", key_ttl=30, remote_schema=True, schema_parser="ecmwf") assert not c.debug assert c.notification_engine.polling_interval == 60 assert c.notification_engine.type == EngineType.ETCD_REST assert c.configuration_engine.type == EngineType.ETCD_REST assert c.auth_type == AuthType.ECMWF assert c.username == "test2" assert c.username_file is None assert c.notification_engine.port == 2379 assert c.notification_engine.host == "localhost" assert c.notification_engine.timeout == 10 assert c.notification_engine.service == "aviso/v4" assert not c.notification_engine.https assert not c.notification_engine.catchup assert not c.configuration_engine.https assert c.configuration_engine.port == 2379 assert c.configuration_engine.host == "localhost" assert c.configuration_engine.max_file_size == 200 assert c.configuration_engine.timeout == 10 assert not c.quiet assert not c.no_fail assert c.key_ttl == 30 assert c.remote_schema assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_save_delete_state(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # first save state revision last_rev = 10 assert engine._save_last_revision(last_rev) assert engine._last_saved_revision() == last_rev # delete it engine._delete_saved_revision() assert engine._last_saved_revision() == -1
def test_env_variables(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) os.environ["AVISO_NOTIFICATION_HOST"] = "test_env" os.environ["AVISO_NOTIFICATION_PORT"] = "3" os.environ["AVISO_NOTIFICATION_HTTPS"] = "True" os.environ["AVISO_NOTIFICATION_CATCHUP"] = "False" os.environ["AVISO_NOTIFICATION_ENGINE"] = "ETCD_GRPC" os.environ["AVISO_NOTIFICATION_HTTPS"] = "True" os.environ["AVISO_NOTIFICATION_SERVICE"] = "aviso/v3" os.environ["AVISO_CONFIGURATION_HTTPS"] = "True" os.environ["AVISO_CONFIGURATION_ENGINE"] = "ETCD_GRPC" os.environ["AVISO_CONFIGURATION_HOST"] = "test_env" os.environ["AVISO_CONFIGURATION_PORT"] = "3" os.environ["AVISO_DEBUG"] = "True" os.environ["AVISO_QUIET"] = "True" os.environ["AVISO_NO_FAIL"] = "True" os.environ["AVISO_USERNAME"] = "******" os.environ["AVISO_KEY_FILE"] = "tests/unit/fixtures/bad/key" os.environ["AVISO_POLLING_INTERVAL"] = "3" os.environ["AVISO_MAX_FILE_SIZE"] = "300" os.environ["AVISO_TIMEOUT"] = "null" os.environ["AVISO_AUTH_TYPE"] = "etcd" os.environ["AVISO_KEY_TTL"] = "20" os.environ["AVISO_USERNAME_FILE"] = "tests/unit/fixtures/username" os.environ["AVISO_REMOTE_SCHEMA"] = "true" os.environ["AVISO_SCHEMA_PARSER"] = "ecmwf" # create a config with the configuration file but the environment variables take priority c = UserConfig() assert c.debug assert c.notification_engine.polling_interval == 3 assert c.notification_engine.type == EngineType.ETCD_GRPC assert c.configuration_engine.type == EngineType.ETCD_GRPC assert c.auth_type == AuthType.ETCD assert c.username == "test_user" assert c.username_file == "tests/unit/fixtures/username" assert c.notification_engine.port == 3 assert c.notification_engine.host == "test_env" assert c.notification_engine.timeout is None assert c.notification_engine.https assert not c.notification_engine.catchup assert c.notification_engine.service == "aviso/v3" assert c.configuration_engine.https assert c.configuration_engine.port == 3 assert c.configuration_engine.host == "test_env" assert c.configuration_engine.max_file_size == 300 assert c.configuration_engine.timeout is None assert c.quiet assert c.no_fail assert c.key_ttl == 20 assert c.remote_schema assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_bad_format(conf: user_config.UserConfig, schema): logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create the notification listener factory authenticator = auth.Auth.get_auth(conf) engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator) listener_factory = elf.EventListenerFactory(engine_factory, schema) # open the listener yaml file with open("tests/unit/fixtures/bad/badFormat.yaml", "r") as f: listeners_dict = yaml.safe_load(f.read()) # parse it try: listeners: list = listener_factory.create_listeners(listeners_dict) except ValueError as e: assert e.args[0] == "Value 2021-01-01 is not valid for key date"
def test_regex_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"regex": "a"} validator = RegexHandler(key="test", **schema) try: result = validator.process("bbb") except ValueError as e: assert e.args[0] == "Value bbb is not valid for key test" result = validator.process("aaa") assert result == 'aaa'
def test_no_listeners(conf: user_config.UserConfig, schema): logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create the notification listener factory authenticator = auth.Auth.get_auth(conf) engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator) listener_factory = elf.EventListenerFactory(engine_factory, schema) # open the listener yaml file with open("tests/unit/fixtures/bad/noListeners.yaml", "r") as f: listeners_dict = yaml.safe_load(f.read()) # parse it try: listeners: list = listener_factory.create_listeners(listeners_dict) except AssertionError as e: assert e.args[0] == "Event listeners definition must start with the keyword 'listeners'"
def test_bad_trigger_type(conf: user_config.UserConfig, schema): logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create the notification listener factory authenticator = auth.Auth.get_auth(conf) engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator) listener_factory = elf.EventListenerFactory(engine_factory, schema) # open the listener yaml file with open("tests/unit/fixtures/bad/badTriggerType.yaml", "r") as f: listeners_dict = yaml.safe_load(f.read()) # parse it try: listeners: list = listener_factory.create_listeners(listeners_dict) except KeyError as e: assert e.args[0] == "Trigger type logger not recognised"
def test_multiple_listener(conf: user_config.UserConfig, schema): logger.debug(os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create the notification listener factory authenticator = auth.Auth.get_auth(conf) engine_factory: ef.EngineFactory = ef.EngineFactory(conf.notification_engine, authenticator) listener_factory = elf.EventListenerFactory(engine_factory, schema) # open the listener yaml file with open("tests/unit/fixtures/multiple_flight_listeners.yaml", "r") as f: listeners_dict = yaml.safe_load(f.read()) # parse it listeners: list = listener_factory.create_listeners(listeners_dict) assert listeners.__len__() == 3 for listener in listeners: assert listener.keys is not None assert listener.keys[0] # this will fail if the path was an empty string
def test_string_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"canonic": "upper"} validator = StringHandler(key="test", **schema) result = validator.process("aaa") assert result == 'AAA' schema = {"canonic": "uppercase"} validator = StringHandler(key="test", **schema) try: result = validator.process("aaa") except AttributeError as e: assert e.args[0] == "Case uppercase not recognised"
def test_enum_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"values": ["1", "2", "3"]} validator = EnumHandler(key="test", **schema) try: result = validator.process("11") except ValueError as e: assert e.args[0] == "Key test accepts only the following values: 1,2,3" try: result = validator.process(1) except ValueError as e: assert e.args[0] == "Key test is not of a valid type" result = validator.process("1") assert result == "1"
def test_float_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"canonic": "{:2.2f}"} validator = FloatHandler(key="test", **schema) try: result = validator.process("aaa") except ValueError as e: assert e.args[0] == "Key test has to be a float" try: result = validator.process("12") except ValueError as e: assert e.args[0] == "Key test has to be a float" result = validator.process(12.33243254323) assert result == '12.33' result = validator.process(12) assert result == '12.00'
def test_listen_old_state(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) callback_list = [] def callback(key, value): callback_list.append(1) # listen to a test key from no state assert engine.listen(["test"], callback) # wait a fraction and check the function has been triggered time.sleep(0.5) assert len(callback_list) == 0 # create independent change to the test key to trigger the notification kvs = [{"key": "test1", "value": "1"}] assert engine.push(kvs) # wait a fraction and check the function has been triggered time.sleep(2) assert len(callback_list) == 1 # stop listening resp = engine.stop() assert resp # repeat the push operation kvs = [{"key": "test1", "value": "2"}] assert engine.push(kvs) kvs = [{"key": "test2", "value": "2"}] assert engine.push(kvs) # wait a fraction and check the function has NOT been triggered time.sleep(2) assert len(callback_list) == 1 # listen again assert engine.listen(["test"], callback) # wait a fraction and check the function has been triggered time.sleep(2) # check the state has been used to retrieved the key updates while we were not listening assert len(callback_list) == 3
def test_int_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"range": [0, 20], "canonic": "{0:0>4}", "error": "error"} try: validator = IntHandler(key="test", **schema) except TypeError as e: assert e.args[ 0] == "__init__() got an unexpected keyword argument 'error'" schema = {"range": [0, 20], "canonic": "{0:0>4}"} validator = IntHandler(key="test", **schema) try: result = validator.process("aaa") except ValueError as e: assert e.args[0] == "Key test has to be an integer" try: result = validator.process("12") except ValueError as e: assert e.args[0] == "Key test has to be an integer" result = validator.process(12.33) assert result == '0012' result = validator.process(12) assert result == '0012' try: result = validator.process(25) except ValueError as e: assert e.args[ 0] == "Value 25 for key test is outside the range defined" try: result = validator.process(-2) except ValueError as e: assert e.args[ 0] == "Value -2 for key test is outside the range defined"
def test_time_handler(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) schema = {"values": ["0", "6", "12", "18"], "canonic": "{0:0>2}"} validator = TimeHandler(key="test", **schema) try: result = validator.process("11") except ValueError as e: assert e.args[ 0] == "Key test accepts only the following values: 0,6,12,18" try: result = validator.process(12) except ValueError as e: assert e.args[0] == "Key test is not of a valid type" result = validator.process("12") assert result == "12" result = validator.process("0") assert result == "00"
def test_listen(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) callback_list = [] def callback(key, value): callback_list.append(1) # listen to a test key assert engine.listen(["test"], callback) # wait a fraction and check the function has been triggered time.sleep(0.5) # create independent change to the test key to trigger the notification kvs = [{"key": "test1", "value": "1"}] assert engine.push(kvs) # wait a fraction and check the function has been triggered time.sleep(2) assert len(callback_list) == 1 # repeat the push operation kvs = [{"key": "test1", "value": "2"}] assert engine.push(kvs) # wait a fraction and check the function has been triggered time.sleep(2) assert len(callback_list) == 2 # stop listening resp = engine.stop() assert resp # repeat the push operation kvs = [{"key": "test1", "value": "2"}] assert engine.push(kvs) # wait a fraction and check the function has NOT been triggered time.sleep(2) assert len(callback_list) == 2
def test_push_delete(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) # create 2 kvs and pushed them kv1 = {"key": "test1", "value": "1"} kv2 = {"key": "test2", "value": "2"} kvs = [kv1, kv2] resp = engine.push(kvs) assert resp # verify they exist resp = engine.pull(key="test") assert len(resp) == 2 # modify one and delete the other kv1 = {"key": "test1", "value": "3"} kvs = [kv1] kvs_delete = ["test2"] resp = engine.push(kvs, kvs_delete) assert resp # verify there is only one now resp = engine.pull(key="test") assert len(resp) == 1
def test_constructor_with_env_var(): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) os.environ["AVISO_NOTIFICATION_HOST"] = "test_env" os.environ["AVISO_NOTIFICATION_PORT"] = "3" os.environ["AVISO_NOTIFICATION_HTTPS"] = "True" os.environ["AVISO_NOTIFICATION_CATCHUP"] = "True" os.environ["AVISO_NOTIFICATION_SERVICE"] = "aviso/v3" os.environ["AVISO_CONFIGURATION_HTTPS"] = "True" os.environ["AVISO_CONFIGURATION_HOST"] = "test_env" os.environ["AVISO_CONFIGURATION_PORT"] = "3" os.environ["AVISO_DEBUG"] = "True" os.environ["AVISO_QUIET"] = "True" os.environ["AVISO_NO_FAIL"] = "True" os.environ["AVISO_USERNAME"] = "******" os.environ["AVISO_KEY_FILE"] = "tests/unit/fixtures/bad/key" os.environ["AVISO_NOTIFICATION_ENGINE"] = "ETCD_GRPC" os.environ["AVISO_CONFIGURATION_ENGINE"] = "ETCD_GRPC" os.environ["AVISO_POLLING_INTERVAL"] = "3" os.environ["AVISO_MAX_FILE_SIZE"] = "300" os.environ["AVISO_TIMEOUT"] = "20" os.environ["AVISO_AUTH_TYPE"] = "etcd" os.environ["AVISO_KEY_TTL"] = "20" os.environ["AVISO_USERNAME_FILE"] = "tests/unit/fixtures/username" os.environ["AVISO_REMOTE_SCHEMA"] = "false" os.environ["AVISO_SCHEMA_PARSER"] = "generic" # create a config with passing the configuration file as well as the parameters. The parameters will take priority notification_engine = { "host": "localhost", "port": 2379, "type": "ETCD_REST", "polling_interval": 60, "timeout": 10, "https": False, "service": "aviso/v4", "catchup": False } configuration_engine = { "host": "localhost", "port": 2379, "type": "ETCD_REST", "max_file_size": 200, "timeout": 10, "https": False } c = UserConfig(conf_path=test_config_folder + "config.yaml", notification_engine=notification_engine, configuration_engine=configuration_engine, debug=False, quiet=False, no_fail=False, username="******", key_file="tests/unit/fixtures/key", auth_type="ecmwf", key_ttl=30, remote_schema=True, schema_parser="ecmwf") assert not c.debug assert c.notification_engine.polling_interval == 60 assert c.notification_engine.type == EngineType.ETCD_REST assert c.configuration_engine.type == EngineType.ETCD_REST assert c.auth_type == AuthType.ECMWF assert c.username == "test_user" assert c.username_file == "tests/unit/fixtures/username" assert c.notification_engine.port == 2379 assert c.notification_engine.host == "localhost" assert c.notification_engine.timeout == 10 assert c.notification_engine.service == "aviso/v4" assert not c.notification_engine.https assert not c.notification_engine.catchup assert not c.configuration_engine.https assert c.configuration_engine.port == 2379 assert c.configuration_engine.host == "localhost" assert c.configuration_engine.max_file_size == 200 assert c.configuration_engine.timeout == 10 assert not c.quiet assert not c.no_fail assert c.key_ttl == 30 assert c.remote_schema assert c.schema_parser == ListenerSchemaParserType.ECMWF
def test_authenticate(engine): logger.debug( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]) assert engine._authenticate()