def test_parser_read_configuration_variable_required_true_value_given_envvar( self): os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable( "TEST_ENVVAR_ESTIMATE_TRUE_INT", required=True) self.assertEqual(reply_json, 123)
def test_parser_read_configuration_variable_envvar_prefix(self): parser = ParseIt(envvar_prefix="prefix_test_", config_folder_location=test_files_location) os.environ["PREFIX_TEST_TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" reply = parser.read_configuration_variable( "test_envvar_estimate_true_int") self.assertEqual(reply, 123)
def test_parser_custom_suffix_mapping_set(self): parser = ParseIt(config_folder_location=test_files_location, custom_suffix_mapping={"yaml": ["custom"]}, config_type_priority=["custom"] + VALID_FILE_TYPE_EXTENSIONS) expected_config_type_priority = ["custom"] + VALID_FILE_TYPE_EXTENSIONS expected_valid_type_extension = [ 'json', 'yaml', 'yml', 'toml', 'tml', 'hcl', 'tf', 'conf', 'cfg', 'ini', 'xml', 'custom' ] expected_suffix_file_type_mapping = { 'json': ['json'], 'yaml': ['yaml', 'yml', 'custom'], 'toml': ['toml', 'tml'], 'hcl': ['hcl', 'tf'], 'ini': ['conf', 'cfg', 'ini'], 'xml': ['xml'] } reply = parser.read_configuration_variable("test_string") self.assertEqual("testing_custom", reply) reply = parser.read_configuration_variable("test_json") self.assertEqual({'test_json_key': 'test_json_value'}, reply) self.assertEqual(expected_config_type_priority, parser.config_type_priority) self.assertEqual(expected_valid_type_extension, parser.valid_file_type_extension) self.assertEqual(expected_suffix_file_type_mapping, parser.suffix_file_type_mapping)
def init(): # read envvars print("reading envvars") parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"]) nebula_host = parser.read_configuration_variable("nebula_host", required=True) nebula_username = parser.read_configuration_variable("nebula_username", default_value=None) nebula_password = parser.read_configuration_variable("nebula_password", default_value=None) nebula_token = parser.read_configuration_variable("nebula_token", default_value=None) nebula_port = parser.read_configuration_variable("nebula_port", default_value=80) nebula_protocol = parser.read_configuration_variable("nebula_protocol", default_value="http") nebula_job_file = parser.read_configuration_variable("nebula_job_file", default_value="nebula.json") nebula_job_type = parser.read_configuration_variable("nebula_job_type", default_value="app") nebula_job_file = os.getcwd() + "/" + nebula_job_file envvar_dict = read_all_envvars_to_dict() # get the job json print("reading nebula job json file") nebula_job_json = read_file(nebula_job_file) # populate the job json with the template data print("populating nebula job json file with the templated data") nebula_job_json = populate_template_string(nebula_job_json, envvar_dict) # create nebula connection object print("contacting nebula API") nebula_connection = NebulaDeploy(host=nebula_host, username=nebula_username, password=nebula_password, token=nebula_token, port=nebula_port, protocol=nebula_protocol) # update nebula if nebula_job_type == "app": nebula_connection.create_or_update_nebula_app(nebula_job_json) elif nebula_job_type == "cron_job": nebula_connection.create_or_update_nebula_cron_job(nebula_job_json) print("finished updating nebula")
def read_configurations(config_folder: str = "config") -> dict: """ Will create a config dict that includes all of the configurations for the autoscaler by aggregating from all valid config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared Arguments: :param config_folder: the folder which all configuration file will be read from recursively Returns: :return config: a dict of all configurations needed for autoscaler to work """ print("reading config variables") config = {} parser = ParseIt(config_location=config_folder, recurse=True) config["kube_token"] = parser.read_configuration_variable( "kube_token", default_value=None) config["kube_api_endpoint"] = parser.read_configuration_variable( "kube_api_endpoint", default_value=None) kubeconfig_file = os.path.expanduser("~/.kube/config") config["kubeconfig_path"] = parser.read_configuration_variable( "kubeconfig_path", default_value=kubeconfig_file) config["kubeconfig_context"] = parser.read_configuration_variable( "kubeconfig_context", default_value=None) config["max_memory_usage"] = parser.read_configuration_variable( "max_memory_usage", default_value=80) config["min_memory_usage"] = parser.read_configuration_variable( "min_memory_usage", default_value=50) config["max_cpu_usage"] = parser.read_configuration_variable( "max_cpu_usage", default_value=80) config["min_cpu_usage"] = parser.read_configuration_variable( "min_cpu_usage", default_value=50) config["seconds_to_check"] = parser.read_configuration_variable( "seconds_to_check", default_value=30) config["spotinst_token"] = parser.read_configuration_variable( "spotinst_token", required=True) config["kube_connection_method"] = decide_kube_connection_method( kube_api_endpoint=config["kube_api_endpoint"], kubeconfig_path=config["kubeconfig_path"]) config["elastigroup_id"] = parser.read_configuration_variable( "elastigroup_id", required=True) config["min_node_count"] = parser.read_configuration_variable( "min_node_count", default_value=2) config["max_node_count"] = parser.read_configuration_variable( "max_node_count", default_value=100) config["spotinst_account"] = parser.read_configuration_variable( "spotinst_account", required=True) config["scale_up_count"] = parser.read_configuration_variable( "scale_up_count", default_value=1) config["scale_down_count"] = parser.read_configuration_variable( "scale_down_count", default_value=1) config["scale_up_active"] = parser.read_configuration_variable( "scale_up_active", default_value=True) config["scale_down_active"] = parser.read_configuration_variable( "scale_down_active", default_value=True) config["scale_on_pending_pods"] = parser.read_configuration_variable( "scale_on_pending_pods", default_value=True) return config
def read_configurations(config_folder: str = "config"): """ Will create a config dict that includes all of the configurations for terraformize by aggregating from all valid config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared Arguments: :param config_folder: the folder which all configuration file will be read from recursively Returns: :return config: a dict of all configurations needed for terraformize to work """ print("reading config variables") config = {} parser = ParseIt(config_location=config_folder, recurse=True) config["basic_auth_user"] = parser.read_configuration_variable( "basic_auth_user", default_value=None) config["basic_auth_password"] = parser.read_configuration_variable( "basic_auth_password", default_value=None) config["auth_token"] = parser.read_configuration_variable( "auth_token", default_value=None) config["terraform_binary_path"] = parser.read_configuration_variable( "terraform_binary_path", default_value=None) config["terraform_modules_path"] = parser.read_configuration_variable( "terraform_modules_path", default_value="/www/terraform_modules") config["auth_enabled"] = auth_enabled(config["basic_auth_user"], config["basic_auth_password"], config["auth_token"]) config["parallelism"] = parser.read_configuration_variable( "parallelism", default_value=10) return config
def _load_configs(self): """Load main configs and module configs.""" for cfile in self.cfiles: try: parser = ParseIt(config_location=cfile, config_type_priority=["yaml"]) config = parser.read_all_configuration_variables() if self.module: if self.module in self.modules: modules = config.pop("modules_activated") self.__setattr__(self.module, Config()) for name, value in modules[self.module].items(): self.__getattribute__(self.module).__setattr__(name, value) else: raise ModuleNotImplemented( "The module `{}` was not found in the modules directory.".format( self.module ) ) for name, value in config.items(): self.__setattr__(name, value) break except FileNotFoundError: pass
def test_parser_read_configuration_variable_type_estimate_true(self): parser = ParseIt(type_estimate=True, config_folder_location=test_files_location) os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_STRING"] = "test" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE"] = "true" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE"] = "false" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_LIST"] = "['test', False, 3]" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_DICT"] = "{'string': 'string', 'int': 1}" reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT") self.assertNotEqual(reply, "123") self.assertEqual(reply, 123) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_STRING") self.assertEqual(reply, "test") reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE") self.assertNotEqual(reply, "true") self.assertEqual(reply, True) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE") self.assertNotEqual(reply, "false") self.assertEqual(reply, False) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_LIST") self.assertNotEqual(reply, "['test', False, 3]") self.assertEqual(reply, ['test', False, 3]) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_DICT") self.assertNotEqual(reply, "{'string': 'string', 'int': 1}") self.assertEqual(reply, {'string': 'string', 'int': 1})
def test_parser_read_configuration_variable_config_folder_location(self): parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable("file_type") self.assertEqual(reply_json, "json") reply_json = parser.read_configuration_variable("test_float") self.assertEqual(reply_json, 123.123) reply_yaml = parser.read_configuration_variable("test_yaml") self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'}) reply_xml = parser.read_configuration_variable("xml_root") expected_reply_xml = { 'file_type': 'xml', 'test_bool_false': False, 'test_bool_true': True, 'test_float': 123.123, 'test_int': 123, 'test_xml': { 'test_xml_key': 'test_xml_value' }, 'test_list': { 'element': [ 'test1', 'test2', 'test3' ] }, 'test_string': 'testing' } self.assertEqual(reply_xml, expected_reply_xml) reply_hcl = parser.read_configuration_variable("test_hcl") expected_reply_hcl = { 'test_hcl_name': { 'test_hcl_key': 'test_hcl_value' } } self.assertEqual(reply_hcl, expected_reply_hcl)
def test_parser_read_configuration_variable_force_envvars_uppercase_false(self): parser = ParseIt(force_envvars_uppercase=False, config_folder_location=test_files_location) os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" os.environ["test_envvar_estimate_true_int"] = "456" reply = parser.read_configuration_variable("test_envvar_estimate_true_int") self.assertNotEqual(reply, 123) self.assertEqual(reply, 456)
def test_parser__parse_file_per_type_ini(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser._parse_file_per_type("ini", test_files_location + "/test.ini") expected_reply = { 'DEFAULT': { 'file_type': 'ini', 'test_string': 'testing', 'test_bool_true': 'true', 'test_bool_false': 'false', 'test_int': '123.0', 'test_float': '123.123', 'test_list': '["test1", "test2", "test3"]' }, 'test_ini': { 'test_ini_key': 'test_ini_value' } } self.assertEqual(reply, expected_reply) reply = parser._parse_file_per_type("conf", test_files_location + "/test.ini") self.assertEqual(reply, expected_reply) reply = parser._parse_file_per_type("cfg", test_files_location + "/test.ini") self.assertEqual(reply, expected_reply)
def init(): # read envvars print("reading envvars") parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"]) metronome_host = parser.read_configuration_variable( "metronome_host", default_value="http://metronome.mesos:9000") metronome_job_file = parser.read_configuration_variable( "metronome_job_file", default_value="metronome.json") metronome_job_file = os.getcwd() + "/" + metronome_job_file envvar_dict = read_all_envvars_to_dict() # get the job json print("reading metronome job json file") metronome_job_json = read_file(metronome_job_file) # populate the job json with the template data print("populating metronome job json file with the templated data") metronome_job_json = populate_template_string(metronome_job_json, envvar_dict) # create/update metronome job print("contacting metronome API") metronome_connection = Metronome(metronome_host) metronome_connection.create_or_update_metronome_job(metronome_job_json) print("finished updating metronome")
def test_parser_read_configuration_recurse_false(self): parser = ParseIt(config_folder_location=test_files_location, recurse=False) reply = parser.read_configuration_variable("test_json_subfolder") expected_config_files_dict = { 'json': [ 'test.json' ], 'yaml': [ 'test.yaml' ], 'yml': [], 'toml': [ 'test.toml' ], 'tml': [], 'conf': [], 'hcl': ['test.hcl'], 'tf': [], 'cfg': [], 'ini': [ 'test.ini' ], 'xml': [ 'test.xml' ] } self.assertEqual(parser.config_files_dict, expected_config_files_dict) self.assertIsNone(reply)
def cli(ctx, verbose, version, config): parse_it = ParseIt(config_location=config) if version: print(__title__, __version__) setup_logging(verbose) ctx.ensure_object(dict) ctx.obj['CA'] = parse_it.read_configuration_variable("CA") ctx.obj['DOMAIN'] = parse_it.read_configuration_variable("DOMAIN")
def test_parser_read_configuration_from_cli_arg(self): testargs = [ "parse_it_mock_script.py", "--test_cli_key", "test_value", "--test_cli_int", "123" ] with mock.patch('sys.argv', testargs): parser = ParseIt() reply = parser.read_configuration_variable("test_cli_key") self.assertEqual(reply, "test_value") reply = parser.read_configuration_variable("test_cli_int") self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_config_type_priority(self): os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" parser = ParseIt(config_type_priority=["yaml", "toml", "ini", "json", "envvars"], config_folder_location=test_files_location) reply = parser.read_configuration_variable("file_type") self.assertEqual(reply, "yaml") reply = parser.read_configuration_variable("test_toml") self.assertEqual(reply, {'test_toml_key': 'test_toml_value'}) reply = parser.read_configuration_variable("test_ini") self.assertEqual(reply, {'test_ini_key': 'test_ini_value'}) reply = parser.read_configuration_variable("test_json") self.assertEqual(reply, {'test_json_key': 'test_json_value'}) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT") self.assertEqual(reply, 123)
def init(): """ Run the logic which will take the variables (OWM api key & location) no matter how they are provided and will alert if it looks like it will rain there tomorrow """ parser = ParseIt(recurse=False, config_type_priority=["envvars"]) owm_api_key = parser.read_configuration_variable("owm_api_key", required=True) city = parser.read_configuration_variable("city", required=True) country_code = parser.read_configuration_variable("country_code", required=True) smtp_server = parser.read_configuration_variable("smtp_server", required=True) sender_email = parser.read_configuration_variable("sender_email", required=True) receiver_email = parser.read_configuration_variable("receiver_email", required=True) email_password = parser.read_configuration_variable("email_password", required=True) email_port = parser.read_configuration_variable("email_port", required=True) telegram_token = parser.read_configuration_variable("telegram_token", required=True) chat_id = parser.read_configuration_variable("chat_id", required=True) owm_object = WeatherForecast(owm_api_key, city, country_code) telegram_object = Telegram(telegram_token) if owm_object.rain_tomorrow() is True and owm_object.storm_tomorrow( ) is True: print("It will be rainy and stormy tomorrow, sending alert") telegram_object.send_alert(chat_id, "stormy and rain") email_alert(smtp_server, sender_email, receiver_email, email_password, "stormy and rain", email_port) print("rain alert sent") elif owm_object.rain_tomorrow() is True: print("It will rain tomorrow, sending alert") telegram_object.send_alert(chat_id, "rain") email_alert(smtp_server, sender_email, receiver_email, email_password, "rain", email_port) print("rain alert sent") elif owm_object.storm_tomorrow() is True: print("It will be a storm tomorrow, sending alert") telegram_object.send_alert(chat_id, "storm") email_alert(smtp_server, sender_email, receiver_email, email_password, "storm", email_port) print("storm alert sent") else: print("It will not rain or be stormy tomorrow")
def test_parser__parse_file_per_type_json(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser._parse_file_per_type("json", test_files_location + "/test.json") expected_reply = { 'file_type': 'json', 'test_string': 'testing', 'test_bool_true': True, 'test_bool_false': False, 'test_int': 123, 'test_float': 123.123, 'test_list': ['test1', 'test2', 'test3'], 'test_json': { 'test_json_key': 'test_json_value' } } self.assertEqual(reply, expected_reply)
def setup(): config_type_priority = [ # 'cli_args', # allow using typer "envvars", "env", "yml", "yaml", "json", "toml", "conf", "cfg", "ini", ] return ParseIt( config_location=CONFIG_PATH, recurse=True, config_type_priority=config_type_priority, )
def test_parser__parse_file_per_type_custom_yaml(self): parser = ParseIt(config_folder_location=test_files_location, custom_suffix_mapping={"yaml": ["custom"]}, config_type_priority=["custom"] + VALID_FILE_TYPE_EXTENSIONS) reply = parser._parse_file_per_type( "custom", test_files_location + "/test.custom") expected_reply = { 'file_type': 'custom_yaml_suffix', 'test_string': 'testing_custom', 'test_bool_true': True, 'test_bool_false': False, 'test_int': 123, 'test_float': 123.123, 'test_list': ['test1', 'test2', 'test3'], 'test_yaml': { 'test_yaml_key': 'custom_test_yaml_value' } } self.assertEqual(reply, expected_reply)
def test_parser__parse_file_per_type_xml(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser._parse_file_per_type("xml", test_files_location + "/test.xml") expected_reply = { 'xml_root': { 'file_type': 'xml', 'test_bool_false': 'false', 'test_bool_true': 'true', 'test_float': '123.123', 'test_int': '123', 'test_xml': { 'test_xml_key': 'test_xml_value' }, 'test_list': { 'element': ['test1', 'test2', 'test3'] }, 'test_string': 'testing' } } self.assertEqual(reply, expected_reply)
def test_parser__parse_file_per_type_hcl(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser._parse_file_per_type("hcl", test_files_location + "/test.hcl") expected_reply = { 'file_type': 'hcl', 'test_string': 'testing', 'test_bool_true': True, 'test_bool_false': False, 'test_int': 123, 'test_float': 123.123, 'test_list': ['test1', 'test2', 'test3'], 'test_hcl': { "test_hcl_name": { 'test_hcl_key': 'test_hcl_value' } } } self.assertEqual(reply, expected_reply) reply = parser._parse_file_per_type("tf", test_files_location + "/test.hcl") self.assertEqual(reply, expected_reply)
def test_parser_init(self): expected_config_files_dict = { 'json': [ 'test.json', 'test_subfolder_1/test_sub_subfolder_2/test_subfolder_2.json', 'test_subfolder_1/test_sub_subfolder_3/test_subfolder_3.json', 'test_subfolder_1/test_subfolder_1.json' ], 'yaml': ['test.yaml'], 'yml': [], 'toml': ['test.toml'], 'tml': [], 'hcl': ['test.hcl'], 'tf': [], 'conf': [], 'cfg': [], 'ini': ['test.ini'], 'xml': ['test.xml'] } expected_config_type_priority = [ 'cli_args', 'env_vars', 'json', 'yaml', 'yml', 'toml', 'tml', 'hcl', 'tf', 'conf', 'cfg', 'ini', 'xml' ] parser = ParseIt(config_type_priority=None, global_default_value=None, type_estimate=True, force_envvars_uppercase=True, config_folder_location=test_files_location, envvar_prefix=None) self.assertEqual(parser.config_files_dict, expected_config_files_dict) self.assertEqual(parser.config_folder_location, test_files_location) self.assertEqual(parser.config_type_priority, expected_config_type_priority) self.assertEqual(parser.envvar_prefix, '') self.assertTrue(parser.force_envvars_uppercase) self.assertIsNone(parser.global_default_value) self.assertTrue(parser.type_estimate)
# retry getting the device_group info @retry(wait_exponential_multiplier=200, wait_exponential_max=1000, stop_max_attempt_number=10) def get_device_group_info(nebula_connection_object, device_group_to_get_info): return nebula_connection_object.list_device_group_info( device_group_to_get_info) if __name__ == "__main__": try: # read config file at startup print("reading config variables") parser = ParseIt(config_location="config", recurse=True) print("reading config variables") # the following config variables are for configuring Nebula workers nebula_manager_auth_user = parser.read_configuration_variable( "nebula_manager_auth_user", default_value=None) nebula_manager_auth_password = parser.read_configuration_variable( "nebula_manager_auth_password", default_value=None) nebula_manager_auth_token = parser.read_configuration_variable( "nebula_manager_auth_token", default_value=None) nebula_manager_host = parser.read_configuration_variable( "nebula_manager_host", default_value=None) nebula_manager_port = parser.read_configuration_variable( "nebula_manager_port", default_value=80) nebula_manager_protocol = parser.read_configuration_variable( "nebula_manager_protocol", default_value="http")
from unittest import TestCase from alert_on_rain.functions.weather_forecast.forecast import * from parse_it import ParseIt parser = ParseIt(recurse=False, config_type_priority=["envvars"]) owm_api_key = parser.read_configuration_variable("owm_api_key", required=True) class BaseTests(TestCase): def test_weather_forecast_init(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") self.assertEqual(owm_object.three_hour_forecast.forecast.interval, "3h") def test_weather_forecast_rain_tomorrow(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") rain_tomorrow = owm_object.rain_tomorrow() self.assertIsInstance(rain_tomorrow, bool) def test_will_be_stormy_tomorrow(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") storm_tomorrow = owm_object.storm_tomorrow() self.assertIsInstance(storm_tomorrow, bool)
def test_parser__parse_file_per_type_wrong_type(self): parser = ParseIt(config_folder_location=test_files_location) with self.assertRaises(ValueError): parser._parse_file_per_type("non_existing_type", test_files_location + "/test.json")
def test_parser_config_found_in_key_false(self): parser = ParseIt(config_folder_location=test_files_location) config_found_reply, config_value_reply = parser._check_config_in_dict("wrong_key", {"test_key": "test_value"}) self.assertFalse(config_found_reply) self.assertEqual(config_value_reply, None)
def test_parser_read_configuration_variable_global_default_value(self): parser = ParseIt(global_default_value="my_last_resort", config_folder_location=test_files_location) reply = parser.read_configuration_variable("this_does_not_exist") self.assertEqual(reply, "my_last_resort")
def test_parser_read_configuration_variable_required_true_value_given_file(self): parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable("file_type", required=True) self.assertEqual(reply_json, "json")
def test_parser_config_type_priority_wrong_type_given(self): with self.assertRaises(ValueError): parser = ParseIt(config_type_priority=['non_existing_type', 'json']) parser.read_configuration_variable("file_type123", required=True)