def test_parser_custom_suffix_mapping_set(self): parser = ParseIt(config_folder_location=test_files_location, custom_suffix_mapping={"yaml": ["custom"]}, config_type_priority=["custom"] + VALID_FILE_TYPE_EXTENSIONS) expected_config_type_priority = ["custom"] + VALID_FILE_TYPE_EXTENSIONS expected_valid_type_extension = [ 'json', 'yaml', 'yml', 'toml', 'tml', 'hcl', 'tf', 'conf', 'cfg', 'ini', 'xml', 'custom' ] expected_suffix_file_type_mapping = { 'json': ['json'], 'yaml': ['yaml', 'yml', 'custom'], 'toml': ['toml', 'tml'], 'hcl': ['hcl', 'tf'], 'ini': ['conf', 'cfg', 'ini'], 'xml': ['xml'] } reply = parser.read_configuration_variable("test_string") self.assertEqual("testing_custom", reply) reply = parser.read_configuration_variable("test_json") self.assertEqual({'test_json_key': 'test_json_value'}, reply) self.assertEqual(expected_config_type_priority, parser.config_type_priority) self.assertEqual(expected_valid_type_extension, parser.valid_file_type_extension) self.assertEqual(expected_suffix_file_type_mapping, parser.suffix_file_type_mapping)
def read_configurations(config_folder: str = "config"): """ Will create a config dict that includes all of the configurations for terraformize by aggregating from all valid config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared Arguments: :param config_folder: the folder which all configuration file will be read from recursively Returns: :return config: a dict of all configurations needed for terraformize to work """ print("reading config variables") config = {} parser = ParseIt(config_location=config_folder, recurse=True) config["basic_auth_user"] = parser.read_configuration_variable( "basic_auth_user", default_value=None) config["basic_auth_password"] = parser.read_configuration_variable( "basic_auth_password", default_value=None) config["auth_token"] = parser.read_configuration_variable( "auth_token", default_value=None) config["terraform_binary_path"] = parser.read_configuration_variable( "terraform_binary_path", default_value=None) config["terraform_modules_path"] = parser.read_configuration_variable( "terraform_modules_path", default_value="/www/terraform_modules") config["auth_enabled"] = auth_enabled(config["basic_auth_user"], config["basic_auth_password"], config["auth_token"]) return config
def test_parser_read_configuration_variable_config_folder_location(self): parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable("file_type") self.assertEqual(reply_json, "json") reply_json = parser.read_configuration_variable("test_float") self.assertEqual(reply_json, 123.123) reply_yaml = parser.read_configuration_variable("test_yaml") self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'}) reply_xml = parser.read_configuration_variable("xml_root") expected_reply_xml = { 'file_type': 'xml', 'test_bool_false': False, 'test_bool_true': True, 'test_float': 123.123, 'test_int': 123, 'test_xml': { 'test_xml_key': 'test_xml_value' }, 'test_list': { 'element': [ 'test1', 'test2', 'test3' ] }, 'test_string': 'testing' } self.assertEqual(reply_xml, expected_reply_xml) reply_hcl = parser.read_configuration_variable("test_hcl") expected_reply_hcl = { 'test_hcl_name': { 'test_hcl_key': 'test_hcl_value' } } self.assertEqual(reply_hcl, expected_reply_hcl)
def init(): # read envvars print("reading envvars") parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"]) metronome_host = parser.read_configuration_variable( "metronome_host", default_value="http://metronome.mesos:9000") metronome_job_file = parser.read_configuration_variable( "metronome_job_file", default_value="metronome.json") metronome_job_file = os.getcwd() + "/" + metronome_job_file envvar_dict = read_all_envvars_to_dict() # get the job json print("reading metronome job json file") metronome_job_json = read_file(metronome_job_file) # populate the job json with the template data print("populating metronome job json file with the templated data") metronome_job_json = populate_template_string(metronome_job_json, envvar_dict) # create/update metronome job print("contacting metronome API") metronome_connection = Metronome(metronome_host) metronome_connection.create_or_update_metronome_job(metronome_job_json) print("finished updating metronome")
def cli(ctx, verbose, version, config): parse_it = ParseIt(config_location=config) if version: print(__title__, __version__) setup_logging(verbose) ctx.ensure_object(dict) ctx.obj['CA'] = parse_it.read_configuration_variable("CA") ctx.obj['DOMAIN'] = parse_it.read_configuration_variable("DOMAIN")
def test_parser_read_configuration_from_cli_arg(self): testargs = [ "parse_it_mock_script.py", "--test_cli_key", "test_value", "--test_cli_int", "123" ] with mock.patch('sys.argv', testargs): parser = ParseIt() reply = parser.read_configuration_variable("test_cli_key") self.assertEqual(reply, "test_value") reply = parser.read_configuration_variable("test_cli_int") self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_config_type_priority(self): os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" parser = ParseIt(config_type_priority=["yaml", "toml", "ini", "json", "envvars"], config_folder_location=test_files_location) reply = parser.read_configuration_variable("file_type") self.assertEqual(reply, "yaml") reply = parser.read_configuration_variable("test_toml") self.assertEqual(reply, {'test_toml_key': 'test_toml_value'}) reply = parser.read_configuration_variable("test_ini") self.assertEqual(reply, {'test_ini_key': 'test_ini_value'}) reply = parser.read_configuration_variable("test_json") self.assertEqual(reply, {'test_json_key': 'test_json_value'}) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT") self.assertEqual(reply, 123)
def init(): """ Run the logic which will take the variables (OWM api key & location) no matter how they are provided and will alert if it looks like it will rain there tomorrow """ parser = ParseIt(recurse=False, config_type_priority=["envvars"]) owm_api_key = parser.read_configuration_variable("owm_api_key", required=True) city = parser.read_configuration_variable("city", required=True) country_code = parser.read_configuration_variable("country_code", required=True) smtp_server = parser.read_configuration_variable("smtp_server", required=True) sender_email = parser.read_configuration_variable("sender_email", required=True) receiver_email = parser.read_configuration_variable("receiver_email", required=True) email_password = parser.read_configuration_variable("email_password", required=True) email_port = parser.read_configuration_variable("email_port", required=True) telegram_token = parser.read_configuration_variable("telegram_token", required=True) chat_id = parser.read_configuration_variable("chat_id", required=True) owm_object = WeatherForecast(owm_api_key, city, country_code) telegram_object = Telegram(telegram_token) if owm_object.rain_tomorrow() is True and owm_object.storm_tomorrow( ) is True: print("It will be rainy and stormy tomorrow, sending alert") telegram_object.send_alert(chat_id, "stormy and rain") email_alert(smtp_server, sender_email, receiver_email, email_password, "stormy and rain", email_port) print("rain alert sent") elif owm_object.rain_tomorrow() is True: print("It will rain tomorrow, sending alert") telegram_object.send_alert(chat_id, "rain") email_alert(smtp_server, sender_email, receiver_email, email_password, "rain", email_port) print("rain alert sent") elif owm_object.storm_tomorrow() is True: print("It will be a storm tomorrow, sending alert") telegram_object.send_alert(chat_id, "storm") email_alert(smtp_server, sender_email, receiver_email, email_password, "storm", email_port) print("storm alert sent") else: print("It will not rain or be stormy tomorrow")
def test_parser_read_configuration_variable_envvar_prefix(self): parser = ParseIt(envvar_prefix="prefix_test_", config_folder_location=test_files_location) os.environ["PREFIX_TEST_TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" reply = parser.read_configuration_variable( "test_envvar_estimate_true_int") self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_required_true_value_given_envvar( self): os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable( "TEST_ENVVAR_ESTIMATE_TRUE_INT", required=True) self.assertEqual(reply_json, 123)
def test_parser_read_configuration_variable_force_envvars_uppercase_false(self): parser = ParseIt(force_envvars_uppercase=False, config_folder_location=test_files_location) os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" os.environ["test_envvar_estimate_true_int"] = "456" reply = parser.read_configuration_variable("test_envvar_estimate_true_int") self.assertNotEqual(reply, 123) self.assertEqual(reply, 456)
def test_parser_read_configuration_recurse_false(self): parser = ParseIt(config_folder_location=test_files_location, recurse=False) reply = parser.read_configuration_variable("test_json_subfolder") expected_config_files_dict = { 'json': [ 'test.json' ], 'yaml': [ 'test.yaml' ], 'yml': [], 'toml': [ 'test.toml' ], 'tml': [], 'conf': [], 'hcl': ['test.hcl'], 'tf': [], 'cfg': [], 'ini': [ 'test.ini' ], 'xml': [ 'test.xml' ] } self.assertEqual(parser.config_files_dict, expected_config_files_dict) self.assertIsNone(reply)
def init(): # read envvars print("reading envvars") parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"]) nebula_host = parser.read_configuration_variable("nebula_host", required=True) nebula_username = parser.read_configuration_variable("nebula_username", default_value=None) nebula_password = parser.read_configuration_variable("nebula_password", default_value=None) nebula_token = parser.read_configuration_variable("nebula_token", default_value=None) nebula_port = parser.read_configuration_variable("nebula_port", default_value=80) nebula_protocol = parser.read_configuration_variable("nebula_protocol", default_value="http") nebula_job_file = parser.read_configuration_variable("nebula_job_file", default_value="nebula.json") nebula_job_type = parser.read_configuration_variable("nebula_job_type", default_value="app") nebula_job_file = os.getcwd() + "/" + nebula_job_file envvar_dict = read_all_envvars_to_dict() # get the job json print("reading nebula job json file") nebula_job_json = read_file(nebula_job_file) # populate the job json with the template data print("populating nebula job json file with the templated data") nebula_job_json = populate_template_string(nebula_job_json, envvar_dict) # create nebula connection object print("contacting nebula API") nebula_connection = NebulaDeploy(host=nebula_host, username=nebula_username, password=nebula_password, token=nebula_token, port=nebula_port, protocol=nebula_protocol) # update nebula if nebula_job_type == "app": nebula_connection.create_or_update_nebula_app(nebula_job_json) elif nebula_job_type == "cron_job": nebula_connection.create_or_update_nebula_cron_job(nebula_job_json) print("finished updating nebula")
def test_parser_read_configuration_variable_type_estimate_true(self): parser = ParseIt(type_estimate=True, config_folder_location=test_files_location) os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_STRING"] = "test" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE"] = "true" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE"] = "false" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_LIST"] = "['test', False, 3]" os.environ["TEST_ENVVAR_ESTIMATE_TRUE_DICT"] = "{'string': 'string', 'int': 1}" reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT") self.assertNotEqual(reply, "123") self.assertEqual(reply, 123) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_STRING") self.assertEqual(reply, "test") reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE") self.assertNotEqual(reply, "true") self.assertEqual(reply, True) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE") self.assertNotEqual(reply, "false") self.assertEqual(reply, False) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_LIST") self.assertNotEqual(reply, "['test', False, 3]") self.assertEqual(reply, ['test', False, 3]) reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_DICT") self.assertNotEqual(reply, "{'string': 'string', 'int': 1}") self.assertEqual(reply, {'string': 'string', 'int': 1})
import redis, os, sys, secrets, requests from functions.hashing.hashing import * from flask import request, Flask, json, jsonify from parse_it import ParseIt # the variables below will only be used by the requester API print("reading config variables") parser = ParseIt() redis_host = parser.read_configuration_variable("redis_host") redis_port = parser.read_configuration_variable("redis_port") receiver_webhook_url = parser.read_configuration_variable("receiver_webhook_url") requester_webhook_url = parser.read_configuration_variable("requester_webhook_url") # create flask app app = Flask(__name__) # if a redis is configured this must be an example requester API so connect to it if redis_port is not None and redis_host is not None: r = redis.Redis(host=redis_host, port=redis_port, db=0) # the receiver endpoint - this is the api2api endpoint that will be contacted by the requester api @app.route('/receiver', methods=["GET"]) def receiver(): # check the request authorization header and get the hashed value and webhook address from it auth_header = request.headers['Authorization'] auth_type, auth_url, auth_hashed_token = auth_header.split(" ") # callback to the webhook of the sender in the authorization header with the hashed value in the body & the url of # itself payload = json.dumps({"hash": auth_hashed_token, "url": receiver_webhook_url})
def read_configurations(config_folder: str = "config") -> dict: """ Will create a config dict that includes all of the configurations for the autoscaler by aggregating from all valid config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared Arguments: :param config_folder: the folder which all configuration file will be read from recursively Returns: :return config: a dict of all configurations needed for autoscaler to work """ print("reading config variables") config = {} parser = ParseIt(config_location=config_folder, recurse=True) config["kube_token"] = parser.read_configuration_variable( "kube_token", default_value=None) config["kube_api_endpoint"] = parser.read_configuration_variable( "kube_api_endpoint", default_value=None) kubeconfig_file = os.path.expanduser("~/.kube/config") config["kubeconfig_path"] = parser.read_configuration_variable( "kubeconfig_path", default_value=kubeconfig_file) config["kubeconfig_context"] = parser.read_configuration_variable( "kubeconfig_context", default_value=None) config["max_memory_usage"] = parser.read_configuration_variable( "max_memory_usage", default_value=80) config["min_memory_usage"] = parser.read_configuration_variable( "min_memory_usage", default_value=50) config["max_cpu_usage"] = parser.read_configuration_variable( "max_cpu_usage", default_value=80) config["min_cpu_usage"] = parser.read_configuration_variable( "min_cpu_usage", default_value=50) config["seconds_to_check"] = parser.read_configuration_variable( "seconds_to_check", default_value=30) config["spotinst_token"] = parser.read_configuration_variable( "spotinst_token", required=True) config["kube_connection_method"] = decide_kube_connection_method( kube_api_endpoint=config["kube_api_endpoint"], kubeconfig_path=config["kubeconfig_path"]) config["elastigroup_id"] = parser.read_configuration_variable( "elastigroup_id", required=True) config["min_node_count"] = parser.read_configuration_variable( "min_node_count", default_value=2) config["max_node_count"] = parser.read_configuration_variable( "max_node_count", default_value=100) config["spotinst_account"] = parser.read_configuration_variable( "spotinst_account", required=True) config["scale_up_count"] = parser.read_configuration_variable( "scale_up_count", default_value=1) config["scale_down_count"] = parser.read_configuration_variable( "scale_down_count", default_value=1) config["scale_up_active"] = parser.read_configuration_variable( "scale_up_active", default_value=True) config["scale_down_active"] = parser.read_configuration_variable( "scale_down_active", default_value=True) config["scale_on_pending_pods"] = parser.read_configuration_variable( "scale_on_pending_pods", default_value=True) return config
def test_parser_read_configuration_variable_global_default_value(self): parser = ParseIt(global_default_value="my_last_resort", config_folder_location=test_files_location) reply = parser.read_configuration_variable("this_does_not_exist") self.assertEqual(reply, "my_last_resort")
def test_parser_read_configuration_variable_required_true_value_given_file(self): parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable("file_type", required=True) self.assertEqual(reply_json, "json")
def test_parser_config_type_priority_wrong_type_given(self): with self.assertRaises(ValueError): parser = ParseIt(config_type_priority=['non_existing_type', 'json']) parser.read_configuration_variable("file_type123", required=True)
def test_parser_read_configuration_variable_required_true_value_not_given(self): parser = ParseIt(config_folder_location=test_files_location) with self.assertRaises(ValueError): parser.read_configuration_variable("file_type123", required=True)
def test_parser_read_configuration_variable_default_value(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser.read_configuration_variable("file_type123", default_value="test123") self.assertEqual(reply, "test123")
from functions.reporting.kafka import * from functions.db.mongo import * import datetime from parse_it import ParseIt if __name__ == "__main__": try: print("reading config variables") parser = ParseIt(config_location="config", recurse=True) # the following config variables are for configure the reporter kafka_bootstrap_servers = parser.read_configuration_variable("kafka_bootstrap_servers", required=True) kafka_security_protocol = parser.read_configuration_variable("kafka_security_protocol", default_value="PLAINTEXT") kafka_sasl_mechanism = parser.read_configuration_variable("kafka_sasl_mechanism", default_value=None) kafka_sasl_plain_username = parser.read_configuration_variable("kafka_sasl_plain_username", default_value=None) kafka_sasl_plain_password = parser.read_configuration_variable("kafka_sasl_plain_password", default_value=None) kafka_ssl_keyfile = parser.read_configuration_variable("kafka_ssl_keyfile", default_value=None) kafka_ssl_password = parser.read_configuration_variable("kafka_ssl_password", default_value=None) kafka_ssl_certfile = parser.read_configuration_variable("kafka_ssl_certfile", default_value=None) kafka_ssl_cafile = parser.read_configuration_variable("kafka_ssl_cafile", default_value=None) kafka_ssl_crlfile = parser.read_configuration_variable("kafka_ssl_crlfile", default_value=None) kafka_sasl_kerberos_service_name = parser.read_configuration_variable("kafka_sasl_kerberos_service_name", default_value="kafka") kafka_sasl_kerberos_domain_name = parser.read_configuration_variable("kafka_sasl_kerberos_domain_name", default_value="kafka") kafka_topic = parser.read_configuration_variable("kafka_topic", default_value="nebula-reports") kafka_auto_offset_reset = parser.read_configuration_variable("kafka_auto_offset_reset", default_value="earliest") kafka_group_id = parser.read_configuration_variable("kafka_group_id", default_value="nebula-reporter-group")
from unittest import TestCase from alert_on_rain.functions.weather_forecast.forecast import * from parse_it import ParseIt parser = ParseIt(recurse=False, config_type_priority=["envvars"]) owm_api_key = parser.read_configuration_variable("owm_api_key", required=True) class BaseTests(TestCase): def test_weather_forecast_init(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") self.assertEqual(owm_object.three_hour_forecast.forecast.interval, "3h") def test_weather_forecast_rain_tomorrow(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") rain_tomorrow = owm_object.rain_tomorrow() self.assertIsInstance(rain_tomorrow, bool) def test_will_be_stormy_tomorrow(self): owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL") storm_tomorrow = owm_object.storm_tomorrow() self.assertIsInstance(storm_tomorrow, bool)
def test_parser_read_configuration_variable(self): parser = ParseIt(config_folder_location=test_files_location) reply_json = parser.read_configuration_variable("file_type") self.assertEqual(reply_json, "json") reply_yaml = parser.read_configuration_variable("test_yaml") self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'})
def test_parser_read_configuration_subfolder(self): parser = ParseIt(config_folder_location=test_files_location) reply = parser.read_configuration_variable("test_json_subfolder") self.assertTrue(reply)
stop_max_attempt_number=10) def get_device_group_info(nebula_connection_object, device_group_to_get_info): return nebula_connection_object.list_device_group_info( device_group_to_get_info) if __name__ == "__main__": try: # read config file at startup print("reading config variables") parser = ParseIt(config_location="config", recurse=True) print("reading config variables") # the following config variables are for configuring Nebula workers nebula_manager_auth_user = parser.read_configuration_variable( "nebula_manager_auth_user", default_value=None) nebula_manager_auth_password = parser.read_configuration_variable( "nebula_manager_auth_password", default_value=None) nebula_manager_auth_token = parser.read_configuration_variable( "nebula_manager_auth_token", default_value=None) nebula_manager_host = parser.read_configuration_variable( "nebula_manager_host", default_value=None) nebula_manager_port = parser.read_configuration_variable( "nebula_manager_port", default_value=80) nebula_manager_protocol = parser.read_configuration_variable( "nebula_manager_protocol", default_value="http") nebula_manager_uri = parser.read_configuration_variable( "nebula_manager_uri", default_value=None) nebula_manager_request_timeout = parser.read_configuration_variable( "nebula_manager_request_timeout", default_value=60) nebula_manager_check_in_time = parser.read_configuration_variable(
elif permission_object_type == "admin": if check_authorized(permission_needed={"admin": permission_needed}, permission_object_type=permission_object_type) is True: result = func(*args, **kwargs) return result return wrapped return callable_function # read config file at startup print("reading config variables") parser = ParseIt(config_location="config", recurse=True) print("reading config variables") basic_auth_user = parser.read_configuration_variable("basic_auth_user", default_value=None) basic_auth_password = parser.read_configuration_variable("basic_auth_password", default_value=None) auth_token = parser.read_configuration_variable("auth_token", default_value=None) mongo_url = parser.read_configuration_variable("mongo_url", required=True) schema_name = parser.read_configuration_variable("schema_name", default_value="nebula") auth_enabled = parser.read_configuration_variable("auth_enabled", default_value=True) cache_time = parser.read_configuration_variable("cache_time", default_value=10) cache_max_size = parser.read_configuration_variable("cache_max_size", default_value=1024) mongo_max_pool_size = parser.read_configuration_variable("mongo_max_pool_size", default_value=25) # login to db at startup mongo_connection = MongoConnection(mongo_url, schema_name, max_pool_size=mongo_max_pool_size) print("opened MongoDB connection") # ensure mongo is indexed properly mongo_connection.mongo_create_index("apps", "app_name")