def test_parser__parse_file_per_type_ini(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser._parse_file_per_type("ini",
test_files_location + "/test.ini")
expected_reply = {
'DEFAULT': {
'file_type': 'ini',
'test_string': 'testing',
'test_bool_true': 'true',
'test_bool_false': 'false',
'test_int': '123.0',
'test_float': '123.123',
'test_list': '["test1", "test2", "test3"]'
},
'test_ini': {
'test_ini_key': 'test_ini_value'
}
}
self.assertEqual(reply, expected_reply)
reply = parser._parse_file_per_type("conf",
test_files_location + "/test.ini")
self.assertEqual(reply, expected_reply)
reply = parser._parse_file_per_type("cfg",
test_files_location + "/test.ini")
self.assertEqual(reply, expected_reply)
def init():
# read envvars
print("reading envvars")
parser = ParseIt(recurse=False,
envvar_prefix="plugin_",
config_type_priority=["env_vars"])
metronome_host = parser.read_configuration_variable(
"metronome_host", default_value="http://metronome.mesos:9000")
metronome_job_file = parser.read_configuration_variable(
"metronome_job_file", default_value="metronome.json")
metronome_job_file = os.getcwd() + "/" + metronome_job_file
envvar_dict = read_all_envvars_to_dict()
# get the job json
print("reading metronome job json file")
metronome_job_json = read_file(metronome_job_file)
# populate the job json with the template data
print("populating metronome job json file with the templated data")
metronome_job_json = populate_template_string(metronome_job_json,
envvar_dict)
# create/update metronome job
print("contacting metronome API")
metronome_connection = Metronome(metronome_host)
metronome_connection.create_or_update_metronome_job(metronome_job_json)
print("finished updating metronome")
def test_parser_custom_suffix_mapping_set(self):
parser = ParseIt(config_folder_location=test_files_location,
custom_suffix_mapping={"yaml": ["custom"]},
config_type_priority=["custom"] +
VALID_FILE_TYPE_EXTENSIONS)
expected_config_type_priority = ["custom"] + VALID_FILE_TYPE_EXTENSIONS
expected_valid_type_extension = [
'json', 'yaml', 'yml', 'toml', 'tml', 'hcl', 'tf', 'conf', 'cfg',
'ini', 'xml', 'custom'
]
expected_suffix_file_type_mapping = {
'json': ['json'],
'yaml': ['yaml', 'yml', 'custom'],
'toml': ['toml', 'tml'],
'hcl': ['hcl', 'tf'],
'ini': ['conf', 'cfg', 'ini'],
'xml': ['xml']
}
reply = parser.read_configuration_variable("test_string")
self.assertEqual("testing_custom", reply)
reply = parser.read_configuration_variable("test_json")
self.assertEqual({'test_json_key': 'test_json_value'}, reply)
self.assertEqual(expected_config_type_priority,
parser.config_type_priority)
self.assertEqual(expected_valid_type_extension,
parser.valid_file_type_extension)
self.assertEqual(expected_suffix_file_type_mapping,
parser.suffix_file_type_mapping)
def test_parser_read_configuration_variable_force_envvars_uppercase_false(self):
parser = ParseIt(force_envvars_uppercase=False, config_folder_location=test_files_location)
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
os.environ["test_envvar_estimate_true_int"] = "456"
reply = parser.read_configuration_variable("test_envvar_estimate_true_int")
self.assertNotEqual(reply, 123)
self.assertEqual(reply, 456)
def test_parser_read_configuration_variable_envvar_prefix(self):
parser = ParseIt(envvar_prefix="prefix_test_",
config_folder_location=test_files_location)
os.environ["PREFIX_TEST_TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
reply = parser.read_configuration_variable(
"test_envvar_estimate_true_int")
self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_required_true_value_given_envvar(
self):
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable(
"TEST_ENVVAR_ESTIMATE_TRUE_INT", required=True)
self.assertEqual(reply_json, 123)
def _load_configs(self):
"""Load main configs and module configs."""
for cfile in self.cfiles:
try:
parser = ParseIt(config_location=cfile, config_type_priority=["yaml"])
config = parser.read_all_configuration_variables()
if self.module:
if self.module in self.modules:
modules = config.pop("modules_activated")
self.__setattr__(self.module, Config())
for name, value in modules[self.module].items():
self.__getattribute__(self.module).__setattr__(name, value)
else:
raise ModuleNotImplemented(
"The module `{}` was not found in the modules directory.".format(
self.module
)
)
for name, value in config.items():
self.__setattr__(name, value)
break
except FileNotFoundError:
pass
def test_parser_read_configuration_recurse_false(self):
parser = ParseIt(config_folder_location=test_files_location, recurse=False)
reply = parser.read_configuration_variable("test_json_subfolder")
expected_config_files_dict = {
'json': [
'test.json'
],
'yaml': [
'test.yaml'
],
'yml': [],
'toml': [
'test.toml'
],
'tml': [],
'conf': [],
'hcl': ['test.hcl'],
'tf': [],
'cfg': [],
'ini': [
'test.ini'
],
'xml': [
'test.xml'
]
}
self.assertEqual(parser.config_files_dict, expected_config_files_dict)
self.assertIsNone(reply)
def cli(ctx, verbose, version, config):
parse_it = ParseIt(config_location=config)
if version:
print(__title__, __version__)
setup_logging(verbose)
ctx.ensure_object(dict)
ctx.obj['CA'] = parse_it.read_configuration_variable("CA")
ctx.obj['DOMAIN'] = parse_it.read_configuration_variable("DOMAIN")
def init():
# read envvars
print("reading envvars")
parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"])
nebula_host = parser.read_configuration_variable("nebula_host", required=True)
nebula_username = parser.read_configuration_variable("nebula_username", default_value=None)
nebula_password = parser.read_configuration_variable("nebula_password", default_value=None)
nebula_token = parser.read_configuration_variable("nebula_token", default_value=None)
nebula_port = parser.read_configuration_variable("nebula_port", default_value=80)
nebula_protocol = parser.read_configuration_variable("nebula_protocol", default_value="http")
nebula_job_file = parser.read_configuration_variable("nebula_job_file", default_value="nebula.json")
nebula_job_type = parser.read_configuration_variable("nebula_job_type", default_value="app")
nebula_job_file = os.getcwd() + "/" + nebula_job_file
envvar_dict = read_all_envvars_to_dict()
# get the job json
print("reading nebula job json file")
nebula_job_json = read_file(nebula_job_file)
# populate the job json with the template data
print("populating nebula job json file with the templated data")
nebula_job_json = populate_template_string(nebula_job_json, envvar_dict)
# create nebula connection object
print("contacting nebula API")
nebula_connection = NebulaDeploy(host=nebula_host, username=nebula_username, password=nebula_password,
token=nebula_token, port=nebula_port, protocol=nebula_protocol)
# update nebula
if nebula_job_type == "app":
nebula_connection.create_or_update_nebula_app(nebula_job_json)
elif nebula_job_type == "cron_job":
nebula_connection.create_or_update_nebula_cron_job(nebula_job_json)
print("finished updating nebula")
def test_parser_read_configuration_from_cli_arg(self):
testargs = [
"parse_it_mock_script.py", "--test_cli_key", "test_value",
"--test_cli_int", "123"
]
with mock.patch('sys.argv', testargs):
parser = ParseIt()
reply = parser.read_configuration_variable("test_cli_key")
self.assertEqual(reply, "test_value")
reply = parser.read_configuration_variable("test_cli_int")
self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_type_estimate_true(self):
parser = ParseIt(type_estimate=True, config_folder_location=test_files_location)
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_STRING"] = "test"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE"] = "true"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE"] = "false"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_LIST"] = "['test', False, 3]"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_DICT"] = "{'string': 'string', 'int': 1}"
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT")
self.assertNotEqual(reply, "123")
self.assertEqual(reply, 123)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_STRING")
self.assertEqual(reply, "test")
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE")
self.assertNotEqual(reply, "true")
self.assertEqual(reply, True)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE")
self.assertNotEqual(reply, "false")
self.assertEqual(reply, False)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_LIST")
self.assertNotEqual(reply, "['test', False, 3]")
self.assertEqual(reply, ['test', False, 3])
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_DICT")
self.assertNotEqual(reply, "{'string': 'string', 'int': 1}")
self.assertEqual(reply, {'string': 'string', 'int': 1})
def read_configurations(config_folder: str = "config"):
"""
Will create a config dict that includes all of the configurations for terraformize by aggregating from all valid
config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared
Arguments:
:param config_folder: the folder which all configuration file will be read from recursively
Returns:
:return config: a dict of all configurations needed for terraformize to work
"""
print("reading config variables")
config = {}
parser = ParseIt(config_location=config_folder, recurse=True)
config["basic_auth_user"] = parser.read_configuration_variable(
"basic_auth_user", default_value=None)
config["basic_auth_password"] = parser.read_configuration_variable(
"basic_auth_password", default_value=None)
config["auth_token"] = parser.read_configuration_variable(
"auth_token", default_value=None)
config["terraform_binary_path"] = parser.read_configuration_variable(
"terraform_binary_path", default_value=None)
config["terraform_modules_path"] = parser.read_configuration_variable(
"terraform_modules_path", default_value="/www/terraform_modules")
config["auth_enabled"] = auth_enabled(config["basic_auth_user"],
config["basic_auth_password"],
config["auth_token"])
config["parallelism"] = parser.read_configuration_variable(
"parallelism", default_value=10)
return config
def test_parser__parse_file_per_type_json(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser._parse_file_per_type("json",
test_files_location + "/test.json")
expected_reply = {
'file_type': 'json',
'test_string': 'testing',
'test_bool_true': True,
'test_bool_false': False,
'test_int': 123,
'test_float': 123.123,
'test_list': ['test1', 'test2', 'test3'],
'test_json': {
'test_json_key': 'test_json_value'
}
}
self.assertEqual(reply, expected_reply)
def test_parser_read_configuration_variable_config_folder_location(self):
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable("file_type")
self.assertEqual(reply_json, "json")
reply_json = parser.read_configuration_variable("test_float")
self.assertEqual(reply_json, 123.123)
reply_yaml = parser.read_configuration_variable("test_yaml")
self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'})
reply_xml = parser.read_configuration_variable("xml_root")
expected_reply_xml = {
'file_type': 'xml',
'test_bool_false': False,
'test_bool_true': True,
'test_float': 123.123,
'test_int': 123,
'test_xml': {
'test_xml_key': 'test_xml_value'
},
'test_list': {
'element': [
'test1',
'test2',
'test3'
]
},
'test_string': 'testing'
}
self.assertEqual(reply_xml, expected_reply_xml)
reply_hcl = parser.read_configuration_variable("test_hcl")
expected_reply_hcl = {
'test_hcl_name': {
'test_hcl_key': 'test_hcl_value'
}
}
self.assertEqual(reply_hcl, expected_reply_hcl)
def test_parser__parse_file_per_type_custom_yaml(self):
parser = ParseIt(config_folder_location=test_files_location,
custom_suffix_mapping={"yaml": ["custom"]},
config_type_priority=["custom"] +
VALID_FILE_TYPE_EXTENSIONS)
reply = parser._parse_file_per_type(
"custom", test_files_location + "/test.custom")
expected_reply = {
'file_type': 'custom_yaml_suffix',
'test_string': 'testing_custom',
'test_bool_true': True,
'test_bool_false': False,
'test_int': 123,
'test_float': 123.123,
'test_list': ['test1', 'test2', 'test3'],
'test_yaml': {
'test_yaml_key': 'custom_test_yaml_value'
}
}
self.assertEqual(reply, expected_reply)
def test_parser__parse_file_per_type_xml(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser._parse_file_per_type("xml",
test_files_location + "/test.xml")
expected_reply = {
'xml_root': {
'file_type': 'xml',
'test_bool_false': 'false',
'test_bool_true': 'true',
'test_float': '123.123',
'test_int': '123',
'test_xml': {
'test_xml_key': 'test_xml_value'
},
'test_list': {
'element': ['test1', 'test2', 'test3']
},
'test_string': 'testing'
}
}
self.assertEqual(reply, expected_reply)
def test_parser__parse_file_per_type_hcl(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser._parse_file_per_type("hcl",
test_files_location + "/test.hcl")
expected_reply = {
'file_type': 'hcl',
'test_string': 'testing',
'test_bool_true': True,
'test_bool_false': False,
'test_int': 123,
'test_float': 123.123,
'test_list': ['test1', 'test2', 'test3'],
'test_hcl': {
"test_hcl_name": {
'test_hcl_key': 'test_hcl_value'
}
}
}
self.assertEqual(reply, expected_reply)
reply = parser._parse_file_per_type("tf",
test_files_location + "/test.hcl")
self.assertEqual(reply, expected_reply)
def test_parser_read_configuration_variable_config_type_priority(self):
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
parser = ParseIt(config_type_priority=["yaml", "toml", "ini", "json", "envvars"],
config_folder_location=test_files_location)
reply = parser.read_configuration_variable("file_type")
self.assertEqual(reply, "yaml")
reply = parser.read_configuration_variable("test_toml")
self.assertEqual(reply, {'test_toml_key': 'test_toml_value'})
reply = parser.read_configuration_variable("test_ini")
self.assertEqual(reply, {'test_ini_key': 'test_ini_value'})
reply = parser.read_configuration_variable("test_json")
self.assertEqual(reply, {'test_json_key': 'test_json_value'})
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT")
self.assertEqual(reply, 123)
def setup():
config_type_priority = [
# 'cli_args', # allow using typer
"envvars",
"env",
"yml",
"yaml",
"json",
"toml",
"conf",
"cfg",
"ini",
]
return ParseIt(
config_location=CONFIG_PATH,
recurse=True,
config_type_priority=config_type_priority,
)
def test_parser_init(self):
expected_config_files_dict = {
'json': [
'test.json',
'test_subfolder_1/test_sub_subfolder_2/test_subfolder_2.json',
'test_subfolder_1/test_sub_subfolder_3/test_subfolder_3.json',
'test_subfolder_1/test_subfolder_1.json'
],
'yaml': ['test.yaml'],
'yml': [],
'toml': ['test.toml'],
'tml': [],
'hcl': ['test.hcl'],
'tf': [],
'conf': [],
'cfg': [],
'ini': ['test.ini'],
'xml': ['test.xml']
}
expected_config_type_priority = [
'cli_args', 'env_vars', 'json', 'yaml', 'yml', 'toml', 'tml',
'hcl', 'tf', 'conf', 'cfg', 'ini', 'xml'
]
parser = ParseIt(config_type_priority=None,
global_default_value=None,
type_estimate=True,
force_envvars_uppercase=True,
config_folder_location=test_files_location,
envvar_prefix=None)
self.assertEqual(parser.config_files_dict, expected_config_files_dict)
self.assertEqual(parser.config_folder_location, test_files_location)
self.assertEqual(parser.config_type_priority,
expected_config_type_priority)
self.assertEqual(parser.envvar_prefix, '')
self.assertTrue(parser.force_envvars_uppercase)
self.assertIsNone(parser.global_default_value)
self.assertTrue(parser.type_estimate)
from unittest import TestCase
from alert_on_rain.functions.weather_forecast.forecast import *
from parse_it import ParseIt
parser = ParseIt(recurse=False, config_type_priority=["envvars"])
owm_api_key = parser.read_configuration_variable("owm_api_key", required=True)
class BaseTests(TestCase):
def test_weather_forecast_init(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
self.assertEqual(owm_object.three_hour_forecast.forecast.interval,
"3h")
def test_weather_forecast_rain_tomorrow(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
rain_tomorrow = owm_object.rain_tomorrow()
self.assertIsInstance(rain_tomorrow, bool)
def test_will_be_stormy_tomorrow(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
storm_tomorrow = owm_object.storm_tomorrow()
self.assertIsInstance(storm_tomorrow, bool)
# retry getting the device_group info
@retry(wait_exponential_multiplier=200,
wait_exponential_max=1000,
stop_max_attempt_number=10)
def get_device_group_info(nebula_connection_object, device_group_to_get_info):
return nebula_connection_object.list_device_group_info(
device_group_to_get_info)
if __name__ == "__main__":
try:
# read config file at startup
print("reading config variables")
parser = ParseIt(config_location="config", recurse=True)
print("reading config variables")
# the following config variables are for configuring Nebula workers
nebula_manager_auth_user = parser.read_configuration_variable(
"nebula_manager_auth_user", default_value=None)
nebula_manager_auth_password = parser.read_configuration_variable(
"nebula_manager_auth_password", default_value=None)
nebula_manager_auth_token = parser.read_configuration_variable(
"nebula_manager_auth_token", default_value=None)
nebula_manager_host = parser.read_configuration_variable(
"nebula_manager_host", default_value=None)
nebula_manager_port = parser.read_configuration_variable(
"nebula_manager_port", default_value=80)
nebula_manager_protocol = parser.read_configuration_variable(
"nebula_manager_protocol", default_value="http")
def test_parser__parse_file_per_type_wrong_type(self):
parser = ParseIt(config_folder_location=test_files_location)
with self.assertRaises(ValueError):
parser._parse_file_per_type("non_existing_type", test_files_location + "/test.json")
def test_parser_read_configuration_variable_default_value(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser.read_configuration_variable("file_type123", default_value="test123")
self.assertEqual(reply, "test123")
def test_parser_config_found_in_key_false(self):
parser = ParseIt(config_folder_location=test_files_location)
config_found_reply, config_value_reply = parser._check_config_in_dict("wrong_key", {"test_key": "test_value"})
self.assertFalse(config_found_reply)
self.assertEqual(config_value_reply, None)
def test_parser_read_configuration_variable_required_true_value_not_given(self):
parser = ParseIt(config_folder_location=test_files_location)
with self.assertRaises(ValueError):
parser.read_configuration_variable("file_type123", required=True)
def test_parser_config_type_priority_wrong_type_given(self):
with self.assertRaises(ValueError):
parser = ParseIt(config_type_priority=['non_existing_type', 'json'])
parser.read_configuration_variable("file_type123", required=True)
def test_parser_read_configuration_variable_required_true_value_given_file(self):
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable("file_type", required=True)
self.assertEqual(reply_json, "json")
def test_parser_read_configuration_variable_global_default_value(self):
parser = ParseIt(global_default_value="my_last_resort", config_folder_location=test_files_location)
reply = parser.read_configuration_variable("this_does_not_exist")
self.assertEqual(reply, "my_last_resort")
