def test_parser_custom_suffix_mapping_set(self):
parser = ParseIt(config_folder_location=test_files_location,
custom_suffix_mapping={"yaml": ["custom"]},
config_type_priority=["custom"] +
VALID_FILE_TYPE_EXTENSIONS)
expected_config_type_priority = ["custom"] + VALID_FILE_TYPE_EXTENSIONS
expected_valid_type_extension = [
'json', 'yaml', 'yml', 'toml', 'tml', 'hcl', 'tf', 'conf', 'cfg',
'ini', 'xml', 'custom'
]
expected_suffix_file_type_mapping = {
'json': ['json'],
'yaml': ['yaml', 'yml', 'custom'],
'toml': ['toml', 'tml'],
'hcl': ['hcl', 'tf'],
'ini': ['conf', 'cfg', 'ini'],
'xml': ['xml']
}
reply = parser.read_configuration_variable("test_string")
self.assertEqual("testing_custom", reply)
reply = parser.read_configuration_variable("test_json")
self.assertEqual({'test_json_key': 'test_json_value'}, reply)
self.assertEqual(expected_config_type_priority,
parser.config_type_priority)
self.assertEqual(expected_valid_type_extension,
parser.valid_file_type_extension)
self.assertEqual(expected_suffix_file_type_mapping,
parser.suffix_file_type_mapping)
def read_configurations(config_folder: str = "config"):
"""
Will create a config dict that includes all of the configurations for terraformize by aggregating from all valid
config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared
Arguments:
:param config_folder: the folder which all configuration file will be read from recursively
Returns:
:return config: a dict of all configurations needed for terraformize to work
"""
print("reading config variables")
config = {}
parser = ParseIt(config_location=config_folder, recurse=True)
config["basic_auth_user"] = parser.read_configuration_variable(
"basic_auth_user", default_value=None)
config["basic_auth_password"] = parser.read_configuration_variable(
"basic_auth_password", default_value=None)
config["auth_token"] = parser.read_configuration_variable(
"auth_token", default_value=None)
config["terraform_binary_path"] = parser.read_configuration_variable(
"terraform_binary_path", default_value=None)
config["terraform_modules_path"] = parser.read_configuration_variable(
"terraform_modules_path", default_value="/www/terraform_modules")
config["auth_enabled"] = auth_enabled(config["basic_auth_user"],
config["basic_auth_password"],
config["auth_token"])
return config
def test_parser_read_configuration_variable_config_folder_location(self):
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable("file_type")
self.assertEqual(reply_json, "json")
reply_json = parser.read_configuration_variable("test_float")
self.assertEqual(reply_json, 123.123)
reply_yaml = parser.read_configuration_variable("test_yaml")
self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'})
reply_xml = parser.read_configuration_variable("xml_root")
expected_reply_xml = {
'file_type': 'xml',
'test_bool_false': False,
'test_bool_true': True,
'test_float': 123.123,
'test_int': 123,
'test_xml': {
'test_xml_key': 'test_xml_value'
},
'test_list': {
'element': [
'test1',
'test2',
'test3'
]
},
'test_string': 'testing'
}
self.assertEqual(reply_xml, expected_reply_xml)
reply_hcl = parser.read_configuration_variable("test_hcl")
expected_reply_hcl = {
'test_hcl_name': {
'test_hcl_key': 'test_hcl_value'
}
}
self.assertEqual(reply_hcl, expected_reply_hcl)
def init():
# read envvars
print("reading envvars")
parser = ParseIt(recurse=False,
envvar_prefix="plugin_",
config_type_priority=["env_vars"])
metronome_host = parser.read_configuration_variable(
"metronome_host", default_value="http://metronome.mesos:9000")
metronome_job_file = parser.read_configuration_variable(
"metronome_job_file", default_value="metronome.json")
metronome_job_file = os.getcwd() + "/" + metronome_job_file
envvar_dict = read_all_envvars_to_dict()
# get the job json
print("reading metronome job json file")
metronome_job_json = read_file(metronome_job_file)
# populate the job json with the template data
print("populating metronome job json file with the templated data")
metronome_job_json = populate_template_string(metronome_job_json,
envvar_dict)
# create/update metronome job
print("contacting metronome API")
metronome_connection = Metronome(metronome_host)
metronome_connection.create_or_update_metronome_job(metronome_job_json)
print("finished updating metronome")
def cli(ctx, verbose, version, config):
parse_it = ParseIt(config_location=config)
if version:
print(__title__, __version__)
setup_logging(verbose)
ctx.ensure_object(dict)
ctx.obj['CA'] = parse_it.read_configuration_variable("CA")
ctx.obj['DOMAIN'] = parse_it.read_configuration_variable("DOMAIN")
def test_parser_read_configuration_from_cli_arg(self):
testargs = [
"parse_it_mock_script.py", "--test_cli_key", "test_value",
"--test_cli_int", "123"
]
with mock.patch('sys.argv', testargs):
parser = ParseIt()
reply = parser.read_configuration_variable("test_cli_key")
self.assertEqual(reply, "test_value")
reply = parser.read_configuration_variable("test_cli_int")
self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_config_type_priority(self):
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
parser = ParseIt(config_type_priority=["yaml", "toml", "ini", "json", "envvars"],
config_folder_location=test_files_location)
reply = parser.read_configuration_variable("file_type")
self.assertEqual(reply, "yaml")
reply = parser.read_configuration_variable("test_toml")
self.assertEqual(reply, {'test_toml_key': 'test_toml_value'})
reply = parser.read_configuration_variable("test_ini")
self.assertEqual(reply, {'test_ini_key': 'test_ini_value'})
reply = parser.read_configuration_variable("test_json")
self.assertEqual(reply, {'test_json_key': 'test_json_value'})
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT")
self.assertEqual(reply, 123)
def init():
"""
Run the logic which will take the variables (OWM api key & location) no matter how they are provided and will alert
if it looks like it will rain there tomorrow
"""
parser = ParseIt(recurse=False, config_type_priority=["envvars"])
owm_api_key = parser.read_configuration_variable("owm_api_key",
required=True)
city = parser.read_configuration_variable("city", required=True)
country_code = parser.read_configuration_variable("country_code",
required=True)
smtp_server = parser.read_configuration_variable("smtp_server",
required=True)
sender_email = parser.read_configuration_variable("sender_email",
required=True)
receiver_email = parser.read_configuration_variable("receiver_email",
required=True)
email_password = parser.read_configuration_variable("email_password",
required=True)
email_port = parser.read_configuration_variable("email_port",
required=True)
telegram_token = parser.read_configuration_variable("telegram_token",
required=True)
chat_id = parser.read_configuration_variable("chat_id", required=True)
owm_object = WeatherForecast(owm_api_key, city, country_code)
telegram_object = Telegram(telegram_token)
if owm_object.rain_tomorrow() is True and owm_object.storm_tomorrow(
) is True:
print("It will be rainy and stormy tomorrow, sending alert")
telegram_object.send_alert(chat_id, "stormy and rain")
email_alert(smtp_server, sender_email, receiver_email, email_password,
"stormy and rain", email_port)
print("rain alert sent")
elif owm_object.rain_tomorrow() is True:
print("It will rain tomorrow, sending alert")
telegram_object.send_alert(chat_id, "rain")
email_alert(smtp_server, sender_email, receiver_email, email_password,
"rain", email_port)
print("rain alert sent")
elif owm_object.storm_tomorrow() is True:
print("It will be a storm tomorrow, sending alert")
telegram_object.send_alert(chat_id, "storm")
email_alert(smtp_server, sender_email, receiver_email, email_password,
"storm", email_port)
print("storm alert sent")
else:
print("It will not rain or be stormy tomorrow")
def test_parser_read_configuration_variable_envvar_prefix(self):
parser = ParseIt(envvar_prefix="prefix_test_",
config_folder_location=test_files_location)
os.environ["PREFIX_TEST_TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
reply = parser.read_configuration_variable(
"test_envvar_estimate_true_int")
self.assertEqual(reply, 123)
def test_parser_read_configuration_variable_required_true_value_given_envvar(
self):
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable(
"TEST_ENVVAR_ESTIMATE_TRUE_INT", required=True)
self.assertEqual(reply_json, 123)
def test_parser_read_configuration_variable_force_envvars_uppercase_false(self):
parser = ParseIt(force_envvars_uppercase=False, config_folder_location=test_files_location)
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
os.environ["test_envvar_estimate_true_int"] = "456"
reply = parser.read_configuration_variable("test_envvar_estimate_true_int")
self.assertNotEqual(reply, 123)
self.assertEqual(reply, 456)
def test_parser_read_configuration_recurse_false(self):
parser = ParseIt(config_folder_location=test_files_location, recurse=False)
reply = parser.read_configuration_variable("test_json_subfolder")
expected_config_files_dict = {
'json': [
'test.json'
],
'yaml': [
'test.yaml'
],
'yml': [],
'toml': [
'test.toml'
],
'tml': [],
'conf': [],
'hcl': ['test.hcl'],
'tf': [],
'cfg': [],
'ini': [
'test.ini'
],
'xml': [
'test.xml'
]
}
self.assertEqual(parser.config_files_dict, expected_config_files_dict)
self.assertIsNone(reply)
def init():
# read envvars
print("reading envvars")
parser = ParseIt(recurse=False, envvar_prefix="plugin_", config_type_priority=["env_vars"])
nebula_host = parser.read_configuration_variable("nebula_host", required=True)
nebula_username = parser.read_configuration_variable("nebula_username", default_value=None)
nebula_password = parser.read_configuration_variable("nebula_password", default_value=None)
nebula_token = parser.read_configuration_variable("nebula_token", default_value=None)
nebula_port = parser.read_configuration_variable("nebula_port", default_value=80)
nebula_protocol = parser.read_configuration_variable("nebula_protocol", default_value="http")
nebula_job_file = parser.read_configuration_variable("nebula_job_file", default_value="nebula.json")
nebula_job_type = parser.read_configuration_variable("nebula_job_type", default_value="app")
nebula_job_file = os.getcwd() + "/" + nebula_job_file
envvar_dict = read_all_envvars_to_dict()
# get the job json
print("reading nebula job json file")
nebula_job_json = read_file(nebula_job_file)
# populate the job json with the template data
print("populating nebula job json file with the templated data")
nebula_job_json = populate_template_string(nebula_job_json, envvar_dict)
# create nebula connection object
print("contacting nebula API")
nebula_connection = NebulaDeploy(host=nebula_host, username=nebula_username, password=nebula_password,
token=nebula_token, port=nebula_port, protocol=nebula_protocol)
# update nebula
if nebula_job_type == "app":
nebula_connection.create_or_update_nebula_app(nebula_job_json)
elif nebula_job_type == "cron_job":
nebula_connection.create_or_update_nebula_cron_job(nebula_job_json)
print("finished updating nebula")
def test_parser_read_configuration_variable_type_estimate_true(self):
parser = ParseIt(type_estimate=True, config_folder_location=test_files_location)
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_INT"] = "123"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_STRING"] = "test"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE"] = "true"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE"] = "false"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_LIST"] = "['test', False, 3]"
os.environ["TEST_ENVVAR_ESTIMATE_TRUE_DICT"] = "{'string': 'string', 'int': 1}"
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_INT")
self.assertNotEqual(reply, "123")
self.assertEqual(reply, 123)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_STRING")
self.assertEqual(reply, "test")
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_TRUE")
self.assertNotEqual(reply, "true")
self.assertEqual(reply, True)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_BOOL_FALSE")
self.assertNotEqual(reply, "false")
self.assertEqual(reply, False)
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_LIST")
self.assertNotEqual(reply, "['test', False, 3]")
self.assertEqual(reply, ['test', False, 3])
reply = parser.read_configuration_variable("TEST_ENVVAR_ESTIMATE_TRUE_DICT")
self.assertNotEqual(reply, "{'string': 'string', 'int': 1}")
self.assertEqual(reply, {'string': 'string', 'int': 1})
import redis, os, sys, secrets, requests
from functions.hashing.hashing import *
from flask import request, Flask, json, jsonify
from parse_it import ParseIt
# the variables below will only be used by the requester API
print("reading config variables")
parser = ParseIt()
redis_host = parser.read_configuration_variable("redis_host")
redis_port = parser.read_configuration_variable("redis_port")
receiver_webhook_url = parser.read_configuration_variable("receiver_webhook_url")
requester_webhook_url = parser.read_configuration_variable("requester_webhook_url")
# create flask app
app = Flask(__name__)
# if a redis is configured this must be an example requester API so connect to it
if redis_port is not None and redis_host is not None:
r = redis.Redis(host=redis_host, port=redis_port, db=0)
# the receiver endpoint - this is the api2api endpoint that will be contacted by the requester api
@app.route('/receiver', methods=["GET"])
def receiver():
# check the request authorization header and get the hashed value and webhook address from it
auth_header = request.headers['Authorization']
auth_type, auth_url, auth_hashed_token = auth_header.split(" ")
# callback to the webhook of the sender in the authorization header with the hashed value in the body & the url of
# itself
payload = json.dumps({"hash": auth_hashed_token, "url": receiver_webhook_url})
def read_configurations(config_folder: str = "config") -> dict:
"""
Will create a config dict that includes all of the configurations for the autoscaler by aggregating from all valid
config sources (files, envvars, cli args, etc) & using sane defaults on config params that are not declared
Arguments:
:param config_folder: the folder which all configuration file will be read from recursively
Returns:
:return config: a dict of all configurations needed for autoscaler to work
"""
print("reading config variables")
config = {}
parser = ParseIt(config_location=config_folder, recurse=True)
config["kube_token"] = parser.read_configuration_variable(
"kube_token", default_value=None)
config["kube_api_endpoint"] = parser.read_configuration_variable(
"kube_api_endpoint", default_value=None)
kubeconfig_file = os.path.expanduser("~/.kube/config")
config["kubeconfig_path"] = parser.read_configuration_variable(
"kubeconfig_path", default_value=kubeconfig_file)
config["kubeconfig_context"] = parser.read_configuration_variable(
"kubeconfig_context", default_value=None)
config["max_memory_usage"] = parser.read_configuration_variable(
"max_memory_usage", default_value=80)
config["min_memory_usage"] = parser.read_configuration_variable(
"min_memory_usage", default_value=50)
config["max_cpu_usage"] = parser.read_configuration_variable(
"max_cpu_usage", default_value=80)
config["min_cpu_usage"] = parser.read_configuration_variable(
"min_cpu_usage", default_value=50)
config["seconds_to_check"] = parser.read_configuration_variable(
"seconds_to_check", default_value=30)
config["spotinst_token"] = parser.read_configuration_variable(
"spotinst_token", required=True)
config["kube_connection_method"] = decide_kube_connection_method(
kube_api_endpoint=config["kube_api_endpoint"],
kubeconfig_path=config["kubeconfig_path"])
config["elastigroup_id"] = parser.read_configuration_variable(
"elastigroup_id", required=True)
config["min_node_count"] = parser.read_configuration_variable(
"min_node_count", default_value=2)
config["max_node_count"] = parser.read_configuration_variable(
"max_node_count", default_value=100)
config["spotinst_account"] = parser.read_configuration_variable(
"spotinst_account", required=True)
config["scale_up_count"] = parser.read_configuration_variable(
"scale_up_count", default_value=1)
config["scale_down_count"] = parser.read_configuration_variable(
"scale_down_count", default_value=1)
config["scale_up_active"] = parser.read_configuration_variable(
"scale_up_active", default_value=True)
config["scale_down_active"] = parser.read_configuration_variable(
"scale_down_active", default_value=True)
config["scale_on_pending_pods"] = parser.read_configuration_variable(
"scale_on_pending_pods", default_value=True)
return config
def test_parser_read_configuration_variable_global_default_value(self):
parser = ParseIt(global_default_value="my_last_resort", config_folder_location=test_files_location)
reply = parser.read_configuration_variable("this_does_not_exist")
self.assertEqual(reply, "my_last_resort")
def test_parser_read_configuration_variable_required_true_value_given_file(self):
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable("file_type", required=True)
self.assertEqual(reply_json, "json")
def test_parser_config_type_priority_wrong_type_given(self):
with self.assertRaises(ValueError):
parser = ParseIt(config_type_priority=['non_existing_type', 'json'])
parser.read_configuration_variable("file_type123", required=True)
def test_parser_read_configuration_variable_required_true_value_not_given(self):
parser = ParseIt(config_folder_location=test_files_location)
with self.assertRaises(ValueError):
parser.read_configuration_variable("file_type123", required=True)
def test_parser_read_configuration_variable_default_value(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser.read_configuration_variable("file_type123", default_value="test123")
self.assertEqual(reply, "test123")
from functions.reporting.kafka import *
from functions.db.mongo import *
import datetime
from parse_it import ParseIt
if __name__ == "__main__":
try:
print("reading config variables")
parser = ParseIt(config_location="config", recurse=True)
# the following config variables are for configure the reporter
kafka_bootstrap_servers = parser.read_configuration_variable("kafka_bootstrap_servers", required=True)
kafka_security_protocol = parser.read_configuration_variable("kafka_security_protocol",
default_value="PLAINTEXT")
kafka_sasl_mechanism = parser.read_configuration_variable("kafka_sasl_mechanism", default_value=None)
kafka_sasl_plain_username = parser.read_configuration_variable("kafka_sasl_plain_username", default_value=None)
kafka_sasl_plain_password = parser.read_configuration_variable("kafka_sasl_plain_password", default_value=None)
kafka_ssl_keyfile = parser.read_configuration_variable("kafka_ssl_keyfile", default_value=None)
kafka_ssl_password = parser.read_configuration_variable("kafka_ssl_password", default_value=None)
kafka_ssl_certfile = parser.read_configuration_variable("kafka_ssl_certfile", default_value=None)
kafka_ssl_cafile = parser.read_configuration_variable("kafka_ssl_cafile", default_value=None)
kafka_ssl_crlfile = parser.read_configuration_variable("kafka_ssl_crlfile", default_value=None)
kafka_sasl_kerberos_service_name = parser.read_configuration_variable("kafka_sasl_kerberos_service_name",
default_value="kafka")
kafka_sasl_kerberos_domain_name = parser.read_configuration_variable("kafka_sasl_kerberos_domain_name",
default_value="kafka")
kafka_topic = parser.read_configuration_variable("kafka_topic", default_value="nebula-reports")
kafka_auto_offset_reset = parser.read_configuration_variable("kafka_auto_offset_reset",
default_value="earliest")
kafka_group_id = parser.read_configuration_variable("kafka_group_id", default_value="nebula-reporter-group")
from unittest import TestCase
from alert_on_rain.functions.weather_forecast.forecast import *
from parse_it import ParseIt
parser = ParseIt(recurse=False, config_type_priority=["envvars"])
owm_api_key = parser.read_configuration_variable("owm_api_key", required=True)
class BaseTests(TestCase):
def test_weather_forecast_init(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
self.assertEqual(owm_object.three_hour_forecast.forecast.interval,
"3h")
def test_weather_forecast_rain_tomorrow(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
rain_tomorrow = owm_object.rain_tomorrow()
self.assertIsInstance(rain_tomorrow, bool)
def test_will_be_stormy_tomorrow(self):
owm_object = WeatherForecast(owm_api_key, "Tel Aviv", "IL")
storm_tomorrow = owm_object.storm_tomorrow()
self.assertIsInstance(storm_tomorrow, bool)
def test_parser_read_configuration_variable(self):
parser = ParseIt(config_folder_location=test_files_location)
reply_json = parser.read_configuration_variable("file_type")
self.assertEqual(reply_json, "json")
reply_yaml = parser.read_configuration_variable("test_yaml")
self.assertEqual(reply_yaml, {'test_yaml_key': 'test_yaml_value'})
def test_parser_read_configuration_subfolder(self):
parser = ParseIt(config_folder_location=test_files_location)
reply = parser.read_configuration_variable("test_json_subfolder")
self.assertTrue(reply)
stop_max_attempt_number=10)
def get_device_group_info(nebula_connection_object, device_group_to_get_info):
return nebula_connection_object.list_device_group_info(
device_group_to_get_info)
if __name__ == "__main__":
try:
# read config file at startup
print("reading config variables")
parser = ParseIt(config_location="config", recurse=True)
print("reading config variables")
# the following config variables are for configuring Nebula workers
nebula_manager_auth_user = parser.read_configuration_variable(
"nebula_manager_auth_user", default_value=None)
nebula_manager_auth_password = parser.read_configuration_variable(
"nebula_manager_auth_password", default_value=None)
nebula_manager_auth_token = parser.read_configuration_variable(
"nebula_manager_auth_token", default_value=None)
nebula_manager_host = parser.read_configuration_variable(
"nebula_manager_host", default_value=None)
nebula_manager_port = parser.read_configuration_variable(
"nebula_manager_port", default_value=80)
nebula_manager_protocol = parser.read_configuration_variable(
"nebula_manager_protocol", default_value="http")
nebula_manager_uri = parser.read_configuration_variable(
"nebula_manager_uri", default_value=None)
nebula_manager_request_timeout = parser.read_configuration_variable(
"nebula_manager_request_timeout", default_value=60)
nebula_manager_check_in_time = parser.read_configuration_variable(
elif permission_object_type == "admin":
if check_authorized(permission_needed={"admin": permission_needed},
permission_object_type=permission_object_type) is True:
result = func(*args, **kwargs)
return result
return wrapped
return callable_function
# read config file at startup
print("reading config variables")
parser = ParseIt(config_location="config", recurse=True)
print("reading config variables")
basic_auth_user = parser.read_configuration_variable("basic_auth_user", default_value=None)
basic_auth_password = parser.read_configuration_variable("basic_auth_password", default_value=None)
auth_token = parser.read_configuration_variable("auth_token", default_value=None)
mongo_url = parser.read_configuration_variable("mongo_url", required=True)
schema_name = parser.read_configuration_variable("schema_name", default_value="nebula")
auth_enabled = parser.read_configuration_variable("auth_enabled", default_value=True)
cache_time = parser.read_configuration_variable("cache_time", default_value=10)
cache_max_size = parser.read_configuration_variable("cache_max_size", default_value=1024)
mongo_max_pool_size = parser.read_configuration_variable("mongo_max_pool_size", default_value=25)
# login to db at startup
mongo_connection = MongoConnection(mongo_url, schema_name, max_pool_size=mongo_max_pool_size)
print("opened MongoDB connection")
# ensure mongo is indexed properly
mongo_connection.mongo_create_index("apps", "app_name")