def __init__(self, config: str, runner_index: Optional[int],
total_runners: Optional[int]):
"""Load config.
Args:
config: Path to config.
"""
# Use envyaml so that users can set environment variables within the yaml file.
data = EnvYAML(config)
# Read parameters specified inside table_header
self.table_data = [data["table_header"]]
self.max_string_width = data.get("table_column_max_width", 0)
# Get global tests parameters
default_args = data.get("default_args", {})
jobs = (data["tests"] if runner_index is None
or total_runners is None else np.array_split(
data["tests"], int(total_runners))[int(runner_index) - 1])
# Loop over all jobs
# and parse parameters to DriveTestCmd
# Join args and default args.
self.pipeline = [
DriveTestCmd(**{
**default_args,
**args
}) for args in jobs
]
def test_it_should_get_default_values():
env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env")
assert env.get("empty.novalues", "default") is None
assert env.get("empty.noenvvalue", "env-value") == "" # value actually set to ""
assert env["empty.noenvvalue"] == ""
def test_it_should_access_environ():
os.environ["ENV_VAR"] = "test-env-var"
env = EnvYAML("tests/env.empty.yaml")
assert env.environ() == os.environ
assert env.environ()["ENV_VAR"] == os.environ["ENV_VAR"]
del os.environ["ENV_VAR"]
def test_it_should_read_config_from_env_variable():
# Set env file
os.environ["ENV_YAML_FILE"] = "tests/env.test.yaml"
os.environ["ENV_FILE"] = "tests/test.env"
env = EnvYAML()
assert env["env_file.project.name"] == "project-x-42"
assert isinstance(env.export(), dict) and len(env.export()) >= 4
def set_config(general_config=None):
"""
Set the path to the config file. This function must be run before running anything else.
Parameters
----------
path : str
Path to config file
Returns
-------
general_config : str
The path to the config file. This return value is useful when we're calling `set_config()` without a parameter -- it enables the caller the retrieve the cached config_path
"""
# If we're specifically configuring the config path
if general_config is not None:
add_cache_entry("config", general_config)
else:
general_config = get_cache_entry("config")
if general_config is None:
raise Exception("Please configure the path to your MC2 config")
_CONF["general_config"] = general_config
config = EnvYAML(_CONF["general_config"])
_CONF["current_user"] = config["user"]["username"]
# Networking configs
manual_head_node = config["launch"].get("head")
manual_worker_nodes = config["launch"].get("workers") or []
_CONF["azure_config"] = config["launch"].get("azure_config")
_CONF["use_azure"] = not (manual_head_node or manual_worker_nodes)
# SSH information
if _CONF["use_azure"]:
azure_config = EnvYAML(_CONF["azure_config"], strict=False)
remote_username = azure_config["auth"]["ssh_user"]
ssh_key = azure_config["auth"]["ssh_private_key"]
# Since `set_config` may be called before Azure VMs may have been
# launched, we can't initialize the IPs of the nodes. This value will
# be set later once we can guarantee the nodes are launched
_CONF["head"] = {
"username": remote_username,
"ssh_key": ssh_key,
"identity": "head node",
}
_CONF["workers"] = [{
"username": remote_username,
"ssh_key": ssh_key,
"identity": "worker node",
} for _ in range(azure_config["num_workers"])]
else:
_CONF["head"] = manual_head_node
_CONF["workers"] = manual_worker_nodes
return general_config
def create_experiment(self, file):
"""
create user experient
"""
env = EnvYAML(file)
project_id = env['Experiment.projectId']
if project_id is None:
print("no project id was set")
return
with open(file, 'r') as yaml_in:
yaml_object = yaml.safe_load(
yaml_in) # yaml_object will be a list or a dict
experiment_dict = yaml_object['Experiment']
experiment_dict['experimentId'] = "".join(
str(uuid.uuid1()).split("-"))
experiment_dict['userId'] = self._user_id
now = datetime.now()
experiment_dict['date'] = now.strftime("%m-%d-%Y")
# project id from the env variable
experiment_dict['projectId'] = project_id
response = self.mutate_single(EXPERIMENT_CREATE_FRAGMENT,
experiment_dict)
new_experiment_created = response["createExperiment"]["ok"]
return self.experiment(new_experiment_created)
def test_it_should_proper_handle_dollar_sign_with_number():
env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env")
assert (
env["sql"]
== 'SELECT * FROM "users" WHERE "user" = $1 AND "login" = $2 AND "pwd" = $3'
)
def generate_symmetric_key():
"""
Generate a new symmetric key and save it path specified by user in config YAML passed to `set_config()`
Parameters
----------
num_bytes : int
Number of bytes for key
"""
num_bytes = _LIB.cipher_key_size()
if _CONF.get("general_config") is None:
raise MC2ClientConfigError("Configuration not set")
symmetric_key_path = EnvYAML(
_CONF["general_config"])["user"]["symmetric_key"]
if os.path.exists(symmetric_key_path):
logger.warning(
"Skipping symmetric key generation - key already exists at {}".
format(symmetric_key_path))
return
key = AESGCM.generate_key(bit_length=num_bytes * 8)
with open(symmetric_key_path, "wb") as symm_key:
symm_key.write(key)
logger.info("Generated symmetric key and outputted to {}".format(
symmetric_key_path))
def yaml_form(request):
if request.method == "POST":
form = SnippetForm(request.POST)
if form.is_valid():
form.save()
name = form.cleaned_data['region']
# with yaml library
with open("template.yaml", 'r') as stream:
data_loaded = yaml.safe_load(stream)
data_loaded['Resources']['MyEC2Instance']['Properties'][
'AvailabilityZone'] = name
with open("new_template.yaml", "w") as f:
yaml.dump(data_loaded, f)
# with env variables and envyaml library (env variables in template must be referenced with $VAR)
os.environ['NAME'] = name
data = EnvYAML('template.yaml')
print(data)
with open("new_template.yaml", "w") as f:
yaml.dump(data['Resources'], f)
# with yaml.add_implicit_resolver and yaml.add_constructor (env variables in template mus be referenced with ${VAR})
os.environ['NAME'] = name
with open('template.yaml', 'r') as stream:
p = yaml.load(stream, Loader=yaml.FullLoader)
print(p['Resources'])
with open("new_template.yaml", "w") as f:
yaml.dump(p, f)
form = SnippetForm()
return render(request, 'form.html', {'form': form})
def config(tmp_path):
tests_dir = pathlib.Path(__file__).parent.absolute()
original_config_path = os.path.join(tests_dir, "test.yaml")
test_cert = os.path.join(tmp_path, "test.crt")
test_priv_key = os.path.join(tmp_path, "test.pem")
test_symm_key = os.path.join(tmp_path, "test_sym.key")
# Rewrite config YAML with test paths
config = EnvYAML(original_config_path)
config["user"]["certificate"] = test_cert
config["user"]["private_key"] = test_priv_key
config["user"]["symmetric_key"] = test_symm_key
# Point to root certificate
config["user"]["root_private_key"] = os.path.join(tests_dir,
"keys/root.pem")
config["user"]["root_certificate"] = os.path.join(tests_dir,
"keys/root.crt")
test_config_path = os.path.join(tmp_path, "config.yaml")
with open(test_config_path, "w") as out:
yaml.dump(dict(config), out, default_flow_style=False)
mc2.set_config(test_config_path)
return tests_dir
def __init__(self):
self.__config_path = os.getenv("EDGE_CONF_FILE")
if self.__config_path is None:
raise Exception("The config path is not correct")
self.__config: EnvYAML = EnvYAML(self.__config_path)
self.__rabbitMQConfig: RMQConfig = None
self.__severConfig: APIServerConfig = None
def test_it_should_be_valid_in_check():
env = EnvYAML(env_file="tests/env.test.yaml")
if "test.one" in env:
assert env["test.one"] == 123
assert "test.not_exists" not in env
def test_it_should_access_environment_variables():
os.environ["ENV_VAR"] = "test-env-var"
env = EnvYAML("tests/env.empty.yaml")
assert env["ENV_VAR"] == "test-env-var"
del os.environ["ENV_VAR"]
def load_yaml(path):
"""
Args:
path (str): File path of yaml configuration file
Returns:
Dict[str, Any]:
"""
return EnvYAML(path, include_environment=False).export()
def decrypt_data(encrypted_file, plaintext_file, enc_format):
"""
Encrypt a file in a certain format
Parameters
----------
encrypted_file : str
Path to encrypted data to decrypt
plaintext_file : str
Path to decrypted data
enc_format : str
The encryption format (i.e. `sql` for Opaque SQL or `xgb` for Secure XGBoost)
"""
if _CONF.get("general_config") is None:
raise MC2ClientConfigError("Configuration not set")
if not os.path.exists(encrypted_file):
raise FileNotFoundError(
"Cannot find file to decrypt: {}".format(encrypted_file))
cleaned_format = "".join(enc_format.split()).lower()
symmetric_key_path = EnvYAML(
_CONF["general_config"])["user"]["symmetric_key"]
if not os.path.exists(symmetric_key_path):
raise FileNotFoundError(
"Symmetric key not found at {}".format(symmetric_key_path))
result = ctypes.c_int()
if cleaned_format == "xgb":
_LIB.sxgb_decrypt_data(
c_str(encrypted_file),
c_str(plaintext_file),
c_str(symmetric_key_path),
ctypes.byref(result),
)
elif cleaned_format == "sql":
# Convert from SequenceFile format to Flatbuffers bytes
data_files = sorted(convert_from_sequencefiles(encrypted_file))
# Decrypt serialized encrypted data
_LIB.opaque_decrypt_data(
from_pystr_to_cstr(data_files),
ctypes.c_size_t(len(data_files)),
c_str(plaintext_file),
c_str(symmetric_key_path),
ctypes.byref(result),
)
# Remove intermediate Flatbuffers bytes files
for tmp_file in data_files:
os.remove(tmp_file)
else:
raise CryptoError("Encryption format not currently supported")
if result.value != 0:
raise CryptoError("Decryption failed")
def test_it_should_correct_handle_user_variables():
# set home to defined user
os.environ["HOME"] = "/home/user"
# read env file
env = EnvYAML(yaml_file="tests/env.default.yaml", env_file="tests/test.env")
# print(env['my_source_directory'])
assert env["my_source_directory"] == "/home/user/mapped/source/directory"
def test_it_should_access_keys_and_lists():
env = EnvYAML("tests/env.test.yaml")
assert isinstance(env["keys_and_lists"], dict)
assert isinstance(env["keys_and_lists.one"], list)
assert isinstance(env["keys_and_lists.two"], list)
assert env["keys_and_lists.one.0"] == "one"
assert env["keys_and_lists.two.1.super.one"] == "one"
def connect_to_db(db_param_filename):
# Set up database connection
connection_info = EnvYAML(db_param_filename)
print(connection_info)
return psycopg2.connect(database=connection_info['database'],
host=connection_info['host'],
port=connection_info['port'],
user=connection_info['user'],
password=connection_info['password'])
def test_it_should_has_no_stict_exception_when_set_env():
# set special env to supress strict mode globaly
os.environ[EnvYAML.ENVYAML_STRICT_DISABLE] = ""
env = EnvYAML("tests/env.ignored.yaml", strict=True)
assert env["env_file.config"] == "$ENV_CONFIG_VERSION"
del os.environ[EnvYAML.ENVYAML_STRICT_DISABLE]
def test_it_should_parser_environment_inside_array_and_object():
env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env")
# assert array
assert env["var_in_array.to.0"] == "env-username"
# assert dictionary
assert env["var_in_dict.extra.user"] == "env-username"
assert env["var_in_dict.extra.password"] == "env-password-with-escape"
def test_it_should_return_default_value():
env = EnvYAML(
yaml_file="tests/env.default.yaml", env_file="tests/test.env", strict=True
)
assert env["simple_d"] is None
assert env["simple_e"] == ""
assert env["config.complex"] == "xxxXyyy"
assert env["config.with_default"] == "DEFAULT"
def get_logger_config(self):
'''
Loads the file with logger configuration.
'''
try:
path = Path('config.yaml')
config = EnvYAML(path)
logging.config.dictConfig(config)
except Exception as e:
print(f'Error with logger configuration {repr(e)}')
def test_it_should_not_flatten():
env = EnvYAML(
yaml_file="tests/env.default.yaml",
env_file="tests/test.env",
strict=True,
flatten=False,
)
assert env["config"]["with_default"] == "DEFAULT"
assert "config.with_default" not in env
def test_it_should_pass_escaped_variable():
env = EnvYAML("tests/env.default.yaml", "tests/test.env")
assert env["test_escape.one"] == "$.foo"
assert env["test_escape.two"] == "$meet"
assert env["test_escape.three"] == "${bracket}"
assert env["test_escape.four"] == "SomePa$$$word"
assert env["test_escape.five"] == "SomePa$$${word}"
assert env["test_escape.six"] == "$env-password-with-escape"
assert env["test_escape.seven"] == "$env-password-with-escape"
assert env["test_escape.eight"] == "$DEFAULT"
def test_it_should_read_default_file():
env = EnvYAML()
# access by item name
assert env["one.two.three.value"] == "one-two-three-value"
# access list item
assert env["list_test"][0] == "one"
# access list item (array)
assert env["list_test.0"] == "one"
assert isinstance(env["keys"], dict) and len(env["keys"]) == 2
