def __init__(self, config: str, runner_index: Optional[int], total_runners: Optional[int]): """Load config. Args: config: Path to config. """ # Use envyaml so that users can set environment variables within the yaml file. data = EnvYAML(config) # Read parameters specified inside table_header self.table_data = [data["table_header"]] self.max_string_width = data.get("table_column_max_width", 0) # Get global tests parameters default_args = data.get("default_args", {}) jobs = (data["tests"] if runner_index is None or total_runners is None else np.array_split( data["tests"], int(total_runners))[int(runner_index) - 1]) # Loop over all jobs # and parse parameters to DriveTestCmd # Join args and default args. self.pipeline = [ DriveTestCmd(**{ **default_args, **args }) for args in jobs ]
def test_it_should_get_default_values(): env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env") assert env.get("empty.novalues", "default") is None assert env.get("empty.noenvvalue", "env-value") == "" # value actually set to "" assert env["empty.noenvvalue"] == ""
def test_it_should_access_environ(): os.environ["ENV_VAR"] = "test-env-var" env = EnvYAML("tests/env.empty.yaml") assert env.environ() == os.environ assert env.environ()["ENV_VAR"] == os.environ["ENV_VAR"] del os.environ["ENV_VAR"]
def test_it_should_read_config_from_env_variable(): # Set env file os.environ["ENV_YAML_FILE"] = "tests/env.test.yaml" os.environ["ENV_FILE"] = "tests/test.env" env = EnvYAML() assert env["env_file.project.name"] == "project-x-42" assert isinstance(env.export(), dict) and len(env.export()) >= 4
def set_config(general_config=None): """ Set the path to the config file. This function must be run before running anything else. Parameters ---------- path : str Path to config file Returns ------- general_config : str The path to the config file. This return value is useful when we're calling `set_config()` without a parameter -- it enables the caller the retrieve the cached config_path """ # If we're specifically configuring the config path if general_config is not None: add_cache_entry("config", general_config) else: general_config = get_cache_entry("config") if general_config is None: raise Exception("Please configure the path to your MC2 config") _CONF["general_config"] = general_config config = EnvYAML(_CONF["general_config"]) _CONF["current_user"] = config["user"]["username"] # Networking configs manual_head_node = config["launch"].get("head") manual_worker_nodes = config["launch"].get("workers") or [] _CONF["azure_config"] = config["launch"].get("azure_config") _CONF["use_azure"] = not (manual_head_node or manual_worker_nodes) # SSH information if _CONF["use_azure"]: azure_config = EnvYAML(_CONF["azure_config"], strict=False) remote_username = azure_config["auth"]["ssh_user"] ssh_key = azure_config["auth"]["ssh_private_key"] # Since `set_config` may be called before Azure VMs may have been # launched, we can't initialize the IPs of the nodes. This value will # be set later once we can guarantee the nodes are launched _CONF["head"] = { "username": remote_username, "ssh_key": ssh_key, "identity": "head node", } _CONF["workers"] = [{ "username": remote_username, "ssh_key": ssh_key, "identity": "worker node", } for _ in range(azure_config["num_workers"])] else: _CONF["head"] = manual_head_node _CONF["workers"] = manual_worker_nodes return general_config
def create_experiment(self, file): """ create user experient """ env = EnvYAML(file) project_id = env['Experiment.projectId'] if project_id is None: print("no project id was set") return with open(file, 'r') as yaml_in: yaml_object = yaml.safe_load( yaml_in) # yaml_object will be a list or a dict experiment_dict = yaml_object['Experiment'] experiment_dict['experimentId'] = "".join( str(uuid.uuid1()).split("-")) experiment_dict['userId'] = self._user_id now = datetime.now() experiment_dict['date'] = now.strftime("%m-%d-%Y") # project id from the env variable experiment_dict['projectId'] = project_id response = self.mutate_single(EXPERIMENT_CREATE_FRAGMENT, experiment_dict) new_experiment_created = response["createExperiment"]["ok"] return self.experiment(new_experiment_created)
def test_it_should_proper_handle_dollar_sign_with_number(): env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env") assert ( env["sql"] == 'SELECT * FROM "users" WHERE "user" = $1 AND "login" = $2 AND "pwd" = $3' )
def generate_symmetric_key(): """ Generate a new symmetric key and save it path specified by user in config YAML passed to `set_config()` Parameters ---------- num_bytes : int Number of bytes for key """ num_bytes = _LIB.cipher_key_size() if _CONF.get("general_config") is None: raise MC2ClientConfigError("Configuration not set") symmetric_key_path = EnvYAML( _CONF["general_config"])["user"]["symmetric_key"] if os.path.exists(symmetric_key_path): logger.warning( "Skipping symmetric key generation - key already exists at {}". format(symmetric_key_path)) return key = AESGCM.generate_key(bit_length=num_bytes * 8) with open(symmetric_key_path, "wb") as symm_key: symm_key.write(key) logger.info("Generated symmetric key and outputted to {}".format( symmetric_key_path))
def yaml_form(request): if request.method == "POST": form = SnippetForm(request.POST) if form.is_valid(): form.save() name = form.cleaned_data['region'] # with yaml library with open("template.yaml", 'r') as stream: data_loaded = yaml.safe_load(stream) data_loaded['Resources']['MyEC2Instance']['Properties'][ 'AvailabilityZone'] = name with open("new_template.yaml", "w") as f: yaml.dump(data_loaded, f) # with env variables and envyaml library (env variables in template must be referenced with $VAR) os.environ['NAME'] = name data = EnvYAML('template.yaml') print(data) with open("new_template.yaml", "w") as f: yaml.dump(data['Resources'], f) # with yaml.add_implicit_resolver and yaml.add_constructor (env variables in template mus be referenced with ${VAR}) os.environ['NAME'] = name with open('template.yaml', 'r') as stream: p = yaml.load(stream, Loader=yaml.FullLoader) print(p['Resources']) with open("new_template.yaml", "w") as f: yaml.dump(p, f) form = SnippetForm() return render(request, 'form.html', {'form': form})
def config(tmp_path): tests_dir = pathlib.Path(__file__).parent.absolute() original_config_path = os.path.join(tests_dir, "test.yaml") test_cert = os.path.join(tmp_path, "test.crt") test_priv_key = os.path.join(tmp_path, "test.pem") test_symm_key = os.path.join(tmp_path, "test_sym.key") # Rewrite config YAML with test paths config = EnvYAML(original_config_path) config["user"]["certificate"] = test_cert config["user"]["private_key"] = test_priv_key config["user"]["symmetric_key"] = test_symm_key # Point to root certificate config["user"]["root_private_key"] = os.path.join(tests_dir, "keys/root.pem") config["user"]["root_certificate"] = os.path.join(tests_dir, "keys/root.crt") test_config_path = os.path.join(tmp_path, "config.yaml") with open(test_config_path, "w") as out: yaml.dump(dict(config), out, default_flow_style=False) mc2.set_config(test_config_path) return tests_dir
def __init__(self): self.__config_path = os.getenv("EDGE_CONF_FILE") if self.__config_path is None: raise Exception("The config path is not correct") self.__config: EnvYAML = EnvYAML(self.__config_path) self.__rabbitMQConfig: RMQConfig = None self.__severConfig: APIServerConfig = None
def test_it_should_be_valid_in_check(): env = EnvYAML(env_file="tests/env.test.yaml") if "test.one" in env: assert env["test.one"] == 123 assert "test.not_exists" not in env
def test_it_should_access_environment_variables(): os.environ["ENV_VAR"] = "test-env-var" env = EnvYAML("tests/env.empty.yaml") assert env["ENV_VAR"] == "test-env-var" del os.environ["ENV_VAR"]
def load_yaml(path): """ Args: path (str): File path of yaml configuration file Returns: Dict[str, Any]: """ return EnvYAML(path, include_environment=False).export()
def decrypt_data(encrypted_file, plaintext_file, enc_format): """ Encrypt a file in a certain format Parameters ---------- encrypted_file : str Path to encrypted data to decrypt plaintext_file : str Path to decrypted data enc_format : str The encryption format (i.e. `sql` for Opaque SQL or `xgb` for Secure XGBoost) """ if _CONF.get("general_config") is None: raise MC2ClientConfigError("Configuration not set") if not os.path.exists(encrypted_file): raise FileNotFoundError( "Cannot find file to decrypt: {}".format(encrypted_file)) cleaned_format = "".join(enc_format.split()).lower() symmetric_key_path = EnvYAML( _CONF["general_config"])["user"]["symmetric_key"] if not os.path.exists(symmetric_key_path): raise FileNotFoundError( "Symmetric key not found at {}".format(symmetric_key_path)) result = ctypes.c_int() if cleaned_format == "xgb": _LIB.sxgb_decrypt_data( c_str(encrypted_file), c_str(plaintext_file), c_str(symmetric_key_path), ctypes.byref(result), ) elif cleaned_format == "sql": # Convert from SequenceFile format to Flatbuffers bytes data_files = sorted(convert_from_sequencefiles(encrypted_file)) # Decrypt serialized encrypted data _LIB.opaque_decrypt_data( from_pystr_to_cstr(data_files), ctypes.c_size_t(len(data_files)), c_str(plaintext_file), c_str(symmetric_key_path), ctypes.byref(result), ) # Remove intermediate Flatbuffers bytes files for tmp_file in data_files: os.remove(tmp_file) else: raise CryptoError("Encryption format not currently supported") if result.value != 0: raise CryptoError("Decryption failed")
def test_it_should_correct_handle_user_variables(): # set home to defined user os.environ["HOME"] = "/home/user" # read env file env = EnvYAML(yaml_file="tests/env.default.yaml", env_file="tests/test.env") # print(env['my_source_directory']) assert env["my_source_directory"] == "/home/user/mapped/source/directory"
def test_it_should_access_keys_and_lists(): env = EnvYAML("tests/env.test.yaml") assert isinstance(env["keys_and_lists"], dict) assert isinstance(env["keys_and_lists.one"], list) assert isinstance(env["keys_and_lists.two"], list) assert env["keys_and_lists.one.0"] == "one" assert env["keys_and_lists.two.1.super.one"] == "one"
def connect_to_db(db_param_filename): # Set up database connection connection_info = EnvYAML(db_param_filename) print(connection_info) return psycopg2.connect(database=connection_info['database'], host=connection_info['host'], port=connection_info['port'], user=connection_info['user'], password=connection_info['password'])
def test_it_should_has_no_stict_exception_when_set_env(): # set special env to supress strict mode globaly os.environ[EnvYAML.ENVYAML_STRICT_DISABLE] = "" env = EnvYAML("tests/env.ignored.yaml", strict=True) assert env["env_file.config"] == "$ENV_CONFIG_VERSION" del os.environ[EnvYAML.ENVYAML_STRICT_DISABLE]
def test_it_should_parser_environment_inside_array_and_object(): env = EnvYAML("tests/env.test.yaml", env_file="tests/test.env") # assert array assert env["var_in_array.to.0"] == "env-username" # assert dictionary assert env["var_in_dict.extra.user"] == "env-username" assert env["var_in_dict.extra.password"] == "env-password-with-escape"
def test_it_should_return_default_value(): env = EnvYAML( yaml_file="tests/env.default.yaml", env_file="tests/test.env", strict=True ) assert env["simple_d"] is None assert env["simple_e"] == "" assert env["config.complex"] == "xxxXyyy" assert env["config.with_default"] == "DEFAULT"
def get_logger_config(self): ''' Loads the file with logger configuration. ''' try: path = Path('config.yaml') config = EnvYAML(path) logging.config.dictConfig(config) except Exception as e: print(f'Error with logger configuration {repr(e)}')
def test_it_should_not_flatten(): env = EnvYAML( yaml_file="tests/env.default.yaml", env_file="tests/test.env", strict=True, flatten=False, ) assert env["config"]["with_default"] == "DEFAULT" assert "config.with_default" not in env
def test_it_should_pass_escaped_variable(): env = EnvYAML("tests/env.default.yaml", "tests/test.env") assert env["test_escape.one"] == "$.foo" assert env["test_escape.two"] == "$meet" assert env["test_escape.three"] == "${bracket}" assert env["test_escape.four"] == "SomePa$$$word" assert env["test_escape.five"] == "SomePa$$${word}" assert env["test_escape.six"] == "$env-password-with-escape" assert env["test_escape.seven"] == "$env-password-with-escape" assert env["test_escape.eight"] == "$DEFAULT"
def test_it_should_read_default_file(): env = EnvYAML() # access by item name assert env["one.two.three.value"] == "one-two-three-value" # access list item assert env["list_test"][0] == "one" # access list item (array) assert env["list_test.0"] == "one" assert isinstance(env["keys"], dict) and len(env["keys"]) == 2