(sample_task_object, err) = MLObject.create_object_from_file('tasks/download_data.yaml') if err != {}: pp.pprint(err) else: pp.pprint(sample_task_object.inputs) pp.pprint(sample_task_object.outputs) load_path = Path('tasks').glob("*.yaml") load_list = list(load_path) for this_file in load_list: file_text = this_file.read_text() (loaded_object, err) = MLObject.create_object_from_string(file_text) if err != {}: print(f"ERROR LOADING FILE: {this_file}") pp.pprint(err) print("\n\n") else: print(f"File loaded: {this_file}") pp.pprint(loaded_object) print("\n\n") from mlspeclib import MLObject, MLSchema from pathlib import Path MLSchema.populate_registry() MLSchema.append_schema_to_registry(Path("schemas"))
def load_contract_object( parameters: dict, workflow_object: MLObject, step_name: str, contract_type: str ): """ Creates an MLObject based on an input string, and validates it against the workflow object and step_name provided. Will fail if the .validate() fails on the object or the schema mismatches what is seen in the workflow. """ rootLogger = setupLogger().get_root_logger() if contract_type not in CONTRACT_TYPES: raise KnownException( f"{contract_type} not in the expected list of contract types: {CONTRACT_TYPES}." ) if isinstance(parameters, dict): parameters_string = YAML.safe_dump(parameters) elif isinstance(parameters, str): parameters_string = parameters else: raise KnownException( f"load_contract_object was called with neither a string nor a dict. Value: {parameters}" ) (contract_object, errors) = MLObject.create_object_from_string(parameters_string) if errors is not None and len(errors) > 0: rootLogger.debug(f"{contract_type} object loading errors: {errors}") raise KnownException( f"Error when trying to validate the contract object {step_name}.{contract_type}. Errors: {errors}" ) if step_name not in workflow_object["steps"]: raise KnownException( f"Workflow object does not contain the step '{step_name}'." ) if contract_type not in workflow_object["steps"][step_name]: raise KnownException( f"Workflow object for step '{step_name}' does not contain a spec for the contract type: '{contract_type}'." ) if ( contract_object.schema_type != workflow_object["steps"][step_name][contract_type].schema_type ) or ( contract_object.schema_version != workflow_object["steps"][step_name][contract_type].schema_version ): raise_schema_mismatch( expected_type=workflow_object["steps"][step_name][ contract_type ].schema_type, actual_type=contract_object.schema_type, expected_version=workflow_object["steps"][step_name][ contract_type ].schema_version, actual_version=contract_object.schema_version, ) rootLogger.debug( f"Successfully loaded and validated contract object: {contract_object.schema_type} on step {step_name}.{contract_type}" ) return contract_object
def test_e2e(self): parameters_from_environment = {} integration_tests_dir = parameters_from_environment.get( "INPUT_integration_tests_directory", "integration") # parameters_dir_name = parameters_from_environment.get("INPUT_parameters_directory", "/src/.parameters") variables_file_name = parameters_from_environment.get( "INPUT_integration_tests_variable_file_name", "integration_test_variables.yaml", ) print(os.environ) for var in os.environ: if "INPUT" in var: parameters_from_environment[var] = os.environ.get(var, default=None) parameters_from_file = {} parameters_file_location = Path( integration_tests_dir) / variables_file_name if parameters_file_location.exists(): parameters_from_file = YAML.safe_load( parameters_file_location.read_text("utf-8")) # Building everything into parameters that we'll eventually write to environment variables to execute Docker parameters = {**parameters_from_file, **parameters_from_environment} schemas_dir_name = parameters.get( "INPUT_integration_tests_schemas_dir_name", "/src/parameters/schemas") repo_name = parameters.get("INPUT_container_repo_name") container_name = parameters.get("INPUT_container_name") parameters["INPUT_workflow_version"] = parameters.get( "INPUT_workflow_version", str("999999999999.9." + str(random.randint(0, 9999)))) workflow_version = parameters["INPUT_workflow_version"] MLSchema.append_schema_to_registry(Path(schemas_dir_name)) workflow_input = parameters.get("INPUT_workflow") if isinstance(workflow_input, dict): workflow_string = YAML.safe_dump(workflow_input) else: workflow_string = workflow_input workflow_dict = YAML.safe_load(workflow_string) workflow_dict["workflow_version"] = workflow_version workflow_dict["run_id"] = str(uuid.uuid4()) parameters["GITHUB_RUN_ID"] = workflow_dict["run_id"] parameters["GITHUB_WORKSPACE"] = "/src" workflow_dict["step_id"] = str(uuid.uuid4()) workflow_dict["run_date"] = datetime.datetime.now() workflow_string = YAML.safe_dump(workflow_dict) (workflow_object, errors) = MLObject.create_object_from_string(workflow_string) credentials_packed = parameters_from_environment.get( "INPUT_METASTORE_CREDENTIALS", None) if credentials_packed is None: credentials_packed = (Path(integration_tests_dir) / "metastore_credentials.yaml").read_text( encoding="utf-8") # TODO Sometimes secrets have no spacer. Should figure this out. parameters["INPUT_METASTORE_CREDENTIALS"] = credentials_packed ms = Metastore(credentials_packed) debug_args = "" environment_vars_list = [] workflow_node_id = None try: workflow_node_id = ms.create_workflow_node(workflow_object, workflow_dict["run_id"]) ms.create_workflow_steps(workflow_node_id, workflow_object) parameters["INPUT_workflow_node_id"] = workflow_node_id for param in parameters: if isinstance(parameters[param], dict): env_value = YAML.safe_dump(parameters[param]) else: env_value = parameters[param] debug_args += f" -e '{param}={env_value}'" environment_vars_list.append("-e") environment_vars_list.append(f"{param}={env_value}") exec_statement = [ "docker", "pull", f"{repo_name}/{container_name}", ] print(f"docker pull --no-cache {repo_name}/{container_name}") # self.rootLogger.debug(f"exec_statement = {exec_statement}") p = subprocess.Popen(exec_statement, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() # self.rootLogger.debug(f"out = {str(out)}") # self.rootLogger.debug(f"error = {str(err)}") # self.assertTrue(str(err, "utf-8") == "") exec_statement = (["docker", "run"] + environment_vars_list + [f"{repo_name}/{container_name}"]) # print(f"args statement: '{debug_args}'") print( f"docker run \\\n {debug_args} \\\n -ti --entrypoint=/bin/bash {repo_name}/{container_name}" ) # self.rootLogger.debug(f"exec_statement = {exec_statement}") p = subprocess.Popen(exec_statement, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() self.rootLogger.debug(f"out = {str(out)}") self.rootLogger.debug(f"error = {str(err)}") self.assertTrue(str(err, "utf-8") == "") result = ms.execute_query( f"g.V().has('workflow_node_id', '{workflow_node_id}').count()") self.assertTrue(result[0] == 8) finally: try: if workflow_node_id is not None: ms.execute_query( f"g.V().has('workflow_node_id', '{workflow_node_id}').drop()" ) except ValueError: pass
Path('tests/sample_process_data_execution.py').absolute()) credentials_packed = parameters["INPUT_METASTORE_CREDENTIALS"] ms = Metastore(credentials_packed) workflow_dict = parameters["INPUT_WORKFLOW"] workflow_dict["workflow_version"] = str("999999999999.9." + str(random.randint(0, 9999))) workflow_dict["run_id"] = str(uuid.uuid4()) workflow_dict["step_id"] = str(uuid.uuid4()) workflow_dict["run_date"] = datetime.datetime.now() MLSchema.append_schema_to_registry(Path("tests/schemas_for_test")) (workflow_object, err) = MLObject.create_object_from_string(workflow_dict) if len(err) != 0: raise ValueError(f"Error creating mock workflow_object. Errors: {err}") parameters["INPUT_WORKFLOW_NODE_ID"] = ms.create_workflow_node( workflow_object, str(uuid.uuid4())) parameters["GITHUB_RUN_ID"] = workflow_dict["run_id"] parameters["GITHUB_WORKSPACE"] = "/src" rootLogger.debug(os.environ) bar = buffer.getvalue() environment_vars = "" for param in parameters: