def test_pipeline_operations_and_handle_artifact_file_details(): pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_3_node_sample.json") pipeline = PipelineParser().parse(pipeline_json) assert len(pipeline.operations) == 3 for op in pipeline.operations.values(): assert "." not in op.name
def test_pipeline_with_dirty_list_values(valid_operation): pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_with_invalid_list_values.json") pipeline = PipelineParser().parse(pipeline_json) assert pipeline.name == "{{name}}" assert pipeline.runtime == "{{runtime}}" assert pipeline.runtime_config == "{{runtime-config}}" assert len(pipeline.operations) == 1 assert pipeline.operations["{{uuid}}"] == valid_operation
def test_pipeline_get_envs(): # Ensure pipeline operation env lists are properly converted to dictionaries. pipeline_json = _read_pipeline_resource( "resources/sample_pipelines/pipeline_dependency_complex.json") pipeline = PipelineParser().parse(pipeline_json) for op in pipeline.operations.values(): assert isinstance(op, GenericOperation) op_envs = op.env_vars.to_dict() assert op_envs["OP_NAME"] == op.name
async def post(self, *args, **kwargs): self.log.debug("Pipeline Export handler now executing post request") parent = self.settings.get("elyra") payload = self.get_json_body() self.log.debug( f"JSON payload: {json.dumps(payload, indent=2, separators=(',', ': '))}" ) pipeline_definition = payload["pipeline"] pipeline_export_format = payload["export_format"] pipeline_export_path = payload["export_path"] pipeline_overwrite = payload["overwrite"] response = await PipelineValidationManager.instance().validate( pipeline=pipeline_definition) self.log.debug( f"Validation checks completed. Results as follows: {response.to_json()}" ) if not response.has_fatal: pipeline = PipelineParser( root_dir=self.settings["server_root_dir"], parent=parent).parse(pipeline_definition) pipeline_exported_path = await PipelineProcessorManager.instance( ).export(pipeline, pipeline_export_format, pipeline_export_path, pipeline_overwrite) json_msg = json.dumps({"export_path": pipeline_export_path}) self.set_status(201) self.set_header("Content-Type", "application/json") location = url_path_join(self.base_url, "api", "contents", pipeline_exported_path) self.set_header("Location", location) else: json_msg = json.dumps({ "reason": responses.get(400), "message": "Errors found in pipeline", "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "issues": response.to_json().get("issues"), }) self.set_status(400) self.set_header("Content-Type", "application/json") await self.finish(json_msg)
def _execute_pipeline(pipeline_definition) -> PipelineProcessorResponse: try: # parse pipeline pipeline_object = PipelineParser().parse(pipeline_definition) # process pipeline with warnings.catch_warnings(): warnings.simplefilter("ignore") response = asyncio.get_event_loop().run_until_complete( PipelineProcessorManager.instance().process(pipeline_object) ) return response except ValueError as ve: raise click.ClickException(f"Error parsing pipeline: \n {ve}") except RuntimeError as re: raise click.ClickException(f"Error processing pipeline: \n {re} \n {re.__cause__}")
def test_pipeline_execution_order_in_simple_pipeline(): expected_operation_names = ["f", "a", "c", "g"] pipeline_json = _read_pipeline_resource( "resources/sample_pipelines/pipeline_dependency_simple.json") pipeline = PipelineParser().parse(pipeline_json) current_ordered_operation_names = _get_operation_names( pipeline.operations.values()) assert current_ordered_operation_names != expected_operation_names operations = LocalPipelineProcessor._sort_operations( operations_by_id=pipeline.operations) ordered_operation_names = _get_operation_names(operations) assert ordered_operation_names == expected_operation_names
async def post(self, *args, **kwargs): self.log.debug("Pipeline SchedulerHandler now executing post request") parent = self.settings.get("elyra") pipeline_definition = self.get_json_body() self.log.debug(f"JSON payload: {pipeline_definition}") response = await PipelineValidationManager.instance().validate( pipeline=pipeline_definition) self.log.debug( f"Validation checks completed. Results as follows: {response.to_json()}" ) if not response.has_fatal: self.log.debug( "Processing the pipeline submission and executing request") pipeline = PipelineParser( root_dir=self.settings["server_root_dir"], parent=parent).parse(pipeline_definition) response = await PipelineProcessorManager.instance().process( pipeline) json_msg = json.dumps(response.to_json()) self.set_status(200) else: json_msg = json.dumps({ "reason": responses.get(400), "message": "Errors found in pipeline", "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "issues": response.to_json().get("issues"), }) self.set_status(400) self.set_header("Content-Type", "application/json") await self.finish(json_msg)
def parsed_pipeline(): pipeline_resource = _read_pipeline_resource(PIPELINE_FILE) return PipelineParser().parse(pipeline_definitions=pipeline_resource)
def export(pipeline_path, runtime_config, output, overwrite): """ Export a pipeline to a runtime-specific format """ click.echo() print_banner("Elyra pipeline export") rtc = _get_runtime_config(runtime_config) runtime_schema = rtc.schema_name runtime_type = rtc.metadata.get("runtime_type") pipeline_definition = _preprocess_pipeline(pipeline_path, runtime=runtime_schema, runtime_config=runtime_config) # Verify that the pipeline's runtime type is compatible with the # runtime configuration pipeline_runtime_type = _get_pipeline_runtime_type(pipeline_definition) if pipeline_runtime_type and pipeline_runtime_type != "Generic" and pipeline_runtime_type != runtime_type: raise click.BadParameter( f"The runtime configuration type '{runtime_type}' does not match " f"the pipeline's runtime type '{pipeline_runtime_type}'.", param_hint="--runtime-config", ) resources = RuntimeTypeResources.get_instance_by_type(RuntimeProcessorType.get_instance_by_name(runtime_type)) supported_export_formats = resources.get_export_extensions() if len(supported_export_formats) == 0: raise click.ClickException(f"Runtime type '{runtime_type}' does not support export.") # If, in the future, a runtime supports multiple export output formats, # the user can choose one. For now, choose the only option. selected_export_format = supported_export_formats[0] selected_export_format_suffix = f".{selected_export_format}" # generate output file name from the user-provided input if output is None: # user did not specify an output; use current directory # and derive the file name from the pipeline file name output_path = Path.cwd() filename = f"{Path(pipeline_path).stem}{selected_export_format_suffix}" else: if output.suffix == selected_export_format_suffix: # user provided a file name output_path = output.parent filename = output.name else: # user provided a directory output_path = output filename = f"{Path(pipeline_path).stem}{selected_export_format_suffix}" output_file = output_path.resolve() / filename # verify that the output path meets the prerequisites if not output_file.parent.is_dir(): try: output_file.parent.mkdir(parents=True, exist_ok=True) except Exception as ex: raise click.BadParameter(f"Cannot create output directory: {ex}", param_hint="--output") # handle output overwrite if output_file.exists() and not overwrite: raise click.ClickException( f"Output file '{str(output_file)}' exists and " "option '--overwrite' was not specified." ) if pipeline_runtime_type: _build_component_cache() # validate the pipeline try: _validate_pipeline_definition(pipeline_definition) except Exception: raise click.ClickException("Pipeline validation FAILED. The pipeline was not exported.") with Spinner(text="Exporting pipeline ..."): try: # parse pipeline pipeline_object = PipelineParser().parse(pipeline_definition) # process pipeline with warnings.catch_warnings(): warnings.simplefilter("ignore") asyncio.get_event_loop().run_until_complete( PipelineProcessorManager.instance().export( pipeline_object, selected_export_format, str(output_file), True ) ) except ValueError as ve: raise click.ClickException(f"Error parsing pipeline: \n {ve}") except RuntimeError as re: raise click.ClickException(f"Error exporting pipeline: \n {re} \n {re.__cause__}") click.echo(f"Pipeline was exported to '{str(output_file)}'.")
def test_multinode_pipeline(): pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_3_node_sample.json") pipeline = PipelineParser().parse(pipeline_json) assert len(pipeline.operations) == 3
def test_multiple_pipeline_definition(): pipeline_json = _read_pipeline_resource("resources/sample_pipelines/" "pipeline_multiple_pipeline_definitions.json") with pytest.raises(ValueError): PipelineParser().parse(pipeline_json)
def parsed_pipeline(request): pipeline_resource = _read_pipeline_resource(request.param) return PipelineParser().parse(pipeline_json=pipeline_resource)
def pipeline(): pipeline_resource = _read_pipeline_resource( 'resources/sample_pipelines/pipeline_3_node_sample.json') return PipelineParser.parse(pipeline_resource)
def pipeline(): ComponentCache.instance().wait_for_all_cache_tasks() pipeline_resource = _read_pipeline_resource( "resources/sample_pipelines/pipeline_3_node_sample.json") return PipelineParser.parse(pipeline_resource)