def test_pipeline_operations_and_handle_artifact_file_details():
pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_3_node_sample.json")
pipeline = PipelineParser().parse(pipeline_json)
assert len(pipeline.operations) == 3
for op in pipeline.operations.values():
assert "." not in op.name
def test_pipeline_with_dirty_list_values(valid_operation):
pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_with_invalid_list_values.json")
pipeline = PipelineParser().parse(pipeline_json)
assert pipeline.name == "{{name}}"
assert pipeline.runtime == "{{runtime}}"
assert pipeline.runtime_config == "{{runtime-config}}"
assert len(pipeline.operations) == 1
assert pipeline.operations["{{uuid}}"] == valid_operation
def test_pipeline_get_envs():
# Ensure pipeline operation env lists are properly converted to dictionaries.
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_dependency_complex.json")
pipeline = PipelineParser().parse(pipeline_json)
for op in pipeline.operations.values():
assert isinstance(op, GenericOperation)
op_envs = op.env_vars.to_dict()
assert op_envs["OP_NAME"] == op.name
async def post(self, *args, **kwargs):
self.log.debug("Pipeline Export handler now executing post request")
parent = self.settings.get("elyra")
payload = self.get_json_body()
self.log.debug(
f"JSON payload: {json.dumps(payload, indent=2, separators=(',', ': '))}"
)
pipeline_definition = payload["pipeline"]
pipeline_export_format = payload["export_format"]
pipeline_export_path = payload["export_path"]
pipeline_overwrite = payload["overwrite"]
response = await PipelineValidationManager.instance().validate(
pipeline=pipeline_definition)
self.log.debug(
f"Validation checks completed. Results as follows: {response.to_json()}"
)
if not response.has_fatal:
pipeline = PipelineParser(
root_dir=self.settings["server_root_dir"],
parent=parent).parse(pipeline_definition)
pipeline_exported_path = await PipelineProcessorManager.instance(
).export(pipeline, pipeline_export_format, pipeline_export_path,
pipeline_overwrite)
json_msg = json.dumps({"export_path": pipeline_export_path})
self.set_status(201)
self.set_header("Content-Type", "application/json")
location = url_path_join(self.base_url, "api", "contents",
pipeline_exported_path)
self.set_header("Location", location)
else:
json_msg = json.dumps({
"reason":
responses.get(400),
"message":
"Errors found in pipeline",
"timestamp":
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"issues":
response.to_json().get("issues"),
})
self.set_status(400)
self.set_header("Content-Type", "application/json")
await self.finish(json_msg)
def _execute_pipeline(pipeline_definition) -> PipelineProcessorResponse:
try:
# parse pipeline
pipeline_object = PipelineParser().parse(pipeline_definition)
# process pipeline
with warnings.catch_warnings():
warnings.simplefilter("ignore")
response = asyncio.get_event_loop().run_until_complete(
PipelineProcessorManager.instance().process(pipeline_object)
)
return response
except ValueError as ve:
raise click.ClickException(f"Error parsing pipeline: \n {ve}")
except RuntimeError as re:
raise click.ClickException(f"Error processing pipeline: \n {re} \n {re.__cause__}")
def test_pipeline_execution_order_in_simple_pipeline():
expected_operation_names = ["f", "a", "c", "g"]
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_dependency_simple.json")
pipeline = PipelineParser().parse(pipeline_json)
current_ordered_operation_names = _get_operation_names(
pipeline.operations.values())
assert current_ordered_operation_names != expected_operation_names
operations = LocalPipelineProcessor._sort_operations(
operations_by_id=pipeline.operations)
ordered_operation_names = _get_operation_names(operations)
assert ordered_operation_names == expected_operation_names
async def post(self, *args, **kwargs):
self.log.debug("Pipeline SchedulerHandler now executing post request")
parent = self.settings.get("elyra")
pipeline_definition = self.get_json_body()
self.log.debug(f"JSON payload: {pipeline_definition}")
response = await PipelineValidationManager.instance().validate(
pipeline=pipeline_definition)
self.log.debug(
f"Validation checks completed. Results as follows: {response.to_json()}"
)
if not response.has_fatal:
self.log.debug(
"Processing the pipeline submission and executing request")
pipeline = PipelineParser(
root_dir=self.settings["server_root_dir"],
parent=parent).parse(pipeline_definition)
response = await PipelineProcessorManager.instance().process(
pipeline)
json_msg = json.dumps(response.to_json())
self.set_status(200)
else:
json_msg = json.dumps({
"reason":
responses.get(400),
"message":
"Errors found in pipeline",
"timestamp":
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"issues":
response.to_json().get("issues"),
})
self.set_status(400)
self.set_header("Content-Type", "application/json")
await self.finish(json_msg)
def parsed_pipeline():
pipeline_resource = _read_pipeline_resource(PIPELINE_FILE)
return PipelineParser().parse(pipeline_definitions=pipeline_resource)
def export(pipeline_path, runtime_config, output, overwrite):
"""
Export a pipeline to a runtime-specific format
"""
click.echo()
print_banner("Elyra pipeline export")
rtc = _get_runtime_config(runtime_config)
runtime_schema = rtc.schema_name
runtime_type = rtc.metadata.get("runtime_type")
pipeline_definition = _preprocess_pipeline(pipeline_path, runtime=runtime_schema, runtime_config=runtime_config)
# Verify that the pipeline's runtime type is compatible with the
# runtime configuration
pipeline_runtime_type = _get_pipeline_runtime_type(pipeline_definition)
if pipeline_runtime_type and pipeline_runtime_type != "Generic" and pipeline_runtime_type != runtime_type:
raise click.BadParameter(
f"The runtime configuration type '{runtime_type}' does not match "
f"the pipeline's runtime type '{pipeline_runtime_type}'.",
param_hint="--runtime-config",
)
resources = RuntimeTypeResources.get_instance_by_type(RuntimeProcessorType.get_instance_by_name(runtime_type))
supported_export_formats = resources.get_export_extensions()
if len(supported_export_formats) == 0:
raise click.ClickException(f"Runtime type '{runtime_type}' does not support export.")
# If, in the future, a runtime supports multiple export output formats,
# the user can choose one. For now, choose the only option.
selected_export_format = supported_export_formats[0]
selected_export_format_suffix = f".{selected_export_format}"
# generate output file name from the user-provided input
if output is None:
# user did not specify an output; use current directory
# and derive the file name from the pipeline file name
output_path = Path.cwd()
filename = f"{Path(pipeline_path).stem}{selected_export_format_suffix}"
else:
if output.suffix == selected_export_format_suffix:
# user provided a file name
output_path = output.parent
filename = output.name
else:
# user provided a directory
output_path = output
filename = f"{Path(pipeline_path).stem}{selected_export_format_suffix}"
output_file = output_path.resolve() / filename
# verify that the output path meets the prerequisites
if not output_file.parent.is_dir():
try:
output_file.parent.mkdir(parents=True, exist_ok=True)
except Exception as ex:
raise click.BadParameter(f"Cannot create output directory: {ex}", param_hint="--output")
# handle output overwrite
if output_file.exists() and not overwrite:
raise click.ClickException(
f"Output file '{str(output_file)}' exists and " "option '--overwrite' was not specified."
)
if pipeline_runtime_type:
_build_component_cache()
# validate the pipeline
try:
_validate_pipeline_definition(pipeline_definition)
except Exception:
raise click.ClickException("Pipeline validation FAILED. The pipeline was not exported.")
with Spinner(text="Exporting pipeline ..."):
try:
# parse pipeline
pipeline_object = PipelineParser().parse(pipeline_definition)
# process pipeline
with warnings.catch_warnings():
warnings.simplefilter("ignore")
asyncio.get_event_loop().run_until_complete(
PipelineProcessorManager.instance().export(
pipeline_object, selected_export_format, str(output_file), True
)
)
except ValueError as ve:
raise click.ClickException(f"Error parsing pipeline: \n {ve}")
except RuntimeError as re:
raise click.ClickException(f"Error exporting pipeline: \n {re} \n {re.__cause__}")
click.echo(f"Pipeline was exported to '{str(output_file)}'.")
def test_multinode_pipeline():
pipeline_json = _read_pipeline_resource("resources/sample_pipelines/pipeline_3_node_sample.json")
pipeline = PipelineParser().parse(pipeline_json)
assert len(pipeline.operations) == 3
def test_multiple_pipeline_definition():
pipeline_json = _read_pipeline_resource("resources/sample_pipelines/" "pipeline_multiple_pipeline_definitions.json")
with pytest.raises(ValueError):
PipelineParser().parse(pipeline_json)
def parsed_pipeline(request):
pipeline_resource = _read_pipeline_resource(request.param)
return PipelineParser().parse(pipeline_json=pipeline_resource)
def pipeline():
pipeline_resource = _read_pipeline_resource(
'resources/sample_pipelines/pipeline_3_node_sample.json')
return PipelineParser.parse(pipeline_resource)
def pipeline():
ComponentCache.instance().wait_for_all_cache_tasks()
pipeline_resource = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_3_node_sample.json")
return PipelineParser.parse(pipeline_resource)
