def _check_missing_primary_pipeline_field(field: str, error_msg: str):
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_valid.json")
pipeline_json["pipelines"][0].pop(field)
pipeline_definition = PipelineDefinition(pipeline_definition=pipeline_json)
assert pipeline_definition.is_valid() is False
assert error_msg in pipeline_definition.validate()
def test_validation_flags_missing_version_field():
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_valid.json")
pipeline_json["pipelines"][0]["app_data"].pop("version")
pipeline_definition = PipelineDefinition(pipeline_definition=pipeline_json)
assert pipeline_definition.is_valid() is False
assert "Primary pipeline is missing the 'version' field." in pipeline_definition.validate(
)
def _check_pipeline_field_type(field: str, wrong_type_value: any,
error_msg: str):
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_valid.json")
pipeline_json.pop(field)
pipeline_json[field] = wrong_type_value
pipeline_definition = PipelineDefinition(pipeline_definition=pipeline_json)
assert pipeline_definition.is_valid() is False
assert error_msg in pipeline_definition.validate()
async def validate(self, pipeline: Dict) -> ValidationResponse:
"""
Validates the pipeline JSON payload
:param pipeline: the pipeline definition to be validated
:return: ValidationResponse containing any and all issues discovered during the validation
"""
response = ValidationResponse()
pipeline_definition = PipelineDefinition(pipeline_definition=pipeline)
issues = pipeline_definition.validate()
for issue in issues:
response.add_message(severity=ValidationSeverity.Error, message_type="invalidJSON", message=issue)
try:
primary_pipeline = pipeline_definition.primary_pipeline
except ValueError:
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidJSON",
message="Invalid JSON detected, unable to continue.",
)
return response
# Validation can be driven from runtime_config since both runtime and pipeline_type can
# be derived from that and we should not use the 'runtime' and 'runtime_type' fields in
# the pipeline.
# Note: validation updates the pipeline definition with the correct values
# of 'runtime' and 'runtime_type' obtained from 'runtime_config'. We may want to move this
# into PipelineDefinition, but then parsing tests have issues because parsing (tests) assume
# no validation has been applied to the pipeline.
runtime_config = primary_pipeline.runtime_config
if runtime_config is None:
runtime_config = "local"
pipeline_runtime = PipelineValidationManager._determine_runtime(runtime_config)
if PipelineProcessorManager.instance().is_supported_runtime(pipeline_runtime):
# Set the runtime since its derived from runtime_config and valid
primary_pipeline.set("runtime", pipeline_runtime)
else:
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidRuntime",
message="Unsupported pipeline runtime",
data={"pipelineRuntime": pipeline_runtime},
)
self._validate_pipeline_structure(pipeline_definition=pipeline_definition, response=response)
pipeline_type = PipelineValidationManager._determine_runtime_type(runtime_config)
await self._validate_compatibility(
pipeline_definition=pipeline_definition,
pipeline_type=pipeline_type,
pipeline_runtime=pipeline_runtime,
response=response,
)
self._validate_pipeline_graph(pipeline=pipeline, response=response)
if response.has_fatal:
return response
# Set runtime_type since its derived from runtime_config, in case its needed
primary_pipeline.set("runtime_type", pipeline_type)
await self._validate_node_properties(
pipeline_definition=pipeline_definition,
pipeline_type=pipeline_type,
pipeline_runtime=pipeline_runtime,
response=response,
)
return response