def test_decorator_metadata(self):
"""Test @pipeline decorator with metadata."""
@pipeline(name='p1', description='description1')
def my_pipeline1(a: {'Schema':
{
'file_type': 'csv'
}} = 'good',
b: Integer() = 12):
pass
golden_meta = PipelineMeta(name='p1', description='description1')
golden_meta.inputs.append(
ParameterMeta(name='a',
description='',
param_type=TypeMeta(name='Schema',
properties={'file_type': 'csv'}),
default='good'))
golden_meta.inputs.append(
ParameterMeta(name='b',
description='',
param_type=TypeMeta(name='Integer'),
default=12))
pipeline_meta = Pipeline.get_pipeline_functions()[my_pipeline1]
self.assertEqual(pipeline_meta, golden_meta)
def test_component_metadata(self):
"""Test component decorator metadata."""
class MockContainerOp:
def _set_metadata(self, component_meta):
self._metadata = component_meta
@component
def componentA(
a: {'ArtifactA': {
'file_type': 'csv'
}},
b: Integer() = 12,
c: {'ArtifactB': {
'path_type': 'file',
'file_type': 'tsv'
}} = 'gs://hello/world'
) -> {
'model': Integer()
}:
return MockContainerOp()
containerOp = componentA(1, 2, c=3)
golden_meta = ComponentMeta(name='componentA', description='')
golden_meta.inputs.append(
ParameterMeta(name='a',
description='',
param_type=TypeMeta(name='ArtifactA',
properties={'file_type':
'csv'})))
golden_meta.inputs.append(
ParameterMeta(name='b',
description='',
param_type=TypeMeta(name='Integer',
properties={
'openapi_schema_validator': {
"type": "integer"
}
}),
default=12))
golden_meta.inputs.append(
ParameterMeta(name='c',
description='',
param_type=TypeMeta(name='ArtifactB',
properties={
'path_type': 'file',
'file_type': 'tsv'
}),
default='gs://hello/world'))
golden_meta.outputs.append(
ParameterMeta(name='model',
description='',
param_type=TypeMeta(name='Integer',
properties={
'openapi_schema_validator': {
"type": "integer"
}
})))
self.assertEqual(containerOp._metadata, golden_meta)
def test_extract_pipelineparam_with_types(self):
"""Test _extract_pipelineparams. """
p1 = PipelineParam(name='param1',
op_name='op1',
param_type=TypeMeta(
name='customized_type_a',
properties={'property_a': 'value_a'}))
p2 = PipelineParam(name='param2',
param_type=TypeMeta(name='customized_type_b'))
p3 = PipelineParam(name='param3',
value='value3',
param_type=TypeMeta(
name='customized_type_c',
properties={'property_c': 'value_c'}))
stuff_chars = ' between '
payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3)
params = _extract_pipelineparams(payload)
self.assertListEqual([p1, p2, p3], params)
# Expecting the _extract_pipelineparam to dedup the pipelineparams among all the payloads.
payload = [
str(p1) + stuff_chars + str(p2),
str(p2) + stuff_chars + str(p3)
]
params = _extract_pipelineparams(payload)
self.assertListEqual([p1, p2, p3], params)
def test_deserialize(self):
component_dict = {
'GCSPath': {
'bucket_type': 'directory',
'file_type': 'csv'
}
}
golden_type_meta = TypeMeta(name='GCSPath',
properties={
'bucket_type': 'directory',
'file_type': 'csv'
})
self.assertEqual(TypeMeta.deserialize(component_dict),
golden_type_meta)
component_str = 'GCSPath'
golden_type_meta = TypeMeta(name='GCSPath')
self.assertEqual(TypeMeta.deserialize(component_str), golden_type_meta)
def test_eq(self):
type_a = TypeMeta(name='GCSPath',
properties={
'bucket_type': 'directory',
'file_type': 'csv'
})
type_b = TypeMeta(name='GCSPath',
properties={
'bucket_type': 'directory',
'file_type': 'tsv'
})
type_c = TypeMeta(name='GCSPatha',
properties={
'bucket_type': 'directory',
'file_type': 'csv'
})
type_d = TypeMeta(name='GCSPath',
properties={
'bucket_type': 'directory',
'file_type': 'csv'
})
self.assertNotEqual(type_a, type_b)
self.assertNotEqual(type_a, type_c)
self.assertEqual(type_a, type_d)
def test_to_dict(self):
component_meta = ComponentMeta(
name='foobar',
description='foobar example',
inputs=[
ParameterMeta(name='input1',
description='input1 desc',
param_type=TypeMeta(name='GCSPath',
properties={
'bucket_type':
'directory',
'file_type': 'csv'
}),
default='default1'),
ParameterMeta(name='input2',
description='input2 desc',
param_type=TypeMeta(name='TFModel',
properties={
'input_data': 'tensor',
'version': '1.8.0'
}),
default='default2'),
ParameterMeta(name='input3',
description='input3 desc',
param_type=TypeMeta(name='Integer'),
default='default3'),
],
outputs=[
ParameterMeta(name='output1',
description='output1 desc',
param_type=TypeMeta(
name='Schema',
properties={'file_type': 'tsv'}),
default='default_output1')
])
golden_meta = {
'name':
'foobar',
'description':
'foobar example',
'inputs': [{
'name': 'input1',
'description': 'input1 desc',
'type': {
'GCSPath': {
'bucket_type': 'directory',
'file_type': 'csv'
}
},
'default': 'default1'
}, {
'name': 'input2',
'description': 'input2 desc',
'type': {
'TFModel': {
'input_data': 'tensor',
'version': '1.8.0'
}
},
'default': 'default2'
}, {
'name': 'input3',
'description': 'input3 desc',
'type': 'Integer',
'default': 'default3'
}],
'outputs': [{
'name': 'output1',
'description': 'output1 desc',
'type': {
'Schema': {
'file_type': 'tsv'
}
},
'default': 'default_output1'
}]
}
self.assertEqual(component_meta.to_dict(), golden_meta)