def test_extract_pipeline_channels(self):
p1 = pipeline_channel.PipelineParameterChannel(
name='channel1',
channel_type='String',
value='abc',
)
p2 = pipeline_channel.PipelineArtifactChannel(
name='channel2',
channel_type='customized_type_b',
task_name='task2',
)
p3 = pipeline_channel.PipelineArtifactChannel(
name='channel3',
channel_type={'customized_type_c': {
'property_c': 'value_c'
}},
task_name='task3',
)
stuff_chars = ' between '
payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3)
params = pipeline_channel.extract_pipeline_channels_from_string(payload)
self.assertListEqual([p1, p2, p3], params)
# Expecting the extract_pipelineparam_from_any to dedup pipeline channels
# among all the payloads.
payload = [
str(p1) + stuff_chars + str(p2),
str(p2) + stuff_chars + str(p3)
]
params = pipeline_channel.extract_pipeline_channels_from_any(payload)
self.assertListEqual([p1, p2, p3], params)
def test_task_name_and_value_both_set(self):
with self.assertRaisesRegex(ValueError,
'task_name and value cannot be both set.'):
p = pipeline_channel.PipelineParameterChannel(
name='abc',
channel_type='Integer',
task_name='task1',
value=123,
)
def test_invalid_name(self):
with self.assertRaisesRegex(
ValueError,
'Only letters, numbers, spaces, "_", and "-" are allowed in the '
'name. Must begin with a letter. Got name: 123_abc'):
p = pipeline_channel.PipelineParameterChannel(
name='123_abc',
channel_type='String',
)
def test_invalid_type(self):
with self.assertRaisesRegex(TypeError,
'Artifact is not a parameter type.'):
p = pipeline_channel.PipelineParameterChannel(
name='channel1',
channel_type='Artifact',
)
with self.assertRaisesRegex(TypeError,
'String is not an artifact type.'):
p = pipeline_channel.PipelineArtifactChannel(
name='channel1',
channel_type='String',
task_name='task1',
)
def test_create_loop_argument_varaible(self, subvar_name, valid):
loop_argument = for_loop.LoopArgument.from_pipeline_channel(
pipeline_channel.PipelineParameterChannel(
name='param1',
channel_type='List[Dict[str, str]]',
))
if valid:
loop_arg_var = for_loop.LoopArgumentVariable(
loop_argument=loop_argument,
subvar_name=subvar_name,
)
self.assertEqual(loop_arg_var.loop_argument, loop_argument)
self.assertEqual(loop_arg_var.subvar_name, subvar_name)
else:
with self.assertRaisesRegex(ValueError,
'Tried to create subvariable'):
for_loop.LoopArgumentVariable(
loop_argument=loop_argument,
subvar_name=subvar_name,
)
class PipelineChannelTest(parameterized.TestCase):
def test_instantiate_pipline_channel(self):
with self.assertRaisesRegex(
TypeError, "Can't instantiate abstract class PipelineChannel"):
p = pipeline_channel.PipelineChannel(
name='channel',
channel_type='String',
)
def test_invalid_name(self):
with self.assertRaisesRegex(
ValueError,
'Only letters, numbers, spaces, "_", and "-" are allowed in the '
'name. Must begin with a letter. Got name: 123_abc'):
p = pipeline_channel.PipelineParameterChannel(
name='123_abc',
channel_type='String',
)
def test_task_name_and_value_both_set(self):
with self.assertRaisesRegex(ValueError,
'task_name and value cannot be both set.'):
p = pipeline_channel.PipelineParameterChannel(
name='abc',
channel_type='Integer',
task_name='task1',
value=123,
)
def test_invalid_type(self):
with self.assertRaisesRegex(TypeError,
'Artifact is not a parameter type.'):
p = pipeline_channel.PipelineParameterChannel(
name='channel1',
channel_type='Artifact',
)
with self.assertRaisesRegex(TypeError,
'String is not an artifact type.'):
p = pipeline_channel.PipelineArtifactChannel(
name='channel1',
channel_type='String',
task_name='task1',
)
@parameterized.parameters(
{
'pipeline_channel':
pipeline_channel.PipelineParameterChannel(
name='channel1',
task_name='task1',
channel_type='String',
),
'str_repr':
'{{channel:task=task1;name=channel1;type=String;}}',
},
{
'pipeline_channel':
pipeline_channel.PipelineParameterChannel(
name='channel2',
channel_type='Integer',
),
'str_repr':
'{{channel:task=;name=channel2;type=Integer;}}',
},
{
'pipeline_channel':
pipeline_channel.PipelineArtifactChannel(
name='channel3',
channel_type={'type_a': {
'property_b': 'c'
}},
task_name='task3',
),
'str_repr':
'{{channel:task=task3;name=channel3;type={"type_a": {"property_b": "c"}};}}',
},
{
'pipeline_channel':
pipeline_channel.PipelineParameterChannel(
name='channel4',
channel_type='Float',
value=1.23,
),
'str_repr':
'{{channel:task=;name=channel4;type=Float;}}',
},
{
'pipeline_channel':
pipeline_channel.PipelineArtifactChannel(
name='channel5',
channel_type='Artifact',
task_name='task5',
),
'str_repr':
'{{channel:task=task5;name=channel5;type=Artifact;}}',
},
)
def test_str_repr(self, pipeline_channel, str_repr):
self.assertEqual(str_repr, str(pipeline_channel))
def test_extract_pipeline_channels(self):
p1 = pipeline_channel.PipelineParameterChannel(
name='channel1',
channel_type='String',
value='abc',
)
p2 = pipeline_channel.PipelineArtifactChannel(
name='channel2',
channel_type='customized_type_b',
task_name='task2',
)
p3 = pipeline_channel.PipelineArtifactChannel(
name='channel3',
channel_type={'customized_type_c': {
'property_c': 'value_c'
}},
task_name='task3',
)
stuff_chars = ' between '
payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3)
params = pipeline_channel.extract_pipeline_channels_from_string(payload)
self.assertListEqual([p1, p2, p3], params)
# Expecting the extract_pipelineparam_from_any to dedup pipeline channels
# among all the payloads.
payload = [
str(p1) + stuff_chars + str(p2),
str(p2) + stuff_chars + str(p3)
]
params = pipeline_channel.extract_pipeline_channels_from_any(payload)
self.assertListEqual([p1, p2, p3], params)
class PipelineSpecBuilderTest(parameterized.TestCase):
def setUp(self):
self.maxDiff = None
@parameterized.parameters(
{
'channel':
pipeline_channel.PipelineParameterChannel(
name='output1', task_name='task1', channel_type='String'),
'expected':
'pipelinechannel--task1-output1',
},
{
'channel':
pipeline_channel.PipelineArtifactChannel(
name='output1', task_name='task1', channel_type='Artifact'),
'expected':
'pipelinechannel--task1-output1',
},
{
'channel':
pipeline_channel.PipelineParameterChannel(name='param1',
channel_type='String'),
'expected':
'pipelinechannel--param1',
},
)
def test_additional_input_name_for_pipeline_channel(
self, channel, expected):
self.assertEqual(
expected,
pipeline_spec_builder._additional_input_name_for_pipeline_channel(
channel))
@parameterized.parameters(
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER,
'default_value': None,
'expected': struct_pb2.Value(),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER,
'default_value': 1,
'expected': struct_pb2.Value(number_value=1),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE,
'default_value': 1.2,
'expected': struct_pb2.Value(number_value=1.2),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.STRING,
'default_value': 'text',
'expected': struct_pb2.Value(string_value='text'),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.BOOLEAN,
'default_value': True,
'expected': struct_pb2.Value(bool_value=True),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.BOOLEAN,
'default_value': False,
'expected': struct_pb2.Value(bool_value=False),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.STRUCT,
'default_value': {
'a': 1,
'b': 2,
},
'expected':
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=1),
'b': struct_pb2.Value(number_value=2),
})),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.LIST,
'default_value': ['a', 'b'],
'expected':
struct_pb2.Value(list_value=struct_pb2.ListValue(values=[
struct_pb2.Value(string_value='a'),
struct_pb2.Value(string_value='b'),
])),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.LIST,
'default_value': [{
'a': 1,
'b': 2
}, {
'a': 10,
'b': 20
}],
'expected':
struct_pb2.Value(list_value=struct_pb2.ListValue(values=[
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=1),
'b': struct_pb2.Value(number_value=2),
})),
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=10),
'b': struct_pb2.Value(number_value=20),
})),
])),
},
)
def test_fill_in_component_input_default_value(self, parameter_type,
default_value, expected):
component_spec = pipeline_spec_pb2.ComponentSpec(
input_definitions=pipeline_spec_pb2.ComponentInputsSpec(
parameters={
'input1':
pipeline_spec_pb2.ComponentInputsSpec.ParameterSpec(
parameter_type=parameter_type)
}))
pipeline_spec_builder._fill_in_component_input_default_value(
component_spec=component_spec,
input_name='input1',
default_value=default_value)
self.assertEqual(
expected,
component_spec.input_definitions.parameters['input1'].
default_value,
)
class ForLoopTest(parameterized.TestCase):
@parameterized.parameters(
{
'collection_type': 'List[int]',
'item_type': 'int',
},
{
'collection_type': 'typing.List[str]',
'item_type': 'str',
},
{
'collection_type': 'typing.Tuple[ float ]',
'item_type': 'float',
},
{
'collection_type': 'typing.Sequence[Dict[str, str]]',
'item_type': 'Dict[str, str]',
},
{
'collection_type': 'List',
'item_type': None,
},
)
def test_get_loop_item_type(self, collection_type, item_type):
self.assertEqual(for_loop._get_loop_item_type(collection_type),
item_type)
@parameterized.parameters(
{
'dict_type': 'Dict[str, int]',
'value_type': 'int',
},
{
'dict_type': 'typing.Mapping[str,float]',
'value_type': 'float',
},
{
'dict_type': 'typing.Mapping[str, Dict[str, str] ]',
'value_type': 'Dict[str, str]',
},
{
'dict_type': 'dict',
'value_type': None,
},
)
def test_get_subvar_type(self, dict_type, value_type):
self.assertEqual(for_loop._get_subvar_type(dict_type), value_type)
@parameterized.parameters(
{
'channel':
pipeline_channel.PipelineParameterChannel(
name='param1',
channel_type='List[str]',
),
'expected_serialization_value':
'{{channel:task=;name=param1-loop-item;type=str;}}',
},
{
'channel':
pipeline_channel.PipelineParameterChannel(
name='output1',
channel_type='List[Dict[str, str]]',
task_name='task1',
),
'expected_serialization_value':
'{{channel:task=task1;name=output1-loop-item;type=Dict[str, str];}}',
},
)
def test_loop_argument_from_pipeline_channel(self, channel,
expected_serialization_value):
loop_argument = for_loop.LoopArgument.from_pipeline_channel(channel)
self.assertEqual(loop_argument.items_or_pipeline_channel, channel)
self.assertEqual(str(loop_argument), expected_serialization_value)
@parameterized.parameters(
{
'raw_items': ['a', 'b', 'c'],
'name_code':
'1',
'expected_serialization_value':
'{{channel:task=;name=loop-item-param-1;type=str;}}',
},
{
'raw_items': [
{
'A_a': 1
},
{
'A_a': 2
},
],
'name_code':
'2',
'expected_serialization_value':
'{{channel:task=;name=loop-item-param-2;type=dict;}}',
},
)
def test_loop_argument_from_raw_items(self, raw_items, name_code,
expected_serialization_value):
loop_argument = for_loop.LoopArgument.from_raw_items(
raw_items, name_code)
self.assertEqual(loop_argument.items_or_pipeline_channel, raw_items)
self.assertEqual(str(loop_argument), expected_serialization_value)
@parameterized.parameters(
{
'name': 'abc-loop-item',
'expected_result': True
},
{
'name': 'abc-loop-item-subvar-a',
'expected_result': True
},
{
'name': 'loop-item-param-1',
'expected_result': True
},
{
'name': 'loop-item-param-1-subvar-a',
'expected_result': True
},
{
'name': 'param1',
'expected_result': False
},
)
def test_name_is_loop_argument(self, name, expected_result):
self.assertEqual(for_loop.LoopArgument.name_is_loop_argument(name),
expected_result)
@parameterized.parameters(
{
'subvar_name': 'a',
'valid': True
},
{
'subvar_name': 'A_a',
'valid': True
},
{
'subvar_name': 'a0',
'valid': True
},
{
'subvar_name': 'a-b',
'valid': False
},
{
'subvar_name': '0',
'valid': False
},
{
'subvar_name': 'a#',
'valid': False
},
)
def test_create_loop_argument_varaible(self, subvar_name, valid):
loop_argument = for_loop.LoopArgument.from_pipeline_channel(
pipeline_channel.PipelineParameterChannel(
name='param1',
channel_type='List[Dict[str, str]]',
))
if valid:
loop_arg_var = for_loop.LoopArgumentVariable(
loop_argument=loop_argument,
subvar_name=subvar_name,
)
self.assertEqual(loop_arg_var.loop_argument, loop_argument)
self.assertEqual(loop_arg_var.subvar_name, subvar_name)
else:
with self.assertRaisesRegex(ValueError,
'Tried to create subvariable'):
for_loop.LoopArgumentVariable(
loop_argument=loop_argument,
subvar_name=subvar_name,
)