def test_compile_simple_pipeline(self):
tmpdir = tempfile.mkdtemp()
try:
producer_op = components.load_component_from_text("""
name: producer
inputs:
- {name: input_param, type: String}
outputs:
- {name: output_model, type: Model}
- {name: output_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: input_param}
- {outputPath: output_model}
- {outputPath: output_value}
""")
consumer_op = components.load_component_from_text("""
name: consumer
inputs:
- {name: input_model, type: Model}
- {name: input_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputPath: input_model}
- {inputValue: input_value}
""")
@dsl.pipeline(name='test-pipeline')
def simple_pipeline(pipeline_input: str = 'Hello KFP!'):
producer = producer_op(input_param=pipeline_input)
consumer = consumer_op(
input_model=producer.outputs['output_model'],
input_value=producer.outputs['output_value'])
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(pipeline_func=simple_pipeline,
package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file, 'r') as f:
print(f.read())
finally:
shutil.rmtree(tmpdir)
def random_num_op(low, high):
"""Generate a random number between low and high."""
return components.load_component_from_text("""
name: Generate random number
outputs:
- {name: output, type: Integer}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2
- "%s"
- "%s"
- {outputPath: output}
""" % (low, high))
# See the License for the specific language governing permissions and
# limitations under the License.
"""Pipeline using ExitHandler."""
import kfp.v2.components.experimental as components
import kfp.v2.dsl.experimental as dsl
from kfp.v2.compiler.experimental import compiler
print_op = components.load_component_from_text("""
name: print op
inputs:
- {name: msg, type: String}
implementation:
container:
image: alpine
command:
- sh
- -c
- |
set -e -x
echo "$0"
- {inputValue: msg}
""")
fail_op = components.load_component_from_text("""
name: fail op
inputs:
- {name: msg, type: String}
implementation:
container:
image: alpine
import kfp.v2.components.experimental as components
import kfp.v2.dsl.experimental as dsl
from kfp.v2.compiler.experimental import compiler
# @component
# def args_generator_op() -> List[str]:
# return [{'A_a': '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]
args_generator_op = components.load_component_from_text("""
name: Args generator op
outputs:
- {name: output, type: "List[Dict[str, str]]"}
implementation:
container:
image: alpine
command:
- sh
- -c
- |
set -e -x
echo "[{'A_a': '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]" > "$0"
- {outputPath: output}
""")
# @component
# def print_op(msg: str):
# print(msg)
print_op = components.load_component_from_text("""
name: print op
inputs:
# See the License for the specific language governing permissions and
# limitations under the License.
import kfp.v2.components.experimental as components
import kfp.v2.dsl.experimental as dsl
from kfp.v2.compiler.experimental import compiler
component_op_1 = components.load_component_from_text("""
name: Write to GCS
inputs:
- {name: text, type: String, description: 'Content to be written to GCS'}
outputs:
- {name: output_gcs_path, type: GCSPath, description: 'GCS file path'}
implementation:
container:
image: google/cloud-sdk:slim
command:
- sh
- -c
- |
set -e -x
echo "$0" | gsutil cp - "$1"
- {inputValue: text}
- {outputUri: output_gcs_path}
""")
component_op_2 = components.load_component_from_text("""
name: Read from GCS
inputs:
- {name: input_gcs_path, type: GCSPath, description: 'GCS file path'}
implementation:
container:
- -c
args:
- mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2
- "%s"
- "%s"
- {outputPath: output}
""" % (low, high))
flip_coin_op = components.load_component_from_text("""
name: Flip coin
outputs:
- {name: output, type: String}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- mkdir -p "$(dirname $0)" && python -c "import random; result = \'heads\' if random.randint(0,1) == 0 else \'tails\'; print(result, end='')" | tee $0
- {outputPath: output}
""")
print_op = components.load_component_from_text("""
name: Print
inputs:
- {name: msg, type: String}
implementation:
container:
image: python:alpine3.6
command: