def pipeline_exit_handler(url: str = 'gs://ml-pipeline/shakespeare1.txt'):
"""A sample pipeline showing exit handler."""
exit_task = echo_msg('exit!')
with dsl.ExitHandler(exit_task):
download_task = gcs_download_op(url)
echo_task = print_file(download_task.output)
def download_and_print(url='gs://ml-pipeline/shakespeare1.txt'):
"""A sample pipeline showing exit handler."""
exit_task = echo_op('exit!')
with dsl.ExitHandler(exit_task):
download_task = gcs_download_op(url)
echo_task = echo_op(download_task.output)
def save_most_frequent_word():
exit_op = ExitHandlerOp('exiting')
with dsl.ExitHandler(exit_op):
counter = GetFrequentWordOp(name='get-Frequent', message=message_param)
counter.container.set_memory_request('200M')
saver = SaveMessageOp(name='save',
message=counter.output,
output_path=output_path_param)
saver.container.set_cpu_limit('0.5')
saver.container.set_gpu_limit('2')
saver.add_node_selector_constraint('cloud.google.com/gke-accelerator',
'nvidia-tesla-k80')
saver.apply(
gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12'))
def save_most_frequent_word(message: str, outputpath: str):
"""A pipeline function describing the orchestration of the workflow."""
exit_op = ExitHandlerOp('exiting')
with dsl.ExitHandler(exit_op):
counter = GetFrequentWordOp(name='get-Frequent', message=message)
counter.container.set_memory_request('200M')
saver = SaveMessageOp(
name='save', message=counter.output, output_path=outputpath)
saver.container.set_cpu_limit('0.5')
saver.container.set_gpu_limit('2')
saver.add_node_selector_constraint('cloud.google.com/gke-accelerator',
'nvidia-tesla-k80')
saver.apply(
gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12'))
def xgb_train_pipeline(
output='gs://{{kfp-default-bucket}}',
project='{{kfp-project-id}}',
diagnostic_mode='HALT_ON_ERROR',
rounds=5,
):
output_template = str(output) + '/' + dsl.RUN_ID_PLACEHOLDER + '/data'
region = 'us-central1'
workers = 2
quota_check = [{'region': region, 'metric': 'CPUS', 'quota_needed': 12.0}]
train_data = 'gs://ml-pipeline/sample-data/sfpd/train.csv'
eval_data = 'gs://ml-pipeline/sample-data/sfpd/eval.csv'
schema = 'gs://ml-pipeline/sample-data/sfpd/schema.json'
true_label = 'ACTION'
target = 'resolution'
required_apis = 'dataproc.googleapis.com'
cluster_name = 'xgb-%s' % dsl.RUN_ID_PLACEHOLDER
# Current GCP pyspark/spark op do not provide outputs as return values, instead,
# we need to use strings to pass the uri around.
analyze_output = output_template
transform_output_train = os.path.join(output_template, 'train', 'part-*')
transform_output_eval = os.path.join(output_template, 'eval', 'part-*')
train_output = os.path.join(output_template, 'train_output')
predict_output = os.path.join(output_template, 'predict_output')
_diagnose_me_op = diagnose_me_op(bucket=output,
execution_mode=diagnostic_mode,
project_id=project,
target_apis=required_apis,
quota_check=quota_check)
with dsl.ExitHandler(exit_op=dataproc_delete_cluster_op(
project_id=project, region=region, name=cluster_name)):
_create_cluster_op = dataproc_create_cluster_op(
project_id=project,
region=region,
name=cluster_name,
initialization_actions=[
os.path.join(_PYSRC_PREFIX, 'initialization_actions.sh'),
],
image_version='1.5').after(_diagnose_me_op)
_analyze_op = dataproc_analyze_op(
project=project,
region=region,
cluster_name=cluster_name,
schema=schema,
train_data=train_data,
output=output_template).after(_create_cluster_op).set_display_name(
'Analyzer')
_transform_op = dataproc_transform_op(
project=project,
region=region,
cluster_name=cluster_name,
train_data=train_data,
eval_data=eval_data,
target=target,
analysis=analyze_output,
output=output_template).after(_analyze_op).set_display_name(
'Transformer')
_train_op = dataproc_train_op(
project=project,
region=region,
cluster_name=cluster_name,
train_data=transform_output_train,
eval_data=transform_output_eval,
target=target,
analysis=analyze_output,
workers=workers,
rounds=rounds,
output=train_output).after(_transform_op).set_display_name(
'Trainer')
_predict_op = dataproc_predict_op(
project=project,
region=region,
cluster_name=cluster_name,
data=transform_output_eval,
model=train_output,
target=target,
analysis=analyze_output,
output=predict_output).after(_train_op).set_display_name(
'Predictor')
_cm_op = confusion_matrix_op(
predictions=os.path.join(predict_output, 'part-*.csv'),
output_dir=output_template).after(_predict_op)
_roc_op = roc_op(predictions_dir=os.path.join(predict_output,
'part-*.csv'),
true_class=true_label,
true_score_column=true_label,
output_dir=output_template).after(_predict_op)