def run_pipeline(client: kfp.Client, pipeline_name: str, pipeline_id: str, pipeline_paramters_path: dict): experiment_id = find_experiment_id( experiment_name=os.environ["INPUT_EXPERIMENT_NAME"], client=client) if not experiment_id: raise ValueError("Failed to find experiment with the name: {}".format( os.environ["INPUT_EXPERIMENT_NAME"])) logging.info(f"The expriment id is: {experiment_id}") namespace = None if (os.getenv("INPUT_PIPELINE_NAMESPACE") != None) and (str.isspace(os.getenv("INPUT_PIPELINE_NAMESPACE")) == False) and os.getenv("INPUT_PIPELINE_NAMESPACE"): namespace = os.environ["INPUT_PIPELINE_NAMESPACE"] logging.info(f"The namespace that will be used is: {namespace}") # [TODO] What would be a good way to name the jobs job_name = pipeline_name + datetime.now().strftime("%Y_%m_%d_%H_%M_%S") logging.info(f"The job name is: {job_name}") pipeline_params = read_pipeline_params( pipeline_paramters_path=pipeline_paramters_path) pipeline_params = pipeline_params if pipeline_params != None else {} logging.info( f"experiment_id: {experiment_id}, job_name:{job_name}, pipeline_params:{pipeline_params}, pipeline_id:{pipeline_id}, namespace:{namespace}" ) client.run_pipeline( experiment_id=experiment_id, job_name=job_name, # Read this as a yaml, people seam to prefer that to json. params=pipeline_params, pipeline_id=pipeline_id, namespace=namespace) logging.info( "Successfully started the pipeline, head over to kubeflow to check it out" )
def run_pipeline_func(client: kfp.Client, pipeline_name: str, pipeline_id: str, pipeline_paramters_path: dict, recurring_flag: bool = False, cron_exp: str = ''): pipeline_params = read_pipeline_params( pipeline_paramters_path=pipeline_paramters_path) pipeline_params = pipeline_params if pipeline_params is not None else {} experiment_id = None experiment_name = "{}-{}".format(pipeline_name, os.environ["INPUT_EXPERIMENT_NAME"]) try: experiment_id = client.get_experiment( experiment_name=experiment_name).to_dict()["id"] except ValueError: experiment_id = client.create_experiment( name=experiment_name).to_dict()["id"] namespace = os.getenv("INPUT_PIPELINE_NAMESPACE") if not str.isspace( os.getenv("INPUT_PIPELINE_NAMESPACE")) else None job_name = 'Run {} on {}'.format( pipeline_name, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) logging.info(f"experiment_id: {experiment_id}, \ job_name: {job_name}, \ pipeline_params: {pipeline_params}, \ pipeline_id: {pipeline_id}, \ namespace: {namespace}, \ cron_exp: {cron_exp}") if recurring_flag == "true": client.create_recurring_run(experiment_id=experiment_id, job_name=job_name, params=pipeline_params, pipeline_id=pipeline_id, cron_expression=cron_exp) logging.info( "Successfully started the recurring pipeline, head over to kubeflow to check it out" ) client.run_pipeline(experiment_id=experiment_id, job_name=job_name, params=pipeline_params, pipeline_id=pipeline_id) logging.info( "Successfully started the pipeline, head over to kubeflow to check it out" )
def run_pipeline_in_experiment(api_pipeline: ApiPipeline, parameters: dict = None, run_name: str = None, wait_for_status: bool = False): try: client = KfpClient(_pipeline_service_url) experiment = client.create_experiment('PIPELINE_RUNS') run_result = client.run_pipeline(experiment_id=experiment.id, job_name=run_name or api_pipeline.name, params=parameters, pipeline_id=api_pipeline.id) run_id = run_result.id if wait_for_status: run_details = wait_for_run_status(client, run_id, 10) run_status = json.loads(run_details.pipeline_runtime.workflow_manifest)["status"] if run_status \ and run_status.get("phase", "").lower() in ["failed", "error"] \ and run_status.get("message"): raise RuntimeError(f"Run {run_id} failed with error: {run_status['message']}") return run_id except Exception as e: print(f"Exception trying to run pipeline {api_pipeline.id} '{api_pipeline.name}'" f" with parameters {parameters}:" f" %s\n" % e) raise ApiError(message=f"{e.body}\nKFP URL: {_pipeline_service_url}", http_status_code=e.status or 422) return None
def main(): args = parse_arguments() test_cases = [] test_name = args.testname + ' Sample Test' ###### Initialization ###### client = Client(namespace=args.namespace) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = args.testname + ' sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = args.testname + '_sample' params = {} response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### start_time = datetime.now() response = client.wait_for_run_completion(run_id, 1200) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format( args.namespace, workflow_id)) print("=========Argo Workflow Log=========") print(argo_log) if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit(1) ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def _mock_pipelines_creation(kfp_client_mock: kfp.Client): def _mock_create_experiment(name, description=None, namespace=None): return kfp_server_api.models.ApiExperiment( id="some-exp-id", name=name, description=description, ) def _mock_run_pipeline( experiment_id, job_name, pipeline_package_path=None, params={}, pipeline_id=None, version_id=None, ): return kfp_server_api.models.ApiRun(id="some-run-id", name=job_name) kfp_client_mock.create_experiment = _mock_create_experiment kfp_client_mock.run_pipeline = _mock_run_pipeline
def run_pipeline( pipeline, arguments=None, project=None, experiment=None, run=None, namespace=None, artifact_path=None, ops=None, url=None, ttl=None, ): """remote KubeFlow pipeline execution Submit a workflow task to KFP via mlrun API service :param pipeline: KFP pipeline function or path to .yaml/.zip pipeline file :param arguments: pipeline arguments :param experiment: experiment name :param run: optional, run name :param namespace: Kubernetes namespace (if not using default) :param url: optional, url to mlrun API service :param artifact_path: target location/url for mlrun artifacts :param ops: additional operators (.apply() to all pipeline functions) :param ttl: pipeline ttl in secs (after that the pods will be removed) :returns: kubeflow pipeline id """ remote = not get_k8s_helper( silent=True).is_running_inside_kubernetes_cluster() artifact_path = artifact_path or mlconf.artifact_path if artifact_path and "{{run.uid}}" in artifact_path: artifact_path.replace("{{run.uid}}", "{{workflow.uid}}") if artifact_path and "{{run.project}}" in artifact_path: if not project: raise ValueError("project name must be specified with this" + f" artifact_path template {artifact_path}") artifact_path.replace("{{run.project}}", project) if not artifact_path: raise ValueError("artifact path was not specified") namespace = namespace or mlconf.namespace arguments = arguments or {} if remote or url: mldb = get_run_db(url) if mldb.kind != "http": raise ValueError( "run pipeline require access to remote api-service" ", please set the dbpath url") id = mldb.submit_pipeline( pipeline, arguments, experiment=experiment, run=run, namespace=namespace, ops=ops, artifact_path=artifact_path, ) else: client = Client(namespace=namespace) if isinstance(pipeline, str): experiment = client.create_experiment(name=experiment) run_result = client.run_pipeline(experiment.id, run, pipeline, params=arguments) else: conf = new_pipe_meta(artifact_path, ttl, ops) run_result = client.create_run_from_pipeline_func( pipeline, arguments, run_name=run, experiment_name=experiment, pipeline_conf=conf, ) id = run_result.run_id logger.info("Pipeline run id={}, check UI or DB for progress".format(id)) return id
class PySampleChecker(object): def __init__(self, testname, input, output, result, namespace='kubeflow'): """Util class for checking python sample test running results. :param testname: test name. :param input: The path of a pipeline file that will be submitted. :param output: The path of the test output. :param result: The path of the test result that will be exported. :param namespace: namespace of the deployed pipeline system. Default: kubeflow """ self._testname = testname self._input = input self._output = output self._result = result self._namespace = namespace self._run_pipeline = None self._test_timeout = None self._test_cases = [] self._test_name = self._testname + ' Sample Test' self._client = None self._experiment_id = None self._job_name = None self._test_args = None self._run_id = None def run(self): """Run compiled KFP pipeline.""" ###### Initialization ###### host = 'ml-pipeline.%s.svc.cluster.local:8888' % self._namespace self._client = Client(host=host) ###### Check Input File ###### utils.add_junit_test(self._test_cases, 'input generated yaml file', os.path.exists(self._input), 'yaml file is not generated') if not os.path.exists(self._input): utils.write_junit_xml(self._test_name, self._result, self._test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = self._testname + ' sample experiment' response = self._client.create_experiment(experiment_name) self._experiment_id = response.id utils.add_junit_test(self._test_cases, 'create experiment', True) ###### Create Job ###### self._job_name = self._testname + '_sample' ###### Figure out arguments from associated config files. ####### self._test_args = {} config_schema = yamale.make_schema(SCHEMA_CONFIG) try: with open(DEFAULT_CONFIG, 'r') as f: raw_args = yaml.safe_load(f) default_config = yamale.make_data(DEFAULT_CONFIG) yamale.validate( config_schema, default_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: raise RuntimeError('Illegal default config:{}'.format(yamlerr)) except OSError as ose: raise FileExistsError('Default config not found:{}'.format(ose)) else: self._test_timeout = raw_args['test_timeout'] self._run_pipeline = raw_args['run_pipeline'] try: with open( os.path.join(CONFIG_DIR, '%s.config.yaml' % self._testname), 'r') as f: raw_args = yaml.safe_load(f) test_config = yamale.make_data( os.path.join(CONFIG_DIR, '%s.config.yaml' % self._testname)) yamale.validate( config_schema, test_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: print( 'No legit yaml config file found, use default args:{}'.format( yamlerr)) except OSError as ose: print( 'Config file with the same name not found, use default args:{}' .format(ose)) else: self._test_args.update(raw_args['arguments']) if 'output' in self._test_args.keys( ): # output is a special param that has to be specified dynamically. self._test_args['output'] = self._output if 'test_timeout' in raw_args.keys(): self._test_timeout = raw_args['test_timeout'] if 'run_pipeline' in raw_args.keys(): self._run_pipeline = raw_args['run_pipeline'] # Submit for pipeline running. if self._run_pipeline: response = self._client.run_pipeline(self._experiment_id, self._job_name, self._input, self._test_args) self._run_id = response.id utils.add_junit_test(self._test_cases, 'create pipeline run', True) def check(self): """Check pipeline run results.""" if self._run_pipeline: ###### Monitor Job ###### try: start_time = datetime.now() response = self._client.wait_for_run_completion( self._run_id, self._test_timeout) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(self._test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) finally: ###### Output Argo Log for Debugging ###### workflow_json = self._client._get_workflow_json(self._run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command( 'argo logs -n {} -w {}'.format(self._namespace, workflow_id)) print('=========Argo Workflow Log=========') print(argo_log) if not succ: utils.write_junit_xml(self._test_name, self._result, self._test_cases) exit(1) ###### Validate the results for specific test cases ###### #TODO: Add result check for tfx-cab-classification after launch. if self._testname == 'xgboost_training_cm': # For xgboost sample, check its confusion matrix. cm_tar_path = './confusion_matrix.tar.gz' utils.get_artifact_in_minio(workflow_json, 'confusion-matrix', cm_tar_path, 'mlpipeline-ui-metadata') with tarfile.open(cm_tar_path) as tar_handle: file_handles = tar_handle.getmembers() assert len(file_handles) == 1 with tar_handle.extractfile(file_handles[0]) as f: cm_data = f.read() utils.add_junit_test( self._test_cases, 'confusion matrix format', (len(cm_data) > 0), 'the confusion matrix file is empty') ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(self._test_name, self._result, self._test_cases)
def main(): args = parse_arguments() test_cases = [] test_name = 'Kubeflow Sample Test' ###### Initialization ###### client = Client() ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) exit() ###### Create Experiment ###### experiment_name = 'kubeflow sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = 'kubeflow_sample' params = { 'output': args.output, 'project': 'ml-pipeline-test', 'evaluation': 'gs://ml-pipeline-dataset/sample-test/flower/eval15.csv', 'train': 'gs://ml-pipeline-dataset/sample-test/flower/train30.csv', 'hidden-layer-size': '10,5', 'steps': '5' } response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### start_time = datetime.now() response = client.wait_for_run_completion(run_id, 1200) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit() ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] #TODO: remove the namespace dependency or make is configurable. argo_log, _ = utils.run_bash_command( 'argo logs -n kubeflow -w {}'.format(workflow_id)) print("=========Argo Workflow Log=========") print(argo_log) ###### Validate the results ###### # confusion matrix should show three columns for the flower data # target, predicted, count cm_tar_path = './confusion_matrix.tar.gz' cm_filename = 'mlpipeline-ui-metadata.json' utils.get_artifact_in_minio(workflow_json, 'confusionmatrix', cm_tar_path) tar_handler = tarfile.open(cm_tar_path) tar_handler.extractall() with open(cm_filename, 'r') as f: cm_data = json.load(f) utils.add_junit_test( test_cases, 'confusion matrix format', (len(cm_data['outputs'][0]['schema']) == 3), 'the column number of the confusion matrix output is not equal to three' ) ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def run_pipeline(pipeline, arguments=None, experiment=None, run=None, namespace=None, artifact_path=None, ops=None, url=None, remote=False): """remote KubeFlow pipeline execution Submit a workflow task to KFP via mlrun API service :param pipeline KFP pipeline function or path to .yaml/.zip pipeline file :param arguments pipeline arguments :param experiment experiment name :param run optional, run name :param namespace Kubernetes namespace (if not using default) :param url optional, url to mlrun API service :param artifact_path target location/url for mlrun artifacts :param ops additional operators (.apply() to all pipeline functions) :param remote use mlrun remote API service vs direct KFP APIs :return kubeflow pipeline id """ namespace = namespace or mlconf.namespace arguments = arguments or {} if remote or url: mldb = get_run_db(url).connect() if mldb.kind != 'http': raise ValueError( 'run pipeline require access to remote api-service' ', please set the dbpath url') id = mldb.submit_pipeline(pipeline, arguments, experiment=experiment, run=run, namespace=namespace, ops=ops, artifact_path=artifact_path) else: client = Client(namespace=namespace) if isinstance(pipeline, str): experiment = client.create_experiment(name=experiment) run_result = client.run_pipeline(experiment.id, run, pipeline, params=arguments) else: conf = new_pipe_meta(artifact_path, ops) run_result = client.create_run_from_pipeline_func( pipeline, arguments, run_name=run, experiment_name=experiment, pipeline_conf=conf) id = run_result.run_id logger.info('Pipeline run id={}, check UI or DB for progress'.format(id)) return id
def main(): args = parse_arguments() test_cases = [] test_name = args.testname + ' Sample Test' ###### Initialization ###### host = 'ml-pipeline.%s.svc.cluster.local:8888' % args.namespace client = Client(host=host) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = args.testname + ' sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = args.testname + '_sample' ###### Test-specific arguments ####### if args.testname == 'tfx_cab_classification': params = { 'output': args.output, 'project': 'ml-pipeline-test', 'column-names': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/column-names.json', 'evaluation': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/eval20.csv', 'train': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/train50.csv', 'hidden-layer-size': '5', 'steps': '5' } elif args.testname == 'xgboost_training_cm': params = { 'output': args.output, 'project': 'ml-pipeline-test', 'train-data': 'gs://ml-pipeline-dataset/sample-test/sfpd/train_50.csv', 'eval-data': 'gs://ml-pipeline-dataset/sample-test/sfpd/eval_20.csv', 'schema': 'gs://ml-pipeline-dataset/sample-test/sfpd/schema.json', 'rounds': '20', 'workers': '2' } else: # Basic tests require no additional params. params = {} response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### try: start_time = datetime.now() if args.testname == 'xgboost_training_cm': response = client.wait_for_run_completion(run_id, 1800) else: response = client.wait_for_run_completion(run_id, 1200) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) finally: ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format( args.namespace, workflow_id)) print('=========Argo Workflow Log=========') print(argo_log) if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit(1) ###### Validate the results for specific test cases ###### #TODO: Add result check for tfx-cab-classification after launch. if args.testname == 'xgboost_training_cm': cm_tar_path = './confusion_matrix.tar.gz' utils.get_artifact_in_minio(workflow_json, 'confusion-matrix', cm_tar_path, 'mlpipeline-ui-metadata') with tarfile.open(cm_tar_path) as tar_handle: file_handles = tar_handle.getmembers() assert len(file_handles) == 1 with tar_handle.extractfile(file_handles[0]) as f: cm_data = f.read() utils.add_junit_test(test_cases, 'confusion matrix format', (len(cm_data) > 0), 'the confusion matrix file is empty') ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def main(): args = parse_arguments() test_cases = [] test_name = 'XGBoost Sample Test' ###### Initialization ###### host = 'ml-pipeline.%s.svc.cluster.local:8888' % args.namespace client = Client(host=host) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = 'xgboost sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = 'xgboost_sample' params = { 'output': args.output, 'project': 'ml-pipeline-test', 'train-data': 'gs://ml-pipeline-dataset/sample-test/sfpd/train_50.csv', 'eval-data': 'gs://ml-pipeline-dataset/sample-test/sfpd/eval_20.csv', 'schema': 'gs://ml-pipeline-dataset/sample-test/sfpd/schema.json', 'rounds': '20', 'workers': '2' } response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### start_time = datetime.now() response = client.wait_for_run_completion(run_id, 1800) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format( args.namespace, workflow_id)) print("=========Argo Workflow Log=========") print(argo_log) ###### If the job fails, skip the result validation ###### if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit(1) ###### Validate the results ###### # confusion matrix should show three columns for the flower data # target, predicted, count cm_tar_path = './confusion_matrix.tar.gz' cm_filename = 'mlpipeline-ui-metadata.json' utils.get_artifact_in_minio(workflow_json, 'confusion-matrix', cm_tar_path) tar_handler = tarfile.open(cm_tar_path) tar_handler.extractall() with open(cm_filename, 'r') as f: cm_data = f.read() utils.add_junit_test(test_cases, 'confusion matrix format', (len(cm_data) > 0), 'the confusion matrix file is empty') ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def main(): args = parse_arguments() test_cases = [] test_name = 'Resnet CMLE Test' ###### Initialization ###### host = 'ml-pipeline.%s.svc.cluster.local:8888' % args.namespace client = Client(host=host) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = 'resnet cmle sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = 'cmle_sample' params = { 'output': args.output, 'project_id': 'ml-pipeline-test', 'region': 'us-central1', 'model': 'bolts', 'version': 'beta1', 'tf_version': '1.9', # Watch out! If 1.9 is no longer supported we need to set it to a newer version. 'train_csv': 'gs://ml-pipeline-dataset/sample-test/bolts/bolt_images_train_sample1000.csv', 'validation_csv': 'gs://ml-pipeline-dataset/sample-test/bolts/bolt_images_validate_sample200.csv', 'labels': 'gs://bolts_image_dataset/labels.txt', 'depth': 50, 'train_batch_size': 32, 'eval_batch_size': 32, 'steps_per_eval': 128, 'train_steps': 128, 'num_train_images': 1000, 'num_eval_images': 200, 'num_label_classes': 10 } response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### try: start_time = datetime.now() response = client.wait_for_run_completion(run_id, 1800) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) finally: ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format( args.namespace, workflow_id)) print("=========Argo Workflow Log=========") print(argo_log) if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit(1) ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def main(): args = parse_arguments() test_cases = [] test_name = 'TFX Sample Test' ###### Initialization ###### client = Client(namespace=args.namespace) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated') if not os.path.exists(args.input): utils.write_junit_xml(test_name, args.result, test_cases) exit() ###### Create Experiment ###### experiment_name = 'TFX sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = 'TFX_sample' params = {'output': args.output, 'project': 'ml-pipeline-test', 'column-names': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/column-names.json', 'evaluation': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/eval20.csv', 'train': 'gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/train50.csv', 'hidden-layer-size': '5', 'steps': '5'} response = client.run_pipeline(experiment_id, job_name, args.input, params) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### start_time = datetime.now() response = client.wait_for_run_completion(run_id, 1200) succ = (response.run.status.lower()=='succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) if not succ: utils.write_junit_xml(test_name, args.result, test_cases) exit() ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format(args.namespace, workflow_id)) print("=========Argo Workflow Log=========") print(argo_log) ###### Validate the results ###### #TODO: enable after launch # model analysis html is validated # argo_workflow_id = workflow_json['metadata']['name'] # gcs_html_path = os.path.join(os.path.join(args.output, str(argo_workflow_id)), 'analysis/output_display.html') # print('Output display HTML path is ' + gcs_html_path) # utils.run_bash_command('gsutil cp ' + gcs_html_path + './') # display_file = open('./output_display.html', 'r') # is_next_line_state = False # for line in display_file: # if is_next_line_state: # state = line.strip() # break # if line.strip() == '<script type="application/vnd.jupyter.widget-state+json">': # is_next_line_state = True # import json # state_json = json.loads(state) # succ = ('state' in state_json and 'version_major' in state_json and 'version_minor' in state_json) # utils.add_junit_test(test_cases, 'output display html', succ, 'the state json does not contain state, version_major, or version_inor') ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, args.result, test_cases)
def check(self): """Run sample test and check results.""" test_cases = [] test_name = self._testname + ' Sample Test' ###### Initialization ###### host = 'ml-pipeline.%s.svc.cluster.local:8888' % self._namespace client = Client(host=host) ###### Check Input File ###### utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(self._input), 'yaml file is not generated') if not os.path.exists(self._input): utils.write_junit_xml(test_name, self._result, test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### experiment_name = self._testname + ' sample experiment' response = client.create_experiment(experiment_name) experiment_id = response.id utils.add_junit_test(test_cases, 'create experiment', True) ###### Create Job ###### job_name = self._testname + '_sample' ###### Figure out arguments from associated config files. ####### test_args = {} try: with open(DEFAULT_CONFIG, 'r') as f: raw_args = yaml.safe_load(f) except yaml.YAMLError as yamlerr: raise RuntimeError('Illegal default config:{}'.format(yamlerr)) except OSError as ose: raise FileExistsError('Default config not found:{}'.format(ose)) else: test_timeout = raw_args['test_timeout'] try: with open(os.path.join(CONFIG_DIR, '%s.config.yaml' % self._testname), 'r') as f: raw_args = yaml.safe_load(f) except yaml.YAMLError as yamlerr: print('No legit yaml config file found, use default args:{}'.format(yamlerr)) except OSError as ose: print('Config file with the same name not found, use default args:{}'.format(ose)) else: test_args.update(raw_args['arguments']) if 'output' in test_args.keys(): # output is a special param that has to be specified dynamically. test_args['output'] = self._output if 'test_timeout' in raw_args.keys(): test_timeout = raw_args['test_timeout'] response = client.run_pipeline(experiment_id, job_name, self._input, test_args) run_id = response.id utils.add_junit_test(test_cases, 'create pipeline run', True) ###### Monitor Job ###### try: start_time = datetime.now() response = client.wait_for_run_completion(run_id, test_timeout) succ = (response.run.status.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time) finally: ###### Output Argo Log for Debugging ###### workflow_json = client._get_workflow_json(run_id) workflow_id = workflow_json['metadata']['name'] argo_log, _ = utils.run_bash_command('argo logs -n {} -w {}'.format( self._namespace, workflow_id)) print('=========Argo Workflow Log=========') print(argo_log) if not succ: utils.write_junit_xml(test_name, self._result, test_cases) exit(1) ###### Validate the results for specific test cases ###### #TODO: Add result check for tfx-cab-classification after launch. if self._testname == 'xgboost_training_cm': # For xgboost sample, check its confusion matrix. cm_tar_path = './confusion_matrix.tar.gz' utils.get_artifact_in_minio(workflow_json, 'confusion-matrix', cm_tar_path, 'mlpipeline-ui-metadata') with tarfile.open(cm_tar_path) as tar_handle: file_handles = tar_handle.getmembers() assert len(file_handles) == 1 with tar_handle.extractfile(file_handles[0]) as f: cm_data = f.read() utils.add_junit_test(test_cases, 'confusion matrix format', (len(cm_data) > 0), 'the confusion matrix file is empty') ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. ###### Write out the test result in junit xml ###### utils.write_junit_xml(test_name, self._result, test_cases)