def describe(self): LOGGER.info(f"Pipeline Stage is {self._stage}") LOGGER.info("DSL is:") if self._stage == "fit": LOGGER.info(f"{self._train_dsl}") else: LOGGER.info(f"{self._predict_dsl}") LOGGER.info(f"Pipeline Create Time: {self._create_time}")
def get_output_data_table(self, job_id, cpn_name, role, party_id): """ Parameters ---------- job_id: str cpn_name: str role: str party_id: int Returns ------- dict single output example: { table_name: [], table_namespace: [] } multiple output example: { train_data: { table_name: [], table_namespace: [] }, validate_data: { table_name: [], table_namespace: [] } test_data: { table_name: [], table_namespace: [] } } """ party_id = str(party_id) result = self.client.component.output_data_table( job_id=job_id, role=role, party_id=party_id, component_name=cpn_name) data = {} try: if 'retcode' not in result or result["retcode"] != 0: raise ValueError if "data" not in result: raise ValueError all_data = result["data"] n = len(all_data) # single data table if n == 1: single_data = all_data[0] del single_data["data_name"] data = single_data # multiple data table elif n > 1: for single_data in all_data: data_name = single_data["data_name"] del single_data["data_name"] data[data_name] = single_data # no data table obtained else: LOGGER.info(f"No output data table found in {result}") except ValueError: raise ValueError("Job submit failed, err msg: {}".format(result)) return data
def monitor_job_status(self, job_id, role, party_id): party_id = str(party_id) start_time = time.time() pre_cpn = None LOGGER.info(f"Job id is {job_id}") while True: ret_code, ret_msg, data = self.query_job(job_id, role, party_id) status = data["f_status"] if status == JobStatus.SUCCESS: # print("job is success!!!") elapse_seconds = timedelta(seconds=int(time.time() - start_time)) sys.stdout.write(f"\n\r") LOGGER.info(f"Job is success!!! Job id is {job_id}") LOGGER.info(f"Total time: {elapse_seconds}") return StatusCode.SUCCESS elif status == JobStatus.FAILED: sys.stdout.write(f"\n\r") raise ValueError( f"Job is failed, please check out job {job_id} by fate board or fate_flow cli" ) elif status == JobStatus.WAITING: elapse_seconds = timedelta(seconds=int(time.time() - start_time)) sys.stdout.write( f"Job is still waiting, time elapse: {elapse_seconds}\r") sys.stdout.flush() elif status == JobStatus.CANCELED: elapse_seconds = timedelta(seconds=int(time.time() - start_time)) sys.stdout.write(f"\n\r") LOGGER.info( f"Job is canceled, time elapse: {elapse_seconds}\r") return StatusCode.CANCELED elif status == JobStatus.TIMEOUT: elapse_seconds = timedelta(seconds=int(time.time() - start_time)) sys.stdout.write(f"\n\r") raise ValueError( f"Job is timeout, time elapse: {elapse_seconds}\r") elif status == JobStatus.RUNNING: ret_code, _, data = self.query_task(job_id=job_id, role=role, party_id=party_id, status=JobStatus.RUNNING) if ret_code != 0 or len(data) == 0: time.sleep(conf.TIME_QUERY_FREQS) continue elapse_seconds = timedelta(seconds=int(time.time() - start_time)) if len(data) == 1: cpn = data[0]["f_component_name"] else: cpn = [] for cpn_data in data: cpn.append(cpn_data["f_component_name"]) if cpn != pre_cpn: sys.stdout.write(f"\n \r") pre_cpn = cpn sys.stdout.write( f"Running component {cpn}, time elapse: {elapse_seconds}\r" ) sys.stdout.flush() else: raise ValueError(f"Unknown status: {status}") time.sleep(conf.TIME_QUERY_FREQS)