def describe(self):
LOGGER.info(f"Pipeline Stage is {self._stage}")
LOGGER.info("DSL is:")
if self._stage == "fit":
LOGGER.info(f"{self._train_dsl}")
else:
LOGGER.info(f"{self._predict_dsl}")
LOGGER.info(f"Pipeline Create Time: {self._create_time}")
def get_output_data_table(self, job_id, cpn_name, role, party_id):
"""
Parameters
----------
job_id: str
cpn_name: str
role: str
party_id: int
Returns
-------
dict
single output example:
{
table_name: [],
table_namespace: []
}
multiple output example:
{
train_data: {
table_name: [],
table_namespace: []
},
validate_data: {
table_name: [],
table_namespace: []
}
test_data: {
table_name: [],
table_namespace: []
}
}
"""
party_id = str(party_id)
result = self.client.component.output_data_table(
job_id=job_id,
role=role,
party_id=party_id,
component_name=cpn_name)
data = {}
try:
if 'retcode' not in result or result["retcode"] != 0:
raise ValueError
if "data" not in result:
raise ValueError
all_data = result["data"]
n = len(all_data)
# single data table
if n == 1:
single_data = all_data[0]
del single_data["data_name"]
data = single_data
# multiple data table
elif n > 1:
for single_data in all_data:
data_name = single_data["data_name"]
del single_data["data_name"]
data[data_name] = single_data
# no data table obtained
else:
LOGGER.info(f"No output data table found in {result}")
except ValueError:
raise ValueError("Job submit failed, err msg: {}".format(result))
return data
def monitor_job_status(self, job_id, role, party_id):
party_id = str(party_id)
start_time = time.time()
pre_cpn = None
LOGGER.info(f"Job id is {job_id}")
while True:
ret_code, ret_msg, data = self.query_job(job_id, role, party_id)
status = data["f_status"]
if status == JobStatus.SUCCESS:
# print("job is success!!!")
elapse_seconds = timedelta(seconds=int(time.time() -
start_time))
sys.stdout.write(f"\n\r")
LOGGER.info(f"Job is success!!! Job id is {job_id}")
LOGGER.info(f"Total time: {elapse_seconds}")
return StatusCode.SUCCESS
elif status == JobStatus.FAILED:
sys.stdout.write(f"\n\r")
raise ValueError(
f"Job is failed, please check out job {job_id} by fate board or fate_flow cli"
)
elif status == JobStatus.WAITING:
elapse_seconds = timedelta(seconds=int(time.time() -
start_time))
sys.stdout.write(
f"Job is still waiting, time elapse: {elapse_seconds}\r")
sys.stdout.flush()
elif status == JobStatus.CANCELED:
elapse_seconds = timedelta(seconds=int(time.time() -
start_time))
sys.stdout.write(f"\n\r")
LOGGER.info(
f"Job is canceled, time elapse: {elapse_seconds}\r")
return StatusCode.CANCELED
elif status == JobStatus.TIMEOUT:
elapse_seconds = timedelta(seconds=int(time.time() -
start_time))
sys.stdout.write(f"\n\r")
raise ValueError(
f"Job is timeout, time elapse: {elapse_seconds}\r")
elif status == JobStatus.RUNNING:
ret_code, _, data = self.query_task(job_id=job_id,
role=role,
party_id=party_id,
status=JobStatus.RUNNING)
if ret_code != 0 or len(data) == 0:
time.sleep(conf.TIME_QUERY_FREQS)
continue
elapse_seconds = timedelta(seconds=int(time.time() -
start_time))
if len(data) == 1:
cpn = data[0]["f_component_name"]
else:
cpn = []
for cpn_data in data:
cpn.append(cpn_data["f_component_name"])
if cpn != pre_cpn:
sys.stdout.write(f"\n \r")
pre_cpn = cpn
sys.stdout.write(
f"Running component {cpn}, time elapse: {elapse_seconds}\r"
)
sys.stdout.flush()
else:
raise ValueError(f"Unknown status: {status}")
time.sleep(conf.TIME_QUERY_FREQS)
