def _call_back(resp): if isinstance(resp, UploadDataResponse): data_progress.submitted(resp.job_id) echo.file(f"[dataset]{resp.job_id}") if isinstance(resp, QueryJobResponse): data_progress.update() update_bar(0)
def _raise(): exception_id = str(uuid.uuid1()) job_progress.exception(exception_id) suite.update_status(job_name=job.job_name, exception_id=exception_id) echo.file(f"exception({exception_id})") LOGGER.exception(f"exception id: {exception_id}")
def _call_back(resp: SubmitJobResponse): if isinstance(resp, SubmitJobResponse): job_progress.submitted(resp.job_id) echo.file(f"[jobs] {resp.job_id} ", nl=False) suite.update_status(job_name=job.job_name, job_id=resp.job_id) if isinstance(resp, QueryJobResponse): job_progress.running(resp.status, resp.progress) update_bar(0)
def _call_back(resp: SubmitJobResponse): if isinstance(resp, SubmitJobResponse): progress_tracking = "/".join([str(_config.jobs_progress), str(_config.jobs_num)]) if _config.jobs_num != len(suite.jobs): job_progress.set_progress_tracking(progress_tracking) job_progress.submitted(resp.job_id) echo.file(f"[jobs] {resp.job_id} ", nl=False) suite.update_status(job_name=job.job_name, job_id=resp.job_id) if isinstance(resp, QueryJobResponse): job_progress.running(resp.status, resp.progress) update_bar(0)
def _upload_data(clients: Clients, suite, config: Config, output_path=None): with click.progressbar(length=len(suite.dataset), label="dataset", show_eta=False, show_pos=True, width=24) as bar: for i, data in enumerate(suite.dataset): data.update(config) table_name = data.config['table_name'] if data.config.get( 'table_name', None) is not None else data.config.get('name') data_progress = DataProgress( f"{data.role_str}<-{data.config['namespace']}.{table_name}") def update_bar(n_step): bar.item_show_func = lambda x: data_progress.show() time.sleep(0.1) bar.update(n_step) def _call_back(resp): if isinstance(resp, UploadDataResponse): data_progress.submitted(resp.job_id) echo.file(f"[dataset]{resp.job_id}") if isinstance(resp, QueryJobResponse): data_progress.update() update_bar(0) try: echo.stdout_newline() status, data_path = clients[data.role_str].upload_data( data, _call_back, output_path) time.sleep(1) data_progress.update() if status != 'success': raise RuntimeError( f"uploading {i + 1}th data for {suite.path} {status}") bar.update(1) if _config.data_switch: from fate_test.scripts import generate_mock_data generate_mock_data.remove_file(data_path) except Exception: exception_id = str(uuid.uuid1()) echo.file(f"exception({exception_id})") LOGGER.exception(f"exception id: {exception_id}") echo.echo( f"upload {i + 1}th data {data.config} to {data.role_str} fail, exception_id: {exception_id}" )
def _upload_data(clients: Clients, suite, config: Config): with click.progressbar(length=len(suite.dataset), label="dataset", show_eta=False, show_pos=True, width=24) as bar: for i, data in enumerate(suite.dataset): data.update(config) data_progress = DataProgress( f"{data.role_str}<-{data.config['namespace']}.{data.config['table_name']}" ) def update_bar(n_step): bar.item_show_func = lambda x: data_progress.show() time.sleep(0.1) bar.update(n_step) def _call_back(resp): if isinstance(resp, UploadDataResponse): data_progress.submitted(resp.job_id) echo.file(f"[dataset]{resp.job_id}") if isinstance(resp, QueryJobResponse): data_progress.update() update_bar(0) try: echo.stdout_newline() response = clients[data.role_str].upload_data(data, _call_back) data_progress.update() if not response.status.is_success(): raise RuntimeError( f"uploading {i + 1}th data for {suite.path} {response.status}" ) bar.update(1) except Exception as e: exception_id = str(uuid.uuid1()) echo.file(f"exception({exception_id})") LOGGER.exception(f"exception id: {exception_id}") echo.echo( f"upload {i + 1}th data {data.config} to {data.role_str} fail, exception_id: {exception_id}" )
def _raise(err_msg, status="failed"): exception_id = str(uuid.uuid1()) suite.update_status(job_name=job_name, exception_id=exception_id, status=status) echo.file(f"exception({exception_id}), error message:\n{err_msg}")