def _wrapper(*args, **kwargs): if RuntimeConfig.PROCESS_ROLE in [ProcessRole.SERVER]: for i in range(3): try: stat_logger.info("detect session {} by table {} {}".format( session.get_session_id(), DETECT_TABLE[0], DETECT_TABLE[1])) stat_logger.info("start count table {} {}".format(DETECT_TABLE[0], DETECT_TABLE[1])) count = session.table(namespace=DETECT_TABLE[0], name=DETECT_TABLE[1]).count() stat_logger.info("table {} {} count is {}".format(DETECT_TABLE[0], DETECT_TABLE[1], count)) if count != DETECT_TABLE[2]: raise Exception("session {} count error".format(session.get_session_id())) stat_logger.info("session {} is ok".format(session.get_session_id())) break except Exception as e: stat_logger.exception(e) stat_logger.info("start init new session") try: clean_server_used_session() init_session_for_flow_server() except Exception as e: stat_logger.exception(e) stat_logger.info("init new session failed.") else: stat_logger.error("init new session failed.") else: # If in executor pass. TODO: detect and restore the session in executor pass return func(*args, **kwargs)
def run(): parser = argparse.ArgumentParser() parser.add_argument('-j', '--job_id', required=True, type=str, help="job id") parser.add_argument('-w', '--work_mode', required=True, type=str, help="work mode") parser.add_argument('-b', '--backend', required=True, type=str, help="backend") args = parser.parse_args() job_id = args.job_id work_mode = int(args.work_mode) backend = int(args.backend) session.init(job_id=job_id, mode=work_mode, backend=backend, set_log_dir=False) try: schedule_logger(job_id.split('_')[0]).info( 'start stop session {}'.format(session.get_session_id())) session.stop() schedule_logger(job_id.split('_')[0]).info( 'stop session {} success'.format(session.get_session_id())) except Exception as e: pass
def init_session_for_flow_server(): # Options are used with different backend on demand session.init(job_id="session_used_by_fate_flow_server_{}".format(fate_uuid()), mode=RuntimeConfig.WORK_MODE, backend=RuntimeConfig.BACKEND, options={"eggroll.session.processors.per.node": 1}) # init session detect table detect_table = session.table(namespace=DETECT_TABLE[0], name=DETECT_TABLE[1], partition=DETECT_TABLE[2]) detect_table.destroy() detect_table = session.table(namespace=DETECT_TABLE[0], name=DETECT_TABLE[1], partition=DETECT_TABLE[2]) detect_table.put_all(enumerate(range(DETECT_TABLE[2]))) stat_logger.info("init detect table {} {} for session {}".format(detect_table.get_namespace(), detect_table.get_name(), session.get_session_id())) stat_logger.info("init session {} for fate flow server successfully".format(session.get_session_id()))
def clean_server_used_session(): used_session_id = None try: used_session_id = session.get_session_id() session.stop() except: pass session.exit() stat_logger.info("clean session {} for fate flow server done".format(used_session_id))
def test_remote(self): table = session.table(name='remote_name', namespace='remote_namespace', partition=1) table.put_all(range(12)) from arch.api import federation federation.init(session.get_session_id(), runtime_conf=None) federation.remote(table, name="roll_pair_name.table", tag="roll_pair_tag")
def run(): parser = argparse.ArgumentParser() parser.add_argument('-j', '--job_id', required=True, type=str, help="job id") parser.add_argument('-w', '--work_mode', required=True, type=str, help="work mode") parser.add_argument('-b', '--backend', required=True, type=str, help="backend") parser.add_argument('-c', '--command', required=True, type=str, help="command") args = parser.parse_args() session_job_id = args.job_id fate_job_id = session_job_id.split('_')[0] work_mode = int(args.work_mode) backend = int(args.backend) command = args.command session.init(job_id=session_job_id, mode=work_mode, backend=backend, set_log_dir=False) try: schedule_logger(fate_job_id).info('start {} session {}'.format(command, session.get_session_id())) if command == 'stop': session.stop() elif command == 'kill': session.kill() else: schedule_logger(fate_job_id).info('{} session {} failed, this command is not supported'.format(command, session.get_session_id())) schedule_logger(fate_job_id).info('{} session {} success'.format(command, session.get_session_id())) except Exception as e: pass
def test_parallelize(self): print("session id:{}".format(session.get_session_id())) print(session.parallelize(range(10)).get_all())