def save_job(cls, job: Job, *, module=None, module_data=None) -> NoReturn:
"""
Save job
:param job: job to save
:param module: job module
:param module_data:
"""
job_path = JobPath(job.job_id)
job_path.makedirs()
cls.__save_json(job_path.metadata_file, job.metadata)
cls.__save_json(job_path.job_config_file, job.job_config)
cls.__save_json(job_path.train_config_file, job.train_config)
cls.__save_json(job_path.aggregate_config_file, job.aggregate_config)
if module_data is not None:
ZipUtils.extract_data(module_data, GflConf.temp_dir)
ModuleUtils.migrate_module(
PathUtils.join(GflConf.temp_dir, job.job_id),
job_path.module_name, job_path.module_dir)
elif module is not None:
ModuleUtils.submit_module(module, job_path.module_name,
job_path.module_dir)
else:
ModuleUtils.submit_module(job.module, job_path.module_name,
job_path.module_dir)
def create_tables(job_id: str):
job_path = JobPath(job_id)
job_path.makedirs()
with SqliteContext(job_path.sqlite_file) as (_, cursor):
for s in create_job_kv:
cursor.execute(s)
for s in create_job_client:
cursor.execute(s)
for s in create_job_params:
cursor.execute(s)
def save_job(job: Job, module=None) -> NoReturn:
"""
Save job
:param job: job to save
:param module: job module
"""
if module is None:
module = job.module
job_path = JobPath(job.job_id)
job_path.makedirs()
__save_json(job_path.metadata_file, job.metadata)
__save_json(job_path.job_config_file, job.job_config)
__save_json(job_path.train_config_file, job.train_config)
__save_json(job_path.aggregate_config_file, job.aggregate_config)
ModuleUtils.submit_module(module, job_path.module_name,
job_path.module_dir)
def save_topology_manager(cls, job_id, topology_manager):
"""
Save topology_manager
topology_manager->topology_config->json
:param job_id: Job ID
:param topology_manager: topology_manager to save
"""
job_path = JobPath(job_id)
job_path.makedirs()
topology_config = TopologyConfig()
topology_config.with_topology(topology_manager.topology)
topology_config.with_client_nodes(topology_manager.client_address_list)
topology_config.with_index2node(topology_manager.index2node)
topology_config.with_train_node_num(topology_manager.train_node_num)
topology_config.with_server_nodes(topology_manager.server_address_list)
# 目前暂时考虑中心化的场景
topology_config.with_isCentralized(True)
cls.__save_json(job_path.topology_config_file, topology_config)