def save_dataset(dataset: Dataset, module=None) -> NoReturn:
"""
Save dataset
:param dataset: dataset to save
:param module: dataset module
"""
if module is None:
module = dataset.module
dataset_path = DatasetPath(dataset.dataset_id)
dataset_path.makedirs()
__save_json(dataset_path.metadata_file, dataset.metadata)
__save_json(dataset_path.dataset_config_file, dataset.dataset_config)
ModuleUtils.submit_module(module, dataset_path.module_name,
dataset_path.module_dir)
def save_job(job: Job, module=None) -> NoReturn:
"""
Save job
:param job: job to save
:param module: job module
"""
if module is None:
module = job.module
job_path = JobPath(job.job_id)
job_path.makedirs()
__save_json(job_path.metadata_file, job.metadata)
__save_json(job_path.job_config_file, job.job_config)
__save_json(job_path.train_config_file, job.train_config)
__save_json(job_path.aggregate_config_file, job.aggregate_config)
ModuleUtils.submit_module(module, job_path.module_name,
job_path.module_dir)
def pack_dataset(dataset: Dataset):
data = {
"dataset_id": dataset.dataset_id,
"metadata": dataset.metadata,
"dataset_config": dataset.dataset_config
}
module_path = ModuleUtils.get_module_path(dataset.module)
module_zip_data = ZipUtils.get_compress_data(module_path,
dataset.dataset_id)
return json.dumps(data).encode("utf-8"), module_zip_data
def pack_job(job: Job):
data = {
"job_id": job.job_id,
"metadata": job.metadata.to_dict(),
"job_config": job.job_config.to_dict(),
"train_config": job.train_config.to_dict(),
"aggregate_config": job.aggregate_config.to_dict()
}
module_path = ModuleUtils.get_module_path(job.module)
module_zip_data = ZipUtils.get_compress_data(module_path, job.job_id)
return json.dumps(data).encode("utf-8"), module_zip_data
def save_dataset(cls,
dataset: Dataset,
*,
module=None,
module_data=None) -> NoReturn:
"""
Save dataset
:param dataset: dataset to save
:param module: dataset module
"""
if module is None:
module = dataset.module
dataset_path = DatasetPath(dataset.dataset_id)
dataset_path.makedirs()
cls.__save_json(dataset_path.metadata_file, dataset.metadata)
cls.__save_json(dataset_path.dataset_config_file,
dataset.dataset_config)
if module_data is not None:
ZipUtils.extract_data(module_data, GflConf.temp_dir)
ModuleUtils.migrate_module(
PathUtils.join(GflConf.temp_dir, dataset.dataset_id),
dataset_path.module_name, dataset_path.module_dir)
elif module is not None:
ModuleUtils.submit_module(module, dataset_path.module_name,
dataset_path.module_dir)
else:
ModuleUtils.submit_module(dataset.module, dataset_path.module_name,
dataset_path.module_dir)
def save_job(cls, job: Job, *, module=None, module_data=None) -> NoReturn:
"""
Save job
:param job: job to save
:param module: job module
:param module_data:
"""
job_path = JobPath(job.job_id)
job_path.makedirs()
cls.__save_json(job_path.metadata_file, job.metadata)
cls.__save_json(job_path.job_config_file, job.job_config)
cls.__save_json(job_path.train_config_file, job.train_config)
cls.__save_json(job_path.aggregate_config_file, job.aggregate_config)
if module_data is not None:
ZipUtils.extract_data(module_data, GflConf.temp_dir)
ModuleUtils.migrate_module(
PathUtils.join(GflConf.temp_dir, job.job_id),
job_path.module_name, job_path.module_dir)
elif module is not None:
ModuleUtils.submit_module(module, job_path.module_name,
job_path.module_dir)
else:
ModuleUtils.submit_module(job.module, job_path.module_name,
job_path.module_dir)
def new_object(cls, *, module=None, obj=None, strategy=None, is_instance=None, **kwargs):
if strategy is not None:
name = strategy.value
if is_instance is None:
is_instance = False
is_builtin = True
else:
name = ModuleUtils.get_name(module, obj)
if name is None:
raise ValueError("")
if is_instance is None:
is_instance = not (type(obj) == type)
is_builtin = False
args = kwargs.copy()
return ConfigObject(name=name, is_instance=is_instance, is_builtin=is_builtin, args=args)
def load_dataset(cls, dataset_id: str) -> Dataset:
"""
Load dataset from JSON file.
:param dataset_id: dataset ID
"""
dataset_path = DatasetPath(dataset_id)
metadata = cls.__load_json(dataset_path.metadata_file, DatasetMetadata)
dataset_config = cls.__load_json(dataset_path.dataset_config_file,
DatasetConfig)
module = ModuleUtils.import_module(dataset_path.module_dir,
dataset_path.module_name)
dataset_config.module = module
dataset = Dataset(dataset_id=dataset_id,
metadata=metadata,
dataset_config=dataset_config)
dataset.module = module
return dataset
def load_job(cls, job_id: str) -> Job:
"""
Load job from JSON file.
:param job_id: dataset ID
"""
job_path = JobPath(job_id)
metadata = cls.__load_json(job_path.metadata_file, JobMetadata)
job_config = cls.__load_json(job_path.job_config_file, JobConfig)
train_config = cls.__load_json(job_path.train_config_file, TrainConfig)
aggregate_config = cls.__load_json(job_path.aggregate_config_file,
AggregateConfig)
module = ModuleUtils.import_module(job_path.module_dir,
job_path.module_name)
job_config.module = module
train_config.module = module
aggregate_config.module = module
job = Job(job_id=job_id,
metadata=metadata,
job_config=job_config,
train_config=train_config,
aggregate_config=aggregate_config)
job.module = module
return job
from gfl.utils import ModuleUtils
if not ModuleUtils.exists_module("torch"):
print("torch is required.")
exit(1)