def save_dataset(dataset: Dataset, module=None) -> NoReturn: """ Save dataset :param dataset: dataset to save :param module: dataset module """ if module is None: module = dataset.module dataset_path = DatasetPath(dataset.dataset_id) dataset_path.makedirs() __save_json(dataset_path.metadata_file, dataset.metadata) __save_json(dataset_path.dataset_config_file, dataset.dataset_config) ModuleUtils.submit_module(module, dataset_path.module_name, dataset_path.module_dir)
def save_job(job: Job, module=None) -> NoReturn: """ Save job :param job: job to save :param module: job module """ if module is None: module = job.module job_path = JobPath(job.job_id) job_path.makedirs() __save_json(job_path.metadata_file, job.metadata) __save_json(job_path.job_config_file, job.job_config) __save_json(job_path.train_config_file, job.train_config) __save_json(job_path.aggregate_config_file, job.aggregate_config) ModuleUtils.submit_module(module, job_path.module_name, job_path.module_dir)
def pack_dataset(dataset: Dataset): data = { "dataset_id": dataset.dataset_id, "metadata": dataset.metadata, "dataset_config": dataset.dataset_config } module_path = ModuleUtils.get_module_path(dataset.module) module_zip_data = ZipUtils.get_compress_data(module_path, dataset.dataset_id) return json.dumps(data).encode("utf-8"), module_zip_data
def pack_job(job: Job): data = { "job_id": job.job_id, "metadata": job.metadata.to_dict(), "job_config": job.job_config.to_dict(), "train_config": job.train_config.to_dict(), "aggregate_config": job.aggregate_config.to_dict() } module_path = ModuleUtils.get_module_path(job.module) module_zip_data = ZipUtils.get_compress_data(module_path, job.job_id) return json.dumps(data).encode("utf-8"), module_zip_data
def save_dataset(cls, dataset: Dataset, *, module=None, module_data=None) -> NoReturn: """ Save dataset :param dataset: dataset to save :param module: dataset module """ if module is None: module = dataset.module dataset_path = DatasetPath(dataset.dataset_id) dataset_path.makedirs() cls.__save_json(dataset_path.metadata_file, dataset.metadata) cls.__save_json(dataset_path.dataset_config_file, dataset.dataset_config) if module_data is not None: ZipUtils.extract_data(module_data, GflConf.temp_dir) ModuleUtils.migrate_module( PathUtils.join(GflConf.temp_dir, dataset.dataset_id), dataset_path.module_name, dataset_path.module_dir) elif module is not None: ModuleUtils.submit_module(module, dataset_path.module_name, dataset_path.module_dir) else: ModuleUtils.submit_module(dataset.module, dataset_path.module_name, dataset_path.module_dir)
def save_job(cls, job: Job, *, module=None, module_data=None) -> NoReturn: """ Save job :param job: job to save :param module: job module :param module_data: """ job_path = JobPath(job.job_id) job_path.makedirs() cls.__save_json(job_path.metadata_file, job.metadata) cls.__save_json(job_path.job_config_file, job.job_config) cls.__save_json(job_path.train_config_file, job.train_config) cls.__save_json(job_path.aggregate_config_file, job.aggregate_config) if module_data is not None: ZipUtils.extract_data(module_data, GflConf.temp_dir) ModuleUtils.migrate_module( PathUtils.join(GflConf.temp_dir, job.job_id), job_path.module_name, job_path.module_dir) elif module is not None: ModuleUtils.submit_module(module, job_path.module_name, job_path.module_dir) else: ModuleUtils.submit_module(job.module, job_path.module_name, job_path.module_dir)
def new_object(cls, *, module=None, obj=None, strategy=None, is_instance=None, **kwargs): if strategy is not None: name = strategy.value if is_instance is None: is_instance = False is_builtin = True else: name = ModuleUtils.get_name(module, obj) if name is None: raise ValueError("") if is_instance is None: is_instance = not (type(obj) == type) is_builtin = False args = kwargs.copy() return ConfigObject(name=name, is_instance=is_instance, is_builtin=is_builtin, args=args)
def load_dataset(cls, dataset_id: str) -> Dataset: """ Load dataset from JSON file. :param dataset_id: dataset ID """ dataset_path = DatasetPath(dataset_id) metadata = cls.__load_json(dataset_path.metadata_file, DatasetMetadata) dataset_config = cls.__load_json(dataset_path.dataset_config_file, DatasetConfig) module = ModuleUtils.import_module(dataset_path.module_dir, dataset_path.module_name) dataset_config.module = module dataset = Dataset(dataset_id=dataset_id, metadata=metadata, dataset_config=dataset_config) dataset.module = module return dataset
def load_job(cls, job_id: str) -> Job: """ Load job from JSON file. :param job_id: dataset ID """ job_path = JobPath(job_id) metadata = cls.__load_json(job_path.metadata_file, JobMetadata) job_config = cls.__load_json(job_path.job_config_file, JobConfig) train_config = cls.__load_json(job_path.train_config_file, TrainConfig) aggregate_config = cls.__load_json(job_path.aggregate_config_file, AggregateConfig) module = ModuleUtils.import_module(job_path.module_dir, job_path.module_name) job_config.module = module train_config.module = module aggregate_config.module = module job = Job(job_id=job_id, metadata=metadata, job_config=job_config, train_config=train_config, aggregate_config=aggregate_config) job.module = module return job
from gfl.utils import ModuleUtils if not ModuleUtils.exists_module("torch"): print("torch is required.") exit(1)