Ejemplo n.º 1
0
    def save_dataset(cls,
                     dataset: Dataset,
                     *,
                     module=None,
                     module_data=None) -> NoReturn:
        """
        Save dataset

        :param dataset: dataset to save
        :param module: dataset module
        """
        if module is None:
            module = dataset.module
        dataset_path = DatasetPath(dataset.dataset_id)
        dataset_path.makedirs()
        cls.__save_json(dataset_path.metadata_file, dataset.metadata)
        cls.__save_json(dataset_path.dataset_config_file,
                        dataset.dataset_config)
        if module_data is not None:
            ZipUtils.extract_data(module_data, GflConf.temp_dir)
            ModuleUtils.migrate_module(
                PathUtils.join(GflConf.temp_dir, dataset.dataset_id),
                dataset_path.module_name, dataset_path.module_dir)
        elif module is not None:
            ModuleUtils.submit_module(module, dataset_path.module_name,
                                      dataset_path.module_dir)
        else:
            ModuleUtils.submit_module(dataset.module, dataset_path.module_name,
                                      dataset_path.module_dir)
Ejemplo n.º 2
0
    def save_job(cls, job: Job, *, module=None, module_data=None) -> NoReturn:
        """
        Save job

        :param job: job to save
        :param module: job module
        :param module_data:
        """
        job_path = JobPath(job.job_id)
        job_path.makedirs()
        cls.__save_json(job_path.metadata_file, job.metadata)
        cls.__save_json(job_path.job_config_file, job.job_config)
        cls.__save_json(job_path.train_config_file, job.train_config)
        cls.__save_json(job_path.aggregate_config_file, job.aggregate_config)
        if module_data is not None:
            ZipUtils.extract_data(module_data, GflConf.temp_dir)
            ModuleUtils.migrate_module(
                PathUtils.join(GflConf.temp_dir, job.job_id),
                job_path.module_name, job_path.module_dir)
        elif module is not None:
            ModuleUtils.submit_module(module, job_path.module_name,
                                      job_path.module_dir)
        else:
            ModuleUtils.submit_module(job.module, job_path.module_name,
                                      job_path.module_dir)
Ejemplo n.º 3
0
    def save_job_zip(cls, job_id: str, job: File) -> NoReturn:
        """
        Save the zip file of the job.

        :param job_id: job ID
        :param job: zip file
        """
        job_path = JobPath(job_id)
        if job.ipfs_hash is not None and job.ipfs_hash != "":
            file_obj = Ipfs.get(job.ipfs_hash)
        else:
            file_obj = File.file
        ZipUtils.extract(file_obj, job_path.root_dir)
Ejemplo n.º 4
0
    def save_dataset_zip(cls, dataset_id: str, dataset: File) -> NoReturn:
        """
        Save the zip file of the dataset.

        :param dataset_id: dataset ID
        :param dataset: zip file
        """
        dataset_path = DatasetPath(dataset_id)
        if dataset.ipfs_hash is not None and dataset.ipfs_hash != "":
            file_obj = Ipfs.get(dataset.ipfs_hash)
        else:
            file_obj = dataset.file
        ZipUtils.extract(file_obj, dataset_path.root_dir)
Ejemplo n.º 5
0
    def load_dataset_zip(cls, dataset_id: str) -> File:
        """
        Load dataset from a ZIP file.

        :param dataset_id: dataset ID
        """
        dataset_path = DatasetPath(dataset_id)
        file_obj = BytesIO()
        ZipUtils.compress(dataset_path.metadata_file, file_obj)
        if GflConf.get_property("ipfs.enabled"):
            file_obj.seek(0)
            ipfs_hash = Ipfs.put(file_obj.read())
            return File(ipfs_hash=ipfs_hash, file=None)
        else:
            return File(ipfs_hash=None, file=file_obj)
Ejemplo n.º 6
0
def load_job_zip(job_id: str) -> File:
    """
    Load job from a ZIP file.

    :param job_id: dataset ID
    """
    job_path = JobPath(job_id)
    file_obj = BytesIO()
    ZipUtils.compress([job_path.metadata_file, job_path.config_dir], file_obj)
    if GflConf.get_property("ipfs.enabled"):
        file_obj.seek(0)
        ipfs_hash = Ipfs.put(file_obj.read())
        return File(ipfs_hash == ipfs_hash)
    else:
        return File(file=file_obj)
Ejemplo n.º 7
0
def pack_dataset(dataset: Dataset):
    data = {
        "dataset_id": dataset.dataset_id,
        "metadata": dataset.metadata,
        "dataset_config": dataset.dataset_config
    }
    module_path = ModuleUtils.get_module_path(dataset.module)
    module_zip_data = ZipUtils.get_compress_data(module_path,
                                                 dataset.dataset_id)
    return json.dumps(data).encode("utf-8"), module_zip_data
Ejemplo n.º 8
0
def pack_job(job: Job):
    data = {
        "job_id": job.job_id,
        "metadata": job.metadata.to_dict(),
        "job_config": job.job_config.to_dict(),
        "train_config": job.train_config.to_dict(),
        "aggregate_config": job.aggregate_config.to_dict()
    }
    module_path = ModuleUtils.get_module_path(job.module)
    module_zip_data = ZipUtils.get_compress_data(module_path, job.job_id)
    return json.dumps(data).encode("utf-8"), module_zip_data
Ejemplo n.º 9
0

from gfl.utils import ZipUtils


if __name__ == "__main__":
    data = ZipUtils.get_compress_data("__init__.py", "")
    ZipUtils.extract_data(data, "t2")