def _init_dataloader(self, mode, loader=None, transforms=None): """Init dataloader.""" if loader is not None: return loader if mode == "train" and self.hps is not None and self.hps.get( "dataset") is not None: if self.hps.get("dataset") and self.hps.get("dataset").get('type'): dataset_cls = ClassFactory.get_cls( ClassType.DATASET, self.hps.get("dataset").get('type')) else: dataset_cls = ClassFactory.get_cls(ClassType.DATASET) dataset = dataset_cls(mode=mode, hps=self.hps.get("dataset")) elif self.hps: if self.hps.get("dataset") and self.hps.get("dataset").get('type'): dataset_cls = ClassFactory.get_cls( ClassType.DATASET, self.hps.get("dataset").get('type')) dataset = dataset_cls(mode=mode, hps=self.hps.get("dataset")) else: dataset_cls = ClassFactory.get_cls(ClassType.DATASET) dataset = dataset_cls(mode=mode) else: dataset_cls = ClassFactory.get_cls(ClassType.DATASET) dataset = dataset_cls(mode=mode) if transforms is not None: dataset.transforms = transforms if self.distributed and mode == "train": dataset.set_distributed(self._world_size, self._rank_id) # adapt the dataset to specific backend dataloader = Adapter(dataset).loader return dataloader
def from_module(module): """From Model.""" name = module.__class__.__name__ if ClassFactory.is_exists(ClassType.NETWORK, name): module_cls = ClassFactory.get_cls(ClassType.NETWORK, name) if hasattr(module_cls, "from_module"): return module_cls.from_module(module) return module
def register_datasets(backend): """Import and register datasets automatically.""" if backend == "pytorch": from . import pytorch elif backend == "tensorflow": from . import tensorflow elif backend == "mindspore": import mindspore.dataset from . import mindspore ClassFactory.lazy_register("vega.datasets.common", {"imagenet": ["Imagenet"]}) from . import common from . import transforms
def _train_single_model(self, model_desc=None, model_id=None, weights_file=None): cls_trainer = ClassFactory.get_cls(ClassType.TRAINER, PipeStepConfig.trainer.type) step_name = self.task.step_name if model_desc is not None: sample = dict(worker_id=model_id, desc=model_desc, step_name=step_name) record = ReportRecord().load_dict(sample) logging.debug("update record=%s", str(record)) trainer = cls_trainer(model_desc=model_desc, id=model_id, pretrained_model_file=weights_file) else: trainer = cls_trainer(None, 0) record = ReportRecord(trainer.step_name, trainer.worker_id, desc=trainer.model_desc) ReportClient().update(**record.to_dict()) # resume training if vega.is_torch_backend() and General._resume: trainer.load_checkpoint = True trainer._resume_training = True if self._distributed_training: self._do_distributed_fully_train(trainer) else: self._do_single_fully_train(trainer)
def _do_hccl_fully_train(self, trainer): origin_worker_id = trainer.worker_id model_desc = trainer.model_desc del trainer os.environ['RANK_SIZE'] = os.environ['ORIGIN_RANK_SIZE'] os.environ['RANK_TABLE_FILE'] = os.environ['ORIGIN_RANK_TABLE_FILE'] origin_parallel_fully_train = General.parallel_fully_train origin_parallel = General._parallel General.parallel_fully_train = True General.dft = True General._parallel = True cls_trainer = ClassFactory.get_cls(ClassType.TRAINER, PipeStepConfig.trainer.type) self.master = create_master() workers_num = int(os.environ['RANK_SIZE']) for i in range(workers_num): worker_id = "{}-{}".format(origin_worker_id, i) trainer = cls_trainer(model_desc, id=worker_id) evaluator = self._get_evaluator( worker_id) if os.environ['DEVICE_ID'] == "0" else None self.master.run(trainer, evaluator) self.master.join() self.master.close() General.parallel_fully_train = origin_parallel_fully_train General.dft = False General._parallel = origin_parallel
def _get_evaluator(self, worker_id): if not PipeStepConfig.evaluator_enable: return None cls_evaluator = ClassFactory.get_cls('evaluator', "Evaluator") evaluator = cls_evaluator({ "step_name": self.task.step_name, "worker_id": worker_id }) return evaluator
def _train_single_model(self, model_desc, model_id, hps, multi_task): cls_trainer = ClassFactory.get_cls(ClassType.TRAINER, PipeStepConfig.trainer.type) step_name = self.task.step_name sample = dict(worker_id=model_id, desc=model_desc, step_name=step_name) record = ReportRecord().load_dict(sample) logging.debug("update record=%s", str(record)) trainer = cls_trainer(model_desc=model_desc, id=model_id, hps=hps, multi_task=multi_task) ReportClient().update(**record.to_dict()) if self._distributed_training: self._do_distributed_fully_train(trainer) else: self._do_single_fully_train(trainer)
def to_model(self): """Transform a NetworkDesc to a special model.""" logging.debug("Start to Create a Network.") module = ClassFactory.get_cls(ClassType.NETWORK, "Module") model = module.from_desc(self._desc) if not model: raise Exception("Failed to create model, model desc={}".format( self._desc)) model.desc = self._desc if hasattr(model, '_apply_names'): model._apply_names() return model
def __new__(cls, model=None, id=None, hps=None, load_ckpt_flag=False, model_desc=None, **kwargs): """Create Trainer clss.""" if vega.is_torch_backend(): trainer_cls = ClassFactory.get_cls(ClassType.TRAINER, "TrainerTorch") elif vega.is_tf_backend(): trainer_cls = ClassFactory.get_cls(ClassType.TRAINER, "TrainerTf") else: trainer_cls = ClassFactory.get_cls(ClassType.TRAINER, "TrainerMs") return trainer_cls(model=model, id=id, hps=hps, load_ckpt_flag=load_ckpt_flag, model_desc=model_desc, **kwargs)
def __init__(self, search_space, num_samples, max_epochs, repeat_times, min_epochs=1, eta=3, multi_obj=False, random_samples=None, prob_crossover=0.6, prob_mutatation=0.2, tuner="GP"): """Init BOHB.""" super().__init__(search_space, num_samples, max_epochs, min_epochs, eta) # init all the configs self.repeat_times = repeat_times self.max_epochs = max_epochs self.iter_list, self.min_epoch_list = self._get_total_iters( num_samples, max_epochs, self.repeat_times, min_epochs, eta) self.additional_samples = self._get_additional_samples(eta) if random_samples is not None: self.random_samples = random_samples else: self.random_samples = max(self.iter_list[0][0], 2) self.tuner_name = "GA" if multi_obj else tuner logger.info( f"bohb info, iter list: {self.iter_list}, min epoch list: {self.min_epoch_list}, " f"addition samples: {self.additional_samples}, tuner: {self.tuner_name}, " f"random samples: {self.random_samples}") # create sha list self.sha_list = self._create_sha_list(search_space, self.iter_list, self.min_epoch_list, self.repeat_times) # create tuner if multi_obj: self.tuner = GeneticAlgorithm(search_space, random_samples=self.random_samples, prob_crossover=prob_crossover, prob_mutatation=prob_mutatation) elif self.tuner_name == "hebo": self.tuner = ClassFactory.get_cls(ClassType.SEARCH_ALGORITHM, "HeboAdaptor")(search_space) else: self.tuner = TunerBuilder(search_space, tuner=tuner)
def __init__(self, desc=None): """Init SearchSpace.""" super(SearchSpace, self).__init__() if desc is None: desc = SearchSpaceConfig().to_dict() if desc.type is not None: desc = ClassFactory.get_cls(ClassType.SEARCHSPACE, desc.type).get_space(desc) for name, item in desc.items(): self.__setattr__(name, item) self.__setitem__(name, item) self._params = OrderedDict() self._condition_dict = OrderedDict() self._forbidden_list = [] self._hp_count = 0 self._dag = DAG() if desc is not None: self.form_desc(desc)
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import necks.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.networks.pytorch.necks", { "ffm": ["network:FeatureFusionModule"], "fpn": ["FPN"] })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import mindspore network.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.networks.mindspore", { "dnet": ["DNet"], "super_network": ["DartsNetwork", "CARSDartsNetwork", "GDASDartsNetwork"], "backbones.load_official_model": ["OffcialModelLoader"], "backbones.resnet_ms": ["ResNetMs"], "losses.mix_auxiliary_loss": ["MixAuxiliaryLoss"], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import data augmentation algorithms.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.algorithms.data_augmentation", { "pba_hpo": ["PBAHpo"], "pba_trainer_callback": ["PbaTrainerCallback"], "cyclesr": ["CyclesrTrainerCallback"], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Import and register evaluator automatically.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.evaluator", { "device_evaluator": ["DeviceEvaluator"], "host_evaluator": ["HostEvaluator"], "evaluator": ["Evaluator"], })
def transform_architecture(model, pretrained_model_file=None): """Transform architecture.""" if not hasattr(model, "_arch_params") or not model._arch_params or \ PipeStepConfig.pipe_step.get("type") == "TrainPipeStep": return model model._apply_names() logging.info( "Start to transform architecture, model arch params type: {}".format( model._arch_params_type)) ConnectionsArchParamsCombiner().combine(model) if vega.is_ms_backend(): from mindspore.train.serialization import load_checkpoint changed_name_list = [] mask_weight_list = [] for name, module in model.named_modules(): if not ClassFactory.is_exists(model._arch_params_type, module.model_name): continue changed_name_list, mask_weight_list = decode_fn_ms( module, changed_name_list, mask_weight_list) assert len(changed_name_list) == len(mask_weight_list) # change model and rebuild model_desc = model.desc root_name = [ name for name in list(model_desc.keys()) if name not in ('type', '_arch_params') ] for changed_name, mask in zip(changed_name_list, mask_weight_list): name = changed_name.split('.') name[0] = root_name[int(name[0])] assert len(name) <= 6 if len(name) == 6: model_desc[name[0]][name[1]][name[2]][name[3]][name[4]][ name[5]] = sum(mask) if len(name) == 5: model_desc[name[0]][name[1]][name[2]][name[3]][name[4]] = sum( mask) if len(name) == 4: model_desc[name[0]][name[1]][name[2]][name[3]] = sum(mask) if len(name) == 3: model_desc[name[0]][name[1]][name[2]] = sum(mask) if len(name) == 2: model_desc[name[0]][name[1]] = sum(mask) network = NetworkDesc(model_desc) model = network.to_model() model_desc.pop( '_arch_params') if '_arch_params' in model_desc else model_desc model.desc = model_desc # change weight if hasattr(model, "pretrained"): pretrained_weight = model.pretrained(pretrained_model_file) load_checkpoint(pretrained_weight, net=model) os.remove(pretrained_weight) else: for name, module in model.named_modules(): if not ClassFactory.is_exists(model._arch_params_type, module.model_name): continue arch_cls = ClassFactory.get_cls(model._arch_params_type, module.model_name) decode_fn(module, arch_cls) module.register_forward_pre_hook(arch_cls.fit_weights) module.register_forward_hook(module.clear_module_arch_params) return model
from .search_algorithm import SearchAlgorithm from .ea_conf import EAConfig from .pareto_front_conf import ParetoFrontConfig from .pareto_front import ParetoFront from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.core.search_algs", { "ps_differential": ["DifferentialAlgorithm"], })
def __init__(self, student, teacher, header=None): super(TinyBertDistil, self).__init__(student, teacher) self.loss_mse = ops.MSELoss() self.head = ClassFactory.get_instance(ClassType.NETWORK, header)
from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.datasets.pytorch", { "coco_transforms": ["CocoCategoriesTransform", "PolysToMaskTransform"], })
# it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import dataset.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.datasets.common", { "avazu": ["AvazuDataset"], "cifar10": ["Cifar10"], "cifar100": ["Cifar100"], "div2k": ["DIV2K"], "cls_ds": ["ClassificationDataset"], "cityscapes": ["Cityscapes"], "div2k_unpair": ["Div2kUnpair"], "fmnist": ["FashionMnist"], # "imagenet": ["Imagenet"], "mnist": ["Mnist"], "sr_datasets": ["Set5", "Set14", "BSDS100"], "auto_lane_datasets": ["AutoLaneDataset"], "coco": ["CocoDataset", "DetectionDataset"], "glue": ["GlueDataset"], "spatiotemporal": ["SpatiotemporalDataset"], "reds": ["REDS"], "nasbench": ["Nasbench"], })
def __init__(self, student, teacher): super().__init__() self.student = ClassFactory.get_instance(ClassType.NETWORK, student) self.teacher = ClassFactory.get_instance(ClassType.NETWORK, teacher) self.teacher.freeze('teacher')
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Import and register trainer automatically.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.trainer", { "trainer_torch": ["TrainerTorch"], "trainer_tf": ["TrainerTf"], "trainer_ms": ["TrainerMs"], "trainer": ["Trainer"], "script_runner": ["ScriptRunner"], })
"""Lazy import tensorflow networks.""" from .network import Sequential from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.networks.tensorflow", { "resnet_tf": ["ResNetTF", 'ResNetSlim'], # backbones "backbones.resnet_det": ["ResNetDet"], # customs "customs.edvr.edvr": ["EDVR"], "customs.gcn_regressor": ["GCNRegressor"], # detectors "detectors.faster_rcnn_trainer_callback": ["FasterRCNNTrainerCallback"], "detectors.faster_rcnn": ["FasterRCNN"], "detectors.tf_optimizer": ["TFOptimizer"], # losses "losses.cross_entropy_loss": ["CrossEntropyLoss"], "losses.mix_auxiliary_loss": ["MixAuxiliaryLoss"], "losses.charbonnier": ["CharbonnierLoss"], # necks "necks.mask_rcnn_box": ["MaskRCNNBox"], }) ClassFactory.lazy_register( "vega.networks.tensorflow.utils", { "anchor_utils.anchor_generator": ["AnchorGenerator"], "hyperparams.initializer": ["Initializer"],
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Import and register modules automatically.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.modules", { "module": ["network:Module"], }) def register_modules(): """Import and register modules automatically.""" from . import blocks from . import cells from . import connections from . import operators from . import preprocess from . import loss from . import getters from . import necks from . import backbones from . import distillation
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import custom network.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.networks.pytorch.customs", { "nago": ["network:NAGO"], "deepfm": ["network:DeepFactorizationMachineModel"], "autogate": ["network:AutoGateModel"], "autogroup": ["network:AutoGroupModel"], "bisenet": ["network:BiSeNet"], "modnas": ["network:ModNasArchSpace"], "mobilenetv2": ["network:MobileNetV2"], "gcn_regressor": ["network:GCNRegressor"], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import compression algorithms.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.algorithms.compression", { "prune_ea": ["PruneCodec", "PruneEA", "PruneSearchSpace", "PruneTrainerCallback"], "prune_ea_mobilenet": ["PruneMobilenetCodec", "PruneMobilenetTrainerCallback"], "quant_ea": ["QuantCodec", "QuantEA", "QuantTrainerCallback"], })
from .callback import Callback from .callback_list import CallbackList from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.trainer.callbacks", { "metrics_evaluator": ["trainer.callback:MetricsEvaluator"], "progress_logger": ["trainer.callback:ProgressLogger"], "performance_saver": ["trainer.callback:PerformanceSaver"], "lr_scheduler": ["trainer.callback:LearningRateScheduler"], "model_builder": ["trainer.callback:ModelBuilder"], "model_statistics": ["trainer.callback:ModelStatistics"], "model_checkpoint": ["trainer.callback:ModelCheckpoint"], "report_callback": ["trainer.callback:ReportCallback"], "runtime_callback": ["trainer.callback:RuntimeCallback"], "detection_progress_logger": ["trainer.callback:DetectionProgressLogger"], "detection_metrics_evaluator": ["trainer.callback:DetectionMetricsEvaluator"], "visual_callback": ["trainer.callback:VisualCallBack"], "model_tuner": ["trainer.callback:ModelTuner"], "timm_trainer_callback": ["trainer.callback:TimmTrainerCallback"], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import pytorch blocks.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.networks.pytorch.blocks", { "block": ["Block"], "conv_ws": ["ConvWS2d"], "stem": ['PreTwoStem'], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import head network.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register( "vega.networks.pytorch.heads", { "auto_lane_head": ["network:AutoLaneHead"], "auxiliary_head": ["network:AuxiliaryHead"], })
# -*- coding:utf-8 -*- # Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. # This program is free software; you can redistribute it and/or modify # it under the terms of the MIT License. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MIT License for more details. """Lazy import detector.""" from vega.common.class_factory import ClassFactory ClassFactory.lazy_register("vega.networks.pytorch.detectors", { "auto_lane_detector": ["AutoLaneDetector"], })