def get_loaders(self, stage: str) -> "OrderedDict[str, DataLoader]": data_params = dict(self.stages_config[stage]["data_params"]) batch_size = data_params.pop("batch_size", 1) num_workers = data_params.pop("num_workers") drop_last = data_params.pop("drop_last", False) per_gpu_scaling = data_params.pop("per_gpu_scaling", False) distributed_rank = self.distributed_params.get("rank", -1) distributed = distributed_rank > -1 if per_gpu_scaling and not distributed: num_gpus = max(1, torch.cuda.device_count()) batch_size *= num_gpus num_workers *= num_gpus datasets = self.get_datasets(stage=stage, **data_params) loaders = OrderedDict() for name, ds_ in datasets.items(): assert isinstance(ds_, (Dataset, dict)), \ f"{ds_} should be Dataset of Dict" loader_params = { "batch_size": batch_size, "num_workers": num_workers, "pin_memory": torch.cuda.is_available(), "drop_last": drop_last, } if isinstance(ds_, Dataset): loader_params["dataset"] = ds_ elif isinstance(ds_, dict): assert "dataset" in ds_, \ "You need to specify dataset for dataloader" loader_params = merge_dicts(ds_, loader_params) else: raise NotImplementedError if distributed: sampler = loader_params.get("sampler") if sampler is not None: assert isinstance(sampler, DistributedSampler) else: loader_params["sampler"] = DistributedSampler( dataset=loader_params["dataset"]) loader_params["shuffle"] = (name.startswith("train") and loader_params.get("sampler") is None) if "batch_sampler" in loader_params: if distributed: raise ValueError("batch_sampler option is mutually " "exclusive with distributed") for k in ("batch_size", "shuffle", "sampler", "drop_last"): loader_params.pop(k, None) loaders[name] = DataLoader(**loader_params) return loaders
def parse_args_uargs(args, unknown_args): """ Function for parsing configuration files Args: args: recognized arguments unknown_args: unrecognized arguments Returns: tuple: updated arguments, dict with config """ args_ = copy.deepcopy(args) # load params config = {} for config_path in args_.configs: config_ = utils.load_config(config_path, ordered=True) config = utils.merge_dicts(config, config_) config, args_ = parse_config_args(config=config, args=args_, unknown_args=unknown_args) # hack with argparse in config config_args = config.get("args", None) if config_args is not None: for key, value in config_args.items(): arg_value = getattr(args_, key, None) if arg_value is None or (key in ["logdir", "baselogdir"] and arg_value == ""): arg_value = value setattr(args_, key, arg_value) return args_, config
def __init__(self, config: Dict): self._config = deepcopy(config) self.__prepare_logdir() self._config["stages"]["state_params"] = merge_dicts( deepcopy(self._config["stages"].get("state_params", {})), deepcopy(self._config.get("args", {})), {"logdir": self._logdir}) self.stages_config = self._get_stages_config(self._config["stages"])
def process_model_params( model: Model, layerwise_params: Dict[str, dict] = None, no_bias_weight_decay: bool = True, lr_scaling: float = 1.0, ) -> List[Union[torch.nn.Parameter, dict]]: """ Gains model parameters for ``torch.optim.Optimizer`` Args: model (torch.nn.Module): Model to process layerwise_params (Dict): Order-sensitive dict where each key is regex pattern and values are layer-wise options for layers matching with a pattern no_bias_weight_decay (bool): If true, removes weight_decay for all ``bias`` parameters in the model lr_scaling (float): layer-wise learning rate scaling, if 1.0, learning rates will not be scaled Returns: iterable: parameters for an optimizer Examples: >>> model = catalyst.contrib.models.segmentation.ResnetUnet() >>> layerwise_params = collections.OrderedDict([ >>> ("conv1.*", dict(lr=0.001, weight_decay=0.0003)), >>> ("conv.*", dict(lr=0.002)) >>> ]) >>> params = process_model_params(model, layerwise_params) >>> optimizer = torch.optim.Adam(params, lr=0.0003) """ params = list(model.named_parameters()) layerwise_params = layerwise_params or collections.OrderedDict() model_params = [] for name, parameters in params: options = {} for pattern, options_ in layerwise_params.items(): if re.match(pattern, name) is not None: # all new LR rules write on top of the old ones options = utils.merge_dicts(options, options_) # no bias decay from https://arxiv.org/abs/1812.01187 if no_bias_weight_decay and name.endswith("bias"): options["weight_decay"] = 0.0 # lr linear scaling from https://arxiv.org/pdf/1706.02677.pdf if "lr" in options: options["lr"] *= lr_scaling model_params.append({"params": parameters, **options}) return model_params
def process_class_config(class_config: Dict): default_w = 1.0 default_classification_w = 0.05 default_subclasses = dict() default_params = { "weight": default_w, "classification_w": default_classification_w, "subclasses": default_subclasses, "classified": True } for d_class_name in class_config.keys(): class_config[d_class_name] = merge_dicts( default_params, class_config.get(d_class_name) or {} ) for c_class_name in class_config[d_class_name]["subclasses"]: class_config[d_class_name]["subclasses"][c_class_name] = merge_dicts( {"weight": default_w, "aliases": [c_class_name]}, class_config[d_class_name]["subclasses"][c_class_name] or {} ) return class_config
def __getitem__(self, index: int) -> Any: """Get item from all datasets. Args: index (int): index to value from all datasets Returns: list: list of value in every dataset """ dcts = [x[index] for x in self.datasets] dct = merge_dicts(*dcts) if self.dict_transform is not None: dct = self.dict_transform(dct) return dct
def _get_stages_config(self, stages_config): stages_defaults = {} stages_config_out = OrderedDict() for key in self.STAGE_KEYWORDS: stages_defaults[key] = deepcopy(stages_config.get(key, {})) for stage in stages_config: if stage in self.STAGE_KEYWORDS \ or stages_config.get(stage) is None: continue stages_config_out[stage] = {} for key in self.STAGE_KEYWORDS: stages_config_out[stage][key] = merge_dicts( deepcopy(stages_defaults.get(key, {})), deepcopy(stages_config[stage].get(key, {})), ) return stages_config_out
def parse_args_uargs(args, unknown_args): """ Function for parsing configuration files Args: args: recognized arguments unknown_args: unrecognized arguments Returns: tuple: updated arguments, dict with config """ args_ = copy.deepcopy(args) # load params config = {} for config_path in args_.configs: with open(config_path, "r") as fin: if config_path.endswith("json"): config_ = json.load(fin, object_pairs_hook=OrderedDict) elif config_path.endswith("yml"): config_ = load_ordered_yaml(fin) else: raise Exception("Unknown file format") config = utils.merge_dicts(config, config_) config, args_ = parse_config_args(config=config, args=args_, unknown_args=unknown_args) # hack with argparse in config config_args = config.get("args", None) if config_args is not None: for key, value in config_args.items(): arg_value = getattr(args_, key, None) if arg_value is None or \ (key in ["logdir", "baselogdir"] and arg_value == ""): arg_value = value setattr(args_, key, arg_value) return args_, config
def get_loaders_from_params( batch_size: int = 1, num_workers: int = 0, drop_last: bool = False, per_gpu_scaling: bool = False, loaders_params: Dict[str, Any] = None, samplers_params: Dict[str, Any] = None, initial_seed: int = 42, get_datasets_fn: Callable = None, **data_params, ) -> "OrderedDict[str, DataLoader]": """ Creates pytorch dataloaders from datasets and additional parameters. Args: batch_size (int): ``batch_size`` parameter from ``torch.utils.data.DataLoader`` num_workers (int): ``num_workers`` parameter from ``torch.utils.data.DataLoader`` drop_last (bool): ``drop_last`` parameter from ``torch.utils.data.DataLoader`` per_gpu_scaling (bool): boolean flag, if ``True``, uses ``batch_size=batch_size*num_available_gpus`` loaders_params (Dict[str, Any]): additional loaders parameters samplers_params (Dict[str, Any]): additional sampler parameters initial_seed (int): initial seed for ``torch.utils.data.DataLoader`` workers get_datasets_fn(Callable): callable function to get dictionary with ``torch.utils.data.Datasets`` **data_params: additional data parameters or dictionary with ``torch.utils.data.Datasets`` to use for pytorch dataloaders creation Returns: OrderedDict[str, DataLoader]: dictionary with ``torch.utils.data.DataLoader`` Raises: NotImplementedError: if datasource is out of `Dataset` or dict ValueError: if batch_sampler option is mutually exclusive with distributed """ default_batch_size = batch_size default_num_workers = num_workers loaders_params = loaders_params or {} assert isinstance(loaders_params, dict), (f"`loaders_params` should be a Dict. " f"Got: {loaders_params}") samplers_params = samplers_params or {} assert isinstance( samplers_params, dict), f"`samplers_params` should be a Dict. Got: {samplers_params}" distributed_rank = get_rank() distributed = distributed_rank > -1 if get_datasets_fn is not None: datasets = get_datasets_fn(**data_params) else: datasets = dict(**data_params) loaders = OrderedDict() for name, datasource in datasets.items(): # noqa: WPS426 assert isinstance( datasource, (Dataset, dict )), f"{datasource} should be Dataset or Dict. Got: {datasource}" loader_params = loaders_params.pop(name, {}) assert isinstance(loader_params, dict), f"{loader_params} should be Dict" sampler_params = samplers_params.pop(name, None) if sampler_params is None: if isinstance(datasource, dict) and "sampler" in datasource: sampler = datasource.pop("sampler", None) else: sampler = None else: sampler = SAMPLER.get_from_params(**sampler_params) if isinstance(datasource, dict) and "sampler" in datasource: datasource.pop("sampler", None) batch_size = loader_params.pop("batch_size", default_batch_size) num_workers = loader_params.pop("num_workers", default_num_workers) if per_gpu_scaling and not distributed: num_gpus = max(1, torch.cuda.device_count()) batch_size *= num_gpus num_workers *= num_gpus loader_params = { "batch_size": batch_size, "num_workers": num_workers, "pin_memory": torch.cuda.is_available(), "drop_last": drop_last, **loader_params, } if isinstance(datasource, Dataset): loader_params["dataset"] = datasource elif isinstance(datasource, dict): assert ( "dataset" in datasource), "You need to specify dataset for dataloader" loader_params = merge_dicts(datasource, loader_params) else: raise NotImplementedError if distributed: if sampler is not None: if not isinstance(sampler, DistributedSampler): sampler = DistributedSamplerWrapper(sampler=sampler) else: sampler = DistributedSampler(dataset=loader_params["dataset"]) loader_params["shuffle"] = name.startswith("train") and sampler is None loader_params["sampler"] = sampler if "batch_sampler" in loader_params: if distributed: raise ValueError("batch_sampler option is mutually " "exclusive with distributed") for k in ("batch_size", "shuffle", "sampler", "drop_last"): loader_params.pop(k, None) if "worker_init_fn" not in loader_params: loader_params["worker_init_fn"] = lambda x: set_global_seed( initial_seed + x) loaders[name] = DataLoader(**loader_params) return loaders