def test_decode_none() -> None: c = OmegaConf.create({"x": "${oc.decode:null}"}) assert c.x is None
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None: """merge src into dest and return a new copy, does not modified input""" from omegaconf import AnyNode, DictConfig, OmegaConf, ValueNode assert isinstance(dest, DictConfig) assert isinstance(src, DictConfig) src_type = src._metadata.object_type src_ref_type = get_ref_type(src) assert src_ref_type is not None # If source DictConfig is: # - an interpolation => set the destination DictConfig to be the same interpolation # - None => set the destination DictConfig to None if src._is_interpolation() or src._is_none(): dest._set_value(src._value()) _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) return dest._validate_merge(value=src) def expand(node: Container) -> None: rt = node._metadata.ref_type val: Any if rt is not Any: if is_dict_annotation(rt): val = {} elif is_list_annotation(rt): val = [] else: val = rt elif isinstance(node, DictConfig): val = {} else: assert False node._set_value(val) if ( src._is_missing() and not dest._is_missing() and is_structured_config(src_ref_type) ): # Replace `src` with a prototype of its corresponding structured config # whose fields are all missing (to avoid overwriting fields in `dest`). src = _create_structured_with_missing_fields( ref_type=src_ref_type, object_type=src_type ) if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing(): expand(dest) for key, src_value in src.items_ex(resolve=False): src_node = src._get_node(key, validate_access=False) dest_node = dest._get_node(key, validate_access=False) assert src_node is None or isinstance(src_node, Node) assert dest_node is None or isinstance(dest_node, Node) if isinstance(dest_node, DictConfig): dest_node._validate_merge(value=src_node) missing_src_value = _is_missing_value(src_value) if ( isinstance(dest_node, Container) and OmegaConf.is_none(dest, key) and not missing_src_value and not OmegaConf.is_none(src_value) ): expand(dest_node) if dest_node is not None and dest_node._is_interpolation(): target_node = dest_node._dereference_node( throw_on_resolution_failure=False ) if isinstance(target_node, Container): dest[key] = target_node dest_node = dest._get_node(key) if ( dest_node is None and is_structured_config(dest._metadata.element_type) and not missing_src_value ): # merging into a new node. Use element_type as a base dest[key] = DictConfig(content=dest._metadata.element_type, parent=dest) dest_node = dest._get_node(key) if dest_node is not None: if isinstance(dest_node, BaseContainer): if isinstance(src_value, BaseContainer): dest_node._merge_with(src_value) elif not missing_src_value: dest.__setitem__(key, src_value) else: if isinstance(src_value, BaseContainer): dest.__setitem__(key, src_value) else: assert isinstance(dest_node, ValueNode) assert isinstance(src_node, ValueNode) # Compare to literal missing, ignoring interpolation src_node_missing = src_value == "???" try: if isinstance(dest_node, AnyNode): if src_node_missing: node = copy.copy(src_node) # if src node is missing, use the value from the dest_node, # but validate it against the type of the src node before assigment node._set_value(dest_node._value()) else: node = src_node dest.__setitem__(key, node) else: if not src_node_missing: dest_node._set_value(src_value) except (ValidationError, ReadonlyConfigError) as e: dest._format_and_raise(key=key, value=src_value, cause=e) else: from omegaconf import open_dict if is_structured_config(src_type): # verified to be compatible above in _validate_merge with open_dict(dest): dest[key] = src._get_node(key) else: dest[key] = src._get_node(key) _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) # explicit flags on the source config are replacing the flag values in the destination flags = src._metadata.flags assert flags is not None for flag, value in flags.items(): if value is not None: dest._set_flag(flag, value)
def test_save_illegal_type() -> None: with pytest.raises(TypeError): OmegaConf.save(OmegaConf.create(), 1000) # type: ignore
def setup(run_name, training_root, validation_root, base_directory, cfg_filename, device, num_workers, resume, machine_rank): experiment_dir = base_directory / run_name if communication.get_local_rank() == 0: # Want to prevent multiple workers from trying to write a directory # This is required in the logging below experiment_dir.mkdir(parents=True, exist_ok=True) communication.synchronize() # Ensure folders are in place. # Load configs from YAML file to check which model needs to be loaded. cfg_from_file = OmegaConf.load(cfg_filename) model_name = cfg_from_file.model_name + 'Config' try: model_cfg = str_to_class(f'direct.nn.{cfg_from_file.model_name.lower()}.config', model_name) except (AttributeError, ModuleNotFoundError) as e: logger.error(f'Model configuration does not exist for {cfg_from_file.model_name} (err = {e}).') sys.exit(-1) # Load the default configs to ensure type safety base_cfg = OmegaConf.structured(DefaultConfig) base_cfg = OmegaConf.merge(base_cfg, {'model': model_cfg, 'training': TrainingConfig()}) cfg = OmegaConf.merge(base_cfg, cfg_from_file) # Setup logging log_file = experiment_dir / f'log_{machine_rank}_{communication.get_local_rank()}.txt' direct.utils.logging.setup( use_stdout=communication.get_local_rank() == 0 or cfg.debug, filename=log_file, log_level=('INFO' if not cfg.debug else 'DEBUG') ) logger.info(f'Machine rank: {machine_rank}.') logger.info(f'Local rank: {communication.get_local_rank()}.') logger.info(f'Logging: {log_file}.') logger.info(f'Saving to: {experiment_dir}.') logger.info(f'Run name: {run_name}.') logger.info(f'Config file: {cfg_filename}.') logger.info(f'Python version: {sys.version}.') logger.info(f'PyTorch version: {torch.__version__}.') # noqa logger.info(f'CUDA {torch.version.cuda} - cuDNN {torch.backends.cudnn.version()}.') logger.info(f'Configuration: {pformat(dict(cfg))}.') # Create the model logger.info('Building model.') model = MRIReconstruction(2, **cfg.model).to(device) n_params = sum(p.numel() for p in model.parameters()) logger.info(f'Number of parameters: {n_params} ({n_params / 10.0**3:.2f}k).') logger.debug(model) # Create training and validation data train_mask_func, val_mask_func = build_masking_functions(**cfg.masking) train_transforms, val_transforms = build_mri_transforms( train_mask_func, val_mask_func=val_mask_func, crop=cfg.dataset.transforms.crop) training_data, validation_data = build_datasets( cfg.dataset.name, training_root, train_sensitivity_maps=None, train_transforms=train_transforms, validation_root=validation_root, val_sensitivity_maps=None, val_transforms=val_transforms) # Create the optimizers logger.info('Building optimizers.') optimizer: torch.optim.Optimizer = str_to_class('torch.optim', cfg.training.optimizer)( # noqa model.parameters(), lr=cfg.training.lr, weight_decay=cfg.training.weight_decay ) # noqa # Build the LR scheduler, we use a fixed LR schedule step size, no adaptive training schedule. solver_steps = list(range(cfg.training.lr_step_size, cfg.training.num_iterations, cfg.training.lr_step_size)) lr_scheduler = WarmupMultiStepLR( optimizer, solver_steps, cfg.training.lr_gamma, warmup_factor=1 / 3., warmup_iters=cfg.training.lr_warmup_iter, warmup_method='linear') # Just to make sure. torch.cuda.empty_cache() # Setup training engine. engine = RIMEngine(cfg, model, device=device) engine.train( optimizer, lr_scheduler, training_data, experiment_dir, validation_data=validation_data, resume=resume, num_workers=num_workers)
@fixture( params=[ hydra._internal.instantiate._instantiate2.instantiate, ], ids=[ "instantiate2", ], ) def instantiate_func(request: Any) -> Any: return request.param @fixture( params=[ lambda cfg: copy.deepcopy(cfg), lambda cfg: OmegaConf.create(cfg), ], ids=[ "dict", "dict_config", ], ) def config(request: Any, src: Any) -> Any: config = request.param(src) cfg_copy = copy.deepcopy(config) yield config assert config == cfg_copy @mark.parametrize( "recursive",
def __init__(self) -> None: root_dir = Path(__file__).parent.parent self.config_dir: PosixPath = root_dir / "configs" self.configs: DictConfig = OmegaConf.create()
def add_tokenizer(self, langpair: str) -> None: langpair = normalize_langpair(langpair) tokenizer_config = ( self.config_dir / "tokenizer" / f"sentencepiece_bpe_wmt14_{langpair}.yaml" ) self.configs.update({"tokenizer": OmegaConf.load(tokenizer_config)})
def create_readonly(cfg: Any) -> Any: cfg = OmegaConf.create(cfg) OmegaConf.set_readonly(cfg, True) return cfg
result = trainer.fit(model) assert result == 1 assert trainer.custom_kwarg == 'custom' assert trainer.fast_dev_run # when we pass in an unknown arg, the base class should complain with pytest.raises( TypeError, match=r"__init__\(\) got an unexpected keyword argument 'abcdefg'" ): TrainerSubclass(abcdefg='unknown_arg') @pytest.mark.parametrize('trainer_params', [ OmegaConf.create({ 'max_epochs': 1, 'gpus': 1 }), OmegaConf.create({ 'max_epochs': 1, 'gpus': [0] }), ]) @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine") def test_trainer_omegaconf(trainer_params): Trainer(**trainer_params) def test_trainer_pickle(tmpdir): trainer = Trainer( max_epochs=1,
self.full_key = "" else: if isinstance(self.key, (str, int, Enum, slice)): self.full_key = self.key else: self.full_key = "" params = [ ############## # DictConfig # ############## # update pytest.param( Expected( create=lambda: OmegaConf.structured(StructuredWithMissing), op=lambda cfg: OmegaConf.update(cfg, "num", "hello"), exception_type=ValidationError, msg="Value 'hello' could not be converted to Integer", parent_node=lambda cfg: cfg, child_node=lambda cfg: cfg._get_node("num"), object_type=StructuredWithMissing, ref_type=Optional[StructuredWithMissing], key="num", ), id="structured:update_with_invalid_value", ), pytest.param( Expected( create=lambda: OmegaConf.structured(StructuredWithMissing), op=lambda cfg: OmegaConf.update(cfg, "num", None),
def create_struct(cfg: Any) -> Any: cfg = OmegaConf.create(cfg) OmegaConf.set_struct(cfg, True) return cfg
def test_RNNTDecoder(self): vocab = list(range(10)) vocab = [str(x) for x in vocab] vocab_size = len(vocab) pred_config = OmegaConf.create({ '_target_': 'nemo.collections.asr.modules.RNNTDecoder', 'prednet': { 'pred_hidden': 32, 'pred_rnn_layers': 1, }, 'vocab_size': vocab_size, 'blank_as_pad': True, }) prednet = modules.RNNTDecoder.from_config_dict(pred_config) # num params pred_hidden = pred_config.prednet.pred_hidden embed = (vocab_size + 1) * pred_hidden # embedding with blank rnn = (2 * 4 * (pred_hidden * pred_hidden + pred_hidden) ) # (ih + hh) * (ifco gates) * (indim * hiddendim + bias) assert prednet.num_weights == (embed + rnn) # State initialization x_ = torch.zeros(4, dtype=torch.float32) states = prednet.initialize_state(x_) for state_i in states: assert state_i.dtype == x_.dtype assert state_i.device == x_.device assert state_i.shape[1] == len(x_) # Blank hypotheses test blank = vocab_size hyp = Hypothesis(score=0.0, y_sequence=[blank]) cache = {} pred, states, _ = prednet.score_hypothesis(hyp, cache) assert pred.shape == torch.Size([1, 1, pred_hidden]) assert len(states) == 2 for state_i in states: assert state_i.dtype == pred.dtype assert state_i.device == pred.device assert state_i.shape[1] == len(pred) # Blank stateless predict g, states = prednet.predict(y=None, state=None, add_sos=False, batch_size=1) assert g.shape == torch.Size([1, 1, pred_hidden]) assert len(states) == 2 for state_i in states: assert state_i.dtype == g.dtype assert state_i.device == g.device assert state_i.shape[1] == len(g) # Blank stateful predict g, states2 = prednet.predict(y=None, state=states, add_sos=False, batch_size=1) assert g.shape == torch.Size([1, 1, pred_hidden]) assert len(states2) == 2 for state_i, state_j in zip(states, states2): assert (state_i - state_j).square().sum().sqrt() > 0.0 # Predict with token and state token = torch.full([1, 1], fill_value=0, dtype=torch.long) g, states = prednet.predict(y=token, state=states2, add_sos=False, batch_size=None) assert g.shape == torch.Size([1, 1, pred_hidden]) assert len(states) == 2 # Predict with blank token and no state token = torch.full([1, 1], fill_value=blank, dtype=torch.long) g, states = prednet.predict(y=token, state=None, add_sos=False, batch_size=None) assert g.shape == torch.Size([1, 1, pred_hidden]) assert len(states) == 2
import hydra from utils import wrap_continuous_env, wrap_discrete_env, get_latest from omegaconf import DictConfig, OmegaConf from hydra.utils import instantiate, call, get_original_cwd OmegaConf.register_new_resolver("parse_string", lambda input : input.lower().replace(" ", "_")) OmegaConf.register_new_resolver("get_wrapper_func", lambda continuous : wrap_continuous_env if continuous else wrap_discrete_env) OmegaConf.register_new_resolver("original_dir", lambda relative_path : get_original_cwd() + relative_path) OmegaConf.register_new_resolver("get_latest", get_latest) OmegaConf.register_new_resolver("wandb_mode", lambda save : "online" if save else "disabled") @hydra.main(config_path="../config", config_name="config") def main(config: DictConfig): env = call(config.environment) callbacks = list(instantiate(config.callbacks)["callbacks"]) model = call(config.model, env) model.learn(total_timesteps=config.run["max_timesteps"], callback=callbacks) if config.run["save"]: model.save(config.model["save_path"]) if __name__ == "__main__": main()
def load_config(configs, database_config, parameters_replacement): loaded_json = [] for config in configs: try: tmp_json = json.loads(config.read()) except Exception as exception: raise Exception( "Error loading input file called {}. Reason: '{}'".format( config.name, exception ) ) # Special check for metadata files, if they exist the idea is to replace the # non existent constants with their default values if ( config.name.endswith("metadata.json") and tmp_json["Constants"] and type(tmp_json["Constants"]) is list ): tmp_json["Constants"] = dict( map( lambda constant: ( constant["Name"], constant["DefaultValue"], ), tmp_json["Constants"], ) ) loaded_json.append(convert_dot_field_to_dict(tmp_json)) modified_env_vars = OmegaConf.create() if database_config: modified_env_vars.update(database_config) for prefix in [ENV_VAR_RUNTIME_PREFIX, ENV_VAR_BROKER_PREFIX]: env_vars = dict( filter( lambda key: key[0].startswith(prefix) and key[0] in whitelist, dict(os.environ).items(), ) ) for key, value in env_vars.items(): new_key = __curate_key(key, prefix) OmegaConf.update(modified_env_vars, new_key, value) # Fetch and update any constants passed as env var const_env_vars = dict( filter( lambda key: key[0].startswith(CONSTANTS_ENV_VAR_PREFIX), dict(os.environ).items(), ) ) modified_constants = OmegaConf.create({"Constants": {}}) for key, value in const_env_vars.items(): new_key = key.replace(CONSTANTS_ENV_VAR_PREFIX, "", 1) new_key = new_key.replace("_", ".", 1) OmegaConf.update(modified_constants.Constants, new_key, value) parameters_replacement_dict = OmegaConf.create() for key, value in parameters_replacement: OmegaConf.update(parameters_replacement_dict, key, value) try: complete_conf = OmegaConf.merge( *loaded_json, modified_env_vars, modified_constants, parameters_replacement_dict ) bootstrap_servers = get_value_for_constant( complete_conf, complete_conf.DataBrokerConfiguration.publishedServices[ 0 ].brokerUrl, ) OmegaConf.update( complete_conf, BOOTSTRAP_SERVERS_KEY, bootstrap_servers ) if not OmegaConf.select(complete_conf, NODE_COUNT_KEY): complete_conf[NODE_COUNT_KEY] = 1 __generate_source_topic_names(complete_conf) return complete_conf except Exception as exception: raise Exception( "Error while reading input config files. Reason: '{}'".format( exception ) )
) from tests.test_modules.generated import PeskySentinelUsageConf # chdir_hydra_root(subdir="tools/configen") ## # To re-generate the expected config run the following command from configen's root directory (tools/configen). # # PYTHONPATH=. configen --config-dir tests/gen-test-expected/ # ## conf: ConfigenConf = OmegaConf.structured( ConfigenConf( header= """# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved # Generated by configen, do not edit. # See https://github.com/facebookresearch/hydra/tree/master/tools/configen # fmt: off # isort:skip_file # flake8: noqa """)) MODULE_NAME = "tests.test_modules" def test_generated_code() -> None: classes = [ "Empty", "UntypedArg", "IntArg", "Args", "Kwargs",
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Simple core for Discord bot with dynamic extension support. """ from datetime import datetime, timedelta, timezone import discord from discord.ext import commands from discord.utils import sleep_until from omegaconf import OmegaConf conf = OmegaConf.load('config.yaml').discopig GUILD = conf.DISCORD_GUILD TOKEN = conf.DISCORD_TOKEN CATEGORY = conf.CATEGORY LOBBY = conf.LOBBY PREFIX = conf.CMD_PREFIX bot = commands.Bot(command_prefix=PREFIX) async def send_dm(user, txt): channel = user.dm_channel if channel is None: channel = await user.create_dm() msg = await channel.send(txt) return msg
def main(cfg: DictConfig) -> None: # set up mlflow experiment id mlflow.set_tracking_uri(f"file://{to_absolute_path(cfg.path_to_mlflow)}") experiment = mlflow.get_experiment_by_name(cfg.experiment_name) if experiment is not None: run_kwargs = {'experiment_id': experiment.experiment_id} if cfg["pretrained"] is not None: # initialise with pretrained run, otherwise create a new run run_kwargs['run_id'] = cfg["pretrained"]["run_id"] else: # create new experiment experiment_id = mlflow.create_experiment(cfg.experiment_name) run_kwargs = {'experiment_id': experiment_id} # run the training with mlflow tracking with mlflow.start_run(**run_kwargs) as main_run: if cfg["pretrained"] is not None: mlflow.start_run(experiment_id=run_kwargs['experiment_id'], nested=True) active_run = mlflow.active_run() run_id = active_run.info.run_id setup_gpu(cfg.gpu_cfg) training_cfg = OmegaConf.to_object( cfg.training_cfg) # convert to python dictionary scaling_cfg = to_absolute_path(cfg.scaling_cfg) dataloader = DataLoader.DataLoader(training_cfg, scaling_cfg) setup = dataloader.config["SetupNN"] TauLosses.SetSFs(*setup["TauLossesSFs"]) print("loss consts:", TauLosses.Le_sf, TauLosses.Lmu_sf, TauLosses.Ltau_sf, TauLosses.Ljet_sf) if setup["using_new_loss"]: tf.config.run_functions_eagerly(True) netConf_full = dataloader.get_net_config() if dataloader.input_type == "Adversarial": model = create_model( netConf_full, dataloader.model_name, loss=setup["loss"], use_newloss=setup["using_new_loss"], use_AdvDataset=True, adv_param=dataloader.adversarial_parameter, n_adv_tau=dataloader.adv_batch_size, adv_learning_rate=dataloader.adv_learning_rate) else: model = create_model(netConf_full, dataloader.model_name, loss=setup["loss"], use_newloss=setup["using_new_loss"]) if cfg.pretrained is None: print( "Warning: no pretrained NN -> training will be started from scratch" ) old_opt = None else: print("Warning: training will be started from pretrained model.") print( f"Model: run_id={cfg.pretrained.run_id}, experiment_id={cfg.pretrained.experiment_id}, model={cfg.pretrained.starting_model}" ) path_to_pretrain = to_absolute_path( f'{cfg.path_to_mlflow}/{cfg.pretrained.experiment_id}/{cfg.pretrained.run_id}/artifacts/' ) old_model = load_model( path_to_pretrain + f"/model_checkpoints/{cfg.pretrained.starting_model}", compile=False, custom_objects=None) for layer in model.layers: weights_found = False for old_layer in old_model.layers: if layer.name == old_layer.name: layer.set_weights(old_layer.get_weights()) weights_found = True break if not weights_found: print(f"Weights for layer '{layer.name}' not found.") old_opt = old_model.optimizer old_vars = [var.name for var in old_model.trainable_variables] compile_model(model, setup["optimizer_name"], setup["learning_rate"], setup["metrics"], setup["schedule_decay"]) fit_hist = run_training(model, dataloader, False, cfg.log_suffix, setup["using_new_loss"], old_opt=old_opt) # log NN params for net_type in [ 'tau_net', 'comp_net', 'comp_merge_net', 'conv_2d_net', 'dense_net' ]: mlflow.log_params({ f'{net_type}_{k}': v for k, v in cfg.training_cfg.SetupNN[net_type].items() }) mlflow.log_params({ f'TauLossesSFs_{i}': v for i, v in enumerate(cfg.training_cfg.SetupNN.TauLossesSFs) }) with open( to_absolute_path( f'{cfg.path_to_mlflow}/{run_kwargs["experiment_id"]}/{run_id}/artifacts/model_summary.txt' )) as f: for l in f: if (s := 'Trainable params: ') in l: mlflow.log_param('n_train_params', int(l.split(s)[-1].replace(',', ''))) # log training related files mlflow.log_dict(training_cfg, 'input_cfg/training_cfg.yaml') mlflow.log_artifact(scaling_cfg, 'input_cfg') mlflow.log_artifact(to_absolute_path("Training_CNN.py"), 'input_cfg') mlflow.log_artifact(to_absolute_path("common.py"), 'input_cfg') # log hydra files mlflow.log_artifacts('.hydra', 'input_cfg/hydra') mlflow.log_artifact('Training_CNN.log', 'input_cfg/hydra') # log misc. info mlflow.log_param('run_id', run_id) mlflow.log_param('git_commit', _get_git_commit(to_absolute_path('.'))) print( f'\nTraining has finished! Corresponding MLflow experiment name (ID): {cfg.experiment_name}({run_kwargs["experiment_id"]}), and run ID: {run_id}\n' ) mlflow.end_run() # Temporary workaround to kill additional subprocesses that have not exited correctly try: current_process = psutil.Process() children = current_process.children(recursive=True) for child in children: child.kill() except: pass
def _get_callback(params): target = params.pop("_target_") callback_class = hydra.utils.get_class(target) params = OmegaConf.to_container(params, resolve=True) callback = callback_class(**params) return callback
def add_data(self, langpair: str) -> None: langpair = normalize_langpair(langpair) data_config = self.config_dir / "data" / f"wmt14.{langpair}.yaml" self.configs.update({"data": OmegaConf.load(data_config)})
def hparams(self) -> OrderedDict: """Returns hyperparameters""" return OrderedDict(OmegaConf.to_container(self._config, resolve=True))
def add_model(self, is_base: bool = True) -> None: model_type = "base" if is_base else "big" model_config = self.config_dir / "model" / f"transformer-{model_type}.yaml" self.configs.update({"model": OmegaConf.load(model_config)})
def get_architect(self) -> LocalArchitect: """We need to specify that the architect is launching on localhost for testing""" arch_args = LocalArchitectArgs(hostname="http://localhost", port="3000") args = OmegaConf.structured(MephistoConfig(architect=arch_args)) self.curr_architect = self.ArchitectClass(self.db, args, SharedTaskState(), self.task_run, self.build_dir) return self.curr_architect
def my_app(cfg: Config) -> None: print(OmegaConf.to_yaml(cfg))
def exp_manager(trainer: 'pytorch_lightning.Trainer', cfg: Optional[Union[DictConfig, Dict]] = None) -> Path: """ exp_manager is a helper function used to manage folders for experiments. It follows the pytorch lightning paradigm of exp_dir/model_or_experiment_name/version. If the lightning trainer has a logger, exp_manager will get exp_dir, name, and version from the logger. Otherwise it will use the exp_dir and name arguments to create the logging directory. exp_manager also allows for explicit folder creation via explicit_log_dir. The version will be a datetime string or an integer. Note, exp_manager does not handle versioning on slurm multi-node runs. Datestime version can be disabled if use_datetime_version is set to False. It optionally creates TensorBoardLogger, WandBLogger, ModelCheckpoint objects from pytorch lightning. It copies sys.argv, and git information if available to the logging directory. It creates a log file for each process to log their output into. exp_manager additionally has a resume feature which can be used to continuing training from the constructed log_dir. Args: trainer (pytorch_lightning.Trainer): The lightning trainer. cfg (DictConfig, dict): Can have the following keys: - explicit_log_dir (str, Path): Can be used to override exp_dir/name/version folder creation. Defaults to None, which will use exp_dir, name, and version to construct the logging directory. - exp_dir (str, Path): The base directory to create the logging directory. Defaults to None, which logs to ./nemo_experiments. - name (str): The name of the experiment. Defaults to None which turns into "default" via name = name or "default". - version (str): The version of the experiment. Defaults to None which uses either a datetime string or lightning's TensorboardLogger system of using version_{int}. - use_datetime_version (bool): Whether to use a datetime string for version. Defaults to True. - resume_if_exists (bool): Whether this experiment is resuming from a previous run. If True, it sets trainer.resume_from_checkpoint so that the trainer should auto-resume. exp_manager will move files under log_dir to log_dir/run_{int}. Defaults to False. - resume_past_end (bool): exp_manager errors out if resume_if_exists is True and a checkpoint matching *end.ckpt indicating a previous training run fully completed. This behaviour can be disabled, in which case the *end.ckpt will be loaded by setting resume_past_end to True. Defaults to False. - resume_ignore_no_checkpoint (bool): exp_manager errors out if resume_if_exists is True and no checkpoint could be found. This behaviour can be disabled, in which case exp_manager will print a message and continue without restoring, by setting resume_ignore_no_checkpoint to True. Defaults to False. - create_tensorboard_logger (bool): Whether to create a tensorboard logger and attach it to the pytorch lightning trainer. Defaults to True. - summary_writer_kwargs (dict): A dictionary of kwargs that can be passed to lightning's TensorboardLogger class. Note that log_dir is passed by exp_manager and cannot exist in this dict. Defaults to None. - create_wandb_logger (bool): Whether to create a Weights and Baises logger and attach it to the pytorch lightning trainer. Defaults to False. - wandb_logger_kwargs (dict): A dictionary of kwargs that can be passed to lightning's WandBLogger class. Note that name and project are required parameters if create_wandb_logger is True. Defaults to None. - create_checkpoint_callback (bool): Whether to create a ModelCheckpoint callback and attach it to the pytorch lightning trainer. The ModelCheckpoint saves the top 3 models with the best "val_loss", the most recent checkpoint under *last.ckpt, and the final checkpoint after training completes under *end.ckpt. Defaults to True. - files_to_copy (list): A list of files to copy to the experiment logging directory. Defaults to None which copies no files. returns: log_dir (Path): The final logging directory where logging files are saved. Usually the concatenation of exp_dir, name, and version. """ if cfg is None: logging.error("exp_manager did not receive a cfg argument. It will be disabled.") return # Ensure passed cfg is compliant with ExpManagerConfig schema = OmegaConf.structured(ExpManagerConfig) if isinstance(cfg, dict): cfg = OmegaConf.create(cfg) elif not isinstance(cfg, DictConfig): raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") cfg = OmegaConf.create(OmegaConf.to_container(cfg, resolve=True)) cfg = OmegaConf.merge(schema, cfg) error_checks(trainer, cfg) # Ensures that trainer options are compliant with NeMo and exp_manager arguments log_dir, exp_dir, name, version = get_log_dir( trainer=trainer, exp_dir=cfg.exp_dir, name=cfg.name, version=cfg.version, explicit_log_dir=cfg.explicit_log_dir, use_datetime_version=cfg.use_datetime_version, ) if cfg.resume_if_exists: check_resume(trainer, log_dir, cfg.resume_past_end, cfg.resume_ignore_no_checkpoint) checkpoint_name = name # If name returned from get_log_dir is "", use cfg.name for checkpointing if checkpoint_name is None or checkpoint_name == '': checkpoint_name = cfg.name or "default" cfg.name = name # Used for configure_loggers so that the log_dir is properly set even if name is "" cfg.version = version # Create the logging directory if it does not exist os.makedirs(log_dir, exist_ok=True) # Cannot limit creation to global zero as all ranks write to own log file logging.info(f'Experiments will be logged at {log_dir}') trainer._default_root_dir = log_dir # Handle Loggers by creating file and handle DEBUG statements # Note: trainer.global_rank and trainer.is_global_zero are not set until trainer.fit, so have to hack around it global_rank = trainer.node_rank * trainer.num_gpus + trainer.local_rank log_file = log_dir / f'nemo_log_globalrank-{global_rank}_localrank-{trainer.local_rank}.txt' logging.add_file_handler(log_file) logging.rank = global_rank # For some reason, LearningRateLogger requires trainer to have a logger. Safer to create logger on all ranks # not just global rank 0. if cfg.create_tensorboard_logger or cfg.create_wandb_logger: configure_loggers( trainer, exp_dir, cfg.name, cfg.version, cfg.create_tensorboard_logger, cfg.summary_writer_kwargs, cfg.create_wandb_logger, cfg.wandb_logger_kwargs, ) if is_global_rank_zero(): if cfg.create_checkpoint_callback: configure_checkpointing(trainer, log_dir, checkpoint_name, cfg.checkpoint_callback_params) # Move files_to_copy to folder and add git information if present if cfg.files_to_copy: for _file in cfg.files_to_copy: copy(Path(_file), log_dir) # Create files for cmd args and git info with open(log_dir / 'cmd-args.log', 'w') as _file: _file.write(" ".join(sys.argv)) # Try to get git hash git_repo, git_hash = get_git_hash() if git_repo: with open(log_dir / 'git-info.log', 'w') as _file: _file.write(f'commit hash: {git_hash}') _file.write(get_git_diff()) # Add err_file logging to global_rank zero logging.add_err_file_handler(log_dir / 'nemo_error_log.txt') # Add lightning file logging to global_rank zero add_filehandlers_to_pl_logger(log_dir / 'lightning_logs.txt', log_dir / 'nemo_error_log.txt') return log_dir
CONFIG_DEFAULT: DictConfig = OmegaConf.create({ 'config': { 'file_in': 'itodo.yml', 'file_out': None, }, 'searchDir': ['../modules', '../pyutil'], 'file_todo': 'todo-inline.md', 'verbose': False, # 'output' : { # 'links' : True, # 'tag-sort' : True, # 'file-sort' : True, # 'dir-sort' : True, # 'format' : 'md', # }, 'read': { 'tags': { 'list': [ "CRIT", "TODO", "FIXME", "FIX", "BUG", "DEBUG", "UGLY", "HACK", "NOTE", "IDEA", "REVIEW", "OPTIMIZE", "CONFIG", "!!!", "OLD" ], }, # 'comments' : { # 'require' : False, # 'list' : [ # '//', # '/*' # '#', # '<!--', # '%', # '\\', # ], # }, # 'prefix' : { # 'PREFIX' : '@', # 'require' : False, # 'always_accept' : False, # }, 'SOURCE_FILES': [ 'c', 'cpp', 'h', 'hpp', 'py', 'm', 'tex', 'sh', 'java', 'js', ], 'EXCLUDE': [ 'inline_todo.py', 'itodo.yml', 'todo-inline.md', ], 'MAX_SEARCH_LEN': 15, 'context': { 'enabled': True, 'lines': 5, # number of lines to show before and after the tag }, }, 'write': { 'attr_sort_order': ['tag', 'file', 'lineNum'], 'item_format': 'md_det', } })
from omegaconf import OmegaConf import os cfg = OmegaConf.load( os.path.join(os.getenv("PROJECT_DIR"), "config/config.yaml"))
from omegaconf import OmegaConf from src.utils.utils import read_data, get_all_db_files, get_complete_table import re import unidecode from nltk.corpus import stopwords from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import RepeatedStratifiedKFold from sklearn import metrics # - paths = OmegaConf.load("config/paths.yaml") # # Settings DATASET = "dataset_v09" TEST_SET_NUM = "2" SEED = int(TEST_SET_NUM) # # Functions def clean_message(message): stop_words = set(stopwords.words("spanish")) message = message.lower() message = unidecode.unidecode(message)
def __init__(self, labels: List, model_cfg: Union[UniDirectionalConfig, BiDirectionalConfig], precision: int, optim_cfg: Union[AdamConfig, SGDConfig], spect_cfg: SpectConfig): super().__init__() self.save_hyperparameters() self.model_cfg = model_cfg self.precision = precision self.optim_cfg = optim_cfg self.spect_cfg = spect_cfg self.bidirectional = True if OmegaConf.get_type( model_cfg) is BiDirectionalConfig else False print('\n Bidirectional or Not \n', self.bidirectional) print('\n OMEGA CONF \n', model_cfg) self.labels = labels num_classes = len(self.labels) self.conv = MaskConv( nn.Sequential( nn.Conv2d(1, 32, kernel_size=(41, 11), stride=(2, 2), padding=(20, 5)), nn.BatchNorm2d(32), nn.Hardtanh(0, 20, inplace=True), nn.Conv2d(32, 32, kernel_size=(21, 11), stride=(2, 1), padding=(10, 5)), nn.BatchNorm2d(32), nn.Hardtanh(0, 20, inplace=True))) # Based on above convolutions and spectrogram size using conv formula (W - F + 2P)/ S+1 rnn_input_size = int( math.floor((self.spect_cfg.sample_rate * self.spect_cfg.window_size) / 2) + 1) rnn_input_size = int(math.floor(rnn_input_size + 2 * 20 - 41) / 2 + 1) rnn_input_size = int(math.floor(rnn_input_size + 2 * 10 - 21) / 2 + 1) rnn_input_size *= 32 self.rnns = nn.Sequential( BatchRNN(input_size=rnn_input_size, hidden_size=self.model_cfg.hidden_size, rnn_type=self.model_cfg.rnn_type.value, bidirectional=self.bidirectional, batch_norm=False), *(BatchRNN(input_size=self.model_cfg.hidden_size, hidden_size=self.model_cfg.hidden_size, rnn_type=self.model_cfg.rnn_type.value, bidirectional=self.bidirectional) for x in range(self.model_cfg.hidden_layers - 1))) self.lookahead = nn.Sequential( # consider adding batch norm? Lookahead(self.model_cfg.hidden_size, context=self.model_cfg.lookahead_context), nn.Hardtanh(0, 20, inplace=True)) if not self.bidirectional else None fully_connected = nn.Sequential( nn.BatchNorm1d(self.model_cfg.hidden_size), nn.Linear(self.model_cfg.hidden_size, num_classes, bias=False)) self.fc = nn.Sequential(SequenceWise(fully_connected), ) self.inference_softmax = InferenceBatchSoftmax() self.criterion = LWLRAP(precision=self.precision) self.lwlrap = LWLRAP(precision=self.precision) self.loss = RMSELoss() self.softmax = nn.Softmax(dim=-1) # self.loss = nn.BCEWithLogitsLoss() self.sigmoid = nn.Sigmoid()
def test_load_from_invalid() -> None: with pytest.raises(TypeError): OmegaConf.load(3.1415) # type: ignore
def test_decode_error(monkeypatch: Any, value: Any, exc: Any) -> None: c = OmegaConf.create({"x": f"${{oc.decode:{value}}}"}) with exc: c.x