def test_decode_none() -> None:
c = OmegaConf.create({"x": "${oc.decode:null}"})
assert c.x is None
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None:
"""merge src into dest and return a new copy, does not modified input"""
from omegaconf import AnyNode, DictConfig, OmegaConf, ValueNode
assert isinstance(dest, DictConfig)
assert isinstance(src, DictConfig)
src_type = src._metadata.object_type
src_ref_type = get_ref_type(src)
assert src_ref_type is not None
# If source DictConfig is:
# - an interpolation => set the destination DictConfig to be the same interpolation
# - None => set the destination DictConfig to None
if src._is_interpolation() or src._is_none():
dest._set_value(src._value())
_update_types(node=dest, ref_type=src_ref_type, object_type=src_type)
return
dest._validate_merge(value=src)
def expand(node: Container) -> None:
rt = node._metadata.ref_type
val: Any
if rt is not Any:
if is_dict_annotation(rt):
val = {}
elif is_list_annotation(rt):
val = []
else:
val = rt
elif isinstance(node, DictConfig):
val = {}
else:
assert False
node._set_value(val)
if (
src._is_missing()
and not dest._is_missing()
and is_structured_config(src_ref_type)
):
# Replace `src` with a prototype of its corresponding structured config
# whose fields are all missing (to avoid overwriting fields in `dest`).
src = _create_structured_with_missing_fields(
ref_type=src_ref_type, object_type=src_type
)
if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing():
expand(dest)
for key, src_value in src.items_ex(resolve=False):
src_node = src._get_node(key, validate_access=False)
dest_node = dest._get_node(key, validate_access=False)
assert src_node is None or isinstance(src_node, Node)
assert dest_node is None or isinstance(dest_node, Node)
if isinstance(dest_node, DictConfig):
dest_node._validate_merge(value=src_node)
missing_src_value = _is_missing_value(src_value)
if (
isinstance(dest_node, Container)
and OmegaConf.is_none(dest, key)
and not missing_src_value
and not OmegaConf.is_none(src_value)
):
expand(dest_node)
if dest_node is not None and dest_node._is_interpolation():
target_node = dest_node._dereference_node(
throw_on_resolution_failure=False
)
if isinstance(target_node, Container):
dest[key] = target_node
dest_node = dest._get_node(key)
if (
dest_node is None
and is_structured_config(dest._metadata.element_type)
and not missing_src_value
):
# merging into a new node. Use element_type as a base
dest[key] = DictConfig(content=dest._metadata.element_type, parent=dest)
dest_node = dest._get_node(key)
if dest_node is not None:
if isinstance(dest_node, BaseContainer):
if isinstance(src_value, BaseContainer):
dest_node._merge_with(src_value)
elif not missing_src_value:
dest.__setitem__(key, src_value)
else:
if isinstance(src_value, BaseContainer):
dest.__setitem__(key, src_value)
else:
assert isinstance(dest_node, ValueNode)
assert isinstance(src_node, ValueNode)
# Compare to literal missing, ignoring interpolation
src_node_missing = src_value == "???"
try:
if isinstance(dest_node, AnyNode):
if src_node_missing:
node = copy.copy(src_node)
# if src node is missing, use the value from the dest_node,
# but validate it against the type of the src node before assigment
node._set_value(dest_node._value())
else:
node = src_node
dest.__setitem__(key, node)
else:
if not src_node_missing:
dest_node._set_value(src_value)
except (ValidationError, ReadonlyConfigError) as e:
dest._format_and_raise(key=key, value=src_value, cause=e)
else:
from omegaconf import open_dict
if is_structured_config(src_type):
# verified to be compatible above in _validate_merge
with open_dict(dest):
dest[key] = src._get_node(key)
else:
dest[key] = src._get_node(key)
_update_types(node=dest, ref_type=src_ref_type, object_type=src_type)
# explicit flags on the source config are replacing the flag values in the destination
flags = src._metadata.flags
assert flags is not None
for flag, value in flags.items():
if value is not None:
dest._set_flag(flag, value)
def test_save_illegal_type() -> None:
with pytest.raises(TypeError):
OmegaConf.save(OmegaConf.create(), 1000) # type: ignore
def setup(run_name, training_root, validation_root, base_directory,
cfg_filename, device, num_workers, resume, machine_rank):
experiment_dir = base_directory / run_name
if communication.get_local_rank() == 0:
# Want to prevent multiple workers from trying to write a directory
# This is required in the logging below
experiment_dir.mkdir(parents=True, exist_ok=True)
communication.synchronize() # Ensure folders are in place.
# Load configs from YAML file to check which model needs to be loaded.
cfg_from_file = OmegaConf.load(cfg_filename)
model_name = cfg_from_file.model_name + 'Config'
try:
model_cfg = str_to_class(f'direct.nn.{cfg_from_file.model_name.lower()}.config', model_name)
except (AttributeError, ModuleNotFoundError) as e:
logger.error(f'Model configuration does not exist for {cfg_from_file.model_name} (err = {e}).')
sys.exit(-1)
# Load the default configs to ensure type safety
base_cfg = OmegaConf.structured(DefaultConfig)
base_cfg = OmegaConf.merge(base_cfg, {'model': model_cfg, 'training': TrainingConfig()})
cfg = OmegaConf.merge(base_cfg, cfg_from_file)
# Setup logging
log_file = experiment_dir / f'log_{machine_rank}_{communication.get_local_rank()}.txt'
direct.utils.logging.setup(
use_stdout=communication.get_local_rank() == 0 or cfg.debug,
filename=log_file,
log_level=('INFO' if not cfg.debug else 'DEBUG')
)
logger.info(f'Machine rank: {machine_rank}.')
logger.info(f'Local rank: {communication.get_local_rank()}.')
logger.info(f'Logging: {log_file}.')
logger.info(f'Saving to: {experiment_dir}.')
logger.info(f'Run name: {run_name}.')
logger.info(f'Config file: {cfg_filename}.')
logger.info(f'Python version: {sys.version}.')
logger.info(f'PyTorch version: {torch.__version__}.') # noqa
logger.info(f'CUDA {torch.version.cuda} - cuDNN {torch.backends.cudnn.version()}.')
logger.info(f'Configuration: {pformat(dict(cfg))}.')
# Create the model
logger.info('Building model.')
model = MRIReconstruction(2, **cfg.model).to(device)
n_params = sum(p.numel() for p in model.parameters())
logger.info(f'Number of parameters: {n_params} ({n_params / 10.0**3:.2f}k).')
logger.debug(model)
# Create training and validation data
train_mask_func, val_mask_func = build_masking_functions(**cfg.masking)
train_transforms, val_transforms = build_mri_transforms(
train_mask_func, val_mask_func=val_mask_func, crop=cfg.dataset.transforms.crop)
training_data, validation_data = build_datasets(
cfg.dataset.name, training_root, train_sensitivity_maps=None, train_transforms=train_transforms,
validation_root=validation_root, val_sensitivity_maps=None, val_transforms=val_transforms)
# Create the optimizers
logger.info('Building optimizers.')
optimizer: torch.optim.Optimizer = str_to_class('torch.optim', cfg.training.optimizer)( # noqa
model.parameters(), lr=cfg.training.lr, weight_decay=cfg.training.weight_decay
) # noqa
# Build the LR scheduler, we use a fixed LR schedule step size, no adaptive training schedule.
solver_steps = list(range(cfg.training.lr_step_size, cfg.training.num_iterations, cfg.training.lr_step_size))
lr_scheduler = WarmupMultiStepLR(
optimizer, solver_steps, cfg.training.lr_gamma, warmup_factor=1 / 3.,
warmup_iters=cfg.training.lr_warmup_iter, warmup_method='linear')
# Just to make sure.
torch.cuda.empty_cache()
# Setup training engine.
engine = RIMEngine(cfg, model, device=device)
engine.train(
optimizer, lr_scheduler, training_data, experiment_dir,
validation_data=validation_data, resume=resume, num_workers=num_workers)
@fixture(
params=[
hydra._internal.instantiate._instantiate2.instantiate,
],
ids=[
"instantiate2",
],
)
def instantiate_func(request: Any) -> Any:
return request.param
@fixture(
params=[
lambda cfg: copy.deepcopy(cfg),
lambda cfg: OmegaConf.create(cfg),
],
ids=[
"dict",
"dict_config",
],
)
def config(request: Any, src: Any) -> Any:
config = request.param(src)
cfg_copy = copy.deepcopy(config)
yield config
assert config == cfg_copy
@mark.parametrize(
"recursive",
def __init__(self) -> None:
root_dir = Path(__file__).parent.parent
self.config_dir: PosixPath = root_dir / "configs"
self.configs: DictConfig = OmegaConf.create()
def add_tokenizer(self, langpair: str) -> None:
langpair = normalize_langpair(langpair)
tokenizer_config = (
self.config_dir / "tokenizer" / f"sentencepiece_bpe_wmt14_{langpair}.yaml"
)
self.configs.update({"tokenizer": OmegaConf.load(tokenizer_config)})
def create_readonly(cfg: Any) -> Any:
cfg = OmegaConf.create(cfg)
OmegaConf.set_readonly(cfg, True)
return cfg
result = trainer.fit(model)
assert result == 1
assert trainer.custom_kwarg == 'custom'
assert trainer.fast_dev_run
# when we pass in an unknown arg, the base class should complain
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'abcdefg'"
):
TrainerSubclass(abcdefg='unknown_arg')
@pytest.mark.parametrize('trainer_params', [
OmegaConf.create({
'max_epochs': 1,
'gpus': 1
}),
OmegaConf.create({
'max_epochs': 1,
'gpus': [0]
}),
])
@pytest.mark.skipif(not torch.cuda.is_available(),
reason="test requires GPU machine")
def test_trainer_omegaconf(trainer_params):
Trainer(**trainer_params)
def test_trainer_pickle(tmpdir):
trainer = Trainer(
max_epochs=1,
self.full_key = ""
else:
if isinstance(self.key, (str, int, Enum, slice)):
self.full_key = self.key
else:
self.full_key = ""
params = [
##############
# DictConfig #
##############
# update
pytest.param(
Expected(
create=lambda: OmegaConf.structured(StructuredWithMissing),
op=lambda cfg: OmegaConf.update(cfg, "num", "hello"),
exception_type=ValidationError,
msg="Value 'hello' could not be converted to Integer",
parent_node=lambda cfg: cfg,
child_node=lambda cfg: cfg._get_node("num"),
object_type=StructuredWithMissing,
ref_type=Optional[StructuredWithMissing],
key="num",
),
id="structured:update_with_invalid_value",
),
pytest.param(
Expected(
create=lambda: OmegaConf.structured(StructuredWithMissing),
op=lambda cfg: OmegaConf.update(cfg, "num", None),
def create_struct(cfg: Any) -> Any:
cfg = OmegaConf.create(cfg)
OmegaConf.set_struct(cfg, True)
return cfg
def test_RNNTDecoder(self):
vocab = list(range(10))
vocab = [str(x) for x in vocab]
vocab_size = len(vocab)
pred_config = OmegaConf.create({
'_target_': 'nemo.collections.asr.modules.RNNTDecoder',
'prednet': {
'pred_hidden': 32,
'pred_rnn_layers': 1,
},
'vocab_size': vocab_size,
'blank_as_pad': True,
})
prednet = modules.RNNTDecoder.from_config_dict(pred_config)
# num params
pred_hidden = pred_config.prednet.pred_hidden
embed = (vocab_size + 1) * pred_hidden # embedding with blank
rnn = (2 * 4 * (pred_hidden * pred_hidden + pred_hidden)
) # (ih + hh) * (ifco gates) * (indim * hiddendim + bias)
assert prednet.num_weights == (embed + rnn)
# State initialization
x_ = torch.zeros(4, dtype=torch.float32)
states = prednet.initialize_state(x_)
for state_i in states:
assert state_i.dtype == x_.dtype
assert state_i.device == x_.device
assert state_i.shape[1] == len(x_)
# Blank hypotheses test
blank = vocab_size
hyp = Hypothesis(score=0.0, y_sequence=[blank])
cache = {}
pred, states, _ = prednet.score_hypothesis(hyp, cache)
assert pred.shape == torch.Size([1, 1, pred_hidden])
assert len(states) == 2
for state_i in states:
assert state_i.dtype == pred.dtype
assert state_i.device == pred.device
assert state_i.shape[1] == len(pred)
# Blank stateless predict
g, states = prednet.predict(y=None,
state=None,
add_sos=False,
batch_size=1)
assert g.shape == torch.Size([1, 1, pred_hidden])
assert len(states) == 2
for state_i in states:
assert state_i.dtype == g.dtype
assert state_i.device == g.device
assert state_i.shape[1] == len(g)
# Blank stateful predict
g, states2 = prednet.predict(y=None,
state=states,
add_sos=False,
batch_size=1)
assert g.shape == torch.Size([1, 1, pred_hidden])
assert len(states2) == 2
for state_i, state_j in zip(states, states2):
assert (state_i - state_j).square().sum().sqrt() > 0.0
# Predict with token and state
token = torch.full([1, 1], fill_value=0, dtype=torch.long)
g, states = prednet.predict(y=token,
state=states2,
add_sos=False,
batch_size=None)
assert g.shape == torch.Size([1, 1, pred_hidden])
assert len(states) == 2
# Predict with blank token and no state
token = torch.full([1, 1], fill_value=blank, dtype=torch.long)
g, states = prednet.predict(y=token,
state=None,
add_sos=False,
batch_size=None)
assert g.shape == torch.Size([1, 1, pred_hidden])
assert len(states) == 2
import hydra
from utils import wrap_continuous_env, wrap_discrete_env, get_latest
from omegaconf import DictConfig, OmegaConf
from hydra.utils import instantiate, call, get_original_cwd
OmegaConf.register_new_resolver("parse_string", lambda input : input.lower().replace(" ", "_"))
OmegaConf.register_new_resolver("get_wrapper_func", lambda continuous : wrap_continuous_env if continuous else wrap_discrete_env)
OmegaConf.register_new_resolver("original_dir", lambda relative_path : get_original_cwd() + relative_path)
OmegaConf.register_new_resolver("get_latest", get_latest)
OmegaConf.register_new_resolver("wandb_mode", lambda save : "online" if save else "disabled")
@hydra.main(config_path="../config", config_name="config")
def main(config: DictConfig):
env = call(config.environment)
callbacks = list(instantiate(config.callbacks)["callbacks"])
model = call(config.model, env)
model.learn(total_timesteps=config.run["max_timesteps"], callback=callbacks)
if config.run["save"]: model.save(config.model["save_path"])
if __name__ == "__main__":
main()
def load_config(configs, database_config, parameters_replacement):
loaded_json = []
for config in configs:
try:
tmp_json = json.loads(config.read())
except Exception as exception:
raise Exception(
"Error loading input file called {}. Reason: '{}'".format(
config.name, exception
)
)
# Special check for metadata files, if they exist the idea is to replace the
# non existent constants with their default values
if (
config.name.endswith("metadata.json")
and tmp_json["Constants"]
and type(tmp_json["Constants"]) is list
):
tmp_json["Constants"] = dict(
map(
lambda constant: (
constant["Name"],
constant["DefaultValue"],
),
tmp_json["Constants"],
)
)
loaded_json.append(convert_dot_field_to_dict(tmp_json))
modified_env_vars = OmegaConf.create()
if database_config:
modified_env_vars.update(database_config)
for prefix in [ENV_VAR_RUNTIME_PREFIX, ENV_VAR_BROKER_PREFIX]:
env_vars = dict(
filter(
lambda key: key[0].startswith(prefix) and key[0] in whitelist,
dict(os.environ).items(),
)
)
for key, value in env_vars.items():
new_key = __curate_key(key, prefix)
OmegaConf.update(modified_env_vars, new_key, value)
# Fetch and update any constants passed as env var
const_env_vars = dict(
filter(
lambda key: key[0].startswith(CONSTANTS_ENV_VAR_PREFIX),
dict(os.environ).items(),
)
)
modified_constants = OmegaConf.create({"Constants": {}})
for key, value in const_env_vars.items():
new_key = key.replace(CONSTANTS_ENV_VAR_PREFIX, "", 1)
new_key = new_key.replace("_", ".", 1)
OmegaConf.update(modified_constants.Constants, new_key, value)
parameters_replacement_dict = OmegaConf.create()
for key, value in parameters_replacement:
OmegaConf.update(parameters_replacement_dict, key, value)
try:
complete_conf = OmegaConf.merge(
*loaded_json,
modified_env_vars,
modified_constants,
parameters_replacement_dict
)
bootstrap_servers = get_value_for_constant(
complete_conf,
complete_conf.DataBrokerConfiguration.publishedServices[
0
].brokerUrl,
)
OmegaConf.update(
complete_conf, BOOTSTRAP_SERVERS_KEY, bootstrap_servers
)
if not OmegaConf.select(complete_conf, NODE_COUNT_KEY):
complete_conf[NODE_COUNT_KEY] = 1
__generate_source_topic_names(complete_conf)
return complete_conf
except Exception as exception:
raise Exception(
"Error while reading input config files. Reason: '{}'".format(
exception
)
)
)
from tests.test_modules.generated import PeskySentinelUsageConf
# chdir_hydra_root(subdir="tools/configen")
##
# To re-generate the expected config run the following command from configen's root directory (tools/configen).
#
# PYTHONPATH=. configen --config-dir tests/gen-test-expected/
#
##
conf: ConfigenConf = OmegaConf.structured(
ConfigenConf(
header=
"""# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
# Generated by configen, do not edit.
# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
# fmt: off
# isort:skip_file
# flake8: noqa
"""))
MODULE_NAME = "tests.test_modules"
def test_generated_code() -> None:
classes = [
"Empty",
"UntypedArg",
"IntArg",
"Args",
"Kwargs",
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Simple core for Discord bot with dynamic extension support.
"""
from datetime import datetime, timedelta, timezone
import discord
from discord.ext import commands
from discord.utils import sleep_until
from omegaconf import OmegaConf
conf = OmegaConf.load('config.yaml').discopig
GUILD = conf.DISCORD_GUILD
TOKEN = conf.DISCORD_TOKEN
CATEGORY = conf.CATEGORY
LOBBY = conf.LOBBY
PREFIX = conf.CMD_PREFIX
bot = commands.Bot(command_prefix=PREFIX)
async def send_dm(user, txt):
channel = user.dm_channel
if channel is None:
channel = await user.create_dm()
msg = await channel.send(txt)
return msg
def main(cfg: DictConfig) -> None:
# set up mlflow experiment id
mlflow.set_tracking_uri(f"file://{to_absolute_path(cfg.path_to_mlflow)}")
experiment = mlflow.get_experiment_by_name(cfg.experiment_name)
if experiment is not None:
run_kwargs = {'experiment_id': experiment.experiment_id}
if cfg["pretrained"] is not None: # initialise with pretrained run, otherwise create a new run
run_kwargs['run_id'] = cfg["pretrained"]["run_id"]
else: # create new experiment
experiment_id = mlflow.create_experiment(cfg.experiment_name)
run_kwargs = {'experiment_id': experiment_id}
# run the training with mlflow tracking
with mlflow.start_run(**run_kwargs) as main_run:
if cfg["pretrained"] is not None:
mlflow.start_run(experiment_id=run_kwargs['experiment_id'],
nested=True)
active_run = mlflow.active_run()
run_id = active_run.info.run_id
setup_gpu(cfg.gpu_cfg)
training_cfg = OmegaConf.to_object(
cfg.training_cfg) # convert to python dictionary
scaling_cfg = to_absolute_path(cfg.scaling_cfg)
dataloader = DataLoader.DataLoader(training_cfg, scaling_cfg)
setup = dataloader.config["SetupNN"]
TauLosses.SetSFs(*setup["TauLossesSFs"])
print("loss consts:", TauLosses.Le_sf, TauLosses.Lmu_sf,
TauLosses.Ltau_sf, TauLosses.Ljet_sf)
if setup["using_new_loss"]: tf.config.run_functions_eagerly(True)
netConf_full = dataloader.get_net_config()
if dataloader.input_type == "Adversarial":
model = create_model(
netConf_full,
dataloader.model_name,
loss=setup["loss"],
use_newloss=setup["using_new_loss"],
use_AdvDataset=True,
adv_param=dataloader.adversarial_parameter,
n_adv_tau=dataloader.adv_batch_size,
adv_learning_rate=dataloader.adv_learning_rate)
else:
model = create_model(netConf_full,
dataloader.model_name,
loss=setup["loss"],
use_newloss=setup["using_new_loss"])
if cfg.pretrained is None:
print(
"Warning: no pretrained NN -> training will be started from scratch"
)
old_opt = None
else:
print("Warning: training will be started from pretrained model.")
print(
f"Model: run_id={cfg.pretrained.run_id}, experiment_id={cfg.pretrained.experiment_id}, model={cfg.pretrained.starting_model}"
)
path_to_pretrain = to_absolute_path(
f'{cfg.path_to_mlflow}/{cfg.pretrained.experiment_id}/{cfg.pretrained.run_id}/artifacts/'
)
old_model = load_model(
path_to_pretrain +
f"/model_checkpoints/{cfg.pretrained.starting_model}",
compile=False,
custom_objects=None)
for layer in model.layers:
weights_found = False
for old_layer in old_model.layers:
if layer.name == old_layer.name:
layer.set_weights(old_layer.get_weights())
weights_found = True
break
if not weights_found:
print(f"Weights for layer '{layer.name}' not found.")
old_opt = old_model.optimizer
old_vars = [var.name for var in old_model.trainable_variables]
compile_model(model, setup["optimizer_name"], setup["learning_rate"],
setup["metrics"], setup["schedule_decay"])
fit_hist = run_training(model,
dataloader,
False,
cfg.log_suffix,
setup["using_new_loss"],
old_opt=old_opt)
# log NN params
for net_type in [
'tau_net', 'comp_net', 'comp_merge_net', 'conv_2d_net',
'dense_net'
]:
mlflow.log_params({
f'{net_type}_{k}': v
for k, v in cfg.training_cfg.SetupNN[net_type].items()
})
mlflow.log_params({
f'TauLossesSFs_{i}': v
for i, v in enumerate(cfg.training_cfg.SetupNN.TauLossesSFs)
})
with open(
to_absolute_path(
f'{cfg.path_to_mlflow}/{run_kwargs["experiment_id"]}/{run_id}/artifacts/model_summary.txt'
)) as f:
for l in f:
if (s := 'Trainable params: ') in l:
mlflow.log_param('n_train_params',
int(l.split(s)[-1].replace(',', '')))
# log training related files
mlflow.log_dict(training_cfg, 'input_cfg/training_cfg.yaml')
mlflow.log_artifact(scaling_cfg, 'input_cfg')
mlflow.log_artifact(to_absolute_path("Training_CNN.py"), 'input_cfg')
mlflow.log_artifact(to_absolute_path("common.py"), 'input_cfg')
# log hydra files
mlflow.log_artifacts('.hydra', 'input_cfg/hydra')
mlflow.log_artifact('Training_CNN.log', 'input_cfg/hydra')
# log misc. info
mlflow.log_param('run_id', run_id)
mlflow.log_param('git_commit', _get_git_commit(to_absolute_path('.')))
print(
f'\nTraining has finished! Corresponding MLflow experiment name (ID): {cfg.experiment_name}({run_kwargs["experiment_id"]}), and run ID: {run_id}\n'
)
mlflow.end_run()
# Temporary workaround to kill additional subprocesses that have not exited correctly
try:
current_process = psutil.Process()
children = current_process.children(recursive=True)
for child in children:
child.kill()
except:
pass
def _get_callback(params):
target = params.pop("_target_")
callback_class = hydra.utils.get_class(target)
params = OmegaConf.to_container(params, resolve=True)
callback = callback_class(**params)
return callback
def add_data(self, langpair: str) -> None:
langpair = normalize_langpair(langpair)
data_config = self.config_dir / "data" / f"wmt14.{langpair}.yaml"
self.configs.update({"data": OmegaConf.load(data_config)})
def hparams(self) -> OrderedDict:
"""Returns hyperparameters"""
return OrderedDict(OmegaConf.to_container(self._config, resolve=True))
def add_model(self, is_base: bool = True) -> None:
model_type = "base" if is_base else "big"
model_config = self.config_dir / "model" / f"transformer-{model_type}.yaml"
self.configs.update({"model": OmegaConf.load(model_config)})
def get_architect(self) -> LocalArchitect:
"""We need to specify that the architect is launching on localhost for testing"""
arch_args = LocalArchitectArgs(hostname="http://localhost", port="3000")
args = OmegaConf.structured(MephistoConfig(architect=arch_args))
self.curr_architect = self.ArchitectClass(self.db, args, SharedTaskState(), self.task_run, self.build_dir)
return self.curr_architect
def my_app(cfg: Config) -> None:
print(OmegaConf.to_yaml(cfg))
def exp_manager(trainer: 'pytorch_lightning.Trainer', cfg: Optional[Union[DictConfig, Dict]] = None) -> Path:
"""
exp_manager is a helper function used to manage folders for experiments. It follows the pytorch lightning paradigm
of exp_dir/model_or_experiment_name/version. If the lightning trainer has a logger, exp_manager will get exp_dir,
name, and version from the logger. Otherwise it will use the exp_dir and name arguments to create the logging
directory. exp_manager also allows for explicit folder creation via explicit_log_dir.
The version will be a datetime string or an integer. Note, exp_manager does not handle versioning on slurm
multi-node runs. Datestime version can be disabled if use_datetime_version is set to False.
It optionally creates TensorBoardLogger, WandBLogger, ModelCheckpoint objects from pytorch lightning. It copies
sys.argv, and git information if available to the logging directory. It creates a log file for each process to log
their output into.
exp_manager additionally has a resume feature which can be used to continuing training from the constructed log_dir.
Args:
trainer (pytorch_lightning.Trainer): The lightning trainer.
cfg (DictConfig, dict): Can have the following keys:
- explicit_log_dir (str, Path): Can be used to override exp_dir/name/version folder creation. Defaults to
None, which will use exp_dir, name, and version to construct the logging directory.
- exp_dir (str, Path): The base directory to create the logging directory. Defaults to None, which logs to
./nemo_experiments.
- name (str): The name of the experiment. Defaults to None which turns into "default" via name = name or
"default".
- version (str): The version of the experiment. Defaults to None which uses either a datetime string or
lightning's TensorboardLogger system of using version_{int}.
- use_datetime_version (bool): Whether to use a datetime string for version. Defaults to True.
- resume_if_exists (bool): Whether this experiment is resuming from a previous run. If True, it sets
trainer.resume_from_checkpoint so that the trainer should auto-resume. exp_manager will move files
under log_dir to log_dir/run_{int}. Defaults to False.
- resume_past_end (bool): exp_manager errors out if resume_if_exists is True and a checkpoint matching
*end.ckpt indicating a previous training run fully completed. This behaviour can be disabled, in which
case the *end.ckpt will be loaded by setting resume_past_end to True. Defaults to False.
- resume_ignore_no_checkpoint (bool): exp_manager errors out if resume_if_exists is True and no checkpoint
could be found. This behaviour can be disabled, in which case exp_manager will print a message and
continue without restoring, by setting resume_ignore_no_checkpoint to True. Defaults to False.
- create_tensorboard_logger (bool): Whether to create a tensorboard logger and attach it to the pytorch
lightning trainer. Defaults to True.
- summary_writer_kwargs (dict): A dictionary of kwargs that can be passed to lightning's TensorboardLogger
class. Note that log_dir is passed by exp_manager and cannot exist in this dict. Defaults to None.
- create_wandb_logger (bool): Whether to create a Weights and Baises logger and attach it to the pytorch
lightning trainer. Defaults to False.
- wandb_logger_kwargs (dict): A dictionary of kwargs that can be passed to lightning's WandBLogger
class. Note that name and project are required parameters if create_wandb_logger is True.
Defaults to None.
- create_checkpoint_callback (bool): Whether to create a ModelCheckpoint callback and attach it to the
pytorch lightning trainer. The ModelCheckpoint saves the top 3 models with the best "val_loss", the most
recent checkpoint under *last.ckpt, and the final checkpoint after training completes under *end.ckpt.
Defaults to True.
- files_to_copy (list): A list of files to copy to the experiment logging directory. Defaults to None which
copies no files.
returns:
log_dir (Path): The final logging directory where logging files are saved. Usually the concatenation of
exp_dir, name, and version.
"""
if cfg is None:
logging.error("exp_manager did not receive a cfg argument. It will be disabled.")
return
# Ensure passed cfg is compliant with ExpManagerConfig
schema = OmegaConf.structured(ExpManagerConfig)
if isinstance(cfg, dict):
cfg = OmegaConf.create(cfg)
elif not isinstance(cfg, DictConfig):
raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig")
cfg = OmegaConf.create(OmegaConf.to_container(cfg, resolve=True))
cfg = OmegaConf.merge(schema, cfg)
error_checks(trainer, cfg) # Ensures that trainer options are compliant with NeMo and exp_manager arguments
log_dir, exp_dir, name, version = get_log_dir(
trainer=trainer,
exp_dir=cfg.exp_dir,
name=cfg.name,
version=cfg.version,
explicit_log_dir=cfg.explicit_log_dir,
use_datetime_version=cfg.use_datetime_version,
)
if cfg.resume_if_exists:
check_resume(trainer, log_dir, cfg.resume_past_end, cfg.resume_ignore_no_checkpoint)
checkpoint_name = name
# If name returned from get_log_dir is "", use cfg.name for checkpointing
if checkpoint_name is None or checkpoint_name == '':
checkpoint_name = cfg.name or "default"
cfg.name = name # Used for configure_loggers so that the log_dir is properly set even if name is ""
cfg.version = version
# Create the logging directory if it does not exist
os.makedirs(log_dir, exist_ok=True) # Cannot limit creation to global zero as all ranks write to own log file
logging.info(f'Experiments will be logged at {log_dir}')
trainer._default_root_dir = log_dir
# Handle Loggers by creating file and handle DEBUG statements
# Note: trainer.global_rank and trainer.is_global_zero are not set until trainer.fit, so have to hack around it
global_rank = trainer.node_rank * trainer.num_gpus + trainer.local_rank
log_file = log_dir / f'nemo_log_globalrank-{global_rank}_localrank-{trainer.local_rank}.txt'
logging.add_file_handler(log_file)
logging.rank = global_rank
# For some reason, LearningRateLogger requires trainer to have a logger. Safer to create logger on all ranks
# not just global rank 0.
if cfg.create_tensorboard_logger or cfg.create_wandb_logger:
configure_loggers(
trainer,
exp_dir,
cfg.name,
cfg.version,
cfg.create_tensorboard_logger,
cfg.summary_writer_kwargs,
cfg.create_wandb_logger,
cfg.wandb_logger_kwargs,
)
if is_global_rank_zero():
if cfg.create_checkpoint_callback:
configure_checkpointing(trainer, log_dir, checkpoint_name, cfg.checkpoint_callback_params)
# Move files_to_copy to folder and add git information if present
if cfg.files_to_copy:
for _file in cfg.files_to_copy:
copy(Path(_file), log_dir)
# Create files for cmd args and git info
with open(log_dir / 'cmd-args.log', 'w') as _file:
_file.write(" ".join(sys.argv))
# Try to get git hash
git_repo, git_hash = get_git_hash()
if git_repo:
with open(log_dir / 'git-info.log', 'w') as _file:
_file.write(f'commit hash: {git_hash}')
_file.write(get_git_diff())
# Add err_file logging to global_rank zero
logging.add_err_file_handler(log_dir / 'nemo_error_log.txt')
# Add lightning file logging to global_rank zero
add_filehandlers_to_pl_logger(log_dir / 'lightning_logs.txt', log_dir / 'nemo_error_log.txt')
return log_dir
CONFIG_DEFAULT: DictConfig = OmegaConf.create({
'config': {
'file_in': 'itodo.yml',
'file_out': None,
},
'searchDir': ['../modules', '../pyutil'],
'file_todo':
'todo-inline.md',
'verbose':
False,
# 'output' : {
# 'links' : True,
# 'tag-sort' : True,
# 'file-sort' : True,
# 'dir-sort' : True,
# 'format' : 'md',
# },
'read': {
'tags': {
'list': [
"CRIT", "TODO", "FIXME", "FIX", "BUG", "DEBUG", "UGLY", "HACK",
"NOTE", "IDEA", "REVIEW", "OPTIMIZE", "CONFIG", "!!!", "OLD"
],
},
# 'comments' : {
# 'require' : False,
# 'list' : [
# '//',
# '/*'
# '#',
# '<!--',
# '%',
# '\\',
# ],
# },
# 'prefix' : {
# 'PREFIX' : '@',
# 'require' : False,
# 'always_accept' : False,
# },
'SOURCE_FILES': [
'c',
'cpp',
'h',
'hpp',
'py',
'm',
'tex',
'sh',
'java',
'js',
],
'EXCLUDE': [
'inline_todo.py',
'itodo.yml',
'todo-inline.md',
],
'MAX_SEARCH_LEN':
15,
'context': {
'enabled': True,
'lines': 5, # number of lines to show before and after the tag
},
},
'write': {
'attr_sort_order': ['tag', 'file', 'lineNum'],
'item_format': 'md_det',
}
})
from omegaconf import OmegaConf
import os
cfg = OmegaConf.load(
os.path.join(os.getenv("PROJECT_DIR"), "config/config.yaml"))
from omegaconf import OmegaConf
from src.utils.utils import read_data, get_all_db_files, get_complete_table
import re
import unidecode
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn import metrics
# -
paths = OmegaConf.load("config/paths.yaml")
# # Settings
DATASET = "dataset_v09"
TEST_SET_NUM = "2"
SEED = int(TEST_SET_NUM)
# # Functions
def clean_message(message):
stop_words = set(stopwords.words("spanish"))
message = message.lower()
message = unidecode.unidecode(message)
def __init__(self, labels: List, model_cfg: Union[UniDirectionalConfig,
BiDirectionalConfig],
precision: int, optim_cfg: Union[AdamConfig, SGDConfig],
spect_cfg: SpectConfig):
super().__init__()
self.save_hyperparameters()
self.model_cfg = model_cfg
self.precision = precision
self.optim_cfg = optim_cfg
self.spect_cfg = spect_cfg
self.bidirectional = True if OmegaConf.get_type(
model_cfg) is BiDirectionalConfig else False
print('\n Bidirectional or Not \n', self.bidirectional)
print('\n OMEGA CONF \n', model_cfg)
self.labels = labels
num_classes = len(self.labels)
self.conv = MaskConv(
nn.Sequential(
nn.Conv2d(1,
32,
kernel_size=(41, 11),
stride=(2, 2),
padding=(20, 5)), nn.BatchNorm2d(32),
nn.Hardtanh(0, 20, inplace=True),
nn.Conv2d(32,
32,
kernel_size=(21, 11),
stride=(2, 1),
padding=(10, 5)), nn.BatchNorm2d(32),
nn.Hardtanh(0, 20, inplace=True)))
# Based on above convolutions and spectrogram size using conv formula (W - F + 2P)/ S+1
rnn_input_size = int(
math.floor((self.spect_cfg.sample_rate *
self.spect_cfg.window_size) / 2) + 1)
rnn_input_size = int(math.floor(rnn_input_size + 2 * 20 - 41) / 2 + 1)
rnn_input_size = int(math.floor(rnn_input_size + 2 * 10 - 21) / 2 + 1)
rnn_input_size *= 32
self.rnns = nn.Sequential(
BatchRNN(input_size=rnn_input_size,
hidden_size=self.model_cfg.hidden_size,
rnn_type=self.model_cfg.rnn_type.value,
bidirectional=self.bidirectional,
batch_norm=False),
*(BatchRNN(input_size=self.model_cfg.hidden_size,
hidden_size=self.model_cfg.hidden_size,
rnn_type=self.model_cfg.rnn_type.value,
bidirectional=self.bidirectional)
for x in range(self.model_cfg.hidden_layers - 1)))
self.lookahead = nn.Sequential(
# consider adding batch norm?
Lookahead(self.model_cfg.hidden_size,
context=self.model_cfg.lookahead_context),
nn.Hardtanh(0, 20,
inplace=True)) if not self.bidirectional else None
fully_connected = nn.Sequential(
nn.BatchNorm1d(self.model_cfg.hidden_size),
nn.Linear(self.model_cfg.hidden_size, num_classes, bias=False))
self.fc = nn.Sequential(SequenceWise(fully_connected), )
self.inference_softmax = InferenceBatchSoftmax()
self.criterion = LWLRAP(precision=self.precision)
self.lwlrap = LWLRAP(precision=self.precision)
self.loss = RMSELoss()
self.softmax = nn.Softmax(dim=-1)
# self.loss = nn.BCEWithLogitsLoss()
self.sigmoid = nn.Sigmoid()
def test_load_from_invalid() -> None:
with pytest.raises(TypeError):
OmegaConf.load(3.1415) # type: ignore
def test_decode_error(monkeypatch: Any, value: Any, exc: Any) -> None:
c = OmegaConf.create({"x": f"${{oc.decode:{value}}}"})
with exc:
c.x
