def __init__(self, name=None):
# Initialize the inferface.
super().__init__()
# Retrieve dictionary of parameters (keys, values) passed to init.
self._init_params = self.__extract_init_params()
# Get object UUID.
self._uuid = str(uuid.uuid4())
# Register module and store the generated name.
self._name = self._app_state.register_module(self, name)
# Set "both" as default operation mode.
self._operation_mode = OperationMode.both
# Get default factory.
self._factory = NeuralModuleFactory.get_default_factory()
# Set module properties from factory else use defaults
self._placement = self._factory.placement
# If one needs to change that should override it manually.
# Optimization level.
self._opt_level = self._factory.optim_level
def __init__(
self, *,
pretrained_model_name=None,
create_port_args=None,
factory=None,
placement=None,
**kwargs
):
self._pretrained_model_name = pretrained_model_name
self._local_parameters = self.update_local_params()
if create_port_args is None:
create_port_args = {}
self._input_ports, self._output_ports = self.create_ports(
**create_port_args)
default_factory = NeuralModuleFactory.get_default_factory()
if (factory is None) and (default_factory is not None):
factory = default_factory
# Set module properties from factory else use defaults
self._placement = factory.placement if factory is not None\
else DeviceType.GPU
self._opt_level = factory.optim_level if factory is not None\
else Optimization.mxprO0
self._logger = factory.logger if factory is not None\
else logging
# Update module properties using overrides if overrides exist
if placement is not None:
self._placement = placement
self._factory = factory
self._uuid = str(uuid.uuid4())
def neural_factory(request):
""" Fixture creating a Neural Factory object parametrized by the command line --cpu argument """
# Get flag.
if request.config.getoption("--cpu"):
device = DeviceType.CPU
else:
device = DeviceType.GPU
# Initialize the default Neural Factory - on GPU.
request.cls.nf = NeuralModuleFactory(placement=device)
# Print standard header.
logging.info("Using {} during testing".format(request.cls.nf.placement))
def __init__(self):
# Get default factory.
self._factory = NeuralModuleFactory.get_default_factory()
# Set module properties from factory else use defaults
self._placement = self._factory.placement
# If one needs to change that should override it manually.
# Optimization level.
self._opt_level = self._factory.optim_level
# Get object UUID.
self._uuid = str(uuid.uuid4())
# Retrieve dictionary of parameters (keys, values) passed to init.
self._init_params = self.__extract_init_params()
def __init__(self, model_config, encoder_weights, decoder_weights):
super().__init__()
with open(model_config, 'r') as config:
model_args = YAML.load(config)
_ = NeuralModuleFactory(placement=DeviceType.CPU)
encoder_params = model_args['init_params']['encoder_params']['init_params']
self.encoder = JasperEncoder(**encoder_params)
self.encoder.load_state_dict(torch.load(encoder_weights, map_location='cpu'))
decoder_params = model_args['init_params']['decoder_params']['init_params']
self.decoder = JasperDecoderForCTC(**decoder_params)
self.decoder.load_state_dict(torch.load(decoder_weights, map_location='cpu'))
self.encoder._prepare_for_deployment()
self.decoder._prepare_for_deployment()
convert_to_2d(self.encoder)
convert_to_2d(self.decoder)
from nemo.collections.cv.modules.data_layers import CIFAR100DataLayer
from nemo.collections.cv.modules.losses import NLLLoss
from nemo.collections.cv.modules.non_trainables import NonLinearity, ReshapeTensor
from nemo.collections.cv.modules.trainables import FeedForwardNetwork, ImageEncoder
from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback
from nemo.utils import logging
if __name__ == "__main__":
# Create the default parser.
parser = argparse.ArgumentParser(parents=[nm_argparse.NemoArgParser()],
conflict_handler='resolve')
# Parse the arguments
args = parser.parse_args()
# Instantiate Neural Factory.
nf = NeuralModuleFactory(local_rank=args.local_rank,
placement=DeviceType.CPU)
# Data layer - upscale the CIFAR100 images to ImageNet resolution.
cifar100_dl = CIFAR100DataLayer(height=224, width=224, train=True)
# The "model".
image_encoder = ImageEncoder(model_type="vgg16",
return_feature_maps=True,
pretrained=True,
name="vgg16")
reshaper = ReshapeTensor(input_sizes=[-1, 7, 7, 512],
output_sizes=[-1, 25088])
ffn = FeedForwardNetwork(input_size=25088,
output_size=100,
hidden_sizes=[1000, 1000],
dropout_rate=0.1)
nl = NonLinearity(type="logsoftmax", sizes=[-1, 100])
# "Deserialize" dim.
deserialized_params["dim"] = init_params["dim"]
# Custom "deserialization" of the status.
if init_params["status"] == 0:
deserialized_params["status"] = Status.success
else:
deserialized_params["status"] = Status.error
# Return deserialized parameters.
return deserialized_params
# Run on CPU.
nf = NeuralModuleFactory(placement=DeviceType.CPU)
# Instantitate RealFunctionDataLayer defaults to f=torch.sin, sampling from x=[-1, 1]
dl = RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=32)
# Instantiate a simple feed-forward, single layer neural network.
fx = CustomTaylorNet(dim=4, status=Status.error)
# Instantitate loss.
mse_loss = MSELoss()
# Export the model configuration.
fx.export_to_config("/tmp/custom_taylor_net.yml")
# Create a second instance, using the parameters loaded from the previously created configuration.
# Please note that we are calling the overriden method from the CustomTaylorNet class.
else:
schema_config = {
"MAX_NUM_CAT_SLOT": 6,
"MAX_NUM_NONCAT_SLOT": 12,
"MAX_NUM_VALUE_PER_CAT_SLOT": 12,
"MAX_NUM_INTENT": 4,
}
if not os.path.exists(args.data_dir):
raise ValueError(f'Data not found at {args.data_dir}')
nf = NeuralModuleFactory(
backend=Backend.PyTorch,
local_rank=args.local_rank,
optimization_level=args.amp_opt_level,
log_dir=args.work_dir,
create_tb_writer=True,
checkpoint_dir=args.checkpoint_dir,
files_to_copy=[__file__],
add_time_to_log_dir=not args.no_time_to_log_dir,
)
pretrained_bert_model = nemo_nlp.nm.trainables.get_pretrained_lm_model(
pretrained_model_name=args.pretrained_model_name,
config=args.bert_config,
vocab=args.vocab_file,
checkpoint=args.bert_checkpoint,
)
schema_config["EMBEDDING_DIMENSION"] = pretrained_bert_model.hidden_size
schema_config["MAX_SEQ_LENGTH"] = args.max_seq_length
if args.command == 'train':
train_examples = torch.load(args.train_file_preprocessed)
if args.eval_file_preprocessed:
eval_examples, eval_special_tokens = torch.load(
args.eval_file_preprocessed)
test_examples, test_special_tokens = torch.load(
args.test_file_preprocessed)
label_map = utils.read_label_map(args.label_map_file)
num_tags = len(label_map)
nf = NeuralModuleFactory(
local_rank=args.local_rank,
optimization_level=args.amp_opt_level,
log_dir=args.work_dir,
create_tb_writer=True,
files_to_copy=[__file__],
add_time_to_log_dir=False,
)
encoder = nemo_nlp.nm.trainables.huggingface.get_huggingface_lm_model(
pretrained_model_name=args.pretrained_model_name)
tokenizer = nemo_nlp.data.NemoBertTokenizer(
pretrained_model=args.pretrained_model_name)
hidden_size = encoder.hidden_size
# Size of the output vocabulary which contains the tags + begin and end
# tokens used by the Transformer decoder.
output_vocab_size = num_tags
parser.add_argument("--work_dir", default='outputs', type=str, help='Path to where to store logs')
args = parser.parse_args()
# Get the absolute path.
abs_data_dir = expanduser(args.data_dir)
# Check if data dir exists
if not exists(abs_data_dir):
raise ValueError(f"Data folder `{abs_data_dir}` not found")
if args.show_all_output:
logging.setLevel('DEBUG')
# Initialize NF.
nf = NeuralModuleFactory(placement=DeviceType.CPU, local_rank=None, log_dir=args.work_dir, checkpoint_dir=None)
# Initialize the modules.
# List of the domains to be considered.
domains = {"attraction": 0, "restaurant": 1, "train": 2, "hotel": 3, "taxi": 5}
# Create DataDescriptor that contains information about domains, slots, and associated vocabulary
data_desc = MultiWOZDataDesc(abs_data_dir, domains)
vocab_size = len(data_desc.vocab)
# Encoder changing the "user utterance" into format accepted by TRADE encoderRNN.
user_utterance_encoder = UserUtteranceEncoder(data_desc=data_desc)
# TRADE modules.
trade_encoder = EncoderRNN(
def sentence_classification(args):
# TODO: construct name of experiment based on args
"""
name = construct_name(
args.exp_name,
args.lr,
args.batch_size,
args.num_epochs,
args.weight_decay,
args.optimizer)
work_dir = name
if args.work_dir:
work_dir = os.path.join(args.work_dir, name)
"""
# Instantiate neural modules
nf = NeuralModuleFactory(
backend=nemo.core.Backend.PyTorch,
local_rank=args.local_rank,
optimization_level=args.amp_opt_level,
log_dir=args.work_dir,
create_tb_writer=True,
files_to_copy=[__file__],
add_time_to_log_dir=True)
# Pre-trained BERT
tokenizer = BertTokenizer.from_pretrained(args.pretrained_bert_model)
if args.bert_checkpoint is None:
bert = nemo_nlp.BERT(pretrained_model_name=args.pretrained_bert_model)
# save bert config for inference after fine-tuning
bert_config = bert.config.to_dict()
with open(args.work_dir + '/' + args.pretrained_bert_model + '_config.json', 'w+') as json_file:
json.dump(bert_config, json_file)
else:
if args.bert_config is not None:
with open(args.bert_config) as json_file:
bert_config = json.load(json_file)
bert = nemo_nlp.BERT(**bert_config)
bert.restore_from(args.bert_checkpoint)
# MLP
bert_hidden_size = bert.local_parameters['hidden_size']
mlp = nemo_nlp.SequenceClassifier(
hidden_size=bert_hidden_size,
num_classes=args.num_classes,
num_layers=args.num_layers,
log_softmax=False,
dropout=args.dropout)
# TODO: save mlp/all model configs (bake in to Neural Module?)
if args.mlp_checkpoint:
mlp.restore_from(args.mlp_checkpoint)
# Loss function for classification
loss_fn = CrossEntropyLoss()
# Data layers, pipelines, and callbacks
callbacks = [] # callbacks depend on files present
if args.train_file:
if args.preproc:
train_data_layer = preproc_data_layer.PreprocBertSentenceClassificationDataLayer(
input_file=args.train_file,
shuffle=True,
num_samples=args.num_samples, # lower for dev, -1 for all dataset
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
else:
train_data_layer = nemo_nlp.BertSentenceClassificationDataLayer(
input_file=args.train_file,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
shuffle=True,
num_samples=args.num_samples, # lower for dev, -1 for all dataset
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
train_logits, train_loss, steps_per_epoch, train_labels = create_pipeline(
nf,
train_data_layer,
bert,
mlp,
loss_fn)
train_callback = nemo.core.SimpleLossLoggerCallback(
tensors=[train_loss, train_logits],
print_func=lambda x: nf.logger.info(f'Train loss: {str(np.round(x[0].item(), 3))}'),
tb_writer=nf.tb_writer,
get_tb_values=lambda x: [["train_loss", x[0]]],
step_freq=steps_per_epoch)
callbacks.append(train_callback)
if args.num_checkpoints != 0:
ckpt_callback = nemo.core.CheckpointCallback(
folder=nf.checkpoint_dir,
epoch_freq=args.save_epoch_freq,
step_freq=args.save_step_freq,
checkpoints_to_keep=args.num_checkpoints)
callbacks.append(ckpt_callback)
if args.eval_file:
if args.preproc:
eval_data_layer = preproc_data_layer.PreprocBertSentenceClassificationDataLayer(
input_file=args.eval_file,
shuffle=False,
num_samples=args.num_samples,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
else:
eval_data_layer = nemo_nlp.BertSentenceClassificationDataLayer(
input_file=args.eval_file,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
shuffle=False,
num_samples=args.num_samples,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
eval_logits, eval_loss, _, eval_labels = create_pipeline(
nf,
eval_data_layer,
bert,
mlp,
loss_fn)
eval_callback = nemo.core.EvaluatorCallback(
eval_tensors=[eval_logits, eval_labels],
user_iter_callback=lambda x, y: eval_iter_callback(
x, y, eval_data_layer),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x, f'{nf.work_dir}/graphs'),
tb_writer=nf.tb_writer,
eval_step=steps_per_epoch)
callbacks.append(eval_callback)
if args.inference_file:
if args.preproc:
inference_data_layer = preproc_data_layer.PreprocBertSentenceClassificationDataLayer(
input_file=args.inference_file,
shuffle=False,
num_samples=args.num_samples,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
else:
inference_data_layer = nemo_nlp.BertSentenceClassificationDataLayer(
input_file=args.inference_file,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
shuffle=False,
num_samples=args.num_samples,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
# TODO: Finish inference
inference_callback = None
# Training, eval and inference
if args.train_file:
lr_policy_fn = get_lr_policy(
args.lr_policy,
total_steps=args.num_epochs * steps_per_epoch,
warmup_ratio=args.lr_warmup_proportion)
nf.train(
tensors_to_optimize=[train_loss],
callbacks=callbacks,
lr_policy=lr_policy_fn,
optimizer=args.optimizer_kind,
optimization_params={'num_epochs': args.num_epochs, 'lr': args.lr})
import nemo.utils.argparse as nm_argparse
from nemo.collections.cv.modules.data_layers import MNISTDataLayer
from nemo.collections.cv.modules.losses import NLLLoss
from nemo.collections.cv.modules.trainables import LeNet5
from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode
from nemo.utils import logging
if __name__ == "__main__":
# Create the default parser.
parser = argparse.ArgumentParser(parents=[nm_argparse.NemoArgParser()],
conflict_handler='resolve')
# Parse the arguments
args = parser.parse_args()
# Instantiate Neural Factory.
nf = NeuralModuleFactory(local_rank=args.local_rank)
# Data layers for training and validation.
dl = MNISTDataLayer(height=32, width=32, train=True)
dl_e = MNISTDataLayer(height=32, width=32, train=False)
# The "model".
lenet5 = LeNet5()
# Loss.
nll_loss = NLLLoss()
# Create a training graph.
with NeuralGraph(operation_mode=OperationMode.training) as training_graph:
_, x, y, _ = dl()
p = lenet5(images=x)
loss = nll_loss(predictions=p, targets=y)
