def input_interaction(modeloptions, embfile, emb_dim, modelcheckpoint,
checkpointfolder, device):
if embfile is not None:
print("Embedding file given! Load embeddings...")
word2idx, idx2word, embeddings = load_embeddings(embfile, emb_dim)
else:
print("Embedding file not given! Load embedding dicts for checkpoint "
"folder...")
embeddings = None
with open(os.path.join(checkpointfolder, 'word2idx.pickle'), 'rb') as \
handle:
word2idx = pickle.load(handle)
with open(os.path.join(checkpointfolder, 'idx2word.pickle'), 'rb') as \
handle:
idx2word = pickle.load(handle)
vocab_size = len(word2idx)
pad_index = word2idx[HRED_SPECIAL_TOKENS.PAD.value]
sos_index = word2idx[HRED_SPECIAL_TOKENS.SOU.value]
eos_index = word2idx[HRED_SPECIAL_TOKENS.EOU.value]
print("Loaded Embeddings...")
# --- load model using loaded embeddings ---
model = create_model(modeloptions, embeddings, emb_dim, vocab_size,
sos_index, device)
model = from_checkpoint(modelcheckpoint, model, map_location='cpu')
model = model.to(device)
print("Loaded Model...")
# --- create searcher for encoding user's input and for providing an
# answer ---
searcher = create_searcher(model, device)
searcher = searcher.to(device)
searcher.eval()
print("Interacting...")
evaluate_input(searcher, word2idx, idx2word, device)
def load_trainer_factory(options,
emb_dim,
vocab_size,
embeddings,
pad_index,
sos_index,
loadfrom,
checkpoint_dir=None,
device=DEVICE):
model = HREDSeq2Seq(options, emb_dim, vocab_size, embeddings, embeddings,
sos_index, device)
import ipdb
ipdb.set_trace()
model = from_checkpoint(loadfrom, model, map_location='cpu')
model = model.to(device)
numparams = sum([p.numel() for p in model.parameters() if p.requires_grad])
print('Trainable Parameters: {}'.format(numparams))
optimizer = Adam([p for p in model.parameters() if p.requires_grad],
lr=1e-3,
weight_decay=1e-6)
criterion = SequenceCrossEntropyLoss(pad_index)
perplexity = Perplexity(pad_index)
metrics = {'loss': Loss(criterion), 'ppl': Loss(perplexity)}
trainer = HREDTrainer(model,
optimizer,
checkpoint_dir=checkpoint_dir,
metrics=metrics,
non_blocking=True,
retain_graph=False,
patience=5,
device=device,
loss_fn=criterion)
return trainer
def input_interaction(modeloptions, embfile, emb_dim, checkpointfile,
outputfile, device):
word2idx, idx2word, embeddings = load_embeddings(embfile, emb_dim)
vocab_size = len(word2idx)
pad_index = word2idx[HRED_SPECIAL_TOKENS.PAD.value]
sos_index = word2idx[HRED_SPECIAL_TOKENS.SOU.value]
eos_index = word2idx[HRED_SPECIAL_TOKENS.EOU.value]
print("Loaded Embeddings...")
# --- load model using loaded embeddings ---
model = create_model(modeloptions, embeddings, emb_dim, vocab_size,
sos_index, device)
model = from_checkpoint(checkpointfile, model, map_location='cpu')
model = model.to(device)
print("Loaded Model...")
# --- create searcher for encoding user's input and for providing an
# answer ---
searcher = create_searcher(model, device)
searcher = searcher.to(device)
searcher.eval()
print("Interacting...")
evaluate_input(searcher, word2idx, idx2word, device)
def load_trainer_factory(options,
emb_dim,
vocab_size,
embeddings,
pad_index,
sos_index,
loadfrom,
checkpoint_dir=None,
device=DEVICE):
model = HREDSeq2Seq(options, emb_dim, vocab_size, embeddings, embeddings,
sos_index, device)
import ipdb
ipdb.set_trace()
model = from_checkpoint(loadfrom, model, map_location='cpu')
model = model.to(device)
numparams = sum([p.numel() for p in model.parameters() if p.requires_grad])
print('Trainable Parameters: {}'.format(numparams))
optimizer = Adam([p for p in model.parameters() if p.requires_grad],
lr=1e-3,
weight_decay=1e-6)
criterion = SequenceCrossEntropyLoss(pad_index)
perplexity = Perplexity(pad_index)
metrics = [perplexity]
trainer = HREDIterationsTrainer(model,
optimizer,
criterion,
metrics,
checkpoint_dir=checkpoint_dir,
save_every=options.save_every,
print_every=options.print_every,
device=device)
return trainer
def __init__(self: TrainerType,
model: nn.Module,
optimizer: Optimizer,
checkpoint_dir: str = '../../checkpoints',
experiment_name: str = 'experiment',
model_checkpoint: Optional[str] = None,
optimizer_checkpoint: Optional[str] = None,
metrics: types.GenericDict = None,
patience: int = 10,
validate_every: int = 1,
accumulation_steps: int = 1,
loss_fn: Union[_Loss, DataParallelCriterion] = None,
non_blocking: bool = True,
retain_graph: bool = False,
dtype: torch.dtype = torch.float,
device: str = 'cpu',
parallel: bool = False) -> None:
self.dtype = dtype
self.retain_graph = retain_graph
self.non_blocking = non_blocking
self.device = device
self.loss_fn = loss_fn
self.validate_every = validate_every
self.patience = patience
self.accumulation_steps = accumulation_steps
self.checkpoint_dir = checkpoint_dir
model_checkpoint = self._check_checkpoint(model_checkpoint)
optimizer_checkpoint = self._check_checkpoint(optimizer_checkpoint)
self.model = cast(
nn.Module,
from_checkpoint(model_checkpoint,
model,
map_location=torch.device('cpu')))
self.model = self.model.type(dtype).to(device)
self.optimizer = from_checkpoint(optimizer_checkpoint, optimizer)
self.parallel = parallel
if parallel:
if device == 'cpu':
raise ValueError("parallel can be used only with cuda device")
self.model = DataParallelModel(self.model).to(device)
self.loss_fn = DataParallelCriterion(self.loss_fn) # type: ignore
if metrics is None:
metrics = {}
if 'loss' not in metrics:
if self.parallel:
metrics['loss'] = Loss(
lambda x, y: self.loss_fn(x, y).mean()) # type: ignore
else:
metrics['loss'] = Loss(self.loss_fn)
self.trainer = Engine(self.train_step)
self.train_evaluator = Engine(self.eval_step)
self.valid_evaluator = Engine(self.eval_step)
for name, metric in metrics.items():
metric.attach(self.train_evaluator, name)
metric.attach(self.valid_evaluator, name)
self.pbar = ProgressBar()
self.val_pbar = ProgressBar(desc='Validation')
if checkpoint_dir is not None:
self.checkpoint = CheckpointHandler(checkpoint_dir,
experiment_name,
score_name='validation_loss',
score_function=self._score_fn,
n_saved=2,
require_empty=False,
save_as_state_dict=True)
self.early_stop = EarlyStopping(patience, self._score_fn, self.trainer)
self.val_handler = EvaluationHandler(pbar=self.pbar,
validate_every=1,
early_stopping=self.early_stop)
self.attach()
log.info(
f'Trainer configured to run {experiment_name}\n'
f'\tpretrained model: {model_checkpoint} {optimizer_checkpoint}\n'
f'\tcheckpoint directory: {checkpoint_dir}\n'
f'\tpatience: {patience}\n'
f'\taccumulation steps: {accumulation_steps}\n'
f'\tnon blocking: {non_blocking}\n'
f'\tretain graph: {retain_graph}\n'
f'\tdevice: {device}\n'
f'\tmodel dtype: {dtype}\n'
f'\tparallel: {parallel}')
def __init__(self: TrainerType,
model: nn.Module,
optimizer: Optimizer,
checkpoint_dir: str = '../../checkpoints',
experiment_name: str = 'experiment',
model_checkpoint: Optional[str] = None,
optimizer_checkpoint: Optional[str] = None,
metrics: types.GenericDict = None,
patience: int = 10,
validate_every: int = 1,
accumulation_steps: int = 1,
loss_fn: _Loss = nn.CrossEntropyLoss(),
non_blocking: bool = True,
dtype: torch.dtype = torch.float,
device: str = 'cpu') -> None:
self.dtype = dtype
self.non_blocking = non_blocking
self.device = device
self.loss_fn = loss_fn
self.validate_every = validate_every
self.patience = patience
self.accumulation_steps = accumulation_steps
self.checkpoint_dir = checkpoint_dir
model_checkpoint = self._check_checkpoint(model_checkpoint)
optimizer_checkpoint = self._check_checkpoint(optimizer_checkpoint)
if metrics is None:
metrics = {}
if 'loss' not in metrics:
metrics['loss'] = Loss(self.loss_fn)
self.model = cast(
nn.Module,
from_checkpoint(model_checkpoint,
model,
map_location=torch.device('cpu')))
self.model = self.model.type(dtype).to(device)
self.optimizer = from_checkpoint(optimizer_checkpoint, optimizer)
self.trainer = Engine(self.train_step)
self.train_evaluator = Engine(self.eval_step)
self.valid_evaluator = Engine(self.eval_step)
for name, metric in metrics.items():
metric.attach(self.train_evaluator, name)
metric.attach(self.valid_evaluator, name)
self.pbar = ProgressBar()
self.val_pbar = ProgressBar(desc='Validation')
self.checkpoint = CheckpointHandler(checkpoint_dir,
experiment_name,
score_name='validation_loss',
score_function=self._score_fn,
n_saved=2,
require_empty=False,
save_as_state_dict=True)
self.early_stop = EarlyStopping(patience, self._score_fn, self.trainer)
self.val_handler = EvaluationHandler(pbar=self.pbar,
validate_every=1,
early_stopping=self.early_stop)
self.attach()