def main(lr, num_epochs, datadir, batch_size, nworkers, outdir):
# code for GPU support : Start
cudnn.benchmark = True
network = resnet18().cuda()
# code for GPU support : End
# path = "changed_parameters/"
# epochList = ["epoch_15.pth","epoch_20.pth","epoch_30.pth","epoch_90.pth","epoch_99.pth"]
# snapshotLoad = torch.load("changed_parameters/epoch_99.pth")
# network.load_state_dict(snapshotLoad.get("model_state"))
train_iter = make_image_loader(pt.join(datadir, 'train.msgpack'),
batch_size, nworkers, *ARGS_VAL)
val_iter = make_image_loader(pt.join(datadir, 'val.msgpack'), batch_size,
nworkers, *ARGS_VAL)
# code without GPU support
# net = resnet18()
loss = CrossEntropyLoss(output_key="net_out").cuda()
val_loss = CrossEntropyLoss(output_key="net_out").cuda()
optimizer = optim.SGD(network.parameters(),
lr=lr,
weight_decay=0.0004,
momentum=0.9)
policy = PolyPolicy(optimizer, num_epochs, power=1)
# trainer.logger.info(run_id=_run._id)
# # trainer.set_hook('train_begin', set_eval)
# with train_iter, val_iter:
# trainer.train(num_epochs, start_epoch=start_epoch)
trainer = Trainer(network, optimizer, loss, AccuracyMetric(), None, policy,
train_iter, val_iter, outdir, val_loss)
with train_iter, val_iter:
trainer.train(num_epochs)
def main():
dataset = Dataset()
fcn_model = loader.get_fcn_model_module().FCNModel()
trainer = Trainer()
trainer.train(fcn_model, dataset)
if not config.one_batch_overfit:
detector = FCNDetector(fcn_model.model)
detector.weights_path = osp.join(fcn_model.weights_dir, 'best_weights.hdf5')
estimate_quality(detector, dataset)
def main(config, model_weight=None, opt_weight=None):
def print_config(config):
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(vars(config))
print_config(config)
loader = DataLoader(
config.train,
config.valid,
(config.lang[:2], config.lang[-2:]),
batch_size=config.batch_size,
device=-1,
max_length=config.max_length
)
input_size, output_size = len(loader.src.vocab), len(loader.tgt.vocab)
model = get_model(input_size, output_size, config)
crit = get_crit(output_size, data_loader.PAD)
if model_weight:
model.load_state_dict(model_weight)
if config.gpu_id >= 0:
model.cuda(config.gpu_id)
crit.cuda(config.gpu_id)
optimizer = get_optimizer(model, config)
if opt_weight:
optimizer.load_state_dict(opt_weight)
lr_scheduler = None
if config.verbose >= 2:
print(model)
print(crit)
print(optimizer)
trainer = Trainer(IgniteEngine, config)
trainer.train(
model,
crit,
optimizer,
train_loader=loader.train_iter,
valid_loader=loader.valid_iter,
src_vocab=loader.src.vocab,
tgt_vocab=loader.tgt.vocab,
n_epochs=config.n_epochs,
lr_scheduler=lr_scheduler
)
def main(config):
if config.gpu_id < 0:
print("Device: CPU")
else:
print("Device:", torch.cuda.get_device_name(config.gpu_id))
print("Building Vocab...")
data_handler = CbowDataHandler(
file_name=config.train_fn,
window_size=config.window_size,
train_ratio=config.train_ratio,
batch_size=config.batch_size,
)
print('|train| =', len(data_handler.train_loader.dataset), '|valid| =',
len(data_handler.valid_loader.dataset))
print('|vocab_size| =', data_handler.vocab_size)
model = CBOW(
vocab_size=data_handler.vocab_size,
embd_size=config.embd_size,
window_size=config.window_size,
hidden_size=config.hidden_size,
)
# crit = nn.CrossEntropyLoss()
# optimizer = optim.SGD(model.parameters(), lr=learning_rate)
optimizer = optim.Adam(model.parameters(), lr=config.learning_rate)
crit = nn.NLLLoss()
print(model)
if config.gpu_id >= 0:
model.cuda(config.gpu_id)
crit.cuda(config.gpu_id)
trainer = Trainer(config)
trainer.train(model, crit, optimizer, data_handler.train_loader,
data_handler.valid_loader)
# Test
test_data = ['맞교환', '백색', '합판', '이메일']
ctx_idxs = [data_handler.w2i[w] for w in test_data]
ctx_var = Variable(torch.LongTensor([ctx_idxs])).to(config.gpu_id)
model.zero_grad()
y = model(ctx_var)
_, predicted = torch.max(y.data, 1)
predicted_word = data_handler.i2w[int(predicted[0])]
print('input:', test_data)
print('predicted:', predicted_word)
def main(config):
loaders = DataLoader(train_fn=config.train_fn,
batch_size=config.batch_size,
min_freq=config.min_vocab_freq,
max_vocab=config.max_vocab_size,
device=config.gpu_id)
print(
'|train| =',
len(loaders.train_loader.dataset),
'|valid| =',
len(loaders.valid_loader.dataset),
)
vocab_size = len(loaders.text.vocab)
n_classes = len(loaders.label.vocab)
print('|vocab| =', vocab_size, ' |classes| =', n_classes)
if config.rnn is False and config.cnn is False:
raise Exception(
'You need to specify an archtiecture to train. (--rnn or --cnn)')
if config.rnn:
#Declare model and loss.
model = RNNClassifier(
input_size=vocab_size,
word_vec_size=config.word_vec_size,
hidden_size=config.hidden_size,
n_classes=n_classes,
n_layers=config.n_layers,
dropout_p=config.dropout,
)
optimizer = optim.Adam(model.parameters())
crit = nn.NLLLoss()
print(model)
if config.gpu_id >= 0:
model.cuda(config.gpu_id)
crit.cuda(config.gpu_id)
rnn_trainer = Trainer(config)
rnn_model = rnn_trainer.train(model, crit, optimizer,
loaders.train_loader,
loaders.valid_loader)
torch.save(
{
'rnn': rnn_model.state_dict() if config.rnn else None,
'cnn': cnn_model.state_dict() if config.cnn else None,
'config': config,
'vocab': loaders.text.vocab,
'classes': loaders.label.vocab,
}, config.model_fn)
def main():
args = parse_args()
if args.vk_group_name is not None:
from modules import VKParser
vk_parser = VKParser(group_name=args.vk_group_name,
app_id=args.vk_appid,
login=args.vk_login,
password=args.vk_pass)
if args.vk_file is not None:
vk_parser.dump_posts(args.vk_file)
return
trainer = Trainer(n=args.model_n, lc=args.lc)
if args.vk_group_name is not None:
trainer.train(vk_parser.post_iter(args.vk_offset, args.vk_count),
re=re_ru_words_punc,
output_newlines=True)
elif args.input_dir is not None:
for entry in scandir(args.input_dir):
if entry.name.endswith('.txt') and entry.is_file():
with open(entry.path, 'r', encoding='utf8') as file:
trainer.train(file,
re=re_ru_words_punc,
output_newlines=True)
else:
trainer.train(stdin, re=re_ru_words_punc, output_newlines=True)
with open(args.model, 'wb') as file:
trainer.get_model().dump(file)
def main():
# parse arguments
args = parse_agrs()
# fix random seeds
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(args.seed)
# create tokenizer
tokenizer = Tokenizer(args)
# create data loader
train_dataloader = R2DataLoader(args,
tokenizer,
split='train',
shuffle=True)
val_dataloader = R2DataLoader(args, tokenizer, split='val', shuffle=False)
test_dataloader = R2DataLoader(args,
tokenizer,
split='test',
shuffle=False)
# build model architecture
model = R2GenModel(args, tokenizer)
# get function handles of loss and metrics
criterion = compute_loss
metrics = compute_scores
# build optimizer, learning rate scheduler
optimizer = build_optimizer(args, model)
lr_scheduler = build_lr_scheduler(args, optimizer)
# build trainer and start to train
trainer = Trainer(model, criterion, metrics, optimizer, args, lr_scheduler,
train_dataloader, val_dataloader, test_dataloader)
trainer.train()
def main(in_dataset_folder, in_noisy_dataset_folder, in_custom_vocab_file,
in_model_folder, in_config):
with open(in_config, encoding='utf-8') as config_in:
config = json.load(config_in)
train_json = load_hcn_json(os.path.join(in_dataset_folder, 'train.json'))
dev_json = load_hcn_json(os.path.join(in_dataset_folder, 'dev.json'))
# test_json = load_hcn_json(os.path.join(in_dataset_folder, 'test.json'))
test_ood_json = load_hcn_json(
os.path.join(in_noisy_dataset_folder, 'test_ood.json'))
kb = make_augmented_knowledge_base(
os.path.join(BABI_FOLDER, 'dialog-babi-task6-dstc2-kb.txt'),
os.path.join(BABI_FOLDER, 'dialog-babi-task6-dstc2-candidates.txt'))
action_templates = train_json['actions']
max_noisy_dialog_length = max(
[len(dialog['turns']) for dialog in test_ood_json['dialogs']])
config['max_input_length'] = max_noisy_dialog_length
et = EntityTracker(kb)
post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(
test_ood_json)
if in_custom_vocab_file is not None:
with open(in_custom_vocab_file) as vocab_in:
rev_vocab = [line.rstrip() for line in vocab_in]
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
else:
utterances_tokenized = []
for dialog in train_json['dialogs']:
for utterance in dialog['turns']:
utterances_tokenized.append(utterance['input'].split())
vocab, rev_vocab = make_vocabulary(
utterances_tokenized,
config['max_vocabulary_size'],
special_tokens=[PAD, START, UNK, EOS] + list(kb.keys()))
ctx_features = []
for dialog in train_json['dialogs']:
for utterance in dialog['turns']:
if 'context_features' in utterance:
ctx_features.append(utterance['context_features'])
ctx_features_vocab, ctx_features_rev_vocab = make_vocabulary(
ctx_features, config['max_vocabulary_size'], special_tokens=[])
config['vocabulary_size'] = len(vocab)
print('Training with config: {}'.format(json.dumps(config)))
data_preparation_function = getattr(utils.preprocessing,
config['data_preparation_function'])
data_train = data_preparation_function(train_json, vocab,
ctx_features_vocab, et, **config)
data_dev = data_preparation_function(dev_json, vocab, ctx_features_vocab,
et, **config)
# data_test = data_preparation_function(test_json, vocab, ctx_features_vocab, et, **config)
data_test_ood = data_preparation_function(test_ood_json, vocab,
ctx_features_vocab, et, **config)
dropout_turn_generation_function = getattr(
utils.preprocessing, config['dropout_turn_generation_function'])
random_input = dropout_turn_generation_function(
10000, 3, config['max_sequence_length'], train_json, vocab,
config['turn_word_dropout_prob'])
save_model(rev_vocab, config, kb, action_templates, in_model_folder)
net = getattr(modules, config['model_name'])(vocab, config,
len(ctx_features_vocab),
len(action_templates))
trainer = Trainer(data_train, data_dev, data_test_ood, action_templates,
random_input, post_ood_turns_noisy, config, net,
in_model_folder)
trainer.train()