def step(task, *args): loss = task.train_cost(*args) task.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm_(task.parameters(), task.config['max_norm']) return loss
def experiment(net, data, run_config): net.cuda() net.train() scorer = run_config['Scorer'] last_epoch = 0 for _, task in run_config['tasks']: task.optimizer.zero_grad() with open("result.json", "w") as out: for epoch in range(last_epoch + 1, run_config['epochs'] + 1): costs = dict(SpeechText=Counter(), SpeechCorrText=Counter(), SpeechImage=Counter(), TextImage=Counter()) for _j, items in enumerate( zip( data['SpeechImage'].iter_train_batches(reshuffle=True), data['SpeechText'].iter_train_batches(reshuffle=True), data['SpeechCorrText'].iter_train_batches( reshuffle=True), data['TextImage'].iter_train_batches(reshuffle=True))): j = _j + 1 item = dict(SpeechImage=items[0], SpeechText=items[1], SpeechCorrText=items[2], TextImage=items[3]) for name, task in run_config['tasks']: spk = item[name]['speaker'][0] if len( set(item[name]['speaker'])) == 1 else 'MIXED' args = task.args(item[name]) args = [torch.from_numpy(x).cuda() for x in args] loss = task.cost(*args) task.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm_(task.parameters(), task.config['max_norm']) task.optimizer.step() costs[name] += Counter({'cost': loss.data.item(), 'N': 1}) print( epoch, j, j * data[name].batch_size, name, spk, "train", "".join([str(costs[name]['cost'] / costs[name]['N'])])) if j % run_config['validate_period'] == 0: loss = valid_loss(net, name, task, data) print(epoch, j, 0, name, "VALID", "valid", "".join([str(numpy.mean(loss))])) sys.stdout.flush() torch.save(net, "model.{}.pkl".format(epoch)) with testing(net): result = dict(epoch=epoch, rsa=scorer.rsa_image(net), retrieval=scorer.retrieval(net), speaker_id=scorer.speaker_id(net)) out.write(json.dumps(result)) out.write("\n") out.flush() torch.save(net, "model.pkl")
def experiment(net, data, run_config): def valid_loss(name, task): result = [] for item in data[name].iter_valid_batches(): args = task.args(item) args = [ torch.autograd.Variable(torch.from_numpy(x), volatile=True).cuda() for x in args ] result.append( [x.data.cpu().numpy() for x in task.test_cost(*args)]) return result net.cuda() net.train() scorer = Scorer( data['Image'].provider, dict(split='val', tokenize=lambda x: x['audio'], batch_size=data['Image'].batch_size)) last_epoch = 0 #tasks = [('Text', net.Text), ('Image', net.Image)] for _, task in run_config['tasks']: task.optimizer.zero_grad() with open("result.json", "w") as out: for epoch in range(last_epoch + 1, run_config['epochs'] + 1): costs = dict(Text=Counter(), Image=Counter()) for _j, items in enumerate( zip(data['Image'].iter_train_batches(reshuffle=True), data['Text'].iter_train_batches(reshuffle=True))): j = _j + 1 item = dict(Image=items[0], Text=items[1]) for name, task in run_config['tasks']: spk = item[name]['speaker'][0] if len( set(item[name]['speaker'])) == 1 else 'MIXED' args = task.args(item[name]) args = [ torch.autograd.Variable(torch.from_numpy(x)).cuda() for x in args ] loss = task.cost(*args) task.optimizer.zero_grad() loss.backward() _ = nn.utils.clip_grad_norm(task.parameters(), task.config['max_norm']) task.optimizer.step() costs[name] += Counter({'cost': loss.data[0], 'N': 1}) print( epoch, j, j * data[name].batch_size, name, spk, "train", "".join([str(costs[name]['cost'] / costs[name]['N'])])) if j % run_config['validate_period'] == 0: loss = valid_loss(name, task) print(epoch, j, 0, name, "VALID", "valid", "".join([str(numpy.mean(loss))])) # with testing(net): # result = dict(epoch=epoch, rsa=scorer.rsa_image(net), # retrieval=scorer.retrieval(net), # speaker_id=scorer.speaker_id(net)) # out.write(json.dumps(result)) # out.write("\n") # out.flush() sys.stdout.flush() torch.save(net, "model.{}.pkl".format(epoch)) # loaded = S.load("model.{}.pkl".format(epoch)) with testing(net): result = dict(epoch=epoch, rsa=scorer.rsa_image(net), retrieval=scorer.retrieval(net), speaker_id=scorer.speaker_id(net)) out.write(json.dumps(result)) out.write("\n") out.flush() torch.save(net, "model.pkl")