def lstm_model(train, val, test):
# vocab = Vocab()
# vocab.from_dataset(train+val+test)
vocab = generate_vocab()
hparams = {
"vocab": vocab,
"embedding_dim": 128,
"hidden_dim": 128,
"n_epochs": 5,
"batch_size": 32,
"lr": 0.001,
"num_layers": 1,
"dropout": 0,
}
model = LSTMModel(hparams)
model.fit(train, val)
print("lstm model perplexity:", evaluate(model, test))
progression = model.generate(10)
print("generated progression:", len(progression), progression)
# convert_to_chord_name(progression))
print("generated progression perplexity:", evaluate(model, [progression]))
print("accuracy:", accuracy(model, test))
print("vocab size:", len(vocab))
def run(trainData, valData, foldno):
question_feats, user_feats, user_keys, user_keys_map = loadData()
nbmodels = getModels(trainData, question_feats)
predictions = getPredictions(valData, nbmodels, question_feats, user_feats,
user_keys, user_keys_map)
fname = '../localvalidation/content_ques_topics' + str(foldno) + '.csv'
with open(fname, 'w') as f1:
f1.write('qid,uid,label\n')
for i in range(0, len(predictions)):
f1.write(valData[i][0] + ',' + valData[i][1] + ',' +
str(predictions[i]) + '\n')
return evaluate.ndcg(fname)
print evaluate.accuracy(fname)
print evaluate.logloss(fname)
return
def test(net, test_dataloader, device, criterion, epoch, epoch_test_loss, f_test_acc):
running_loss = 0
net.eval()
for i, data in enumerate(test_dataloader, 0):
# get the inputs
test_batch, label_batch = data
test_batch = test_batch.to(device)
label_batch = label_batch.to(device)
test_batch, label_batch = Variable(test_batch), Variable(label_batch)
m, h, w = test_batch.shape
test_batch = test_batch.view(m, 1, h, w)
label_batch = label_batch.type(torch.long)
# forward
outputs = net(test_batch)
loss = criterion(outputs, label_batch)
running_loss += loss.item()
# print statistics
if i % 100 == 0: # print every 100 iterations
test_acc = accuracy(outputs, label_batch)
for j in range(6):
f_test_acc[j].append(test_acc[j])
running_loss = running_loss/len(test_dataloader)
epoch_test_loss.append(running_loss)
def train(net, train_dataloader, device, optimizer, criterion, epoch, epoch_train_loss, f_train_acc):
running_loss = 0
for i, data in enumerate(train_dataloader, 0):
# get the inputs
train_batch, label_batch = data
train_batch = train_batch.to(device)
label_batch = label_batch.to(device)
train_batch, label_batch = Variable(train_batch), Variable(label_batch)
m, h ,w = train_batch.shape
train_batch = train_batch.view(m,1,h,w)
label_batch = label_batch.type(torch.long)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(train_batch)
loss = criterion(outputs, label_batch)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
if i % 100 == 0: # print every 100 iterations.
train_acc = accuracy(outputs, label_batch)
for j in range(6):
f_train_acc[j].append(train_acc[j])
running_loss = running_loss/len(train_dataloader)
epoch_train_loss.append(running_loss)
def train(config, train_loader, model, criterion, optimizer, epoch,
output_dir, tb_log_dir, writer_dict):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, (input, target, target_weight, meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
# compute output
output = model(input)
target = target.cuda(non_blocking=True)
target_weight = target_weight.cuda(non_blocking=True)
loss = criterion(output, target, target_weight)
# compute gradient and do update step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
_, avg_acc, cnt, pred = accuracy(output.detach().cpu().numpy(),
target.detach().cpu().numpy())
acc.update(avg_acc, cnt)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % config.PRINT_FREQ == 0:
msg = 'Epoch: [{0}][{1}/{2}]\t' \
'Time {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\t' \
'Speed {speed:.1f} samples/s\t' \
'Data {data_time.val:.3f}s ({data_time.avg:.3f}s)\t' \
'Loss {loss.val:.5f} ({loss.avg:.5f})\t' \
'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch, i, len(train_loader), batch_time=batch_time,
speed=input.size(0)/batch_time.val,
data_time=data_time, loss=losses, acc=acc)
logger.info(msg)
writer = writer_dict['writer']
global_steps = writer_dict['train_global_steps']
writer.add_scalar('train_loss', losses.val, global_steps)
writer.add_scalar('train_acc', acc.val, global_steps)
writer_dict['train_global_steps'] = global_steps + 1
prefix = '{}_{}'.format(os.path.join(output_dir, 'train'), i)
save_debug_images(config, input, meta, target, pred*4, output,
prefix)
def train_step(network, data_loader, loss_calculator, optimizer, device, epoch, print_freq=100):
network.train()
# set benchmark flag to faster runtime
torch.backends.cudnn.benchmark = True
starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
train_time = AverageMeter()
data_time = AverageMeter()
loss_time = AverageMeter()
forward_time = AverageMeter()
backward_time = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
tic = time.time()
s_tic = time.time()
for iteration, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - tic)
inputs, targets = inputs.to(device), targets.to(device)
starter.record()
outputs = network(inputs)
ender.record()
# WAIT FOR GPU SYNC
torch.cuda.synchronize()
forward_time.update(starter.elapsed_time(ender)/1000)
tic = time.time()
loss = loss_calculator.calc_loss(outputs, targets)
loss_time.update(time.time() - tic)
tic = time.time()
optimizer.zero_grad()
loss.backward()
optimizer.step()
backward_time.update(time.time() - tic)
train_time.update(time.time() - s_tic)
prec1, prec5 = accuracy(outputs.data, targets, topk=(1,5))
top1.update(prec1.item(), inputs.size(0))
top5.update(prec5.item(), inputs.size(0))
if iteration % print_freq == 0:
logs_ = '%s: '%time.ctime()
logs_ += 'Epoch [%d], '%epoch
logs_ += 'Iteration [%d/%d/], '%(iteration, len(data_loader))
logs_ += 'Data(s): %2.3f, Loss(s): %2.3f, '%(data_time.avg, loss_time.avg)
logs_ += 'Forward(s): %2.3f, Backward(s): %2.3f, '%(forward_time.avg, backward_time.avg)
logs_ += 'Train(s): %2.3f, '%(train_time.avg)
logs_ += 'Top1: %2.3f, Top5: %2.4f, '%(top1.avg, top5.avg)
logs_ += 'Loss: %2.3f'%loss_calculator.get_loss_log()
print(logs_)
tic = time.time()
s_tic = time.time()
return None
def __get_evaluation_score(self, model_result):
if self.scoring == MODEL_SCORE_EVALUATION_ACCURACY:
return accuracy(model_result)
elif self.scoring == MODEL_SCORE_EVALUATION_F1_MACRO:
return macro_f1(model_result)
elif self.scoring == MODEL_SCORE_EVALUATION_F1_WEIGHTED:
return weighted_f1(model_result)
else:
raise ValueError(
'Unsupported scoring strategy for model evaluation!')
def execute(dataset, hmm_file, tag_file):
sys.stderr.write("loading learnt parameters...\n")
hmm, tagset = hmm_utils.get_param_tagset(hmm_file, tag_file)
sys.stderr.write("reading dev data...\n")
test_sentences, test_tags = data_reader.read_tagging_data(dataset)
test_sentences = replace_test(test_sentences, hmm, tagset)
i = 0
converges = 0
avg_iterations = 0
start_time = time.time()
for sentence in test_sentences:
if len(sentence) > 0 :#True: #len(tree) < 100:
i += 1
truetags = test_tags[test_sentences.index(sentence)]
sys.stderr.write('\n' + str(i)+ '\n')
tagprint(test_sents_not_rare[test_sentences.index(sentence)])
best_tags, num_iterations, tags1, tags2 = dd_tagger_fst.run(sentence, tagset, hmm)
tagprint(best_tags)
tagprint(tags2)
if num_iterations != -1:
sys.stderr.write("fst tagger accuracy = " + str(evaluate.accuracy(truetags, tags2)) + "\n")
sys.stderr.write("best tags accuracy = " + str(evaluate.accuracy(truetags, best_tags)) + "\n")
sys.stderr.write("converges in " + str(num_iterations) + " iterations \n")
converges += 1
avg_iterations += num_iterations
else:
print
sys.stderr.write("does not converge :(\n")
tagprint(truetags)
print
#if i==100:
#break
sys.stderr.write("\n" + str(avg_iterations/converges) + " iterations on average\n")
sys.stderr.write(str(converges*100/i) + " % convergence\n")
sys.stderr.write("time_taken = "+ str(time.time() - start_time) + "\n")
def evaluation(param, test_file='validation'):
if param['evaluation_metric'] == 'auc':
acc = e.accuracy(param, test_file)
print('Iteration %s, Accuracy: %s' % (param['iteration'], acc))
elif param['evaluation_metric'] == 'mean_rank':
hits, top_hits, mean_rank = e.test(param,
test_file='validation',
is_filetered=False)
print('Iteration %s, mean_rank: %s, top_hits: %s, hits %s' %
(param['iteration'], mean_rank, top_hits, hits))
def model(self):
"""
ResNet architecture
:return:
"""
# first convolution layers
x = self.conv(x=self.input_x,
k_size=7,
filters_out=64,
strides=2,
activation=True,
name='First_Conv')
x = tf.layers.max_pooling2d(x,
pool_size=[3, 3],
strides=2,
padding='same',
name='max_pool')
# stack blocks
x = self.stack_block(x)
x = tf.layers.average_pooling2d(x,
pool_size=x.get_shape()[1:3],
strides=1,
name='average_pool')
x = tf.reshape(x, [-1, 1 * 1 * self.conv_out_depth])
fc_W = tf.truncated_normal_initializer(stddev=0.1)
logits = tf.layers.dense(inputs=x,
units=self.num_classes,
kernel_initializer=fc_W,
name='dense_layer')
# computer prediction
self.prediction = tf.argmax(logits, axis=-1)
# probability
self.probability = tf.reduce_max(tf.nn.softmax(logits), axis=-1)
# compute accuracy
self.acc = accuracy(logits, self.input_y)
# loss function
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=self.input_y))
# global steps
self.global_step = tf.train.get_or_create_global_step()
# decay learning rate
learning_rate = tf.train.exponential_decay(
learning_rate=self.learning_rate,
global_step=self.global_step,
decay_rate=self.learning_decay_rate,
decay_steps=self.learning_decay_steps,
staircase=True)
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(
self.loss)
def test(config, net, device, test_loader, epoch):
total_correct = 0
total_cnt = 0
for i, (img, label, _1, _2) in enumerate(test_loader):
img = img.to(device)
pred = net(img)
n_correct, cnt = accuracy(pred, label)
total_correct += n_correct
total_cnt += cnt
if i % config.print_freq == 0:
logging.info(
f'Epoch[{epoch}][{i}/{len(test_loader)}], Test accuracy: {n_correct / cnt:.3f}({total_correct / total_cnt:.3f})'
)
return total_correct / total_cnt
def main(net, loader, device, config):
checkpoint = os.path.join(config.weight, config.model + '.pth')
net.load_state_dict(torch.load(checkpoint))
total_correct = 0
total_cnt = 0
for i, (img, label, _1, _2) in enumerate(loader):
img = img.to(device)
pred, fmaps = net(img)
n_correct, cnt = accuracy(pred, label)
total_correct += n_correct
total_cnt += cnt
gradcam = GradCAM(i, config, checkpoint, img, pred, fmaps, label)
gradcam.generate()
print('acc: %.4f' % (total_correct / total_cnt))
def eval_model(checkpoint_dir, input_type):
"""Execute the proper evalutation of the MODEL, using the model
found in checkpoint_dir, using the specified input_tyoe
Args:
model: the model to evaluate
input_type: the Type.inputType enum that defines the input
Returns:
val: the evaluation results
"""
InputType.check(input_type)
if isinstance(MODEL, Classifier):
return evaluate.accuracy(checkpoint_dir, MODEL, DATASET, input_type,
ARGS.batch_size)
if isinstance(MODEL, Autoencoder) or isinstance(MODEL, Regressor):
return evaluate.error(checkpoint_dir, MODEL, DATASET, input_type,
ARGS.batch_size)
raise ValueError("Evaluate method not defined for this model type")
def train_step(network, train_data_loader, test_data_loader, optimizer, device,
epoch):
network.train()
# set benchmark flag to faster runtime
torch.backends.cudnn.benchmark = True
top1 = AverageMeter()
top5 = AverageMeter()
loss_calculator = LossCalculator()
prev_time = datetime.now()
for iteration, (inputs, targets) in enumerate(train_data_loader):
inputs, targets = inputs.to(device), targets.to(device)
outputs = network(inputs)
loss = loss_calculator.calc_loss(outputs, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
prec1, prec5 = accuracy(outputs, targets, topk=(1, 5))
top1.update(prec1.item(), inputs.size(0))
top5.update(prec5.item(), inputs.size(0))
cur_time = datetime.now()
h, remainder = divmod((cur_time - prev_time).seconds, 3600)
m, s = divmod(remainder, 60)
time_str = "Time: {:0>2d}:{:0>2d}:{:0>2d}".format(h, m, s)
train_acc_str = '[Train] Top1: %2.4f, Top5: %2.4f, ' % (top1.avg, top5.avg)
train_loss_str = 'Loss: %.4f. ' % loss_calculator.get_loss_log()
test_top1, test_top5, test_loss = test_step(network, test_data_loader,
device)
test_acc_str = '[Test] Top1: %2.4f, Top5: %2.4f, ' % (test_top1, test_top5)
test_loss_str = 'Loss: %.4f. ' % test_loss
print('Epoch %d. ' % epoch + train_acc_str + train_loss_str +
test_acc_str + test_loss_str + time_str)
return None
def compare(kbestall, goldall, k):
count = 0
avgacc = 0.0 # corpus acc for k best
for i, gold in goldall.iteritems():
if len(kbestall[i]) >= k:
kbest = kbestall[i][:k]
count += 1
else:
kbest = kbestall[i]
maxacc = 0.0
for best in kbest:
if len(gold[0]) != len(best):
print i
a = evaluate.accuracy(gold[0], best)
if a > maxacc:
maxacc = a
avgacc += maxacc
conv_rate = count/len(goldall)
avgacc /= len(goldall)
return conv_rate, avgacc
def train(hyperparams: dict):
"""
Train the model with <hyperparams>.
"""
train, val, test = load_data(0.8) # TODO: train/val split ratio?
model = Equilibrium(hyperparams["alpha"])
# get number of examples
train_size = train[0].shape[1]
for epoch in range(hyperparams["epochs"]):
# shuffle train set
shuffled_indices = [i for i in range(train_size)]
np.random.shuffle(shuffled_indices)
for minibatch in range(
int(np.ceil(train_size / hyperparams["minibatch"]))):
if minibatch % 100 == 0:
print("Epoch: %d, Minibatch: %d, Accuracy: %.2f" %
(epoch, minibatch, accuracy(model, *train, hyperparams)))
x_sample = train[0][:, shuffled_indices[minibatch *
hyperparams["minibatch"]:
(minibatch + 1) *
hyperparams["minibatch"]]]
y_sample = train[1][:, shuffled_indices[minibatch *
hyperparams["minibatch"]:
(minibatch + 1) *
hyperparams["minibatch"]]]
beta = (2 * np.random.randint(0, 2) -
1) * hyperparams["beta"] # random sign
s_neg = model.negative_phase(x_sample, hyperparams["t-"],
hyperparams["epsilon"])
s_pos = model.positive_phase(x_sample, y_sample, hyperparams["t+"],
hyperparams["epsilon"], beta)
model.update_weights(beta, hyperparams["etas"], s_pos, s_neg,
x_sample)
def test_model(model, data, eva_hypers):
"""Evaluate an already trained RNN on given test data.
Input
-----
model : tuple(list(np.ndarray), dict)
A tuple containing the model parameters, and a dictionary with its
hyperparameters.
dataset : tuple(tuple(np.ndarray), dict)
A tuple containing test inputs and their corresponding labels, and
dictionary with information about the data.
eva_hypers: dict
A dictionary which specifies how exactly to evaluate the model.
"""
test_data, dat_hypers = data
n_input = dat_hypers['n_input']
n_class = dat_hypers['n_class']
X_test = test_data[0]
y_test = test_data[1]
model_params, mod_hypers = model
model = DRBM(n_input=n_input,
n_class=n_class,
hypers=mod_hypers,
init_params=model_params)
# Cross entropy
y_prob = model.predict_proba(X_test)
test_nll = negative_log_likelihood(y_prob, y_test)
# Accuracy
y_pred = model.predict_function(X_test)
test_acc = accuracy(y_pred, y_test)
print "Test negative log-likelihood (offline): %.3f\n" % (test_nll)
print "Test accuracy (offline): %.3f\n" % (test_acc)
return y_prob, test_nll, y_pred, test_acc
def main(net, loader, device, config):
checkpoint = os.path.join(config.weight, config.model + '.pth')
net.load_state_dict(torch.load(checkpoint))
Xs = [None, None, None, None]
Ys = [None, None, None, None]
total_correct = 0
total_cnt = 0
for i, (img, label, _1, _2) in enumerate(loader):
img = img.to(device)
pred, features = net(img)
n_correct, cnt = accuracy(pred, label)
total_correct += n_correct
total_cnt += cnt
if i % config.print_freq == 0:
logging.info(
f'Test[{i}/{len(test_loader)}], Test accuracy: {n_correct / cnt:.3f}({total_correct / total_cnt:.3f})'
)
label = label.cpu().detach().numpy()
for i in range(4):
feature = features[i].cpu().detach().numpy()
if Xs[i] is None:
Xs[i] = feature
Ys[i] = label
else:
Xs[i] = np.concatenate((Xs[i], feature), axis=0)
Ys[i] = np.concatenate((Ys[i], label), axis=0)
for i in range(4):
print(Xs[i].shape)
print('Start visualize feature %d in t-SNE...' % i)
tSNE(Xs[i], Ys[i], [j for j in range(10)], 'feature%d_tsne' % i)
# print('Start visualize feature %d in PCA...' % i)
# pca(Xs[i], Ys[i], [j for j in range(10)], 'feature%d_pca'%i)
print('Final test accuracy: %.4f' % (total_correct / total_cnt))
def validate(dict_out, config, val_loader, val_dataset, model, criterion, output_dir,
tb_log_dir, writer_dict=None):
### output dictionary
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(val_dataset)
all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32)
all_boxes = np.zeros((num_samples, 6))
image_path = []
filenames = []
imgnums = []
idx = 0
with torch.no_grad():
end = time.time()
for i, (input, target, target_weight, meta) in enumerate(val_loader):
# compute output
output = model(input)
if config.TEST.FLIP_TEST:
# this part is ugly, because pytorch has not supported negative index
# input_flipped = model(input[:, :, :, ::-1])
input_flipped = np.flip(input.cpu().numpy(), 3).copy()
input_flipped = torch.from_numpy(input_flipped).cuda()
output_flipped = model(input_flipped)
output_flipped = flip_back(output_flipped.cpu().numpy(),
val_dataset.flip_pairs)
output_flipped = torch.from_numpy(output_flipped.copy()).cuda()
# feature is not aligned, shift flipped heatmap for higher accuracy
if config.TEST.SHIFT_HEATMAP:
output_flipped[:, :, :, 1:] = \
output_flipped.clone()[:, :, :, 0:-1]
# output_flipped[:, :, :, 0] = 0
output = (output + output_flipped) * 0.5
target = target.cuda(non_blocking=True)
target_weight = target_weight.cuda(non_blocking=True)
loss = criterion(output, target, target_weight)
num_images = input.size(0)
# measure accuracy and record loss
losses.update(loss.item(), num_images)
_, avg_acc, cnt, pred = accuracy(output.cpu().numpy(),
target.cpu().numpy())
acc.update(avg_acc, cnt)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
c = meta['center'].numpy()
s = meta['scale'].numpy()
score = meta['score'].numpy()
preds, maxvals = get_final_preds(
config, output.clone().cpu().numpy(), c, s)
all_preds[idx:idx + num_images, :, 0:2] = preds[:, :, 0:2]
all_preds[idx:idx + num_images, :, 2:3] = maxvals
# double check this all_boxes parts
all_boxes[idx:idx + num_images, 0:2] = c[:, 0:2]
all_boxes[idx:idx + num_images, 2:4] = s[:, 0:2]
all_boxes[idx:idx + num_images, 4] = np.prod(s*200, 1)
all_boxes[idx:idx + num_images, 5] = score
image_path.extend(meta['image'])
if config.DATASET.DATASET == 'posetrack':
filenames.extend(meta['filename'])
imgnums.extend(meta['imgnum'].numpy())
idx += num_images
# if i % config.PRINT_FREQ == 0:
msg = 'Test: [{0}/{1}]\t' \
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \
'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \
'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format(
i, len(val_loader), batch_time=batch_time,
loss=losses, acc=acc)
logger.info(msg)
#prefix = '{}_{}'.format(os.path.join(output_dir, 'val'), i)
#save_debug_images(config, input, meta, target, pred*4, output,
# prefix)
for b in range(len(meta['joints'])):
reshape_pred = np.concatenate((pred[b]*4, np.ones((17, 1))), axis=1).reshape(-1)
reshape_pred= reshape_pred.tolist()
anno_dict = {'area': float((meta['bbox'][2][b]*meta['bbox'][3][b]).data.numpy()),
'bbox': [float(meta['bbox'][0][b].data.numpy()),float(meta['bbox'][1][b].data.numpy()),
float(meta['bbox'][2][b].data.numpy()),float(meta['bbox'][3][b].data.numpy())],
'category_id' : 1,
'id': int(meta['id'][b].data.numpy()),
'image_id': int(meta['image_id'][b].data.numpy()),
'iscrowd': 0,
'keypoints': reshape_pred,
'num_keypoints': len(meta['joints'][b].data.numpy())
}
dict_out['annotations'].append(anno_dict)
### save json
# with open('all_posetrack_train_pred.json', 'w') as fp:
# json.dump(dict_out, fp)
name_values, perf_indicator = val_dataset.evaluate(
config, all_preds, output_dir, all_boxes, image_path,
filenames, imgnums)
_, full_arch_name = get_model_name(config)
if isinstance(name_values, list):
for name_value in name_values:
_print_name_value(name_value, full_arch_name)
else:
_print_name_value(name_values, full_arch_name)
if writer_dict:
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', losses.avg, global_steps)
writer.add_scalar('valid_acc', acc.avg, global_steps)
if isinstance(name_values, list):
for name_value in name_values:
writer.add_scalars('valid', dict(name_value), global_steps)
else:
writer.add_scalars('valid', dict(name_values), global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
return perf_indicator
for qid, uid in valData:
if qid not in nbmodels:
predictions.append(0)
continue
prob = nbmodels[qid].predict_proba([user_feats[uid]])
if nbmodels[qid].classes_[0] == 1:
predictions.append(prob[0][0])
else:
predictions.append(prob[0][1])
return predictions
user_feats, trainData, valData = loadData()
nbmodels = getModels(trainData, user_feats)
#training_predictions = getPredictions([[x[0], x[1]] for x in trainData], nbmodels, user_feats)
predictions = getPredictions(valData, nbmodels, user_feats)
# with open('../validation/contentbased_char_tfidfrevtrain.csv', 'w') as f1:
# f1.write('qid,uid,label\n')
# for i in range(0, len(training_predictions)):
# f1.write(trainData[i][0]+','+trainData[i][1]+','+str(training_predictions[i])+'\n')
fname = '../localvalidation/content_rev_user_tags.csv'
with open(fname, 'w') as f1:
f1.write('qid,uid,label\n')
for i in range(0, len(predictions)):
f1.write(valData[i][0] + ',' + valData[i][1] + ',' +
str(predictions[i]) + '\n')
print evaluate.ndcg(fname)
print evaluate.accuracy(fname)
for class_type in data_info:
data_info[class_type] = [(calc_mean(x), calc_variance(x))
for x in data_info[class_type]]
return data_info
def predict(data_info, test):
probs = []
test = np.float64(test)
for class_type in range(len(data_info)):
prob = np.ones(len(test))
class_info = data_info[class_type]
for i in range(len(class_info)):
mean, variance = class_info[i]
prob *= calc_probability(mean, variance, test[:, i])
probs.append(prob)
probs = np.array(probs)
label_prob = [(max(enumerate(x), key=lambda x: x[1])) for x in probs.T]
return np.array(list(zip(*label_prob)))
if __name__ == '__main__':
labelmap = {'Iris-setosa': 0, 'Iris-versicolor': 1, 'Iris-virginica': 2}
data = load_data('data/iris.csv', labelmap, None)
data, label = data[:, :-1], data[:, -1]
train, test, train_label, test_label = train_test_split(data, label)
data_info = get_mean_stdev(train, train_label)
result = predict(data_info, test)[0]
print(accuracy(test_label, result))
print("knn_k", knn_k)
cluster(input_file)
print('label', label)
print('class_label', class_label)
print('n_labels', next_label)
from evaluate import class_match, accuracy, consistency
mapping = None
if class_label is not None:
mapping, count = class_match(label, class_label)
print(mapping)
print(accuracy(label, class_label, mapping))
c = consistency(label, class_label, mapping, count)
table_filename = input_file + '_table.html'
open(table_filename, "wb").write(c.encode())
a = os.system('firefox ' + table_filename)
# can still write output labels, even if no "reference" class data
output_file = input_file + '_output.csv'
print(rho)
write_output(input_file, output_file, label, mapping, rho)
# maybe do later?
# put field names on the three axes (in the 3d plot)
def train(hyperparams: dict):
"""
Train the model with <hyperparams>.
"""
train, val, test = load_data(0.8) # TODO: train/val split ratio?
train = test
model = EquilibriumNet(*hyperparams["alpha"])
# get number of examples
train_size = train[0].shape[1]
for epoch in range(hyperparams["epochs"]):
# shuffle train set
shuffled_indices = [i for i in range(train_size)]
np.random.shuffle(shuffled_indices)
for minibatch in range(int(np.ceil(train_size /
model.minibatch_size))):
x_sample = train[0][:, shuffled_indices[minibatch *
hyperparams["alpha"][-1]:
(minibatch + 1) *
hyperparams["alpha"][-1]]]
y_sample = train[1][:, shuffled_indices[minibatch *
hyperparams["alpha"][-1]:
(minibatch + 1) *
hyperparams["alpha"][-1]]]
x_sample = torch.tensor(x_sample.T).float()
y_sample = torch.tensor(y_sample).float()
beta = (2 * np.random.randint(0, 2) -
1) * hyperparams["beta"] # random sign
s_neg = model.negative_phase(x_sample, hyperparams["t-"],
hyperparams["epsilon"])
s_pos = model.positive_phase(x_sample, y_sample, hyperparams["t+"],
hyperparams["epsilon"], beta)
model.update_weights(beta, hyperparams["etas"], s_pos, s_neg,
x_sample)
if minibatch % 100 == 0:
print("Epoch: %d, Minibatch: %d" % (epoch, minibatch))
total_accuracy = 0
num_accuracies = 0
for minibatch in range(int(np.ceil(train_size /
model.minibatch_size))):
x_sample = train[0][:, shuffled_indices[minibatch *
hyperparams["alpha"][-1]:
(minibatch + 1) *
hyperparams["alpha"][-1]]]
y_sample = train[1][:, shuffled_indices[minibatch *
hyperparams["alpha"][-1]:
(minibatch + 1) *
hyperparams["alpha"][-1]]]
x_sample = torch.tensor(x_sample.T).float()
y_sample = torch.tensor(y_sample).float()
total_accuracy += accuracy(model, x_sample, y_sample, hyperparams)
num_accuracies += 1
print("Epoch: %d Accuracy: %.2f" %
(epoch, total_accuracy / num_accuracies))
def get_validation_accuracy(self):
return evaluate.accuracy(self.model, self.validation_loader)
def get_accuracy(self):
acc = evaluate.accuracy(self.model, self.test_loader)
return acc
# print "-----------------------------------"
# print "Evaluation file " + str(section) + ":"
#
# for line in open("eval/temp" + str(section)):
# print line,
# print
#
# print "-----------------------------------"
# print "Testing file " + str(section) + ":"
#
# for line in open("test/temp" + str(section)):
# print line,
# print
#
# print "==================================="
# print
# Training and testing
avg_accuracy = 0.0
for section in range(10):
pos_tagger_hmm.train("train/danish" + str(section))
pos_tagger_hmm.test("test/danish" + str(section), True, "hyp/danish" + str(section))
avg_accuracy += evaluate.accuracy("eval/danish" + str(section), "hyp/danish" + str(section))
print "Average accuracy: " + str(avg_accuracy / 10.0)
#pos_tagger_hmm.train("danish.train")
#pos_tagger_hmm.test("danish.test", False, "output.csv")
#pos_tagger_hmm.train("train/hmm_testing0")
#pos_tagger_hmm.test("test/hmm_testing0", True, "out.txt")
def evaluate(self, test_data):
test_x, test_y = test_data[:, 0], test_data[:, 1]
# print(test_y)
# print(np.array([int(x) for x in self.predict(test_x)[0]]))
return accuracy(test_y, self.predict(test_x)[0])
prev = tag
score += get_local_score("", prev, "STOP", hmm)
return score
def get_aug_hmm(seq, sent, hmm, dd_u):
score = get_hmm_only_score(seq, sent, hmm)
for i in range(len(seq)):
score -= dd_u[i][seq[i]]
return score
if __name__ == "__main__":
sentences, truetags = data_reader.read_tagging_data(sys.argv[1])
hmm, tagset = hmm_utils.get_param_tagset(sys.argv[2], sys.argv[3])
sentences = replace_test(sentences, hmm, tagset)
print sentences
i = 0
tot_acc = 0.0
for sentence in sentences:
# for tag in tagset:
# print tag,"\t",
# print
tags = run(sentence, tagset, hmm, None)
#tot_acc += evaluate.accuracy(truetags, tags)
print sentence
print tags, evaluate.accuracy(truetags[i], tags)
print truetags[i], " :gold"
i+=1
print "---------------------------"
print tot_acc/i
save_interval = len(corpus.data_all['train']) // args.batch_size
best_dev_score = -99999
iterations = args.epochs * len(corpus.data_all['train']) // args.batch_size
print('max iterations: ' + str(iterations))
for iter in range(iterations):
optimizer.zero_grad()
data = corpus.get_batch(args.batch_size, 'train', div=True)
output1, output2, _ = model(data)
labels = data[2].cuda() if args.cuda else data[2]
# predict only with matching score
_, pred = output1.max(1)
score = accuracy(labels, pred)
loss1 = criterion(output1, labels)
loss2 = criterion(output2, labels)
#train via interploated loss of matching and ordering score
loss = (1 - args.lamda) * loss1 + args.lamda * loss2
loss.backward()
optimizer.step()
optimizer.zero_grad()
total_loss += float(loss.data)
total_loss1 += float(loss1.data)
total_loss2 += float(loss2.data)
if iter % interval == 0:
cur_loss = total_loss / interval if iter != 0 else total_loss
def main(unused_argv):
train_dataset, validate_dataset, test_dataset = input.input(
shuffle_files=False)
#Text information
info = tf.constant([
"Batch size = %s" % f.FLAGS.batch_size,
"Epochs = %s" % f.FLAGS.num_epochs,
"Learning rate = %s" % f.FLAGS.learning_rate,
"Batch normalization = No",
"Window size = %s" % f.FLAGS.window_size, "Shuffle Files = No",
"CNN model = %s" % f.FLAGS.cnn_model, "Shuffle Samples = YES"
])
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [
None, input.SAMPLE_DEPTH, input.SAMPLE_HEIGHT, input.SAMPLE_WIDTH
])
y_ = tf.placeholder(tf.float32, [None, 2])
dropout_rate = tf.placeholder(tf.float32)
is_training = tf.placeholder(tf.bool)
with tf.name_scope('logits'):
if f.FLAGS.cnn_model == "lenet5":
logits = lenet5.model_fn(sample_input=x,
is_training=is_training,
summaries=summaries)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=logits)
mean_cross_entropy_loss = tf.reduce_mean(cross_entropy)
loss_summ = tf.summary.scalar('Mean_cross_entropy_loss',
mean_cross_entropy_loss)
summaries['train'].append(loss_summ)
#summaries['validate'].append(loss_summ)
with tf.name_scope('adam_optimizer'):
optimizer = tf.train.AdamOptimizer(
f.FLAGS.learning_rate).minimize(mean_cross_entropy_loss)
with tf.name_scope('accuracy'):
preds = tf.argmax(logits, 1)
correct_preds = tf.argmax(y_, 1)
equal = tf.equal(preds, correct_preds)
training_accuracy_op = tf.reduce_mean(tf.cast(equal, tf.float32))
summaries['train'].append(
tf.summary.scalar('Training_Accuracy', training_accuracy_op))
with tf.name_scope('Evaluation_Metrics'):
tp_op = evaluate.tp(logits=logits, labels=y_)
fp_op = evaluate.fp(logits=logits, labels=y_)
tn_op = evaluate.tn(logits=logits, labels=y_)
fn_op = evaluate.fn(logits=logits, labels=y_)
tp_sum = tf.placeholder(tf.float32)
tn_sum = tf.placeholder(tf.float32)
fp_sum = tf.placeholder(tf.float32)
fn_sum = tf.placeholder(tf.float32)
precision_op = evaluate.precision(tp=tp_sum,
fp=fp_sum,
tn=tn_sum,
fn=fn_sum)
accuracy_op = evaluate.accuracy(tp=tp_sum,
fp=fp_sum,
tn=tn_sum,
fn=fn_sum)
recall_op = evaluate.recall(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum)
fscore_op = evaluate.fscore(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum)
precision_summ = tf.summary.scalar('Precision', precision_op)
accuracy_summ = tf.summary.scalar('Accuracy', accuracy_op)
recall_summ = tf.summary.scalar('Recall', recall_op)
fscore_summ = tf.summary.scalar('Fscore', fscore_op)
summaries['validate'].append(accuracy_summ)
summaries['validate'].append(precision_summ)
summaries['validate'].append(recall_summ)
summaries['validate'].append(fscore_summ)
summaries['test'].append(accuracy_summ)
summaries['test'].append(precision_summ)
summaries['test'].append(recall_summ)
summaries['test'].append(fscore_summ)
print("Saving graph to %s" % f.FLAGS.log_dir)
train_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/train")
validate_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/validate")
test_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/test")
train_writer.add_graph(tf.get_default_graph())
train_summaries = tf.summary.merge(summaries['train'])
validate_summaries = tf.summary.merge(summaries['validate'])
test_summaries = tf.summary.merge(summaries['test'])
with tf.Session() as sess:
train_writer.add_summary(sess.run(tf.summary.text("Information",
info)))
train_iter = train_dataset.make_initializable_iterator()
train_next_elem = train_iter.get_next()
sess.run(tf.global_variables_initializer())
global_step = 0
display_freq = 10
validate_freq = 50
test_freq = 50
for epoch in range(1, f.FLAGS.num_epochs + 1):
sess.run(train_iter.initializer)
step_time = 0.0
fetch_time = 0.0
while True:
try:
a = time.time()
global_step += 1
sample, label = sess.run(train_next_elem)
fetch_time += time.time() - a
#print (sample.shape, label.shape)
#print (label)
#for s in sample[0][0]:
# print (s)
a = time.time()
_, summ = sess.run([optimizer, train_summaries],
feed_dict={
x: sample,
y_: label,
dropout_rate: 0.5,
is_training: True
})
train_writer.add_summary(summ, global_step)
step_time += time.time() - a
except tf.errors.OutOfRangeError:
break
if global_step % display_freq == 0:
batch_loss, batch_accuracy = sess.run(
[mean_cross_entropy_loss, training_accuracy_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
print(
"Epoch {:3}\t Step {:5}:\t Loss={:.3f}, \tTraining Accuracy={:.5f} \tStep Time {:4.2f}m, Fetch Time {:4.2f}m"
.format(epoch, global_step, batch_loss, batch_accuracy,
step_time / 60, fetch_time / 60))
step_time = 0.0
fetch_time = 0.0
#Validate and test after each epoch
val_it = validate_dataset.make_one_shot_iterator()
val_next_elem = val_it.get_next()
tot_tp, tot_tn, tot_fp, tot_fn = 0, 0, 0, 0
while True:
try:
sample, label = sess.run(val_next_elem)
tp, fp, tn, fn = sess.run(
[tp_op, fp_op, tn_op, fn_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
except tf.errors.OutOfRangeError:
break
tot_tp += tp
tot_fp += fp
tot_fn += fn
tot_tn += tn
precision, recall, accuracy, fscore, summ = sess.run([
precision_op, recall_op, accuracy_op, fscore_op,
validate_summaries
],
feed_dict={
tp_sum:
tot_tp,
tn_sum:
tot_tn,
fp_sum:
tot_fp,
fn_sum:
tot_fn
})
validate_writer.add_summary(summ, global_step)
print("Epoch %d, Step %d" % (epoch, global_step))
print("=" * 10, "Validating Results", "=" * 10)
print("TP: %g\nTN: %g\nFP: %g\nFN: %g" %
(tot_tp, tot_tn, tot_fp, tot_fn))
print(
"\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g"
% (precision, recall, fscore, accuracy))
test_it = test_dataset.make_one_shot_iterator()
test_next_elem = test_it.get_next()
tot_tp, tot_tn, tot_fp, tot_tn = 0, 0, 0, 0
while True:
try:
sample, label = sess.run(test_next_elem)
tp, fp, tn, fn = sess.run(
[tp_op, fp_op, tn_op, fn_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
except tf.errors.OutOfRangeError:
break
tot_tp += tp
tot_fp += fp
tot_fn += fn
tot_tn += tn
precision, recall, accuracy, fscore, summ = sess.run([
precision_op, recall_op, accuracy_op, fscore_op, test_summaries
],
feed_dict={
tp_sum:
tot_tp,
tn_sum:
tot_tn,
fp_sum:
tot_fp,
fn_sum:
tot_fn
})
test_writer.add_summary(summ, global_step)
print("=" * 10, "Testing Results", "=" * 10)
print("TP: %g\nTN: %g\nFP: %g\nFN: %g" %
(tot_tp, tot_tn, tot_fp, tot_fn))
print(
"\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g"
% (precision, recall, fscore, accuracy))
print("=" * 10, "===============", "=" * 10)
def execute(dataset, hmm_file, tag_file):
# sys.stderr.write("loading learnt parameters...\n")
hmm, tagset = hmm_utils.get_param_tagset(hmm_file, tag_file)
# sys.stderr.write("reading dev data...\n")
ts, test_tags = data_reader.read_tagging_data(dataset)
test_sentences = replace_test(ts, hmm, tagset)
i = 0
converges = 0
avg_iterations = 0
start_time = time.time()
fst_acc = 0
best_acc = 0
wrong = 0
for sentence in test_sentences:
if True:
i += 1
truetags = test_tags[test_sentences.index(sentence)]
sys.stderr.write("\n" + str(i) + "\n")
# sys.stderr.write(' '.join(ts[i-1]) + "\n")
print " ".join(sentence)
# print ' '.join(ts[i-1])
best_tags, num_iterations, tags1, tags2 = dd_tagger_fst.run(sentence, tagset, hmm)
if tags2 == best_tags:
sys.stderr.write("YOU ARE WRONG!\n")
wrong += 1
if num_iterations != -1:
facc = evaluate.accuracy(truetags, tags2)
# sys.stderr.write("fst tagger accuracy = " + str(facc) + "\n")
fst_acc += facc
bacc = evaluate.accuracy(truetags, best_tags)
# sys.stderr.write("best tags accuracy = " + str(bacc) + "\n")
best_acc += bacc
sys.stderr.write("converges in " + str(num_iterations) + " iterations \n")
converges += 1
avg_iterations += num_iterations
else:
sys.stderr.write("does not converge :(\n")
print " ".join(best_tags)
print " ".join(tags2)
# print "gold : ", ' '.join(truetags)
print
# if i == 100:
# break
sys.stderr.write("\nsystem performance\n--------------------\n")
sys.stderr.write("\n" + str(wrong * 100 / converges) + "% sequences are wrong:\n")
sys.stderr.write("\naverage accuracy of best: " + str(best_acc / converges) + "\n")
sys.stderr.write("average accuracy of 2nd best: " + str(fst_acc / converges) + "\n")
sys.stderr.write("\nsystem efficiency\n---------------------\n")
sys.stderr.write("\n" + str(avg_iterations / converges) + " iterations on average\n")
sys.stderr.write(str(converges * 100 / i) + " % convergence\n")
sys.stderr.write("time_taken = " + str(time.time() - start_time) + "\n")
def optimize(self, model, dataset):
"""Learn a model using batch gradient descent.
Input
-----
model : Python class
Definition of model.
dataset : tuple(tuple(np.ndarray))
Each tuple contains inputs and targets for training, validation and
test data.
Output
------
best_model_params: list(np.ndarray)
List of all the model parameters
best_valid_acc: float
Best validation set accuracy.
"""
# Load training and validation data, and check for sparsity
X_train, y_train = dataset[0]
X_valid, y_valid = dataset[1]
if self.eval_test == True:
X_test, y_test = dataset[2]
# Generate file name to save intermediate training models
os.mkdir('.' + model.uid)
temp_file_name = os.path.join('.' + model.uid, 'best_model.pkl')
n_valid = len(X_valid)
if self.eval_test == True:
n_test = len(X_test)
# Initialize learning rate and schedule
learning_rate = self.learning_rate
threshold = self.threshold
if self.schedule == 'constant':
rate_update = lambda coeff: self.learning_rate
elif self.schedule == 'linear':
rate_update = lambda coeff: self.learning_rate / (1 + coeff)
elif self.schedule == 'exponential':
rate_update = lambda coeff: self.learning_rate / \
10**(coeff/self.threshold)
elif self.schedule == 'power':
rate_update = lambda coeff: self.learning_rate / \
(1 + coeff/self.threshold)
elif self.schedule == 'inv-log':
rate_update = lambda coeff: self.learning_rate / \
(1+np.log(coeff+1))
max_epoch = self.max_epoch
validation_frequency = self.validation_frequency
best_valid_acc = -1.0
if self.eval_test == True:
best_test_acc = -1.0
cPickle.dump(model, open(temp_file_name, 'wb'))
best_params = model.get_model_parameters()
# Early stopping parameters and other checks
patience = self.patience
pissed_off = 0
for epoch in xrange(max_epoch):
# Set effective momentum for the current epoch.
effective_momentum = self.final_momentum \
if epoch > self.momentum_switchover \
else self.initial_momentum
# Check if it's time to stop learning.
if threshold == 0:
if pissed_off == patience: # Exit and return best model
model = cPickle.load(open(temp_file_name, 'rb'))
best_params = model.get_model_parameters()
print('Learning terminated after %d epochs.\n' %
(epoch + 1))
break
else: # Reload previous best model and continue
pissed_off += 1
learning_rate = rate_update(pissed_off)
threshold = self.threshold
model = cPickle.load(open(temp_file_name, 'rb'))
print(
'Re-initialising to previous best model with '
'validation prediction accuracy %.3f.\n'
'\tCurrent pissed off level: %d/%d.\n'
'\tCurrent learning rate: %.4f.\n' %
(best_valid_acc, pissed_off, patience, learning_rate))
RNG.seed(0xbeef)
RNG.shuffle(X_train)
RNG.seed(0xbeef)
RNG.shuffle(y_train)
costs = []
for X, y in zip(X_train, y_train):
costs.append(
model.train_function(X, y, learning_rate,
effective_momentum))
mean_train_loss = np.mean(costs)
print('Epoch %i/%i, train loss: %.3f bits' %
(epoch + 1, max_epoch, mean_train_loss))
if (epoch + 1) % validation_frequency == 0:
# Compute validation negative log-likelihood
valid_pred = []
if self.eval_test == True:
test_pred = []
for i in xrange(n_valid):
valid_pred.append( # Don't pass labels
model.predict_function(X_valid[i]))
this_valid_acc = accuracy(
np.concatenate(tuple(valid_pred), axis=0),
np.concatenate(tuple(y_valid), axis=0))
if self.eval_test == True:
for i in xrange(n_test):
test_pred.append( # Don't pass labels
model.predict_function(X_test[i]))
this_test_acc = accuracy(
np.concatenate(tuple(test_pred), axis=0),
np.concatenate(tuple(y_test), axis=0))
print("\tValidation accuracy: %.3f (previous best: %.3f)" %
(this_valid_acc, best_valid_acc))
if self.eval_test == True:
print("\tTest accuracy: %.3f (previous best: %.3f)" %
(this_test_acc, best_test_acc))
print "\tCurrent learning rate: %.4f" % (learning_rate)
if this_valid_acc > best_valid_acc:
best_valid_acc = this_valid_acc
best_params = model.get_model_parameters()
cPickle.dump(model, open(temp_file_name, 'wb'))
threshold = self.threshold
else:
threshold -= 1
# Clean temporary files and folder
os.remove(temp_file_name)
os.rmdir('.' + model.uid)
return best_params, best_valid_acc
