def lstm_model(train, val, test): # vocab = Vocab() # vocab.from_dataset(train+val+test) vocab = generate_vocab() hparams = { "vocab": vocab, "embedding_dim": 128, "hidden_dim": 128, "n_epochs": 5, "batch_size": 32, "lr": 0.001, "num_layers": 1, "dropout": 0, } model = LSTMModel(hparams) model.fit(train, val) print("lstm model perplexity:", evaluate(model, test)) progression = model.generate(10) print("generated progression:", len(progression), progression) # convert_to_chord_name(progression)) print("generated progression perplexity:", evaluate(model, [progression])) print("accuracy:", accuracy(model, test)) print("vocab size:", len(vocab))
def run(trainData, valData, foldno): question_feats, user_feats, user_keys, user_keys_map = loadData() nbmodels = getModels(trainData, question_feats) predictions = getPredictions(valData, nbmodels, question_feats, user_feats, user_keys, user_keys_map) fname = '../localvalidation/content_ques_topics' + str(foldno) + '.csv' with open(fname, 'w') as f1: f1.write('qid,uid,label\n') for i in range(0, len(predictions)): f1.write(valData[i][0] + ',' + valData[i][1] + ',' + str(predictions[i]) + '\n') return evaluate.ndcg(fname) print evaluate.accuracy(fname) print evaluate.logloss(fname) return
def test(net, test_dataloader, device, criterion, epoch, epoch_test_loss, f_test_acc): running_loss = 0 net.eval() for i, data in enumerate(test_dataloader, 0): # get the inputs test_batch, label_batch = data test_batch = test_batch.to(device) label_batch = label_batch.to(device) test_batch, label_batch = Variable(test_batch), Variable(label_batch) m, h, w = test_batch.shape test_batch = test_batch.view(m, 1, h, w) label_batch = label_batch.type(torch.long) # forward outputs = net(test_batch) loss = criterion(outputs, label_batch) running_loss += loss.item() # print statistics if i % 100 == 0: # print every 100 iterations test_acc = accuracy(outputs, label_batch) for j in range(6): f_test_acc[j].append(test_acc[j]) running_loss = running_loss/len(test_dataloader) epoch_test_loss.append(running_loss)
def train(net, train_dataloader, device, optimizer, criterion, epoch, epoch_train_loss, f_train_acc): running_loss = 0 for i, data in enumerate(train_dataloader, 0): # get the inputs train_batch, label_batch = data train_batch = train_batch.to(device) label_batch = label_batch.to(device) train_batch, label_batch = Variable(train_batch), Variable(label_batch) m, h ,w = train_batch.shape train_batch = train_batch.view(m,1,h,w) label_batch = label_batch.type(torch.long) # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = net(train_batch) loss = criterion(outputs, label_batch) loss.backward() optimizer.step() # print statistics running_loss += loss.item() if i % 100 == 0: # print every 100 iterations. train_acc = accuracy(outputs, label_batch) for j in range(6): f_train_acc[j].append(train_acc[j]) running_loss = running_loss/len(train_dataloader) epoch_train_loss.append(running_loss)
def train(config, train_loader, model, criterion, optimizer, epoch, output_dir, tb_log_dir, writer_dict): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() acc = AverageMeter() # switch to train mode model.train() end = time.time() for i, (input, target, target_weight, meta) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) # compute output output = model(input) target = target.cuda(non_blocking=True) target_weight = target_weight.cuda(non_blocking=True) loss = criterion(output, target, target_weight) # compute gradient and do update step optimizer.zero_grad() loss.backward() optimizer.step() # measure accuracy and record loss losses.update(loss.item(), input.size(0)) _, avg_acc, cnt, pred = accuracy(output.detach().cpu().numpy(), target.detach().cpu().numpy()) acc.update(avg_acc, cnt) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % config.PRINT_FREQ == 0: msg = 'Epoch: [{0}][{1}/{2}]\t' \ 'Time {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\t' \ 'Speed {speed:.1f} samples/s\t' \ 'Data {data_time.val:.3f}s ({data_time.avg:.3f}s)\t' \ 'Loss {loss.val:.5f} ({loss.avg:.5f})\t' \ 'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, speed=input.size(0)/batch_time.val, data_time=data_time, loss=losses, acc=acc) logger.info(msg) writer = writer_dict['writer'] global_steps = writer_dict['train_global_steps'] writer.add_scalar('train_loss', losses.val, global_steps) writer.add_scalar('train_acc', acc.val, global_steps) writer_dict['train_global_steps'] = global_steps + 1 prefix = '{}_{}'.format(os.path.join(output_dir, 'train'), i) save_debug_images(config, input, meta, target, pred*4, output, prefix)
def train_step(network, data_loader, loss_calculator, optimizer, device, epoch, print_freq=100): network.train() # set benchmark flag to faster runtime torch.backends.cudnn.benchmark = True starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) train_time = AverageMeter() data_time = AverageMeter() loss_time = AverageMeter() forward_time = AverageMeter() backward_time = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() tic = time.time() s_tic = time.time() for iteration, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - tic) inputs, targets = inputs.to(device), targets.to(device) starter.record() outputs = network(inputs) ender.record() # WAIT FOR GPU SYNC torch.cuda.synchronize() forward_time.update(starter.elapsed_time(ender)/1000) tic = time.time() loss = loss_calculator.calc_loss(outputs, targets) loss_time.update(time.time() - tic) tic = time.time() optimizer.zero_grad() loss.backward() optimizer.step() backward_time.update(time.time() - tic) train_time.update(time.time() - s_tic) prec1, prec5 = accuracy(outputs.data, targets, topk=(1,5)) top1.update(prec1.item(), inputs.size(0)) top5.update(prec5.item(), inputs.size(0)) if iteration % print_freq == 0: logs_ = '%s: '%time.ctime() logs_ += 'Epoch [%d], '%epoch logs_ += 'Iteration [%d/%d/], '%(iteration, len(data_loader)) logs_ += 'Data(s): %2.3f, Loss(s): %2.3f, '%(data_time.avg, loss_time.avg) logs_ += 'Forward(s): %2.3f, Backward(s): %2.3f, '%(forward_time.avg, backward_time.avg) logs_ += 'Train(s): %2.3f, '%(train_time.avg) logs_ += 'Top1: %2.3f, Top5: %2.4f, '%(top1.avg, top5.avg) logs_ += 'Loss: %2.3f'%loss_calculator.get_loss_log() print(logs_) tic = time.time() s_tic = time.time() return None
def __get_evaluation_score(self, model_result): if self.scoring == MODEL_SCORE_EVALUATION_ACCURACY: return accuracy(model_result) elif self.scoring == MODEL_SCORE_EVALUATION_F1_MACRO: return macro_f1(model_result) elif self.scoring == MODEL_SCORE_EVALUATION_F1_WEIGHTED: return weighted_f1(model_result) else: raise ValueError( 'Unsupported scoring strategy for model evaluation!')
def execute(dataset, hmm_file, tag_file): sys.stderr.write("loading learnt parameters...\n") hmm, tagset = hmm_utils.get_param_tagset(hmm_file, tag_file) sys.stderr.write("reading dev data...\n") test_sentences, test_tags = data_reader.read_tagging_data(dataset) test_sentences = replace_test(test_sentences, hmm, tagset) i = 0 converges = 0 avg_iterations = 0 start_time = time.time() for sentence in test_sentences: if len(sentence) > 0 :#True: #len(tree) < 100: i += 1 truetags = test_tags[test_sentences.index(sentence)] sys.stderr.write('\n' + str(i)+ '\n') tagprint(test_sents_not_rare[test_sentences.index(sentence)]) best_tags, num_iterations, tags1, tags2 = dd_tagger_fst.run(sentence, tagset, hmm) tagprint(best_tags) tagprint(tags2) if num_iterations != -1: sys.stderr.write("fst tagger accuracy = " + str(evaluate.accuracy(truetags, tags2)) + "\n") sys.stderr.write("best tags accuracy = " + str(evaluate.accuracy(truetags, best_tags)) + "\n") sys.stderr.write("converges in " + str(num_iterations) + " iterations \n") converges += 1 avg_iterations += num_iterations else: print sys.stderr.write("does not converge :(\n") tagprint(truetags) print #if i==100: #break sys.stderr.write("\n" + str(avg_iterations/converges) + " iterations on average\n") sys.stderr.write(str(converges*100/i) + " % convergence\n") sys.stderr.write("time_taken = "+ str(time.time() - start_time) + "\n")
def evaluation(param, test_file='validation'): if param['evaluation_metric'] == 'auc': acc = e.accuracy(param, test_file) print('Iteration %s, Accuracy: %s' % (param['iteration'], acc)) elif param['evaluation_metric'] == 'mean_rank': hits, top_hits, mean_rank = e.test(param, test_file='validation', is_filetered=False) print('Iteration %s, mean_rank: %s, top_hits: %s, hits %s' % (param['iteration'], mean_rank, top_hits, hits))
def model(self): """ ResNet architecture :return: """ # first convolution layers x = self.conv(x=self.input_x, k_size=7, filters_out=64, strides=2, activation=True, name='First_Conv') x = tf.layers.max_pooling2d(x, pool_size=[3, 3], strides=2, padding='same', name='max_pool') # stack blocks x = self.stack_block(x) x = tf.layers.average_pooling2d(x, pool_size=x.get_shape()[1:3], strides=1, name='average_pool') x = tf.reshape(x, [-1, 1 * 1 * self.conv_out_depth]) fc_W = tf.truncated_normal_initializer(stddev=0.1) logits = tf.layers.dense(inputs=x, units=self.num_classes, kernel_initializer=fc_W, name='dense_layer') # computer prediction self.prediction = tf.argmax(logits, axis=-1) # probability self.probability = tf.reduce_max(tf.nn.softmax(logits), axis=-1) # compute accuracy self.acc = accuracy(logits, self.input_y) # loss function self.loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=self.input_y)) # global steps self.global_step = tf.train.get_or_create_global_step() # decay learning rate learning_rate = tf.train.exponential_decay( learning_rate=self.learning_rate, global_step=self.global_step, decay_rate=self.learning_decay_rate, decay_steps=self.learning_decay_steps, staircase=True) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.loss)
def test(config, net, device, test_loader, epoch): total_correct = 0 total_cnt = 0 for i, (img, label, _1, _2) in enumerate(test_loader): img = img.to(device) pred = net(img) n_correct, cnt = accuracy(pred, label) total_correct += n_correct total_cnt += cnt if i % config.print_freq == 0: logging.info( f'Epoch[{epoch}][{i}/{len(test_loader)}], Test accuracy: {n_correct / cnt:.3f}({total_correct / total_cnt:.3f})' ) return total_correct / total_cnt
def main(net, loader, device, config): checkpoint = os.path.join(config.weight, config.model + '.pth') net.load_state_dict(torch.load(checkpoint)) total_correct = 0 total_cnt = 0 for i, (img, label, _1, _2) in enumerate(loader): img = img.to(device) pred, fmaps = net(img) n_correct, cnt = accuracy(pred, label) total_correct += n_correct total_cnt += cnt gradcam = GradCAM(i, config, checkpoint, img, pred, fmaps, label) gradcam.generate() print('acc: %.4f' % (total_correct / total_cnt))
def eval_model(checkpoint_dir, input_type): """Execute the proper evalutation of the MODEL, using the model found in checkpoint_dir, using the specified input_tyoe Args: model: the model to evaluate input_type: the Type.inputType enum that defines the input Returns: val: the evaluation results """ InputType.check(input_type) if isinstance(MODEL, Classifier): return evaluate.accuracy(checkpoint_dir, MODEL, DATASET, input_type, ARGS.batch_size) if isinstance(MODEL, Autoencoder) or isinstance(MODEL, Regressor): return evaluate.error(checkpoint_dir, MODEL, DATASET, input_type, ARGS.batch_size) raise ValueError("Evaluate method not defined for this model type")
def train_step(network, train_data_loader, test_data_loader, optimizer, device, epoch): network.train() # set benchmark flag to faster runtime torch.backends.cudnn.benchmark = True top1 = AverageMeter() top5 = AverageMeter() loss_calculator = LossCalculator() prev_time = datetime.now() for iteration, (inputs, targets) in enumerate(train_data_loader): inputs, targets = inputs.to(device), targets.to(device) outputs = network(inputs) loss = loss_calculator.calc_loss(outputs, targets) optimizer.zero_grad() loss.backward() optimizer.step() prec1, prec5 = accuracy(outputs, targets, topk=(1, 5)) top1.update(prec1.item(), inputs.size(0)) top5.update(prec5.item(), inputs.size(0)) cur_time = datetime.now() h, remainder = divmod((cur_time - prev_time).seconds, 3600) m, s = divmod(remainder, 60) time_str = "Time: {:0>2d}:{:0>2d}:{:0>2d}".format(h, m, s) train_acc_str = '[Train] Top1: %2.4f, Top5: %2.4f, ' % (top1.avg, top5.avg) train_loss_str = 'Loss: %.4f. ' % loss_calculator.get_loss_log() test_top1, test_top5, test_loss = test_step(network, test_data_loader, device) test_acc_str = '[Test] Top1: %2.4f, Top5: %2.4f, ' % (test_top1, test_top5) test_loss_str = 'Loss: %.4f. ' % test_loss print('Epoch %d. ' % epoch + train_acc_str + train_loss_str + test_acc_str + test_loss_str + time_str) return None
def compare(kbestall, goldall, k): count = 0 avgacc = 0.0 # corpus acc for k best for i, gold in goldall.iteritems(): if len(kbestall[i]) >= k: kbest = kbestall[i][:k] count += 1 else: kbest = kbestall[i] maxacc = 0.0 for best in kbest: if len(gold[0]) != len(best): print i a = evaluate.accuracy(gold[0], best) if a > maxacc: maxacc = a avgacc += maxacc conv_rate = count/len(goldall) avgacc /= len(goldall) return conv_rate, avgacc
def train(hyperparams: dict): """ Train the model with <hyperparams>. """ train, val, test = load_data(0.8) # TODO: train/val split ratio? model = Equilibrium(hyperparams["alpha"]) # get number of examples train_size = train[0].shape[1] for epoch in range(hyperparams["epochs"]): # shuffle train set shuffled_indices = [i for i in range(train_size)] np.random.shuffle(shuffled_indices) for minibatch in range( int(np.ceil(train_size / hyperparams["minibatch"]))): if minibatch % 100 == 0: print("Epoch: %d, Minibatch: %d, Accuracy: %.2f" % (epoch, minibatch, accuracy(model, *train, hyperparams))) x_sample = train[0][:, shuffled_indices[minibatch * hyperparams["minibatch"]: (minibatch + 1) * hyperparams["minibatch"]]] y_sample = train[1][:, shuffled_indices[minibatch * hyperparams["minibatch"]: (minibatch + 1) * hyperparams["minibatch"]]] beta = (2 * np.random.randint(0, 2) - 1) * hyperparams["beta"] # random sign s_neg = model.negative_phase(x_sample, hyperparams["t-"], hyperparams["epsilon"]) s_pos = model.positive_phase(x_sample, y_sample, hyperparams["t+"], hyperparams["epsilon"], beta) model.update_weights(beta, hyperparams["etas"], s_pos, s_neg, x_sample)
def test_model(model, data, eva_hypers): """Evaluate an already trained RNN on given test data. Input ----- model : tuple(list(np.ndarray), dict) A tuple containing the model parameters, and a dictionary with its hyperparameters. dataset : tuple(tuple(np.ndarray), dict) A tuple containing test inputs and their corresponding labels, and dictionary with information about the data. eva_hypers: dict A dictionary which specifies how exactly to evaluate the model. """ test_data, dat_hypers = data n_input = dat_hypers['n_input'] n_class = dat_hypers['n_class'] X_test = test_data[0] y_test = test_data[1] model_params, mod_hypers = model model = DRBM(n_input=n_input, n_class=n_class, hypers=mod_hypers, init_params=model_params) # Cross entropy y_prob = model.predict_proba(X_test) test_nll = negative_log_likelihood(y_prob, y_test) # Accuracy y_pred = model.predict_function(X_test) test_acc = accuracy(y_pred, y_test) print "Test negative log-likelihood (offline): %.3f\n" % (test_nll) print "Test accuracy (offline): %.3f\n" % (test_acc) return y_prob, test_nll, y_pred, test_acc
def main(net, loader, device, config): checkpoint = os.path.join(config.weight, config.model + '.pth') net.load_state_dict(torch.load(checkpoint)) Xs = [None, None, None, None] Ys = [None, None, None, None] total_correct = 0 total_cnt = 0 for i, (img, label, _1, _2) in enumerate(loader): img = img.to(device) pred, features = net(img) n_correct, cnt = accuracy(pred, label) total_correct += n_correct total_cnt += cnt if i % config.print_freq == 0: logging.info( f'Test[{i}/{len(test_loader)}], Test accuracy: {n_correct / cnt:.3f}({total_correct / total_cnt:.3f})' ) label = label.cpu().detach().numpy() for i in range(4): feature = features[i].cpu().detach().numpy() if Xs[i] is None: Xs[i] = feature Ys[i] = label else: Xs[i] = np.concatenate((Xs[i], feature), axis=0) Ys[i] = np.concatenate((Ys[i], label), axis=0) for i in range(4): print(Xs[i].shape) print('Start visualize feature %d in t-SNE...' % i) tSNE(Xs[i], Ys[i], [j for j in range(10)], 'feature%d_tsne' % i) # print('Start visualize feature %d in PCA...' % i) # pca(Xs[i], Ys[i], [j for j in range(10)], 'feature%d_pca'%i) print('Final test accuracy: %.4f' % (total_correct / total_cnt))
def validate(dict_out, config, val_loader, val_dataset, model, criterion, output_dir, tb_log_dir, writer_dict=None): ### output dictionary batch_time = AverageMeter() losses = AverageMeter() acc = AverageMeter() # switch to evaluate mode model.eval() num_samples = len(val_dataset) all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3), dtype=np.float32) all_boxes = np.zeros((num_samples, 6)) image_path = [] filenames = [] imgnums = [] idx = 0 with torch.no_grad(): end = time.time() for i, (input, target, target_weight, meta) in enumerate(val_loader): # compute output output = model(input) if config.TEST.FLIP_TEST: # this part is ugly, because pytorch has not supported negative index # input_flipped = model(input[:, :, :, ::-1]) input_flipped = np.flip(input.cpu().numpy(), 3).copy() input_flipped = torch.from_numpy(input_flipped).cuda() output_flipped = model(input_flipped) output_flipped = flip_back(output_flipped.cpu().numpy(), val_dataset.flip_pairs) output_flipped = torch.from_numpy(output_flipped.copy()).cuda() # feature is not aligned, shift flipped heatmap for higher accuracy if config.TEST.SHIFT_HEATMAP: output_flipped[:, :, :, 1:] = \ output_flipped.clone()[:, :, :, 0:-1] # output_flipped[:, :, :, 0] = 0 output = (output + output_flipped) * 0.5 target = target.cuda(non_blocking=True) target_weight = target_weight.cuda(non_blocking=True) loss = criterion(output, target, target_weight) num_images = input.size(0) # measure accuracy and record loss losses.update(loss.item(), num_images) _, avg_acc, cnt, pred = accuracy(output.cpu().numpy(), target.cpu().numpy()) acc.update(avg_acc, cnt) # measure elapsed time batch_time.update(time.time() - end) end = time.time() c = meta['center'].numpy() s = meta['scale'].numpy() score = meta['score'].numpy() preds, maxvals = get_final_preds( config, output.clone().cpu().numpy(), c, s) all_preds[idx:idx + num_images, :, 0:2] = preds[:, :, 0:2] all_preds[idx:idx + num_images, :, 2:3] = maxvals # double check this all_boxes parts all_boxes[idx:idx + num_images, 0:2] = c[:, 0:2] all_boxes[idx:idx + num_images, 2:4] = s[:, 0:2] all_boxes[idx:idx + num_images, 4] = np.prod(s*200, 1) all_boxes[idx:idx + num_images, 5] = score image_path.extend(meta['image']) if config.DATASET.DATASET == 'posetrack': filenames.extend(meta['filename']) imgnums.extend(meta['imgnum'].numpy()) idx += num_images # if i % config.PRINT_FREQ == 0: msg = 'Test: [{0}/{1}]\t' \ 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \ 'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, acc=acc) logger.info(msg) #prefix = '{}_{}'.format(os.path.join(output_dir, 'val'), i) #save_debug_images(config, input, meta, target, pred*4, output, # prefix) for b in range(len(meta['joints'])): reshape_pred = np.concatenate((pred[b]*4, np.ones((17, 1))), axis=1).reshape(-1) reshape_pred= reshape_pred.tolist() anno_dict = {'area': float((meta['bbox'][2][b]*meta['bbox'][3][b]).data.numpy()), 'bbox': [float(meta['bbox'][0][b].data.numpy()),float(meta['bbox'][1][b].data.numpy()), float(meta['bbox'][2][b].data.numpy()),float(meta['bbox'][3][b].data.numpy())], 'category_id' : 1, 'id': int(meta['id'][b].data.numpy()), 'image_id': int(meta['image_id'][b].data.numpy()), 'iscrowd': 0, 'keypoints': reshape_pred, 'num_keypoints': len(meta['joints'][b].data.numpy()) } dict_out['annotations'].append(anno_dict) ### save json # with open('all_posetrack_train_pred.json', 'w') as fp: # json.dump(dict_out, fp) name_values, perf_indicator = val_dataset.evaluate( config, all_preds, output_dir, all_boxes, image_path, filenames, imgnums) _, full_arch_name = get_model_name(config) if isinstance(name_values, list): for name_value in name_values: _print_name_value(name_value, full_arch_name) else: _print_name_value(name_values, full_arch_name) if writer_dict: writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', losses.avg, global_steps) writer.add_scalar('valid_acc', acc.avg, global_steps) if isinstance(name_values, list): for name_value in name_values: writer.add_scalars('valid', dict(name_value), global_steps) else: writer.add_scalars('valid', dict(name_values), global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return perf_indicator
for qid, uid in valData: if qid not in nbmodels: predictions.append(0) continue prob = nbmodels[qid].predict_proba([user_feats[uid]]) if nbmodels[qid].classes_[0] == 1: predictions.append(prob[0][0]) else: predictions.append(prob[0][1]) return predictions user_feats, trainData, valData = loadData() nbmodels = getModels(trainData, user_feats) #training_predictions = getPredictions([[x[0], x[1]] for x in trainData], nbmodels, user_feats) predictions = getPredictions(valData, nbmodels, user_feats) # with open('../validation/contentbased_char_tfidfrevtrain.csv', 'w') as f1: # f1.write('qid,uid,label\n') # for i in range(0, len(training_predictions)): # f1.write(trainData[i][0]+','+trainData[i][1]+','+str(training_predictions[i])+'\n') fname = '../localvalidation/content_rev_user_tags.csv' with open(fname, 'w') as f1: f1.write('qid,uid,label\n') for i in range(0, len(predictions)): f1.write(valData[i][0] + ',' + valData[i][1] + ',' + str(predictions[i]) + '\n') print evaluate.ndcg(fname) print evaluate.accuracy(fname)
for class_type in data_info: data_info[class_type] = [(calc_mean(x), calc_variance(x)) for x in data_info[class_type]] return data_info def predict(data_info, test): probs = [] test = np.float64(test) for class_type in range(len(data_info)): prob = np.ones(len(test)) class_info = data_info[class_type] for i in range(len(class_info)): mean, variance = class_info[i] prob *= calc_probability(mean, variance, test[:, i]) probs.append(prob) probs = np.array(probs) label_prob = [(max(enumerate(x), key=lambda x: x[1])) for x in probs.T] return np.array(list(zip(*label_prob))) if __name__ == '__main__': labelmap = {'Iris-setosa': 0, 'Iris-versicolor': 1, 'Iris-virginica': 2} data = load_data('data/iris.csv', labelmap, None) data, label = data[:, :-1], data[:, -1] train, test, train_label, test_label = train_test_split(data, label) data_info = get_mean_stdev(train, train_label) result = predict(data_info, test)[0] print(accuracy(test_label, result))
print("knn_k", knn_k) cluster(input_file) print('label', label) print('class_label', class_label) print('n_labels', next_label) from evaluate import class_match, accuracy, consistency mapping = None if class_label is not None: mapping, count = class_match(label, class_label) print(mapping) print(accuracy(label, class_label, mapping)) c = consistency(label, class_label, mapping, count) table_filename = input_file + '_table.html' open(table_filename, "wb").write(c.encode()) a = os.system('firefox ' + table_filename) # can still write output labels, even if no "reference" class data output_file = input_file + '_output.csv' print(rho) write_output(input_file, output_file, label, mapping, rho) # maybe do later? # put field names on the three axes (in the 3d plot)
def train(hyperparams: dict): """ Train the model with <hyperparams>. """ train, val, test = load_data(0.8) # TODO: train/val split ratio? train = test model = EquilibriumNet(*hyperparams["alpha"]) # get number of examples train_size = train[0].shape[1] for epoch in range(hyperparams["epochs"]): # shuffle train set shuffled_indices = [i for i in range(train_size)] np.random.shuffle(shuffled_indices) for minibatch in range(int(np.ceil(train_size / model.minibatch_size))): x_sample = train[0][:, shuffled_indices[minibatch * hyperparams["alpha"][-1]: (minibatch + 1) * hyperparams["alpha"][-1]]] y_sample = train[1][:, shuffled_indices[minibatch * hyperparams["alpha"][-1]: (minibatch + 1) * hyperparams["alpha"][-1]]] x_sample = torch.tensor(x_sample.T).float() y_sample = torch.tensor(y_sample).float() beta = (2 * np.random.randint(0, 2) - 1) * hyperparams["beta"] # random sign s_neg = model.negative_phase(x_sample, hyperparams["t-"], hyperparams["epsilon"]) s_pos = model.positive_phase(x_sample, y_sample, hyperparams["t+"], hyperparams["epsilon"], beta) model.update_weights(beta, hyperparams["etas"], s_pos, s_neg, x_sample) if minibatch % 100 == 0: print("Epoch: %d, Minibatch: %d" % (epoch, minibatch)) total_accuracy = 0 num_accuracies = 0 for minibatch in range(int(np.ceil(train_size / model.minibatch_size))): x_sample = train[0][:, shuffled_indices[minibatch * hyperparams["alpha"][-1]: (minibatch + 1) * hyperparams["alpha"][-1]]] y_sample = train[1][:, shuffled_indices[minibatch * hyperparams["alpha"][-1]: (minibatch + 1) * hyperparams["alpha"][-1]]] x_sample = torch.tensor(x_sample.T).float() y_sample = torch.tensor(y_sample).float() total_accuracy += accuracy(model, x_sample, y_sample, hyperparams) num_accuracies += 1 print("Epoch: %d Accuracy: %.2f" % (epoch, total_accuracy / num_accuracies))
def get_validation_accuracy(self): return evaluate.accuracy(self.model, self.validation_loader)
def get_accuracy(self): acc = evaluate.accuracy(self.model, self.test_loader) return acc
# print "-----------------------------------" # print "Evaluation file " + str(section) + ":" # # for line in open("eval/temp" + str(section)): # print line, # print # # print "-----------------------------------" # print "Testing file " + str(section) + ":" # # for line in open("test/temp" + str(section)): # print line, # print # # print "===================================" # print # Training and testing avg_accuracy = 0.0 for section in range(10): pos_tagger_hmm.train("train/danish" + str(section)) pos_tagger_hmm.test("test/danish" + str(section), True, "hyp/danish" + str(section)) avg_accuracy += evaluate.accuracy("eval/danish" + str(section), "hyp/danish" + str(section)) print "Average accuracy: " + str(avg_accuracy / 10.0) #pos_tagger_hmm.train("danish.train") #pos_tagger_hmm.test("danish.test", False, "output.csv") #pos_tagger_hmm.train("train/hmm_testing0") #pos_tagger_hmm.test("test/hmm_testing0", True, "out.txt")
def evaluate(self, test_data): test_x, test_y = test_data[:, 0], test_data[:, 1] # print(test_y) # print(np.array([int(x) for x in self.predict(test_x)[0]])) return accuracy(test_y, self.predict(test_x)[0])
prev = tag score += get_local_score("", prev, "STOP", hmm) return score def get_aug_hmm(seq, sent, hmm, dd_u): score = get_hmm_only_score(seq, sent, hmm) for i in range(len(seq)): score -= dd_u[i][seq[i]] return score if __name__ == "__main__": sentences, truetags = data_reader.read_tagging_data(sys.argv[1]) hmm, tagset = hmm_utils.get_param_tagset(sys.argv[2], sys.argv[3]) sentences = replace_test(sentences, hmm, tagset) print sentences i = 0 tot_acc = 0.0 for sentence in sentences: # for tag in tagset: # print tag,"\t", # print tags = run(sentence, tagset, hmm, None) #tot_acc += evaluate.accuracy(truetags, tags) print sentence print tags, evaluate.accuracy(truetags[i], tags) print truetags[i], " :gold" i+=1 print "---------------------------" print tot_acc/i
save_interval = len(corpus.data_all['train']) // args.batch_size best_dev_score = -99999 iterations = args.epochs * len(corpus.data_all['train']) // args.batch_size print('max iterations: ' + str(iterations)) for iter in range(iterations): optimizer.zero_grad() data = corpus.get_batch(args.batch_size, 'train', div=True) output1, output2, _ = model(data) labels = data[2].cuda() if args.cuda else data[2] # predict only with matching score _, pred = output1.max(1) score = accuracy(labels, pred) loss1 = criterion(output1, labels) loss2 = criterion(output2, labels) #train via interploated loss of matching and ordering score loss = (1 - args.lamda) * loss1 + args.lamda * loss2 loss.backward() optimizer.step() optimizer.zero_grad() total_loss += float(loss.data) total_loss1 += float(loss1.data) total_loss2 += float(loss2.data) if iter % interval == 0: cur_loss = total_loss / interval if iter != 0 else total_loss
def main(unused_argv): train_dataset, validate_dataset, test_dataset = input.input( shuffle_files=False) #Text information info = tf.constant([ "Batch size = %s" % f.FLAGS.batch_size, "Epochs = %s" % f.FLAGS.num_epochs, "Learning rate = %s" % f.FLAGS.learning_rate, "Batch normalization = No", "Window size = %s" % f.FLAGS.window_size, "Shuffle Files = No", "CNN model = %s" % f.FLAGS.cnn_model, "Shuffle Samples = YES" ]) with tf.name_scope('input'): x = tf.placeholder(tf.float32, [ None, input.SAMPLE_DEPTH, input.SAMPLE_HEIGHT, input.SAMPLE_WIDTH ]) y_ = tf.placeholder(tf.float32, [None, 2]) dropout_rate = tf.placeholder(tf.float32) is_training = tf.placeholder(tf.bool) with tf.name_scope('logits'): if f.FLAGS.cnn_model == "lenet5": logits = lenet5.model_fn(sample_input=x, is_training=is_training, summaries=summaries) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=logits) mean_cross_entropy_loss = tf.reduce_mean(cross_entropy) loss_summ = tf.summary.scalar('Mean_cross_entropy_loss', mean_cross_entropy_loss) summaries['train'].append(loss_summ) #summaries['validate'].append(loss_summ) with tf.name_scope('adam_optimizer'): optimizer = tf.train.AdamOptimizer( f.FLAGS.learning_rate).minimize(mean_cross_entropy_loss) with tf.name_scope('accuracy'): preds = tf.argmax(logits, 1) correct_preds = tf.argmax(y_, 1) equal = tf.equal(preds, correct_preds) training_accuracy_op = tf.reduce_mean(tf.cast(equal, tf.float32)) summaries['train'].append( tf.summary.scalar('Training_Accuracy', training_accuracy_op)) with tf.name_scope('Evaluation_Metrics'): tp_op = evaluate.tp(logits=logits, labels=y_) fp_op = evaluate.fp(logits=logits, labels=y_) tn_op = evaluate.tn(logits=logits, labels=y_) fn_op = evaluate.fn(logits=logits, labels=y_) tp_sum = tf.placeholder(tf.float32) tn_sum = tf.placeholder(tf.float32) fp_sum = tf.placeholder(tf.float32) fn_sum = tf.placeholder(tf.float32) precision_op = evaluate.precision(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) accuracy_op = evaluate.accuracy(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) recall_op = evaluate.recall(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) fscore_op = evaluate.fscore(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) precision_summ = tf.summary.scalar('Precision', precision_op) accuracy_summ = tf.summary.scalar('Accuracy', accuracy_op) recall_summ = tf.summary.scalar('Recall', recall_op) fscore_summ = tf.summary.scalar('Fscore', fscore_op) summaries['validate'].append(accuracy_summ) summaries['validate'].append(precision_summ) summaries['validate'].append(recall_summ) summaries['validate'].append(fscore_summ) summaries['test'].append(accuracy_summ) summaries['test'].append(precision_summ) summaries['test'].append(recall_summ) summaries['test'].append(fscore_summ) print("Saving graph to %s" % f.FLAGS.log_dir) train_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/train") validate_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/validate") test_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/test") train_writer.add_graph(tf.get_default_graph()) train_summaries = tf.summary.merge(summaries['train']) validate_summaries = tf.summary.merge(summaries['validate']) test_summaries = tf.summary.merge(summaries['test']) with tf.Session() as sess: train_writer.add_summary(sess.run(tf.summary.text("Information", info))) train_iter = train_dataset.make_initializable_iterator() train_next_elem = train_iter.get_next() sess.run(tf.global_variables_initializer()) global_step = 0 display_freq = 10 validate_freq = 50 test_freq = 50 for epoch in range(1, f.FLAGS.num_epochs + 1): sess.run(train_iter.initializer) step_time = 0.0 fetch_time = 0.0 while True: try: a = time.time() global_step += 1 sample, label = sess.run(train_next_elem) fetch_time += time.time() - a #print (sample.shape, label.shape) #print (label) #for s in sample[0][0]: # print (s) a = time.time() _, summ = sess.run([optimizer, train_summaries], feed_dict={ x: sample, y_: label, dropout_rate: 0.5, is_training: True }) train_writer.add_summary(summ, global_step) step_time += time.time() - a except tf.errors.OutOfRangeError: break if global_step % display_freq == 0: batch_loss, batch_accuracy = sess.run( [mean_cross_entropy_loss, training_accuracy_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) print( "Epoch {:3}\t Step {:5}:\t Loss={:.3f}, \tTraining Accuracy={:.5f} \tStep Time {:4.2f}m, Fetch Time {:4.2f}m" .format(epoch, global_step, batch_loss, batch_accuracy, step_time / 60, fetch_time / 60)) step_time = 0.0 fetch_time = 0.0 #Validate and test after each epoch val_it = validate_dataset.make_one_shot_iterator() val_next_elem = val_it.get_next() tot_tp, tot_tn, tot_fp, tot_fn = 0, 0, 0, 0 while True: try: sample, label = sess.run(val_next_elem) tp, fp, tn, fn = sess.run( [tp_op, fp_op, tn_op, fn_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) except tf.errors.OutOfRangeError: break tot_tp += tp tot_fp += fp tot_fn += fn tot_tn += tn precision, recall, accuracy, fscore, summ = sess.run([ precision_op, recall_op, accuracy_op, fscore_op, validate_summaries ], feed_dict={ tp_sum: tot_tp, tn_sum: tot_tn, fp_sum: tot_fp, fn_sum: tot_fn }) validate_writer.add_summary(summ, global_step) print("Epoch %d, Step %d" % (epoch, global_step)) print("=" * 10, "Validating Results", "=" * 10) print("TP: %g\nTN: %g\nFP: %g\nFN: %g" % (tot_tp, tot_tn, tot_fp, tot_fn)) print( "\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g" % (precision, recall, fscore, accuracy)) test_it = test_dataset.make_one_shot_iterator() test_next_elem = test_it.get_next() tot_tp, tot_tn, tot_fp, tot_tn = 0, 0, 0, 0 while True: try: sample, label = sess.run(test_next_elem) tp, fp, tn, fn = sess.run( [tp_op, fp_op, tn_op, fn_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) except tf.errors.OutOfRangeError: break tot_tp += tp tot_fp += fp tot_fn += fn tot_tn += tn precision, recall, accuracy, fscore, summ = sess.run([ precision_op, recall_op, accuracy_op, fscore_op, test_summaries ], feed_dict={ tp_sum: tot_tp, tn_sum: tot_tn, fp_sum: tot_fp, fn_sum: tot_fn }) test_writer.add_summary(summ, global_step) print("=" * 10, "Testing Results", "=" * 10) print("TP: %g\nTN: %g\nFP: %g\nFN: %g" % (tot_tp, tot_tn, tot_fp, tot_fn)) print( "\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g" % (precision, recall, fscore, accuracy)) print("=" * 10, "===============", "=" * 10)
def execute(dataset, hmm_file, tag_file): # sys.stderr.write("loading learnt parameters...\n") hmm, tagset = hmm_utils.get_param_tagset(hmm_file, tag_file) # sys.stderr.write("reading dev data...\n") ts, test_tags = data_reader.read_tagging_data(dataset) test_sentences = replace_test(ts, hmm, tagset) i = 0 converges = 0 avg_iterations = 0 start_time = time.time() fst_acc = 0 best_acc = 0 wrong = 0 for sentence in test_sentences: if True: i += 1 truetags = test_tags[test_sentences.index(sentence)] sys.stderr.write("\n" + str(i) + "\n") # sys.stderr.write(' '.join(ts[i-1]) + "\n") print " ".join(sentence) # print ' '.join(ts[i-1]) best_tags, num_iterations, tags1, tags2 = dd_tagger_fst.run(sentence, tagset, hmm) if tags2 == best_tags: sys.stderr.write("YOU ARE WRONG!\n") wrong += 1 if num_iterations != -1: facc = evaluate.accuracy(truetags, tags2) # sys.stderr.write("fst tagger accuracy = " + str(facc) + "\n") fst_acc += facc bacc = evaluate.accuracy(truetags, best_tags) # sys.stderr.write("best tags accuracy = " + str(bacc) + "\n") best_acc += bacc sys.stderr.write("converges in " + str(num_iterations) + " iterations \n") converges += 1 avg_iterations += num_iterations else: sys.stderr.write("does not converge :(\n") print " ".join(best_tags) print " ".join(tags2) # print "gold : ", ' '.join(truetags) print # if i == 100: # break sys.stderr.write("\nsystem performance\n--------------------\n") sys.stderr.write("\n" + str(wrong * 100 / converges) + "% sequences are wrong:\n") sys.stderr.write("\naverage accuracy of best: " + str(best_acc / converges) + "\n") sys.stderr.write("average accuracy of 2nd best: " + str(fst_acc / converges) + "\n") sys.stderr.write("\nsystem efficiency\n---------------------\n") sys.stderr.write("\n" + str(avg_iterations / converges) + " iterations on average\n") sys.stderr.write(str(converges * 100 / i) + " % convergence\n") sys.stderr.write("time_taken = " + str(time.time() - start_time) + "\n")
def optimize(self, model, dataset): """Learn a model using batch gradient descent. Input ----- model : Python class Definition of model. dataset : tuple(tuple(np.ndarray)) Each tuple contains inputs and targets for training, validation and test data. Output ------ best_model_params: list(np.ndarray) List of all the model parameters best_valid_acc: float Best validation set accuracy. """ # Load training and validation data, and check for sparsity X_train, y_train = dataset[0] X_valid, y_valid = dataset[1] if self.eval_test == True: X_test, y_test = dataset[2] # Generate file name to save intermediate training models os.mkdir('.' + model.uid) temp_file_name = os.path.join('.' + model.uid, 'best_model.pkl') n_valid = len(X_valid) if self.eval_test == True: n_test = len(X_test) # Initialize learning rate and schedule learning_rate = self.learning_rate threshold = self.threshold if self.schedule == 'constant': rate_update = lambda coeff: self.learning_rate elif self.schedule == 'linear': rate_update = lambda coeff: self.learning_rate / (1 + coeff) elif self.schedule == 'exponential': rate_update = lambda coeff: self.learning_rate / \ 10**(coeff/self.threshold) elif self.schedule == 'power': rate_update = lambda coeff: self.learning_rate / \ (1 + coeff/self.threshold) elif self.schedule == 'inv-log': rate_update = lambda coeff: self.learning_rate / \ (1+np.log(coeff+1)) max_epoch = self.max_epoch validation_frequency = self.validation_frequency best_valid_acc = -1.0 if self.eval_test == True: best_test_acc = -1.0 cPickle.dump(model, open(temp_file_name, 'wb')) best_params = model.get_model_parameters() # Early stopping parameters and other checks patience = self.patience pissed_off = 0 for epoch in xrange(max_epoch): # Set effective momentum for the current epoch. effective_momentum = self.final_momentum \ if epoch > self.momentum_switchover \ else self.initial_momentum # Check if it's time to stop learning. if threshold == 0: if pissed_off == patience: # Exit and return best model model = cPickle.load(open(temp_file_name, 'rb')) best_params = model.get_model_parameters() print('Learning terminated after %d epochs.\n' % (epoch + 1)) break else: # Reload previous best model and continue pissed_off += 1 learning_rate = rate_update(pissed_off) threshold = self.threshold model = cPickle.load(open(temp_file_name, 'rb')) print( 'Re-initialising to previous best model with ' 'validation prediction accuracy %.3f.\n' '\tCurrent pissed off level: %d/%d.\n' '\tCurrent learning rate: %.4f.\n' % (best_valid_acc, pissed_off, patience, learning_rate)) RNG.seed(0xbeef) RNG.shuffle(X_train) RNG.seed(0xbeef) RNG.shuffle(y_train) costs = [] for X, y in zip(X_train, y_train): costs.append( model.train_function(X, y, learning_rate, effective_momentum)) mean_train_loss = np.mean(costs) print('Epoch %i/%i, train loss: %.3f bits' % (epoch + 1, max_epoch, mean_train_loss)) if (epoch + 1) % validation_frequency == 0: # Compute validation negative log-likelihood valid_pred = [] if self.eval_test == True: test_pred = [] for i in xrange(n_valid): valid_pred.append( # Don't pass labels model.predict_function(X_valid[i])) this_valid_acc = accuracy( np.concatenate(tuple(valid_pred), axis=0), np.concatenate(tuple(y_valid), axis=0)) if self.eval_test == True: for i in xrange(n_test): test_pred.append( # Don't pass labels model.predict_function(X_test[i])) this_test_acc = accuracy( np.concatenate(tuple(test_pred), axis=0), np.concatenate(tuple(y_test), axis=0)) print("\tValidation accuracy: %.3f (previous best: %.3f)" % (this_valid_acc, best_valid_acc)) if self.eval_test == True: print("\tTest accuracy: %.3f (previous best: %.3f)" % (this_test_acc, best_test_acc)) print "\tCurrent learning rate: %.4f" % (learning_rate) if this_valid_acc > best_valid_acc: best_valid_acc = this_valid_acc best_params = model.get_model_parameters() cPickle.dump(model, open(temp_file_name, 'wb')) threshold = self.threshold else: threshold -= 1 # Clean temporary files and folder os.remove(temp_file_name) os.rmdir('.' + model.uid) return best_params, best_valid_acc