def train(self, train_data, dev_data, epochs): """ This function trains the model. No need to change this function. """ acc = 0 best_weights = {} best_acc = 0 for n in range(epochs): shuffle(train_data) stdout.write("Epoch %u : " % (n + 1)) for ex in train_data: self.N += 1 self.estimate_ex(ex) sys_classes = self.classify(dev_data) acc, _ = evaluate(sys_classes, dev_data) sys_classes_1 = self.classify(train_data) t_acc, _ = evaluate(sys_classes_1, train_data) print("Train accuracy %.2f%%, Dev accuracy %.2f%%" % (t_acc, acc)) if acc > best_acc: best_weights = self.W best_acc = acc print('New best val accuracy model for these parameters') if t_acc > 99: print( 'Training accuracy preeetty good for now pls stop mister') self.W = best_weights return best_acc self.W = best_weights return best_acc
def test(self, test_set): real_seq = list() for seq in test_set[0]: real_seq.append(list(map(lambda x: self.idx2words[x], seq))) real_label = list() for seq in test_set[1]: real_label.append(list(map(lambda x: self.idx2labels[x], seq))) real_test_set = (real_seq, real_label) slot_predict = None intent_predict = None saver = tf.train.Saver() with tf.Session(config=self.config) as sess: saver.restore(sess, self.model_path) feed_dict, seq_len_list = self.get_feed_dict(test_set[0]) if self.slot_filling: if self.CRF: logits, transition_params = sess.run( [self.logits_slot, self.transition_params], feed_dict=feed_dict) slot_predicts = list() for logit, seq_len in zip(logits, seq_len_list): viterbi_seq, _ = viterbi_decode( logit[:seq_len], transition_params) slot_predicts.append(viterbi_seq) else: slot_predicts = sess.run(self.labels_softmax, feed_dict=feed_dict) slot_predict = list() for i in range(len(test_set[0])): seq_len = len(test_set[0][i]) predicted_seq = list( map(lambda x: self.idx2labels[x], slot_predicts[i][:seq_len])) slot_predict.append(predicted_seq) if self.intent_detection: intent_predicts = sess.run(self.intents_softmax, feed_dict=feed_dict) for i in range(len(test_set[0])): if test_set[2][i] == intent_predicts[i]: # TODO intent eval pass evaluate(real_test_set, slot_predict, intent_predict, self.error_example_output, self.true_example_output, self.slot_distinct)
def train(self, train_data, dev_data, mode, epochs): """ This function trains the model. No need to change this function. """ for n in range(epochs): shuffle(train_data) stdout.write("Epoch %u : " % (n + 1)) for ex in train_data: self.N += 1 self.estimate_ex(ex, mode) sys_classes = self.classify(dev_data, mode) acc, _ = evaluate(sys_classes, dev_data) sys_classes_1 = self.classify(train_data, mode) t_acc, _ = evaluate(sys_classes_1, train_data) print("Train accuracy %.2f%%, Dev accuracy %.2f%%" % (t_acc, acc)) # Let's not overfit too much :^) if t_acc >= 99.5: print('Training accuracy large enough, stopping training') break
def main(): train_utts = data.read_file("Brown_train_unseg.txt") lexicon = build_lexicon(train_utts[:]) test_utts = data.read_file("Brown_test_unseg.txt") goldsegs_train = data.get_goldsegs("Brown_train_gold.txt") goldsegs_test = data.get_goldsegs("Brown_test_gold.txt") for utts, goldsegs, title in ((train_utts, goldsegs_train, "Training"), (test_utts, goldsegs_test, "Testing")): joineds, segs = get_segpoints(lexicon, utts) stats = data.evaluate(goldsegs, segs) print(title) print("P: %s\tR: %s\t\tF1: %s\n" % tuple([round(stat * 100, 2) for stat in stats]))
def main(): train_utts = data.read_file("Brown_train_unseg.txt") probdict = get_bigramprobdict(train_utts) test_utts = data.read_file("Brown_test_unseg.txt") goldsegs_train = data.get_goldsegs("Brown_train_gold.txt") goldsegs_test = data.get_goldsegs("Brown_test_gold.txt") for utts, goldsegs, title in ((train_utts, goldsegs_train, "Training"), (test_utts, goldsegs_test, "Testing")): sylls = [data.tokenize_syllables(utt) for utt in utts] segs = get_segpoints(sylls, probdict) # Use data.apply_boundaries to helpvisualize the segmentations for debugging and answering # analysis questions stats = data.evaluate(goldsegs, segs) print(title) print("P: %s\tR: %s\t\tF1: %s\n" % tuple([round(stat * 100, 2) for stat in stats]))
def main(): train_utts = data.read_file("Brown_train_unseg.txt") train_tokenized = [data.tokenize_syllables(utt) for utt in train_utts] baselinesegs_train = baseline_segs(train_tokenized) test_utts = data.read_file("Brown_test_unseg.txt") test_tokenized = [data.tokenize_syllables(utt) for utt in test_utts] baselinesegs_test = baseline_segs(test_tokenized) goldsegs_train = data.get_goldsegs("Brown_train_gold.txt") goldsegs_test = data.get_goldsegs("Brown_test_gold.txt") for basesegs, goldsegs, title in ((baselinesegs_train, goldsegs_train, "Training"), (baselinesegs_test, goldsegs_test, "Testing")): stats = data.evaluate(goldsegs, basesegs) print(title) print("P: %s\tR: %s\t\tF1: %s\n" % tuple([round(stat * 100, 2) for stat in stats]))
pyp.ylim(0, 100) pyp.xlim(0, 100) pyp.scatter(outsT[0, :], outsT[1, :], 20) pyp.gca().set_aspect('equal', adjustable='box') r = np.corrcoef(outsT[0, :], outsT[1, :])[0, 1] print('R2 value is ' + str(r * r)) slope, intercept, r_value, p_value, std_err = stats.linregress( outsT[0, :], outsT[1, :]) line = slope * outsT[0, :] + intercept pyp.plot(outsT[0, :], line) stng = 'y = ' + str(slope)[:5] + 'x + ' + str(intercept)[:5] + '\nR2 = ' + str( r * r)[:5] pyp.text(15, 70, stng) # %% Apply to validation set as pre-test (outs, names) = data.evaluate(Model, 'val', sourceparam) # %% Plot validation results pyp.figure(figsize=(8, 8)) pyp.ylim(0, 100) pyp.xlim(0, 100) pyp.scatter(outs[0, :], outs[1, :], 20) pyp.gca().set_aspect('equal', adjustable='box') r = np.corrcoef(outs[0, :], outs[1, :])[0, 1] print('R2 value is ' + str(r * r)) slope, intercept, r_value, p_value, std_err = stats.linregress( outs[0, :], outs[1, :]) line = slope * outs[0, :] + intercept pyp.plot(outs[0, :], line) stng = 'y = ' + str(slope)[:5] + 'x + ' + str(intercept)[:5] + '\nR2 = ' + str( r * r)[:5]
def write(data, olution): name = file_name + '_' + str(evaluate(data, solution)) + output_extension path = join(output_solve_folder, name) write_data(path, solution)
def train(c): c.setdefault(hebbian=False) net = eval(c.model)(c) emb_params = count_params(net.embed) + count_params( net.loss.projections) + count_params(net.loss.clusters) opt = get_opt(c, net) net, opt, step = c.init_model(net, opt=opt, step='max', train=True) step_lr = scheduler(c, opt, step) if c.get('distill'): data_tr_distill = DistillationSampleIterator(c, c.train_batch) iter_tr_distill = iter(data_tr_distill) else: data_tr = SampleIterator(c, c.train_batch, split='valid' if c.debug else 'train') iter_tr = iter(data_tr) data_val = SequentialIterator(c, c.eval_batch, split='valid') s = Namespace(net=net, opt=opt, step=step) c.on_train_start(s) c.log('Embedding has %s parameters' % emb_params) if c.hebbian: counters = [ torch.ones(end - start, dtype=torch.long, device=c.device) for start, end in zip([0] + c.cutoffs, c.cutoffs + [c.n_vocab]) ] temp_counters = [torch.zeros_like(x) for x in counters] best_val_loss = np.inf if s.results is not None and 'val_loss' in s.results.columns: best_val_loss = s.results['val_loss'].dropna().max() try: while step < s.step_max: step_lr(step) t_s = time() if c.get('distill'): hard_labels, soft_labels, soft_probs = next(iter_tr_distill) hard_labels = to_torch(hard_labels, c.device).t() soft_labels = to_torch(soft_labels, c.device).permute(1, 0, 2)[1:] soft_probs = to_torch(soft_probs, c.device).permute(1, 0, 2)[1:] inputs, hard_labels = hard_labels[:-1], hard_labels[1:] preds = net(inputs=inputs, labels=hard_labels, soft_labels=soft_labels, soft_probs=soft_probs, current_step=step) else: x = to_torch(next(iter_tr), c.device).t() inputs, labels = x[:-1], x[1:] preds = net(inputs, labels) loss = preds['loss'] opt.zero_grad() if torch.isnan(loss): raise RuntimeError('Encountered nan loss during training') if c.opt_level == 'O0': loss.backward() else: with amp.scale_loss(loss, opt) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), c.get('clip_grad', 0.5)) opt.step() if c.hebbian: hebbian_weight_update(c, net, preds['hiddens'], counters, temp_counters) time_model = np.round(time() - t_s, 5) loss = from_torch(loss) perplexity = np.nan if loss > 5 else np.e**loss step_result = pd.Series( dict( loss=loss, perplexity=perplexity, time=time_model, )).add_prefix('train_') step_result['lr'] = next(iter(opt.param_groups))['lr'] if c.get('use_cache'): step_result['theta'] = from_torch(preds['theta']) step_result['lambda'] = from_torch(preds['lambda']) s.step = step = step + 1 if step % c.step_eval == 0: step_result = step_result.append( pd.Series(evaluate(c, data_val, net)).add_prefix('val_')) s.record_step = step_result['val_loss'] < best_val_loss clear_gpu_memory() s.step_result = step_result c.on_step_end(s) except Exception as e: import traceback err = traceback.format_exc() if c.main: c.log(err) else: print(err) finally: c.on_train_end(s)
if __name__ == '__main__': c = Config.from_args().setdefault(model='model.Transformer') evals = [x for x in ['valid', 'test'] if c.get(x)] if len(evals): net = eval(c.model)(c) net, step = c.init_model(net, step=c.get('step', 'max'), train=False) print('Model at step', step) emb_params = count_params(net.embed) + count_params( net.loss.projections) + count_params(net.loss.clusters) print('Model has %s parameters. Embedding has %s parameters' % (count_params(net), emb_params)) cache_search_path = c.res / ('cache_step%s_n%s.yaml' % (step, c.get('n_cache'))) if c.get('use_cache_search', True) and cache_search_path.exists(): for k in 'cache_theta_init', 'cache_lambda_init': if c.get(k): c.unvar(k) params = cache_search_path.load() c.var(**params) print('Loaded cache search parameters') print(params) for split in evals: data = SequentialIterator(c, c.eval_batch, split=split) print(split, evaluate(c, data, net)) else: train(c)
lambda_init = from_torch(state['loss.cache_lambda_inv_sigmoid'].sigmoid()) theta_init = from_torch(F.softplus(state['loss.cache_theta_inv_softplus'])) print('trained lambda', lambda_init) print('trained theta', theta_init) else: # initial cache parameters if we didn't train cache parameters lambda_init = 0.1 theta_init = 0.016 print('initial lambda', lambda_init) print('initial theta', theta_init) c.var(use_cache=True, n_cache=c.n_cache, cache_lambda=lambda_init, cache_theta=theta_init) ppl = evaluate(c, data, net)['perplexity'] lam, theta = lambda_init, theta_init lam_delta, theta_delta = lam / 20, theta / 20 print('Initial val PPL=%s lambda=%.3g (%.3g) theta=%.3g (%.3g)' % (ppl, lam, lam_delta, theta, theta_delta)) while True: ppl_plus = evaluate(c.var(cache_lambda=lam + lam_delta, cache_theta=theta), data, net)['perplexity'] ppl_minus = evaluate( c.var(cache_lambda=lam - lam_delta, cache_theta=theta), data, net)['perplexity'] new_ppl, new_lam = min((ppl_plus, lam + lam_delta), (ppl_minus, (lam - lam_delta))) if new_ppl < ppl: ppl, lam = new_ppl, new_lam lam_delta *= 1.2
feed_dict=trainBatch) if (i + 1) % 20 == 0: tfSummaryWriter.add_summary(summary, i + 1) if (i + 1) % 100 == 0: print( "it. {0}/{1} (lr={5:.2e},{4:.2e}), loss={2:.4f}, time={3:.4f}". format(i + 1, maxIterN, trainBatchLoss, time.time() - timeStart, currLearningRate[0], currLearningRate[1])) if (i + 1) % 2000 == 0: # update image summaries if imageSummaries is not None: summary = sess.run(imageSummaries, feed_dict=trainBatch) tfSummaryWriter.add_summary(summary, i + 1) # evaluate on validation and test sets testAccuracy = data.evaluate(testData, imageRawBatch, pInitBatch, labelBatch, prediction, sess, params) testError = (1 - testAccuracy) * 100 summary = sess.run(testErrorSummary, feed_dict={testErrorPH: testError}) tfSummaryWriter.add_summary(summary, i + 1) # save model savePath = tfSaver.save( sess, "models_{2}/{0}_it{1}k.ckpt".format(saveFname, (i + 1) // 1000, suffix)) print("model saved: {0}".format(savePath)) if (i + 1) % 10000 == 0: # save intermediate model tfSaverInterm.save( sess, "models_{2}/interm/{0}_it{1}k.ckpt".format( saveFname, (i + 1) // 1000, suffix))
imageRawBatch: trainData["image"][randIdx], labelBatch: trainData["label"][randIdx], learningRate: currLearningRate } # run one step _,trainBatchLoss,summary = sess.run([trainStep,loss,lossSummary],feed_dict=trainBatch) if (i+1)%20==0: tfSummaryWriter.add_summary(summary,i+1) if (i+1)%100==0: print("it. {0}/{1} (lr={5:.2e},{4:.2e}), loss={2:.4f}, time={3:.4f}" .format(i+1,maxIterN,trainBatchLoss,time.time()-timeStart,currLearningRate[0],currLearningRate[1])) if (i+1)%2000==0: # update image summaries if imageSummaries is not None: summary = sess.run(imageSummaries,feed_dict=trainBatch) tfSummaryWriter.add_summary(summary,i+1) # evaluate on validation and test sets testAccuracy = data.evaluate(testData,imageRawBatch,pInitBatch,labelBatch,prediction,sess,opt) testError = (1-testAccuracy)*100 summary = sess.run(testErrorSummary,feed_dict={testErrorPH: testError}) tfSummaryWriter.add_summary(summary,i+1) # save model savePath = tfSaver.save(sess,"models_{2}/{0}_it{1}k.ckpt".format(opt.model,(i+1)//1000,opt.group)) print("model saved: {0}".format(savePath)) if (i+1)%10000==0: # save intermediate model tfSaverInterm.save(sess,"models_{2}/interm/{0}_it{1}k.ckpt".format(opt.model,(i+1)//1000,opt.group)) # save final model tfSaverFinal.save(sess,"models_{1}/final/{0}.ckpt".format(opt.model,opt.group)) print("======= backpropagation done =======")
def validate(X, y, Z, actual): predicted = train_and_predict(X, y, Z) return data.evaluate(predicted, actual)
feed_dict=trainBatch) if (i + 1) % 10 == 0: tfSummaryWriter.add_summary(summary, i + 1) if (i + 1) % 100 == 0: print( "it. {0}/{1} (lr={5:.2e},{4:.2e}), loss={2:.4f}, time={3:.4f}". format(i + 1, maxIterN, trainBatchLoss, time.time() - timeStart, currLearningRate[0], currLearningRate[1])) if (i + 1) % 5000 == 0: # update image summaries if imageSummaries is not None: summary = sess.run(imageSummaries, feed_dict=trainBatch) tfSummaryWriter.add_summary(summary, i + 1) # evaluate on validation and test sets validAccuracy = data.evaluate(validData, imageRawBatch, labelBatch, prediction, sess, params) validError = (1 - validAccuracy) * 100 print('Iter {} Accuracy: {}'.format(i, validAccuracy)) if validAccuracy > best_validation_accuracy: best_validation_accuracy = validAccuracy else: params.baseLR = params.baseLR / 10 if params.baseLR <= 0.0001: params.baseLR = 0.0001 summary = sess.run(validSummary, feed_dict={validErrorPH: validError}) tfSummaryWriter.add_summary(summary, i + 1) # save model savePath = tfSaver.save( sess, "models_{2}/{0}_it{1}k.ckpt".format(saveFname, (i + 1) // 1000,
def train(c, net, compression_scheduler=None): import distiller.apputils as apputils from distiller.data_loggers import TensorBoardLogger, PythonLogger msglogger = apputils.config_pylogger('logging.conf', None) tflogger = TensorBoardLogger(msglogger.logdir) tflogger.log_gradients = True pylogger = PythonLogger(msglogger) c.setdefault(hebbian=False) emb_params = count_params(net.embed) + count_params(net.loss.projections) + count_params(net.loss.clusters) opt = get_opt(c, net) net, opt, step = c.init_model(net, opt=opt, step='max', train=True) step_lr = scheduler(c, opt, step) data_tr = SampleIterator(c, c.train_batch, split='valid' if c.debug else 'train') iter_tr = iter(data_tr) data_val = SequentialIterator(c, c.eval_batch, split='valid') s = Namespace(net=net, opt=opt, step=step) c.on_train_start(s) c.log('Embedding has %s parameters' % emb_params) if c.get("steps_per_epoch"): steps_per_epoch = c.steps_per_epoch else: steps_per_epoch = len(data_tr.tokens) // data_tr.bs // c.train_chunk print("#### steps per epoch %d ####" % steps_per_epoch) if c.hebbian: counters = [torch.ones(end - start, dtype=torch.long, device=c.device) for start, end in zip([0] + c.cutoffs, c.cutoffs + [c.n_vocab])] temp_counters = [torch.zeros_like(x) for x in counters] best_val_loss = np.inf if s.results is not None and 'val_loss' in s.results.columns: best_val_loss = s.results['val_loss'].dropna().max() try: while step < s.step_max: batch = step % steps_per_epoch epoch = step // steps_per_epoch if step % steps_per_epoch == 0: c.log("====> batch=%d, epoch=%d, step=%d" % (batch, epoch, step)) if compression_scheduler: compression_scheduler.on_epoch_begin(epoch) if compression_scheduler: compression_scheduler.on_minibatch_begin(epoch, minibatch_id=batch, minibatches_per_epoch=steps_per_epoch) step_lr(step) x = to_torch(next(iter_tr), c.device).t() t_s = time() inputs, labels = x[:-1], x[1:] preds = net(inputs, labels) loss = preds['loss'] if compression_scheduler: _ = compression_scheduler.before_backward_pass(epoch, minibatch_id=batch, minibatches_per_epoch=steps_per_epoch, loss=loss, return_loss_components=False) opt.zero_grad() if torch.isnan(loss): raise RuntimeError('Encountered nan loss during training') loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), c.get('clip_grad', 0.5)) opt.step() if c.hebbian: hebbian_weight_update(c, net, preds['hiddens'], counters, temp_counters) time_model = np.round(time() - t_s, 5) loss = from_torch(loss) perplexity = np.nan if loss > 5 else np.e ** loss step_result = pd.Series(dict( loss=loss, perplexity=perplexity, time=time_model )).add_prefix('train_') step_result['lr'] = next(iter(opt.param_groups))['lr'] if c.use_cache: step_result['theta'] = preds['theta'] step_result['lambda'] = preds['lambda'].item() if compression_scheduler: compression_scheduler.on_minibatch_end(epoch, minibatch_id=batch, minibatches_per_epoch=steps_per_epoch) if step % steps_per_epoch == 0: if compression_scheduler: compression_scheduler.on_epoch_end(epoch) s.step = step = step + 1 if step % c.step_eval == 0: distiller.log_weights_sparsity(net, epoch, loggers=[tflogger, pylogger]) t, total = distiller.weights_sparsity_tbl_summary(net, return_total_sparsity=True) c.log("total sparsity: %.3lf" % total) step_result = step_result.append( pd.Series(evaluate(c, data_val, net)).add_prefix('val_') ) s.record_step = step_result['val_loss'] < best_val_loss clear_gpu_memory() s.step_result = step_result c.on_step_end(s) except Exception as e: import traceback err = traceback.format_exc() if c.main: c.log(err) else: print(err) finally: c.on_train_end(s)
for l in [0.1, 0.01, 0.001, 1]: print(f'Trying with learning rate {l}') model = LogisticRegression(data["training"]) model.Lambda = l val_acc = model.train(data["training"], data["development.gold"], epochs) if val_acc > best_model[1]: best_model = (deepcopy(model), val_acc) print( f'Found new best model with lambda of {l} and val accuracy of {val_acc} ' ) if CLASSIFY_TEST_SET: print("Labeling test set with the best val accuracy model.") test_output = best_model[0].classify(data["test.input"]) acc, fscores = evaluate(test_output, data["test.gold"]) print("Final test accuracy: %.2f" % acc) print("Per class F1-fscore:") for c in fscores: print(" %s %.2f" % (c, fscores[c])) write_semeval(data["test.input"], test_output, output_file) print(f'Best val accuracy was {best_model[1]}') """ Write your code for analyzing model weights here. You can use the Python dict encoder which will be defined. """ model = best_model[0] for c in model.Y: for w in ['happy', 'great', 'awful', 'sucks']:
def train(c): import distiller net = Transformer(c) opt = get_opt(c, net) net, opt, step = c.init_model(net, opt=opt, step='max', train=True) step_lr = scheduler(c, opt, step) data_tr = SampleIterator(c, c.train_batch, split='valid' if c.debug else 'train') iter_tr = iter(data_tr) data_val = SequentialIterator(c, c.eval_batch, split='valid') data_test = SequentialIterator(c, c.eval_batch, split='test') print('Before quantization') tbl, sparsity = distiller.weights_sparsity_tbl_summary( net, return_total_sparsity=True) step_result = pd.Series(evaluate(c, data_val, net)).add_prefix('val_') step_result = step_result.append( pd.Series(evaluate(c, data_test, net)).add_prefix('test_')) step_result['sparsity'] = sparsity print(step_result) compression_scheduler = distiller.config.file_config(net, opt, c.compress) print('After initial quantization') s = Namespace(net=net, opt=opt, step=step) c.on_train_start(s) tbl, sparsity = distiller.weights_sparsity_tbl_summary( net, return_total_sparsity=True) step_result = pd.Series(evaluate(c, data_val, net)).add_prefix('val_') step_result = step_result.append( pd.Series(evaluate(c, data_test, net)).add_prefix('test_')) step_result['sparsity'] = sparsity print(step_result) npm = [] for name, param in net.named_parameters(): if param.dim() in [2, 4] and any(type in name for type in ['weight', 'bias']): npm.append((name, param, param.abs() == 0)) best_val_loss = np.inf if s.results is not None and 'val_loss' in s.results.columns: best_val_loss = s.results['val_loss'].dropna().max() try: steps_per_epoch = c.step_eval while step < s.step_max: epoch = step // steps_per_epoch batch = step % steps_per_epoch if batch == 0: compression_scheduler.on_epoch_begin(epoch) compression_scheduler.on_minibatch_begin(epoch, batch, steps_per_epoch) step_lr(step) x = to_torch(next(iter_tr), c.device).t() t_s = time() inputs, labels = x[:-1], x[1:] preds = net(inputs, labels) loss = preds['loss'] compression_scheduler.before_backward_pass(epoch, batch, steps_per_epoch, loss, False) opt.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), c.get('clip_grad', 0.5)) compression_scheduler.before_parameter_optimization( epoch, batch, steps_per_epoch, opt) opt.step() for name, param, mask in npm: param.data[mask] = 0 compression_scheduler.on_minibatch_end(epoch, batch, steps_per_epoch) if (batch + 1) == steps_per_epoch: compression_scheduler.on_epoch_end(epoch) time_model = np.round(time() - t_s, 5) loss = from_torch(loss) perplexity = np.nan if loss > 5 else np.e**loss step_result = pd.Series( dict( loss=loss, perplexity=perplexity, time=time_model, )).add_prefix('train_') step_result['lr'] = next(iter(opt.param_groups))['lr'] s.step = step = step + 1 if step % c.step_eval == 0: tbl, sparsity = distiller.weights_sparsity_tbl_summary( net, return_total_sparsity=True) step_result = step_result.append( pd.Series(evaluate(c, data_val, net)).add_prefix('val_')) step_result = step_result.append( pd.Series(evaluate(c, data_test, net)).add_prefix('test_')) step_result['sparsity'] = sparsity s.record_step = step_result['val_loss'] < best_val_loss clear_gpu_memory() s.step_result = step_result c.on_step_end(s) except Exception as e: import traceback err = traceback.format_exc() if c.main: c.log(err) else: print(err) finally: c.on_train_end(s) return net, step
data = read_20newsgroup_datasets(data_dir) shuffle(data["train"]) print("Extracting features.") extract_features(data) # You can explore the effect of lambda using the test set. lambdas = {10:0.1, 50:0.0001, 100:0.0001} for labeled_size in [10,50,100]: print("Experiment with %u labeled examples:" % labeled_size) labeled_train_data = data['train'][:labeled_size] unlabeled_train_data = data['train'][labeled_size:] print(" Training fully supervised model.") model = NaiveBayes(data['train']) model.train(labeled_train_data, unlabeled_train_data, semisupervised=0) test_output = model.classify(data["test"]) acc, fscores = evaluate(test_output, data["test"]) print(" Test accuracy: %.2f" % acc) print() print(" Training semi-supervised model.") model = NaiveBayes(data['train'],lambdas[labeled_size]) model.train(labeled_train_data, unlabeled_train_data, semisupervised=1) test_output = model.classify(data["test"]) acc, fscores = evaluate(test_output, data["test"]) print(" Test accuracy: %.2f" % acc) print()