def train(): logger.info('start...') if FLAGS.child_fixed_arc is None: images, labels = read_data(FLAGS.data_path) else: images, labels = read_data(FLAGS.data_path, num_valids=0) logger.info("Original Image Shape: {0}".format(images['train'].shape)) g = tf.Graph() with g.as_default(): ops = get_ops(images, labels) child_ops = ops["child"] generator_ops = ops["generator"] dc = DagController( num_cells=FLAGS.child_num_cells, num_layers=FLAGS.child_num_layers, cd_length=FLAGS.child_num_cells+2, opt_num=FLAGS.child_num_branches, path_pool_size=FLAGS.path_pool_size, k_init_selection_num=FLAGS.k_init_selection_num, k_best_selection_num=FLAGS.k_best_selection_num, max_generation=FLAGS.max_generation) if FLAGS.child_fixed_arc is None: dc.evolve_ops_dag(child_ops, generator_ops) else: dc.eval_dag_arc(child_ops)
def train_and_eval(model_dir, model_type, train_steps, train_file_name, valid_file_name, test_file_name, result_file): model_dir = tempfile.mkdtemp() if not model_dir else model_dir m = build_model(model_dir, model_type) # set num_epochs to None to get infinite stream of data. rf = codecs.open(result_file, mode='w', encoding='utf-8') session_config = tf.ConfigProto(allow_soft_placement=True) session_config.gpu_options.allow_growth = True with tf.Session(config=session_config) as sess: #m.train(input_fn=read_data(train_file_name, num_epochs=None, shuffle=True), steps=train_steps) m.train(input_fn=read_data_with_sampling(train_file_name, num_epochs=None, shuffle=True), steps=train_steps) eval_result = m.evaluate(input_fn=read_data(valid_file_name, num_epochs=1, shuffle=False), steps=None) print("model directory = %s" % model_dir) for key in sorted(eval_result): print("%s: %s" % (key, eval_result[key])) predictions = m.predict(input_fn=read_data(test_file_name, num_epochs=1, shuffle=False), predict_keys="classes") predictions = list(predictions) for p in predictions: rf.write(str(p["classes"][0], encoding='utf-8')) rf.write("\n")
def load_data(): print("Loading data from %s" % FLAGS.data_dir) nl_extention = ".ids%d.nl" % FLAGS.nl_vocab_size cm_extension = ".ids%d.cm" % FLAGS.cm_vocab_size nl_txt_train = os.path.join(data_dir, "train") + ".nl" cm_txt_train = os.path.join(data_dir, "train") + ".cm" nl_txt_dev = os.path.join(data_dir, "dev") + ".nl" cm_txt_dev = os.path.join(data_dir, "dev") + ".cm" nl_txt_test = os.path.join(data_dir, "test") + ".nl" cm_txt_test = os.path.join(data_dir, "test") + ".cm" nl_train = os.path.join(data_dir, "train") + nl_extention cm_train = os.path.join(data_dir, "train") + cm_extension nl_dev = os.path.join(data_dir, "dev") + nl_extention cm_dev = os.path.join(data_dir, "dev") + cm_extension nl_test = os.path.join(data_dir, "test") + nl_extention cm_test = os.path.join(data_dir, "test") + cm_extension train_set = data_utils.read_data(nl_txt_train, cm_txt_train, nl_train, cm_train, None, FLAGS.max_train_data_size) dev_set = data_utils.read_data(nl_txt_dev, cm_txt_dev, nl_dev, cm_dev, None) test_set = data_utils.read_data(nl_txt_test, cm_txt_test, nl_test, cm_test, None) return train_set, dev_set, test_set
def load_data(self, debug=False): """Loads train/valid/test data and sentence encoding""" en_train, fr_train, en_dev, fr_dev, en_vocab_path, fr_vocab_path = data_utils.prepare_data( 'tmp', 40000, 40000) self.source_vocab_to_id, self.source_id_to_vocab = data_utils.initialize_vocabulary( en_vocab_path) self.target_vocab_to_id, self.target_id_to_vocab = data_utils.initialize_vocabulary( fr_vocab_path) source_path = './tmp/train.ids40000.questions' target_path = './tmp/train.ids40000.answers' if self.config.train_mode: source_path = './tmp/train.ids40000.questions' target_path = './tmp/train.ids40000.answers' sources, targets = data_utils.read_data(source_path, target_path) else: source_path = './tmp/test.ids40000.questions' target_path = './tmp/test.ids40000.answers' sources, targets = data_utils.read_data(source_path, target_path) self.train, self.valid, self.max_t_len, self.max_input_len, self.max_sen_len = data_utils.pad_length_bucket( sources, targets, self.config) source_vocab_path = './tmp/vocab40000.questions' target_vocab_path = './tmp/vocab40000.answers' self.source_vocab_size = data_utils.get_vocab_size(source_vocab_path) self.target_vocab_size = data_utils.get_vocab_size(target_vocab_path) self.word_embedding = np.random.uniform( -self.config.embedding_init, self.config.embedding_init, (self.source_vocab_size, self.config.embed_size))
def main(argv=None): train_data, user_size, item_size = data_utils.read_data( FLAGS.dataset, FLAGS.embed_size, FLAGS.batch_size, True) test_data, user_size, item_size = data_utils.read_data( FLAGS.dataset, FLAGS.embed_size, FLAGS.batch_size, False) train(train_data, test_data, user_size, item_size)
def decode(): config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # Load test data. print("Reading data in %s" % FLAGS.data_dir) test_set = None if FLAGS.decode_train: test_set = data_utils.read_data(FLAGS.data_dir, 'train') else: test_set = data_utils.read_data(FLAGS.data_dir, 'test') # Create model and load parameters. model = create_model(sess, test_set, True) model.batch_size = 1 # We decode one sentence at a time. test_set.pad(test_set.rank_list_size, model.hparams.reverse_input) rerank_scores = [] # Decode from test data. for i in xrange(len(test_set.initial_list)): encoder_inputs, embeddings, decoder_targets, target_weights, target_initial_scores = model.get_data_by_index( test_set.initial_list, test_set.gold_list, test_set.gold_weights, test_set.initial_scores, test_set.features, i) _, test_loss, output_logits, summary = model.step( sess, encoder_inputs, embeddings, decoder_targets, target_weights, target_initial_scores, True) #The output is a list of rerank index for decoder_inputs (which represents the gold rank list) rerank_scores.append(output_logits[0][0]) if i % FLAGS.steps_per_checkpoint == 0: print("Decoding %.2f \r" % (float(i) / len(test_set.initial_list))), #get rerank indexes with new scores rerank_lists = [] for i in xrange(len(rerank_scores)): scores = rerank_scores[i] rerank_lists.append( sorted(range(len(scores)), key=lambda k: scores[k], reverse=True)) if FLAGS.decode_train: data_utils.output_ranklist(test_set, rerank_lists, FLAGS.test_dir, model.hparams.reverse_input, 'train') else: data_utils.output_ranklist(test_set, rerank_lists, FLAGS.test_dir, model.hparams.reverse_input, 'test') return
def load_data(self, debug=False): """Loads train/valid/test data and sentence encoding""" ''' en_train, fr_train, en_dev, fr_dev, _, _ = data_utils.prepare_data( FLAGS.data_dir, FLAGS.en_vocab_size, FLAGS.fr_vocab_size) ''' en_train, fr_train, en_dev, fr_dev, en_vocab_path, fr_vocab_path = data_utils.prepare_data( 'tmp', 40000, 40000) self.source_vocab_to_id, self.source_id_to_vocab = data_utils.initialize_vocabulary( en_vocab_path) self.target_vocab_to_id, self.target_id_to_vocab = data_utils.initialize_vocabulary( fr_vocab_path) #print self.source_vocab_to_id #print self.source_id_to_vocab ''' print self.target_vocab_to_id print self.target_id_to_vocab ''' ''' for i in range(0, 10): print i print self.target_id_to_vocab[int(float(i))] #adsfas ''' source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.questions' target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.answers' if self.config.train_mode: source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.questions' target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.answers' sources, targets = data_utils.read_data(source_path, target_path) else: source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/test.ids40000.questions' target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/test.ids40000.answers' sources, targets = data_utils.read_data(source_path, target_path) self.train, self.valid, self.max_t_len, self.max_input_len, self.max_sen_len = data_utils.pad_length_bucket( sources, targets, self.config) source_vocab_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/vocab40000.questions' target_vocab_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/vocab40000.answers' self.source_vocab_size = data_utils.get_vocab_size(source_vocab_path) self.target_vocab_size = data_utils.get_vocab_size(target_vocab_path) self.word_embedding = np.random.uniform( -self.config.embedding_init, self.config.embedding_init, (self.source_vocab_size, self.config.embed_size))
def train(): print "Preparing data in %s" % settings.data_dir sr_train_ids_path, tg_train_ids_path,sr_dev_ids_path, tg_dev_ids_path,sr_vocab_path, tg_vocab_path = data_utils.prepare_data(settings.data_dir) print "Reading training data from %s" % settings.data_dir train_set = data_utils.read_data(sr_train_ids_path,tg_train_ids_path,settings.max_train_num) train_batches,train_bucket_ids = data_utils.batchize(train_set) print "Reading development data from %s" % settings.data_dir dev_set = data_utils.read_data(sr_dev_ids_path,tg_dev_ids_path) dev_batches,dev_bucket_ids = data_utils.batchize(dev_set,False) log_file = open(settings.train_dir+'log.txt','w') log_file.write('epoch\tstep\ttime\ttrain-ppx\tdev-ppx\n') log_file.flush() with tf.Session() as sess: print("Creating %d layers of %d units." % (settings.num_layers, settings.size)) model = create_model(sess, False) current_epoch,current_step,train_loss = 0,0,0.0 start_time = time.time() while True: current_epoch+=1 for batch_id in xrange(len(train_batches)): current_step+=1 step_start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.preprocess_batch(train_batches[batch_id], train_bucket_ids[batch_id]) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, train_bucket_ids[batch_id], False) print "global-step %d\tstep-time %.2f\tstep-loss %.2f" % (model.global_step.eval(),time.time()-step_start_time,step_loss) train_loss+=step_loss/settings.steps_per_checkpoint if current_step % settings.steps_per_checkpoint == 0: # evaluate in training set train_ppx = math.exp(train_loss)/model.batch_size if train_loss < 300 else float('inf') # evaluate in development set dev_loss=0.0 for dev_batch_id in xrange(len(dev_batches)): encoder_inputs, decoder_inputs, target_weights = model.preprocess_batch(dev_batches[dev_batch_id], dev_bucket_ids[dev_batch_id]) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, dev_bucket_ids[dev_batch_id], True) dev_loss+=step_loss/len(dev_batches) dev_ppx = math.exp(dev_loss)/model.batch_size if dev_loss < 300 else float('inf') log_file.write("%d\t%d\t%.2f\t%.2f\t%.2f\n" % (current_epoch,model.global_step.eval(),time.time()-start_time,train_ppx,dev_ppx)) log_file.flush() sys.stdout.flush() train_loss,dev_loss = 0.0,0.0 checkpoint_path = os.path.join(settings.train_dir, "summary.ckpt") model.saver.save(sess, checkpoint_path,global_step=model.global_step) train_batches,train_bucket_ids = data_utils.batchize(train_set)
def get_copy_dict(vocab): lines = du.read_data(filepaths['copypp']) ids = du.convert_to_token_ids(lines, vocab) copy_dict = {} for line in ids: copy_dict[line[0]] = line[1:] return copy_dict
def parse_data(data_dir, ti='train', tp='train', rank_cut=10, target='./'): train_session = '' train_size = '' train_svm = '' train_rank = '' print("Reading data in %s" % data_dir) train_set = data_utils.read_data(data_dir, ti, rank_cut) l = len(train_set.initial_list) fout1 = open(target + 'letor.%s' % tp, 'w') fout2 = open(target + 'letor.%s.query' % tp, 'w') fout3 = open(target + 'letor.%s.svm' % tp, 'w') fout4 = open(target + 'letor.%s.rank' % tp, 'w') rg = 1 qid = 0 for r in range(rg): ser = 0 for i in range(l): if i % 1000 == 0: print(i, l, qid) train_size += str(len(train_set.initial_list[i])) + '\n' gold_label_list = [ 0 if train_set.initial_list[i][x] < 0 else train_set.gold_weights[i][x] for x in xrange(len(train_set.initial_list[i])) ] click_list, _, _ = click_model.sampleClicksForOneList( list(gold_label_list)) while sum(click_list) == 0: click_list, _, _ = click_model.sampleClicksForOneList( list(gold_label_list)) for s in range(len(click_list)): feat_str = '' hit = 0 for cnt, f in enumerate(train_set.features[ser + s]): if f != 0: hit += 1 feat_str += ' ' + str(cnt + 1) + ':' + str(f) if hit == 0: print(feat_str) return train_session += str(click_list[s]) + feat_str + '\n' train_svm += str( click_list[s]) + ' qid:' + str(qid) + feat_str + '\n' train_rank += str(s) + '\n' qid += 1 ser += len(click_list) fout1.write(train_session) fout2.write(train_size) fout3.write(train_svm) fout4.write(train_rank) train_size = '' train_session = '' train_svm = '' train_rank = '' fout1.close() fout2.close() fout3.close() fout4.close() return train_set
def main(unused_argv): os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' if not os.path.isdir(args.train_output_dir): tf.logging.info("Path {} does not exist. Creating.".format( args.train_output_dir)) os.makedirs(args.train_output_dir) if not os.path.isdir(args.model_dir): tf.logging.info("Path {} does not exist. Creating.".format( args.model_dir)) os.makedirs(args.model_dir) if not os.path.isdir(args.test_dir): tf.logging.info("Path {} does not exist. Creating.".format( args.test_dir)) os.makedirs(args.test_dir) images, labels = read_data(args.data) g1 = tf.get_default_graph() M = Model(images, labels, args, g1) M.generate_ops() num_valid = len(images['valid']) num_batch = (num_valid + args.train_valid_batch_size - 1) // args.train_valid_batch_size num_train_examples = np.shape(images["train"])[0] batch_size = args.batch_size tf.logging.info("batch_size is {}".format(batch_size)) num_train_batches = (num_train_examples + batch_size - 1) // batch_size start_time = time.time() with g1.as_default(): GA = Game(M) saver = tf.train.Saver(max_to_keep=10) checkpoint_saver_hook = tf.train.CheckpointSaverHook( args.train_output_dir, save_steps=num_train_batches * args.num_epochs_per_iter, saver=saver) hooks = [checkpoint_saver_hook] if args.sync_replicas: sync_replicas_hook = M.optimizer.make_session_run_hook(True) hooks.append(sync_replicas_hook) tf.logging.info("-" * 80) tf.logging.info("Starting session") config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True with tf.train.SingularMonitoredSession( config=config, hooks=hooks, checkpoint_dir=args.train_output_dir) as sess: GA.sample_and_train(sess)
def train(): params = get_child_model_params() images, labels = read_data(params['data_dir'], num_valids=0) g = tf.Graph() with g.as_default(): ops = get_ops(images, labels, params) saver = tf.train.Saver(max_to_keep=30) checkpoint_saver_hook = tf.train.CheckpointSaverHook( params['model_dir'], save_steps=ops["num_train_batches"], saver=saver) hooks = [checkpoint_saver_hook] if params['sync_replicas']: sync_replicas_hook = ops["optimizer"].make_session_run_hook(True) hooks.append(sync_replicas_hook) tf.logging.info("-" * 80) tf.logging.info("Starting session") config = tf.ConfigProto(allow_soft_placement=True) with tf.train.SingularMonitoredSession( config=config, hooks=hooks, checkpoint_dir=params['model_dir']) as sess: start_time = time.time() while True: run_ops = [ ops["loss"], ops["lr"], ops["grad_norm"], ops["train_acc"], ops["train_op"], ] loss, lr, gn, tr_acc, _ = sess.run(run_ops) global_step = sess.run(ops["global_step"]) if params['sync_replicas']: actual_step = global_step * params['num_aggregate'] else: actual_step = global_step epoch = actual_step // ops["num_train_batches"] curr_time = time.time() if global_step % 50 == 0: log_string = "" log_string += "epoch={:<6d}".format(epoch) log_string += "ch_step={:<6d}".format(global_step) log_string += " loss={:<8.6f}".format(loss) log_string += " lr={:<8.4f}".format(lr) log_string += " |g|={:<8.4f}".format(gn) log_string += " tr_acc={:<3d}/{:>3d}".format( tr_acc, params['batch_size']) log_string += " mins={:<10.2f}".format( float(curr_time - start_time) / 60) tf.logging.info(log_string) if actual_step % ops["eval_every"] == 0: ops["eval_func"](sess, "test") if epoch >= params['num_epochs']: tf.logging.info('Training finished!') break
def test_serialize_deserialize() -> None: unlabeled_data = data_utils.read_data("test_data/56980685061237.npz") labels = data_utils.read_labels("test_data/labels.csv") data = data_utils.label_data(unlabeled_data, labels) for training_point in data_utils.generate_training_points(data): serialized = trainer.serialize(training_point) inputs, outputs = trainer.deserialize(serialized) assert set(inputs.keys()) == set(trainer.INPUTS_SPEC.keys()) assert set(outputs.keys()) == set(trainer.OUTPUTS_SPEC.keys())
def main(): # 获得 训练集,验证集,测试集 tuple_data_ = reader.read_data( 'F:/Datas/msr_training/msr_training.utf8.ic', target_dict={ 'B': 0, 'M': 1, 'E': 2, 'S': 3 }) # 根据模型规模config{small,medium,large,or test} # 获得2套模型参数,参数封装在config类的属性当中 config = Config() eval_config = Config() eval_config.batch_size = 1 eval_config.num_steps = 1 with tf.Graph().as_default(), tf.Session() as session: # 创建一个初始化的容器 initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) # 分别创建训练模型,验证模型,测试模型 # reuse=None 不使用以往参数 with tf.variable_scope("model", reuse=None, initializer=initializer): m = CTBModel(is_training=True, config=config) # 使用以往的参数 with tf.variable_scope("model", reuse=True, initializer=initializer): mtest = CTBModel(is_training=False, config=eval_config) tf.global_variables_initializer().run() for i in range(config.max_max_epoch): # 0.5**(0,..,0 and 1 and 2,.. ) lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0) # 学习率在max_epoch之后开始随迭代次数下降 m.assign_lr(session, config.learning_rate * lr_decay) print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr))) train_perplexity, accuracy = run_epoch(session, m, tuple_data_, m.train_op, verbose=True) print("Epoch: %d Train Perplexity: %.3f, Train Accuracy: %.3f%%" % (i + 1, train_perplexity, accuracy)) print("----------- Begin to Test!----------") test_perplexity, accuracy = run_epoch(session, mtest, tuple_data_, tf.no_op(), verbose=True) print("Test Perplexity: %.3f , Test Accuracy:%.3f%%" % (test_perplexity, accuracy)) saver = tf.train.Saver() save_path = saver.save(session, "./CTB_Model/CTB_Variables.ckpt") print("Save to path: ", save_path)
def main(_): if not FLAGS.data_dir: raise ValueError("Must set --data_dir to data directory") vocab_path = data_utils.create_vocabulary(os.path.join(FLAGS.data_dir, 'train'), FLAGS.data_dir) train_data = data_utils.read_data(os.path.join(FLAGS.data_dir, 'train'), vocab_path) valid_data = data_utils.read_data(os.path.join(FLAGS.data_dir, 'dev'), vocab_path) test_data = valid_data config = get_config() eval_config = get_config() eval_config.batch_size = 1 eval_config.num_steps = 1 with tf.Graph().as_default(), tf.Session() as session: initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) with tf.variable_scope("model", reuse=None, initializer=initializer): m = PTBModel(is_training=True, config=config) with tf.variable_scope("model", reuse=True, initializer=initializer): mvalid = PTBModel(is_training=False, config=config) mtest = PTBModel(is_training=False, config=eval_config) ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir) if ckpt and gfile.Exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) m.saver.restore(session, ckpt.model_checkpoint_path) else: print("Created model with fresh parameters.") tf.initialize_all_variables().run() for i in range(config.max_max_epoch): lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0) m.assign_lr(session, config.learning_rate * lr_decay) print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr))) train_perplexity = run_epoch(session, m, train_data, m.train_op, verbose=True) print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity)) valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op()) print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity)) test_perplexity = run_epoch(session, mtest, test_data, tf.no_op()) print("Test Perplexity: %.3f" % test_perplexity)
def make_weights(name_dataset): sensitive_words, sensitive_general_words, sensitive_extra_words = get_sensitive_words( ) sensitive_words = list(set(sensitive_words)) data, split = read_data(name_dataset) X = data["text"].apply(clean_text).values label_encoder = LabelEncoder().fit(data["label"].values) y = label_encoder.transform(data["label"].values) X = X[np.concatenate([split[0], split[1]])] y = y[np.concatenate([split[0], split[1]])] Z, idxs_sens = get_tfidf(X, sensitive_words) # obtaining the weights clf = RandomForestClassifier(n_estimators=1000, max_depth=27, random_state=233, n_jobs=14, criterion='entropy') y_pred = cross_val_predict(clf, Z[idxs_sens], y[idxs_sens], cv=250, n_jobs=1, method='predict_proba') print('Refit log loss: %.5f' % (log_loss(y[idxs_sens], y_pred[:, 1]))) p1 = sum(y[idxs_sens]) / len(y[idxs_sens]) p0 = 1 - p1 print(roc_auc_score(to_categorical(y[idxs_sens]), y_pred)) print(accuracy_score(y[idxs_sens], np.argmax(y_pred, 1)), max(p0, p1)) p1 = sum(y) / len(y) p0 = 1 - p1 propensity = np.array([(p1 if y[i] == 1 else p0) for i in range(len(y))]) propensity[idxs_sens] = np.array( [y_pred[i, y[idxs_sens[i]]] for i in range(len(idxs_sens))]) np.save(dir_processed + "propensity_%s.npy" % name_dataset, propensity) # propensity = np.load(dir_processed + "propensity_%s.npy" % name_dataset) weights = 1 / propensity a = np.mean( np.array([weights[i] for i in range(len(weights)) if y[i] == 0])) b = np.mean( np.array([weights[i] for i in range(len(weights)) if y[i] == 1])) print((1 / a) / (1 / a + 1 / b), (1 / b) / (1 / a + 1 / b)) weights = np.array([(weights[i] / a if y[i] == 0 else weights[i] / b) for i in range(len(weights))]) weights /= weights.mean() ret = np.zeros(len(data)) ret[np.concatenate([split[0], split[1]])] = weights np.save(dir_processed + "weights.npy", ret)
def main(): tf.set_random_seed(1234) images, labels = read_data('./data/cifar10', 0.5) train_dataset = tf.data.Dataset.from_tensor_slices( (images["train"], labels["train"])) train_dataset = train_dataset.shuffle(100).batch(16) train_iter = train_dataset.make_initializable_iterator() x_train, y_train = train_iter.get_next() # x_train=images["train"][:16] # y_train=labels["train"][:16] logits, train_loss = Model_test(x_train, y_train, True) w_var = utils.get_var(tf.trainable_variables(), 'weight_var')[1] arch_var = utils.get_var(tf.trainable_variables(), 'arch_var')[1] _, unrolled_train_loss = Model_test(x_train, y_train, True, "unrolled_weight_var") unrolled_w_var = utils.get_var(tf.trainable_variables(), 'unrolled_weight_var')[1] cpoy_weight_opts = [v_.assign(v) for v_, v in zip(unrolled_w_var, w_var)] with tf.control_dependencies(cpoy_weight_opts): unrolled_optimizer = tf.train.GradientDescentOptimizer(0.001) unrolled_optimizer = unrolled_optimizer.minimize( unrolled_train_loss, var_list=unrolled_w_var) #w' with tf.control_dependencies([unrolled_optimizer]): valid_grads = tf.gradients(unrolled_train_loss, unrolled_w_var) R = 0.01 / tf.global_norm(valid_grads) #Original Implementation # opts=[v.assign(v+R*g) for v,g in zip(w_var,valid_grads)] # with tf.control_dependencies(opts): # arch_grad_after=tf.gradients(train_loss,arch_var) optimizer1 = tf.train.GradientDescentOptimizer(R) optimizer1 = optimizer1.apply_gradients(zip(valid_grads, w_var)) with tf.control_dependencies([optimizer1]): arch_grad_after = tf.gradients(train_loss, arch_var) config = tf.ConfigProto() os.environ["CUDA_VISIBLE_DEVICES"] = str(0) config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) sess.run([train_iter.initializer]) print(sess.run(valid_grads)[0]) start = time.time() print(sess.run(arch_grad_after)[0]) print("time_is {}".format(time.time() - start)) print(sess.run(valid_grads)[0])
def train(stock_no, start, end, x_window_size, y_window_size, split_ratio, batch_size, layer_num, hidden_dim, nb_epoch): data, scaler_adjclose, scaler_volume = data_utils.read_data( stock_no, start, end) X, y = data_utils.window_transform_series(data, x_window_size=x_window_size, y_window_size=y_window_size) X_train, X_test, y_train, y_test = data_utils.tts(X, y, split_ratio) filepath = model_utils.model_path(stock_no) model_utils.train(X_train, y_train, X_test, y_test, x_window_size, y_window_size, split_ratio, batch_size, layer_num, hidden_dim, nb_epoch, filepath)
def test(): params = get_child_model_params() images, labels = read_data(params['data_dir'], num_valids=0) g = tf.Graph() with g.as_default(): ops = get_ops(images, labels, params) tf.logging.info("-" * 80) tf.logging.info("Starting session") config = tf.ConfigProto(allow_soft_placement=True) with tf.train.SingularMonitoredSession( config=config, checkpoint_dir=params['model_dir']) as sess: ops["eval_func"](sess, "test")
def data_generator(batch_size): data = read_data(batch_size) iterator = data.make_initializable_iterator() batch_images, batch_outputs, batch_categorys = iterator.get_next() with tf.Session() as sess: sess.run(iterator.initializer) for i in xrange(800000): images, label, category = sess.run( [batch_images, batch_outputs, batch_categorys]) im, hmp, c = gen_traindata(images, label, category) message.put([i, im, hmp, c]) message.put(None)
def main(): click_model_json_file = sys.argv[1] data_dir = sys.argv[2] output_file = sys.argv[3] print("Load data from " + data_dir) train_set = data_utils.read_data(data_dir, 'train') click_model = None with open(click_model_json_file) as fin: model_desc = json.load(fin) click_model = CM.loadModelFromJson(model_desc) print("Estimating...") estimator = RandomizedPropensityEstimator() estimator.estimateParametersFromModel(click_model, train_set) print("Output results...") estimator.outputEstimatorToFile(output_file)
def test_ctrl(): # obtain datasets t = time.time() images, labels = read_data() t = time.time() - t print('read dataset consumes %.2f sec' % t) # config of a model class_num = 10 child_num_layers = 6 out_channels = 32 batch_size = 32 device = 'gpu' epoch_num = 4 # files to print sampled archs child_filename = 'child_file.txt' ctrl_filename = 'controller_file.txt' child_file = open(child_filename, 'w') ctrl_file = open(ctrl_filename, 'w') # create a controller ctrl = Controller(child_num_layers=child_num_layers) # create a child, set epoch to 1; later this will be moved to an over epoch child = Child(images, labels, class_num, child_num_layers, out_channels, batch_size, device, 1) print(len(list(child.net.graph))) # print(child.net.graph) # train multiple epochs for _ in range(epoch_num): # sample an arch ctrl.ctrl.net_sample() sample_arch = ctrl.ctrl.sample_arch print_sample_arch(sample_arch, child_file) # train a child model # t = time.time() # child.train(sample_arch) # t = time.time() - t # print('child training time %.2f sec' % t) # train controller t = time.time() ctrl.train(child, ctrl_file) t = time.time() - t print('ctrller training time %.2f sec' % t)
def main(_): if not FLAGS.data_path: raise ValueError("Must set --data_path to PTB data directory") raw_data = data_utils.read_data(FLAGS.data_path) train_data, valid_data, test_data, _ = raw_data config = get_config() eval_config = get_config() eval_config.batch_size = 1 eval_config.num_steps = 1 with tf.Graph().as_default(), tf.Session() as session: initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) with tf.variable_scope("model", reuse=None, initializer=initializer): m = PTBModel(is_training=True, config=config) with tf.variable_scope("model", reuse=True, initializer=initializer): mvalid = PTBModel(is_training=False, config=config) mtest = PTBModel(is_training=False, config=eval_config) tf.initialize_all_variables().run() for i in range(config.max_max_epoch): lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0) m.assign_lr(session, config.learning_rate * lr_decay) print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr))) train_perplexity = run_epoch(session, m, train_data, m.train_op, verbose=True) print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity)) valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op()) print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity)) test_perplexity = run_epoch(session, mtest, test_data, tf.no_op()) print("Test Perplexity: %.3f" % test_perplexity)
def do_test(config_file): """ do the tests by using configuration file :param config_file: :return: """ config = utils.load_json_data(config_file) partial_to_saturation_col = None if 'partial_to_saturation_col' not in config \ else config['partial_to_saturation_col'] x = du.read_data(config['data_file'], sep='\t' if 'sep' not in config else config['sep'], column_mapping=config['mapping'], partial_to_saturation_col=partial_to_saturation_col, hdf=False if 'hdf' not in config else config['hdf']) if 'binary_columns_to_impute' in config: # impute binary columns logging.info('binary columns to impute [%s]' % config['binary_columns_to_impute']) imputer = BinaryImputator() x = imputer.impute(x, config['binary_columns_to_impute']) if 'comorbidity_cols' in config: populate_col_by_or(x, config['comorbidity_cols'], new_col_name='comorbidity') model_files = config['model_files'] for outcome in config['outcomes']: logging.info('testing for outcome [{0}] with #{1} models'.format(outcome, len(model_files))) result_file = '{0}/{1}_result.tsv'.format(config['result_tsv_folder'], outcome) test_models_and_ensemble(model_files, x, weights=config['weights'][outcome] if 'weights' in config else None, outcome=outcome, threshold=config['threshold'], result_csv=result_file, severity_conf=None if 'severity_scores' not in config else config['severity_scores'], generate_figs=False if 'generate_figs' not in config else config['generate_figs'], auc_fig_file=None if 'auc_fig_file_pattern' not in config else (config['auc_fig_file_pattern'] % outcome), calibration_fig_file=None if 'calibration_fig_file_pattern' not in config else (config['calibration_fig_file_pattern'] % outcome), event_rate=None if 'event_rate' not in config else config['event_rate'], nri_json=None if 'nri_json' not in config else config['nri_json'] ) logging.info('result saved to {0}'.format(result_file))
def main(): config = model_config.Config() BATCH_SIZE = config.batch_size data = data_utils.read_data(config.corpus_path) data_utils.build_vocab(config.corpus_path, config.vocab_path) vocab = data_utils.read_dictionary() # random.shuffle(data) train_data = data[BATCH_SIZE:] test_data = data[:BATCH_SIZE] model = BiLSTM_CRF_Model(config=config, vocab=vocab, tag2label=data_utils.tag2label) model.build_graph() model.fit(train_data, val_data=test_data)
def main(): CLICK_MODEL_JSON = sys.argv[1] # the folder where the input data can be found INPUT_DATA_PATH = sys.argv[2] # the folder where output should be stored OUTPUT_PATH = sys.argv[3] # how many results to show in the results page of the ranker # this should be equal or smaller than the rank cut when creating the data RANK_CUT = int(sys.argv[4]) with open(CLICK_MODEL_JSON) as fin: model_desc = json.load(fin) click_model = cm.loadModelFromJson(model_desc) # process dataset from file train_set = data_utils.read_data(INPUT_DATA_PATH, 'train', RANK_CUT) click_log, relevance_log, feature_log = generate_clicks( 1000000, click_model, train_set.gold_weights, train_set.featuredids) timeit_results = timeit.Timer( partial(generate_clicks, 1000000, click_model, train_set.gold_weights, train_set.featuredids)).repeat(10, 1)
def run(stock_no, start, end, x_window_size, y_window_size, split_ratio, layer_num, hidden_dim): model = model_utils.read_model(stock_no, x_window_size, y_window_size, layer_num, hidden_dim) data, scaler_adjclose, scaler_volume = data_utils.read_data( stock_no, start, end) X = data_utils.testing_values(data, x_window_size, y_window_size) y_pred = model.predict(X.reshape(X.shape[0], X.shape[1], 2)) y_mean = tools.prediction_mean(y_pred).reshape(-1, 1) y_pred = scaler_adjclose.inverse_transform(y_pred) y_mean = scaler_adjclose.inverse_transform(y_mean) data_idx = data_utils.data_idx(stock_no, start, end)[-y_window_size:] y_pred = np.concatenate((y_pred, y_mean), axis=1) y_pred = pd.DataFrame(y_pred, index=data_idx, columns=['t+1', 't+2', 't+3', 'mean']) filepath = outputpath(stock_no, end) y_pred.to_csv(filepath)
def train_MLE(): ''' data_utils.prepare_whole_data(FLAGS.source_data, FLAGS.target_data, FLAGS.src_vocab_size, FLAGS.trg_vocab_size) # read dataset and split to training set and validation set d = data_utils.read_data(FLAGS.source_data + '.token', FLAGS.target_data + '.token', buckets) np.random.seed(SEED) np.random.shuffle(d) print('Total document size: %s' % sum(len(l) for l in d)) print('len(d): ', len(d)) d_train = [[] for _ in range(len(d))] d_valid = [[] for _ in range(len(d))] for i in range(len(d)): d_train[i] = d[i][:int(0.9 * len(d[i]))] d_valid[i] = d[i][int(-0.1 * len(d[i])):] ''' d_train = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets) d_valid = data_utils.read_data(FLAGS.source_data + '_val.token',FLAGS.target_data + '_val.token',buckets) print('Total document size of training data: %s' % sum(len(l) for l in d_train)) print('Total document size of validation data: %s' % sum(len(l) for l in d_valid)) train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))] train_total_size = float(sum(train_bucket_sizes)) train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in range(len(train_bucket_sizes))] print('train_bucket_sizes: ',train_bucket_sizes) print('train_total_size: ',train_total_size) print('train_buckets_scale: ',train_buckets_scale) valid_bucket_sizes = [len(d_valid[b]) for b in range(len(d_valid))] valid_total_size = float(sum(valid_bucket_sizes)) valid_buckets_scale = [sum(valid_bucket_sizes[:i + 1]) / valid_total_size for i in range(len(valid_bucket_sizes))] print('valid_bucket_sizes: ',valid_bucket_sizes) print('valid_total_size: ',valid_total_size) print('valid_buckets_scale: ',valid_buckets_scale) with tf.Session() as sess: model = create_seq2seq(sess, 'MLE') if FLAGS.reset_sampling_prob: with tf.variable_scope('sampling_prob',reuse=tf.AUTO_REUSE): sess.run(tf.assign(model.sampling_probability,reset_prob)) if FLAGS.schedule_sampling: if not FLAGS.keep_best_model: FLAGS.keep_best_model = False print('model.sampling_probability: ',model.sampling_probability_clip) #sess.run(tf.assign(model.sampling_probability,1.0)) step = 0 loss = 0 loss_list = [] perplexity_valid_min = float('Inf') if FLAGS.schedule_sampling: print('sampling_decay_steps: ',FLAGS.sampling_decay_steps) print('sampling_probability: ',sess.run(model.sampling_probability_clip)) print('-----') while(True): step += 1 random_number = np.random.random_sample() # buckets_scale 是累加百分比 bucket_id = min([i for i in range(len(train_buckets_scale)) if train_buckets_scale[i] > random_number]) encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id) ''' debug inds = [0,1,2,3,4,5] for ind in inds: encoder_sent = [b[ind] for b in encoder_input] decoder_sent = [b[ind] for b in decoder_input] print('len of encoder_input: ',len(encoder_input)) print('encoder_input: ',encoder_sent,data_utils.token_to_text(encoder_sent,source_mapping)) print('decoder_input: ',decoder_sent,data_utils.token_to_text(decoder_sent,target_mapping)) print('-------------------------') ''' #print('batch_size: ',model.batch_size) ==> 64 #print('batch_size: ',len(encoder_input[0])) ==> 64 #print('batch_size: ',len(encoder_input)) ==> 15,50,... #print('batch_size: ',len(decoder_input)) ==> 15,50,... #print('batch_size: ',len(weight)) ==> 15,50,... loss_train, _ = model.run(sess, encoder_input, decoder_input, weight, bucket_id) loss += loss_train / FLAGS.check_step #if step!=0 and step % FLAGS.sampling_decay_steps == 0: # sess.run(model.sampling_probability_decay) # print('sampling_probability: ',sess.run(model.sampling_probability)) if step % FLAGS.check_step == 0: perplexity_train = np.exp(loss) with open('%s/loss_train'%FLAGS.model_dir,'a') as f: f.write('%s\n'%perplexity_train) print('Step %s, Training perplexity: %s, Learning rate: %s' % (step, perplexity_train, sess.run(model.learning_rate))) perplexity_valids = [] for i in range(len(d_train)): encoder_input, decoder_input, weight = model.get_batch(d_valid, i) loss_valid, _ = model.run(sess, encoder_input, decoder_input, weight, i, forward_only = True) perplexity_valid = np.exp(loss_valid) print(' Validation perplexity in bucket %s: %s' % (i,perplexity_valid)) perplexity_valids.append(perplexity_valid) if len(loss_list) > 2 and loss > max(loss_list[-3:]): sess.run(model.learning_rate_decay) else: if step!=0: if FLAGS.schedule_sampling: sess.run(model.sampling_probability_decay) print('sampling_probability: ',sess.run(model.sampling_probability_clip)) loss_list.append(loss) loss = 0 if FLAGS.keep_best_model: perplexity_valids_mean = np.mean(perplexity_valids) if perplexity_valids_mean < perplexity_valid_min: perplexity_valid_min = perplexity_valids_mean print('perplexity_valid_min: ',perplexity_valid_min) checkpoint_path = os.path.join(FLAGS.model_dir, "MLE.ckpt") model.saver.save(sess, checkpoint_path, global_step = step) print('Saving model at step %s' % step) else: checkpoint_path = os.path.join(FLAGS.model_dir, "MLE.ckpt") model.saver.save(sess, checkpoint_path, global_step = step) print('Saving model at step %s' % step) with open('%s/loss_val'%FLAGS.model_dir,'a') as f: for perplexity_valid in perplexity_valids: f.write('%s\n'%perplexity_valid) f.write('-----------------\n') if FLAGS.schedule_sampling: if step == FLAGS.sampling_global_step: break
def train_RL(): if FLAGS.sent_word_seg == 'word': import jieba jieba.load_userdict('dict_fasttext.txt') g1 = tf.Graph() g2 = tf.Graph() g3 = tf.Graph() sess1 = tf.Session(graph = g1) sess2 = tf.Session(graph = g2) sess3 = tf.Session(graph = g3) # model is for training seq2seq with Reinforcement Learning with g1.as_default(): model = create_seq2seq(sess1, 'RL') # we set sample size = ? model.batch_size = FLAGS.batch_size # model_LM is for a reward function (language model) with g2.as_default(): model_LM = create_seq2seq(sess2, 'MLE') model_LM.beam_search = False # calculate probibility of only one sentence model_LM.batch_size = 1 def LM(encoder_input, decoder_input, weight, bucket_id): return model_LM.run(sess2, encoder_input, decoder_input, weight, bucket_id, forward_only = True)[1] # new reward function: sentiment score with g3.as_default(): model_SA = run.create_model(sess3, 'test') model_SA.batch_size = 1 def SA(sentence, encoder_length): if FLAGS.sent_word_seg == 'word': sentence = ''.join(sentence) sentence = jieba.lcut(sentence) sentence = ' '.join(sentence) elif FLAGS.sent_word_seg == 'char': sentence = ' '.join(sentence) token_ids = dataset.convert_to_token(sentence, model_SA.vocab_map) print('sentence: ',sentence) print('token_ids: ',token_ids) encoder_input, encoder_length, _ = model_SA.get_batch([(0, token_ids)]) return model_SA.step(sess3, encoder_input, encoder_length)[0][0] ''' data_utils.prepare_whole_data(FLAGS.source_data, FLAGS.target_data, FLAGS.src_vocab_size, FLAGS.trg_vocab_size) d = data_utils.read_data(FLAGS.source_data + '.token', FLAGS.target_data + '.token', buckets) d = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets) train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))] train_total_size = float(sum(train_bucket_sizes)) train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in range(len(train_bucket_sizes))] ''' d_train = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets) train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))] train_total_size = float(sum(train_bucket_sizes)) train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in range(len(train_bucket_sizes))] # make RL object read vocab mapping dict, list model.RL_readmap(src_map_path=source_mapping,trg_map_path=target_mapping) step = 0 while(True): step += 1 random_number = np.random.random_sample() bucket_id = min([i for i in range(len(train_buckets_scale)) if train_buckets_scale[i] > random_number]) print('step: ',step) print('bucket_id: ',bucket_id) # the same encoder_input for sampling batch_size times #encoder_input, decoder_input, weight = model.get_batch(d, bucket_id, rand = False) #encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id, rand = False, initial_id=FLAGS.skip) encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id, rand = True, initial_id=FLAGS.skip) #print('encoder_input: ',len(encoder_input[0])) #print('decoder_input: ',len(decoder_input[0])) print('batch_size: ',model.batch_size) loss = model.run(sess1, encoder_input, decoder_input, weight, bucket_id, X = LM, Y = SA) print('Loss: %s' %loss) print('====================') # debug #encoder_input = np.reshape(np.transpose(encoder_input, (1, 0, 2)), (-1, FLAGS.vocab_size)) #encoder_input = np.split(encoder_input, FLAGS.max_length) #print(model.token2word(encoder_input)[0]) #print(model.token2word(sen)[0]) if step % FLAGS.check_step == 0: print('Loss at step %s: %s' % (step, loss)) checkpoint_path = os.path.join(FLAGS.model_rl_dir, "RL.ckpt") model.saver.save(sess1, checkpoint_path, global_step = step) print('Saving model at step %s' % step) with open('%s/loss_train'%FLAGS.model_rl_dir,'a') as f: f.write('%s\n'%loss) if step == FLAGS.sampling_global_step: break
from collections import defaultdict import math import numpy as np from data_utils import read_data, preprocess_data, clean_text, read_weights if __name__ == '__main__': data, _ = read_data(use_loaded=True) X, y, emb, tokenizer, label_encoder = preprocess_data(data=data, use_loaded=True) with open("data/adjectives_people.txt", "r", encoding="utf-8") as fin: identity_columns = [line.strip() for line in fin.readlines()] cleaned_text = data["text"].apply(clean_text).values debias_weights = np.ones(len(y)) num_pos, num_all = defaultdict(int), defaultdict(int) sum_pos = sum([debias_weights[i] for i in range(len(y)) if y[i] == 0]) sum_all = sum(debias_weights) for idty in identity_columns: for i in range(len(cleaned_text)): ok = False sen = cleaned_text[i] if idty in ["american", "african"]: sen_split = sen.split() for j in range(len(sen_split)): if sen_split[j] == idty: if j == 0 or " ".join([sen_split[j - 1], sen_split[j] ]) != "american african": if j == len(sen_split) - 1 or " ".join([ sen_split[j], sen_split[j + 1]
def train(): images, labels = data_utils.read_data(FLAGS.train_data_dir, FLAGS.val_data_dir, FLAGS.test_data_dir, FLAGS.channel, FLAGS.img_size, FLAGS.n_aug_img) n_data = np.shape(images["train"])[0] print("Number of training data: %d" % (n_data)) g = tf.Graph() with g.as_default(): ops = get_ops(images, labels) child_ops = ops["child"] controller_ops = ops["controller"] saver = tf.train.Saver(max_to_keep=2) checkpoint_saver_hook = tf.train.CheckpointSaverHook( FLAGS.output_dir, save_steps=child_ops["num_train_batches"], saver=saver) hooks = [checkpoint_saver_hook] if FLAGS.child_sync_replicas: sync_replicas_hook = child_ops["optimizer"].make_session_run_hook( True) hooks.append(sync_replicas_hook) if FLAGS.controller_training and FLAGS.controller_sync_replicas: sync_replicas_hook = controller_ops[ "optimizer"].make_session_run_hook(True) hooks.append(sync_replicas_hook) print("-" * 80) print("Starting session") config = tf.ConfigProto(allow_soft_placement=True) with tf.train.SingularMonitoredSession( config=config, hooks=hooks, checkpoint_dir=FLAGS.output_dir) as sess: start_time = time.time() while True: run_ops = [ child_ops["loss"], child_ops["lr"], child_ops["grad_norm"], child_ops["train_acc"], child_ops["train_op"] ] loss, lr, gn, tr_acc, _ = sess.run(run_ops) global_step = sess.run(child_ops["global_step"]) if FLAGS.child_sync_replicas: actual_step = global_step * FLAGS.num_aggregate else: actual_step = global_step epoch = actual_step // ops["num_train_batches"] curr_time = time.time() if global_step % FLAGS.log_every == 0: log_string = "" log_string += "epoch = {:<6d}".format(epoch) log_string += "ch_step = {:<6d}".format(global_step) log_string += " loss = {:<8.6f}".format(loss) log_string += " lr = {:<8.4f}".format(lr) log_string += " |g| = {:<8.4f}".format(gn) log_string += " tr_acc = {:<3d}/{:>3d}".format( tr_acc, FLAGS.batch_size) log_string += " mins = {:<10.2f}".format( float(curr_time - start_time) / 60) print(log_string) if actual_step % ops["eval_every"] == 0: if (FLAGS.controller_training and epoch % FLAGS.controller_train_every == 0): print("Epoch {}: Training controller".format(epoch)) for ct_step in range(FLAGS.controller_train_steps * FLAGS.controller_num_aggregate): run_ops = [ controller_ops["loss"], controller_ops["entropy"], controller_ops["lr"], controller_ops["grad_norm"], controller_ops["valid_acc"], controller_ops["baseline"], controller_ops["skip_rate"], controller_ops["train_op"], ] loss, entropy, lr, gn, val_acc, bl, skip, _ = sess.run( run_ops) controller_step = sess.run( controller_ops["train_step"]) if ct_step % FLAGS.log_every == 0: curr_time = time.time() log_string = "" log_string += "ctrl_step = {:<6d}".format( controller_step) log_string += " loss = {:<7.3f}".format(loss) log_string += " ent = {:<5.2f}".format(entropy) log_string += " lr = {:<6.4f}".format(lr) log_string += " |g| = {:<8.4f}".format(gn) log_string += " acc = {:<6.4f}".format(val_acc) log_string += " bl = {:<5.2f}".format(bl) log_string += " mins = {:<.2f}".format( float(curr_time - start_time) / 60) print(log_string) print("Here are 10 architectures") for _ in range(10): arc, acc = sess.run([ controller_ops["sample_arc"], controller_ops["valid_acc"], ]) if FLAGS.search_for == "micro": normal_arc, reduce_arc = arc print(np.reshape(normal_arc, [-1])) print(np.reshape(reduce_arc, [-1])) else: start = 0 for layer_id in range(FLAGS.child_num_layers): if FLAGS.controller_search_whole_channels: end = start + 1 + layer_id else: end = start + 2 * FLAGS.child_num_branches + layer_id print(np.reshape(arc[start:end], [-1])) start = end print("val_acc = {:<6.4f}".format(acc)) print("-" * 80) print("Epoch {}: Eval".format(epoch)) if FLAGS.child_fixed_arc is None: ops["eval_func"](sess, "valid") ops["eval_func"](sess, "test") if epoch >= FLAGS.num_epochs: break
import data_utils import sys import numpy as np import os if __name__ == "__main__": data_dir = '../generate_dataset/' test_set = data_utils.read_data(data_dir, 'test', 100000) res = open(sys.argv[1]).readlines() res = [float(fi[:-1]) for fi in res] ser = 0 fout = open('./trec_out.txt', 'w') for i, j in zip(test_set.initial_list, test_set.qids): l = np.array(res[ser: ser + len(i)]) r = np.argsort(-l) for s, si in zip(r, range(len(r))): fout.write(('%s Q0 %s %d %f SVM\n') % (j, test_set.dids[i[s]], si+1, l[s])) ser += len(r) fout.close() os.system('./trec_eval -c -m ndcg_cut.1,3,5,10 ' + data_dir + 'test/test.qrels ./trec_out.txt > res.out') print(''.join(open('./res.out').readlines())) os.system('./trec_eval -c ' + data_dir + 'test/test.qrels ./trec_out.txt > res.out') print(''.join(open('./res.out').readlines()))
def train(): # Prepare data. print("Reading data in %s" % FLAGS.data_dir) train_set = data_utils.read_data(FLAGS.data_dir, 'train', FLAGS.train_list_cutoff) valid_set = data_utils.read_data(FLAGS.data_dir, 'valid', FLAGS.train_list_cutoff) print("Rank list size %d" % train_set.rank_list_size) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # Create model. print("Creating model...") model = create_model(sess, train_set, False) print("Created %d layers of %d units." % (model.hparams.num_layers, model.embed_size)) # Create tensorboard summarizations. train_writer = tf.summary.FileWriter(FLAGS.train_dir + '/train_log', sess.graph) valid_writer = tf.summary.FileWriter(FLAGS.train_dir + '/valid_log') #pad data train_set.pad(train_set.rank_list_size) valid_set.pad(valid_set.rank_list_size) # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] best_loss = None while True: # Get a batch and make a step. start_time = time.time() input_feed, _ = model.get_batch(train_set) step_loss, _, summary = model.step(sess, input_feed, False) step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 train_writer.add_summary(summary, current_step) # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % FLAGS.steps_per_checkpoint == 0: # Print statistics for the previous epoch. #loss = math.exp(loss) if loss < 300 else float('inf') print("global step %d learning rate %.4f step-time %.2f loss " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, loss)) #train_writer.add_summary({'step-time':step_time, 'loss':loss}, current_step) # Decrease learning rate if no improvement was seen over last 3 times. #if len(previous_losses) > 2 and loss > max(previous_losses[-3:]): # sess.run(model.learning_rate_decay_op) previous_losses.append(loss) # Validate model it = 0 count_batch = 0.0 valid_loss = 0 while it < len(valid_set.initial_list) - model.batch_size: input_feed, _ = model.get_next_batch(it, valid_set) v_loss, results, summary = model.step( sess, input_feed, True) it += model.batch_size valid_loss += v_loss count_batch += 1.0 valid_writer.add_summary(summary, current_step) valid_loss /= count_batch #valid_loss = math.exp(valid_loss) if valid_loss < 300 else float('inf') print(" eval: loss %.2f" % (valid_loss)) # Save checkpoint and zero timer and loss. # need to rethink #if best_loss == None or best_loss >= eval_ppx: # best_loss = eval_ppx checkpoint_path = os.path.join(FLAGS.train_dir, "DLA.ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) if loss == float('inf'): break step_time, loss = 0.0, 0.0 sys.stdout.flush() if FLAGS.max_train_iteration > 0 and current_step > FLAGS.max_train_iteration: break
if word in dictionary: encoded_word = dictionary[word] else: encoded_word = dictionary['UNK'] encoded_sentence += str(encoded_word) + " " print sentence encoded_sentence = encoded_sentence[:-1] # Remove final space f.write(encoded_sentence + '\n') # Write sentence to file f.close() # Generate dictionary for dataset tokenized_data = data_utils.read_data(num_movie_scripts) print '-------- tokenized_data' print tokenized_data[:10] data, count, dictionary, reverse_dictionary = data_utils.build_dataset(tokenized_data, vocabulary_size) print '-------- data' print data print '-------- count' print count print '-------- dictionary' data_utils.print_dic(dictionary, 5) print dictionary print '-------- reverse_dictionary' data_utils.print_dic(reverse_dictionary, 5) print reverse_dictionary