def train():
logger.info('start...')
if FLAGS.child_fixed_arc is None:
images, labels = read_data(FLAGS.data_path)
else:
images, labels = read_data(FLAGS.data_path, num_valids=0)
logger.info("Original Image Shape: {0}".format(images['train'].shape))
g = tf.Graph()
with g.as_default():
ops = get_ops(images, labels)
child_ops = ops["child"]
generator_ops = ops["generator"]
dc = DagController(
num_cells=FLAGS.child_num_cells,
num_layers=FLAGS.child_num_layers,
cd_length=FLAGS.child_num_cells+2,
opt_num=FLAGS.child_num_branches,
path_pool_size=FLAGS.path_pool_size,
k_init_selection_num=FLAGS.k_init_selection_num,
k_best_selection_num=FLAGS.k_best_selection_num,
max_generation=FLAGS.max_generation)
if FLAGS.child_fixed_arc is None:
dc.evolve_ops_dag(child_ops, generator_ops)
else:
dc.eval_dag_arc(child_ops)
def train_and_eval(model_dir, model_type, train_steps, train_file_name,
valid_file_name, test_file_name, result_file):
model_dir = tempfile.mkdtemp() if not model_dir else model_dir
m = build_model(model_dir, model_type)
# set num_epochs to None to get infinite stream of data.
rf = codecs.open(result_file, mode='w', encoding='utf-8')
session_config = tf.ConfigProto(allow_soft_placement=True)
session_config.gpu_options.allow_growth = True
with tf.Session(config=session_config) as sess:
#m.train(input_fn=read_data(train_file_name, num_epochs=None, shuffle=True), steps=train_steps)
m.train(input_fn=read_data_with_sampling(train_file_name,
num_epochs=None,
shuffle=True),
steps=train_steps)
eval_result = m.evaluate(input_fn=read_data(valid_file_name,
num_epochs=1,
shuffle=False),
steps=None)
print("model directory = %s" % model_dir)
for key in sorted(eval_result):
print("%s: %s" % (key, eval_result[key]))
predictions = m.predict(input_fn=read_data(test_file_name,
num_epochs=1,
shuffle=False),
predict_keys="classes")
predictions = list(predictions)
for p in predictions:
rf.write(str(p["classes"][0], encoding='utf-8'))
rf.write("\n")
def load_data():
print("Loading data from %s" % FLAGS.data_dir)
nl_extention = ".ids%d.nl" % FLAGS.nl_vocab_size
cm_extension = ".ids%d.cm" % FLAGS.cm_vocab_size
nl_txt_train = os.path.join(data_dir, "train") + ".nl"
cm_txt_train = os.path.join(data_dir, "train") + ".cm"
nl_txt_dev = os.path.join(data_dir, "dev") + ".nl"
cm_txt_dev = os.path.join(data_dir, "dev") + ".cm"
nl_txt_test = os.path.join(data_dir, "test") + ".nl"
cm_txt_test = os.path.join(data_dir, "test") + ".cm"
nl_train = os.path.join(data_dir, "train") + nl_extention
cm_train = os.path.join(data_dir, "train") + cm_extension
nl_dev = os.path.join(data_dir, "dev") + nl_extention
cm_dev = os.path.join(data_dir, "dev") + cm_extension
nl_test = os.path.join(data_dir, "test") + nl_extention
cm_test = os.path.join(data_dir, "test") + cm_extension
train_set = data_utils.read_data(nl_txt_train, cm_txt_train, nl_train, cm_train, None,
FLAGS.max_train_data_size)
dev_set = data_utils.read_data(nl_txt_dev, cm_txt_dev, nl_dev, cm_dev, None)
test_set = data_utils.read_data(nl_txt_test, cm_txt_test, nl_test, cm_test, None)
return train_set, dev_set, test_set
def load_data(self, debug=False):
"""Loads train/valid/test data and sentence encoding"""
en_train, fr_train, en_dev, fr_dev, en_vocab_path, fr_vocab_path = data_utils.prepare_data(
'tmp', 40000, 40000)
self.source_vocab_to_id, self.source_id_to_vocab = data_utils.initialize_vocabulary(
en_vocab_path)
self.target_vocab_to_id, self.target_id_to_vocab = data_utils.initialize_vocabulary(
fr_vocab_path)
source_path = './tmp/train.ids40000.questions'
target_path = './tmp/train.ids40000.answers'
if self.config.train_mode:
source_path = './tmp/train.ids40000.questions'
target_path = './tmp/train.ids40000.answers'
sources, targets = data_utils.read_data(source_path, target_path)
else:
source_path = './tmp/test.ids40000.questions'
target_path = './tmp/test.ids40000.answers'
sources, targets = data_utils.read_data(source_path, target_path)
self.train, self.valid, self.max_t_len, self.max_input_len, self.max_sen_len = data_utils.pad_length_bucket(
sources, targets, self.config)
source_vocab_path = './tmp/vocab40000.questions'
target_vocab_path = './tmp/vocab40000.answers'
self.source_vocab_size = data_utils.get_vocab_size(source_vocab_path)
self.target_vocab_size = data_utils.get_vocab_size(target_vocab_path)
self.word_embedding = np.random.uniform(
-self.config.embedding_init, self.config.embedding_init,
(self.source_vocab_size, self.config.embed_size))
def main(argv=None):
train_data, user_size, item_size = data_utils.read_data(
FLAGS.dataset, FLAGS.embed_size, FLAGS.batch_size, True)
test_data, user_size, item_size = data_utils.read_data(
FLAGS.dataset, FLAGS.embed_size, FLAGS.batch_size, False)
train(train_data, test_data, user_size, item_size)
def decode():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Load test data.
print("Reading data in %s" % FLAGS.data_dir)
test_set = None
if FLAGS.decode_train:
test_set = data_utils.read_data(FLAGS.data_dir, 'train')
else:
test_set = data_utils.read_data(FLAGS.data_dir, 'test')
# Create model and load parameters.
model = create_model(sess, test_set, True)
model.batch_size = 1 # We decode one sentence at a time.
test_set.pad(test_set.rank_list_size, model.hparams.reverse_input)
rerank_scores = []
# Decode from test data.
for i in xrange(len(test_set.initial_list)):
encoder_inputs, embeddings, decoder_targets, target_weights, target_initial_scores = model.get_data_by_index(
test_set.initial_list, test_set.gold_list,
test_set.gold_weights, test_set.initial_scores,
test_set.features, i)
_, test_loss, output_logits, summary = model.step(
sess, encoder_inputs, embeddings, decoder_targets,
target_weights, target_initial_scores, True)
#The output is a list of rerank index for decoder_inputs (which represents the gold rank list)
rerank_scores.append(output_logits[0][0])
if i % FLAGS.steps_per_checkpoint == 0:
print("Decoding %.2f \r" %
(float(i) / len(test_set.initial_list))),
#get rerank indexes with new scores
rerank_lists = []
for i in xrange(len(rerank_scores)):
scores = rerank_scores[i]
rerank_lists.append(
sorted(range(len(scores)),
key=lambda k: scores[k],
reverse=True))
if FLAGS.decode_train:
data_utils.output_ranklist(test_set, rerank_lists, FLAGS.test_dir,
model.hparams.reverse_input, 'train')
else:
data_utils.output_ranklist(test_set, rerank_lists, FLAGS.test_dir,
model.hparams.reverse_input, 'test')
return
def load_data(self, debug=False):
"""Loads train/valid/test data and sentence encoding"""
'''
en_train, fr_train, en_dev, fr_dev, _, _ = data_utils.prepare_data(
FLAGS.data_dir, FLAGS.en_vocab_size, FLAGS.fr_vocab_size)
'''
en_train, fr_train, en_dev, fr_dev, en_vocab_path, fr_vocab_path = data_utils.prepare_data(
'tmp', 40000, 40000)
self.source_vocab_to_id, self.source_id_to_vocab = data_utils.initialize_vocabulary(
en_vocab_path)
self.target_vocab_to_id, self.target_id_to_vocab = data_utils.initialize_vocabulary(
fr_vocab_path)
#print self.source_vocab_to_id
#print self.source_id_to_vocab
'''
print self.target_vocab_to_id
print self.target_id_to_vocab
'''
'''
for i in range(0, 10):
print i
print self.target_id_to_vocab[int(float(i))]
#adsfas
'''
source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.questions'
target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.answers'
if self.config.train_mode:
source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.questions'
target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/train.ids40000.answers'
sources, targets = data_utils.read_data(source_path, target_path)
else:
source_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/test.ids40000.questions'
target_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/test.ids40000.answers'
sources, targets = data_utils.read_data(source_path, target_path)
self.train, self.valid, self.max_t_len, self.max_input_len, self.max_sen_len = data_utils.pad_length_bucket(
sources, targets, self.config)
source_vocab_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/vocab40000.questions'
target_vocab_path = '/Users/ethancaballero/Neural-Engineer_Candidates/dmn-tf-alter_working_decoder_d2c/tmp/vocab40000.answers'
self.source_vocab_size = data_utils.get_vocab_size(source_vocab_path)
self.target_vocab_size = data_utils.get_vocab_size(target_vocab_path)
self.word_embedding = np.random.uniform(
-self.config.embedding_init, self.config.embedding_init,
(self.source_vocab_size, self.config.embed_size))
def train():
print "Preparing data in %s" % settings.data_dir
sr_train_ids_path, tg_train_ids_path,sr_dev_ids_path, tg_dev_ids_path,sr_vocab_path, tg_vocab_path = data_utils.prepare_data(settings.data_dir)
print "Reading training data from %s" % settings.data_dir
train_set = data_utils.read_data(sr_train_ids_path,tg_train_ids_path,settings.max_train_num)
train_batches,train_bucket_ids = data_utils.batchize(train_set)
print "Reading development data from %s" % settings.data_dir
dev_set = data_utils.read_data(sr_dev_ids_path,tg_dev_ids_path)
dev_batches,dev_bucket_ids = data_utils.batchize(dev_set,False)
log_file = open(settings.train_dir+'log.txt','w')
log_file.write('epoch\tstep\ttime\ttrain-ppx\tdev-ppx\n')
log_file.flush()
with tf.Session() as sess:
print("Creating %d layers of %d units." %
(settings.num_layers, settings.size))
model = create_model(sess, False)
current_epoch,current_step,train_loss = 0,0,0.0
start_time = time.time()
while True:
current_epoch+=1
for batch_id in xrange(len(train_batches)):
current_step+=1
step_start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.preprocess_batch(train_batches[batch_id], train_bucket_ids[batch_id])
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, train_bucket_ids[batch_id], False)
print "global-step %d\tstep-time %.2f\tstep-loss %.2f" % (model.global_step.eval(),time.time()-step_start_time,step_loss)
train_loss+=step_loss/settings.steps_per_checkpoint
if current_step % settings.steps_per_checkpoint == 0:
# evaluate in training set
train_ppx = math.exp(train_loss)/model.batch_size if train_loss < 300 else float('inf')
# evaluate in development set
dev_loss=0.0
for dev_batch_id in xrange(len(dev_batches)):
encoder_inputs, decoder_inputs, target_weights = model.preprocess_batch(dev_batches[dev_batch_id], dev_bucket_ids[dev_batch_id])
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, dev_bucket_ids[dev_batch_id], True)
dev_loss+=step_loss/len(dev_batches)
dev_ppx = math.exp(dev_loss)/model.batch_size if dev_loss < 300 else float('inf')
log_file.write("%d\t%d\t%.2f\t%.2f\t%.2f\n" % (current_epoch,model.global_step.eval(),time.time()-start_time,train_ppx,dev_ppx))
log_file.flush()
sys.stdout.flush()
train_loss,dev_loss = 0.0,0.0
checkpoint_path = os.path.join(settings.train_dir, "summary.ckpt")
model.saver.save(sess, checkpoint_path,global_step=model.global_step)
train_batches,train_bucket_ids = data_utils.batchize(train_set)
def get_copy_dict(vocab):
lines = du.read_data(filepaths['copypp'])
ids = du.convert_to_token_ids(lines, vocab)
copy_dict = {}
for line in ids:
copy_dict[line[0]] = line[1:]
return copy_dict
def parse_data(data_dir, ti='train', tp='train', rank_cut=10, target='./'):
train_session = ''
train_size = ''
train_svm = ''
train_rank = ''
print("Reading data in %s" % data_dir)
train_set = data_utils.read_data(data_dir, ti, rank_cut)
l = len(train_set.initial_list)
fout1 = open(target + 'letor.%s' % tp, 'w')
fout2 = open(target + 'letor.%s.query' % tp, 'w')
fout3 = open(target + 'letor.%s.svm' % tp, 'w')
fout4 = open(target + 'letor.%s.rank' % tp, 'w')
rg = 1
qid = 0
for r in range(rg):
ser = 0
for i in range(l):
if i % 1000 == 0:
print(i, l, qid)
train_size += str(len(train_set.initial_list[i])) + '\n'
gold_label_list = [
0 if train_set.initial_list[i][x] < 0 else
train_set.gold_weights[i][x]
for x in xrange(len(train_set.initial_list[i]))
]
click_list, _, _ = click_model.sampleClicksForOneList(
list(gold_label_list))
while sum(click_list) == 0:
click_list, _, _ = click_model.sampleClicksForOneList(
list(gold_label_list))
for s in range(len(click_list)):
feat_str = ''
hit = 0
for cnt, f in enumerate(train_set.features[ser + s]):
if f != 0:
hit += 1
feat_str += ' ' + str(cnt + 1) + ':' + str(f)
if hit == 0:
print(feat_str)
return
train_session += str(click_list[s]) + feat_str + '\n'
train_svm += str(
click_list[s]) + ' qid:' + str(qid) + feat_str + '\n'
train_rank += str(s) + '\n'
qid += 1
ser += len(click_list)
fout1.write(train_session)
fout2.write(train_size)
fout3.write(train_svm)
fout4.write(train_rank)
train_size = ''
train_session = ''
train_svm = ''
train_rank = ''
fout1.close()
fout2.close()
fout3.close()
fout4.close()
return train_set
def main(unused_argv):
os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'
if not os.path.isdir(args.train_output_dir):
tf.logging.info("Path {} does not exist. Creating.".format(
args.train_output_dir))
os.makedirs(args.train_output_dir)
if not os.path.isdir(args.model_dir):
tf.logging.info("Path {} does not exist. Creating.".format(
args.model_dir))
os.makedirs(args.model_dir)
if not os.path.isdir(args.test_dir):
tf.logging.info("Path {} does not exist. Creating.".format(
args.test_dir))
os.makedirs(args.test_dir)
images, labels = read_data(args.data)
g1 = tf.get_default_graph()
M = Model(images, labels, args, g1)
M.generate_ops()
num_valid = len(images['valid'])
num_batch = (num_valid + args.train_valid_batch_size -
1) // args.train_valid_batch_size
num_train_examples = np.shape(images["train"])[0]
batch_size = args.batch_size
tf.logging.info("batch_size is {}".format(batch_size))
num_train_batches = (num_train_examples + batch_size - 1) // batch_size
start_time = time.time()
with g1.as_default():
GA = Game(M)
saver = tf.train.Saver(max_to_keep=10)
checkpoint_saver_hook = tf.train.CheckpointSaverHook(
args.train_output_dir,
save_steps=num_train_batches * args.num_epochs_per_iter,
saver=saver)
hooks = [checkpoint_saver_hook]
if args.sync_replicas:
sync_replicas_hook = M.optimizer.make_session_run_hook(True)
hooks.append(sync_replicas_hook)
tf.logging.info("-" * 80)
tf.logging.info("Starting session")
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.train.SingularMonitoredSession(
config=config, hooks=hooks,
checkpoint_dir=args.train_output_dir) as sess:
GA.sample_and_train(sess)
def train():
params = get_child_model_params()
images, labels = read_data(params['data_dir'], num_valids=0)
g = tf.Graph()
with g.as_default():
ops = get_ops(images, labels, params)
saver = tf.train.Saver(max_to_keep=30)
checkpoint_saver_hook = tf.train.CheckpointSaverHook(
params['model_dir'],
save_steps=ops["num_train_batches"],
saver=saver)
hooks = [checkpoint_saver_hook]
if params['sync_replicas']:
sync_replicas_hook = ops["optimizer"].make_session_run_hook(True)
hooks.append(sync_replicas_hook)
tf.logging.info("-" * 80)
tf.logging.info("Starting session")
config = tf.ConfigProto(allow_soft_placement=True)
with tf.train.SingularMonitoredSession(
config=config, hooks=hooks,
checkpoint_dir=params['model_dir']) as sess:
start_time = time.time()
while True:
run_ops = [
ops["loss"],
ops["lr"],
ops["grad_norm"],
ops["train_acc"],
ops["train_op"],
]
loss, lr, gn, tr_acc, _ = sess.run(run_ops)
global_step = sess.run(ops["global_step"])
if params['sync_replicas']:
actual_step = global_step * params['num_aggregate']
else:
actual_step = global_step
epoch = actual_step // ops["num_train_batches"]
curr_time = time.time()
if global_step % 50 == 0:
log_string = ""
log_string += "epoch={:<6d}".format(epoch)
log_string += "ch_step={:<6d}".format(global_step)
log_string += " loss={:<8.6f}".format(loss)
log_string += " lr={:<8.4f}".format(lr)
log_string += " |g|={:<8.4f}".format(gn)
log_string += " tr_acc={:<3d}/{:>3d}".format(
tr_acc, params['batch_size'])
log_string += " mins={:<10.2f}".format(
float(curr_time - start_time) / 60)
tf.logging.info(log_string)
if actual_step % ops["eval_every"] == 0:
ops["eval_func"](sess, "test")
if epoch >= params['num_epochs']:
tf.logging.info('Training finished!')
break
def test_serialize_deserialize() -> None:
unlabeled_data = data_utils.read_data("test_data/56980685061237.npz")
labels = data_utils.read_labels("test_data/labels.csv")
data = data_utils.label_data(unlabeled_data, labels)
for training_point in data_utils.generate_training_points(data):
serialized = trainer.serialize(training_point)
inputs, outputs = trainer.deserialize(serialized)
assert set(inputs.keys()) == set(trainer.INPUTS_SPEC.keys())
assert set(outputs.keys()) == set(trainer.OUTPUTS_SPEC.keys())
def main():
# 获得 训练集,验证集,测试集
tuple_data_ = reader.read_data(
'F:/Datas/msr_training/msr_training.utf8.ic',
target_dict={
'B': 0,
'M': 1,
'E': 2,
'S': 3
})
# 根据模型规模config{small,medium,large,or test}
# 获得2套模型参数,参数封装在config类的属性当中
config = Config()
eval_config = Config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
# 创建一个初始化的容器
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
# 分别创建训练模型,验证模型,测试模型
# reuse=None 不使用以往参数
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = CTBModel(is_training=True, config=config)
# 使用以往的参数
with tf.variable_scope("model", reuse=True, initializer=initializer):
mtest = CTBModel(is_training=False, config=eval_config)
tf.global_variables_initializer().run()
for i in range(config.max_max_epoch):
# 0.5**(0,..,0 and 1 and 2,.. )
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
# 学习率在max_epoch之后开始随迭代次数下降
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity, accuracy = run_epoch(session,
m,
tuple_data_,
m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f, Train Accuracy: %.3f%%" %
(i + 1, train_perplexity, accuracy))
print("----------- Begin to Test!----------")
test_perplexity, accuracy = run_epoch(session,
mtest,
tuple_data_,
tf.no_op(),
verbose=True)
print("Test Perplexity: %.3f , Test Accuracy:%.3f%%" %
(test_perplexity, accuracy))
saver = tf.train.Saver()
save_path = saver.save(session, "./CTB_Model/CTB_Variables.ckpt")
print("Save to path: ", save_path)
def main(_):
if not FLAGS.data_dir:
raise ValueError("Must set --data_dir to data directory")
vocab_path = data_utils.create_vocabulary(os.path.join(FLAGS.data_dir, 'train'), FLAGS.data_dir)
train_data = data_utils.read_data(os.path.join(FLAGS.data_dir, 'train'), vocab_path)
valid_data = data_utils.read_data(os.path.join(FLAGS.data_dir, 'dev'), vocab_path)
test_data = valid_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
if ckpt and gfile.Exists(ckpt.model_checkpoint_path):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
m.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
tf.initialize_all_variables().run()
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, train_data, m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op())
print("Test Perplexity: %.3f" % test_perplexity)
def make_weights(name_dataset):
sensitive_words, sensitive_general_words, sensitive_extra_words = get_sensitive_words(
)
sensitive_words = list(set(sensitive_words))
data, split = read_data(name_dataset)
X = data["text"].apply(clean_text).values
label_encoder = LabelEncoder().fit(data["label"].values)
y = label_encoder.transform(data["label"].values)
X = X[np.concatenate([split[0], split[1]])]
y = y[np.concatenate([split[0], split[1]])]
Z, idxs_sens = get_tfidf(X, sensitive_words)
# obtaining the weights
clf = RandomForestClassifier(n_estimators=1000,
max_depth=27,
random_state=233,
n_jobs=14,
criterion='entropy')
y_pred = cross_val_predict(clf,
Z[idxs_sens],
y[idxs_sens],
cv=250,
n_jobs=1,
method='predict_proba')
print('Refit log loss: %.5f' % (log_loss(y[idxs_sens], y_pred[:, 1])))
p1 = sum(y[idxs_sens]) / len(y[idxs_sens])
p0 = 1 - p1
print(roc_auc_score(to_categorical(y[idxs_sens]), y_pred))
print(accuracy_score(y[idxs_sens], np.argmax(y_pred, 1)), max(p0, p1))
p1 = sum(y) / len(y)
p0 = 1 - p1
propensity = np.array([(p1 if y[i] == 1 else p0) for i in range(len(y))])
propensity[idxs_sens] = np.array(
[y_pred[i, y[idxs_sens[i]]] for i in range(len(idxs_sens))])
np.save(dir_processed + "propensity_%s.npy" % name_dataset, propensity)
# propensity = np.load(dir_processed + "propensity_%s.npy" % name_dataset)
weights = 1 / propensity
a = np.mean(
np.array([weights[i] for i in range(len(weights)) if y[i] == 0]))
b = np.mean(
np.array([weights[i] for i in range(len(weights)) if y[i] == 1]))
print((1 / a) / (1 / a + 1 / b), (1 / b) / (1 / a + 1 / b))
weights = np.array([(weights[i] / a if y[i] == 0 else weights[i] / b)
for i in range(len(weights))])
weights /= weights.mean()
ret = np.zeros(len(data))
ret[np.concatenate([split[0], split[1]])] = weights
np.save(dir_processed + "weights.npy", ret)
def main():
tf.set_random_seed(1234)
images, labels = read_data('./data/cifar10', 0.5)
train_dataset = tf.data.Dataset.from_tensor_slices(
(images["train"], labels["train"]))
train_dataset = train_dataset.shuffle(100).batch(16)
train_iter = train_dataset.make_initializable_iterator()
x_train, y_train = train_iter.get_next()
# x_train=images["train"][:16]
# y_train=labels["train"][:16]
logits, train_loss = Model_test(x_train, y_train, True)
w_var = utils.get_var(tf.trainable_variables(), 'weight_var')[1]
arch_var = utils.get_var(tf.trainable_variables(), 'arch_var')[1]
_, unrolled_train_loss = Model_test(x_train, y_train, True,
"unrolled_weight_var")
unrolled_w_var = utils.get_var(tf.trainable_variables(),
'unrolled_weight_var')[1]
cpoy_weight_opts = [v_.assign(v) for v_, v in zip(unrolled_w_var, w_var)]
with tf.control_dependencies(cpoy_weight_opts):
unrolled_optimizer = tf.train.GradientDescentOptimizer(0.001)
unrolled_optimizer = unrolled_optimizer.minimize(
unrolled_train_loss, var_list=unrolled_w_var)
#w'
with tf.control_dependencies([unrolled_optimizer]):
valid_grads = tf.gradients(unrolled_train_loss, unrolled_w_var)
R = 0.01 / tf.global_norm(valid_grads)
#Original Implementation
# opts=[v.assign(v+R*g) for v,g in zip(w_var,valid_grads)]
# with tf.control_dependencies(opts):
# arch_grad_after=tf.gradients(train_loss,arch_var)
optimizer1 = tf.train.GradientDescentOptimizer(R)
optimizer1 = optimizer1.apply_gradients(zip(valid_grads, w_var))
with tf.control_dependencies([optimizer1]):
arch_grad_after = tf.gradients(train_loss, arch_var)
config = tf.ConfigProto()
os.environ["CUDA_VISIBLE_DEVICES"] = str(0)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
sess.run([train_iter.initializer])
print(sess.run(valid_grads)[0])
start = time.time()
print(sess.run(arch_grad_after)[0])
print("time_is {}".format(time.time() - start))
print(sess.run(valid_grads)[0])
def train(stock_no, start, end, x_window_size, y_window_size, split_ratio,
batch_size, layer_num, hidden_dim, nb_epoch):
data, scaler_adjclose, scaler_volume = data_utils.read_data(
stock_no, start, end)
X, y = data_utils.window_transform_series(data,
x_window_size=x_window_size,
y_window_size=y_window_size)
X_train, X_test, y_train, y_test = data_utils.tts(X, y, split_ratio)
filepath = model_utils.model_path(stock_no)
model_utils.train(X_train, y_train, X_test, y_test, x_window_size,
y_window_size, split_ratio, batch_size, layer_num,
hidden_dim, nb_epoch, filepath)
def test():
params = get_child_model_params()
images, labels = read_data(params['data_dir'], num_valids=0)
g = tf.Graph()
with g.as_default():
ops = get_ops(images, labels, params)
tf.logging.info("-" * 80)
tf.logging.info("Starting session")
config = tf.ConfigProto(allow_soft_placement=True)
with tf.train.SingularMonitoredSession(
config=config, checkpoint_dir=params['model_dir']) as sess:
ops["eval_func"](sess, "test")
def data_generator(batch_size):
data = read_data(batch_size)
iterator = data.make_initializable_iterator()
batch_images, batch_outputs, batch_categorys = iterator.get_next()
with tf.Session() as sess:
sess.run(iterator.initializer)
for i in xrange(800000):
images, label, category = sess.run(
[batch_images, batch_outputs, batch_categorys])
im, hmp, c = gen_traindata(images, label, category)
message.put([i, im, hmp, c])
message.put(None)
def main():
click_model_json_file = sys.argv[1]
data_dir = sys.argv[2]
output_file = sys.argv[3]
print("Load data from " + data_dir)
train_set = data_utils.read_data(data_dir, 'train')
click_model = None
with open(click_model_json_file) as fin:
model_desc = json.load(fin)
click_model = CM.loadModelFromJson(model_desc)
print("Estimating...")
estimator = RandomizedPropensityEstimator()
estimator.estimateParametersFromModel(click_model, train_set)
print("Output results...")
estimator.outputEstimatorToFile(output_file)
def test_ctrl():
# obtain datasets
t = time.time()
images, labels = read_data()
t = time.time() - t
print('read dataset consumes %.2f sec' % t)
# config of a model
class_num = 10
child_num_layers = 6
out_channels = 32
batch_size = 32
device = 'gpu'
epoch_num = 4
# files to print sampled archs
child_filename = 'child_file.txt'
ctrl_filename = 'controller_file.txt'
child_file = open(child_filename, 'w')
ctrl_file = open(ctrl_filename, 'w')
# create a controller
ctrl = Controller(child_num_layers=child_num_layers)
# create a child, set epoch to 1; later this will be moved to an over epoch
child = Child(images, labels, class_num, child_num_layers, out_channels,
batch_size, device, 1)
print(len(list(child.net.graph)))
# print(child.net.graph)
# train multiple epochs
for _ in range(epoch_num):
# sample an arch
ctrl.ctrl.net_sample()
sample_arch = ctrl.ctrl.sample_arch
print_sample_arch(sample_arch, child_file)
# train a child model
# t = time.time()
# child.train(sample_arch)
# t = time.time() - t
# print('child training time %.2f sec' % t)
# train controller
t = time.time()
ctrl.train(child, ctrl_file)
t = time.time() - t
print('ctrller training time %.2f sec' % t)
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
raw_data = data_utils.read_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config)
mtest = PTBModel(is_training=False, config=eval_config)
tf.initialize_all_variables().run()
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
train_data,
m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data,
tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op())
print("Test Perplexity: %.3f" % test_perplexity)
def do_test(config_file):
"""
do the tests by using configuration file
:param config_file:
:return:
"""
config = utils.load_json_data(config_file)
partial_to_saturation_col = None if 'partial_to_saturation_col' not in config \
else config['partial_to_saturation_col']
x = du.read_data(config['data_file'],
sep='\t' if 'sep' not in config else config['sep'],
column_mapping=config['mapping'],
partial_to_saturation_col=partial_to_saturation_col,
hdf=False if 'hdf' not in config else config['hdf'])
if 'binary_columns_to_impute' in config:
# impute binary columns
logging.info('binary columns to impute [%s]' % config['binary_columns_to_impute'])
imputer = BinaryImputator()
x = imputer.impute(x, config['binary_columns_to_impute'])
if 'comorbidity_cols' in config:
populate_col_by_or(x, config['comorbidity_cols'], new_col_name='comorbidity')
model_files = config['model_files']
for outcome in config['outcomes']:
logging.info('testing for outcome [{0}] with #{1} models'.format(outcome, len(model_files)))
result_file = '{0}/{1}_result.tsv'.format(config['result_tsv_folder'], outcome)
test_models_and_ensemble(model_files,
x,
weights=config['weights'][outcome] if 'weights' in config else None,
outcome=outcome,
threshold=config['threshold'],
result_csv=result_file,
severity_conf=None if 'severity_scores' not in config else config['severity_scores'],
generate_figs=False if 'generate_figs' not in config else config['generate_figs'],
auc_fig_file=None if 'auc_fig_file_pattern' not in config
else (config['auc_fig_file_pattern'] % outcome),
calibration_fig_file=None if 'calibration_fig_file_pattern' not in config
else (config['calibration_fig_file_pattern'] % outcome),
event_rate=None if 'event_rate' not in config else config['event_rate'],
nri_json=None if 'nri_json' not in config else config['nri_json']
)
logging.info('result saved to {0}'.format(result_file))
def main():
config = model_config.Config()
BATCH_SIZE = config.batch_size
data = data_utils.read_data(config.corpus_path)
data_utils.build_vocab(config.corpus_path, config.vocab_path)
vocab = data_utils.read_dictionary()
# random.shuffle(data)
train_data = data[BATCH_SIZE:]
test_data = data[:BATCH_SIZE]
model = BiLSTM_CRF_Model(config=config,
vocab=vocab,
tag2label=data_utils.tag2label)
model.build_graph()
model.fit(train_data, val_data=test_data)
def main():
CLICK_MODEL_JSON = sys.argv[1]
# the folder where the input data can be found
INPUT_DATA_PATH = sys.argv[2]
# the folder where output should be stored
OUTPUT_PATH = sys.argv[3]
# how many results to show in the results page of the ranker
# this should be equal or smaller than the rank cut when creating the data
RANK_CUT = int(sys.argv[4])
with open(CLICK_MODEL_JSON) as fin:
model_desc = json.load(fin)
click_model = cm.loadModelFromJson(model_desc)
# process dataset from file
train_set = data_utils.read_data(INPUT_DATA_PATH, 'train', RANK_CUT)
click_log, relevance_log, feature_log = generate_clicks(
1000000, click_model, train_set.gold_weights, train_set.featuredids)
timeit_results = timeit.Timer(
partial(generate_clicks, 1000000, click_model, train_set.gold_weights,
train_set.featuredids)).repeat(10, 1)
def run(stock_no, start, end, x_window_size, y_window_size, split_ratio,
layer_num, hidden_dim):
model = model_utils.read_model(stock_no, x_window_size, y_window_size,
layer_num, hidden_dim)
data, scaler_adjclose, scaler_volume = data_utils.read_data(
stock_no, start, end)
X = data_utils.testing_values(data, x_window_size, y_window_size)
y_pred = model.predict(X.reshape(X.shape[0], X.shape[1], 2))
y_mean = tools.prediction_mean(y_pred).reshape(-1, 1)
y_pred = scaler_adjclose.inverse_transform(y_pred)
y_mean = scaler_adjclose.inverse_transform(y_mean)
data_idx = data_utils.data_idx(stock_no, start, end)[-y_window_size:]
y_pred = np.concatenate((y_pred, y_mean), axis=1)
y_pred = pd.DataFrame(y_pred,
index=data_idx,
columns=['t+1', 't+2', 't+3', 'mean'])
filepath = outputpath(stock_no, end)
y_pred.to_csv(filepath)
def train_MLE():
'''
data_utils.prepare_whole_data(FLAGS.source_data, FLAGS.target_data, FLAGS.src_vocab_size, FLAGS.trg_vocab_size)
# read dataset and split to training set and validation set
d = data_utils.read_data(FLAGS.source_data + '.token', FLAGS.target_data + '.token', buckets)
np.random.seed(SEED)
np.random.shuffle(d)
print('Total document size: %s' % sum(len(l) for l in d))
print('len(d): ', len(d))
d_train = [[] for _ in range(len(d))]
d_valid = [[] for _ in range(len(d))]
for i in range(len(d)):
d_train[i] = d[i][:int(0.9 * len(d[i]))]
d_valid[i] = d[i][int(-0.1 * len(d[i])):]
'''
d_train = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets)
d_valid = data_utils.read_data(FLAGS.source_data + '_val.token',FLAGS.target_data + '_val.token',buckets)
print('Total document size of training data: %s' % sum(len(l) for l in d_train))
print('Total document size of validation data: %s' % sum(len(l) for l in d_valid))
train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))]
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))]
print('train_bucket_sizes: ',train_bucket_sizes)
print('train_total_size: ',train_total_size)
print('train_buckets_scale: ',train_buckets_scale)
valid_bucket_sizes = [len(d_valid[b]) for b in range(len(d_valid))]
valid_total_size = float(sum(valid_bucket_sizes))
valid_buckets_scale = [sum(valid_bucket_sizes[:i + 1]) / valid_total_size
for i in range(len(valid_bucket_sizes))]
print('valid_bucket_sizes: ',valid_bucket_sizes)
print('valid_total_size: ',valid_total_size)
print('valid_buckets_scale: ',valid_buckets_scale)
with tf.Session() as sess:
model = create_seq2seq(sess, 'MLE')
if FLAGS.reset_sampling_prob:
with tf.variable_scope('sampling_prob',reuse=tf.AUTO_REUSE):
sess.run(tf.assign(model.sampling_probability,reset_prob))
if FLAGS.schedule_sampling:
if not FLAGS.keep_best_model:
FLAGS.keep_best_model = False
print('model.sampling_probability: ',model.sampling_probability_clip)
#sess.run(tf.assign(model.sampling_probability,1.0))
step = 0
loss = 0
loss_list = []
perplexity_valid_min = float('Inf')
if FLAGS.schedule_sampling:
print('sampling_decay_steps: ',FLAGS.sampling_decay_steps)
print('sampling_probability: ',sess.run(model.sampling_probability_clip))
print('-----')
while(True):
step += 1
random_number = np.random.random_sample()
# buckets_scale 是累加百分比
bucket_id = min([i for i in range(len(train_buckets_scale))
if train_buckets_scale[i] > random_number])
encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id)
''' debug
inds = [0,1,2,3,4,5]
for ind in inds:
encoder_sent = [b[ind] for b in encoder_input]
decoder_sent = [b[ind] for b in decoder_input]
print('len of encoder_input: ',len(encoder_input))
print('encoder_input: ',encoder_sent,data_utils.token_to_text(encoder_sent,source_mapping))
print('decoder_input: ',decoder_sent,data_utils.token_to_text(decoder_sent,target_mapping))
print('-------------------------')
'''
#print('batch_size: ',model.batch_size) ==> 64
#print('batch_size: ',len(encoder_input[0])) ==> 64
#print('batch_size: ',len(encoder_input)) ==> 15,50,...
#print('batch_size: ',len(decoder_input)) ==> 15,50,...
#print('batch_size: ',len(weight)) ==> 15,50,...
loss_train, _ = model.run(sess, encoder_input, decoder_input, weight, bucket_id)
loss += loss_train / FLAGS.check_step
#if step!=0 and step % FLAGS.sampling_decay_steps == 0:
# sess.run(model.sampling_probability_decay)
# print('sampling_probability: ',sess.run(model.sampling_probability))
if step % FLAGS.check_step == 0:
perplexity_train = np.exp(loss)
with open('%s/loss_train'%FLAGS.model_dir,'a') as f:
f.write('%s\n'%perplexity_train)
print('Step %s, Training perplexity: %s, Learning rate: %s' % (step, perplexity_train,
sess.run(model.learning_rate)))
perplexity_valids = []
for i in range(len(d_train)):
encoder_input, decoder_input, weight = model.get_batch(d_valid, i)
loss_valid, _ = model.run(sess, encoder_input, decoder_input, weight, i, forward_only = True)
perplexity_valid = np.exp(loss_valid)
print(' Validation perplexity in bucket %s: %s' % (i,perplexity_valid))
perplexity_valids.append(perplexity_valid)
if len(loss_list) > 2 and loss > max(loss_list[-3:]):
sess.run(model.learning_rate_decay)
else:
if step!=0:
if FLAGS.schedule_sampling:
sess.run(model.sampling_probability_decay)
print('sampling_probability: ',sess.run(model.sampling_probability_clip))
loss_list.append(loss)
loss = 0
if FLAGS.keep_best_model:
perplexity_valids_mean = np.mean(perplexity_valids)
if perplexity_valids_mean < perplexity_valid_min:
perplexity_valid_min = perplexity_valids_mean
print('perplexity_valid_min: ',perplexity_valid_min)
checkpoint_path = os.path.join(FLAGS.model_dir, "MLE.ckpt")
model.saver.save(sess, checkpoint_path, global_step = step)
print('Saving model at step %s' % step)
else:
checkpoint_path = os.path.join(FLAGS.model_dir, "MLE.ckpt")
model.saver.save(sess, checkpoint_path, global_step = step)
print('Saving model at step %s' % step)
with open('%s/loss_val'%FLAGS.model_dir,'a') as f:
for perplexity_valid in perplexity_valids:
f.write('%s\n'%perplexity_valid)
f.write('-----------------\n')
if FLAGS.schedule_sampling:
if step == FLAGS.sampling_global_step: break
def train_RL():
if FLAGS.sent_word_seg == 'word':
import jieba
jieba.load_userdict('dict_fasttext.txt')
g1 = tf.Graph()
g2 = tf.Graph()
g3 = tf.Graph()
sess1 = tf.Session(graph = g1)
sess2 = tf.Session(graph = g2)
sess3 = tf.Session(graph = g3)
# model is for training seq2seq with Reinforcement Learning
with g1.as_default():
model = create_seq2seq(sess1, 'RL')
# we set sample size = ?
model.batch_size = FLAGS.batch_size
# model_LM is for a reward function (language model)
with g2.as_default():
model_LM = create_seq2seq(sess2, 'MLE')
model_LM.beam_search = False
# calculate probibility of only one sentence
model_LM.batch_size = 1
def LM(encoder_input, decoder_input, weight, bucket_id):
return model_LM.run(sess2, encoder_input, decoder_input, weight, bucket_id, forward_only = True)[1]
# new reward function: sentiment score
with g3.as_default():
model_SA = run.create_model(sess3, 'test')
model_SA.batch_size = 1
def SA(sentence, encoder_length):
if FLAGS.sent_word_seg == 'word':
sentence = ''.join(sentence)
sentence = jieba.lcut(sentence)
sentence = ' '.join(sentence)
elif FLAGS.sent_word_seg == 'char':
sentence = ' '.join(sentence)
token_ids = dataset.convert_to_token(sentence, model_SA.vocab_map)
print('sentence: ',sentence)
print('token_ids: ',token_ids)
encoder_input, encoder_length, _ = model_SA.get_batch([(0, token_ids)])
return model_SA.step(sess3, encoder_input, encoder_length)[0][0]
'''
data_utils.prepare_whole_data(FLAGS.source_data, FLAGS.target_data, FLAGS.src_vocab_size, FLAGS.trg_vocab_size)
d = data_utils.read_data(FLAGS.source_data + '.token', FLAGS.target_data + '.token', buckets)
d = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets)
train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))]
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))]
'''
d_train = data_utils.read_data(FLAGS.source_data + '_train.token',FLAGS.target_data + '_train.token',buckets)
train_bucket_sizes = [len(d_train[b]) for b in range(len(d_train))]
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))]
# make RL object read vocab mapping dict, list
model.RL_readmap(src_map_path=source_mapping,trg_map_path=target_mapping)
step = 0
while(True):
step += 1
random_number = np.random.random_sample()
bucket_id = min([i for i in range(len(train_buckets_scale))
if train_buckets_scale[i] > random_number])
print('step: ',step)
print('bucket_id: ',bucket_id)
# the same encoder_input for sampling batch_size times
#encoder_input, decoder_input, weight = model.get_batch(d, bucket_id, rand = False)
#encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id, rand = False, initial_id=FLAGS.skip)
encoder_input, decoder_input, weight = model.get_batch(d_train, bucket_id, rand = True, initial_id=FLAGS.skip)
#print('encoder_input: ',len(encoder_input[0]))
#print('decoder_input: ',len(decoder_input[0]))
print('batch_size: ',model.batch_size)
loss = model.run(sess1, encoder_input, decoder_input, weight, bucket_id, X = LM, Y = SA)
print('Loss: %s' %loss)
print('====================')
# debug
#encoder_input = np.reshape(np.transpose(encoder_input, (1, 0, 2)), (-1, FLAGS.vocab_size))
#encoder_input = np.split(encoder_input, FLAGS.max_length)
#print(model.token2word(encoder_input)[0])
#print(model.token2word(sen)[0])
if step % FLAGS.check_step == 0:
print('Loss at step %s: %s' % (step, loss))
checkpoint_path = os.path.join(FLAGS.model_rl_dir, "RL.ckpt")
model.saver.save(sess1, checkpoint_path, global_step = step)
print('Saving model at step %s' % step)
with open('%s/loss_train'%FLAGS.model_rl_dir,'a') as f:
f.write('%s\n'%loss)
if step == FLAGS.sampling_global_step: break
from collections import defaultdict
import math
import numpy as np
from data_utils import read_data, preprocess_data, clean_text, read_weights
if __name__ == '__main__':
data, _ = read_data(use_loaded=True)
X, y, emb, tokenizer, label_encoder = preprocess_data(data=data,
use_loaded=True)
with open("data/adjectives_people.txt", "r", encoding="utf-8") as fin:
identity_columns = [line.strip() for line in fin.readlines()]
cleaned_text = data["text"].apply(clean_text).values
debias_weights = np.ones(len(y))
num_pos, num_all = defaultdict(int), defaultdict(int)
sum_pos = sum([debias_weights[i] for i in range(len(y)) if y[i] == 0])
sum_all = sum(debias_weights)
for idty in identity_columns:
for i in range(len(cleaned_text)):
ok = False
sen = cleaned_text[i]
if idty in ["american", "african"]:
sen_split = sen.split()
for j in range(len(sen_split)):
if sen_split[j] == idty:
if j == 0 or " ".join([sen_split[j - 1], sen_split[j]
]) != "american african":
if j == len(sen_split) - 1 or " ".join([
sen_split[j], sen_split[j + 1]
def train():
images, labels = data_utils.read_data(FLAGS.train_data_dir,
FLAGS.val_data_dir,
FLAGS.test_data_dir, FLAGS.channel,
FLAGS.img_size, FLAGS.n_aug_img)
n_data = np.shape(images["train"])[0]
print("Number of training data: %d" % (n_data))
g = tf.Graph()
with g.as_default():
ops = get_ops(images, labels)
child_ops = ops["child"]
controller_ops = ops["controller"]
saver = tf.train.Saver(max_to_keep=2)
checkpoint_saver_hook = tf.train.CheckpointSaverHook(
FLAGS.output_dir,
save_steps=child_ops["num_train_batches"],
saver=saver)
hooks = [checkpoint_saver_hook]
if FLAGS.child_sync_replicas:
sync_replicas_hook = child_ops["optimizer"].make_session_run_hook(
True)
hooks.append(sync_replicas_hook)
if FLAGS.controller_training and FLAGS.controller_sync_replicas:
sync_replicas_hook = controller_ops[
"optimizer"].make_session_run_hook(True)
hooks.append(sync_replicas_hook)
print("-" * 80)
print("Starting session")
config = tf.ConfigProto(allow_soft_placement=True)
with tf.train.SingularMonitoredSession(
config=config, hooks=hooks,
checkpoint_dir=FLAGS.output_dir) as sess:
start_time = time.time()
while True:
run_ops = [
child_ops["loss"], child_ops["lr"], child_ops["grad_norm"],
child_ops["train_acc"], child_ops["train_op"]
]
loss, lr, gn, tr_acc, _ = sess.run(run_ops)
global_step = sess.run(child_ops["global_step"])
if FLAGS.child_sync_replicas:
actual_step = global_step * FLAGS.num_aggregate
else:
actual_step = global_step
epoch = actual_step // ops["num_train_batches"]
curr_time = time.time()
if global_step % FLAGS.log_every == 0:
log_string = ""
log_string += "epoch = {:<6d}".format(epoch)
log_string += "ch_step = {:<6d}".format(global_step)
log_string += " loss = {:<8.6f}".format(loss)
log_string += " lr = {:<8.4f}".format(lr)
log_string += " |g| = {:<8.4f}".format(gn)
log_string += " tr_acc = {:<3d}/{:>3d}".format(
tr_acc, FLAGS.batch_size)
log_string += " mins = {:<10.2f}".format(
float(curr_time - start_time) / 60)
print(log_string)
if actual_step % ops["eval_every"] == 0:
if (FLAGS.controller_training
and epoch % FLAGS.controller_train_every == 0):
print("Epoch {}: Training controller".format(epoch))
for ct_step in range(FLAGS.controller_train_steps *
FLAGS.controller_num_aggregate):
run_ops = [
controller_ops["loss"],
controller_ops["entropy"],
controller_ops["lr"],
controller_ops["grad_norm"],
controller_ops["valid_acc"],
controller_ops["baseline"],
controller_ops["skip_rate"],
controller_ops["train_op"],
]
loss, entropy, lr, gn, val_acc, bl, skip, _ = sess.run(
run_ops)
controller_step = sess.run(
controller_ops["train_step"])
if ct_step % FLAGS.log_every == 0:
curr_time = time.time()
log_string = ""
log_string += "ctrl_step = {:<6d}".format(
controller_step)
log_string += " loss = {:<7.3f}".format(loss)
log_string += " ent = {:<5.2f}".format(entropy)
log_string += " lr = {:<6.4f}".format(lr)
log_string += " |g| = {:<8.4f}".format(gn)
log_string += " acc = {:<6.4f}".format(val_acc)
log_string += " bl = {:<5.2f}".format(bl)
log_string += " mins = {:<.2f}".format(
float(curr_time - start_time) / 60)
print(log_string)
print("Here are 10 architectures")
for _ in range(10):
arc, acc = sess.run([
controller_ops["sample_arc"],
controller_ops["valid_acc"],
])
if FLAGS.search_for == "micro":
normal_arc, reduce_arc = arc
print(np.reshape(normal_arc, [-1]))
print(np.reshape(reduce_arc, [-1]))
else:
start = 0
for layer_id in range(FLAGS.child_num_layers):
if FLAGS.controller_search_whole_channels:
end = start + 1 + layer_id
else:
end = start + 2 * FLAGS.child_num_branches + layer_id
print(np.reshape(arc[start:end], [-1]))
start = end
print("val_acc = {:<6.4f}".format(acc))
print("-" * 80)
print("Epoch {}: Eval".format(epoch))
if FLAGS.child_fixed_arc is None:
ops["eval_func"](sess, "valid")
ops["eval_func"](sess, "test")
if epoch >= FLAGS.num_epochs:
break
import data_utils
import sys
import numpy as np
import os
if __name__ == "__main__":
data_dir = '../generate_dataset/'
test_set = data_utils.read_data(data_dir, 'test', 100000)
res = open(sys.argv[1]).readlines()
res = [float(fi[:-1]) for fi in res]
ser = 0
fout = open('./trec_out.txt', 'w')
for i, j in zip(test_set.initial_list, test_set.qids):
l = np.array(res[ser: ser + len(i)])
r = np.argsort(-l)
for s, si in zip(r, range(len(r))):
fout.write(('%s Q0 %s %d %f SVM\n') %
(j, test_set.dids[i[s]], si+1, l[s]))
ser += len(r)
fout.close()
os.system('./trec_eval -c -m ndcg_cut.1,3,5,10 ' + data_dir +
'test/test.qrels ./trec_out.txt > res.out')
print(''.join(open('./res.out').readlines()))
os.system('./trec_eval -c ' + data_dir +
'test/test.qrels ./trec_out.txt > res.out')
print(''.join(open('./res.out').readlines()))
def train():
# Prepare data.
print("Reading data in %s" % FLAGS.data_dir)
train_set = data_utils.read_data(FLAGS.data_dir, 'train',
FLAGS.train_list_cutoff)
valid_set = data_utils.read_data(FLAGS.data_dir, 'valid',
FLAGS.train_list_cutoff)
print("Rank list size %d" % train_set.rank_list_size)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Create model.
print("Creating model...")
model = create_model(sess, train_set, False)
print("Created %d layers of %d units." %
(model.hparams.num_layers, model.embed_size))
# Create tensorboard summarizations.
train_writer = tf.summary.FileWriter(FLAGS.train_dir + '/train_log',
sess.graph)
valid_writer = tf.summary.FileWriter(FLAGS.train_dir + '/valid_log')
#pad data
train_set.pad(train_set.rank_list_size)
valid_set.pad(valid_set.rank_list_size)
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
best_loss = None
while True:
# Get a batch and make a step.
start_time = time.time()
input_feed, _ = model.get_batch(train_set)
step_loss, _, summary = model.step(sess, input_feed, False)
step_time += (time.time() -
start_time) / FLAGS.steps_per_checkpoint
loss += step_loss / FLAGS.steps_per_checkpoint
current_step += 1
train_writer.add_summary(summary, current_step)
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.steps_per_checkpoint == 0:
# Print statistics for the previous epoch.
#loss = math.exp(loss) if loss < 300 else float('inf')
print("global step %d learning rate %.4f step-time %.2f loss "
"%.2f" % (model.global_step.eval(),
model.learning_rate.eval(), step_time, loss))
#train_writer.add_summary({'step-time':step_time, 'loss':loss}, current_step)
# Decrease learning rate if no improvement was seen over last 3 times.
#if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
# sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
# Validate model
it = 0
count_batch = 0.0
valid_loss = 0
while it < len(valid_set.initial_list) - model.batch_size:
input_feed, _ = model.get_next_batch(it, valid_set)
v_loss, results, summary = model.step(
sess, input_feed, True)
it += model.batch_size
valid_loss += v_loss
count_batch += 1.0
valid_writer.add_summary(summary, current_step)
valid_loss /= count_batch
#valid_loss = math.exp(valid_loss) if valid_loss < 300 else float('inf')
print(" eval: loss %.2f" % (valid_loss))
# Save checkpoint and zero timer and loss. # need to rethink
#if best_loss == None or best_loss >= eval_ppx:
# best_loss = eval_ppx
checkpoint_path = os.path.join(FLAGS.train_dir, "DLA.ckpt")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
if loss == float('inf'):
break
step_time, loss = 0.0, 0.0
sys.stdout.flush()
if FLAGS.max_train_iteration > 0 and current_step > FLAGS.max_train_iteration:
break
if word in dictionary: encoded_word = dictionary[word] else: encoded_word = dictionary['UNK'] encoded_sentence += str(encoded_word) + " " print sentence encoded_sentence = encoded_sentence[:-1] # Remove final space f.write(encoded_sentence + '\n') # Write sentence to file f.close() # Generate dictionary for dataset tokenized_data = data_utils.read_data(num_movie_scripts) print '-------- tokenized_data' print tokenized_data[:10] data, count, dictionary, reverse_dictionary = data_utils.build_dataset(tokenized_data, vocabulary_size) print '-------- data' print data print '-------- count' print count print '-------- dictionary' data_utils.print_dic(dictionary, 5) print dictionary print '-------- reverse_dictionary' data_utils.print_dic(reverse_dictionary, 5) print reverse_dictionary
