def test(model, batch_size, data_dir, dataset, sess):
data = DataGenerator(data_dir, dataset=dataset, portion='test')
iters = int(np.ceil(1.0 * data.size / batch_size))
preds = 0
for i in range(iters):
seqs, lens, gold = data.get_batch(i)
pred = sess.run(model.correct, {
model.x_ph: seqs,
model.y_ph: gold,
model.len_ph: lens
})
if (i + 1) * batch_size < data.size:
preds += len(pred[pred == True])
else:
remain = data.size - i * batch_size
pred = pred[:remain]
preds += len(pred[pred == True])
if i % 100 == 0:
sys.stdout.write('\r{}/{}'.format(i, iters))
sys.stdout.flush()
accuracy = 100. * preds / data.size
print('\rTest accuracy: {:.2f}'.format(accuracy))
def regulate(model, iters, batch_size, data_dir, dataset, embed_path,
save_iter, saver, sess):
data = DataGenerator(data_dir, dataset=dataset, portion='tra')
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
train_op = optimizer.minimize(model.loss, var_list=[model.embed])
var_list = [v.name for v in tf.trainable_variables()]
sess.run(tf.global_variables_initializer())
for i in range(iters):
if i % (data.size / batch_size) == 0:
data.shuffle()
seqs, lens, gold = data.get_batch(i)
loss, acc, w, _ = sess.run(
[model.loss, model.accuracy, var_list, train_op], {
model.x_ph: seqs,
model.y_ph: gold,
model.len_ph: lens
})
if ((i + 1) % 100 == 0):
print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format(
i + 1, loss, acc))
if ((i + 1) == save_iter):
weight_dict = {}
for key, value in zip(var_list, w):
weight_dict[key] = value
embed = weight_dict['embed:0']
np.save(embed_path, embed)
def train(model, iters, batch_size, data_dir, dataset, ckpt_path, save_iter,
saver, sess):
data = DataGenerator(data_dir, dataset=dataset, portion='tra')
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
train_op = optimizer.minimize(model.loss)
sess.run(tf.global_variables_initializer())
for i in range(iters):
if i % (data.size / batch_size) == 0:
data.shuffle()
seqs, lens, gold = data.get_batch(i)
loss, acc, _ = sess.run([model.loss, model.accuracy, train_op], {
model.x_ph: seqs,
model.y_ph: gold,
model.len_ph: lens
})
if ((i + 1) % 100 == 0):
print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format(
i + 1, loss, acc))
if ((i + 1) == save_iter):
saver.save(sess, ckpt_path, write_meta_graph=False)
print('| Saved to {}'.format(ckpt_path))
break
def main():
print('loading modules ...')
gen = DataGenerator('protain/all/train/', 'protain/all/test/', 'protain/all/train.csv')
model = Resnet(resnet_layers=4, channels=[4, 16, 16, 32, 32])
print('Done')
epoch = 10
for i in range(epoch):
val_x, val_y = gen.get_validation_set()
bar = tqdm(gen.get_batch(), total=len(gen.train_ids) // 8)
for x, y in bar:
loss = model.train(x, y)
bar.set_description('loss = {:.5f}'.format(loss))
preds = np.array([[int(y >= 0.5) for y in model.predict([x])[0]] for x in tqdm(val_x)])
print('[epoch {}]: f1_macro = {}'.format(i, f1_score(val_y, preds, average='macro')))
preds_test = [(name, [i for i, y in enumerate(model.predict([x])[0]) if y >= 0.5]) for name, x in gen.get_test_set()]
with open('submission.csv', 'w') as f:
f.write('Id,Predicted\n')
for id_, preds in preds_test:
f.write('{},{}\n'.format(id_, ' '.join(list(map(str, preds)))))
# 由于样本不均衡我们添加上class权重
class_weight = tf.constant([[1, 1.26, 2.64, 7.0, 2.73, 1.72, 2.70, 3.66, 3.30]], dtype=tf.float32)
weight_per_label = tf.transpose(tf.matmul(tf.cast(onehot, tf.float32), tf.transpose(class_weight))) # shape [1, batch_size]
xent = tf.multiply(weight_per_label, tf.nn.softmax_cross_entropy_with_logits(logits=output_concat, labels=onehot)) # shape [1, batch_size]
# cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y_predict, labels=batch_y)
#cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=output_concat, labels=onehot)
cross_entropy_mean = tf.reduce_mean(xent)
regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
loss = tf.add_n([cross_entropy_mean] + regularization_losses)
tf.summary.scalar('train_loss', loss)
return loss
##################### get the input pipline ############################
DataGenerator = DataGenerator.DataGenerator()
TrainDataset = DataGenerator.get_batch(BATCHSIZE, tag="training")
EvalDataset = DataGenerator.get_batch(BATCHSIZE, tag="evaling")
# get the dataset statistics
trainset_length = 380000
eval_set_length = 20000
print("train_set_length:%d" % trainset_length)
print("eval_set_length:%d" % eval_set_length)
iterator = tf.data.Iterator.from_structure(EvalDataset.output_types, EvalDataset.output_shapes)
next_batch = iterator.get_next()
training_init_op = iterator.make_initializer(TrainDataset)
validation_init_op = iterator.make_initializer(EvalDataset)
##################### get the input pipline ############################
result_txt_file = "result_version19_new.txt"
CHECKPOINT_DIR = './checkpoint_version_final/'
NUM_CLASSES = 9
BATCHSIZE = 1
def tool(AreaID):
last = 6 - len(AreaID)
for key in range(last):
AreaID = "0" + AreaID
return AreaID
##################### get the input pipline ############################
DataGenerator = DataGenerator.DataGenerator()
TestDataset = DataGenerator.get_batch(BATCHSIZE, tag="testing")
# get the dataset statistics
test_set_length = 100000
print("test_set_length:%d" % test_set_length)
iterator = TestDataset.make_one_shot_iterator()
next_batch = iterator.get_next()
##################### get the input pipline ############################
##################### setup the network ################################
x = tf.placeholder(tf.float32, shape=(None, 88, 88, 3))
visit_array1 = tf.placeholder(tf.float32, shape=(None, 174, 24, 2))
#visit_array2 = tf.placeholder(tf.float32, shape=(None, 500))
is_training = tf.placeholder('bool', [])
depth = 50 # 可以是50、101、152
def trace(model, model_name, model_size, batch_size, data_dir, dataset,
embed_name, embed, sess):
data = DataGenerator(data_dir, dataset=dataset, portion='val')
iters = int(np.ceil(1.0 * data.size / batch_size))
succ = []
fail = []
ins = []
outs = []
for i in range(iters):
seqs, lens, gold = data.get_batch(i, batch_size=batch_size)
if model_name == 'lstm':
pred, cell_state, history, fc_w, fc_b\
= sess.run([model.correct, model.cell_state, model.history,
model.fc_weight, model.fc_bias],
{model.x_ph: seqs, model.y_ph: gold,
model.len_ph: lens})
else:
pred, history, fc_w, fc_b\
= sess.run([model.correct, model.history,
model.fc_weight, model.fc_bias],
{model.x_ph: seqs, model.y_ph: gold,
model.len_ph: lens})
for j in range(batch_size):
idx = j + i * batch_size
if idx >= data.size:
break
if pred[j]:
succ.append(idx)
else:
fail.append(idx)
step_outs = np.add(np.matmul(history[j], fc_w), fc_b)
_ins = []
_outs = []
for t in range(lens[j] + 1):
if model_name == 'lstm':
c_t = cell_state[j, t]
h_t = history[j, t]
if t < lens[j] or lens[j] == 0:
x_t = embed[seqs[j, t]]
if model_name == 'lstm':
step_in = np.concatenate([x_t, c_t, h_t])
else:
step_in = np.concatenate([x_t, h_t])
_ins.append(step_in)
if t > 0 or lens[j] == 0:
_outs.append(step_outs[t])
ins.append(np.array(_ins))
outs.append(np.array(_outs))
if dataset == 'Yelp' and i >= 99:
break
succ_ins = np.concatenate(np.take(np.array(ins), succ, axis=0))
succ_outs = np.concatenate(np.take(np.array(outs), succ, axis=0))
fail_ins = np.concatenate(np.take(np.array(ins), fail, axis=0))
fail_outs = np.concatenate(np.take(np.array(outs), fail, axis=0))
prefix = '{}/{}/traces/val_{}_{}_{}'.format(data_dir, dataset, model_name,
model_size, embed_name)
np.save('{}_succ_ins'.format(prefix), succ_ins)
np.save('{}_succ_outs'.format(prefix), succ_outs)
np.save('{}_fail_ins'.format(prefix), fail_ins)
np.save('{}_fail_outs'.format(prefix), fail_outs)
size = 0
for pred in ['succ', 'fail']:
for step in ['ins', 'outs']:
path = '{}_{}_{}.npy'.format(prefix, pred, step)
size += os.path.getsize(path)
return (size / 1024. / 1024.)
