def test(model, batch_size, data_dir, dataset, sess): data = DataGenerator(data_dir, dataset=dataset, portion='test') iters = int(np.ceil(1.0 * data.size / batch_size)) preds = 0 for i in range(iters): seqs, lens, gold = data.get_batch(i) pred = sess.run(model.correct, { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if (i + 1) * batch_size < data.size: preds += len(pred[pred == True]) else: remain = data.size - i * batch_size pred = pred[:remain] preds += len(pred[pred == True]) if i % 100 == 0: sys.stdout.write('\r{}/{}'.format(i, iters)) sys.stdout.flush() accuracy = 100. * preds / data.size print('\rTest accuracy: {:.2f}'.format(accuracy))
def regulate(model, iters, batch_size, data_dir, dataset, embed_path, save_iter, saver, sess): data = DataGenerator(data_dir, dataset=dataset, portion='tra') optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) train_op = optimizer.minimize(model.loss, var_list=[model.embed]) var_list = [v.name for v in tf.trainable_variables()] sess.run(tf.global_variables_initializer()) for i in range(iters): if i % (data.size / batch_size) == 0: data.shuffle() seqs, lens, gold = data.get_batch(i) loss, acc, w, _ = sess.run( [model.loss, model.accuracy, var_list, train_op], { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if ((i + 1) % 100 == 0): print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format( i + 1, loss, acc)) if ((i + 1) == save_iter): weight_dict = {} for key, value in zip(var_list, w): weight_dict[key] = value embed = weight_dict['embed:0'] np.save(embed_path, embed)
def train(model, iters, batch_size, data_dir, dataset, ckpt_path, save_iter, saver, sess): data = DataGenerator(data_dir, dataset=dataset, portion='tra') optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) train_op = optimizer.minimize(model.loss) sess.run(tf.global_variables_initializer()) for i in range(iters): if i % (data.size / batch_size) == 0: data.shuffle() seqs, lens, gold = data.get_batch(i) loss, acc, _ = sess.run([model.loss, model.accuracy, train_op], { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if ((i + 1) % 100 == 0): print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format( i + 1, loss, acc)) if ((i + 1) == save_iter): saver.save(sess, ckpt_path, write_meta_graph=False) print('| Saved to {}'.format(ckpt_path)) break
def main(): print('loading modules ...') gen = DataGenerator('protain/all/train/', 'protain/all/test/', 'protain/all/train.csv') model = Resnet(resnet_layers=4, channels=[4, 16, 16, 32, 32]) print('Done') epoch = 10 for i in range(epoch): val_x, val_y = gen.get_validation_set() bar = tqdm(gen.get_batch(), total=len(gen.train_ids) // 8) for x, y in bar: loss = model.train(x, y) bar.set_description('loss = {:.5f}'.format(loss)) preds = np.array([[int(y >= 0.5) for y in model.predict([x])[0]] for x in tqdm(val_x)]) print('[epoch {}]: f1_macro = {}'.format(i, f1_score(val_y, preds, average='macro'))) preds_test = [(name, [i for i, y in enumerate(model.predict([x])[0]) if y >= 0.5]) for name, x in gen.get_test_set()] with open('submission.csv', 'w') as f: f.write('Id,Predicted\n') for id_, preds in preds_test: f.write('{},{}\n'.format(id_, ' '.join(list(map(str, preds)))))
# 由于样本不均衡我们添加上class权重 class_weight = tf.constant([[1, 1.26, 2.64, 7.0, 2.73, 1.72, 2.70, 3.66, 3.30]], dtype=tf.float32) weight_per_label = tf.transpose(tf.matmul(tf.cast(onehot, tf.float32), tf.transpose(class_weight))) # shape [1, batch_size] xent = tf.multiply(weight_per_label, tf.nn.softmax_cross_entropy_with_logits(logits=output_concat, labels=onehot)) # shape [1, batch_size] # cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y_predict, labels=batch_y) #cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=output_concat, labels=onehot) cross_entropy_mean = tf.reduce_mean(xent) regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) loss = tf.add_n([cross_entropy_mean] + regularization_losses) tf.summary.scalar('train_loss', loss) return loss ##################### get the input pipline ############################ DataGenerator = DataGenerator.DataGenerator() TrainDataset = DataGenerator.get_batch(BATCHSIZE, tag="training") EvalDataset = DataGenerator.get_batch(BATCHSIZE, tag="evaling") # get the dataset statistics trainset_length = 380000 eval_set_length = 20000 print("train_set_length:%d" % trainset_length) print("eval_set_length:%d" % eval_set_length) iterator = tf.data.Iterator.from_structure(EvalDataset.output_types, EvalDataset.output_shapes) next_batch = iterator.get_next() training_init_op = iterator.make_initializer(TrainDataset) validation_init_op = iterator.make_initializer(EvalDataset) ##################### get the input pipline ############################
result_txt_file = "result_version19_new.txt" CHECKPOINT_DIR = './checkpoint_version_final/' NUM_CLASSES = 9 BATCHSIZE = 1 def tool(AreaID): last = 6 - len(AreaID) for key in range(last): AreaID = "0" + AreaID return AreaID ##################### get the input pipline ############################ DataGenerator = DataGenerator.DataGenerator() TestDataset = DataGenerator.get_batch(BATCHSIZE, tag="testing") # get the dataset statistics test_set_length = 100000 print("test_set_length:%d" % test_set_length) iterator = TestDataset.make_one_shot_iterator() next_batch = iterator.get_next() ##################### get the input pipline ############################ ##################### setup the network ################################ x = tf.placeholder(tf.float32, shape=(None, 88, 88, 3)) visit_array1 = tf.placeholder(tf.float32, shape=(None, 174, 24, 2)) #visit_array2 = tf.placeholder(tf.float32, shape=(None, 500)) is_training = tf.placeholder('bool', []) depth = 50 # 可以是50、101、152
def trace(model, model_name, model_size, batch_size, data_dir, dataset, embed_name, embed, sess): data = DataGenerator(data_dir, dataset=dataset, portion='val') iters = int(np.ceil(1.0 * data.size / batch_size)) succ = [] fail = [] ins = [] outs = [] for i in range(iters): seqs, lens, gold = data.get_batch(i, batch_size=batch_size) if model_name == 'lstm': pred, cell_state, history, fc_w, fc_b\ = sess.run([model.correct, model.cell_state, model.history, model.fc_weight, model.fc_bias], {model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens}) else: pred, history, fc_w, fc_b\ = sess.run([model.correct, model.history, model.fc_weight, model.fc_bias], {model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens}) for j in range(batch_size): idx = j + i * batch_size if idx >= data.size: break if pred[j]: succ.append(idx) else: fail.append(idx) step_outs = np.add(np.matmul(history[j], fc_w), fc_b) _ins = [] _outs = [] for t in range(lens[j] + 1): if model_name == 'lstm': c_t = cell_state[j, t] h_t = history[j, t] if t < lens[j] or lens[j] == 0: x_t = embed[seqs[j, t]] if model_name == 'lstm': step_in = np.concatenate([x_t, c_t, h_t]) else: step_in = np.concatenate([x_t, h_t]) _ins.append(step_in) if t > 0 or lens[j] == 0: _outs.append(step_outs[t]) ins.append(np.array(_ins)) outs.append(np.array(_outs)) if dataset == 'Yelp' and i >= 99: break succ_ins = np.concatenate(np.take(np.array(ins), succ, axis=0)) succ_outs = np.concatenate(np.take(np.array(outs), succ, axis=0)) fail_ins = np.concatenate(np.take(np.array(ins), fail, axis=0)) fail_outs = np.concatenate(np.take(np.array(outs), fail, axis=0)) prefix = '{}/{}/traces/val_{}_{}_{}'.format(data_dir, dataset, model_name, model_size, embed_name) np.save('{}_succ_ins'.format(prefix), succ_ins) np.save('{}_succ_outs'.format(prefix), succ_outs) np.save('{}_fail_ins'.format(prefix), fail_ins) np.save('{}_fail_outs'.format(prefix), fail_outs) size = 0 for pred in ['succ', 'fail']: for step in ['ins', 'outs']: path = '{}_{}_{}.npy'.format(prefix, pred, step) size += os.path.getsize(path) return (size / 1024. / 1024.)