def train(args):
global PIECES, RECTANGLES, MODE, NUM_CHANNELS, OUT_CHANNELS, CROP_SIZE, MIN_SIZE, MAX_SIZE, FLIP_VERTICAL, FLIP_HORIZONTAL, BAND8, HEATMAP_DIR, FINAL_SIZE
HEATMAP_DIR = args.heatmap_dir
MIN_SIZE = args.min_size
MAX_SIZE = args.max_size
CROP_SIZE = args.crop
FINAL_SIZE = args.final
NUM_CHANNELS = 3
OUT_CHANNELS = args.out_channels
device = '/cpu:0' if args.cpu else '/gpu:{idx}'.format(idx=select_gpu())
print 'device', device
valid_list, train_list, test_list = read_csv()
print len(train_list), train_list[:10], len(valid_list), valid_list[:10]
print len(test_list), test_list[:10]
with tf.device(device):
x = tf.placeholder(tf.float32,
shape=[None, CROP_SIZE, CROP_SIZE, NUM_CHANNELS],
name='x')
y_heat = tf.placeholder(
tf.float32,
shape=[None, FINAL_SIZE, FINAL_SIZE, OUT_CHANNELS],
name='y_heat')
mask = tf.placeholder(tf.float32,
shape=[None, OUT_CHANNELS],
name='mask')
coefficients = tf.placeholder(tf.float32,
shape=[OUT_CHANNELS],
name='coefficients')
signal = x
tf.summary.image('input', x, max_outputs=10)
for i in range(OUT_CHANNELS):
tf.summary.image('heatmap_' + str(i),
tf.slice(y_heat, (0, 0, 0, i), (-1, -1, -1, 1)),
max_outputs=5)
keep_prob = tf.placeholder("float")
learning_rate = tf.placeholder(tf.float32)
phase_train = tf.placeholder(tf.bool)
if args.dilarch == 0:
#front-end
dilarch = [
(8, 1, 3, 0), (8, 1, 3, 1), (16, 1, 3, 0), (16, 1, 3, 1),
(32, 1, 3, 0), (32, 1, 3, 0), (64, 2, 3, 0), (64, 2, 3, 0),
(64, 4, 3, 0), (64, 4, 3, 0), (64, 8, 3, 0), (64, 8, 3, 0),
(64, 16, 3, 0), (64, 16, 3, 0), (64, 4, 3, 0), (64, 4, 3, 0)
] + args.num_dense * [(args.neuron_dense, 1, 1, 0)] + [
(OUT_CHANNELS, 1, 1, 0)
]
dilarch = map(
lambda (a, b, c, d): (int(args.arch_multiplier * a), b, c, d),
dilarch[:-1]) + dilarch[-1:]
if args.filters_bound:
dilarch = map(
lambda (a, b, c, d): (min(a, args.filters_bound), b, c, d),
dilarch[:-1]) + dilarch[-1:]
elif args.dilarch == 1:
dilarch = [(10, 1), (10, 1), (10, 2), (10, 4), (10, 8), (10, 16),
(10, 32), (10, 16), (10, 4), (OUT_CHANNELS, 1)]
dilarch = map(lambda (a, b): (int(args.arch_multiplier * a), b),
dilarch[:-1]) + dilarch[-1:]
else:
assert (0)
print 'dilarch', dilarch
logits = construct_dilated(signal,
input_channels=NUM_CHANNELS,
arch=dilarch,
stddev=args.conv_stddev,
bias=args.conv_bias,
use_batch_norm=args.batch_norm,
final_bn=args.final_bn,
phase_train=phase_train)
sigmoid = tf.nn.sigmoid(logits)
logloss = tf.nn.sigmoid_cross_entropy_with_logits(logits, y_heat)
mean_logloss = tf.reduce_mean(logloss, axis=[1, 2])
print 'mean_logloss', mean_logloss
row_objective = tf.reduce_sum(mean_logloss * mask, axis=0)
print 'row_objective', row_objective
objective = tf.reduce_sum(row_objective * coefficients)
num_batches_train = 400 #len(train_list) / args.batch_size
num_batches_valid = len(valid_list) / args.batch_size
print 'num_batches train {}, valid {}'.format(num_batches_train,
num_batches_valid)
#global_step = tf.Variable(0, trainable=False)
#learning_rate = tf.train.exponential_decay(args.lr, global_step,
# num_batches_train, args.epoch_decay, staircase=False)
if args.optimizer == 'adam':
train_step = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(objective)
elif args.optimizer == 'momentum':
train_step = tf.train.MomentumOptimizer(
learning_rate=learning_rate, momentum=0.9).minimize(objective)
else:
assert (0)
#for var in tf.all_variables():
# print var.name
train_batch = tf_batch(train_list, batch_size=args.batch_size)
valid_batch = tf_batch(valid_list, batch_size=args.batch_size)
test_batch = tf_batch(test_list, batch_size=args.batch_size)
saver = tf.train.Saver(max_to_keep=100)
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as sess:
merged = tf.summary.merge_all()
tensorboard_writer = tf.summary.FileWriter('logs', sess.graph)
print 'session started'
sess.run(tf.global_variables_initializer(),
feed_dict={phase_train: True})
print 'variables initialized'
threads = tf.train.start_queue_runners(
sess, coord=tf.train.Coordinator())
print 'queues started'
if args.means:
mean, stddev = pickle.load(open(args.means))
mean = np.array(mean)
stddev = np.array(stddev)
else:
mean, stddev = estimate_mean(sess,
train_batch,
sampled_batches=5)
print 'mean estimated, shape={}, flattened={}'.format(
mean.shape, mean)
print 'stddev estimated, shape={}, flattened={}'.format(
stddev.shape, stddev)
if args.means_store:
f = open(args.means_store, 'w')
pickle.dump((mean, stddev), f)
f.close()
epoch = 0
if args.restore_path:
saver.restore(sess, args.restore_path)
if args.predict:
#pass_list = [(train_batch, len(train_list)-13*args.batch_size, 'train', True),
# (train_batch, len(train_list), 'train', False),
# (valid_batch, len(valid_list), 'val', False),
# (test_batch, len(test_list), 'test', False)]
pass_list = [
(train_batch, train_list[:200 * 32], 'train', True),
# (train_batch, len(train_list), 'train', False),
(valid_batch, valid_list, 'val', False),
(test_batch, test_list, 'test', False)
]
# if LAPTOP:
# pass_list = pass_list[1:]
#pass_list = [(train_batch, 10, 'train'), (valid_batch, 10, 'val'), (test_batch, 10, 'test')]
valid_dict = load_valid_heatmaps(valid_list)
for (batch_gen, namelist, pref, pt) in pass_list:
size = len(namelist)
print size
print namelist[:10]
namelist = extract_names(namelist)
print namelist[:10]
nameset = {}
pred_dict = {}
tot_cnt = 0
for name in namelist:
nameset[name] = 0
while tot_cnt < size * args.pred_iter:
print 'tot_cnt', tot_cnt
names, batch_x, _, _, aug_data = sess.run(batch_gen)
batch_x -= mean
batch_x /= stddev
print 'names', names[:10], len(names)
names = extract_names(names)
print 'names', names[:10], len(names)
print 'shape', batch_x.shape
[heat_pred] = sess.run([sigmoid],
feed_dict={
x: batch_x,
phase_train: pt
}) #TODO check false
print 'heat_pred.shape', heat_pred.shape
if tot_cnt == 0:
print 'aug_data', aug_data
for i in range(args.batch_size):
name = names[i]
if pref == 'train' or nameset[
name] < args.pred_iter:
if pref != 'train':
nameset[name] += 1
tot_cnt += 1
if not pt:
[dx] = aug_data[i]
if dx > 0:
s = CROP_SIZE + 2 * dx
newheat = np.zeros(
(s, s, OUT_CHANNELS),
dtype=np.float32)
newheat[dx:CROP_SIZE + dx,
dx:CROP_SIZE +
dx, :] = heat_pred[i, :, :, :]
else:
dx = -dx
newheat = heat_pred[i,
dx:CROP_SIZE - dx,
dx:CROP_SIZE -
dx, :]
newheat = cv2.resize(
newheat, (CROP_SIZE, CROP_SIZE))
newheat = newheat[:, :, 0:
1] #storing only the cancer
if pref != 'train':
if nameset[name] == 1:
pred_dict[name] = newheat
else:
pred_dict[name] += newheat
if pref != 'train':
losses = []
for name in namelist:
newheat = pred_dict[name] / nameset[name]
if name in valid_dict:
loss = valid_dict[name] * np.log(newheat[:, :, 0]+1e-6) +\
(1.0 - valid_dict[name]) * (np.log(1.0-newheat[:, :, 0]+1e-6))
loss = np.mean(loss)
print 'loss', loss
losses.append(loss)
#for c in range(OUT_CHANNELS):
for c in range(1):
curpred = newheat[:, :, c]
print 'mean', np.mean(curpred), 'max', np.max(
curpred)
curpred *= 255.0
print 'mean', np.mean(curpred), 'max', np.max(
curpred)
img = np.asarray(curpred, dtype=float)
print 'mean', np.mean(img), 'max', np.max(img)
img = cv2.resize(img, (512, 512))
cv2.imwrite(
'predictions/' + pref + '/' + name + '_' +
str(c) + '.jpg', img)
print 'ave loss', np.mean(losses)
return
epoch_lr = args.lr
while epoch < args.num_epochs:
epoch += 1
print 'epoch {} learning rate {}'.format(epoch, epoch_lr)
epoch_start_time = timeit.default_timer()
train_time, load_time, valid_time, dump_time = 0, 0, 0, 0
pass_list = [
([objective, row_objective,
train_step], num_batches_train, train_batch, True),
([objective,
row_objective], num_batches_valid, valid_batch, False)
]
for compute_list, num_batches, batch_gen, pt in pass_list:
load_time = 0
main_time = 0
pass_start = timeit.default_timer()
results = []
means = []
for i in range(num_batches):
start = timeit.default_timer()
_, batch_x, batch_heat, batch_mask, _ = sess.run(
batch_gen)
if epoch == 1:
print 'batch_heat', np.mean(batch_heat), np.max(
batch_heat), np.sum(batch_heat)
means.append(np.mean(batch_heat))
if i < 10:
print 'batch_mask', batch_mask.shape, batch_mask
batch_x -= mean
batch_x /= stddev
load_time += timeit.default_timer() - start
main_time -= timeit.default_timer()
myfeed = {
x:
batch_x,
y_heat:
batch_heat,
phase_train:
pt,
learning_rate:
epoch_lr,
coefficients:
np.array([1] + (OUT_CHANNELS - 1) * [0.05]),
mask:
batch_mask
}
res = sess.run(compute_list, feed_dict=myfeed)
if pt and i % 10 == 0:
sum = sess.run([merged], feed_dict=myfeed)
tensorboard_writer.add_summary(
sum[0], (epoch - 1) * num_batches + i)
tensorboard_writer.flush()
main_time += timeit.default_timer()
results.append([res[0]] + list(res[1]))
if i < 20000:
print 'train', i, res, np.mean(np.array(
results, dtype=float),
axis=0)
full_time = timeit.default_timer() - pass_start
rest_time = full_time - load_time - main_time
pass_res = np.mean(np.array(results, dtype=float), axis=0)
if epoch == 1:
print 'Heat mean', np.mean(means)
print 'Pass res {} time: {} main_time: {} load_time: {} rest: {}'.format(
pass_res, full_time, main_time, load_time, rest_time)
save_path = saver.save(
sess, args.models_dir + '/epoch{}.ckpt'.format(epoch))
print('Model saved in file: %s' % save_path)
epoch_lr *= args.epoch_decay
epoch_time = timeit.default_timer() - epoch_start_time
rest_time = epoch_time - train_time - load_time - valid_time
print 'Epoch {} time {}, train {}, valid {}, load {}, dumping {}, rest {}'.\
format(epoch, epoch_time, train_time, valid_time, load_time, dump_time, rest_time)
def train(args):
global PIECES, RECTANGLES, MODE, NUM_CHANNELS, IMAGE_CROP, FLIP_VERTICAL, FLIP_HORIZONTAL, BAND8
if args.band8:
BAND8 = 1
IMAGE_CROP = (args.size, args.size)
if args.mode == 'heatmap':
MODE = MODE_HEATMAP
NUM_CHANNELS = 11
elif args.mode == 'rectangles':
MODE = MODE_RECTANGLES
NUM_CHANNELS = 6 + 8 * BAND8
else:
assert args.mode == 'rectangles_fake'
MODE = MODE_RECTANGLES_FAKE
NUM_CHANNELS = 6
PIECES = args.pieces
RECTANGLES = args.rectangles
device = '/cpu:0' if LAPTOP else '/gpu:{idx}'.format(idx=select_gpu())
print 'device', device
train_list, valid_list, test_list = read_csv(args.augment, args.smallval)
print len(train_list), train_list[:10], len(valid_list), valid_list[:10]
out_len = args.pieces * args.pieces * args.rectangles
with tf.device(device):
print 'out_len', out_len
if MODE != MODE_HEATMAP:
x = tf.placeholder(
tf.float32,
shape=[None, IMAGE_SIZE[0], IMAGE_SIZE[1], NUM_CHANNELS],
name='x')
else:
x = tf.placeholder(
tf.float32,
shape=[None, IMAGE_CROP[0], IMAGE_CROP[1], NUM_CHANNELS],
name='x')
y_heat = tf.placeholder(tf.float32,
shape=[None, IMAGE_CROP[0], IMAGE_CROP[1], 3],
name='y_heat')
y = tf.placeholder(tf.float32, shape=[None, out_len], name='y')
y2 = tf.placeholder(tf.float32, shape=[None, out_len * 5], name='y2')
mask = tf.placeholder(tf.float32, shape=[None, out_len], name='mask')
mask2 = tf.placeholder(tf.float32,
shape=[None, out_len * 5],
name='mask2')
signal = x
phase_train = tf.placeholder(tf.bool)
keep_prob = tf.placeholder("float")
learning_rate = tf.placeholder(tf.float32)
if args.arch == 0:
arch = [(4, 0, 3), (4, 1, 3), (8, 0, 3), (8, 1, 3), (16, 0, 3),
(16, 0, 3), (16, 1, 3), (32, 0, 3)]
end_size_dict = {'main': 3, 'offset': 3}
elif args.arch == 1:
arch = [(4, 0, 3), (4, 0, 3), (4, 1, 3), (8, 0, 3), (8, 0, 3),
(8, 1, 3), (16, 0, 3), (16, 0, 3), (16, 1, 3), (32, 0, 3),
(32, 0, 3)]
end_size_dict = {'main': 3, 'offset': 3}
elif args.arch > 1:
assert (0)
if args.arch >= 0:
arch = map(lambda (a, b, c): (int(args.arch_multiplier * a), b, c),
arch)
print 'arch', arch
if args.dilarch == 0:
dilarch = [(5, 1), (5, 1), (5, 2), (5, 4), (5, 8), (5, 16),
(3, 32)]
elif args.dilarch == 1:
dilarch = [(10, 1), (10, 1), (10, 2), (10, 4), (10, 8), (10, 16),
(3, 32)]
elif args.dilarch == 2:
dilarch = [(16, 1), (16, 1), (16, 2), (16, 4), (16, 8), (16, 16),
(16, 32), (3, 16)]
elif args.dilarch == 3:
dilarch = [(16, 1), (16, 1), (16, 2), (16, 4), (16, 8), (16, 16),
(16, 32), (16, 64), (3, 16)]
else:
assert (0)
dilarch = map(lambda (a, b): (int(args.arch_multiplier * a), b),
dilarch[:-1]) + dilarch[-1:]
print 'dilarch', dilarch
if MODE != MODE_HEATMAP:
if args.arch < 0:
if args.arch == -1:
signal = avg_pool_2x2(signal)
signal = avg_pool_2x2(signal)
signal = myconv(signal,
8,
NUM_CHANNELS,
RECTANGLES,
stddev=0.05,
bias=0.0,
stride=2)
else:
assert (0)
else:
signal, signal2, image_size = construct_convnet(
signal,
input_channels=NUM_CHANNELS,
arch=arch,
ending_size_dict=end_size_dict,
stddev=args.conv_stddev,
bias=args.conv_bias,
use_batch_norm=args.batch_norm,
use_dropout=args.dropout > 1e-4,
keep_prob=keep_prob,
phase_train=phase_train,
rectangles=args.rectangles)
assert (image_size == (args.pieces, args.pieces))
print signal
print 'signal2', signal2
signal = tf.reshape(signal, [-1, out_len])
signal2 = tf.reshape(signal2, [-1, out_len * 5])
print 'final signal', signal
print 'final signal2', signal2
#mse = tf.reduce_mean((signal - y)**2)
#mae = tf.reduce_mean(tf.abs(signal - y))
#small_accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.greater(signal_small, 0.0), tf.greater(y_small, 0.5)), tf.float32))
#small_logloss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(signal_small, y_small))
logloss = tf.reduce_sum(
tf.nn.sigmoid_cross_entropy_with_logits(signal, y) * mask,
reduction_indices=1)
mseloss = tf.reduce_sum(((signal2 - y2)**2) * mask2,
reduction_indices=1)
mean_logloss = tf.reduce_mean(logloss)
mean_mseloss = tf.reduce_mean(mseloss)
objective = mean_logloss + mean_mseloss * args.mse_mult
else:
prev_signal, logits = construct_dilated(
signal,
input_channels=NUM_CHANNELS,
arch=dilarch,
stddev=args.conv_stddev,
bias=args.conv_bias,
use_batch_norm=args.batch_norm,
final_bn=args.final_bn,
phase_train=phase_train)
softmax = tf.nn.softmax(logits)
logloss = tf.nn.softmax_cross_entropy_with_logits(logits, y_heat)
mean_logloss = tf.reduce_mean(logloss)
objective = mean_logloss
num_batches_train = len(train_list) / args.batch_size
num_batches_valid = len(valid_list) / args.batch_size
print 'num_batches train {}, valid {}'.format(num_batches_train,
num_batches_valid)
#global_step = tf.Variable(0, trainable=False)
#learning_rate = tf.train.exponential_decay(args.lr, global_step,
# num_batches_train, args.epoch_decay, staircase=False)
if args.optimizer == 'adam':
train_step = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(objective)
elif args.optimizer == 'momentum':
train_step = tf.train.MomentumOptimizer(
learning_rate=learning_rate, momentum=0.9).minimize(objective)
else:
assert (0)
for var in tf.all_variables():
print var.name
train_batch = tf_batch(train_list,
batch_size=args.batch_size,
num_labels=out_len)
valid_batch = tf_batch(valid_list,
batch_size=args.batch_size,
num_labels=out_len)
test_batch = tf_batch(test_list,
batch_size=args.batch_size,
num_labels=out_len)
saver = tf.train.Saver(max_to_keep=100)
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as sess:
print 'session started'
sess.run(tf.initialize_all_variables())
print 'variables initialized'
threads = tf.train.start_queue_runners(
sess, coord=tf.train.Coordinator())
print 'queues started'
mean, stddev = estimate_mean(sess, train_batch, sampled_batches=5)
print 'mean estimated, shape={}, flattened={}'.format(
mean.shape, mean)
epoch = 0
if args.restore_path:
saver.restore(sess, args.restore_path)
if args.predict:
pass_list = [
(train_batch, len(train_list) - 13 * args.batch_size,
'train', True),
(train_batch, len(train_list), 'train', False),
(valid_batch, len(valid_list), 'val', False),
(test_batch, len(test_list), 'test', False)
]
if LAPTOP:
pass_list = pass_list[1:]
#pass_list = [(train_batch, 10, 'train'), (valid_batch, 10, 'val'), (test_batch, 10, 'test')]
for (batch_gen, size, pref, pt) in pass_list:
print size
nameset = {}
while len(nameset) < size:
print len(nameset)
if MODE == MODE_HEATMAP:
names, batch_x, _ = sess.run(batch_gen)
batch_x -= mean
batch_x /= stddev
names = map(lambda s: pref + s,
extract_names(names))
print 'names', names[:10]
#TODO prev_signal read here
features, heat_pred = sess.run(
[prev_signal, softmax],
feed_dict={
x: batch_x,
phase_train: pt
}) #TODO check false
print 'features.shape', features.shape
for i in range(args.batch_size):
if names[i] not in nameset:
nameset[names[i]] = 1
if not pt:
cv2.imwrite(
'predictions/' + names[i] + '.jpg',
np.asarray(heat_pred[i] * 255,
dtype=int))
#f = open('predictions/'+names[i]+'-numpy-tab', 'w')
#np.save(f, features[i])
#f.close()
else:
names, batch_x, _, _ = sess.run(batch_gen)
batch_x -= mean
batch_x /= stddev
[logits, sig2] = sess.run(
[signal, signal2],
feed_dict={
x: batch_x,
phase_train: pt,
keep_prob: 1.0 - args.dropout
})
print 'pre names', names[:2]
names = extract_names(names)
print 'names', names[:2]
print 'logits mean', np.mean(logits)
if args.pred_per_image:
cnt = args.batch_size * args.pred_per_image
threshold = np.partition(
logits.reshape([-1]), -cnt)[-cnt]
else:
threshold = args.dump_threshold
if not pt:
dump_logits(logits, sig2, names, threshold,
nameset)
for i in range(args.batch_size):
if names[i] not in nameset:
nameset[names[i]] = 1
if LAPTOP:
print nameset
return
epoch_lr = args.lr
while epoch < args.num_epochs:
epoch += 1
print 'epoch {} learning rate {}'.format(epoch, epoch_lr)
epoch_start_time = timeit.default_timer()
train_time, load_time, valid_time, dump_time = 0, 0, 0, 0
if MODE == MODE_HEATMAP:
if args.train:
pass_list = [([train_step, mean_logloss],
num_batches_train, train_batch, True),
([mean_logloss], num_batches_valid,
valid_batch, False)]
else:
pass_list = [([mean_logloss], num_batches_train - 13,
train_batch, True),
([mean_logloss], num_batches_train,
train_batch, False),
([mean_logloss], num_batches_valid,
valid_batch, False)]
for compute_list, num_batches, batch_gen, pt in pass_list:
load_time = 0
main_time = 0
pass_start = timeit.default_timer()
results = []
for i in range(num_batches):
start = timeit.default_timer()
_, batch_x, batch_heat = sess.run(batch_gen)
batch_x -= mean
batch_x /= stddev
load_time += timeit.default_timer() - start
main_time -= timeit.default_timer()
res = sess.run(compute_list,
feed_dict={
x: batch_x,
y_heat: batch_heat,
phase_train: pt,
learning_rate: epoch_lr
})
main_time += timeit.default_timer()
results.append(res)
if i < 20000:
print 'train', i, res, np.mean(np.array(
results, dtype=float),
axis=0)
full_time = timeit.default_timer() - pass_start
rest_time = full_time - load_time - main_time
pass_res = np.mean(np.array(results, dtype=float),
axis=0)
print 'Pass res {} time: {} main_time: {} load_time: {} rest: {}'.format(
pass_res, full_time, main_time, load_time,
rest_time)
else:
if args.train:
pass_list = [([train_step, mean_logloss, mean_mseloss],
num_batches_train, train_batch, True),
([mean_logloss, mean_mseloss],
num_batches_valid, valid_batch, False)]
else:
pass_list = [([mean_logloss,
mean_mseloss], num_batches_train - 13,
train_batch, True),
([mean_logloss, mean_mseloss],
num_batches_train, train_batch, False),
([mean_logloss, mean_mseloss],
num_batches_valid, valid_batch, False)]
for compute_list, num_batches, batch_gen, pt in pass_list:
load_time = 0
main_time = 0
pass_start = timeit.default_timer()
results = []
for i in range(num_batches):
start = timeit.default_timer()
_, batch_x, batch_y, batch_y2 = sess.run(batch_gen)
batch_x -= mean
batch_x /= stddev
load_time += timeit.default_timer() - start
main_time -= timeit.default_timer()
[logits] = sess.run(
[signal],
feed_dict={
x: batch_x,
phase_train: pt,
keep_prob: 1.0 - args.dropout
})
print 'logits mean', np.mean(logits)
nonzeros = np.sum(batch_y)
batch_mask, batch_mask2, train_acc = calc_mask(
logits, batch_y, nonzeros, args.nonzeros_mult)
res = sess.run(compute_list,
feed_dict={
x: batch_x,
y: batch_y,
y2: batch_y2,
mask: batch_mask,
mask2: batch_mask2,
phase_train: pt,
learning_rate: epoch_lr
})
res.append(train_acc)
main_time += timeit.default_timer()
results.append(res)
if i < 20000:
print 'train', i, res, np.mean(np.array(
results, dtype=float),
axis=0)
full_time = timeit.default_timer() - pass_start
rest_time = full_time - load_time - main_time
pass_res = np.mean(np.array(results, dtype=float),
axis=0)
print 'Pass res {} time: {} main_time: {} load_time: {} rest: {}'.format(
pass_res, full_time, main_time, load_time,
rest_time)
save_path = saver.save(sess,
'models/epoch{}.ckpt'.format(epoch))
print('Model saved in file: %s' % save_path)
epoch_lr *= args.epoch_decay
epoch_time = timeit.default_timer() - epoch_start_time
rest_time = epoch_time - train_time - load_time - valid_time
print 'Epoch {} time {}, train {}, valid {}, load {}, dumping {}, rest {}'.\
format(epoch, epoch_time, train_time, valid_time, load_time, dump_time, rest_time)
import os
import sqlite3
import logging
from subprocess import Popen, PIPE
from db import DBConnection
from utils import format_name, select_gpu, highlight_str, warning_str
LOG = logging.getLogger(__name__)
logging.basicConfig(
filename='kernel-runner.log',
level=logging.INFO,
format="%(asctime)s - %(name)s (%(lineno)s) - %(levelname)s: %(message)s",
datefmt='%Y.%m.%d %H:%M:%S')
DEVICE_NAME, DEVICE_IDX = select_gpu()
print highlight_str("Device selected: " + DEVICE_NAME)
class ApplicationRunner(DBConnection):
def run(self):
self.cursor = self.connection.cursor()
self.cursor.execute(
"""SELECT * FROM Application AS App INNER JOIN Benchmark AS BM
WHERE App.benchmark = BM._id_""")
db_list = []
for row in self.cursor.fetchall():
print "Profiling " + row['name'] + " Kernel: " + row["acronym"]
cmd = os.environ[row['environment']] + row["binary"] + " " + (
from __future__ import print_function
import os
import numpy as np
import tensorflow as tf
from tensorflow.python import pywrap_tensorflow
import configuration
from configuration import parse_flags
from utils import select_gpu
import skip_thoughts_model
import subprocess
import evaluation
tf.logging.set_verbosity(tf.logging.INFO)
FLAGS = parse_flags()
select_gpu(FLAGS.gpu)
lr = float(FLAGS.lr)
lr_decay = float(FLAGS.lr_decay)
train_dir = os.environ["GROUNDSENT_DIR"]
if FLAGS.train_folder:
train_dir = FLAGS.train_folder
else:
train_dir += "NEW_text" + str(FLAGS.lambda_text) + "_rho" + str(
FLAGS.lambda_rho
) + "_rank" + str(FLAGS.lambda_ranking) + "_rt" + str(
FLAGS.lambda_ranking_text
) + "_rtf" + str(
FLAGS.lambda_ranking_text_f