def backwardpro():
with tf.Graph().as_default():
x = tf.placeholder(tf.float32, [None, forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
'''
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
'''
loss = tf.reduce_mean(tf.square(y_ - y)) + tf.add_n(
tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
train_num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,
global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
wf_batch, pet_batch = generate.get_tfrecord(BATCH_SIZE, isTrain=True)
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(STEPS):
xs, ys = sess.run([wf_batch, pet_batch])
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: xs,
y_: ys
})
if i % 1 == 0:
print(
"After %d training step(s), loss on training batch is %g."
% (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
coord.request_stop()
coord.join(threads)
def test():
with tf.Graph().as_default():
x = tf.placeholder(tf.float32, [None, forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, None)
'''
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
'''
saver = tf.train.Saver()
y_predict = tf.add(tf.div(tf.sign(tf.subtract(y, 0.5)), 2), 0.5)
correct_prediction = tf.equal(y_, y_predict)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
wf_batch, pet_batch = generate.get_tfrecord(TEST_NUM, isTrain=False)
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,
coord=coord)
xs, ys = sess.run([wf_batch, pet_batch])
y_predict = sess.run(y, feed_dict={x: xs, y_: ys})
accuracy_score = sess.run(accuracy,
feed_dict={
x: xs,
y_: ys
})
precision = np.divide(np.sum(np.multiply(ys, y_predict)),
np.sum(y_predict))
recall = np.divide(np.sum(np.multiply(ys, y_predict)),
np.sum(ys))
print("After %s training step(s), test accuracy = %g" %
(global_step, accuracy_score))
print("After %s training step(s), test precision = %g" %
(global_step, precision))
print("After %s training step(s), test recall = %g" %
(global_step, recall))
coord.request_stop()
coord.join(threads)
else:
print("No checkpoint found")
return time.sleep(TEST_INTERVAL_SECS)
def test():
with tf.Graph().as_default():
x = tf.placeholder(tf.float32, [None, forward.INPUT_NODE])
#y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, None)
'''
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
'''
saver = tf.train.Saver()
wf_batch, pet_batch = generate.get_tfrecord(TEST_NUM, isTrain=False)
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,
coord=coord)
xs, ys = sess.run([wf_batch, pet_batch])
y_value = sess.run(y, feed_dict={x: xs})
y_c = np.concatenate([[y_value[:, 1]],
[y_value[:, 0]]]).transpose()
y_predict = np.array(y_value > y_c, dtype=np.uint8)
#y_predict = ys
accuracy_score = np.divide(
np.sum(np.multiply(ys, y_predict)),
np.array(ys[:, 0]).size)
precision = np.divide(
np.sum(np.multiply(ys[:, 0], y_predict[:, 0])),
np.sum(y_predict[:, 0]))
recall = np.divide(
np.sum(np.multiply(ys[:, 0], y_predict[:, 0])),
np.sum(ys[:, 0]))
print("After %s training step(s), test accuracy = %g" %
(global_step, accuracy_score))
print("After %s training step(s), test precision = %g" %
(global_step, precision))
print("After %s training step(s), test recall = %g" %
(global_step, recall))
coord.request_stop()
coord.join(threads)
else:
print("No checkpoint found")
return time.sleep(TEST_INTERVAL_SECS)
def restore_model(wf_test, wf_aver):
with tf.Graph().as_default():
x = tf.placeholder(
tf.float32, [1, 1, generate.Length_waveform, forward.NUM_CHANNELS])
#y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, False, None)
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
#saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
reshaped_xs = np.reshape(
wf_test,
(1, 1, generate.Length_waveform, forward.NUM_CHANNELS))
y_value = sess.run(y, feed_dict={x: reshaped_xs})
pe_num = np.around(
np.polyval(np.array(testnn.REG_RAW), wf_aver))
y_predict = np.zeros_like(y_value)
order_y = np.argsort(y_value[0, :])[::-1]
th_v = y_value[0, int(order_y[int(pe_num)])]
y_predict = np.where(y_value > th_v, 1, 0)
#correction of bias
if np.size(np.where(y_predict == 1)) != 0:
a = np.where(y_predict == 1)[1][0]
b = np.where(y_predict == 1)[1][-1]
p = int(np.around((2. * b - 3. * a) / 5))
y_predict[0, p::] = 0
pf_value = np.where(y_predict == 1)[1]
return pf_value
else:
print("No checkpoint file found")
return -1
def restore_model(wf):
with tf.Graph().as_default():
x = tf.placeholder(tf.float32, [1, forward.INPUT_NODE])
y = forward.forwardpro(x, None)
'''
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
'''
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
pf_prob = sess.run(y, feed_dict={x: wf})
#pf_sign = tf.add(tf.div(tf.sign(tf.subtract(y,0.5)),2),0.5)
pf_value = np.where(pf_prob > 0.5)[1] + 1
return pf_value
else:
print("No checkpoint file found")
return -1
def test():
with tf.Graph().as_default():
x = tf.placeholder(
tf.float32,
[TEST_NUM, 1, generate.Length_waveform, forward.NUM_CHANNELS])
#y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, False, None)
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
#saver = tf.train.Saver()
wf_batch, pet_batch, aver_batch = generate.get_tfrecord(TEST_NUM,
isTrain=False)
'''
y_predict = tf.add(tf.div(tf.sign(tf.subtract(y,0.5)),2),0.5)
correct_prediction = tf.equal(y_, y_predict)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
'''
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,
coord=coord)
xs, ys, vs = sess.run([wf_batch, pet_batch, aver_batch])
reshaped_xs = np.reshape(
xs, (TEST_NUM, 1, generate.Length_waveform,
forward.NUM_CHANNELS))
y_value = sess.run(y, feed_dict={x: reshaped_xs})
pe_num = np.around(np.polyval(REG, vs))
y_predict = np.zeros_like(y_value)
for i in range(TEST_NUM):
order_y = np.argsort(y_value[i, :])[::-1]
th_v = y_value[i, :][int(order_y[int(
np.round((pe_num[i])))])]
y_predict[i, :] = np.where(y_value[i, :] > th_v, 1, 0)
#correction of bias
if np.size(np.where(y_predict[i, :])) != 0:
a = np.where(y_predict[i, :] == 1)[0][0]
b = np.where(y_predict[i, :] == 1)[0][-1]
p = int(np.around((2. * b - 3. * a) / 5))
y_predict[i, p::] = 0
accuracy_score = np.divide(
np.sum(np.multiply(ys, y_predict)), np.sum(ys))
precision = np.divide(np.sum(np.multiply(ys, y_predict)),
np.sum(y_predict))
recall = np.divide(np.sum(np.multiply(ys, y_predict)),
np.sum(ys))
'''
y_predict_value = sess.run(y_predict, feed_dict={x: reshaped_xs, y_: ys})
accuracy_score = sess.run(accuracy, feed_dict={y_: ys, y_predict: y_predict_value})
precision = np.divide(np.sum(np.multiply(ys, y_predict_value)), np.sum(y_predict_value))
recall = np.divide(np.sum(np.multiply(ys, y_predict_value)), np.sum(ys))
'''
print("After %s training step(s), test accuracy = %g" %
(global_step, accuracy_score))
print("After %s training step(s), test precision = %g" %
(global_step, precision))
print("After %s training step(s), test recall = %g" %
(global_step, recall))
coord.request_stop()
coord.join(threads)
else:
print("No checkpoint found")
return time.sleep(TEST_INTERVAL_SECS)
def process_submit():
opd = [('EventID', '<i8'), ('ChannelID', '<i2'), ('PETime', 'f4'),
('Weight', 'f4')]
print(testnn.REG_RAW)
with tf.Graph().as_default():
x = tf.placeholder(
tf.float32, [1, 1, generate.Length_waveform, forward.NUM_CHANNELS])
#y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
y = forward.forwardpro(x, False, None)
ema = tf.train.ExponentialMovingAverage(backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
#saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
with h5py.File(fipt) as ipt, h5py.File(fopt, "w") as opt:
#initial input
ent = ipt['Waveform']
l = len(ent)
#l = 100
print(l)
#dt = np.zeros(l*20, dtype=opd)
dt = np.zeros(l * 1029, dtype=opd)
start = 0
end = 0
count = 0
for i in range(l):
wf = ent[i]['Waveform']
af = np.where(wf[200:606] <= generate.THRES)
if np.size(af) != 0:
minit_v = af[0][0]
tr = range(
minit_v - 10 + 200,
minit_v - 10 + generate.Length_waveform + 200)
wf_test = wf[tr]
wf_aver = np.mean(
np.subtract(generate.PLATNUM,
wf_test)) * (1. / 100)
#must be 1./100
wf_test = wf_test.reshape(
[1, generate.Length_waveform]) * (1. / 1000)
#into the nn
reshaped_xs = np.reshape(
wf_test, (1, 1, generate.Length_waveform,
forward.NUM_CHANNELS))
y_value = sess.run(y, feed_dict={x: reshaped_xs})
pe_num = int(
np.around(
np.polyval(np.array(testnn.REG_RAW),
wf_aver)))
'''
if pe_num < 0:
print('tiny!', i)
pe_num = 1
if pe_num >= 206:
print('huge!', i)
pe_num = 200
y_predict = np.zeros_like(y_value)
order_y = np.argsort(y_value[0, :])[::-1]
th_v = y_value[0, int(order_y[pe_num])]
y_predict = np.where(y_value > th_v, 1, 0)
#correction of bias
if np.size(np.where(y_predict == 1)) != 0:
a = np.where(y_predict == 1)[1][0]
b = np.where(y_predict == 1)[1][-1]
p = int(np.around((2.*b - 3.*a)/5))
y_predict[0, p::] = 0
pf_value = np.where(y_predict == 1)[1] + minit_v - 10 + 200
pf = pf_value
#out nn
if len(pf) == 0 or np.size(af) == 0:
pf = np.array([300])
'''
pf = np.where(y_value > 0, y_value, 0)
pfw = pf[np.where(pf > 0)]
pfw = np.multiply(pfw, np.divide(pe_num, np.sum(pfw)))
pf = np.where(pf > 0)[1] + minit_v - 10 + 200
end = start + len(pf)
'''
dt['PETime'][start:end] = pf
dt['Weight'][start:end] = 1
'''
dt['PETime'][start:end] = pf
dt['Weight'][start:end] = pfw
dt['EventID'][start:end] = ent[i]['EventID']
dt['ChannelID'][start:end] = ent[i]['ChannelID']
start = end
count = count + 1
if count == int(l / 100) + 1:
print(int((i + 1) / (l / 100)),
end='% ',
flush=True)
count = 0
dt = dt[np.where(dt['EventID'] > 0)]
opt.create_dataset('Answer', data=dt, compression='gzip')
else:
print("No checkpoint file found")
return -1
