def train_nr_iterations(x,
y_,
train_step,
sess,
x_batch,
y_batch,
nr_steps,
batch_size=100):
for _ in range(nr_steps):
x_batch_sub, y_batch_sub = hf.gen_sub_set(batch_size, x_batch, y_batch)
sess.run(train_step, feed_dict={x: x_batch_sub, y_: y_batch_sub})
def main(x_batch, y_batch, dep_batch, nr_nodes_hidden, nr_hidden_laysers, number_of_particles, save_name_network, lam=1, tree=True, off_set=0., n=1):
save = True
restore = True
x, y, y_, loss, train_step = def_network(number_of_particles=number_of_particles, nr_nodes_hidden=nr_nodes_hidden,
nr_hidden_layers=nr_hidden_laysers, lam=1, off_set=off_set, tree=tree)
if save or restore:
saver = tf.train.Saver()
sess = tf.Session()
if restore:
restore_name = save_name_network
saver.restore(sess, restore_name)
else:
sess.run(tf.global_variables_initializer())
loss_list = [] # A list to store the loss values.
loss_list_train = [] # List of loss value for training data.
move_av = [0]
print('Partitioning the data')
x_batch_eval = x_batch[0:int(0.01 * len(x_batch))] # 1% as evaluation data set.
x_batch_train = x_batch[int(0.01 * len(x_batch)):-1] # Rest as training data set.
y_batch_eval = y_batch[0:int(0.01 * len(x_batch))]
y_batch_train = y_batch[int(0.01 * len(x_batch)):-1]
dep_batch_eval = dep_batch[0:int(0.01 * len(x_batch))] # Deposited energy i detector.
print('Start training')
start = t.time() # The time at the start of the training.
i = 0
diff = 10 # Old converges conditions.
tol = 0.000001
while m.fabs(diff) > tol and i < 1e5: # Train 500000 steps
x_batch_sub, y_batch_sub = hf.gen_sub_set(100, x_batch_train, y_batch_train) # Batch size = 100
if i % 100 == 0: # Store statistics.
loss_list_train.append(sess.run(loss, feed_dict={x: x_batch_sub, y_: y_batch_sub})) # Train batch loss.
x_batch_eval_sub, y_batch_eval_sub = hf.gen_sub_set(300, x_batch_eval, y_batch_eval) # Evaluation batch 300
loss_value = sess.run(loss, feed_dict={x: x_batch_eval_sub, y_: y_batch_eval_sub}) # Eval batch loss.
if i % 10000 == 0: # Print to terminal.
print('Iteration nr. ', i, 'Loss: ', loss_value)
loss_list.append(loss_value)
move_av.append(hf.moving_average(loss_list, 20)) # Moving average (Old)
#diff = move_av[-2] - move_av[-1]
sess.run(train_step, feed_dict={x: x_batch_sub, y_: y_batch_sub}) # One train step.
i = i + 1
end = t.time() # Time when training done.
Traningtime = end - start
loss_end = sess.run(loss, feed_dict={x: x_batch_eval, y_: y_batch_eval}) # Loss on the whole set.
Training_iterations = i
print('Calculating output')
out_E = []
out_theta = []
out_corr_E = []
out_corr_theta = []
out = sess.run(y, feed_dict={x: x_batch_eval}) # Run the network on the evalset.
for i in range(len(out)):
index_list = tff.min_sqare_loss_combination(out[i], y_batch_eval[i], lam=lam, off_set=off_set) # Find the right
for j in range(int(len(out[0])/2)): # permutation between correct
out_E.append(out[i][2*j]) # and predicted data.
out_theta.append(out[i][2*j+1])
out_corr_E.append(y_batch_eval[i][2*index_list[j]])
out_corr_theta.append(y_batch_eval[i][2*index_list[j]+1])
# Deposited energy i the detector.
dep = [dep_val[0] for dep_val in dep_batch_eval] # As a normal list.
sum_pred = [np.sum(event[[i for i in range(0, len(event), 2)]]) for event in out] # Sum of predicted energy.
err_theta = [(out_theta[i]-out_corr_theta[i]) for i in range(len(out_theta))] # Error in cos(theta).
err_E = [(out_E[i] - out_corr_E[i])/(out_corr_E[i] + off_set) for i in range(len(out_E))] # Relative error in energy.
mean_E = np.mean(err_E) # Mean of energy error.
mean_theta = np.mean(err_theta) # Mean of cos(theta) error.
std_E = np.std(err_E) # Standard divination for energy error.
std_theta = np.std(err_theta) # Standard diviation of cos(theta) error.
if save:
print('Saving model')
save_name = './model3.ckpt'
save_path = saver.save(sess, save_name)
return loss_end, mean_E, mean_theta, std_E, std_theta, err_E, err_theta, out_E, out_corr_E, out_theta,\
out_corr_theta, loss_list, loss_list_train, Traningtime, Training_iterations,\
sum_pred, dep
def early_stopp_std(x_batch, y_batch, x, y_, y, step, loss, sess, save_path, update_iterations, max_iterations, partition=0.05,
retraining=False):
"""A regularization for the network. First arguments are the data for x and y, then the (already declared) trainingstep and lossfunction.
Choose how many iterations it can pass before the validation error must have been updated, max iterations and the partition of the data.
Set retraining to true if the network should be retrained and not restored."""
x_batch_eval = x_batch[0:int(0.01 * len(x_batch))] # batch for evaluating
x_batch_train = x_batch[int(0.01 * len(x_batch)):-1] # batch for training
y_batch_eval = y_batch[0:int(0.01 * len(x_batch))] # x - the input data, y - the correct data
y_batch_train = y_batch[int(0.01 * len(x_batch)):-1]
# Need to def new subsets of trainingdata if retraining is true
if retraining == True:
x_batch_train = x_batch_train[int(partition * len(x_batch_train)):-1]
x_batch_eval = x_batch_train[0:int(partition * len(x_batch_train))]
y_batch_train = y_batch_train[int(partition * len(x_batch)):-1]
y_batch_eval = y_batch_train[0:int(partition * len(x_batch_train))]
opt_mean = [float('Inf')] # inf from the start
opt_value = [float('Inf')] # inf from the beginning
i = 0 # counter for iterations
opt_iteration = 0 # the number of optimal iterations
j = 0 # counter for how many iterations done without update the convergence condition
loss_list = []
loss_list_train = []
mean_list = []
lam = 1
saver = tf.train.Saver()
print('Start training')
start = t.time()
while j < update_iterations and i < max_iterations: # train for maximum max_iterations or until the opt_value has not been updated for update_iterations
x_batch_sub, y_batch_sub = hf.gen_sub_set(100, x_batch_train, y_batch_train) # batch of 100 events
if i % 100 == 0:
loss_list_train.append(sess.run(loss, feed_dict={x: x_batch_sub, y_: y_batch_sub}))
x_batch_eval_sub, y_batch_eval_sub = hf.gen_sub_set(800, x_batch_eval, y_batch_eval)
loss_value = sess.run(loss, feed_dict={x: x_batch_eval_sub, y_: y_batch_eval_sub})
loss_list.append(loss_value)
out_corr_theta = []
out_theta = []
out = sess.run(y, feed_dict={x: x_batch_eval_sub}) # Run the network on the subset.
for k in range(len(out)):
index_list = tff.min_sqare_loss_combination(out[k], y_batch_eval_sub[k], lam=lam) # Find the right permutation
for p in range(int(len(out[0]) / 2)): # between correct and predicted data.
out_theta.append(out[k][2 * p + 1])
out_corr_theta.append(y_batch_eval[k][2 * index_list[p] + 1])
err_theta = [(out_theta[k] - out_corr_theta[k]) for i in range(len(out_theta))] # Error in cos(theta).
err_theta=np.array(err_theta)
mean = np.mean(err_theta)
mean_list.append(mean)
std_theta = np.std(err_theta)
# print(std_theta)
if std_theta <= opt_value and np.absolute(mean) < np.absolute(opt_mean) :
print('New optimal std ', std_theta, 'Mean: ', mean)
j = 0
opt_iteration = i
opt_value = std_theta
opt_mean = mean
opt_time = t.time()
if retraining == False:
saver.save(sess, save_path) # save_path ska vara i format "./tmp/model.ckpt"
# print("model saved in path: %s" % save_path)
else:
j = j + 1
# print('Iteration', i)
if i % 1000 == 0:
print('Iteration nr. ', i, 'Loss: ', loss_value, ' Std: ', std_theta, 'Mean: ', mean)
sess.run(step, feed_dict={x: x_batch_sub, y_: y_batch_sub})
i = i + 1
end = t.time()
if retraining == False:
print("Training finished at iteration " + str(i))
print('Trainingtime until the lowest std', )
total_iterations = i
trainingtime = end - start
trainingtime_opt = opt_time-start
loss_end = sess.run(loss, feed_dict={x: x_batch_eval, y_: y_batch_eval})
return trainingtime, trainingtime_opt, total_iterations, opt_iteration, loss_end, loss_list, loss_list_train, mean_list, None, None, None, None
else: #retrain the network with all trainingdata opt_iteration time
sess.run(tf.global_variables_initializer()) # reinitialize the parameters
x_batch_eval = x_batch[0:int(partition * len(x_batch))]
x_batch_train = x_batch[int(partition * len(x_batch)):-1]
y_batch_eval = y_batch[0:int(partition * len(x_batch))]
y_batch_train = y_batch[int(partition * len(x_batch)):-1]
loss_list2 = []
loss_list_train2 = []
print('Start second training to iteration ' + str(opt_iteration))
start = t.time()
i = 0
while i < opt_iteration:
x_batch_sub, y_batch_sub = hf.gen_sub_set(100, x_batch_train, y_batch_train)
if i % 100 == 0:
loss_list_train2.append(sess.run(loss, feed_dict={x: x_batch_sub, y_: y_batch_sub}))
x_batch_eval_sub, y_batch_eval_sub = hf.gen_sub_set(300, x_batch_eval, y_batch_eval)
loss_value = sess.run(loss, feed_dict={x: x_batch_eval_sub, y_: y_batch_eval_sub})
if i % 10000 == 0:
print('Iteration nr. ', i, 'Loss: ', loss_value)
loss_list2.append(loss_value)
sess.run(step, feed_dict={x: x_batch_sub, y_: y_batch_sub})
i = i + 1
end = t.time()
trainingtime2 = end - start
loss_end2 = sess.run(loss, feed_dict={x: x_batch_eval, y_: y_batch_eval})
return trainingtime, trainingtime_opt, total_iterations, opt_iteration, loss_end, loss_list, loss_list_train, mean_list, trainingtime2, loss_end2, loss_list2, loss_list_train2
def main(x_batch,
y_batch,
dep_batch,
nr_nodes_hidden,
nr_hidden_laysers,
lam=1,
tree=True):
print('Initializing variables')
x = tf.placeholder(dtype=tf.float32, shape=[None,
162]) # Placeholder for input.
y_ = tf.placeholder(dtype=tf.float32,
shape=[None, 10]) # Placeholder for correct output.
y = tff.def_fc_layers(x, 162, 10, nr_nodes_hidden,
nr_hidden_laysers) # Defines a fully connected graf.
loss = tff.split_energy_angle_comb(y, y_, 5, lam=lam,
tree=tree) # Defines loss.
train_step = tf.train.AdamOptimizer(1e-4).minimize(
loss) # Set the training method to Adam opt.
sess = tf.Session()
sess.run(tf.global_variables_initializer())
loss_list = [] # A list to store the loss values.
loss_list_train = [] # List of loss value for training data.
move_av = [0]
print('Partitioning the data')
x_batch_eval = x_batch[0:int(0.01 *
len(x_batch))] # 1% as evaluation data set.
x_batch_train = x_batch[int(0.01 *
len(x_batch)):-1] # Rest as training data set.
y_batch_eval = y_batch[0:int(0.01 * len(x_batch))]
y_batch_train = y_batch[int(0.01 * len(x_batch)):-1]
print('Start training')
start = t.time() # The time at the start of the training.
i = 0
diff = 10 # Old converges conditions.
tol = 0.000001
while m.fabs(diff) > tol and i < 500000: # Train 500000 steps
x_batch_sub, y_batch_sub = hf.gen_sub_set(
100, x_batch_train, y_batch_train) # Batch size = 100
if i % 100 == 0: # Store statistics.
loss_list_train.append(
sess.run(loss, feed_dict={
x: x_batch_sub,
y_: y_batch_sub
})) # Train batch loss.
x_batch_eval_sub, y_batch_eval_sub = hf.gen_sub_set(
300, x_batch_eval, y_batch_eval) # Evaluation batch 300
loss_value = sess.run(loss,
feed_dict={
x: x_batch_eval_sub,
y_: y_batch_eval_sub
}) # Eval batch loss.
if i % 10000 == 0: # Print to terminal.
print('Iteration nr. ', i, 'Loss: ', loss_value)
loss_list.append(loss_value)
move_av.append(hf.moving_average(loss_list,
20)) # Moving average (Old)
#diff = move_av[-2] - move_av[-1]
sess.run(train_step, feed_dict={
x: x_batch_sub,
y_: y_batch_sub
}) # One train step.
i = i + 1
end = t.time() # Time when training done.
Traningtime = end - start
loss_end = sess.run(loss, feed_dict={
x: x_batch_eval,
y_: y_batch_eval
}) # Loss on the whole set.
Training_iterations = i
print('Calculating output')
out_E = []
out_theta = []
out_corr_E = []
out_corr_theta = []
out = sess.run(y,
feed_dict={x:
x_batch_eval}) # Run the network on the evalset.
for i in range(len(out)):
index_list = tff.min_sqare_loss_combination(
out[i], y_batch_eval[i], lam=lam) # Find the right permutation
for j in range(int(len(out[0]) / 2)): # between correct and predicted
out_E.append(out[i][2 * j]) # data.
out_theta.append(out[i][2 * j + 1])
out_corr_E.append(y_batch_eval[i][2 * index_list[j]])
out_corr_theta.append(y_batch_eval[i][2 * index_list[j] + 1])
dep_batch_eval = dep_batch[0:int(
0.01 * len(x_batch))] # Deposited energy i detector.
dep = [dep_val[0] for dep_val in dep_batch_eval] # As a normal list.
sum_pred = [
np.sum(event[[i for i in range(0, len(event), 2)]]) for event in out
] # Sum of predicted energy.
err_theta = [(out_theta[i] - out_corr_theta[i])
for i in range(len(out_theta))] # Error in cos(theta).
err_E = [(out_E[i] - out_corr_E[i]) / out_corr_E[i]
for i in range(len(out_E))] # Relative error in energy.
mean_E = np.mean(err_E) # Mean of energy error.
mean_theta = np.mean(err_theta) # Mean of cos(theta) error.
std_E = np.std(err_E) # Standard divination for energy error.
std_theta = np.std(err_theta) # Standard diviation of cos(theta) error.
return loss_end, mean_E, mean_theta, std_E, std_theta, err_E, err_theta, out_E, out_corr_E, out_theta,\
out_corr_theta, loss_list, loss_list_train, Traningtime, Training_iterations, sum_pred, dep
def train_stop_on_fitted_parameters_energy_theta(x,
y_,
train_step,
y,
loss,
sess,
x_batch,
y_batch,
x_batch_eval,
y_batch_eval,
saver=None,
save=False,
save_path=None,
ckpt_intrevall=3600,
restore=False,
restore_path=None,
lower_iteration_limit=0,
upper_iteration_limit=1e6,
nr_train_step=100,
print_every_nr_rain_step=10,
train_batch_size=100,
eval_batch_size=1000):
if save or restore:
if saver == None:
raise TypeError(
'A tf.Saver must be given if you want to save or restore a network'
)
if save:
if save_path == None:
print('Warning. No path for storing the network is given. '
'The network will be stored in the current folder.')
save_path = './model.ckpt'
if restore:
if restore_path == None:
raise ValueError(
'A path to the network to be restored must be given.')
if restore:
saver.restore(sess, restore_path)
else:
sess.run(tf.global_variables_initializer())
if save:
saver.Save(sess, save_path)
loss_list = []
loss_list_train = []
p1, p2, p3, p4, move_av_p1, move_av_p2, move_av_p3, move_av_p4 = [], [], [], [], [], [], [], []
i = 0
j = 0
cond_1, cond_2, cond_3, cond_4 = True, True, True, True
print('Start training')
start = t.time()
time_last_save = start
while i < lower_iteration_limit or ((cond_1 or cond_2 or cond_3 or cond_4)
and i < upper_iteration_limit):
x_batch_sub, y_batch_sub = hf.gen_sub_set(train_batch_size, x_batch,
y_batch)
loss_list_train.append(
sess.run(loss, feed_dict={
x: x_batch_sub,
y_: y_batch_sub
}))
x_batch_sub, y_batch_sub = hf.gen_sub_set(eval_batch_size,
x_batch_eval, y_batch_eval)
loss_val = sess.run(loss, feed_dict={x: x_batch_sub, y_: y_batch_sub})
loss_list.append(loss_val)
if i % print_every_nr_rain_step * nr_train_step == 0:
print('Iteration number: {} Loss function value: {}'.format(
i, loss_val))
if t.time() - time_last_save > ckpt_intrevall:
print('Storing the model of the network in {}'.format(save_path))
# Gör anpassning osv
for _ in range(nr_train_step):
x_batch_sub, y_batch_sub = hf.gen_sub_set(train_batch_size,
x_batch, y_batch)
sess.run(train_step, feed_dict={x: x_batch_sub, y_: y_batch_sub})
i = i + nr_train_step