def test(self):
self.num_batch = 1
self.initialize()
print("Beginning to test network with parameters", get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
graph_loss = self.model.get_losses()
graph_optmiser = self.model.get_optimizer()
graph_summary = self.model.get_summary()
graph_summary_validation = self.model.get_summary_validation()
graph_confusion_matrix = self.model.get_confusion_matrix()
graph_accuracy = self.model.get_accuracy()
graph_logits, graph_prediction = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
inputs_feed = self.__get_input_feeds(self.test_files, repeat=False)
accuracy_sum = 0
num_examples = 0
confusion_matrix = np.zeros((self.num_classes, self.num_classes), dtype=np.float32)
init = [tf.global_variables_initializer(), tf.local_variables_initializer()]
with tf.Session() as sess:
sess.run(init)
print("Beginning to test network with parameters", get_num_parameters(self.model.get_variable_scope()))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.saver_sparse.restore(sess, self.model_path)
print("\n\nINFO: Loading model\n\n")
iteration_number = 0
labels = np.zeros((1000000, self.num_classes))
scores = np.zeros((1000000, self.num_classes))
print("Starting iterations")
while iteration_number < 1000000:
try:
inputs = sess.run(inputs_feed)
except tf.errors.OutOfRangeError:
break
inputs_train_dict = {
placeholders[0]: inputs[0],
placeholders[1]: inputs[1]
}
labels[iteration_number] = np.squeeze(inputs[1])
t, eval_loss, eval_accuracy, eval_confusion, test_logits, eval_logits = sess.run(
[graph_temp, graph_loss, graph_accuracy, graph_confusion_matrix, graph_prediction, graph_logits],
feed_dict=inputs_train_dict)
confusion_matrix += eval_confusion
accuracy_sum += eval_accuracy * self.num_batch
num_examples += self.num_batch
scores[iteration_number] = np.squeeze(eval_logits)
print("Test - Batch %4d: loss %0.5f accuracy %03.3f accuracy (cumm) %03.3f" % (
iteration_number, eval_loss, eval_accuracy, accuracy_sum / num_examples))
iteration_number += 1
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
classes_names = 'Electron', 'Muon', 'Pion Charged', 'Pion Neutral', 'K0 Long', 'K0 Short' # TODO: Pick from config
test_result = ClassificationModelTestResult()
test_result.initialize(confusion_matrix, labels, scores, self.model.get_human_name(),
get_num_parameters(self.model.get_variable_scope()), classes_names, self.summary_path)
test_result.evaluate(self.test_out_path)
print("Evaluation complete")
print("Evaluation accuracy ", accuracy_sum / num_examples)
print("Confusion matrix:")
print(confusion_matrix)
def train(self):
self.initialize()
print("Beginning to train network with parameters",
get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
graph_loss = self.model.get_losses()
graph_optmiser = self.model.get_optimizer()
graph_summary = self.model.get_summary()
graph_summary_validation = self.model.get_summary_validation()
graph_accuracy = self.model.get_accuracy()
graph_logits, graph_prediction = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
place_holder_control_switch = self.model.get_place_holder_switch_control(
)
if self.from_scratch:
self.clean_summary_dir()
inputs_feed = self._get_input_feeds(self.training_files)
inputs_validation_feed = self._get_input_feeds(self.validation_files)
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
summary_writer = tf.summary.FileWriter(self.summary_path,
sess.graph)
if not self.from_scratch:
self.saver_sparse.restore(sess, self.model_path)
print("\n\nINFO: Loading model\n\n")
with open(self.model_path + '.txt', 'r') as f:
iteration_number = int(f.read())
else:
iteration_number = 0
print("Starting iterations")
while iteration_number < self.train_for_iterations:
number = int(iteration_number / self.switch_after) % 2
switch_value = np.zeros((2), dtype=np.int64)
switch_value[number] = 1
switch_value = [1, 1] # TODO: Remove this later
inputs_train = sess.run(list(inputs_feed))
inputs_train_dict = {
placeholders[0]: inputs_train[0],
placeholders[1]: inputs_train[1],
placeholders[2]: inputs_train[2],
placeholders[3]: inputs_train[3],
placeholders[4]: inputs_train[4],
place_holder_control_switch: switch_value
}
t, eval_loss, _, eval_summary, eval_accuracy, test_logits = sess.run(
[
graph_temp, graph_loss, graph_optmiser, graph_summary,
graph_accuracy, graph_prediction
],
feed_dict=inputs_train_dict)
if iteration_number % self.validate_after == 0:
inputs_validation = sess.run(list(inputs_validation_feed))
inputs_validation_dict = {
placeholders[0]: inputs_validation[0],
placeholders[1]: inputs_validation[1],
placeholders[2]: inputs_validation[2],
placeholders[3]: inputs_validation[3],
placeholders[4]: inputs_validation[4],
place_holder_control_switch: switch_value
}
eval_loss_validation, eval_summary_validation, eval_accuracy_validation = sess.run(
[graph_loss, graph_summary_validation, graph_accuracy],
feed_dict=inputs_validation_dict)
summary_writer.add_summary(eval_summary_validation,
iteration_number)
print(
"Validation - Iteration %4d: loss %0.5f accuracy %03.3f"
% (iteration_number, eval_loss_validation,
eval_accuracy_validation))
print(
"Training - Iteration %4d: loss %0.5f accuracy %03.3f" %
(iteration_number, eval_loss, eval_accuracy))
print(t[0])
iteration_number += 1
summary_writer.add_summary(eval_summary, iteration_number)
if iteration_number % self.save_after_iterations == 0:
print("\n\nINFO: Saving model\n\n")
self.saver_sparse.save(sess, self.model_path)
with open(self.model_path + '.txt', 'w') as f:
f.write(str(iteration_number))
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
def test(self):
self.initialize_test()
print("Beginning to test network with parameters",
get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
graph_loss = self.model.get_losses()
graph_optmiser = self.model.get_optimizer()
graph_summary = self.model.get_summary()
graph_summary_validation = self.model.get_summary_validation()
graph_output = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
inputs_feed = self.reader_factory.get_class(self.reader_type)(
self.test_files, self.num_max_entries, self.num_data_dims,
self.num_batch).get_feeds(shuffle=False)
inference_streamer = InferenceOutputStreamer(
output_path=self.test_out_path, cache_size=100)
inference_streamer.start_thread()
print(type(inputs_feed))
print("****************************************")
print("Test Input shape: ", inputs_feed.get_shape().as_list())
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.saver_sparse.restore(sess, self.model_path)
print("\n\nINFO: Loading model", self.model_path, "\n\n")
print("Starting testing")
iteration_number = 0
while iteration_number < int(
np.ceil(self.num_testing_samples / self.num_batch)):
inputs_test = sess.run(list(inputs_feed))
if len(placeholders) == 5:
inputs_train_dict = {
placeholders[0]:
inputs_test[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_test[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_test[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_test[0][:, :, self.target_indices],
placeholders[4]:
inputs_test[1],
self.model.is_train:
False,
self.model.learning_rate:
0
}
else:
inputs_train_dict = {
placeholders[0]:
inputs_test[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_test[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_test[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_test[0][:, :, self.target_indices],
placeholders[4]:
inputs_test[1],
# placeholders[5]: inputs_test[2],
self.model.is_train:
False,
self.model.learning_rate:
0
}
t, eval_loss, eval_output = sess.run(
[graph_temp, graph_loss, graph_output],
feed_dict=inputs_train_dict)
print("Adding", len(inputs_test[0]), "test results")
for i in range(len(inputs_test[0])):
if not self.output_seed_indices:
inference_streamer.add(
(inputs_test[0][i], (inputs_test[1])[i, 0],
eval_output[i]))
else:
inference_streamer.add(
(inputs_test[0][i], (inputs_test[1])[i, 0],
eval_output[i], inputs_test[2][i]))
print("Testing - Sample %4d: loss %0.5f" %
(iteration_number * self.num_batch, eval_loss))
print(t[0])
iteration_number += 1
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
inference_streamer.close()
def visualize(self):
self.initialize_test()
print("Beginning to visualize network with parameters",
get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
graph_loss = self.model.get_losses()
graph_output = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
layer_feats = self.model.temp_feat_visualize
inputs_feed = self.reader_factory.get_class(self.reader_type)(
self.test_files, self.num_max_entries, self.num_data_dims,
self.num_batch).get_feeds(shuffle=False)
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.saver_sparse.restore(sess, self.model_path)
print("\n\nINFO: Loading model", self.model_path, "\n\n")
print("Starting visualizing")
iteration_number = 0
while iteration_number < int(
np.ceil(self.num_testing_samples / self.num_batch)):
inputs_test = sess.run(list(inputs_feed))
print("Run")
if len(placeholders) == 5:
inputs_train_dict = {
placeholders[0]:
inputs_test[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_test[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_test[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_test[0][:, :, self.target_indices],
placeholders[4]:
inputs_test[1],
self.model.is_train:
False,
self.model.learning_rate:
0
}
else:
inputs_train_dict = {
placeholders[0]:
inputs_test[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_test[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_test[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_test[0][:, :, self.target_indices],
placeholders[4]:
inputs_test[1],
placeholders[5]:
inputs_test[2],
self.model.is_train:
False,
self.model.learning_rate:
0
}
eval_out = sess.run([graph_temp, graph_loss, graph_output] +
layer_feats,
feed_dict=inputs_train_dict)
layer_outs = eval_out[3:]
prediction = eval_out[2]
if iteration_number * self.num_batch + self.num_batch >= 32:
for x in range(32):
event_number = (32 + x) % self.num_batch
print("Event number", event_number)
seed_index = inputs_test[2][event_number, :]
print(seed_index)
spatial_features = inputs_test[0][
event_number, :, :][:,
self.spatial_features_indices]
energy = inputs_test[0][event_number, :, :][:, 0]
gt = inputs_test[0][
event_number, :, :][:, self.target_indices]
predictionx = prediction[event_number]
layer_outsx = [x[event_number] for x in layer_outs]
if 'aggregators' in self.config_name:
plots.plot_clustering_layer_wise_visualize_agg(
spatial_features, energy, predictionx, gt,
layer_outsx, self.config_name)
else:
plots.plot_clustering_layer_wise_visualize(
spatial_features, energy, predictionx, gt,
layer_outsx, self.config_name)
sys.exit(0)
# Put the condition here!
iteration_number += 1
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
def train(self):
self.initialize()
print("Beginning to train network with parameters",
get_num_parameters(self.model.get_variable_scope()))
print("Variable scope:", self.model.get_variable_scope())
placeholders = self.model.get_placeholders()
if self.from_scratch:
subprocess.call("mkdir -p %s" % (self.summary_path), shell=True)
subprocess.call("mkdir -p %s" % (self.test_out_path), shell=True)
subprocess.call("mkdir -p %s" %
(os.path.join(self.test_out_path, 'ops')),
shell=True)
with open(self.model_path + '_code.py', 'w') as f:
f.write(self.model.get_code())
ops_parent = os.path.dirname(ops.__file__)
for ops_file in os.listdir(ops_parent):
if not ops_file.endswith('.py'):
continue
shutil.copy(os.path.join(ops_parent, ops_file),
os.path.join(self.test_out_path, 'ops'))
graph_loss = self.model.get_losses()
graph_optmiser = self.model.get_optimizer()
graph_summary = self.model.get_summary()
graph_summary_validation = self.model.get_summary_validation()
graph_output = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
if self.plot_after != -1:
data_plotting = None # TODO: Load
if self.from_scratch:
self.clean_summary_dir()
inputs_feed = self.reader_factory.get_class(
self.reader_type)(self.training_files, self.num_max_entries,
self.num_data_dims, self.num_batch).get_feeds()
inputs_validation_feed = self.reader_factory.get_class(
self.reader_type)(self.validation_files, self.num_max_entries,
self.num_data_dims,
self.num_batch).get_feeds(shuffle=False)
print("\n****************************************")
print("Feed Input type", type(inputs_feed[0]))
print("Feed Input shape: ", inputs_feed[0].get_shape().as_list())
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
summary_writer = tf.summary.FileWriter(self.summary_path,
sess.graph)
if not self.from_scratch:
self.saver_sparse.restore(sess, self.model_path)
print("\n\nINFO: Loading model\n\n")
with open(self.model_path + '.txt', 'r') as f:
iteration_number = int(f.read())
else:
iteration_number = 0
print("Starting iterations")
while iteration_number < self.train_for_iterations:
inputs_train = sess.run(list(inputs_feed))
print("\n****************************************")
print("Input Train type", type(inputs_train[0]))
print("Input Train shape: ", inputs_train[0].shape)
learning_rate = 1
if hasattr(self.model, "learningrate_scheduler"):
learning_rate = self.model.learningrate_scheduler.get_lr(
iteration_number)
else:
learning_rate = self.model.learning_rate
if iteration_number == 0:
print('learning rate ', learning_rate)
if len(placeholders) == 5:
inputs_train_dict = {
placeholders[0]:
inputs_train[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_train[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_train[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_train[0][:, :, self.target_indices],
placeholders[4]:
inputs_train[1],
self.model.is_train:
True,
self.model.learning_rate:
learning_rate
}
else:
inputs_train_dict = {
placeholders[0]:
inputs_train[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_train[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_train[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_train[0][:, :, self.target_indices],
placeholders[4]:
inputs_train[1],
# placeholders[5]: inputs_train[2],
self.model.is_train:
True,
self.model.learning_rate:
learning_rate
}
t, eval_loss, _, eval_summary, eval_output = sess.run(
[
graph_temp, graph_loss, graph_optmiser, graph_summary,
graph_output
],
feed_dict=inputs_train_dict)
if self.plot_after != -1:
if iteration_number % self.plot_after == 0:
pass
if iteration_number % self.validate_after == 0:
inputs_validation = sess.run(list(inputs_validation_feed))
self.inputs_plot = inputs_validation
if len(placeholders) == 5:
inputs_validation_dict = {
placeholders[0]:
inputs_validation[0]
[:, :, self.spatial_features_indices],
placeholders[1]:
inputs_validation[0]
[:, :, self.spatial_features_local_indices],
placeholders[2]:
inputs_validation[0][:, :,
self.other_features_indices],
placeholders[3]:
inputs_validation[0][:, :, self.target_indices],
placeholders[4]:
inputs_validation[1],
self.model.is_train:
False,
self.model.learning_rate:
learning_rate
}
else:
inputs_validation_dict = {
placeholders[0]:
inputs_validation[0]
[:, :, self.spatial_features_indices],
placeholders[1]:
inputs_validation[0]
[:, :, self.spatial_features_local_indices],
placeholders[2]:
inputs_validation[0][:, :,
self.other_features_indices],
placeholders[3]:
inputs_validation[0][:, :, self.target_indices],
placeholders[4]:
inputs_validation[1],
# placeholders[5]: inputs_validation[2],
self.model.is_train:
False,
self.model.learning_rate:
learning_rate
}
eval_loss_validation, eval_summary_validation = sess.run(
[graph_loss, graph_summary_validation],
feed_dict=inputs_validation_dict)
summary_writer.add_summary(eval_summary_validation,
iteration_number)
print("Validation - Iteration %4d: loss %.6E" %
(iteration_number, eval_loss_validation))
print("Training - Iteration %4d: loss %0.6E" %
(iteration_number, eval_loss))
print(t[0])
iteration_number += 1
summary_writer.add_summary(eval_summary, iteration_number)
if iteration_number % self.save_after_iterations == 0:
print("\n\nINFO: Saving model\n\n")
self.saver_sparse.save(sess, self.model_path)
with open(self.model_path + '.txt', 'w') as f:
f.write(str(iteration_number))
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
def profile(self):
global bb
tf.reset_default_graph()
self.initialize_profile()
print("Beginning to profile network with parameters",
get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
subprocess.call("mkdir -p %s" % (self.profile_out_path), shell=True)
graph_output = self.model.get_compute_graphs()
inputs_feed = self.reader_factory.get_class(
self.reader_type)(self.training_files, self.num_max_entries,
self.num_data_dims, bb).get_feeds()
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
inference_time_values = []
with tf.Session(config=session_conf) as sess:
# with tf.Session() as sess:
sess.run(init)
profiler = Profiler(sess.graph)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
iteration_number = 0
print("Starting iterations")
while iteration_number < 20:
inputs_train = sess.run(list(inputs_feed))
if len(placeholders) == 5:
inputs_train_dict = {
placeholders[0]:
inputs_train[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_train[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_train[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_train[0][:, :, self.target_indices],
placeholders[4]:
inputs_train[1],
self.model.is_train:
True,
self.model.learning_rate:
1
}
else:
inputs_train_dict = {
placeholders[0]:
inputs_train[0][:, :, self.spatial_features_indices],
placeholders[1]:
inputs_train[0][:, :,
self.spatial_features_local_indices],
placeholders[2]:
inputs_train[0][:, :, self.other_features_indices],
placeholders[3]:
inputs_train[0][:, :, self.target_indices],
placeholders[4]:
inputs_train[1],
placeholders[5]:
inputs_train[2],
self.model.is_train:
True,
self.model.learning_rate:
1
}
run_meta = tf.RunMetadata()
start_time = time.time()
eval_output = sess.run(
graph_output,
feed_dict=inputs_train_dict,
options=tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_meta)
print("XC Time: --- %s seconds --- Iteration %d" %
(time.time() - start_time, iteration_number))
profiler.add_step(iteration_number, run_meta)
# Or profile the timing of your model operations.
opts = option_builder.ProfileOptionBuilder.time_and_memory()
profiler.profile_operations(options=opts)
# Or you can generate a timeline:
opts = (option_builder.ProfileOptionBuilder(
option_builder.ProfileOptionBuilder.time_and_memory()
).with_step(iteration_number).with_timeline_output(
os.path.join(self.profile_out_path, 'profile')).build())
x = profiler.profile_graph(options=opts)
inference_time_values.append(x.total_exec_micros)
peak_bytes = x.total_peak_bytes
iteration_number += 1
print(self.config_name, "Batch size: ", bb)
print(repr(np.array(inference_time_values)))
print(
"Mean",
np.mean(np.array(inference_time_values, dtype=np.float32)[1:]))
print(
"Variance",
np.std(np.array(inference_time_values, dtype=np.float32)[1:]))
print("Peak bytes", peak_bytes)
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
def profile(self):
self.initialize()
print("Beginning to profile network with parameters",
get_num_parameters(self.model.get_variable_scope()))
placeholders = self.model.get_placeholders()
graph_loss = self.model.get_losses()
graph_optmiser = self.model.get_optimizer()
graph_summary = self.model.get_summary()
graph_summary_validation = self.model.get_summary_validation()
graph_accuracy = self.model.get_accuracy()
graph_logits, graph_prediction = self.model.get_compute_graphs()
graph_temp = self.model.get_temp()
inputs_feed = self._get_input_feeds(self.training_files)
inputs_validation_feed = self._get_input_feeds(self.validation_files)
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
with tf.Session() as sess:
sess.run(init)
profiler = Profiler(sess.graph)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
summary_writer = tf.summary.FileWriter(self.summary_path,
sess.graph)
iteration_number = 0
print("Starting iterations")
run_ops = [
graph_temp, graph_loss, graph_optmiser, graph_summary,
graph_accuracy, graph_prediction
]
while iteration_number < 1000:
inputs_train = sess.run(list(inputs_feed))
inputs_train_dict = {
placeholders[0]: inputs_train[0],
placeholders[1]: inputs_train[1],
placeholders[2]: inputs_train[2],
placeholders[3]: inputs_train[3],
placeholders[4]: inputs_train[4]
}
if iteration_number % 100 == 0:
run_meta = tf.RunMetadata()
sess.run(run_ops,
feed_dict=inputs_train_dict,
options=tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_meta)
profiler.add_step(iteration_number, run_meta)
# Profile the parameters of your model.
profiler.profile_name_scope(
options=(option_builder.ProfileOptionBuilder.
trainable_variables_parameter()))
# Or profile the timing of your model operations.
opts = option_builder.ProfileOptionBuilder.time_and_memory(
)
profiler.profile_operations(options=opts)
# Or you can generate a timeline:
opts = (option_builder.ProfileOptionBuilder(
option_builder.ProfileOptionBuilder.time_and_memory(
)).with_step(iteration_number).with_timeline_output(
self.config['profiler_output_file_name']).build())
profiler.profile_graph(options=opts)
else:
sess.run(run_ops, feed_dict=inputs_train_dict)
print("Profiling - Iteration %4d" % iteration_number)
iteration_number += 1
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
