class DeepLearningReco(icetray.I3ConditionalModule):
"""Module to apply dnn reco.
Attributes
----------
batch_size : int, optional
The number of events to accumulate and pass through the network in
parallel. A higher batch size than 1 can usually improve recontruction
runtime, but will also increase the memory footprint.
config : dict
Dictionary with configuration settings
data_handler : dnn_reco.data_handler.DataHanlder
A data handler object. Handles nn model input meta data and provides
tensorflow placeholders.
data_transformer : dnn_reco.data_trafo.DataTransformer
The data transformer.
model : dnn_reco.model.NNModel
The neural network model
"""
def __init__(self, context):
"""Initialize DeepLearningReco Module
"""
icetray.I3ConditionalModule.__init__(self, context)
self.AddParameter('ModelPath', 'Path to DNN model', None)
self.AddParameter('DNNDataContainer',
'Data container that will be used to feed model',
None)
self.AddParameter(
'IgnoreMisconfiguredSettingsList',
"The model automatically checks whether the "
"configured settings for the 'DNNDataContainer' "
"match those settings that were exported in this "
"model. If a mismatch is found, an error will be "
"raised. This helps to ensure the correct use of "
"the trained models. Sometimes it is necessary to "
"use the model with slightly different settings. In "
"this case a list of setting names can be passed "
"for which the mismatches will be ignored. Doing so "
"will relax the raised error to a warning that is "
"issued. This should be used with caution.", None)
self.AddParameter('OutputBaseName',
'Output key under which the result will be written',
'DeepLearningReco')
self.AddParameter(
"MeasureTime",
"If True, time for preprocessing and prediction will"
" be measured and printed", False)
self.AddParameter(
"ParallelismThreads",
"Tensorflow config option for 'intra_op_parallelism_"
"threads' and 'inter_op_parallelism_threads'"
"[# CPUs]", None)
def Configure(self):
"""Configure DeepLearningReco module.
Read in configuration and build nn model.
Raises
------
ValueError
If settings do not match the expected settings by the nn model.
"""
self._model_path = self.GetParameter('ModelPath')
self._container = self.GetParameter('DNNDataContainer')
self._output_key = self.GetParameter("OutputBaseName")
self._measure_time = self.GetParameter("MeasureTime")
self._parallelism_threads = self.GetParameter("ParallelismThreads")
self._ingore_list = \
self.GetParameter('IgnoreMisconfiguredSettingsList')
if self._ingore_list is None:
self._ingore_list = []
# read in and combine config files and set up
training_files = glob.glob(
os.path.join(self._model_path, 'config_training_*.yaml'))
last_training_file = np.sort(training_files)[-1]
setup_manager = SetupManager([last_training_file])
self.config = setup_manager.get_config()
# ToDo: Adjust necessary values in config
self.config['model_checkpoint_path'] = os.path.join(
self._model_path, 'model')
self.config['model_is_training'] = False
self.config['trafo_model_path'] = os.path.join(self._model_path,
'trafo_model.npy')
if self._parallelism_threads is not None:
self.config['tf_parallelism_threads'] = self._parallelism_threads
# ----------------------------------------------------------------
# Check if settings of data container match settings in model path
# ----------------------------------------------------------------
cfg_file = os.path.join(self._model_path, 'config_data_settings.yaml')
with open(cfg_file, 'r') as stream:
data_config = yaml.safe_load(stream)
# Backwards compatibility for older exported models which did not
# include this setting. In this case the separated format, e.g.
# icecube array + deepcore array is used as opposed to the string-dom
# format: [batch, 86, 60, num_bins]
if 'is_str_dom_format' not in data_config:
data_config['is_str_dom_format'] = False
for k in self._container.config:
# backwards compatibility for older exported models which did not
# export these settings
if k not in data_config and k in [
'pulse_key', 'dom_exclusions', 'partial_exclusion',
'cascade_key', 'allowed_pulse_keys', 'allowed_cascade_keys'
]:
msg = 'Warning: not checking if parameter {!r} is correctly '
msg += 'configured for model {!r} because the setting '
msg += 'was not exported.'
logging.warning(msg.format(k, self._model_path))
continue
# check for allowed pulse keys
if (k == 'pulse_key' and 'allowed_pulse_keys' in data_config
and data_config['allowed_pulse_keys'] is not None
and self._container.config[k]
in data_config['allowed_pulse_keys']):
# this is an allowed pulse, so everything is ok
continue
# check for allowed cascade keys
if (k == 'cascade_key' and 'allowed_cascade_keys' in data_config
and data_config['allowed_cascade_keys'] is not None
and self._container.config[k]
in data_config['allowed_cascade_keys']):
# this is an allowed cascade key, so everything is ok
continue
if not self._container.config[k] == data_config[k]:
if k in self._ingore_list:
msg = 'Warning: parameter {!r} is set to {!r} which '
msg += 'differs from the model [{!r}] default value {!r}. '
msg += 'This mismatch will be ingored since the parameter '
msg += 'is in the IgnoreMisconfiguredSettingsList. '
msg += 'Make sure this is what you intend to do!'
logging.warning(
msg.format(k, self._container.config[k],
self._model_path, data_config[k]))
else:
msg = 'Fatal: parameter {!r} is set to {!r} which '
msg += 'differs from the model [{!r}] default value {!r}.'
msg += 'If you are sure you want to use this model '
msg += 'with these settings, then you can add the '
msg += 'parameter to the IgnoreMisconfiguredSettingsList.'
raise ValueError(
msg.format(k, self._container.config[k],
self._model_path, data_config[k]))
# ----------------------------------------------------------------
# create variables and frame buffer for batching
self._frame_buffer = deque()
self._pframe_counter = 0
self._batch_event_index = 0
# Create a new tensorflow graph and session for this instance of
# dnn reco
g = tf.Graph()
if 'tf_parallelism_threads' in self.config:
n_cpus = self.config['tf_parallelism_threads']
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': 1},
intra_op_parallelism_threads=n_cpus,
inter_op_parallelism_threads=n_cpus,
))
else:
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': 1},
))
with g.as_default():
# Create Data Handler object
self.data_handler = DataHandler(self.config)
self.data_handler.setup_with_config(
os.path.join(self._model_path, 'config_meta_data.yaml'))
# Get time vars that need to be corrected by global time offset
self._time_indices = []
for i, name in enumerate(self.data_handler.label_names):
if name in self.data_handler.relative_time_keys:
self._time_indices.append(i)
# create data transformer
self.data_transformer = DataTransformer(
data_handler=self.data_handler,
treat_doms_equally=self.config['trafo_treat_doms_equally'],
normalize_dom_data=self.config['trafo_normalize_dom_data'],
normalize_label_data=self.config['trafo_normalize_label_data'],
normalize_misc_data=self.config['trafo_normalize_misc_data'],
log_dom_bins=self.config['trafo_log_dom_bins'],
log_label_bins=self.config['trafo_log_label_bins'],
log_misc_bins=self.config['trafo_log_misc_bins'],
norm_constant=self.config['trafo_norm_constant'])
# load trafo model from file
self.data_transformer.load_trafo_model(
self.config['trafo_model_path'])
# create NN model
self.model = NNModel(is_training=False,
config=self.config,
data_handler=self.data_handler,
data_transformer=self.data_transformer,
sess=sess)
# compile model: initalize and finalize graph
self.model.compile()
# restore model weights
self.model.restore()
# Get trained labels, e.g. labels with weights greater than zero
self._mask_labels = \
self.model.shared_objects['label_weight_config'] > 0
self._non_zero_labels = [
n for n, b in zip(self.data_handler.label_names,
self._mask_labels) if b
]
self._non_zero_log_bins = \
[l for l, b in
zip(self.data_transformer.trafo_model['log_label_bins'],
self._mask_labels) if b]
def Process(self):
"""Process incoming frames.
Pop frames and put them in the frame buffer.
When a physics frame is popped, accumulate the input data to form
a batch of events. Once a full batch of physics events is accumulated,
perform the prediction and push the buffered frames.
The Physics method can then write the results to the physics frame
by using the results:
self.y_pred_batch, self.y_unc_batch
self._runtime_prediction, self._runtime_preprocess_batch
and the current event index self._batch_event_index
"""
frame = self.PopFrame()
# put frame on buffer
self._frame_buffer.append(frame)
# check if the current frame is a physics frame
if frame.Stop == icetray.I3Frame.Physics:
self._pframe_counter += 1
# check if we have a full batch of events
if self._pframe_counter == self._container.batch_size:
# we have now accumulated a full batch of events so
# that we can perform the prediction
self._process_frame_buffer()
def Finish(self):
"""Run prediciton on last incomplete batch of events.
If there are still frames left in the frame buffer there is an
incomplete batch of events, that still needs to be passed through.
This method will run the prediction on the incomplete batch and then
write the results to the physics frame. All frames in the frame buffer
will be pushed.
"""
if self._frame_buffer:
# there is an incomplete batch of events that we need to complete
self._process_frame_buffer()
def _process_frame_buffer(self):
"""Performs prediction for accumulated batch.
Then writes results to physics frames in frame buffer and eventually
pushes all of the frames in the order they came in.
"""
self._perform_prediction(size=self._pframe_counter)
# reset counters and indices
self._batch_event_index = 0
self._pframe_counter = 0
# push frames
while self._frame_buffer:
fr = self._frame_buffer.popleft()
if fr.Stop == icetray.I3Frame.Physics:
# write results at current batch index to frame
self._write_to_frame(fr, self._batch_event_index)
# increase the batch event index
self._batch_event_index += 1
self.PushFrame(fr)
def _perform_prediction(self, size):
"""Perform the prediction for a batch of events.
Parameters
----------
size : int
The size of the current batch.
"""
if size > 0:
if self._measure_time:
start_time = timeit.default_timer()
self.y_pred_batch, self.y_unc_batch = self.model.predict(
x_ic78=self._container.x_ic78[:size],
x_deepcore=self._container.x_deepcore[:size])
# Fix time offset
if self.data_handler.relative_time_keys:
global_time_offset = \
self._container.global_time_offset_batch[:size]
for i in self._time_indices:
self.y_pred_batch[:, i] += global_time_offset
if self._measure_time:
self._runtime_prediction = \
(timeit.default_timer() - start_time) / size
else:
self.y_pred_batch = None
self.y_unc_batch = None
self._runtime_prediction = None
def _write_to_frame(self, frame, batch_event_index):
"""Writes the prediction results of the given batch event index to
the frame.
Parameters
----------
frame : I3Frame
The physics frame to which the results should be written to.
batch_event_index : int
The batch event index. This defines which event in the batch is to
be written to the frame.
"""
if self._measure_time:
start_time = timeit.default_timer()
# Write prediction and uncertainty estimate to frame
results = {}
for name, pred, unc, log_label in zip(
self._non_zero_labels,
self.y_pred_batch[batch_event_index][self._mask_labels],
self.y_unc_batch[batch_event_index][self._mask_labels],
self._non_zero_log_bins):
# save prediction
results[name] = float(pred)
# save uncertainty estimate
if log_label:
results[name + '_log_uncertainty'] = float(unc)
else:
results[name + '_uncertainty'] = float(unc)
# Create combined I3Particle
if 'label_particle_keys' in self.config:
particle_keys = self.config['label_particle_keys']
particle = dataclasses.I3Particle()
if 'energy' in particle_keys:
if particle_keys['energy'] in self._non_zero_labels:
particle.energy = results[particle_keys['energy']]
if 'time' in particle_keys:
if particle_keys['time'] in self._non_zero_labels:
particle.time = results[particle_keys['time']]
if 'length' in particle_keys:
if particle_keys['length'] in self._non_zero_labels:
particle.length = results[particle_keys['length']]
if 'dir_x' in particle_keys:
if particle_keys['dir_x'] in self._non_zero_labels:
particle.dir = dataclasses.I3Direction(
results[particle_keys['dir_x']],
results[particle_keys['dir_y']],
results[particle_keys['dir_z']])
elif 'azimuth' in particle_keys:
if particle_keys['azimuth'] in self._non_zero_labels:
particle.dir = dataclasses.I3Direction(
results[particle_keys['azimuth']],
results[particle_keys['zenith']])
if 'pos_x' in particle_keys:
if particle_keys['pos_x'] in self._non_zero_labels:
particle.pos = dataclasses.I3Position(
results[particle_keys['pos_x']],
results[particle_keys['pos_y']],
results[particle_keys['pos_z']])
# transform zenith and azimuth to proper range:
particle.dir = dataclasses.I3Direction(particle.dir.x,
particle.dir.y,
particle.dir.z)
frame[self._output_key + '_I3Particle'] = particle
# write time measurement to frame
if self._measure_time:
results['runtime_prediction'] = self._runtime_prediction
results['runtime_write'] = \
timeit.default_timer() - start_time
results['runtime_preprocess'] = \
self._container.runtime_batch[batch_event_index]
# write to frame
frame[self._output_key] = dataclasses.I3MapStringDouble(results)
def Configure(self):
"""Configure DeepLearningReco module.
Read in configuration and build nn model.
Raises
------
ValueError
If settings do not match the expected settings by the nn model.
"""
self._model_path = self.GetParameter('ModelPath')
self._container = self.GetParameter('DNNDataContainer')
self._output_key = self.GetParameter("OutputBaseName")
self._measure_time = self.GetParameter("MeasureTime")
self._parallelism_threads = self.GetParameter("ParallelismThreads")
self._ingore_list = \
self.GetParameter('IgnoreMisconfiguredSettingsList')
if self._ingore_list is None:
self._ingore_list = []
# read in and combine config files and set up
training_files = glob.glob(
os.path.join(self._model_path, 'config_training_*.yaml'))
last_training_file = np.sort(training_files)[-1]
setup_manager = SetupManager([last_training_file])
self.config = setup_manager.get_config()
# ToDo: Adjust necessary values in config
self.config['model_checkpoint_path'] = os.path.join(
self._model_path, 'model')
self.config['model_is_training'] = False
self.config['trafo_model_path'] = os.path.join(self._model_path,
'trafo_model.npy')
if self._parallelism_threads is not None:
self.config['tf_parallelism_threads'] = self._parallelism_threads
# ----------------------------------------------------------------
# Check if settings of data container match settings in model path
# ----------------------------------------------------------------
cfg_file = os.path.join(self._model_path, 'config_data_settings.yaml')
with open(cfg_file, 'r') as stream:
data_config = yaml.safe_load(stream)
# Backwards compatibility for older exported models which did not
# include this setting. In this case the separated format, e.g.
# icecube array + deepcore array is used as opposed to the string-dom
# format: [batch, 86, 60, num_bins]
if 'is_str_dom_format' not in data_config:
data_config['is_str_dom_format'] = False
for k in self._container.config:
# backwards compatibility for older exported models which did not
# export these settings
if k not in data_config and k in [
'pulse_key', 'dom_exclusions', 'partial_exclusion',
'cascade_key', 'allowed_pulse_keys', 'allowed_cascade_keys'
]:
msg = 'Warning: not checking if parameter {!r} is correctly '
msg += 'configured for model {!r} because the setting '
msg += 'was not exported.'
logging.warning(msg.format(k, self._model_path))
continue
# check for allowed pulse keys
if (k == 'pulse_key' and 'allowed_pulse_keys' in data_config
and data_config['allowed_pulse_keys'] is not None
and self._container.config[k]
in data_config['allowed_pulse_keys']):
# this is an allowed pulse, so everything is ok
continue
# check for allowed cascade keys
if (k == 'cascade_key' and 'allowed_cascade_keys' in data_config
and data_config['allowed_cascade_keys'] is not None
and self._container.config[k]
in data_config['allowed_cascade_keys']):
# this is an allowed cascade key, so everything is ok
continue
if not self._container.config[k] == data_config[k]:
if k in self._ingore_list:
msg = 'Warning: parameter {!r} is set to {!r} which '
msg += 'differs from the model [{!r}] default value {!r}. '
msg += 'This mismatch will be ingored since the parameter '
msg += 'is in the IgnoreMisconfiguredSettingsList. '
msg += 'Make sure this is what you intend to do!'
logging.warning(
msg.format(k, self._container.config[k],
self._model_path, data_config[k]))
else:
msg = 'Fatal: parameter {!r} is set to {!r} which '
msg += 'differs from the model [{!r}] default value {!r}.'
msg += 'If you are sure you want to use this model '
msg += 'with these settings, then you can add the '
msg += 'parameter to the IgnoreMisconfiguredSettingsList.'
raise ValueError(
msg.format(k, self._container.config[k],
self._model_path, data_config[k]))
# ----------------------------------------------------------------
# create variables and frame buffer for batching
self._frame_buffer = deque()
self._pframe_counter = 0
self._batch_event_index = 0
# Create a new tensorflow graph and session for this instance of
# dnn reco
g = tf.Graph()
if 'tf_parallelism_threads' in self.config:
n_cpus = self.config['tf_parallelism_threads']
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': 1},
intra_op_parallelism_threads=n_cpus,
inter_op_parallelism_threads=n_cpus,
))
else:
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': 1},
))
with g.as_default():
# Create Data Handler object
self.data_handler = DataHandler(self.config)
self.data_handler.setup_with_config(
os.path.join(self._model_path, 'config_meta_data.yaml'))
# Get time vars that need to be corrected by global time offset
self._time_indices = []
for i, name in enumerate(self.data_handler.label_names):
if name in self.data_handler.relative_time_keys:
self._time_indices.append(i)
# create data transformer
self.data_transformer = DataTransformer(
data_handler=self.data_handler,
treat_doms_equally=self.config['trafo_treat_doms_equally'],
normalize_dom_data=self.config['trafo_normalize_dom_data'],
normalize_label_data=self.config['trafo_normalize_label_data'],
normalize_misc_data=self.config['trafo_normalize_misc_data'],
log_dom_bins=self.config['trafo_log_dom_bins'],
log_label_bins=self.config['trafo_log_label_bins'],
log_misc_bins=self.config['trafo_log_misc_bins'],
norm_constant=self.config['trafo_norm_constant'])
# load trafo model from file
self.data_transformer.load_trafo_model(
self.config['trafo_model_path'])
# create NN model
self.model = NNModel(is_training=False,
config=self.config,
data_handler=self.data_handler,
data_transformer=self.data_transformer,
sess=sess)
# compile model: initalize and finalize graph
self.model.compile()
# restore model weights
self.model.restore()
# Get trained labels, e.g. labels with weights greater than zero
self._mask_labels = \
self.model.shared_objects['label_weight_config'] > 0
self._non_zero_labels = [
n for n, b in zip(self.data_handler.label_names,
self._mask_labels) if b
]
self._non_zero_log_bins = \
[l for l, b in
zip(self.data_transformer.trafo_model['log_label_bins'],
self._mask_labels) if b]
def main(config_files):
"""Script to train the NN model.
Creates data handler, data transformer, and build NN model as specified
in the config files. Compiles and trains the NN model.
Parameters
----------
config_files : list of strings
List of yaml config files.
"""
# read in and combine config files and set up
setup_manager = SetupManager(config_files)
config = setup_manager.get_config()
if not config['model_is_training']:
raise ValueError('Model must be in training mode!')
# Create Data Handler object
data_handler = DataHandler(config)
data_handler.setup_with_test_data(config['training_data_file'])
# Create Data iterators for training and validation data
train_data_generator = data_handler.get_batch_generator(
input_data=config['training_data_file'],
batch_size=config['batch_size'],
sample_randomly=True,
pick_random_files_forever=True,
file_capacity=config['file_capacity'],
batch_capacity=config['batch_capacity'],
num_jobs=config['num_jobs'],
num_add_files=config['num_add_files'],
num_repetitions=config['num_repetitions'],
num_splits=config['data_handler_num_splits'],
init_values=config['DOM_init_values'],
nan_fill_value=config['data_handler_nan_fill_value'],
)
val_data_generator = data_handler.get_batch_generator(
input_data=config['validation_data_file'],
batch_size=config['batch_size'],
sample_randomly=True,
pick_random_files_forever=True,
file_capacity=1,
batch_capacity=5,
num_jobs=1,
num_add_files=1,
num_repetitions=1,
num_splits=config['data_handler_num_splits'],
init_values=config['DOM_init_values'],
nan_fill_value=config['data_handler_nan_fill_value'],
)
# create data transformer
data_transformer = DataTransformer(
data_handler=data_handler,
treat_doms_equally=config['trafo_treat_doms_equally'],
normalize_dom_data=config['trafo_normalize_dom_data'],
normalize_label_data=config['trafo_normalize_label_data'],
normalize_misc_data=config['trafo_normalize_misc_data'],
log_dom_bins=config['trafo_log_dom_bins'],
log_label_bins=config['trafo_log_label_bins'],
log_misc_bins=config['trafo_log_misc_bins'],
norm_constant=config['trafo_norm_constant'])
# load trafo model from file
data_transformer.load_trafo_model(config['trafo_model_path'])
with tf.Graph().as_default():
# create NN model
model = NNModel(is_training=True,
config=config,
data_handler=data_handler,
data_transformer=data_transformer)
# compile model: define loss function and optimizer
model.compile()
# restore model weights
if config['model_restore_model']:
model.restore()
# train model
model.fit(
num_training_iterations=config['num_training_iterations'],
train_data_generator=train_data_generator,
val_data_generator=val_data_generator,
evaluation_methods=None,
)
def main(config_files):
"""Script to generate trafo model.
Creates the desired trafo model as defined in the yaml configuration files
and saves the trafo model to disc.
Parameters
----------
config_files : list of strings
List of yaml config files.
"""
# read in and combine config files and set up
setup_manager = SetupManager(config_files)
config = setup_manager.get_config()
# Create Data Handler object
data_handler = DataHandler(config)
data_handler.setup_with_test_data(config['trafo_data_file'])
settings_trafo = {
'input_data': config['trafo_data_file'],
'batch_size': config['batch_size'],
'sample_randomly': True,
'pick_random_files_forever': False,
'file_capacity': 1,
'batch_capacity': 2,
'num_jobs': config['trafo_num_jobs'],
'num_add_files': 1,
'num_repetitions': 1,
'num_splits': config['data_handler_num_splits'],
'init_values': config['DOM_init_values'],
'nan_fill_value': config['data_handler_nan_fill_value'],
}
trafo_data_generator = data_handler.get_batch_generator(**settings_trafo)
# create TrafoModel
data_transformer = DataTransformer(
data_handler=data_handler,
treat_doms_equally=config['trafo_treat_doms_equally'],
normalize_dom_data=config['trafo_normalize_dom_data'],
normalize_label_data=config['trafo_normalize_label_data'],
normalize_misc_data=config['trafo_normalize_misc_data'],
log_dom_bins=config['trafo_log_dom_bins'],
log_label_bins=config['trafo_log_label_bins'],
log_misc_bins=config['trafo_log_misc_bins'],
norm_constant=config['trafo_norm_constant'])
data_transformer.create_trafo_model_iteratively(
data_iterator=trafo_data_generator,
num_batches=config['trafo_num_batches'])
# save trafo model to file
base_name = os.path.basename(config['trafo_model_path'])
if '.' in base_name:
file_ending = base_name.split('.')[-1]
base_name = base_name.replace('.' + file_ending, '')
directory = os.path.dirname(config['trafo_model_path'])
if not os.path.isdir(directory):
os.makedirs(directory)
misc.print_warning('Creating directory: {}'.format(directory))
trafo_config_file = os.path.join(directory,
'config_trafo__{}.yaml'.format(base_name))
with open(trafo_config_file, 'w') as yaml_file:
yaml.dump(config, yaml_file, default_flow_style=False)
data_transformer.save_trafo_model(config['trafo_model_path'])
# kill multiprocessing queues and workers
data_handler.kill()
print('\n=======================================')
print('= Successfully saved trafo model to: =')
print('=======================================')
print('{!r}\n'.format(config['trafo_model_path']))
def _create_model(self, cfg, cfg_sel):
"""Helper function for biased selection. Creates a new DNN model in an
independent graph
Parameters
----------
cfg_sel : dict
The biased selection config that describes how to bias the input
data.
Returns
-------
NNModel
The created DNN model.
"""
# Create a new tf graph and session for this model instance
g = tf.Graph()
if 'tf_parallelism_threads' in cfg_sel:
n_cpus = cfg_sel['tf_parallelism_threads']
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': cfg_sel['GPU_device_count']},
intra_op_parallelism_threads=n_cpus,
inter_op_parallelism_threads=n_cpus,
))
else:
sess = tf.compat.v1.Session(
graph=g,
config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True),
device_count={'GPU': cfg_sel['GPU_device_count']},
))
with g.as_default():
# Create Data Handler object
data_handler = DataHandler(cfg)
data_handler.setup_with_test_data(cfg['training_data_file'])
# create data transformer
data_transformer = DataTransformer(
data_handler=data_handler,
treat_doms_equally=cfg['trafo_treat_doms_equally'],
normalize_dom_data=cfg['trafo_normalize_dom_data'],
normalize_label_data=cfg['trafo_normalize_label_data'],
normalize_misc_data=cfg['trafo_normalize_misc_data'],
log_dom_bins=cfg['trafo_log_dom_bins'],
log_label_bins=cfg['trafo_log_label_bins'],
log_misc_bins=cfg['trafo_log_misc_bins'],
norm_constant=cfg['trafo_norm_constant'])
# load trafo model from file
data_transformer.load_trafo_model(cfg['trafo_model_path'])
# create NN model
model = NNModel(is_training=False,
config=cfg,
data_handler=data_handler,
data_transformer=data_transformer,
sess=sess)
# compile model: initalize and finalize graph
model.compile()
# restore model weights
model.restore()
return model, data_transformer, data_handler
def main(config_files):
"""Script to generate trafo model.
Creates the desired trafo model as defined in the yaml configuration files
and saves the trafo model to disc.
Parameters
----------
config_files : list of strings
List of yaml config files.
"""
# read in and combine config files and set up
setup_manager = SetupManager(config_files)
config = setup_manager.get_config()
# Create Data Handler object
data_handler = DataHandler(config)
data_handler.setup_with_test_data(config['trafo_data_file'])
settings_trafo = {
'input_data': config['training_data_file'],
'batch_size': config['batch_size'],
'sample_randomly': True,
'pick_random_files_forever': False,
'file_capacity': 1,
'batch_capacity': 2,
'num_jobs': config['num_jobs'],
'num_add_files': 0,
'num_repetitions': 1,
'init_values': config['DOM_init_values'],
}
trafo_data_generator = data_handler.get_batch_generator(**settings_trafo)
# create TrafoModel
data_transformer = DataTransformer(
data_handler=data_handler,
treat_doms_equally=config['trafo_treat_doms_equally'],
normalize_dom_data=config['trafo_normalize_dom_data'],
normalize_label_data=config['trafo_normalize_label_data'],
normalize_misc_data=config['trafo_normalize_misc_data'],
log_dom_bins=config['trafo_log_dom_bins'],
log_label_bins=config['trafo_log_label_bins'],
log_misc_bins=config['trafo_log_misc_bins'],
norm_constant=config['trafo_norm_constant'])
# load trafo model from file
data_transformer.load_trafo_model(config['trafo_model_path'])
# Perform some basic tests
for i in range(3):
# get a new batch
batch = next(trafo_data_generator)
for test_data, data_type in zip(batch,
['ic78', 'deepcore',
'label', 'misc']):
if test_data is None:
continue
test_data_trafo = data_transformer.transform(test_data, data_type)
test_data_inv_trafo = data_transformer.inverse_transform(
test_data_trafo, data_type)
deviations = np.reshape(np.abs(test_data_inv_trafo - test_data),
[-1])
print('Checking numpy trafor for {}:'.format(data_type))
print('\tChecking Mean:')
print('\tOrig: {:2.5f}, Trafo: {:2.5f}, Inv-Trafo: {:2.5f}'.format(
np.mean(test_data),
np.mean(test_data_trafo),
np.mean(test_data_inv_trafo)))
print('\tChecking Standard Deviation:')
print('\tOrig: {:2.5f}, Trafo: {:2.5f}, Inv-Trafo: {:2.5f}'.format(
np.std(test_data, ddof=1),
np.std(test_data_trafo, ddof=1),
np.std(test_data_inv_trafo, ddof=1)))
print('\tOriginal is same as inv-trafo(trafo(original)): ',
np.allclose(test_data, test_data_inv_trafo),
(test_data_inv_trafo == test_data).all())
print('\tResiduals: min {:2.8f}, max {:2.8f}, mean {:2.8f}'.format(
np.min(deviations),
np.max(deviations),
np.mean(deviations)))
test_data = tf.constant(test_data)
test_data_trafo = data_transformer.transform(test_data, data_type)
test_data_inv_trafo = data_transformer.inverse_transform(
test_data_trafo, data_type)
sess = tf.Session()
test_data, test_data_trafo, test_data_inv_trafo = sess.run(
[test_data, test_data_trafo, test_data_inv_trafo])
deviations = np.reshape(np.abs(test_data_inv_trafo - test_data),
[-1])
print('Checking tensorflow trafor for {}:'.format(data_type))
print('\tChecking Mean:')
print('\tOrig: {:2.5f}, Trafo: {:2.5f}, Inv-Trafo: {:2.5f}'.format(
np.mean(test_data),
np.mean(test_data_trafo),
np.mean(test_data_inv_trafo)))
print('\tChecking Standard Deviation:')
print('\tOrig: {:2.5f}, Trafo: {:2.5f}, Inv-Trafo: {:2.5f}'.format(
np.std(test_data, ddof=1),
np.std(test_data_trafo, ddof=1),
np.std(test_data_inv_trafo, ddof=1)))
print('\tOriginal is same as inv-trafo(trafo(original)): ',
np.allclose(test_data, test_data_inv_trafo),
(test_data_inv_trafo == test_data).all())
print('\tResiduals: min {:2.8f}, max {:2.8f}, mean {:2.8f}'.format(
np.min(deviations), np.max(deviations), np.mean(deviations)))
def main(config_files, output_folder, data_settings, logs):
"""Script to export dnn reco model.
Parameters
----------
config_files : list of strings
List of yaml config files.
"""
# Check paths and define output names
if not os.path.isdir(output_folder):
print('Creating directory: {!r}'.format(output_folder))
os.makedirs(output_folder)
else:
if len(os.listdir(output_folder)) > 0:
if click.confirm("Directory already exists and contains files! "
"Delete {!r}?".format(output_folder),
default=False):
shutil.rmtree(output_folder)
os.makedirs(output_folder)
else:
raise ValueError('Aborting!')
# read in and combine config files and set up
setup_manager = SetupManager(config_files)
config = setup_manager.get_config()
# Create Data Handler object
data_handler = DataHandler(config)
data_handler.setup_with_test_data(config['training_data_file'])
# create data transformer
data_transformer = DataTransformer(
data_handler=data_handler,
treat_doms_equally=config['trafo_treat_doms_equally'],
normalize_dom_data=config['trafo_normalize_dom_data'],
normalize_label_data=config['trafo_normalize_label_data'],
normalize_misc_data=config['trafo_normalize_misc_data'],
log_dom_bins=config['trafo_log_dom_bins'],
log_label_bins=config['trafo_log_label_bins'],
log_misc_bins=config['trafo_log_misc_bins'],
norm_constant=config['trafo_norm_constant'])
# load trafo model from file
data_transformer.load_trafo_model(config['trafo_model_path'])
with tf.Graph().as_default():
# create NN model
model = NNModel(is_training=True,
config=config,
data_handler=data_handler,
data_transformer=data_transformer)
# compile model: define loss function and optimizer
model.compile()
# -------------------------
# Export latest checkpoints
# -------------------------
checkpoint_dir = os.path.dirname(config['model_checkpoint_path'])
latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
if latest_checkpoint is None:
raise ValueError('Could not find a checkpoint. Aborting export!')
else:
for ending in ['.index', '.meta', '.data-00000-of-00001']:
shutil.copy2(src=latest_checkpoint + ending, dst=output_folder)
shutil.copy2(src=os.path.join(checkpoint_dir, 'checkpoint'),
dst=output_folder)
# -----------------------------
# read and export data settings
# -----------------------------
export_data_settings(data_settings=data_settings,
output_folder=output_folder)
# -----------------------------
# Export trafo model and config
# -----------------------------
base_name = os.path.basename(config['trafo_model_path'])
if '.' in base_name:
file_ending = base_name.split('.')[-1]
base_name = base_name.replace('.' + file_ending, '')
shutil.copy2(src=config['trafo_model_path'],
dst=os.path.join(output_folder, 'trafo_model.npy'))
shutil.copy2(src=os.path.join(os.path.dirname(config['trafo_model_path']),
'config_trafo__{}.yaml'.format(base_name)),
dst=os.path.join(output_folder, 'config_trafo.yaml'))
# ----------------------------
# Export training config files
# ----------------------------
checkpoint_directory = os.path.dirname(config['model_checkpoint_path'])
training_files = glob.glob(
os.path.join(checkpoint_directory, 'config_training_*.yaml'))
for training_file in training_files:
shutil.copy2(src=training_file,
dst=os.path.join(output_folder,
os.path.basename(training_file)))
shutil.copy2(src=os.path.join(checkpoint_directory, 'training_steps.yaml'),
dst=os.path.join(output_folder, 'training_steps.yaml'))
# ----------------------
# Export model meta data
# ----------------------
# Export all the information that the datahandler and data trafo collect
# via the test file
# ToDo: implement DataHandler.setup_with_config(config_meta_data.yaml)
# (instead of DataHandler.setup_with_data_container)
meta_data = {
'label_names': data_handler.label_names,
'label_name_dict': data_handler.label_name_dict,
'label_shape': data_handler.label_shape,
'num_labels': data_handler.num_labels,
'misc_names': data_handler.misc_names,
'misc_name_dict': data_handler.misc_name_dict,
'misc_data_exists': data_handler.misc_data_exists,
'misc_shape': data_handler.misc_shape,
'num_misc': data_handler.num_misc,
}
with open(os.path.join(output_folder, 'config_meta_data.yaml'), 'w') as f:
yaml.dump(meta_data, f, default_flow_style=False)
# ------------------------------------
# Export package versions and git hash
# ------------------------------------
version_control = {
'git_short_sha': config['git_short_sha'],
'git_sha': config['git_sha'],
'git_origin': config['git_origin'],
'git_uncommited_changes': config['git_uncommited_changes'],
'pip_installed_packages': config['pip_installed_packages'],
}
with open(os.path.join(output_folder, 'version_control.yaml'), 'w') as f:
yaml.dump(version_control, f, default_flow_style=False)
# -------------------------------
# Export tensorflow training logs
# -------------------------------
if logs:
log_directory = os.path.dirname(config['log_path'])
shutil.copytree(src=log_directory,
dst=os.path.join(output_folder, 'logs'))
print('\n====================================')
print('= Successfully exported model to: =')
print('====================================')
print('{!r}\n'.format(output_folder))