def __call__(self, *args, **kwargs): # This is the main entrance to run configuration testing logger.debug("In the __call__ function") # Read through all samples in the input file while self.parser.has_next(): logger.info('Starting new sample') # Read input from file self._read_next_input() batch_number = self.parser.samples_read() // self.batch_size logger.info('Testing configuration {}/{} for batch {}'.format( 1, self.num_configurations, batch_number)) self._test_configuration("random") logger.info('Testing configuration {}/{} for batch {}'.format( 2, self.num_configurations, batch_number)) self._test_configuration("sensitivity_analysis") # Run test for each configuration in the configuration list. for idx, config in enumerate(self.configurations[:1]): logger.info('Testing configuration {}/{} for batch {}'.format( idx + 3, self.num_configurations, batch_number)) self._test_configuration(config) logger.info("Done testing all of the input to the given file")
def _read_next_input(self): logger.debug("Reading input") self.features_read = self.parser.next_batch() logger.debug("Read labels: \n{}".format(self.features_read['label'])) # Prepare separate graph for reading input graph = tf.Graph() with graph.as_default(): *_, input_model, input_feed_dict = _construct_X_and_feed_dict( self.features_read) # Required variable in order to use tf.make_template() global_step = tf.Variable(0, name='global_step', trainable=False) logger.debug('Making sirs model for the input') self.input_model = sirs_classifier.create_model( None, global_step, input_model, False, INPUT_SIZE, CONTEXT_SIZE, NUM_CLASSES, USE_TEXT, use_char_autoencoder=USE_AUTOENCODER, use_doc2vec=USE_DOC2VEC) y_hat = self.input_model['y_hat'] y_maxes = tf.reduce_max(y_hat, axis=2) y_argmax = tf.cast(tf.argmax(y_maxes, axis=1), tf.int32) self.input_model['offsets'] = tf.pad( tf.expand_dims(y_argmax, axis=1), [[0, 0], [3, 1]]) with tf.Session() as s: s.run([tf.local_variables_initializer()]) # Create a tf Saver that can be used to restore a pre-trained model below saver = tf.train.Saver() self.restore_checkpoint(s, saver) # Read the next input self.input_selection, y_hat = s.run( [self.input_model['offsets'], self.input_model['y_hat']], feed_dict=input_feed_dict) # Reduce max sequence length self.max_sequence = y_hat.shape[1] # Tell the parser that we read a batch self.parser.did_read_batch() logger.info("Read sample {} with label \n{}".format( self.parser.samples_read(), self.features_read['label']))
def __init__(self, dest) -> None: self._destination_folder = dest[:-1] if dest[-1] in ['/', '\\' ] else dest i = 1 destination = self._destination_folder while os.path.exists(destination): destination = "{}_{:02}".format(self._destination_folder, i) i += 1 self._destination_folder = destination logger.info("Writing results to {}".format(self._destination_folder)) os.makedirs(self._destination_folder)
def __init__(self, input_features, model, destination, batch_size, **kwargs): # Remember the batch_size self.batch_size = batch_size self.confs = kwargs # Remember the file containing the model self.model_file = model # Dict used to holde the input elements (as tensors) self.features_batch = { 'features': None, 'context': None, 'seq_len': None, 'label:': None, 'forloeb': None, } # Result writer used to append benchmark results to files according to destination directory self.writer = ResultWriter(destination) # Gets all the different configurations self.configurations = get_configurations() # Count the configurations and add sensitivity analysis and random self.num_configurations = len(self.configurations) + 2 self.parser = NpyFeatureParser(input_features, batch_size) logger.info("Testing {} samples from {}".format( self.parser.get_record_count(), input_features)) logger.info("Writing results to folder: {}".format(destination)) logger.info("Model used: {}".format(model)) logger.info("Batch size: {}".format(batch_size))
def _show_results(**kwargs): if kwargs['file']: scores = [ScoreParser(f) for f in kwargs['file']] else: scores = [ ScoreParser(join(kwargs['benchmark_dir'], f)) for f in listdir(kwargs['benchmark_dir']) if isfile(join(kwargs['benchmark_dir'], f)) and f.endswith(".res") ] scores.sort(key=lambda x: x.AOPC, reverse=True) to_print = [ sc for sc in scores if sc.title in ['Sensitivity analysis', 'Random'] ] rest = [ sc for sc in scores if sc.title not in ['Sensitivity analysis', 'Random'] ] to_print.extend(rest[:1]) logger.info("------------ - best - -------------------------") for score in to_print: logger.info("\n" + str(score)) if kwargs['plot']: write_scores_to_plot(to_print, **kwargs) logger.info("------------ - worst - -------------------------") logger.info("\n" + str(rest[-1]))
def do_nan_searching(configs, selected_model, model_file, model_name, **kwargs): iterations = kwargs['test_size'] // kwargs['batch_size'] feed = DataFeed() start = kwargs['start'] end = kwargs['end'] if end == -1: end = len(configs) found_nans = 0 for config_idx, config in enumerate(configs[start:end]): graph = tf.Graph() feed.reset_permutation() with graph.as_default(): x = tf.placeholder(tf.float32, shape=[None, 784]) y_ = tf.placeholder(tf.float32, shape=[None, 10]) is_training = False y, _ = selected_model['nn'](x, y_, is_training) print("Testing ({}/{}), Nan-cnt {}: {}".format( config_idx, end - start, found_nans, config)) explanation = lrp.lrp(x, y, config) init = get_initializer(False) important_variables = tf.trainable_variables() important_variables.extend( [v for v in tf.global_variables() if 'moving_' in v.name]) saver = tf.train.Saver(important_variables) with tf.Session() as s: # Initialize stuff and restore model s.run(init) saver.restore(s, model_file) found_nans += do_search(config, s, feed, explanation, iterations, kwargs['batch_size'], x) logger.info( "Found nans for {} configurations in total for model {}".format( found_nans, model_name)) print("Found nans for {} configurations in total for model {}".format( found_nans, model_name)) return model_name, found_nans
def _read_next_input(self): logger.debug("Starting input session") # Start queue runners for the reader queue to work tf.train.start_queue_runners(sess=self.input_session) # Read the next input self.features_read, self.input_selection, y_hat = self.input_session.run( [ self.next_batch, self.input_model['offsets'], self.input_model['y_hat'] ]) # Reduce max sequence length self.max_sequence = y_hat.shape[1] # Tell the parser that we read a batch self.parser.did_read_batch() logger.info("Read sample {} with label \n{}".format( self.parser.samples_read(), self.features_read['label']))
def _do_test(test_file, **kwargs): with tf.Graph().as_default(): parser = FeatureParser(test_file, INPUT_SIZE, CONTEXT_SIZE, **kwargs) features = parser.next_batch() num_records = parser.get_record_count() # Keep scope empty sirs_model = tf.make_template('', _create_model) model = sirs_model(features) valid_scores = [] scores = [] for s in kwargs['scores']: if s in model: valid_scores.append(s) m_score = model[s] if s == 'accuracy': m_score = tf.reduce_mean(tf.cast(m_score, tf.float32)) scores.append(m_score) else: logger.error("Score '{}' is not supported.".format(s)) with tf.Session() as sess: # Initialize variables sess.run([tf.local_variables_initializer()]) # Restore model _restore_checkpoint(sess, kwargs['model']) # Find scores evaluations = _evaluate_model(sess, model, num_records, scores, **kwargs) # Print scores logger.info("Scores: ") for sc in zip(valid_scores, evaluations): print(sc) logger.info("{:<10}: {:10f}".format(*sc))
def write_result(self, config, label, prediction, results): file_name = "{}/{}.res".format(self._destination_folder, config) exists = os.path.isfile(file_name) label = label.reshape((label.shape[0], 1)) prediction = prediction.reshape((prediction.shape[0], 1)) ls = np.concatenate([label, prediction], axis=1) fmt = "{:>10d} " * 2 + "{:>10.6f} " * results.shape[2] with open(file_name, 'a') as f: if not exists: num_iterations = results.shape[2] str_format = "{:10} {:10} " + "{:>10} " * num_iterations headings = str_format.format( 'label', 'pred', *[i for i in range(num_iterations)]) + '\n' f.write(headings) for i, sample in enumerate(results): for row in sample: f.write(fmt.format(*ls[i], *row) + "\n") f.write("\n") logger.info("Saved result for config {}".format(config))
def run_model(selected_model_names, **kwargs): results = [] for selected_model_name in selected_model_names: print("#" * 40) print("Searching in model: {}".format(selected_model_name.title())) selected_model = models[selected_model_name] configs = configurations.get_configurations_for_layers( *selected_model['confs']) model_file = '%s/%s.ckpt' % (model_dir, selected_model_name) res = do_nan_searching(configs, selected_model, model_file, model_name=selected_model_name, **kwargs) results.append(res) logger.info("Summary:") for r in results: logger.info("\t%s: %3i" % r) print("\t%s: %3i" % r)
def write_explanation(self, config, R): self._write_sparse_tensor('rel_{}'.format(config), R) logger.info("Saved relevances for batch")
def write_input(self, X): self._write_sparse_tensor('inputs', X) logger.info("Saved input batch")
def _test_configuration(self, config): # Start new graph for this configuration graph = tf.Graph() with graph.as_default(): logger.debug( "Start of new test graph with config {}".format(config)) X, X_reordered, self.features_batch, feed_dict = _construct_X_and_feed_dict( self.features_read) # Prepare template (that uses parameter sharing across calls to sirs_template) sirs_template = tf.make_template('', self.create_model) logger.debug("Building graph for forward pass") logger.debug(sirs_template) # Compute the DRN graph model = sirs_template(X_reordered) should_write_input = False if isinstance(config, str): # The config is either random or SA if config == 'random': # Compute random relevances logger.debug("Building random graph") R = get_random_relevance(X) should_write_input = True else: # Compute sensitivity analysis logger.debug("Building SA graph") R = get_sensitivity_analysis(X, model['y_hat']) else: logger.debug('Building lrp graph') R = lrp.lrp(X, model['y_hat'], config) logger.debug('Done building lrp graph') logger.debug("Instantiating pertubation class") # Make pertuber for X and R that prepares a number of pertubations of X pertuber = Pertuber(X, R, self.batch_size, **self.confs) # Build the pertubation graph benchmark = pertuber.build_pertubation_graph(sirs_template) # Create a tf Saver that can be used to restore a pre-trained model below saver = tf.train.Saver() with tf.Session(graph=graph) as s: logger.debug("Initializing vars and restoring model") # Initialize the local variables and restore the model that was trained earlier s.run([tf.local_variables_initializer()]) self.restore_checkpoint(s, saver) logger.debug("Restored model. Now starting threads.") # Create the threads that run the model coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=s) try: # Run the benchmarks. Shapes: # Benchmark_result: batch_size, pertubations, num_classes # y batch_size, 1 # y_hat batch_size, num_classes logger.debug("Starting session for benchmarks") benchmark_result, expl, y, y_hat = self.run_model( [benchmark, R, model['y'], model['y_hat']], model, feed_dict=feed_dict, session=s) logger.debug("Session done") # Remove extra dimension from y # y shape: (batch_size,) y = y[:, 0] # Find argmax for y_hat # y_hat shape: (batch_size,) y_hat = np.argmax(y_hat, axis=1) # Write results to file logger.debug("Writing result to file") self.writer.write_result(config, y, y_hat, benchmark_result) logger.debug("Writing explanation to file") self.writer.write_explanation(config, expl) if should_write_input: logger.debug("Writing input to file") self.writer.write_input(self.features_read['features']) except tf.errors.OutOfRangeError: logger.debug("Done with the testing") except KeyboardInterrupt: logger.debug("Process interrupted by user. Wrapping up.") finally: coord.request_stop() logger.debug("Joining threads") coord.join(threads) logger.info("Done with test")
def __init__(self, input_features, model, destination, batch_size, **kwargs): # Remember the batch_size self.batch_size = batch_size self.confs = kwargs # Remember the file containing the model self.model_file = model # Dict used to holde the input elements (as tensors) self.features_batch = { 'features': None, 'context': None, 'seq_len': None, 'label:': None, 'forloeb': None, } # Result writer used to append benchmark results to files according to destination directory self.writer = ResultWriter(destination) # Gets all the different configurations self.configurations = get_configurations() # Count the configurations and add sensitivity analysis and random self.num_configurations = len(self.configurations) + 2 # Prepare separate graph for reading input self.input_graph = tf.Graph() with self.input_graph.as_default(): self.parser = FeatureParser(input_features, INPUT_SIZE, CONTEXT_SIZE, batch_size) self.next_batch = self.parser.next_batch() # Required variable in order to use tf.make_template() global_step = tf.Variable(0, name='global_step', trainable=False) logger.debug('Making sirs model for the input') self.input_model = sirs_classifier.create_model( None, global_step, self.next_batch, False, INPUT_SIZE, CONTEXT_SIZE, NUM_CLASSES, USE_TEXT, use_char_autoencoder=USE_AUTOENCODER, use_doc2vec=USE_DOC2VEC) y_hat = self.input_model['y_hat'] y_maxes = tf.reduce_max(y_hat, axis=2) y_argmax = tf.cast(tf.argmax(y_maxes, axis=1), tf.int32) self.input_model['offsets'] = tf.pad( tf.expand_dims(y_argmax, axis=1), [[0, 0], [3, 1]]) self.input_session = tf.Session(graph=self.input_graph) self.input_session.run([tf.local_variables_initializer()]) # Create a tf Saver that can be used to restore a pre-trained model below saver = tf.train.Saver() self.restore_checkpoint(self.input_session, saver) logger.info("Testing {} samples from {}".format( self.parser.get_record_count(), input_features)) logger.info("Writing results to folder: {}".format(destination)) logger.info("Model used: {}".format(model)) logger.info("Batch size: {}".format(batch_size))
def _test_configuration(self, config): # Start new graph for this configuration graph = tf.Graph() with graph.as_default(): logger.debug( "Start of new test graph with config {}".format(config)) # Dictionary to hold all the 'feed_dict' parameters for session.run to_feed = dict() # Construct sparse tensors from placeholders # X read from the input (this is a SparseTensorValue , i.e. numpy like) X_read = self.features_read['features'] # Placeholders to reconstruct X in this new graph X_indices = tf.placeholder(tf.int64, X_read.indices.shape) X_values = tf.placeholder(tf.float32, X_read.values.shape) X_shape = tf.placeholder(tf.int64, np.size(X_read.dense_shape)) X = tf.SparseTensor(X_indices, X_values, X_shape) # Do sparse reorder to ensure that LRP (and other sparse matrix operations) works X_reordered = tf.sparse_reorder(X) # Fill actual values into the three placeholders above to_feed[X_indices] = X_read.indices to_feed[X_values] = X_read.values to_feed[X_shape] = X_read.dense_shape # Do the same sparse tensor reconstruction trick for the context # C read from the input (this is a SparseTensorValue, i.e. numpy like) C_read = self.features_read['context'] # Placeholders to reconstruct C in this new graph C_indices = tf.placeholder(tf.int64, C_read.indices.shape) C_values = tf.placeholder(tf.float32, C_read.values.shape) C_shape = tf.placeholder(tf.int64, np.size(C_read.dense_shape)) C = tf.SparseTensor(C_indices, C_values, C_shape) # Fill actual values into the three placeholders for C to_feed[C_indices] = C_read.indices to_feed[C_values] = C_read.values to_feed[C_shape] = C_read.dense_shape # Store sparse context tensor self.features_batch['context'] = C # Same circus for seq_len seq_len = tf.placeholder(tf.int64, (None, )) self.features_batch['seq_len'] = seq_len # Same circus for label label = tf.placeholder(tf.int64, self.features_read['label'].shape) self.features_batch['label'] = label # Same circus for forloeb forloeb = tf.placeholder(tf.int64, self.features_read['forloeb'].shape) self.features_batch['forloeb'] = forloeb # Fill actual values into seq_len, label, forloeb to_feed[seq_len] = self.features_read['seq_len'] to_feed[label] = self.features_read['label'] to_feed[forloeb] = self.features_read['forloeb'] # Prepare template (that uses parameter sharing across calls to sirs_template) self.first_template_use = True sirs_template = tf.make_template('', self.create_model) logger.debug("Building graph for forward pass") # Compute the DRN graph model = sirs_template(X_reordered) should_write_input = False if isinstance(config, str): # The config is either random or SA if config == 'random': # Compute random relevances logger.debug("Building random graph") R = get_random_relevance(X) should_write_input = True else: # Compute sensitivity analysis logger.debug("Building SA graph") R = get_sensitivity_analysis(X, model['y_hat']) else: logger.debug('Building lrp graph') R = lrp.lrp(X, model['y_hat'], config) logger.debug('Done building lrp graph') logger.debug("Instantiating pertubation class") # Make pertuber for X and R that prepares a number of pertubations of X pertuber = Pertuber(X, R, self.batch_size, **self.confs) # Build the pertubation graph benchmark = pertuber.build_pertubation_graph(sirs_template) # Create a tf Saver that can be used to restore a pre-trained model below saver = tf.train.Saver() with tf.Session(graph=graph) as s: logger.debug("Initializing vars and restoring model") # Initialize the local variables and restore the model that was trained earlier s.run([tf.local_variables_initializer()]) self.restore_checkpoint(s, saver) logger.debug("Restored model. Now starting threads.") # Create the threads that run the model coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=s) try: # Run the benchmarks. Shapes: # Benchmark_result: batch_size, pertubations, num_classes # y batch_size, 1 # y_hat batch_size, num_classes logger.debug("Starting session for benchmarks") benchmark_result, expl, y, y_hat = self.run_model( [benchmark, R, model['y'], model['y_hat']], model, feed_dict=to_feed, session=s) logger.debug("Session done") # Remove extra dimension from y # y shape: (batch_size,) y = y[:, 0] # Find argmax for y_hat # y_hat shape: (batch_size,) y_hat = np.argmax(y_hat, axis=1) # Write results to file logger.debug("Writing result to file") self.writer.write_result(config, y, y_hat, benchmark_result) logger.debug("Writing explanation to file") self.writer.write_explanation(config, expl) if should_write_input: logger.debug("Writing input to file") self.writer.write_input(X_read) except tf.errors.OutOfRangeError: logger.debug("Done with the testing") except KeyboardInterrupt: logger.debug("Process interrupted by user. Wrapping up.") finally: coord.request_stop() logger.debug("Joining threads") coord.join(threads) logger.info("Done with test")