def __init__( self, sess, seed_inputs, input_tensors, coverage_tensors, metadata_tensors, coverage_function, metadata_function, objective_function, mutation_function, sample_function, threshold, algorithm="kdtree", ): """Init the class. Args: sess: a TF session seed_inputs: np arrays of initial inputs, to seed the corpus with. input_tensors: TF tensors to which we feed batches of input. coverage_tensors: TF tensors we fetch to get coverage batches. metadata_tensors: TF tensors we fetch to get metadata batches. coverage_function: a function that does coverage batches -> coverage object. metadata_function: a function that does metadata batches -> metadata object. objective_function: a function that checks if a CorpusElement satisifies the fuzzing objective (e.g. find a NaN, find a misclassification, etc). mutation_function: a function that does CorpusElement -> mutated data. fetch_function: grabs numpy arrays from the TF runtime using the relevant tensors, to produce coverage_batches and metadata_batches Returns: Initialized object. """ self.coverage_function = coverage_function self.metadata_function = metadata_function self.objective_function = objective_function self.mutation_function = mutation_function # create a single fetch function (to sess.run the tensors) self.fetch_function = build_fetch_function( sess, input_tensors, coverage_tensors, metadata_tensors ) # set up seed corpus seed_corpus = seed_corpus_from_numpy_arrays( seed_inputs, self.coverage_function, self.metadata_function, self.fetch_function ) self.corpus = InputCorpus( seed_corpus, sample_function, threshold, algorithm )
def main(_): """Configures and runs the fuzzer.""" # Log more and return how logging output will be produced tf.logging.set_verbosity(tf.logging.INFO) coverage_function = raw_logit_coverage_function # change function name target_seed = np.random.uniform(low=0.0, high=1.0, size=(1, )) # 语料库,随机值 numpy_arrays = [[target_seed]] targets_tensor = tf.placeholder(tf.float32, [64, 1]) coverage_tensor = tf.identity(targets_tensor) loss_batch_tensor, _ = binary_cross_entropy_with_logits( tf.zeros_like(targets_tensor), tf.nn.sigmoid(targets_tensor)) grads_tensor = tf.gradients(loss_batch_tensor, targets_tensor)[0] tensor_map = { "input": [targets_tensor], "coverage": [coverage_tensor], "metadata": [loss_batch_tensor, grads_tensor], } with tf.Session() as sess: fetch_function = build_fetch_function(sess, tensor_map) size = FLAGS.mutations_per_corpus_item mutation_function = lambda elt: do_basic_mutations( elt, size, a_min=-1000, a_max=1000) # 从语料库选择种子 seed_corpus = seed_corpus_from_numpy_arrays(numpy_arrays, coverage_function, metadata_function, fetch_function) # corpus: mutations_processed, corpus, time, sample_function, updater corpus = InputCorpus(seed_corpus, uniform_sample_function, FLAGS.ann_threshold, "kdtree") fuzzer = Fuzzer( corpus, coverage_function, metadata_function, objective_function, mutation_function, fetch_function, ) # fuzzer run result = fuzzer.loop(FLAGS.total_inputs_to_fuzz) if result is not None: tf.logging.info("Fuzzing succeeded.") tf.logging.info( "Generations to make satisfying element: %s.", result.oldest_ancestor()[1], ) else: tf.logging.info("Fuzzing failed to satisfy objective function.")
def main(_): """Constructs the fuzzer and performs fuzzing.""" # Log more tf.logging.set_verbosity(tf.logging.INFO) # 为将要被记录的日志设置开始入口 # Set the seeds! if FLAGS.seed: random.seed(FLAGS.seed) np.random.seed(FLAGS.seed) coverage_function = all_logit_coverage_function # 覆盖计算方法,所有logit的绝对值之和 image, label = fuzz_utils.basic_mnist_input_corpus( choose_randomly=FLAGS. random_seed_corpus # 这里为False, 返回第一张图片和标签, 图片为28*28*1 ) numpy_arrays = [[image, label]] with tf.Session() as sess: tensor_map = fuzz_utils.get_tensors_from_checkpoint( # 载入checkpoints sess, FLAGS.checkpoint_dir) fetch_function = fuzz_utils.build_fetch_function( sess, tensor_map) # =============== size = FLAGS.mutations_per_corpus_item # 每次变异数量 mutation_function = lambda elt: do_basic_mutations(elt, size) # 变异方法 seed_corpus = seed_corpus_from_numpy_arrays( # 建立seed corpus,输入集 numpy_array是一张图片,返回的 seedcorpus包含一个元素,有metada和coverage信息 numpy_arrays, coverage_function, metadata_function, fetch_function) corpus = InputCorpus( # 建立input corpus seed_corpus, recent_sample_function, FLAGS.ann_threshold, "kdtree" # recent_sample_function用于选择下一个元素 ) fuzzer = Fuzzer( corpus, coverage_function, metadata_function, objective_function, mutation_function, fetch_function, ) result = fuzzer.loop(FLAGS.total_inputs_to_fuzz) if result is not None: tf.logging.info("Fuzzing succeeded.") tf.logging.info( "Generations to make satisfying element: %s.", result.oldest_ancestor()[1], ) else: tf.logging.info("Fuzzing failed to satisfy objective function.")
def main(_): """Constructs the fuzzer and performs fuzzing.""" # Log more tf.logging.set_verbosity(tf.logging.INFO) # Set the seeds! if FLAGS.seed: random.seed(FLAGS.seed) np.random.seed(FLAGS.seed) coverage_function = all_logit_coverage_function # a function to compute coverage image, label = fuzz_utils.basic_mnist_input_corpus( choose_randomly=FLAGS.random_seed_corpus) numpy_arrays = [[image, label]] with tf.Graph().as_default() as g: # change codes and it works sess = tf.Session() # here tensor_map = fuzz_utils.get_tensors_from_checkpoint( sess, FLAGS.checkpoint_dir) fetch_function = fuzz_utils.build_fetch_function(sess, tensor_map) size = FLAGS.mutations_per_corpus_item mutation_function = lambda elt: do_basic_mutations(elt, size) # pram 1 seed_corpus = seed_corpus_from_numpy_arrays(numpy_arrays, coverage_function, metadata_function, fetch_function) corpus = InputCorpus(seed_corpus, recent_sample_function, FLAGS.ann_threshold, "kdtree") # pram 2 fuzzer = Fuzzer( corpus, coverage_function, metadata_function, objective_function, mutation_function, fetch_function, ) result = fuzzer.loop(FLAGS.total_inputs_to_fuzz) # iterations if result is not None: tf.logging.info("Fuzzing succeeded.") tf.logging.info( "Generations to make satisfying element: %s.", result.oldest_ancestor()[1], ) tf.logging.info("Elements for Crashes: {0}".format(result)) else: tf.logging.info("Fuzzing failed to satisfy objective function.")
def main(_): """Constructs the fuzzer and fuzzes.""" # Sets the threshold for what messages will be logged. # Log more tf.logging.set_verbosity(tf.logging.INFO) # all works # coverage_function = raw_logit_coverage_function coverage_function = all_logit_coverage_function # get inputs corpus data image, label = fuzz_utils.basic_mnist_input_corpus( choose_randomly=FLAGS.random_seed_corpus) numpy_arrays = [[image, label]] image_copy = image[:] with tf.Graph().as_default() as g: sess = tf.Session() tensor_map = fuzz_utils.get_tensors_from_checkpoint( sess, FLAGS.checkpoint_dir) fetch_function = fuzz_utils.build_fetch_function(sess, tensor_map) size = FLAGS.mutations_per_corpus_item # 100 def mutation_function(elt): """Mutates the element in question.""" return do_basic_mutations(elt, size, FLAGS.perturbation_constraint) # initialization of seed corpus seed_corpus = seed_corpus_from_numpy_arrays(numpy_arrays, coverage_function, metadata_function, fetch_function) corpus = InputCorpus(seed_corpus, recent_sample_function, FLAGS.ann_threshold, FLAGS.algorithm) fuzzer = Fuzzer( corpus, coverage_function, metadata_function, objective_function, mutation_function, fetch_function, ) result = fuzzer.loop( FLAGS.total_inputs_to_fuzz) # type is CorpusElement if result is not None: # Double check that there is persistent disagreement for idx in range(10): logits, quantized_logits = sess.run( [tensor_map["coverage"][0], tensor_map["coverage"][1]], feed_dict={ tensor_map["input"][0]: np.expand_dims(result.data[0], 0) }, ) # feed_dict: replace tensor value if np.argmax(logits, 1) != np.argmax(quantized_logits, 1): tf.logging.info("disagreement confirmed: idx %s", idx) else: tf.logging.info( "Idx: {0}, SPURIOUS DISAGREEMENT!!! LOGITS: {1}, QUANTIZED_LOGITS: {2}!" .format(idx, logits, quantized_logits)) tf.logging.info( "Fuzzing succeeded. Generations to make satisfying element: %s.", result.oldest_ancestor()[1], ) max_diff = np.max(result.data[0] - image_copy) tf.logging.info( "Max difference between perturbation and original: %s.", max_diff, ) else: tf.logging.info("Fuzzing failed to satisfy objective function.")
def main(_): """Constructs the fuzzer and fuzzes.""" # Log more tf.logging.set_verbosity(tf.logging.INFO) # 为将要被记录的日志设置开始入口 coverage_function = raw_logit_coverage_function image, label = fuzz_utils.basic_mnist_input_corpus( # 一张图片和对应的标签 choose_randomly=FLAGS.random_seed_corpus) numpy_arrays = [[image, label]] image_copy = image[:] with tf.Session() as sess: tensor_map = fuzz_utils.get_tensors_from_checkpoint( sess, FLAGS.checkpoint_dir) fetch_function = fuzz_utils.build_fetch_function(sess, tensor_map) # 获取模型信息 size = FLAGS.mutations_per_corpus_item def mutation_function(elt): # elt为一个元素 """Mutates the element in question.""" return do_basic_mutations( elt, size, FLAGS.perturbation_constraint) # 一个元素多次变异,返回多个变异后的数据 """ numpy_arrays = [[image, label]] 一张图片 coverage_function = raw_logit_coverage_function fetch_function = fuzz_utils.build_fetch_function(sess, tensor_map) """ seed_corpus = seed_corpus_from_numpy_arrays( # 建立seed_corpus numpy_arrays, coverage_function, metadata_function, fetch_function) corpus = InputCorpus( # 建立input corpus seed_corpus, recent_sample_function, FLAGS.ann_threshold, "kdtree") fuzzer = Fuzzer( # 建立Fuzzer对象 corpus, # InputCorpus(Input Corpus) coverage_function, # 计算神经网络覆盖 metadata_function, # 获取metadata objective_function, # 目标方法,检查是否被错误分类 mutation_function, # 变异方法 fetch_function, # 获取模型信息 ) result = fuzzer.loop(FLAGS.total_inputs_to_fuzz) if result is not None: # Double check that there is persistent disagreement for idx in range(10): logits, quantized_logits = sess.run( [tensor_map["coverage"][0], tensor_map["coverage"][1]], feed_dict={ tensor_map["input"][0]: np.expand_dims(result.data[0], 0) }, ) if np.argmax(logits, 1) != np.argmax(quantized_logits, 1): tf.logging.info("disagreement confirmed: idx %s", idx) else: tf.logging.info("SPURIOUS DISAGREEMENT!!!") # 假的不一致 tf.logging.info("Fuzzing succeeded.") tf.logging.info( "Generations to make satisfying element: %s.", result.oldest_ancestor()[1], ) max_diff = np.max(result.data[0] - image_copy) tf.logging.info( "Max difference between perturbation and original: %s.", max_diff, ) else: tf.logging.info("Fuzzing failed to satisfy objective function.")
class Fuzzer(object): """Class representing the fuzzer itself.""" def __init__( self, sess, seed_inputs, input_tensors, coverage_tensors, metadata_tensors, coverage_function, metadata_function, objective_function, mutation_function, sample_function, threshold, algorithm="kdtree", ): """Init the class. Args: sess: a TF session seed_inputs: np arrays of initial inputs, to seed the corpus with. input_tensors: TF tensors to which we feed batches of input. coverage_tensors: TF tensors we fetch to get coverage batches. metadata_tensors: TF tensors we fetch to get metadata batches. coverage_function: a function that does coverage batches -> coverage object. metadata_function: a function that does metadata batches -> metadata object. objective_function: a function that checks if a CorpusElement satisifies the fuzzing objective (e.g. find a NaN, find a misclassification, etc). mutation_function: a function that does CorpusElement -> mutated data. fetch_function: grabs numpy arrays from the TF runtime using the relevant tensors, to produce coverage_batches and metadata_batches Returns: Initialized object. """ self.coverage_function = coverage_function self.metadata_function = metadata_function self.objective_function = objective_function self.mutation_function = mutation_function # create a single fetch function (to sess.run the tensors) self.fetch_function = build_fetch_function( sess, input_tensors, coverage_tensors, metadata_tensors ) # set up seed corpus seed_corpus = seed_corpus_from_numpy_arrays( seed_inputs, self.coverage_function, self.metadata_function, self.fetch_function ) self.corpus = InputCorpus( seed_corpus, sample_function, threshold, algorithm ) def loop(self, iterations): """Fuzzes a machine learning model in a loop, making *iterations* steps.""" for iteration in range(iterations): if iteration % 100 == 0: tf.logging.info("fuzzing iteration: %s", iteration) parent = self.corpus.sample_input() # Get a mutated batch for each input tensor mutated_data_batches = self.mutation_function(parent) # Grab the coverage and metadata for mutated batch from the TF runtime. coverage_batches, metadata_batches = self.fetch_function( mutated_data_batches ) # Get the coverage - one from each batch element mutated_coverage_list = self.coverage_function(coverage_batches) # Get the metadata objects - one from each batch element mutated_metadata_list = self.metadata_function(metadata_batches) # Check for new coverage and create new corpus elements if necessary. # pylint: disable=consider-using-enumerate for idx in range(len(mutated_coverage_list)): new_element = CorpusElement( [batch[idx] for batch in mutated_data_batches], mutated_metadata_list[idx], mutated_coverage_list[idx], parent, ) if self.objective_function(new_element): tf.logging.info( "OBJECTIVE SATISFIED: coverage: %s, metadata: %s", new_element.coverage, new_element.metadata, ) return new_element self.corpus.maybe_add_to_corpus(new_element) return None