def craft(model, x, order, epsilon=1., max_iter=100): nb_elements = x.shape[0] if len(order.shape) == 1: nb_classes = len(order) order = np.repeat(np.array([order]), nb_elements, axis=0) else: nb_classes = order.shape[1] threshold = 1 / (nb_classes + 1) norm = 2 logger.info( 'Crafting Sorted PGD attack; norm:{} threshold:{} epsilon:{}'. format(norm, (threshold, SATA.power), epsilon)) attack_params, attack_name = get_attack_params("targetted_pgd", norm, epsilon) classifier = KerasClassifier(model=model) crafter = SATA(classifier, num_random_init=num_random_init, max_iter=max_iter, eps=1.) crafter.nb_classes = nb_classes crafter.set_params(**attack_params) adv_x, ref_x = crafter.generate(x, order, threshold) return adv_x
def main(): """Load and plot.""" # Load data if not SELF_COMPARISON_DATA_PATH.exists(): logger.error('Self comparison data not found at: {}'.format( SELF_COMPARISON_DATA_PATH)) return data = pandas.read_csv(str(SELF_COMPARISON_DATA_PATH)) # Generate table for each timeout for timeout in PREPROCESSING_TIMEOUTS: # Subset to data by timeout and grab desired columns subset = data[data[headers.TIMEOUT] == timeout][HEADERS] # Group and compute statistics describe = subset.groupby(GROUPBY).describe().round(decimals=1) # Format columns describe = describe.drop(columns=[(headers.SIZE, 'count')]) int_headers = [(headers.SIZE, h) for h in ['min', '25%', '50%', '75%', 'max']] describe[int_headers] = describe[int_headers].astype('int') # Generate latex describe.to_latex( str(IC_TABLES_DIR / IC_TABLE_FORMAT_STR.format(timeout))) # Print with PRINT_CONTEXT: logger.info('Timeout == {}'.format(timeout)) logger.info(str(describe) + '\n')
def run(dataset="cifar10", model_type="basic", epochs=25, exp_id="_gen_dataset"): attack_name = "targeted_pgd" if RANDOM_SEED > 0: random.seed(RANDOM_SEED) np.random.seed(RANDOM_SEED) pass quality = 100 extension = "png" l = 52 nb_messages = 1000 nb_classes = [2, 3] #nb_classes = [4,5] for nb_cls in nb_classes: experiment_id = "{}/{}".format(exp_id, nb_cls) for i in range(nb_messages): logger.info("## class {} iter {}".format(nb_cls, i)) experiment_time = int(time.time()) msg = "".join( [strs[random.randint(0, len(strs) - 1)] for i in range(l)]) _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, nb_classes=nb_cls, experiment_time=experiment_time)
def run_tests(use_gpu=False): if not use_gpu: os.environ["CUDA_VISIBLE_DEVICES"] = "-1" os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import tensorflow as tf from utils.adversarial_models import load_model model, x_train, x_test, y_train, y_test = load_model(dataset="cifar10", model_type="basic", epochs=25) np.random.seed(RANDOM_SEED) nb_elements = 10 nb_classes = 3 order = np.array([2, 3, 5, 8, 9, 4, 6]) order = order[0:nb_classes] order = np.random.randint(0, 10, (nb_elements, nb_classes)) SATA.power = 1 adv_x = SATA.craft(model, x_test[:nb_elements], order, epsilon=3., max_iter=200) adv_y = model.predict(adv_x) logger.info("{}: {}".format( np.sort(-adv_y[0]).shape, list(zip(order, np.argsort(-1 * adv_y, axis=1)))))
def craft(model, x, target, epsilon=3., num_random_init=10, max_iter=100, quality=75, eps_step=0.1): target = to_categorical(target) norm = 2 logger.info( 'Crafting spacially resilient PGD attack; norm:{} epsilon:{}'. format(norm, epsilon)) attack_params, attack_name = get_attack_params("targetted_pgd", norm, epsilon) attack_params["eps_step"] = eps_step classifier = KerasClassifier(model=model) crafter = SpatialResilientPGD(classifier, num_random_init=num_random_init, max_iter=max_iter, eps=1.) crafter.set_params(**attack_params) adv_x = crafter.generate(x, target, quality) return adv_x
def main(): """Validate.""" # Validate directories argv = sys.argv[1:] if len(argv) != 2: raise Exception( 'Usage: python -m experiments.sanity_check_preprocessing ' '<preprocessed data dir> <preprocessed data dir>') dir1 = Path(argv[0]) / 'edgelist' dir2 = Path(argv[1]) / 'edgelist' logger.info('Loading edgelists from `{}`'.format(dir1)) logger.info('Loading edgelists from `{}`'.format(dir2)) # Get common edgelist files edgelists = _glob_edgelists(dir1) & _glob_edgelists(dir2) logger.info('Found {} shared edgelist files'.format(len(edgelists))) # Check each for edgelist in edgelists: logger.info('Validating `{}`'.format(edgelist)) # Construct graphs from edgelists g1 = graphs.read_edgelist(dir1, edgelist) g2 = graphs.read_edgelist(dir2, edgelist) # Check for isomorphism if not graphs.could_be_isomorphic(g1, g2): logger.info('`{}` is not isomorphic'.format(edgelist))
def main(current_server_id, total_servers, seed_filter): """Run experiment.""" # Find the datasets we want input_dir = Path('.') / 'data' / 'preprocessed' / 'edgelist' datasets = names_in_dir(input_dir, '.edgelist') datasets = [dataset.replace('.edgelist', '') for dataset in datasets] datasets = sorted(datasets) # Quantum data has no dashes, synthetics use dashes to separate parameters datasets = list(filter(seed_filter, datasets)) # Split "-server 3 -of 3" across 3 servers dataset_partition = datasets[2::total_servers] dataset_partition = dataset_partition[current_server_id - 1::total_servers][69:] print('Expected data:', dataset_partition) # Collect solvers solvers_dict = { # AI: _run_ai, ILP: _run_ilp, # ILP1T: partial(_run_ilp, threads=1), # IC: _run_ic } solvers = sorted(list(solvers_dict.items())) # Generate experiments experiments = product(dataset_partition, solvers) # Open output file for writing with open(str(EXACT_RESULTS_DATA_PATH), 'w') as output: # Get writer writer = csv.writer(output) # Write header writer.writerow([ headers.DATASET, headers.SOLVER, headers.TIME, headers.SIZE, headers.CERTIFICATE ]) # Run experiments for dataset, (name, solver) in experiments: # Log logger.info('Running {} on {}'.format(name, dataset)) # Solve and write output try: solution = solver(dataset) if solution[0] >= EXACT_TIMEOUT: continue except subprocess.TimeoutExpired: continue writer.writerow([dataset, name, *solution]) output.flush() # Now generate the ground truth table if akiba_iwata was run if (AI, _run_ai) in solvers: _generate_ground_truth()
def main(): """Run experiment.""" # Open output file for writing with open(BASELINE_FILE, 'w') as output: # Get writer writer = csv.writer(output) # Write header writer.writerow([ headers.DATASET, headers.SOLVER, headers.TIME, headers.SIZE, headers.CERTIFICATE ]) # Run experiments for dataset in huffner_experiment_datasets: # Log logger.info('Computing OCT for {}'.format(dataset)) # Solve and write output try: solution = solve(str(HUFFNER_DATA_DIR / (dataset + HUFFNER_DATA_EXT)), timeout=EXACT_TIMEOUT) if solution[0] >= EXACT_TIMEOUT: continue except subprocess.TimeoutExpired: continue writer.writerow([dataset, HUFFNER_BASELINE_SOLVER, *solution]) output.flush()
def run_tests(use_gpu=True): if not use_gpu: os.environ["CUDA_VISIBLE_DEVICES"] = "-1" os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import tensorflow as tf from utils.adversarial_models import load_model model, x_train, x_test, y_train, y_test = load_model( dataset="cifar10", model_type="basic", epochs=25) quality = 100 SpatialResilientPGD.test_quality = 75 nb_elements = 100 np.random.seed(RANDOM_SEED) y = np.random.randint(0,10,nb_elements) adv_x = SpatialResilientPGD.craft(model, x_test[:nb_elements],y, epsilon=3, max_iter=500, quality=quality, eps_step=0.01,num_random_init=50) adv_y = model.predict(adv_x) #logger.info("{}: {}".format(np.sort(-adv_y[0]).shape, list(zip(y, np.argmax(adv_y,axis=1))))) adv_path= "./utils/adv.jpg" save_img(adv_path,adv_x[0], quality=quality) adv_x_post = np.array([_load_image(adv_path)]) adv_y_post = model.predict(adv_x_post) logger.info("{}-{}".format(np.argmax(adv_y_post,axis=1),np.argmax(adv_y,axis=1)))
def _decode(dataset, model_type, epochs, experiment_id, attack_name, experiment_time, extension=None): if not extension: extension = default_extension pictures_path = default_path.format(experiment_id, attack_name, experiment_time) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) score = [] for file in os.listdir(pictures_path): if file.endswith(".{}".format(extension)): path = "{}/{}".format(pictures_path, file) image = load_img(path) if len(image.size) < 3: image = image.convert("RGB") if image.width != 32: image = image.resize((32, 32), Image.BILINEAR) img = img_to_array(image) / palette img_class = np.argmax(model.predict(np.array([img]), verbose=0)) index = file.index("_truth") - 1 real_class = int(file[index:index + 1]) steg_msg = lsb.reveal(path) logger.info("img {} decoded as {} stegano {}".format( file, img_class, steg_msg)) score.append(real_class == img_class) logger.info("decoding score {}".format(np.mean(np.array(score))))
def _decode(dataset, model_type, epochs, experiment_id, attack_name, experiment_time): pictures_path = default_path.format(experiment_id, attack_name, experiment_time) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) score = [] lsb_score = [] stat_score = [] ensemble_score = [] for file in os.listdir(pictures_path): if file.endswith(".{}".format(extension)): path = "{}/{}".format(pictures_path, file) img = img_to_array(load_img(path)) / palette img_class = np.argmax(model.predict(np.array([img]), verbose=0)) index = file.index("_truth") - 1 real_class = int(file[index:index + 1]) score.append(real_class == img_class) steg_msg = lsb.reveal(path) lsb_score.append(0 if lsb_score == None else 1) bitrate_R = attacks.spa(path, 0) bitrate_G = attacks.spa(path, 1) bitrate_B = attacks.spa(path, 2) threshold = 0.05 if bitrate_R < threshold and bitrate_G < threshold and bitrate_B < threshold: stat_score.append(0) else: stat_score.append(1) #logger.info("img {} decoded as {} stegano {}".format(file,img_class,steg_msg)) logger.info("decoding score {}".format(np.mean(np.array(score)))) real_path = "{}/ref".format(pictures_path) for file in os.listdir(pictures_path): if file.endswith(".{}".format(extension)): path = "{}/{}".format(pictures_path, file) e4s_score = _detect_e4s_srm( file_path=path, model_file="e4s_srm_bossbase_lsbm0.10_gs.model") ensemble_score.append(e4s_score) steg_msg = lsb.reveal(path) lsb_score.append(1 if lsb_score == None else 0) spa_score = _detect_spa(path) stat_score.append(spa_score) lsb_score_mean = np.mean(np.array(lsb_score)) stat_score_mean = np.mean(np.array(stat_score)) logger.info("lsb detection score {}, stats detection score {}".format( lsb_score_mean, stat_score_mean))
def _encode(msg,dataset, model_type, epochs, experiment_id,attack_name, attack_strength=2.0, extension=None, transformation=None): if not extension: extension = default_extension encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg,encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) num_classes= 10 combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test,verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] targets = np.array(to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) adv_x = craft_attack(model,x,attack_name,y=targets, epsilon=attack_strength) yadv = np.argmax(model.predict(adv_x), axis=1) pictures_path = default_path.format(experiment_id,attack_name, experiment_time) os.makedirs(pictures_path, exist_ok =True) os.makedirs("{}/ref".format(pictures_path), exist_ok =True) advs = [] for i, _adv in enumerate(adv_x): predicted = yadv[i] encoded = np.argmax(targets[i]) truth = np.argmax(y[i]) adv_path = "{}/{}_predicted{}_encoded{}_truth{}.{}".format(pictures_path,i,predicted,encoded,truth, extension) real_path = "{}/ref/{}.{}".format(pictures_path,i,extension) adv = array_to_img(_adv) if transformation=="rotate": adv = adv.rotate(10) elif transformation=="crop": adv = adv.crop((2,2,30,30)) elif transformation=="upscale": adv = adv.resize((64,64),Image.BILINEAR) elif transformation=="compress": adv = _compress_img(adv) yield {"time":experiment_time,"file":adv_path,"img":adv} # adv.save(adv_path) return experiment_time
def generate(self, x, y, quality): self.targeted = True """ Generate adversarial samples and return them in an array. :param x: An array with the original inputs. :type x: `np.ndarray` :param y: The labels for the data `x`. Only provide this parameter if you'd like to use true labels when crafting adversarial samples. Otherwise, model predictions are used as labels to avoid the "label leaking" effect (explained in this paper: https://arxiv.org/abs/1611.01236). Default is `None`. Labels should be one-hot-encoded. :type y: `np.ndarray` :return: An array holding the adversarial examples. :rtype: `np.ndarray` """ targets = y adv_x_best = None rate_best = 0.0 for i_random_init in range(max(1, self.num_random_init)): adv_x = x #.astype(NUMPY_DTYPE) for i_max_iter in range(self.max_iter): adv_x = self._compute(adv_x, targets, self.eps, self.eps_step, self.num_random_init > 0 and i_max_iter == 0, quality=quality) if self._project: noise = projection(adv_x - x, self.eps, self.norm) adv_x = x + noise rate = 100 * self.compute_success(self.classifier, x, targets, adv_x, self.targeted, quality=quality) logger.info('Success rate {}: {}'.format(i_random_init, rate)) if rate > rate_best or adv_x_best is None: rate_best = rate adv_x_best = adv_x logger.info('Success rate of Spatially resilient PGD attack: %.2f%%', rate_best) SpatialResilientPGD.rate_best = rate_best return adv_x_best
def main(): """Run experiments""" # Log logger.info('Starting Heuristics Experiment') # Generate experiments experiments = product(PREPROCESSING_TIMEOUTS_MILLISECONDS, DATASETS) # Open output file with open(str(HEURISTICS_RESULTS_DATA_FILE), 'w') as output: # Get writer writer = csv.writer(output) # Write header writer.writerow(HEURISTICS_CSV_HEADERS) # Run experiments for experiment in experiments: # Log logger.info( 'Starting experiment timeout={} dataset={}'.format(*experiment) ) # Run subprocess cmd = [str(HEURISTIC_SOLVER), *experiment] proc = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) # Parse results output_string = bytes.decode(proc.stdout, 'utf-8').strip() match = re.match( r'(\d+),(\d+),"(\[((\d+)(,(\d+))*)?\])"', output_string ) size = int(match.group(1)) time = int(match.group(2)) certificate = match.group(3) # Write writer.writerow([ os.path.basename(experiment[1]), experiment[0], size, time, certificate ]) output.flush()
def run(dataset="cifar10", model_type="basic", epochs=25, experiment_id="SP3"): attack_name = "targeted_pgd" logger.info("running {} {} {}".format(dataset, model_type, attack_name)) if RANDOM_SEED > 0: random.seed(RANDOM_SEED) quality = 100 extension = "png" l = 100 msg = "".join([strs[random.randint(0, len(strs) - 1)] for i in range(l)]) # experiment_id = "SP3/1" # exp_time = _encode(msg, dataset, model_type, epochs, experiment_id,attack_name,attack_strength=5.,extension = extension,transformation="rotate") # _decode( dataset, model_type, epochs, experiment_id,attack_name,exp_time,extension = extension) experiment_id = "SP3/2" exp_time = _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, attack_strength=5., extension=extension, transformation="crop") _decode(dataset, model_type, epochs, experiment_id, attack_name, exp_time, extension=extension) experiment_id = "SP3/3" exp_time = _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, attack_strength=5., extension=extension, transformation="upscale") _decode(dataset, model_type, epochs, experiment_id, attack_name, exp_time, extension=extension)
def main(): """Run experiments""" # Compute datasets datasets = list( map(lambda d: str(SNAP_DATA_DIR / (d + SNAP_DATA_EXT)), preprocessed)) # Log logger.info('Starting CPLEX Experiment') # Generate experiments experiments = product(PREPROCESSING_TIMEOUTS, datasets) # Open output file with open(str(CPLEX_RESULTS_DATA_FILE), 'w') as output: # Get writer writer = csv.writer(output) # Write header writer.writerow(HEURISTICS_CSV_HEADERS) # Run experiments for experiment in experiments: try: # Log logger.info('Starting experiment timeout={} dataset={}'.format( *experiment)) # Run cplex solution = solve(read_edgelist(experiment[1]), formulation='VC', solver='CPLEX', threads=4, timelimit=experiment[0], convert_to_oct=True) # Write writer.writerow([ os.path.basename(experiment[1]), experiment[0], solution.opt, solution.time, solution.certificate ]) output.flush() except Exception as e: logger.error(e)
def main(): """Parse arguments and run experiment.""" logger.info('Starting FCL experiment.') # List solvers to use solvers = [(AI, _run_ai), (ILP, _run_ilp), (IC, _run_ic)] # Open results file context with open(RESULTS_FILE, 'w') as results_file_fd: # Create CSV writer and write heuristic header row csv_writer = csv.writer(results_file_fd) csv_writer.writerow(OUTPUT_HEADER) # Run against all available datasets for dataset in _datasets(): logger.info('Processing dataset {}'.format(dataset)) # Execute for each dataset for name in _fcls_from(dataset): logger.info('Solving {}'.format(name)) # Execute for each solver for sn, solver in solvers: logger.info('Running solver {}'.format(sn)) # Try to execute heuristics try: size, time, certificate = solver(dataset, name) # Write results csv_writer.writerow([ '{}/{}'.format(dataset, name), sn, time, size, certificate ]) except Exception as e: # Log error logger.error(e) # Write empty row csv_writer.writerow([ name, sn, float('nan'), float('nan'), float('nan') ]) finally: # Flush results so we at least get partial results if # something catastrophic causes the script to crash. results_file_fd.flush()
def run(dataset="cifar10", model_type="basic", epochs=25, experiment_id="SP2"): attack_name = "targeted_pgd" logger.info("running {} {} {}".format(dataset, model_type, attack_name)) if RANDOM_SEED > 0: random.seed(RANDOM_SEED) experiment_id = "SP2/1" quality = 100 extension = "png" msg1 = "Hello" exp_time = _encode(msg1, dataset, model_type, epochs, experiment_id, attack_name, quality=quality, attack_strength=1., extension=extension) experiment_id = "SP2/2" exp_time = _encode(msg1, dataset, model_type, epochs, experiment_id, attack_name, quality=quality, attack_strength=1., extension=extension, keep_one=True) experiment_id = "SP2/3" l = 100 msg2 = "".join([strs[random.randint(0, len(strs) - 1)] for i in range(l)]) exp_time = _encode(msg2, dataset, model_type, epochs, experiment_id, attack_name, quality=quality, attack_strength=1., extension=extension)
def run(dataset="cifar10", model_type="basic", epochs=25, experiment_id=5): attack_name = "targeted_pgd" logger.info("running {} {} {}".format(dataset, model_type, attack_name)) if RANDOM_SEED > 0: random.seed(RANDOM_SEED) l = 100 msg = "".join([strs[random.randint(0, len(strs) - 1)] for i in range(l)]) exp_time = _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, quality=100, attack_strength=1.) _decode(dataset, model_type, epochs, experiment_id, attack_name, exp_time)
def _encode_adv(msg, dataset, model_type, epochs, experiment_id, attack_name, attack_strength=2.0): encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) num_classes = 10 combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test, verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] targets = np.array( to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) adv_x = craft_attack(model, x, attack_name, y=targets, epsilon=attack_strength) return x, y, adv_x, model, targets
def main(): """Run experiment.""" # Open output file for writing with open(str(HUFFNER_RESULTS_DATA_FILE), 'w') as output: # Get writer writer = csv.writer(output) # Write header writer.writerow(HUFFNER_CSV_HEADERS) # Run all combinations of dataset and timeouts experiments = product(preprocessed, PREPROCESSING_TIMEOUTS) for dataset, timeout in experiments: # Log logger.info('Computing OCT for {} with timeout={}'.format( dataset, timeout)) # Determine preprocessing level and solver name if timeout <= 0.1: preprocessing = PREPROCESSING_ENSEMBLE solver = HUFFNER_P1 else: preprocessing = PREPROCESSING_DENSITY solver = HUFFNER_P2 # Solve solution = solve(str(HUFFNER_DATA_DIR / (dataset + HUFFNER_DATA_EXT)), timeout=timeout, preprocessing=preprocessing, htime=min(0.3 * timeout, 1)) # Write output writer.writerow([dataset, solver, timeout, *solution]) output.flush()
def run(dataset="cifar10", model_type="basic", epochs=25, experiment_id="SP8"): folder = "./experiments/results/experiment{}".format(experiment_id) os.makedirs(folder, exist_ok=True) if RANDOM_SEED > 0: random.seed(RANDOM_SEED) model, x_train, x_test, y_train, y_test = load_model(dataset="cifar10", model_type="basic", epochs=25) combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) best_rates = {} all_rates = {} powers = (1, 1.5, 2, 2.5) dataset_classes = 10 classes = (2, ) classes = (3, 4, 5, 6, 7) epsilons = (0.5, 1., 2, 3) max_iters = (100, 200, 500) nb_elements = 1000 for nb_classes in classes: best_rate = 0 best_combination = {} all_combinations = [] for power in powers: for epsilon in epsilons: for max_iter in max_iters: if RANDOM_SEED > 0: np.random.seed(RANDOM_SEED) order = np.random.randint(0, dataset_classes, (nb_elements, nb_classes)) SATA.power = power adv_x = SATA.craft(model, x_test[:nb_elements], order, epsilon=epsilon, max_iter=max_iter) combination = { "nb_elements": nb_elements, "max_iter": max_iter, "epsilon": epsilon, "power": power, "rate": SATA.rate_best } all_combinations.append(combination) if SATA.rate_best > best_rate: best_rate = SATA.rate_best best_combination = combination logger.info("class {}, combination {}".format( nb_classes, combination)) all_rates[nb_classes] = all_combinations best_rates[nb_classes] = best_combination with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("{}".format(json.dumps({ "all": all_rates, "best": best_rates }))) logger.info("{}".format(best_rates.items()))
def _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, attack_strength=2.0): extension = default_extension encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs, use_tensorboard=True) num_classes = DATASET_CLASSES[dataset] combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test, verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] chunk_size = int(math.log(num_classes) / math.log(10)) #groups = _split_msg(encoded_msg, chunk_size) targets = np.array( to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) class_density = 0.03 # total class * density = total attacked classes epsilon = 5.0 max_iter = 100 SATA.power = 1.5 nb_elements = 1000 adv_x, rate_best = SATA.embed_message(model, x_test[:nb_elements], encoded_msg, epsilon=epsilon, class_density=class_density) #adv_x = craft_attack(model,x,attack_name,y=targets, epsilon=attack_strength) yadv = np.argmax(model.predict(adv_x), axis=1) pictures_path = default_path.format(experiment_id, attack_name, experiment_time) os.makedirs(pictures_path, exist_ok=True) os.makedirs("{}/ref".format(pictures_path), exist_ok=True) SSIM = [] PSNR = [] for i, _adv in enumerate(adv_x): predicted = yadv[i] encoded = np.argmax(targets[i]) truth = np.argmax(y[i]) adv_path = "{}/{}_predicted{}_encoded{}_truth{}.{}".format( pictures_path, i, predicted, encoded, truth, extension) real_path = "{}/ref/{}.{}".format(pictures_path, i, extension) adv = array_to_img(_adv) adv = adv.resize((16, 16), Image.BILINEAR) #adv = adv.convert("L") adv.save(adv_path) adv_loaded = _load_image(adv_path) real_img = array_to_img(x[i]) real_img = real_img.resize((16, 16), Image.BILINEAR) #real_img = real_img.convert("L") #real = np.squeeze(img_to_array(real_img)) real = img_to_array(real_img) ssim = 1 - np.array( list( ssim_distance(None, np.array([real]), adv_x=np.array([adv_loaded]), distance_only=True))) SSIM.append(ssim) psnr = _psnr_loss(adv_loaded, real) PSNR.append(psnr) return np.array(SSIM).mean(), np.array(PSNR).mean()
def run(dataset="cifar100", model_type="basic", epochs=50, exp_id="SP_sata"): if RANDOM_SEED > 0: # random.seed(RANDOM_SEED) # np.random.seed(RANDOM_SEED) pass l = 33000 #6643 num_classes_tbl = (10000, ) #num_classes_tbl =[10000] nb_trials = 100 nb_steps = 1000 nb_pictures = [] nb_pixels = 32 * 32 * 3 def plot(nb_pictures): for i, nb_picture in enumerate(nb_pictures): # plt.figure() # plt.title("Embedding density for N={}".format(num_classes_tbl[i])) # plt.xlabel('Nb classes embedded per image (k)') # plt.ylabel('Embedding Density (Bit Per Pixel)') density = l / (nb_pixels * np.array(nb_picture)) x = np.arange(len(nb_picture)) + 1 x = x * (num_classes_tbl[i] / nb_steps) fig, ax1 = plt.subplots() color = 'tab:red' ax1.set_xlabel('Nb classes embedded per image (k)') ax1.set_ylabel('Embedding Density (Bits Per Pixel)', color=color) ax1.plot(x, density, color=color, linewidth=2) ax1.tick_params(axis='y', labelcolor=color) ax2 = ax1.twinx( ) # instantiate a second axes that shares the same x-axis color = 'tab:blue' ax2.set_ylabel( 'Number of images to embed the message', color=color) # we already handled the x-label with ax1 ax2.plot(x, nb_picture, color=color, linewidth=2) ax2.tick_params(axis='y', labelcolor=color) fig.tight_layout( ) # otherwise the right y-label is slightly clipped plt.title("Embedding density for N={}".format(num_classes_tbl[i])) #plt.plot(x,density) print("density", density) print("nb_picture", nb_picture) plt.show() for num_classes in num_classes_tbl: nb_pictures.append([]) logger.info("total classes {}".format(num_classes)) for nb_embedded_classes in range(1, nb_steps): num_embedded_classes = int(nb_embedded_classes * (num_classes / nb_steps)) nb_imgs = [] for j in range(nb_trials): msg = "".join( [strs[random.randint(0, len(strs) - 1)] for i in range(l)]) encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) groups = SATA.embed_message( None, None, encoded_msg, epsilon=None, class_density=num_embedded_classes / num_classes, num_classes=num_classes, groups_only=True) # logger.info("{}:{}".format(num_embedded_classes,len(groups))) nb_imgs.append(len(groups)) nb_pictures[-1].append(np.array(nb_imgs).mean()) #densities = l/ (nb_pixels * np.array(nb_pictures)) plot(nb_pictures) return folder = "./experiments/results/experiment{}".format("sata") os.makedirs(folder, exist_ok=True) with open("{}/{}.json".format(folder, "_".join(num_classes_tbl)), 'a') as f: f.write("{}".format(json.dumps(nb_pictures)))
def run(dataset="cifar10", model_type="basic", epochs=25, experiment_id="SP9_6"): attack_name = "targeted_pgd" logger.info("running {} {} {}".format(dataset, model_type, attack_name)) folder = "./experiments/results/experiment{}".format(experiment_id) os.makedirs("{}".format(folder), exist_ok=True) if RANDOM_SEED > 0: random.seed(RANDOM_SEED) quality = 100 extension = "png" l = 100 msg = "".join([strs[random.randint(0, len(strs) - 1)] for i in range(l)]) recovery_rates = [] extension = "h5" models_path = "../products/run31c/cifar/ee50_te300_mr0.1_sr0.2_1565783786" max_models = 150 skip = 0 with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("[") for file in os.listdir(models_path): if max_models == 0: break if (file.startswith("e4") or file.startswith("e5") or file.startswith("e6")) and file.endswith( ".{}".format(extension)): if skip > 0: skip = skip - 1 continue max_models = max_models - 1 model_type = "{}/{}".format(models_path, file) exp_time = 0 atk_strength = 5.0 x, y, adv_x, model, targets = _encode_adv( msg, dataset, "basic", epochs, experiment_id, attack_name, attack_strength=atk_strength) pic_folder = "{}/{}".format(folder, file) os.makedirs(pic_folder, exist_ok=True) files = [] for i, adv in enumerate(adv_x): adv_path = "{}/{}.{}".format(pic_folder, i, extension) #save_img(adv_path,adv) #array_to_img(adv).save(adv_path) files.append(adv_path) rate = {"model": file} rate["stat_score_adv"] = _detect_spa(files, imgs=adv_x) rate["stat_score_real"] = _detect_spa(files, imgs=x) # rate["lsb_score_adv"] = _detect_e4s_srm(files, model_file="e4s_srm_bossbase_lsbm0.40_gs.model",imgs=adv_x) # rate["lsb_score_real"] = _detect_e4s_srm(files, model_file="e4s_srm_bossbase_lsbm0.40_gs.model",imgs=x) # rate["hill_score"] = _detect_e4s_srm(files, model_file="e4s_srm_bossbase_hill0.40_gs.model") # rate["wow_score"] = _detect_e4s_srm(files, model_file="e4s_srm_bossbase_wow0.40_gs.model") # rate["uniw_score"] = _detect_e4s_srm(files, model_file="e4s_srm_bossbase_uniw0.40_gs.model") recovery_rates.append(rate) with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("{},".format(json.dumps(rate))) logger.info(recovery_rates) with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("]") return
def run(dataset="cifar10",model_type="basic", epochs = 25, experiment_id="SP9", start=0): attack_name = "targeted_pgd" logger.info("running {} {} {}".format(dataset,model_type, attack_name)) folder = "./experiments/results/experiment{}".format(experiment_id) os.makedirs("{}".format(folder), exist_ok =True) if RANDOM_SEED>0: random.seed(RANDOM_SEED) quality=100 extension = "png" l = 100 msg = "".join([strs[random.randint(0,len(strs)-1)] for i in range(l)]) recovery_rates = [] extension = "h5" models_path = "../products/run31c/cifar/ee50_te300_mr0.1_sr0.2_1565783786" max_models = 150 skip = start with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("[") for file in os.listdir(models_path): if max_models ==0: break if (file.startswith("e4") or file.startswith("e5") or file.startswith("e6") ) and file.endswith(".{}".format(extension)): if skip>0: skip = skip-1 continue max_models = max_models-1 model_type = "{}/{}".format(models_path,file) exp_time = 0 experiment_id = "SP9/1" advs = _encode(msg, dataset, model_type, epochs, experiment_id,attack_name,attack_strength=5.,extension = extension,transformation="rotate") rotate_recovery = _decode( dataset, model_type, epochs, experiment_id,attack_name,exp_time,extension = extension, advs=advs) experiment_id = "SP9/2" advs = _encode(msg, dataset, model_type, epochs, experiment_id,attack_name,attack_strength=5.,extension = extension,transformation="crop") crop_recovery = _decode( dataset, model_type, epochs, experiment_id,attack_name,exp_time,extension = extension, advs=advs) experiment_id = "SP9/3" advs = _encode(msg, dataset, model_type, epochs, experiment_id,attack_name,attack_strength=5.,extension = extension,transformation="upscale") upscale_recovery = _decode( dataset, model_type, epochs, experiment_id,attack_name,exp_time,extension = extension, advs=advs) experiment_id = "SP9/4" advs = _encode(msg, dataset, model_type, epochs, experiment_id,attack_name,attack_strength=5.,extension = "jpg",transformation="compress") compress_recovery = _decode( dataset, model_type, epochs, experiment_id,attack_name,exp_time,extension = "jpg", advs=advs) rate = {"model":file,"rotate_recovery":rotate_recovery,"crop_recovery":crop_recovery,"upscale_recovery":upscale_recovery,"compress_recovery":compress_recovery} recovery_rates.append(rate) with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("{},".format(json.dumps(rate))) logger.info(recovery_rates) with open("{}/{}.json".format(folder, experiment_time), 'a') as f: f.write("]") return
def _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, keep_one=False, quality=100, attack_strength=2.0, extension=None): if not extension: extension = default_extension encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) num_classes = 10 combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test, verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] targets = np.array( to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) if keep_one: x = np.repeat(np.array([x[0, :, :, :]]), y.shape[0], axis=0) y = model.predict(x) adv_x = craft_attack(model, x, attack_name, y=targets, epsilon=attack_strength) yadv = np.argmax(model.predict(adv_x), axis=1) pictures_path = default_path.format(experiment_id, attack_name, experiment_time) os.makedirs(pictures_path, exist_ok=True) os.makedirs("{}/ref".format(pictures_path), exist_ok=True) SSIM = 1 - np.array( list(ssim_distance(model, x, adv_x=adv_x, distance_only=True))) LPIPS = np.array( list(lpips_distance(model, x, adv_x=adv_x, distance_only=True))) PSNR = np.array([_psnr_loss(x[i], adv_x[i]) for i in range(len(x))]) _compressed = _compress_batch(adv_x) SSIM1 = 1 - np.array( list(ssim_distance(model, x, adv_x=_compressed, distance_only=True))) LPIPS1 = np.array( list(lpips_distance(model, x, adv_x=_compressed, distance_only=True))) PSNR1 = np.array([_psnr_loss(x[i], _compressed[i]) for i in range(len(x))]) mean_LPIPS, var_LPIPS = LPIPS.mean(axis=0), LPIPS.var(axis=0) mean_LPIPS1, var_LPIPS1 = LPIPS1.mean(axis=0), LPIPS1.var(axis=0) logger.info("LPIPS mean:{} var:{}".format(mean_LPIPS, var_LPIPS)) logger.info("LPIPS1 mean:{} var:{}".format(mean_LPIPS1, var_LPIPS1)) mean_PSNR, var_PSNR = PSNR.mean(axis=0), PSNR.var(axis=0) mean_PSNR1, var_PSNR1 = PSNR1.mean(axis=0), PSNR1.var(axis=0) logger.info("PSNR mean:{} var:{}".format(mean_PSNR, var_PSNR)) logger.info("PSNR1 mean:{} var:{}".format(mean_PSNR1, var_PSNR1)) mean_SSIM, var_SSIM = SSIM.mean(axis=0), SSIM.var(axis=0) mean_SSIM1, var_SSIM1 = SSIM1.mean(axis=0), SSIM1.var(axis=0) logger.info("SSIM mean:{} var:{}".format(mean_SSIM, var_SSIM)) logger.info("SSIM1 mean:{} var:{}".format(mean_SSIM1, var_SSIM1)) return experiment_time
def _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, keep_one=False, quality=100, attack_strength=2.0, extension=None): if not extension: extension = default_extension encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) num_classes = 10 combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test, verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] targets = np.array( to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) if keep_one: x = np.repeat(np.array([x[0, :, :, :]]), y.shape[0], axis=0) y = model.predict(x) adv_x = craft_attack(model, x, attack_name, y=targets, epsilon=attack_strength) yadv = np.argmax(model.predict(adv_x), axis=1) pictures_path = default_path.format(experiment_id, attack_name, experiment_time) os.makedirs(pictures_path, exist_ok=True) os.makedirs("{}/ref".format(pictures_path), exist_ok=True) for i, adv in enumerate(adv_x): predicted = yadv[i] encoded = np.argmax(targets[i]) truth = np.argmax(y[i]) adv_path = "{}/{}_predicted{}_encoded{}_truth{}.{}".format( pictures_path, i, predicted, encoded, truth, extension) real_path = "{}/ref/{}.{}".format(pictures_path, i, extension) if extension == "png": q = int(10 - quality / 100) save_img(adv_path, adv, compress_level=q) save_img(real_path, x[i], compress_level=q) elif extension == "jpg": save_img(adv_path, adv, quality=quality) save_img(real_path, x[i], quality=quality) return experiment_time
def _encode(msg, dataset, model_type, epochs, experiment_id, attack_name, attack_strength=5.0, nb_classes=2, experiment_time=""): print(dataset, model_type, epochs) extension = default_extension encoded_msg = _encodeString(msg) logger.info("Encode message {}=>{}".format(msg, encoded_msg)) test_size = len(encoded_msg) model, x_train, x_test, y_train, y_test = load_model(dataset=dataset, model_type=model_type, epochs=epochs) num_classes = DATASET_CLASSES[dataset] combined = list(zip(x_test, y_test)) random.shuffle(combined) x_test[:], y_test[:] = zip(*combined) #keep only correctly predicted inputs batch_size = 64 preds_test = np.argmax(model.predict(x_test, verbose=0), axis=1) inds_correct = np.where(preds_test == y_test.argmax(axis=1))[0] x, y = x_test[inds_correct], y_test[inds_correct] x, y = x[:test_size], y[:test_size] chunk_size = int(math.log(num_classes) / math.log(10)) #groups = _split_msg(encoded_msg, chunk_size) targets = np.array( to_categorical([int(i) for i in encoded_msg], num_classes), "int32") #print(targets) epsilon = 5.0 max_iter = 100 SATA.power = 1.5 nb_elements = 1000 sub_x = x_test[:nb_elements] sub_y = y_test[:nb_elements] begin = time.time() adv_x, ref_x, rate_best = SATA.embed_message(model, sub_x, encoded_msg, epsilon=epsilon, nb_classes_per_img=nb_classes) ref_y = np.argmax(model.predict(ref_x, verbose=0), axis=1) end = time.time() #adv_x = craft_attack(model,x,attack_name,y=targets, epsilon=attack_strength) adv_y = np.argmax(model.predict(adv_x), axis=1) nb_required_imgs = adv_y.shape[0] stats = [ end, begin, msg, nb_required_imgs, nb_classes, epsilon, max_iter, SATA.power, nb_elements ] print("nb images required: {}".format(nb_required_imgs)) pictures_path = default_path.format(experiment_id, attack_name, experiment_time) os.makedirs(pictures_path, exist_ok=True) np.save("{}/ref_x.npy".format(pictures_path), ref_x) np.save("{}/ref_y.npy".format(pictures_path), ref_y) np.save("{}/adv_x.npy".format(pictures_path), adv_x) np.save("{}/adv_y.npy".format(pictures_path), adv_y) np.save("{}/stats.npy".format(pictures_path), np.array(stats))
def generate(self, x_all, order, threshold=0.1, nb_classes=10): self.targeted = True y0 = [] for o in order: y = [0] * nb_classes for i, j in enumerate(o): y[j] = max(0, math.pow(1 - i * threshold, SATA.power)) y0.append(y) #print(y) y = np.array(y0) """ Generate adversarial samples and return them in an array. :param x: An array with the original inputs. :type x: `np.ndarray` :param y: The labels for the data `x`. Only provide this parameter if you'd like to use true labels when crafting adversarial samples. Otherwise, model predictions are used as labels to avoid the "label leaking" effect (explained in this paper: https://arxiv.org/abs/1611.01236). Default is `None`. Labels should be one-hot-encoded. :type y: `np.ndarray` :return: An array holding the adversarial examples. :rtype: `np.ndarray` """ targets = y.copy() logger.info('Nb images to embed the message {}'.format( targets.shape[0])) adv_x_best_index = [False] * len(order) adv_x_best = None rate_best = 0.0 for i in range(SATA.num_cover_init): logger.info('Random inputs pick iteration index {}'.format(i)) random.shuffle(x_all) x = x_all.copy()[0:len(order)] SATA.success_rates.append([]) for i_random_init in range(max(1, self.num_random_init)): adv_x = x #.astype(NUMPY_DTYPE) SATA.success_rates[-1].append([]) for i_max_iter in range(self.max_iter): adv_x = self._compute( adv_x, targets, self.eps, self.eps_step, self.num_random_init > 0 and i_max_iter == 0) if self._project: noise = projection(adv_x - x, self.eps, self.norm) adv_x = x + noise adv_x = np.array([ adv_x[i] if adv_x_best is None or not adv_x_best_index[i] else adv_x_best[i] for i in range(len(adv_x)) ]) rate, adv_x_best_index = self.compute_success( self.classifier, x, targets, adv_x, self.targeted) rate = 100 * rate SATA.success_rates[-1][-1].append(rate) if adv_x_best is None: adv_x_best = adv_x rate_best = rate elif rate > rate_best: rate_best = rate adv_x_best = np.where(adv_x_best_index == True, adv_x, adv_x_best) logger.info('Success rate of SATA attack: %.2f%%', rate_best) SATA.rate_best = rate_best if rate_best == 100: return adv_x_best, x return adv_x_best, x