def main(): # tf.logging.set_verbosity(tf.logging.INFO) # instantiate blackbox and substitute model # instantiate blackbox and substitute model forward_model = load_model() # backward_model1 = create_fmodel_18() backward_model2 = create_fmodel_ALP() backward_model3 = create_fmodel_ALP1000() # print(backward_model1[0]) # instantiate differntiable composite model # (predictions from blackbox, gradients from substitute) model = CompositeModel( forward_model=forward_model, backward_models=[backward_model2, backward_model3], weights = [0.5, 0.5]) predictor = tp.OfflinePredictor(tp.PredictConfig( model=SaliencyModel(), session_init=tp.get_model_loader("resnet_v1_50.ckpt"), input_names=['image'], output_names=['saliency'])) for (file_name, image, label) in read_images(): pos_salience = find_salience(predictor, image) adversarial = run_attack(model, image, label, pos_salience) store_adversarial(file_name, adversarial) attack_complete()
def main(): model = load_model() for (file_name, image, label) in read_images(): print('predicted model for: ', file_name, np.argmax(model.predictions(image))) adversarial = run_attack(model, image, label) store_adversarial(file_name, adversarial)
def main(): loader = TinyImageNetLoader() forward_model = load_model() backward_model1 = create_fmodel_ALP() backward_model2 = create_fmodel_ALP1000() model = CompositeModel(forward_model=forward_model, backward_models=[backward_model1, backward_model2], weights=[0.3, 0.7]) i = 0 total_sum = 0.0 prev_avg = 0.0 for (file_name, image, label) in read_images(): is_boundary = True is_adam = True adversarial = run_attack(loader, forward_model, image, label) # Calculate both Adam and Boundary for first 5 images. if i < 5: adversarial_adam = run_attack2(model, image, label, None) if adversarial is not None: error1 = distance(adversarial, image) else: error1 = 100000.0 is_boundary = False if adversarial_adam is not None: error_adam = distance(adversarial_adam, image) else: error_adam = 200000.0 is_adam = False if is_adam and error1 - error_adam > 0.0: adversarial = adversarial_adam if is_adam or is_boundary: i += 1 total_sum += min(error1, error_adam) else: if adversarial is not None: error1 = distance(adversarial, image) prev_avg = total_sum / i i += 1 total_sum += error1 else: error1 = 100000.0 if error1 > 25.0 or error1 > prev_avg or adversarial is None: adversarial_adam = run_attack2(model, image, label, None) if adversarial_adam is not None: error_adam = distance(adversarial_adam, image) else: error_adam = 200000.0 if error1 - error_adam > 0.0: adversarial = adversarial_adam store_adversarial(file_name, adversarial) # Announce that the attack is complete # NOTE: In the absence of this call, your submission will timeout # while being graded. attack_complete()
def main(): forward_model = load_model() for (file_name, image, label) in read_images(): # tf.logging.info('Checking image is np array: %s' % str(type(image) is np.ndarray)) adversarial = run_attack(forward_model, image, label, None) store_adversarial(file_name, adversarial) # Announce that the attack is complete # NOTE: In the absence of this call, your submission will timeout # while being graded. # print("Attack is complete") attack_complete()
def main(): # tf.logging.set_verbosity(tf.logging.INFO) # instantiate blackbox and substitute model # instantiate blackbox and substitute model forward_model = load_model() backward_model1 = create_fmodel_18() backward_model2 = create_fmodel_ALP() # print(backward_model1[0]) # instantiate differntiable composite model # (predictions from blackbox, gradients from substitute) model = CompositeModel(forward_model=forward_model, backward_models=[backward_model1, backward_model2]) for (file_name, image, label) in read_images(): adversarial = run_attack(model, image, label) store_adversarial(file_name, adversarial) attack_complete()
def main(): loader = TinyImageNetLoader() forward_model = load_model() backward_model1 = create_fmodel_ALP() backward_model2 = create_fmodel_ALP1000() model = CompositeModel(forward_model=forward_model, backward_models=[backward_model1, backward_model2], weights=[0.5, 0.5]) for (file_name, image, label) in read_images(): adversarial = run_attack(loader, forward_model, image, label) if adversarial is None: adversarial = run_attack2(model, image, label, None) store_adversarial(file_name, adversarial) # Announce that the attack is complete # NOTE: In the absence of this call, your submission will timeout # while being graded. attack_complete()
def main(): # tf.logging.set_verbosity(tf.logging.INFO) # instantiate blackbox and substitute model forward_model = load_model() backward_model = create_fmodel() # instantiate differntiable composite model # (predictions from blackbox, gradients from substitute) model = CompositeModel(forward_model=forward_model, backward_model=backward_model) for (file_name, image, label) in read_images(): # tf.logging.info('Checking image is np array: %s' % str(type(image) is np.ndarray)) adversarial = run_attack(model, image, label) store_adversarial(file_name, adversarial) # Announce that the attack is complete # NOTE: In the absence of this call, your submission will timeout # while being graded. # print("Attack is complete") attack_complete()
def main(): # instantiate blackbox and substitute model forward_model = load_model() backward_model = create_fmodel() # instantiate differntiable composite model # (predictions from blackbox, gradients from substitute) model = CompositeModel( forward_model=forward_model, backward_model=backward_model) predictor = tp.OfflinePredictor(tp.PredictConfig( model=SaliencyModel(), session_init=tp.get_model_loader("resnet_v1_50.ckpt"), input_names=['image'], output_names=['saliency'])) for (file_name, image, label) in read_images(): pos_salience = find_salience(predictor, image) adversarial = run_attack(model, image, label, pos_salience) store_adversarial(file_name, adversarial) attack_complete()
def main(): loader = TinyImageNetLoader() forward_model = load_model() backward_model1 = create_fmodel_ALP() backward_model2 = create_fmodel_ALP1000() model = CompositeModel(forward_model=forward_model, backward_models=[backward_model1, backward_model2], weights=[0.5, 0.5]) predictor = tp.OfflinePredictor( tp.PredictConfig(model=SaliencyModel(), session_init=tp.get_model_loader("resnet_v1_50.ckpt"), input_names=['image'], output_names=['saliency'])) for (file_name, image, label) in read_images(): adversarial = run_attack(loader, forward_model, image, label) if adversarial is None: pos_salience = find_salience(predictor, image) adversarial = run_attack2(model, image, label, pos_salience) store_adversarial(file_name, adversarial) # Announce that the attack is complete # NOTE: In the absence of this call, your submission will timeout # while being graded. attack_complete()
def main(): n_classes = 200 img_shape = (64, 64, 3) timed_wrapper = TimedWrapper( load_model()) # Measure model prediction runtime. remote_wrapper = RemoteModelWrapper(timed_wrapper, do_hash=True) # Remember best adv. ex. with SampleGenerator(shape=img_shape, n_threads=1, queue_lengths=100) as sample_gen: X_train, y_train, X_val, y_val = load_dataset( "/path/to/tiny/imagenet", ds_cache_path='tiny_imagenet_cached.npz') with MultiBoundaryAttack(model=remote_wrapper, X_known=np.vstack([X_train, X_val]), y_known=np.concatenate([y_train, y_val]), n_classes=n_classes, sample_gen=sample_gen, cached_conf=None) as attack: model_mean_query_time_history = [] time_max = 89 # As allowed in the rules (batch of 10 in 900 seconds) time_bonus = 0 # Bonus to account for unfair models (see below) i = 0 for (file_name, image, label) in read_images(): time_start = default_timer() # Time calculation: 90 seconds per image are allowed. Models are allowed to use (40ms*1000calls) = 40s. # This leaves 50 seconds for the attacker. # # But if the model is MUCH slower than allowed, then the attacker has less time and can't finish. # To balance the scales, we detect this, and allow ourselves to use up some extra seconds. # If we don't do this (and hard-abort at 90 seconds), attacks that don't count time would have an advantage vs us. if i % 5 == 0 and len(model_mean_query_time_history) > 3: avg_model_time = np.mean(model_mean_query_time_history) if avg_model_time > 55e-3: time_left_for_attacker = 89 - (1000 * avg_model_time) time_bonus = min(55 - time_left_for_attacker, 50) print( "Model is slower than allowed (would leave only {:.1f} seconds for the attacker). " "Will now use up to {:.1f} additional seconds per image." .format(time_left_for_attacker, time_bonus)) elif time_bonus > 0: time_bonus = 0 print( "Model speed seems OK now. Reverting to the 90s time limit." ) print("Image {}:".format(i)) image = np.float32(image) remote_wrapper.adv_set_target(orig_img=image, is_targeted=True, label=label) attack.run_attack(image=image, label=label, is_targeted=True, start_with_fgm=True, fgm_acceptable_dist=10, time_max=time_max + time_bonus) safe_adversarial = remote_wrapper.adv_get_best_img() if safe_adversarial is None: safe_adversarial = np.uint8(image) print("Couldn't find an adversarial! This sucks!") else: dist = util.eval_distance(image, safe_adversarial) print("Final distance: {}".format(dist)) # Save model query time stats. rt_median, rt_mean, rt_std = timed_wrapper.get_runtime_stats() print( "Response time of model: median={:.1f}ms, mean={:.1f}ms, std={:.1f}ms" .format(rt_median * 1e3, rt_mean * 1e3, rt_std * 1e3)) timed_wrapper.reset_runtime_stats() if remote_wrapper.adv_get_n_calls() > 100: model_mean_query_time_history.append(rt_mean) time_elapsed_s = default_timer() - time_start print("Queried the model {} times.".format( remote_wrapper.adv_get_n_calls())) print("Attack for this image took {} seconds.".format( time_elapsed_s)) print() store_adversarial(file_name, safe_adversarial) i += 1 attack_complete()
def main(reader, types, save, backward_cfg=None, forward_cfg=None, verbose=False, addi_name=None, adv_pattern_dir=None, save_saliency=None, saliency_type=None, targeted_labels=None, white_box=False): if forward_cfg is None: print("Using container, addi_name: {}".format(addi_name)) # instantiate blackbox and substitute model test_user = pwd.getpwuid(os.getuid()).pw_name container_name = 'avc_test_model_submission_{}{}'.format( test_user, "_{}".format(addi_name) if addi_name else "") form = '{{ .NetworkSettings.IPAddress }}' cmd = "docker inspect --format='{}' {}".format(form, container_name) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p.wait() ip = p.stdout.read()[:-1].decode('UTF-8') p = subprocess.Popen("docker port " + container_name + " | cut -d/ -f1", shell=True, stdout=subprocess.PIPE) port = p.stdout.read()[:-1].decode('UTF-8') p = subprocess.Popen("docker exec " + container_name + " bash -c 'echo $EVALUATOR_SECRET'", shell=True, stdout=subprocess.PIPE) env_sec = p.stdout.read()[:-1].decode('UTF-8') print("Using container: {}. addr: {}:{}".format( container_name, ip, port)) os.environ["MODEL_SERVER"] = ip os.environ["MODEL_PORT"] = port os.environ["EVALUATOR_SECRET"] = env_sec print('model url: ', "http://{ip}:{port}".format(ip=ip, port=port)) forward_model = load_model() else: forward_model = create_fmodel_cfg(forward_cfg) if not white_box and any([n.endswith("transfer") for n in types]): if backward_cfg is None: backward_cfg = os.environ.get("FMODEL_MODEL_CFG", None) assert backward_cfg, "Neither --backward-cfg nor FMODEL_MODEL_CFG env is provided or set" logging.warning( "backward_cfg is not supplied on cmd line; use the environment variable value {} as backward config file" .format(backward_cfg)) backward_model = create_fmodel_cfg(backward_cfg) # instantiate differntiable composite model # (predictions from blackbox, gradients from substitute) transfer_model = foolbox.models.CompositeModel( forward_model=forward_model, backward_model=backward_model) any_targeted = False if any([n.endswith("targeted") for n in types]): any_targeted = True if targeted_labels: with open(targeted_labels, "r") as f: targeted_labels = yaml.load(f) distances = [[] for _ in types] clean_predicts = [] adv_pattern_predicts = [] accuracy_counter = 0 num_test = 0 not_adv = {tp: 0 for tp in types} dataset_num_classes = reader.num_classes def _save_saliency(s_type, model, image, label): saliency_im = vis.visualize(s_type, model, image, label) fig = plt.figure() ax = plt.subplot(131) ax.imshow(image.astype(np.uint8)) ax = plt.subplot(132) ax.imshow(saliency_im.astype(np.uint8)) ax = plt.subplot(133) cv2.addWeighted(saliency_im, 0.5, image, 0.5, 0, saliency_im) ax.imshow(saliency_im.astype(np.uint8)) plt.savefig( os.path.join(save_saliency, s_type, "{}.png".format(os.path.basename(file_name)))) for ind, (file_name, image, label) in enumerate(reader.read_images()): num_test += 1 predict_label = np.argmax(forward_model.predictions(image)) clean_predict = [file_name.split(".")[0], label, predict_label] print("image {}: {} {}".format(file_name, label, predict_label)) if save_saliency: # save saliency map for s_type in saliency_type: _save_saliency(s_type, forward_model, image, label) if predict_label != label: accuracy_counter += 1 if adv_pattern_dir: adv_im = np.load( os.path.join(adv_pattern_dir, os.path.basename(file_name).split(".")[0]) + ".npy") adv_pattern = adv_im - image before = np.argmax(forward_model.predictions(adv_im)) adv_pattern_predict = [file_name.split(".")[0], label, before] if any_targeted: if targeted_labels: target_label = targeted_labels[file_name] else: target_label = (label + random.randint( 1, reader.num_classes - 1)) % reader.num_classes for it, tp in enumerate(types): with substitute_argscope( foolbox.Adversarial, { "distance": foolbox.distances.Linf if "pgd" in tp else foolbox.distances.MSE }): if "_targeted" in tp: # targeted attack! target = target_label else: target = None if not tp.endswith("transfer") or white_box: if tp.startswith("aug_"): adversarial, suc = bms[tp](forward_model, image, label, verbose=verbose) else: adversarial = bms[tp](forward_model, image, label, verbose=verbose, target=target) suc = adversarial is not None else: adversarial = bms[tp](transfer_model, image, label, verbose=verbose, target=target) suc = adversarial is not None if not suc: pixel_dis = float(worst_case_distance(image)) not_adv[tp] += 1 clean_predict.append(label) # predict to be the original albel else: adversarial = adversarial.astype(np.uint8) pixel_dis = float(distance(image, adversarial)) clean_predict.append( np.argmax(forward_model.predictions(adversarial))) if adv_pattern_dir: assert adversarial is not None # if adversarial is not None: # only test augumentation for now so adversairal must not be None after = np.argmax( forward_model.predictions(adversarial + adv_pattern)) adv_pattern_predict.append(after) distances[it].append(pixel_dis) print("image {}: {} attack / distance: {}{}".format( ind + 1, tp, pixel_dis, "; target: {}".format(target_label) if "_targeted" in tp else "")) sys.stdout.flush() if args.save: if adversarial is None: adversarial = image adversarial = adversarial.astype(np.uint8) if args.use_tofile: # save bin file adversarial.tofile( os.path.join( os.environ["OUTPUT_ADVERSARIAL_PATH"], tp, os.path.basename(file_name).split(".")[0] + ".bin")) else: # save npy np.save( os.path.join( os.environ["OUTPUT_ADVERSARIAL_PATH"], tp, os.path.basename(file_name).split(".")[0]), adversarial) clean_predicts.append(clean_predict) if adv_pattern_dir: adv_pattern_predicts.append(adv_pattern_predict) print("test accuracy: {:.2f}%".format(100. - accuracy_counter * 100. / num_test)) print("not find adv samples: \n\t{}".format("\n\t".join([ "{}: {}; {:.3f}%".format(tp, num, float(num) / num_test * 100) for tp, num in not_adv.items() ]))) if args.save: # open("file_predict_label.txt", "w").write("\n".join(["{} {} {}".format(*x) for x in clean_predicts])) predict_fname = os.path.join( os.environ["OUTPUT_ADVERSARIAL_PATH"], "predicts_{}.csv".format( datetime.datetime.now().strftime("%m-%d_%H-%M-%S"))) print("Save predict to csv: ", predict_fname) pd.DataFrame(clean_predicts, columns=["filename", "label", "pred"] + list(types)).to_csv(predict_fname, index=False) input_link = os.path.join(os.environ["OUTPUT_ADVERSARIAL_PATH"], "pred") if os.path.islink(input_link): os.unlink(input_link) # remove already exists symbol link os.symlink(os.path.abspath(os.environ["INPUT_IMG_PATH"]), input_link) if adv_pattern_predicts: if args.save: adv_pattern_predict_fname = os.path.join( os.environ["OUTPUT_ADVERSARIAL_PATH"], "adv_pattern_predicts_{}.csv".format( datetime.datetime.now().strftime("%m-%d_%H-%M-%S"))) pd.DataFrame(adv_pattern_predicts, columns=["filename", "label", "pred"] + list(types)).to_csv(adv_pattern_predict_fname, index=False) adv_pattern_acc = np.sum(np.array([ [p[1]] for p in adv_pattern_predicts ]) == np.array([p[2:] for p in adv_pattern_predicts]), axis=0) / float(len(adv_pattern_predicts)) print("adv pattern acc: \n\t{}".format("\n\t".join([ "{}: {:.3f}%".format(tp, acc * 100) for tp, acc in zip(["pred"] + list(types), adv_pattern_acc) ]))) for tp, dis in zip(types, distances): distance_array = np.array(dis) print("{}: median pixel distance: {}, mean pixel distance: {}".format( tp, np.median(distance_array), distance_array.mean()))
def main(): model = load_model() np.random.seed(22) test_image = np.random.uniform(size=(64, 64, 3)).astype(np.float32) * 255 prediction = model(test_image) print('predicted class: {}'.format(prediction))