def main():
# tf.logging.set_verbosity(tf.logging.INFO)
# instantiate blackbox and substitute model
# instantiate blackbox and substitute model
forward_model = load_model()
# backward_model1 = create_fmodel_18()
backward_model2 = create_fmodel_ALP()
backward_model3 = create_fmodel_ALP1000()
# print(backward_model1[0])
# instantiate differntiable composite model
# (predictions from blackbox, gradients from substitute)
model = CompositeModel(
forward_model=forward_model,
backward_models=[backward_model2, backward_model3],
weights = [0.5, 0.5])
predictor = tp.OfflinePredictor(tp.PredictConfig(
model=SaliencyModel(),
session_init=tp.get_model_loader("resnet_v1_50.ckpt"),
input_names=['image'],
output_names=['saliency']))
for (file_name, image, label) in read_images():
pos_salience = find_salience(predictor, image)
adversarial = run_attack(model, image, label, pos_salience)
store_adversarial(file_name, adversarial)
attack_complete()
def main():
model = load_model()
for (file_name, image, label) in read_images():
print('predicted model for: ', file_name,
np.argmax(model.predictions(image)))
adversarial = run_attack(model, image, label)
store_adversarial(file_name, adversarial)
def main():
loader = TinyImageNetLoader()
forward_model = load_model()
backward_model1 = create_fmodel_ALP()
backward_model2 = create_fmodel_ALP1000()
model = CompositeModel(forward_model=forward_model,
backward_models=[backward_model1, backward_model2],
weights=[0.3, 0.7])
i = 0
total_sum = 0.0
prev_avg = 0.0
for (file_name, image, label) in read_images():
is_boundary = True
is_adam = True
adversarial = run_attack(loader, forward_model, image, label)
# Calculate both Adam and Boundary for first 5 images.
if i < 5:
adversarial_adam = run_attack2(model, image, label, None)
if adversarial is not None:
error1 = distance(adversarial, image)
else:
error1 = 100000.0
is_boundary = False
if adversarial_adam is not None:
error_adam = distance(adversarial_adam, image)
else:
error_adam = 200000.0
is_adam = False
if is_adam and error1 - error_adam > 0.0:
adversarial = adversarial_adam
if is_adam or is_boundary:
i += 1
total_sum += min(error1, error_adam)
else:
if adversarial is not None:
error1 = distance(adversarial, image)
prev_avg = total_sum / i
i += 1
total_sum += error1
else:
error1 = 100000.0
if error1 > 25.0 or error1 > prev_avg or adversarial is None:
adversarial_adam = run_attack2(model, image, label, None)
if adversarial_adam is not None:
error_adam = distance(adversarial_adam, image)
else:
error_adam = 200000.0
if error1 - error_adam > 0.0:
adversarial = adversarial_adam
store_adversarial(file_name, adversarial)
# Announce that the attack is complete
# NOTE: In the absence of this call, your submission will timeout
# while being graded.
attack_complete()
def main():
forward_model = load_model()
for (file_name, image, label) in read_images():
# tf.logging.info('Checking image is np array: %s' % str(type(image) is np.ndarray))
adversarial = run_attack(forward_model, image, label, None)
store_adversarial(file_name, adversarial)
# Announce that the attack is complete
# NOTE: In the absence of this call, your submission will timeout
# while being graded.
# print("Attack is complete")
attack_complete()
def main():
# tf.logging.set_verbosity(tf.logging.INFO)
# instantiate blackbox and substitute model
# instantiate blackbox and substitute model
forward_model = load_model()
backward_model1 = create_fmodel_18()
backward_model2 = create_fmodel_ALP()
# print(backward_model1[0])
# instantiate differntiable composite model
# (predictions from blackbox, gradients from substitute)
model = CompositeModel(forward_model=forward_model,
backward_models=[backward_model1, backward_model2])
for (file_name, image, label) in read_images():
adversarial = run_attack(model, image, label)
store_adversarial(file_name, adversarial)
attack_complete()
def main():
loader = TinyImageNetLoader()
forward_model = load_model()
backward_model1 = create_fmodel_ALP()
backward_model2 = create_fmodel_ALP1000()
model = CompositeModel(forward_model=forward_model,
backward_models=[backward_model1, backward_model2],
weights=[0.5, 0.5])
for (file_name, image, label) in read_images():
adversarial = run_attack(loader, forward_model, image, label)
if adversarial is None:
adversarial = run_attack2(model, image, label, None)
store_adversarial(file_name, adversarial)
# Announce that the attack is complete
# NOTE: In the absence of this call, your submission will timeout
# while being graded.
attack_complete()
def main():
# tf.logging.set_verbosity(tf.logging.INFO)
# instantiate blackbox and substitute model
forward_model = load_model()
backward_model = create_fmodel()
# instantiate differntiable composite model
# (predictions from blackbox, gradients from substitute)
model = CompositeModel(forward_model=forward_model,
backward_model=backward_model)
for (file_name, image, label) in read_images():
# tf.logging.info('Checking image is np array: %s' % str(type(image) is np.ndarray))
adversarial = run_attack(model, image, label)
store_adversarial(file_name, adversarial)
# Announce that the attack is complete
# NOTE: In the absence of this call, your submission will timeout
# while being graded.
# print("Attack is complete")
attack_complete()
def main():
# instantiate blackbox and substitute model
forward_model = load_model()
backward_model = create_fmodel()
# instantiate differntiable composite model
# (predictions from blackbox, gradients from substitute)
model = CompositeModel(
forward_model=forward_model,
backward_model=backward_model)
predictor = tp.OfflinePredictor(tp.PredictConfig(
model=SaliencyModel(),
session_init=tp.get_model_loader("resnet_v1_50.ckpt"),
input_names=['image'],
output_names=['saliency']))
for (file_name, image, label) in read_images():
pos_salience = find_salience(predictor, image)
adversarial = run_attack(model, image, label, pos_salience)
store_adversarial(file_name, adversarial)
attack_complete()
def main():
loader = TinyImageNetLoader()
forward_model = load_model()
backward_model1 = create_fmodel_ALP()
backward_model2 = create_fmodel_ALP1000()
model = CompositeModel(forward_model=forward_model,
backward_models=[backward_model1, backward_model2],
weights=[0.5, 0.5])
predictor = tp.OfflinePredictor(
tp.PredictConfig(model=SaliencyModel(),
session_init=tp.get_model_loader("resnet_v1_50.ckpt"),
input_names=['image'],
output_names=['saliency']))
for (file_name, image, label) in read_images():
adversarial = run_attack(loader, forward_model, image, label)
if adversarial is None:
pos_salience = find_salience(predictor, image)
adversarial = run_attack2(model, image, label, pos_salience)
store_adversarial(file_name, adversarial)
# Announce that the attack is complete
# NOTE: In the absence of this call, your submission will timeout
# while being graded.
attack_complete()
def main():
n_classes = 200
img_shape = (64, 64, 3)
timed_wrapper = TimedWrapper(
load_model()) # Measure model prediction runtime.
remote_wrapper = RemoteModelWrapper(timed_wrapper,
do_hash=True) # Remember best adv. ex.
with SampleGenerator(shape=img_shape, n_threads=1,
queue_lengths=100) as sample_gen:
X_train, y_train, X_val, y_val = load_dataset(
"/path/to/tiny/imagenet", ds_cache_path='tiny_imagenet_cached.npz')
with MultiBoundaryAttack(model=remote_wrapper,
X_known=np.vstack([X_train, X_val]),
y_known=np.concatenate([y_train, y_val]),
n_classes=n_classes,
sample_gen=sample_gen,
cached_conf=None) as attack:
model_mean_query_time_history = []
time_max = 89 # As allowed in the rules (batch of 10 in 900 seconds)
time_bonus = 0 # Bonus to account for unfair models (see below)
i = 0
for (file_name, image, label) in read_images():
time_start = default_timer()
# Time calculation: 90 seconds per image are allowed. Models are allowed to use (40ms*1000calls) = 40s.
# This leaves 50 seconds for the attacker.
#
# But if the model is MUCH slower than allowed, then the attacker has less time and can't finish.
# To balance the scales, we detect this, and allow ourselves to use up some extra seconds.
# If we don't do this (and hard-abort at 90 seconds), attacks that don't count time would have an advantage vs us.
if i % 5 == 0 and len(model_mean_query_time_history) > 3:
avg_model_time = np.mean(model_mean_query_time_history)
if avg_model_time > 55e-3:
time_left_for_attacker = 89 - (1000 * avg_model_time)
time_bonus = min(55 - time_left_for_attacker, 50)
print(
"Model is slower than allowed (would leave only {:.1f} seconds for the attacker). "
"Will now use up to {:.1f} additional seconds per image."
.format(time_left_for_attacker, time_bonus))
elif time_bonus > 0:
time_bonus = 0
print(
"Model speed seems OK now. Reverting to the 90s time limit."
)
print("Image {}:".format(i))
image = np.float32(image)
remote_wrapper.adv_set_target(orig_img=image,
is_targeted=True,
label=label)
attack.run_attack(image=image,
label=label,
is_targeted=True,
start_with_fgm=True,
fgm_acceptable_dist=10,
time_max=time_max + time_bonus)
safe_adversarial = remote_wrapper.adv_get_best_img()
if safe_adversarial is None:
safe_adversarial = np.uint8(image)
print("Couldn't find an adversarial! This sucks!")
else:
dist = util.eval_distance(image, safe_adversarial)
print("Final distance: {}".format(dist))
# Save model query time stats.
rt_median, rt_mean, rt_std = timed_wrapper.get_runtime_stats()
print(
"Response time of model: median={:.1f}ms, mean={:.1f}ms, std={:.1f}ms"
.format(rt_median * 1e3, rt_mean * 1e3, rt_std * 1e3))
timed_wrapper.reset_runtime_stats()
if remote_wrapper.adv_get_n_calls() > 100:
model_mean_query_time_history.append(rt_mean)
time_elapsed_s = default_timer() - time_start
print("Queried the model {} times.".format(
remote_wrapper.adv_get_n_calls()))
print("Attack for this image took {} seconds.".format(
time_elapsed_s))
print()
store_adversarial(file_name, safe_adversarial)
i += 1
attack_complete()
def main(reader,
types,
save,
backward_cfg=None,
forward_cfg=None,
verbose=False,
addi_name=None,
adv_pattern_dir=None,
save_saliency=None,
saliency_type=None,
targeted_labels=None,
white_box=False):
if forward_cfg is None:
print("Using container, addi_name: {}".format(addi_name))
# instantiate blackbox and substitute model
test_user = pwd.getpwuid(os.getuid()).pw_name
container_name = 'avc_test_model_submission_{}{}'.format(
test_user, "_{}".format(addi_name) if addi_name else "")
form = '{{ .NetworkSettings.IPAddress }}'
cmd = "docker inspect --format='{}' {}".format(form, container_name)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
p.wait()
ip = p.stdout.read()[:-1].decode('UTF-8')
p = subprocess.Popen("docker port " + container_name +
" | cut -d/ -f1",
shell=True,
stdout=subprocess.PIPE)
port = p.stdout.read()[:-1].decode('UTF-8')
p = subprocess.Popen("docker exec " + container_name +
" bash -c 'echo $EVALUATOR_SECRET'",
shell=True,
stdout=subprocess.PIPE)
env_sec = p.stdout.read()[:-1].decode('UTF-8')
print("Using container: {}. addr: {}:{}".format(
container_name, ip, port))
os.environ["MODEL_SERVER"] = ip
os.environ["MODEL_PORT"] = port
os.environ["EVALUATOR_SECRET"] = env_sec
print('model url: ', "http://{ip}:{port}".format(ip=ip, port=port))
forward_model = load_model()
else:
forward_model = create_fmodel_cfg(forward_cfg)
if not white_box and any([n.endswith("transfer") for n in types]):
if backward_cfg is None:
backward_cfg = os.environ.get("FMODEL_MODEL_CFG", None)
assert backward_cfg, "Neither --backward-cfg nor FMODEL_MODEL_CFG env is provided or set"
logging.warning(
"backward_cfg is not supplied on cmd line; use the environment variable value {} as backward config file"
.format(backward_cfg))
backward_model = create_fmodel_cfg(backward_cfg)
# instantiate differntiable composite model
# (predictions from blackbox, gradients from substitute)
transfer_model = foolbox.models.CompositeModel(
forward_model=forward_model, backward_model=backward_model)
any_targeted = False
if any([n.endswith("targeted") for n in types]):
any_targeted = True
if targeted_labels:
with open(targeted_labels, "r") as f:
targeted_labels = yaml.load(f)
distances = [[] for _ in types]
clean_predicts = []
adv_pattern_predicts = []
accuracy_counter = 0
num_test = 0
not_adv = {tp: 0 for tp in types}
dataset_num_classes = reader.num_classes
def _save_saliency(s_type, model, image, label):
saliency_im = vis.visualize(s_type, model, image, label)
fig = plt.figure()
ax = plt.subplot(131)
ax.imshow(image.astype(np.uint8))
ax = plt.subplot(132)
ax.imshow(saliency_im.astype(np.uint8))
ax = plt.subplot(133)
cv2.addWeighted(saliency_im, 0.5, image, 0.5, 0, saliency_im)
ax.imshow(saliency_im.astype(np.uint8))
plt.savefig(
os.path.join(save_saliency, s_type,
"{}.png".format(os.path.basename(file_name))))
for ind, (file_name, image, label) in enumerate(reader.read_images()):
num_test += 1
predict_label = np.argmax(forward_model.predictions(image))
clean_predict = [file_name.split(".")[0], label, predict_label]
print("image {}: {} {}".format(file_name, label, predict_label))
if save_saliency: # save saliency map
for s_type in saliency_type:
_save_saliency(s_type, forward_model, image, label)
if predict_label != label:
accuracy_counter += 1
if adv_pattern_dir:
adv_im = np.load(
os.path.join(adv_pattern_dir,
os.path.basename(file_name).split(".")[0]) +
".npy")
adv_pattern = adv_im - image
before = np.argmax(forward_model.predictions(adv_im))
adv_pattern_predict = [file_name.split(".")[0], label, before]
if any_targeted:
if targeted_labels:
target_label = targeted_labels[file_name]
else:
target_label = (label + random.randint(
1, reader.num_classes - 1)) % reader.num_classes
for it, tp in enumerate(types):
with substitute_argscope(
foolbox.Adversarial, {
"distance":
foolbox.distances.Linf
if "pgd" in tp else foolbox.distances.MSE
}):
if "_targeted" in tp: # targeted attack!
target = target_label
else:
target = None
if not tp.endswith("transfer") or white_box:
if tp.startswith("aug_"):
adversarial, suc = bms[tp](forward_model,
image,
label,
verbose=verbose)
else:
adversarial = bms[tp](forward_model,
image,
label,
verbose=verbose,
target=target)
suc = adversarial is not None
else:
adversarial = bms[tp](transfer_model,
image,
label,
verbose=verbose,
target=target)
suc = adversarial is not None
if not suc:
pixel_dis = float(worst_case_distance(image))
not_adv[tp] += 1
clean_predict.append(label) # predict to be the original albel
else:
adversarial = adversarial.astype(np.uint8)
pixel_dis = float(distance(image, adversarial))
clean_predict.append(
np.argmax(forward_model.predictions(adversarial)))
if adv_pattern_dir:
assert adversarial is not None
# if adversarial is not None: # only test augumentation for now so adversairal must not be None
after = np.argmax(
forward_model.predictions(adversarial + adv_pattern))
adv_pattern_predict.append(after)
distances[it].append(pixel_dis)
print("image {}: {} attack / distance: {}{}".format(
ind + 1, tp, pixel_dis, "; target: {}".format(target_label)
if "_targeted" in tp else ""))
sys.stdout.flush()
if args.save:
if adversarial is None:
adversarial = image
adversarial = adversarial.astype(np.uint8)
if args.use_tofile: # save bin file
adversarial.tofile(
os.path.join(
os.environ["OUTPUT_ADVERSARIAL_PATH"], tp,
os.path.basename(file_name).split(".")[0] +
".bin"))
else: # save npy
np.save(
os.path.join(
os.environ["OUTPUT_ADVERSARIAL_PATH"], tp,
os.path.basename(file_name).split(".")[0]),
adversarial)
clean_predicts.append(clean_predict)
if adv_pattern_dir:
adv_pattern_predicts.append(adv_pattern_predict)
print("test accuracy: {:.2f}%".format(100. -
accuracy_counter * 100. / num_test))
print("not find adv samples: \n\t{}".format("\n\t".join([
"{}: {}; {:.3f}%".format(tp, num,
float(num) / num_test * 100)
for tp, num in not_adv.items()
])))
if args.save:
# open("file_predict_label.txt", "w").write("\n".join(["{} {} {}".format(*x) for x in clean_predicts]))
predict_fname = os.path.join(
os.environ["OUTPUT_ADVERSARIAL_PATH"], "predicts_{}.csv".format(
datetime.datetime.now().strftime("%m-%d_%H-%M-%S")))
print("Save predict to csv: ", predict_fname)
pd.DataFrame(clean_predicts,
columns=["filename", "label", "pred"] +
list(types)).to_csv(predict_fname, index=False)
input_link = os.path.join(os.environ["OUTPUT_ADVERSARIAL_PATH"],
"pred")
if os.path.islink(input_link):
os.unlink(input_link) # remove already exists symbol link
os.symlink(os.path.abspath(os.environ["INPUT_IMG_PATH"]), input_link)
if adv_pattern_predicts:
if args.save:
adv_pattern_predict_fname = os.path.join(
os.environ["OUTPUT_ADVERSARIAL_PATH"],
"adv_pattern_predicts_{}.csv".format(
datetime.datetime.now().strftime("%m-%d_%H-%M-%S")))
pd.DataFrame(adv_pattern_predicts,
columns=["filename", "label", "pred"] +
list(types)).to_csv(adv_pattern_predict_fname,
index=False)
adv_pattern_acc = np.sum(np.array([
[p[1]] for p in adv_pattern_predicts
]) == np.array([p[2:] for p in adv_pattern_predicts]),
axis=0) / float(len(adv_pattern_predicts))
print("adv pattern acc: \n\t{}".format("\n\t".join([
"{}: {:.3f}%".format(tp, acc * 100)
for tp, acc in zip(["pred"] + list(types), adv_pattern_acc)
])))
for tp, dis in zip(types, distances):
distance_array = np.array(dis)
print("{}: median pixel distance: {}, mean pixel distance: {}".format(
tp, np.median(distance_array), distance_array.mean()))
def main():
model = load_model()
np.random.seed(22)
test_image = np.random.uniform(size=(64, 64, 3)).astype(np.float32) * 255
prediction = model(test_image)
print('predicted class: {}'.format(prediction))