def load_model(model_config):
"""
Loads a model and preprocessing function from configuration file
preprocessing_fn can be a tuple of functions or None values
If so, it applies to training and inference separately
"""
model_module = import_module(model_config["module"])
model_fn = getattr(model_module, model_config["name"])
weights_file = model_config.get("weights_file", None)
if isinstance(weights_file, str):
weights_path = maybe_download_weights_from_s3(
weights_file, auto_expand_tars=True
)
elif isinstance(weights_file, list):
weights_path = [
maybe_download_weights_from_s3(w, auto_expand_tars=True)
for w in weights_file
]
elif isinstance(weights_file, dict):
weights_path = {
k: maybe_download_weights_from_s3(v) for k, v in weights_file.items()
}
else:
weights_path = None
model = model_fn(
model_config["model_kwargs"], model_config["wrapper_kwargs"], weights_path
)
if not isinstance(model, Classifier):
raise TypeError(f"{model} is not an instance of {Classifier}")
if not weights_file and not model_config["fit"]:
logger.warning(
"No weights file was provided and the model is not configured to train. "
"Are you loading model weights from an online repository?"
)
preprocessing_fn = getattr(model_module, "preprocessing_fn", None)
if preprocessing_fn is not None:
if isinstance(preprocessing_fn, tuple):
if len(preprocessing_fn) != 2:
raise ValueError(
f"preprocessing tuple length {len(preprocessing_fn)} != 2"
)
elif not all([x is None or callable(x) for x in preprocessing_fn]):
raise TypeError(
f"preprocessing_fn tuple elements {preprocessing_fn} must be None or callable"
)
elif not callable(preprocessing_fn):
raise TypeError(
f"preprocessing_fn {preprocessing_fn} must be None, tuple, or callable"
)
return model, preprocessing_fn
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = ResNet50(weights=None, **model_kwargs)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
model.load_weights(filepath)
wrapped_model = KerasClassifier(
model,
clip_values=(
np.array(
[
0.0 - IMAGENET_MEANS[0],
0.0 - IMAGENET_MEANS[1],
0.0 - IMAGENET_MEANS[2],
]
),
np.array(
[
255.0 - IMAGENET_MEANS[0],
255.0 - IMAGENET_MEANS[1],
255.0 - IMAGENET_MEANS[2],
]
),
),
**wrapper_kwargs,
)
return wrapped_model
def test_keras_imagenet_transfer():
classifier_module = import_module(
"armory.baseline_models.keras.inception_resnet_v2")
classifier_fn = getattr(classifier_module, "get_art_model")
weights_path = maybe_download_weights_from_s3(
"inceptionresnetv2_imagenet_v1.h5")
classifier = classifier_fn(model_kwargs={},
wrapper_kwargs={},
weights_path=weights_path)
dataset = adversarial_datasets.imagenet_adversarial(
split="adversarial",
epochs=1,
batch_size=100,
dataset_dir=DATASET_DIR,
)
accuracy_clean = 0
accuracy_adv = 0
for _ in range(dataset.batches_per_epoch):
(x_clean, x_adv), y = dataset.get_batch()
predictions_clean = classifier.predict(x_clean)
accuracy_clean += np.sum(
np.argmax(predictions_clean, axis=1) == y) / len(y)
predictions_adv = classifier.predict(x_adv)
accuracy_adv += np.sum(
np.argmax(predictions_adv, axis=1) == y) / len(y)
assert (accuracy_clean / dataset.batches_per_epoch) > 0.74
assert (accuracy_adv / dataset.batches_per_epoch) < 0.73
def __init__(self, model_dir, params, device=torch.device('cuda')):
# Lazy load model.
if not model_dir in models:
if os.path.exists(f'{model_dir}/weights.pt'):
checkpoint = torch.load(f'{model_dir}/weights.pt')
else:
weights_path = maybe_download_weights_from_s3(model_dir)
checkpoint = torch.load(weights_path)
# checkpoint = torch.load(model_dir)
model = WaveGrad(AttrDict(base_params)).to(device)
model.load_state_dict(checkpoint['model'])
model.eval()
models[model_dir] = model
model = models[model_dir]
model.params.override(params)
beta = np.array(model.params.noise_schedule)
alpha = 1 - beta
alpha_cum = np.cumprod(alpha)
self.alpha = alpha
self.alpha_cum = alpha_cum
self.beta = beta
self.model = model
self.device = device
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = Model(**model_kwargs)
input_ph = model.x_input
labels_ph = model.y_input
training_ph = tf.placeholder(tf.bool, shape=())
tf_sess = tf.Session()
tf_sess.run(tf.global_variables_initializer())
if weights_file:
saver = tf.train.Saver()
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=SAVED_MODEL_DIR)
tar.close()
model_file = tf.train.latest_checkpoint(SAVED_MODEL_DIR +
'/free_checkpoint_tf')
saver.restore(tf_sess, model_file)
wrapped_model = TFClassifier(input_ph=input_ph,
output=model.pre_softmax,
labels_ph=labels_ph,
loss=model.xent,
learning=training_ph,
sess=tf_sess,
clip_values=(0, 255),
**wrapper_kwargs)
return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = models.resnet50(**model_kwargs)
model.to(DEVICE)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
checkpoint = torch.load(filepath, map_location=DEVICE)
model.load_state_dict(checkpoint)
wrapped_model = PyTorchClassifier(
model,
loss=torch.nn.CrossEntropyLoss(),
optimizer=torch.optim.Adam(model.parameters(), lr=0.003),
input_shape=(224, 224, 3),
**wrapper_kwargs,
clip_values=(
np.array([
0.0 - IMAGENET_MEANS[0] / IMAGENET_STDEV[0],
0.0 - IMAGENET_MEANS[1] / IMAGENET_STDEV[1],
0.0 - IMAGENET_MEANS[2] / IMAGENET_STDEV[2],
]),
np.array([
1.0 - IMAGENET_MEANS[0] / IMAGENET_STDEV[0],
1.0 - IMAGENET_MEANS[1] / IMAGENET_STDEV[1],
1.0 - IMAGENET_MEANS[2] / IMAGENET_STDEV[2],
]),
),
)
return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = make_cifar_model(**model_kwargs)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
model.load_weights(filepath)
wrapped_model = KerasClassifier(model, clip_values=(0.0, 1.0), **wrapper_kwargs)
return wrapped_model
def make_model(model_status="ucf101_trained", weights_file=None):
statuses = ("ucf101_trained", "kinetics_pretrained")
if model_status not in statuses:
raise ValueError(f"model_status {model_status} not in {statuses}")
trained = model_status == "ucf101_trained"
if not trained and weights_file is None:
raise ValueError(
"weights_file cannot be None for 'kinetics_pretrained'")
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
opt = parse_opts(arguments=[])
opt.dataset = "UCF101"
opt.only_RGB = True
opt.log = 0
opt.batch_size = 1
opt.arch = f"{opt.model}-{opt.model_depth}"
if trained:
opt.n_classes = 101
else:
opt.n_classes = 400
opt.n_finetune_classes = 101
opt.batch_size = 32
opt.ft_begin_index = 4
opt.pretrain_path = filepath
logger.info(f"Loading model... {opt.model} {opt.model_depth}")
model, parameters = generate_model(opt)
if trained and weights_file is not None:
checkpoint = torch.load(filepath, map_location=DEVICE)
model.load_state_dict(checkpoint["state_dict"])
# Initializing the optimizer
if opt.pretrain_path:
opt.weight_decay = 1e-5
opt.learning_rate = 0.001
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
optimizer = optim.SGD(
parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov,
)
return model, optimizer
def get_art_model(model_kwargs, wrapper_kwargs, weights_file):
model = make_densenet121_resisc_model(**model_kwargs)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
model.load_weights(filepath)
mean, std = mean_std()
wrapped_model = KerasClassifier(
model, clip_values=((0.0 - mean) / std, (1.0 - mean) / std), **wrapper_kwargs
)
return wrapped_model
def sail(weights_file=None):
pretrained = weights_file is not None
filepath = None
if pretrained:
filepath = maybe_download_weights_from_s3(weights_file)
sailNet = dnn_models.get_model(weights_file=filepath)
if pretrained:
sailNet.eval()
else:
sailNet.train()
return sailNet
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
labels_ph = tf.placeholder(tf.int32, shape=[None, 10])
training_ph = tf.placeholder(tf.bool, shape=())
x = tf.layers.conv2d(input_ph,
filters=4,
kernel_size=(5, 5),
activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.conv2d(x,
filters=10,
kernel_size=(5, 5),
activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.flatten(x)
x = tf.layers.dense(x, 100, activation=tf.nn.relu)
logits = tf.layers.dense(x, 10)
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=labels_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if weights_file:
# Load Model using preferred save/restore method
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=paths.runtime_paths().saved_model_dir)
tar.close()
# Restore variables...
wrapped_model = TFClassifier(clip_values=(0.0, 1.0),
input_ph=input_ph,
output=logits,
labels_ph=labels_ph,
train=train_op,
loss=loss,
learning=training_ph,
sess=sess,
**wrapper_kwargs)
return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file):
if weights_file:
# Download tarball of all model weights
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=SAVED_MODEL_DIR)
tar.close()
model = make_ensemble_model(**model_kwargs)
else:
raise NotImplementedError("This demo is for a pretrained ensemble")
mean, std = mean_std()
wrapped_model = KerasClassifier(model,
clip_values=((0.0 - mean) / std,
(1.0 - mean) / std),
**wrapper_kwargs)
return wrapped_model
def test_keras_mnist_pretrained():
classifier_module = import_module("armory.baseline_models.keras.mnist")
classifier_fn = getattr(classifier_module, "get_art_model")
weights_path = maybe_download_weights_from_s3("undefended_mnist_5epochs.h5")
classifier = classifier_fn(
model_kwargs={}, wrapper_kwargs={}, weights_path=weights_path
)
test_dataset = datasets.mnist(
split="test", epochs=1, batch_size=100, dataset_dir=DATASET_DIR,
)
accuracy = 0
for _ in range(test_dataset.batches_per_epoch):
x, y = test_dataset.get_batch()
predictions = classifier.predict(x)
accuracy += np.sum(np.argmax(predictions, axis=1) == y) / len(y)
assert (accuracy / test_dataset.batches_per_epoch) > 0.98
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = make_audio_model(**model_kwargs)
model.to(DEVICE)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
dic = torch.load(filepath)
model.load_state_dict(dic)
logger.info("Model weights loaded successfully")
wrapped_model = SmoothedPytorchClassifier(
model,
loss=torch.nn.CrossEntropyLoss(),
optimizer=torch.optim.Adam(model.parameters(), lr=0.0001),
input_shape=(WINDOW_LENGTH, ),
nb_classes=40,
)
return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = make_cifar_model(**model_kwargs)
model.to(DEVICE)
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
checkpoint = torch.load(filepath, map_location=DEVICE)
model.load_state_dict(checkpoint)
wrapped_model = PyTorchClassifier(
model,
loss=nn.CrossEntropyLoss(),
optimizer=torch.optim.Adam(model.parameters(), lr=0.003),
input_shape=(3, 32, 32),
nb_classes=10,
clip_values=(0.0, 1.0),
**wrapper_kwargs,
)
return wrapped_model
def test_pytorch_xview_pretrained():
detector_module = import_module(
"armory.baseline_models.pytorch.xview_frcnn")
detector_fn = getattr(detector_module, "get_art_model")
weights_path = maybe_download_weights_from_s3(
"xview_model_state_dict_epoch_99_loss_0p67")
detector = detector_fn(
model_kwargs={},
wrapper_kwargs={},
weights_path=weights_path,
)
NUM_TEST_SAMPLES = 250
dataset_config = {
"batch_size": 1,
"framework": "numpy",
"module": "armory.data.datasets",
"name": "xview",
}
test_dataset = load_dataset(
dataset_config,
epochs=1,
split="test",
num_batches=NUM_TEST_SAMPLES,
shuffle_files=False,
)
list_of_ys = []
list_of_ypreds = []
for x, y in test_dataset:
y_pred = detector.predict(x)
list_of_ys.extend(y)
list_of_ypreds.extend(y_pred)
average_precision_by_class = object_detection_AP_per_class(
list_of_ys, list_of_ypreds)
mAP = np.fromiter(average_precision_by_class.values(), dtype=float).mean()
for class_id in [4, 23, 33, 39]:
assert average_precision_by_class[class_id] > 0.9
assert mAP > 0.25
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
input_ph = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
labels_ph = tf.placeholder(tf.int32, shape=[None, 10])
training_ph = tf.placeholder(tf.bool, shape=())
# Conditional for handling training phase or inference phase
output = tf.cond(
training_ph,
true_fn=lambda: _training_pass(input_ph),
false_fn=lambda: _inference_pass(input_ph),
)
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=output,
onehot_labels=labels_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.003)
train_op = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if weights_file:
# Load Model using preferred save/restore method
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=paths.runtime_paths().saved_model_dir)
tar.close()
# Restore variables...
wrapped_model = TFClassifier(clip_values=(0.0, 1.0),
input_ph=input_ph,
output=output,
labels_ph=labels_ph,
train=train_op,
loss=loss,
learning=training_ph,
sess=sess,
**wrapper_kwargs)
return wrapped_model
def make_model(model_status="ucf101_trained", weights_file=None):
statuses = ("ucf101_trained", "kinetics_pretrained")
if model_status not in statuses:
raise ValueError(f"model_status {model_status} not in {statuses}")
trained = model_status == "ucf101_trained"
if not trained and weights_file is None:
raise ValueError(
"weights_file cannot be None for 'kinetics_pretrained'")
if weights_file:
filepath = maybe_download_weights_from_s3(weights_file)
opt = parse_opts(arguments=[])
opt.dataset = "UCF101"
opt.only_RGB = True
opt.log = 0
opt.batch_size = 1
opt.arch = f"{opt.model}-{opt.model_depth}"
if trained:
opt.n_classes = 101
else:
opt.n_classes = 400
opt.n_finetune_classes = 101
opt.batch_size = 32
opt.ft_begin_index = 4
opt.pretrain_path = filepath
logger.info(f"Loading model... {opt.model} {opt.model_depth}")
model, parameters = generate_model(opt)
if trained and weights_file is not None:
checkpoint = torch.load(filepath, map_location=DEVICE)
# Fit the robust model into the original resnext model
state_dict_path = 'model'
if not ('model' in checkpoint):
state_dict_path = 'state_dict'
sd = checkpoint[state_dict_path]
sd = {k[len('module.'):]: v for k, v in sd.items()}
items = list(sd.items())
for key, val in items:
if key.startswith('attacker.'):
sd.pop(key)
if key.startswith('model.'):
new_key = 'module.' + key[len('model.'):]
sd[new_key] = val
sd.pop(key)
if key == 'normalizer.new_mean' or key == 'normalizer.new_std':
sd.pop(key)
model.load_state_dict(sd)
# Initializing the optimizer
if opt.pretrain_path:
opt.weight_decay = 1e-5
opt.learning_rate = 0.001
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
optimizer = optim.SGD(
parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov,
)
return model, optimizer
def test_invalid_weights():
weights_file = "does_not_exist.h5"
with pytest.raises(ValueError,
match="attempting to load a custom set of weights"):
maybe_download_weights_from_s3(weights_file)
def __init__(
self,
clip_values,
step=200,
means=None,
stds=None,
gan_ckpt='styleGAN.pt',
encoder_ckpt='encoder.pt',
optimize_noise=True,
use_noise_regularize=False,
use_lpips=False,
apply_fit=False,
apply_predict=True,
mse=500,
lr_rampup=0.05,
lr_rampdown=0.05,
noise=0.05,
noise_ramp=0.75,
noise_regularize=1e5,
lr=0.1
):
super(InvGAN, self).__init__()
#print("invgan")
#pdb.set_trace()
self._apply_fit = apply_fit
self._apply_predict = apply_predict
# setup normalization parameters
if means is None:
means = (0.0, 0.0, 0.0) # identity operation
if len(means) != 3:
raise ValueError("means must have 3 values, one per channel")
self.means = means
if stds is None:
stds = (1.0, 1.0, 1.0) # identity operation
if len(stds) != 3:
raise ValueError("stds must have 3 values, one per channel")
self.stds = stds
self.clip_values = clip_values
# setup optimization parameters
self.optimize_noise = optimize_noise
self.use_noise_regularize = use_noise_regularize
self.use_lpips = use_lpips
self.step = step
self.mse = mse
self.lr = lr
self.lr_rampup = lr_rampup
self.lr_rampdown = lr_rampdown
self.noise = noise
self.noise_ramp = noise_ramp
self.noise_regularize = noise_regularize
# setup generator
self.generator = Generator(256, 512, 8)
#self.generator.load_state_dict(torch.load(gan_ckpt)['g_ema'])
self.generator.load_state_dict(torch.load(maybe_download_weights_from_s3(gan_ckpt))['g_ema'])
self.generator.eval()
self.generator.cuda()
self.deprocess_layer = NormalizeByChannelMeanStd([-1., -1., -1.], [2., 2., 2.]).cuda()
# setup encoder
self.encoder = Encoder()
#self.encoder.load_state_dict(torch.load(encoder_ckpt)['netE'])
self.encoder.load_state_dict(torch.load(maybe_download_weights_from_s3(encoder_ckpt))['netE'])
self.encoder.eval()
self.encoder.cuda()
# setup loss
if use_lpips:
self.lpips = PerceptualLoss().cuda()
# estimate latent code statistics
n_mean_latent = 10000
with torch.no_grad():
noise_sample = torch.randn(n_mean_latent, 512, device='cuda')
latent_out = self.generator.style(noise_sample)
latent_mean = latent_out.mean(0)
self.latent_std = ((latent_out - latent_mean).pow(2).sum() / n_mean_latent) ** 0.5
def sincnet(weights_file=None):
pretrained = weights_file is not None
if pretrained:
filepath = maybe_download_weights_from_s3(weights_file)
model_params = torch.load(filepath, map_location=DEVICE)
else:
model_params = {}
CNN_params = model_params.get("CNN_model_par")
DNN1_params = model_params.get("DNN1_model_par")
DNN2_params = model_params.get("DNN2_model_par")
# from SincNet/cfg/SincNet_dev_LibriSpeech.cfg
cnn_N_filt = [80, 60, 60]
cnn_len_filt = [251, 5, 5]
cnn_max_pool_len = [3, 3, 3]
cnn_use_laynorm_inp = True
cnn_use_batchnorm_inp = False
cnn_use_laynorm = [True, True, True]
cnn_use_batchnorm = [False, False, False]
cnn_act = ["relu", "relu", "relu"]
cnn_drop = [0.0, 0.0, 0.0]
fc_lay = [2048, 2048, 2048]
fc_drop = [0.0, 0.0, 0.0]
fc_use_laynorm_inp = True
fc_use_batchnorm_inp = False
fc_use_batchnorm = [True, True, True]
fc_use_laynorm = [False, False, False]
fc_act = ["leaky_relu", "linear", "leaky_relu"]
class_lay = [40]
class_drop = [0.0, 0.0]
class_use_laynorm_inp = True
class_use_batchnorm_inp = False
class_use_batchnorm = [False]
class_use_laynorm = [False]
class_act = ["softmax"]
CNN_options = {
"input_dim": WINDOW_LENGTH,
"fs": SAMPLE_RATE,
"cnn_N_filt": cnn_N_filt,
"cnn_len_filt": cnn_len_filt,
"cnn_max_pool_len": cnn_max_pool_len,
"cnn_use_laynorm_inp": cnn_use_laynorm_inp,
"cnn_use_batchnorm_inp": cnn_use_batchnorm_inp,
"cnn_use_laynorm": cnn_use_laynorm,
"cnn_use_batchnorm": cnn_use_batchnorm,
"cnn_act": cnn_act,
"cnn_drop": cnn_drop,
"pretrained": pretrained,
"model_params": CNN_params,
}
DNN1_options = {
"fc_lay": fc_lay,
"fc_drop": fc_drop,
"fc_use_batchnorm": fc_use_batchnorm,
"fc_use_laynorm": fc_use_laynorm,
"fc_use_laynorm_inp": fc_use_laynorm_inp,
"fc_use_batchnorm_inp": fc_use_batchnorm_inp,
"fc_act": fc_act,
"pretrained": pretrained,
"model_params": DNN1_params,
}
DNN2_options = {
"input_dim": fc_lay[-1],
"fc_lay": class_lay,
"fc_drop": class_drop,
"fc_use_batchnorm": class_use_batchnorm,
"fc_use_laynorm": class_use_laynorm,
"fc_use_laynorm_inp": class_use_laynorm_inp,
"fc_use_batchnorm_inp": class_use_batchnorm_inp,
"fc_act": class_act,
}
sincNet = dnn_models.SincWrapper(DNN2_options, DNN1_options, CNN_options)
if pretrained:
sincNet.eval()
sincNet.load_state_dict(DNN2_params)
else:
sincNet.train()
return sincNet
