def load_model(model_config): """ Loads a model and preprocessing function from configuration file preprocessing_fn can be a tuple of functions or None values If so, it applies to training and inference separately """ model_module = import_module(model_config["module"]) model_fn = getattr(model_module, model_config["name"]) weights_file = model_config.get("weights_file", None) if isinstance(weights_file, str): weights_path = maybe_download_weights_from_s3( weights_file, auto_expand_tars=True ) elif isinstance(weights_file, list): weights_path = [ maybe_download_weights_from_s3(w, auto_expand_tars=True) for w in weights_file ] elif isinstance(weights_file, dict): weights_path = { k: maybe_download_weights_from_s3(v) for k, v in weights_file.items() } else: weights_path = None model = model_fn( model_config["model_kwargs"], model_config["wrapper_kwargs"], weights_path ) if not isinstance(model, Classifier): raise TypeError(f"{model} is not an instance of {Classifier}") if not weights_file and not model_config["fit"]: logger.warning( "No weights file was provided and the model is not configured to train. " "Are you loading model weights from an online repository?" ) preprocessing_fn = getattr(model_module, "preprocessing_fn", None) if preprocessing_fn is not None: if isinstance(preprocessing_fn, tuple): if len(preprocessing_fn) != 2: raise ValueError( f"preprocessing tuple length {len(preprocessing_fn)} != 2" ) elif not all([x is None or callable(x) for x in preprocessing_fn]): raise TypeError( f"preprocessing_fn tuple elements {preprocessing_fn} must be None or callable" ) elif not callable(preprocessing_fn): raise TypeError( f"preprocessing_fn {preprocessing_fn} must be None, tuple, or callable" ) return model, preprocessing_fn
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = ResNet50(weights=None, **model_kwargs) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) model.load_weights(filepath) wrapped_model = KerasClassifier( model, clip_values=( np.array( [ 0.0 - IMAGENET_MEANS[0], 0.0 - IMAGENET_MEANS[1], 0.0 - IMAGENET_MEANS[2], ] ), np.array( [ 255.0 - IMAGENET_MEANS[0], 255.0 - IMAGENET_MEANS[1], 255.0 - IMAGENET_MEANS[2], ] ), ), **wrapper_kwargs, ) return wrapped_model
def test_keras_imagenet_transfer(): classifier_module = import_module( "armory.baseline_models.keras.inception_resnet_v2") classifier_fn = getattr(classifier_module, "get_art_model") weights_path = maybe_download_weights_from_s3( "inceptionresnetv2_imagenet_v1.h5") classifier = classifier_fn(model_kwargs={}, wrapper_kwargs={}, weights_path=weights_path) dataset = adversarial_datasets.imagenet_adversarial( split="adversarial", epochs=1, batch_size=100, dataset_dir=DATASET_DIR, ) accuracy_clean = 0 accuracy_adv = 0 for _ in range(dataset.batches_per_epoch): (x_clean, x_adv), y = dataset.get_batch() predictions_clean = classifier.predict(x_clean) accuracy_clean += np.sum( np.argmax(predictions_clean, axis=1) == y) / len(y) predictions_adv = classifier.predict(x_adv) accuracy_adv += np.sum( np.argmax(predictions_adv, axis=1) == y) / len(y) assert (accuracy_clean / dataset.batches_per_epoch) > 0.74 assert (accuracy_adv / dataset.batches_per_epoch) < 0.73
def __init__(self, model_dir, params, device=torch.device('cuda')): # Lazy load model. if not model_dir in models: if os.path.exists(f'{model_dir}/weights.pt'): checkpoint = torch.load(f'{model_dir}/weights.pt') else: weights_path = maybe_download_weights_from_s3(model_dir) checkpoint = torch.load(weights_path) # checkpoint = torch.load(model_dir) model = WaveGrad(AttrDict(base_params)).to(device) model.load_state_dict(checkpoint['model']) model.eval() models[model_dir] = model model = models[model_dir] model.params.override(params) beta = np.array(model.params.noise_schedule) alpha = 1 - beta alpha_cum = np.cumprod(alpha) self.alpha = alpha self.alpha_cum = alpha_cum self.beta = beta self.model = model self.device = device
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = Model(**model_kwargs) input_ph = model.x_input labels_ph = model.y_input training_ph = tf.placeholder(tf.bool, shape=()) tf_sess = tf.Session() tf_sess.run(tf.global_variables_initializer()) if weights_file: saver = tf.train.Saver() filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=SAVED_MODEL_DIR) tar.close() model_file = tf.train.latest_checkpoint(SAVED_MODEL_DIR + '/free_checkpoint_tf') saver.restore(tf_sess, model_file) wrapped_model = TFClassifier(input_ph=input_ph, output=model.pre_softmax, labels_ph=labels_ph, loss=model.xent, learning=training_ph, sess=tf_sess, clip_values=(0, 255), **wrapper_kwargs) return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = models.resnet50(**model_kwargs) model.to(DEVICE) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) checkpoint = torch.load(filepath, map_location=DEVICE) model.load_state_dict(checkpoint) wrapped_model = PyTorchClassifier( model, loss=torch.nn.CrossEntropyLoss(), optimizer=torch.optim.Adam(model.parameters(), lr=0.003), input_shape=(224, 224, 3), **wrapper_kwargs, clip_values=( np.array([ 0.0 - IMAGENET_MEANS[0] / IMAGENET_STDEV[0], 0.0 - IMAGENET_MEANS[1] / IMAGENET_STDEV[1], 0.0 - IMAGENET_MEANS[2] / IMAGENET_STDEV[2], ]), np.array([ 1.0 - IMAGENET_MEANS[0] / IMAGENET_STDEV[0], 1.0 - IMAGENET_MEANS[1] / IMAGENET_STDEV[1], 1.0 - IMAGENET_MEANS[2] / IMAGENET_STDEV[2], ]), ), ) return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = make_cifar_model(**model_kwargs) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) model.load_weights(filepath) wrapped_model = KerasClassifier(model, clip_values=(0.0, 1.0), **wrapper_kwargs) return wrapped_model
def make_model(model_status="ucf101_trained", weights_file=None): statuses = ("ucf101_trained", "kinetics_pretrained") if model_status not in statuses: raise ValueError(f"model_status {model_status} not in {statuses}") trained = model_status == "ucf101_trained" if not trained and weights_file is None: raise ValueError( "weights_file cannot be None for 'kinetics_pretrained'") if weights_file: filepath = maybe_download_weights_from_s3(weights_file) opt = parse_opts(arguments=[]) opt.dataset = "UCF101" opt.only_RGB = True opt.log = 0 opt.batch_size = 1 opt.arch = f"{opt.model}-{opt.model_depth}" if trained: opt.n_classes = 101 else: opt.n_classes = 400 opt.n_finetune_classes = 101 opt.batch_size = 32 opt.ft_begin_index = 4 opt.pretrain_path = filepath logger.info(f"Loading model... {opt.model} {opt.model_depth}") model, parameters = generate_model(opt) if trained and weights_file is not None: checkpoint = torch.load(filepath, map_location=DEVICE) model.load_state_dict(checkpoint["state_dict"]) # Initializing the optimizer if opt.pretrain_path: opt.weight_decay = 1e-5 opt.learning_rate = 0.001 if opt.nesterov: dampening = 0 else: dampening = opt.dampening optimizer = optim.SGD( parameters, lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov, ) return model, optimizer
def get_art_model(model_kwargs, wrapper_kwargs, weights_file): model = make_densenet121_resisc_model(**model_kwargs) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) model.load_weights(filepath) mean, std = mean_std() wrapped_model = KerasClassifier( model, clip_values=((0.0 - mean) / std, (1.0 - mean) / std), **wrapper_kwargs ) return wrapped_model
def sail(weights_file=None): pretrained = weights_file is not None filepath = None if pretrained: filepath = maybe_download_weights_from_s3(weights_file) sailNet = dnn_models.get_model(weights_file=filepath) if pretrained: sailNet.eval() else: sailNet.train() return sailNet
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) labels_ph = tf.placeholder(tf.int32, shape=[None, 10]) training_ph = tf.placeholder(tf.bool, shape=()) x = tf.layers.conv2d(input_ph, filters=4, kernel_size=(5, 5), activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.conv2d(x, filters=10, kernel_size=(5, 5), activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.flatten(x) x = tf.layers.dense(x, 100, activation=tf.nn.relu) logits = tf.layers.dense(x, 10) loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train_op = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) if weights_file: # Load Model using preferred save/restore method filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=paths.runtime_paths().saved_model_dir) tar.close() # Restore variables... wrapped_model = TFClassifier(clip_values=(0.0, 1.0), input_ph=input_ph, output=logits, labels_ph=labels_ph, train=train_op, loss=loss, learning=training_ph, sess=sess, **wrapper_kwargs) return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file): if weights_file: # Download tarball of all model weights filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=SAVED_MODEL_DIR) tar.close() model = make_ensemble_model(**model_kwargs) else: raise NotImplementedError("This demo is for a pretrained ensemble") mean, std = mean_std() wrapped_model = KerasClassifier(model, clip_values=((0.0 - mean) / std, (1.0 - mean) / std), **wrapper_kwargs) return wrapped_model
def test_keras_mnist_pretrained(): classifier_module = import_module("armory.baseline_models.keras.mnist") classifier_fn = getattr(classifier_module, "get_art_model") weights_path = maybe_download_weights_from_s3("undefended_mnist_5epochs.h5") classifier = classifier_fn( model_kwargs={}, wrapper_kwargs={}, weights_path=weights_path ) test_dataset = datasets.mnist( split="test", epochs=1, batch_size=100, dataset_dir=DATASET_DIR, ) accuracy = 0 for _ in range(test_dataset.batches_per_epoch): x, y = test_dataset.get_batch() predictions = classifier.predict(x) accuracy += np.sum(np.argmax(predictions, axis=1) == y) / len(y) assert (accuracy / test_dataset.batches_per_epoch) > 0.98
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = make_audio_model(**model_kwargs) model.to(DEVICE) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) dic = torch.load(filepath) model.load_state_dict(dic) logger.info("Model weights loaded successfully") wrapped_model = SmoothedPytorchClassifier( model, loss=torch.nn.CrossEntropyLoss(), optimizer=torch.optim.Adam(model.parameters(), lr=0.0001), input_shape=(WINDOW_LENGTH, ), nb_classes=40, ) return wrapped_model
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = make_cifar_model(**model_kwargs) model.to(DEVICE) if weights_file: filepath = maybe_download_weights_from_s3(weights_file) checkpoint = torch.load(filepath, map_location=DEVICE) model.load_state_dict(checkpoint) wrapped_model = PyTorchClassifier( model, loss=nn.CrossEntropyLoss(), optimizer=torch.optim.Adam(model.parameters(), lr=0.003), input_shape=(3, 32, 32), nb_classes=10, clip_values=(0.0, 1.0), **wrapper_kwargs, ) return wrapped_model
def test_pytorch_xview_pretrained(): detector_module = import_module( "armory.baseline_models.pytorch.xview_frcnn") detector_fn = getattr(detector_module, "get_art_model") weights_path = maybe_download_weights_from_s3( "xview_model_state_dict_epoch_99_loss_0p67") detector = detector_fn( model_kwargs={}, wrapper_kwargs={}, weights_path=weights_path, ) NUM_TEST_SAMPLES = 250 dataset_config = { "batch_size": 1, "framework": "numpy", "module": "armory.data.datasets", "name": "xview", } test_dataset = load_dataset( dataset_config, epochs=1, split="test", num_batches=NUM_TEST_SAMPLES, shuffle_files=False, ) list_of_ys = [] list_of_ypreds = [] for x, y in test_dataset: y_pred = detector.predict(x) list_of_ys.extend(y) list_of_ypreds.extend(y_pred) average_precision_by_class = object_detection_AP_per_class( list_of_ys, list_of_ypreds) mAP = np.fromiter(average_precision_by_class.values(), dtype=float).mean() for class_id in [4, 23, 33, 39]: assert average_precision_by_class[class_id] > 0.9 assert mAP > 0.25
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): input_ph = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) labels_ph = tf.placeholder(tf.int32, shape=[None, 10]) training_ph = tf.placeholder(tf.bool, shape=()) # Conditional for handling training phase or inference phase output = tf.cond( training_ph, true_fn=lambda: _training_pass(input_ph), false_fn=lambda: _inference_pass(input_ph), ) loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=output, onehot_labels=labels_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.003) train_op = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) if weights_file: # Load Model using preferred save/restore method filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=paths.runtime_paths().saved_model_dir) tar.close() # Restore variables... wrapped_model = TFClassifier(clip_values=(0.0, 1.0), input_ph=input_ph, output=output, labels_ph=labels_ph, train=train_op, loss=loss, learning=training_ph, sess=sess, **wrapper_kwargs) return wrapped_model
def make_model(model_status="ucf101_trained", weights_file=None): statuses = ("ucf101_trained", "kinetics_pretrained") if model_status not in statuses: raise ValueError(f"model_status {model_status} not in {statuses}") trained = model_status == "ucf101_trained" if not trained and weights_file is None: raise ValueError( "weights_file cannot be None for 'kinetics_pretrained'") if weights_file: filepath = maybe_download_weights_from_s3(weights_file) opt = parse_opts(arguments=[]) opt.dataset = "UCF101" opt.only_RGB = True opt.log = 0 opt.batch_size = 1 opt.arch = f"{opt.model}-{opt.model_depth}" if trained: opt.n_classes = 101 else: opt.n_classes = 400 opt.n_finetune_classes = 101 opt.batch_size = 32 opt.ft_begin_index = 4 opt.pretrain_path = filepath logger.info(f"Loading model... {opt.model} {opt.model_depth}") model, parameters = generate_model(opt) if trained and weights_file is not None: checkpoint = torch.load(filepath, map_location=DEVICE) # Fit the robust model into the original resnext model state_dict_path = 'model' if not ('model' in checkpoint): state_dict_path = 'state_dict' sd = checkpoint[state_dict_path] sd = {k[len('module.'):]: v for k, v in sd.items()} items = list(sd.items()) for key, val in items: if key.startswith('attacker.'): sd.pop(key) if key.startswith('model.'): new_key = 'module.' + key[len('model.'):] sd[new_key] = val sd.pop(key) if key == 'normalizer.new_mean' or key == 'normalizer.new_std': sd.pop(key) model.load_state_dict(sd) # Initializing the optimizer if opt.pretrain_path: opt.weight_decay = 1e-5 opt.learning_rate = 0.001 if opt.nesterov: dampening = 0 else: dampening = opt.dampening optimizer = optim.SGD( parameters, lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov, ) return model, optimizer
def test_invalid_weights(): weights_file = "does_not_exist.h5" with pytest.raises(ValueError, match="attempting to load a custom set of weights"): maybe_download_weights_from_s3(weights_file)
def __init__( self, clip_values, step=200, means=None, stds=None, gan_ckpt='styleGAN.pt', encoder_ckpt='encoder.pt', optimize_noise=True, use_noise_regularize=False, use_lpips=False, apply_fit=False, apply_predict=True, mse=500, lr_rampup=0.05, lr_rampdown=0.05, noise=0.05, noise_ramp=0.75, noise_regularize=1e5, lr=0.1 ): super(InvGAN, self).__init__() #print("invgan") #pdb.set_trace() self._apply_fit = apply_fit self._apply_predict = apply_predict # setup normalization parameters if means is None: means = (0.0, 0.0, 0.0) # identity operation if len(means) != 3: raise ValueError("means must have 3 values, one per channel") self.means = means if stds is None: stds = (1.0, 1.0, 1.0) # identity operation if len(stds) != 3: raise ValueError("stds must have 3 values, one per channel") self.stds = stds self.clip_values = clip_values # setup optimization parameters self.optimize_noise = optimize_noise self.use_noise_regularize = use_noise_regularize self.use_lpips = use_lpips self.step = step self.mse = mse self.lr = lr self.lr_rampup = lr_rampup self.lr_rampdown = lr_rampdown self.noise = noise self.noise_ramp = noise_ramp self.noise_regularize = noise_regularize # setup generator self.generator = Generator(256, 512, 8) #self.generator.load_state_dict(torch.load(gan_ckpt)['g_ema']) self.generator.load_state_dict(torch.load(maybe_download_weights_from_s3(gan_ckpt))['g_ema']) self.generator.eval() self.generator.cuda() self.deprocess_layer = NormalizeByChannelMeanStd([-1., -1., -1.], [2., 2., 2.]).cuda() # setup encoder self.encoder = Encoder() #self.encoder.load_state_dict(torch.load(encoder_ckpt)['netE']) self.encoder.load_state_dict(torch.load(maybe_download_weights_from_s3(encoder_ckpt))['netE']) self.encoder.eval() self.encoder.cuda() # setup loss if use_lpips: self.lpips = PerceptualLoss().cuda() # estimate latent code statistics n_mean_latent = 10000 with torch.no_grad(): noise_sample = torch.randn(n_mean_latent, 512, device='cuda') latent_out = self.generator.style(noise_sample) latent_mean = latent_out.mean(0) self.latent_std = ((latent_out - latent_mean).pow(2).sum() / n_mean_latent) ** 0.5
def sincnet(weights_file=None): pretrained = weights_file is not None if pretrained: filepath = maybe_download_weights_from_s3(weights_file) model_params = torch.load(filepath, map_location=DEVICE) else: model_params = {} CNN_params = model_params.get("CNN_model_par") DNN1_params = model_params.get("DNN1_model_par") DNN2_params = model_params.get("DNN2_model_par") # from SincNet/cfg/SincNet_dev_LibriSpeech.cfg cnn_N_filt = [80, 60, 60] cnn_len_filt = [251, 5, 5] cnn_max_pool_len = [3, 3, 3] cnn_use_laynorm_inp = True cnn_use_batchnorm_inp = False cnn_use_laynorm = [True, True, True] cnn_use_batchnorm = [False, False, False] cnn_act = ["relu", "relu", "relu"] cnn_drop = [0.0, 0.0, 0.0] fc_lay = [2048, 2048, 2048] fc_drop = [0.0, 0.0, 0.0] fc_use_laynorm_inp = True fc_use_batchnorm_inp = False fc_use_batchnorm = [True, True, True] fc_use_laynorm = [False, False, False] fc_act = ["leaky_relu", "linear", "leaky_relu"] class_lay = [40] class_drop = [0.0, 0.0] class_use_laynorm_inp = True class_use_batchnorm_inp = False class_use_batchnorm = [False] class_use_laynorm = [False] class_act = ["softmax"] CNN_options = { "input_dim": WINDOW_LENGTH, "fs": SAMPLE_RATE, "cnn_N_filt": cnn_N_filt, "cnn_len_filt": cnn_len_filt, "cnn_max_pool_len": cnn_max_pool_len, "cnn_use_laynorm_inp": cnn_use_laynorm_inp, "cnn_use_batchnorm_inp": cnn_use_batchnorm_inp, "cnn_use_laynorm": cnn_use_laynorm, "cnn_use_batchnorm": cnn_use_batchnorm, "cnn_act": cnn_act, "cnn_drop": cnn_drop, "pretrained": pretrained, "model_params": CNN_params, } DNN1_options = { "fc_lay": fc_lay, "fc_drop": fc_drop, "fc_use_batchnorm": fc_use_batchnorm, "fc_use_laynorm": fc_use_laynorm, "fc_use_laynorm_inp": fc_use_laynorm_inp, "fc_use_batchnorm_inp": fc_use_batchnorm_inp, "fc_act": fc_act, "pretrained": pretrained, "model_params": DNN1_params, } DNN2_options = { "input_dim": fc_lay[-1], "fc_lay": class_lay, "fc_drop": class_drop, "fc_use_batchnorm": class_use_batchnorm, "fc_use_laynorm": class_use_laynorm, "fc_use_laynorm_inp": class_use_laynorm_inp, "fc_use_batchnorm_inp": class_use_batchnorm_inp, "fc_act": class_act, } sincNet = dnn_models.SincWrapper(DNN2_options, DNN1_options, CNN_options) if pretrained: sincNet.eval() sincNet.load_state_dict(DNN2_params) else: sincNet.train() return sincNet