def by_infile(self, infile):
try:
shutil.rmtree(self.OUTPUT_DIR)
except:
pass
self.db_open()
json_data = self.get_events_from_infile(infile)
# build preprocessor
ppr = Preprocessor()
# Process raw data
#X, Y, events_found = ppr.get_raw_data(DIMENSION, [RAW_FILE], bad)
X, Y, events_found = ppr.get_from_json(self.DIMENSION, json_data)
X, Y = ppr.remove_outliers(X, Y)
X, Y = ppr.normalize(X, Y)
trX, trY, teX, teY, vaX, vaY = ppr.partition_for_training(
X, Y, 0.0, 1.0)
ppr.store_training_partitions(trX, trY, teX, teY, vaX, vaY,
self.INPUT_DIR)
# build adapter
adapter = MACAdapter(self.INPUT_DIR, self.DIMENSION, self.FOLDS)
# build model
convnet = ConvNet(self.DIMENSION)
# build server
server = ConvNetServer(adapter,
self.OUTPUT_DIR,
batch_size=self.BATCH_SIZE,
verbose=True,
use=True)
x, durs, _ = server.get_testing_batch()
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
convnet.restore(sess, self.INITIAL_WEIGHTS)
predictions = sess.run((convnet.predictor),
feed_dict={
convnet.x: x,
convnet.durs: durs
})
# Get event ids
_, _, ids = adapter.get_ids()
results = [{
"eventID": int(ids[i]),
"ml": {
"aircraftProbability":
round(np.around(predictions[i][0], decimals=4), 4),
"model":
self.MODEL
}
} for i in range(0, len(ids))]
for result in results:
self.insert_result_for_event(result)
self.db_close()
from datasets.cuhk01 import CUHK01
from datasets.transforms.transforms import Scale, Slice
from datasets.viper import VIPeR
from metric.metric import metric_learn
from models.bkw import BkwNet
from models.convnet import ConvNet
from models.parts import PartsNet
from models.triplet import TripletNet
from models.voting import VotingNet
from plotter import Plot
from trainer import Trainer
name = 'triplet-sgd-nopool' + datetime.now().strftime('_%Y-%m-%d_%H%M%S')
plot = Plot(name)
net = TripletNet(ConvNet())
net.load_state_dict(torch.load('triplet-sgd_2018-05-10_101323_best'))
# net = VotingNet(BkwNet())
net.cuda()
train, val, test = VIPeR.create((316, 380), shuffle_seed=12345)
metric = ITML_Supervised()
metric_learn(metric, net, train)
criterion = nn.TripletMarginLoss().cuda()
optimizer = optim.SGD(net.parameters(), lr=1e-2, momentum=0.9)
scheduler = scheduler.ReduceLROnPlateau(optimizer,
patience=1,
eps=1e-8,
verbose=True)
def run(model: nn.Module,
siamese: bool = False,
epochs: int = 25,
lr: float = 5e-3,
decay: float = 1e-3,
gamma: float = 0.,
trials: int = 0,
seed: int = 27,
batch_size: int = 50,
standardize: bool = True,
filename: str = 'model',
verbose: int = 0):
"""
Run a trial of model trainings.
Args:
model (nn.Module): Model
siamese (bool, optional): Use the siamese weight-sharing version. Defaults to False.
epochs (int, optional): Number of training epochs. Defaults to 25.
lr (float, optional): Learning rate. Defaults to 5e-3.
decay (float, optional): Regularization weight decay. Defaults to 1e-3.
gamma (float, optional): Auxiliary loss gamma hyper-parameter. Defaults to 0..
trials (int, optional): Number of trials to run. Defaults to 0.
seed (int, optional): PyTorch manual seed to set. Defaults to 27.
batch_size (int, optional): Batch size to use for training. Defaults to 50.
standardize (bool, optional): Standardize data. Defaults to True.
filename (str, optional): Filename to save metrics in. Defaults to 'model'.
verbose (int, optional): Print out intermediate information. Defaults to 0.
"""
training_metrics = TrainingMetrics()
testing_metrics = TestingMetrics()
for trial in range(trials):
if verbose > 1:
print(f"Creating {'standardized' if standardize else ''} "
f"DataLoaders with batch size {batch_size}...")
torch.manual_seed(seed + trial)
train_loader, test_loader = load_dataset(batch_size=batch_size,
standardize=standardize)
start = time.time()
if siamese:
model = SiameseConvNet() if model == 'ConvNet' else SiameseMLP()
else:
model = ConvNet() if model == 'ConvNet' else MLP()
if verbose > 1:
print(
f"{model} instanciated with {model.param_count()} parameters.")
train(model,
train_loader,
learning_rate=lr,
weight_decay=decay,
gamma=.5,
epochs=epochs,
metrics=training_metrics,
run=trial,
verbose=verbose)
end = time.time()
if verbose > 1:
print("Evaluating performance on test set...")
testing_metrics.add_entry(model, test_loader, (end - start) / epochs,
verbose)
'''
# Number of trials to do for each fold (stats will be averaged)
TRIALS_PER_FOLD = 5
'''
SCRIPT
'''
# Only run if this is the main module to be run
if __name__ == '__main__':
# build adapter
adapter = MACAdapter(INPUT_DIR, DIMENSION, FOLDS)
# build model
convnet = ConvNet(DIMENSION)
# build server
server = ConvNetServer(adapter, OUTPUT_DIR,
batch_size = BATCH_SIZE,
verbose = False)
# build trainer
trainer = ConvNetTrainer(convnet, server, EPOCHS, STEPS_PER_EPOCH,
optimizer = OPTIMIZER,
opt_kwargs = OPT_KWARGS,
keep_prob = KEEP_PROB,
batch_size = BATCH_SIZE,
display_step = DISPLAY_STEP)
train_data = dset.MNIST('/home-nfs/dan/cifar_data/mnist', train=True, transform=trn.ToTensor())
test_data = dset.MNIST('/home-nfs/dan/cifar_data/mnist', train=False, transform=trn.ToTensor())
num_classes = 10
train_data, val_data = validation_split(train_data, val_share=0.1)
val_loader = torch.utils.data.DataLoader(
val_data, batch_size=args.test_bs, shuffle=False,
num_workers=args.prefetch, pin_memory=True)
test_loader = torch.utils.data.DataLoader(
test_data, batch_size=args.test_bs, shuffle=False,
num_workers=args.prefetch, pin_memory=True)
# Create model
net = ConvNet()
start_epoch = 0
# Restore model
if args.load != '':
for i in range(300 - 1, -1, -1):
if 'baseline' in args.method_name:
subdir = 'baseline'
elif 'oe_tune' in args.method_name:
subdir = 'oe_tune'
else:
subdir = 'oe_scratch'
model_name = os.path.join(os.path.join(args.load, subdir), args.method_name + '_epoch_' + str(i) + '.pt')
def main():
parser = argparse.ArgumentParser(description="Trainer")
parser.add_argument(
"-b",
"--batch-size",
type=int,
default=64,
metavar="B",
help="input batch size for training (default: 64)",
)
parser.add_argument(
"-na",
"--n_accumulation_steps",
default=1,
type=int,
metavar="N",
help="number of mini-batches to accumulate into an effective batch",
)
parser.add_argument(
"--test-batch-size",
type=int,
default=200,
metavar="TB",
help="input batch size for testing (default: 1024)",
)
parser.add_argument(
"-d",
"--data",
type=str,
default="mnist",
metavar="D",
help="dataset to train on (mnist, fashion, cifar10, cifar100"
)
parser.add_argument(
"-n",
"--epochs",
type=int,
default=10,
metavar="N",
help="number of epochs to train (default: 14)",
)
parser.add_argument(
"-r",
"--n-runs",
type=int,
default=1,
metavar="R",
help="number of runs to average on (default: 1)",
)
parser.add_argument(
"--lr",
type=float,
default=0.1,
metavar="LR",
help="learning rate (default: .1)",
)
parser.add_argument(
"--wd",
"--weight-decay",
default=0.,
type=float,
metavar="W",
help="SGD weight decay (default: 1e-4)",
dest="weight_decay",
)
parser.add_argument(
"--momentum", default=0., type=float, metavar="M", help="SGD momentum"
)
parser.add_argument(
"--sigma",
type=float,
default=1.0,
metavar="S",
help="Noise multiplier (default 1.0)",
)
parser.add_argument(
"-c",
"--max-per-sample-grad_norm",
type=float,
default=1.0,
metavar="C",
help="Clip per-sample gradients to this norm (default 1.0)",
)
parser.add_argument(
"--delta",
type=float,
default=1e-5,
metavar="D",
help="Target delta (default: 1e-5)",
)
parser.add_argument(
"--device",
type=str,
default="cuda",
help="GPU ID for this process (default: 'cuda')",
)
parser.add_argument(
"--save-model",
action="store_true",
default=False,
help="Save the trained model (default: false)",
)
parser.add_argument(
"--disable-dp",
action="store_true",
default=False,
help="Disable privacy training and just train with vanilla SGD",
)
parser.add_argument(
"--data-root",
type=str,
default="./data",
help="Where MNIST is/will be stored",
)
args = parser.parse_args()
device = torch.device(args.device)
kwargs = {"num_workers": 1, "pin_memory": True}
train_set, test_set, train_loader, test_loader = SangExp.get_data(
data=args.data,
batch_size=args.batch_size,
test_batch_size=args.test_batch_size,
download_path=args.data_root
)
if args.data == "mnist" or args.data == "fashion":
model = ConvNet()
elif args.data == "cifar10":
model = resnet18(num_classes=10)
elif args.data == "cifar100":
model = resnet18(num_classes=100)
model = convert_batchnorm_modules(model).to(device)
optimizer = optim.SGD(
model.parameters(),
lr = args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay
)
if not args.disable_dp:
privacy_engine = PrivacyEngine(
model,
batch_size=args.batch_size * args.n_accumulation_steps,
sample_size=len(train_set),
alphas=[1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)),
noise_multiplier=args.sigma,
max_grad_norm=args.max_per_sample_grad_norm,
clip_per_layer=False,
enable_stat=False
)
privacy_engine.attach(optimizer)
stats = []
for epoch in range(args.epochs):
stat = []
if not args.disable_dp:
epsilon, best_alpha = train(args, model, device, train_loader, optimizer, epoch)
stat.append(epsilon)
stat.append(best_alpha)
else:
train(args, model, device, train_loader, optimizer, epoch)
acc = test(args, model, device, test_loader)
stat.append(acc)
stats.append(tuple(stat))
name = "save/{}".format(args.data)
if not args.disable_dp:
name += "_dp"
np.save(name, stats)
torch.save(model.state_dict(), name+".pt")
pin_memory=True)
train_loader_out = torch.utils.data.DataLoader(tiny_images,
batch_size=args.oe_batch_size,
shuffle=False,
num_workers=3,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.prefetch,
pin_memory=True)
# Create model
net = ConvNet()
# Restore model if desired
model_found = False
if args.load != '':
for i in range(1000 - 1, -1, -1):
model_name = os.path.join(
args.load, calib_indicator + 'baseline_epoch_' + str(i) + '.pt')
if os.path.isfile(model_name):
net.load_state_dict(torch.load(model_name))
print('Model restored! Epoch:', i)
model_found = True
break
if not model_found:
assert False, "could not resume"
# build preprocessor
ppr = Preprocessor()
# Process raw data
X, Y, events_found = ppr.get_raw_data(DIMENSION, [RAW_FILE], bad)
X, Y = ppr.remove_outliers(X, Y)
X, Y = ppr.normalize(X, Y)
trX, trY, teX, teY, vaX, vaY = ppr.partition_for_training(X, Y, 0.0, 1.0)
ppr.store_training_partitions(trX, trY, teX, teY, vaX, vaY, INPUT_DIR)
# build adapter
adapter = MACAdapter(INPUT_DIR, DIMENSION, FOLDS)
# build model
convnet = ConvNet(DIMENSION)
# build server
server = ConvNetServer(adapter,
OUTPUT_DIR,
batch_size=BATCH_SIZE,
verbose=True,
use=True)
x, durs, _ = server.get_testing_batch()
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
convnet.restore(sess, INITIAL_WEIGHTS)
def run(model, siamese, epochs, lr, decay, gamma, trials, seed, batch_size,
standardize, make_figs, clear_figs, verbose):
# Clear figures directory
if clear_figs:
if verbose > 0:
print("Clearing previous figures...")
metrics.clear_figures()
# Create figures subdirectory for current run
if make_figs:
if verbose > 0:
print("Creating folder for trial figures...")
timestamp = str(dt.datetime.today())
os.makedirs(os.path.join(metrics.FIGURE_DIR, timestamp))
training_metrics = TrainingMetrics()
testing_metrics = TestingMetrics()
for trial in range(trials):
if verbose > 1:
print(f"Creating {'standardized' if standardize else ''} "
f"DataLoaders with batch size {batch_size}...")
torch.manual_seed(seed + trial)
train_loader, test_loader = load_dataset(batch_size=batch_size,
standardize=standardize)
start = time.time()
if siamese:
model = SiameseConvNet() if model == 'ConvNet' else SiameseMLP()
else:
model = ConvNet() if model == 'ConvNet' else MLP()
if verbose > 1:
print(
f"{model} instanciated with {model.param_count()} parameters.")
train(model,
train_loader,
learning_rate=lr,
weight_decay=decay,
gamma=gamma,
epochs=epochs,
metrics=training_metrics,
run=trial,
verbose=verbose)
end = time.time()
testing_metrics.add_entry(model, test_loader, (end - start) / epochs,
verbose)
if make_figs:
training_metrics.plot(timestamp)
testing_metrics.plot(timestamp)
testing_metrics.save()
return training_metrics._average_accuracy(
), testing_metrics._average_accuracy()
# probability value to use for dropout
KEEP_PROB = 1.0
# training batch size
BATCH_SIZE = 400
# step at which to log status at modulo 0
DISPLAY_STEP = 5
'''
SCRIPT
'''
# Only run if this is the main module to be run
if __name__ == '__main__':
# build adapter
adapter = MACAdapter(INPUT_DIR)
# build model
convnet = ConvNet(10)
# build server
server = ConvNetServer(adapter, OUTPUT_DIR, batch_size=BATCH_SIZE)
# build trainer
trainer = ConvNetTrainer(convnet,
server,
EPOCHS,
STEPS_PER_EPOCH,
optimizer=OPTIMIZER,
opt_kwargs=OPT_KWARGS)
convnet.test_shp()
return file_list
if __name__ == "__main__":
opt = eval_parse()
device = torch.device("cuda" if cuda.is_available() else "cpu")
train_iter, val_iter = dataset.setup(opt)
# Get list of model paths
checkpoints = get_model_paths(opt.checkpoint)
for path in checkpoints:
model = ConvNet(opt.nClasses)
model.load_state_dict(torch.load(path, map_location=device))
model = model.to(device)
line = "Loading/evaluating model {}".format(path)
sys.stderr.write('\r\033[K' + line)
sys.stderr.flush()
trainer = Trainer(model, None, train_iter, val_iter, opt)
error_rate = trainer.val()
line ="Error rate: {:.2f} | Model: {}\n".format(error_rate, path)
sys.stderr.write('\r\033[K' + line)
sys.stderr.flush()
num_classes = 10
calib_indicator = ''
if args.calibration:
train_data, val_data = validation_split(train_data, val_share=0.1)
calib_indicator = 'calib_'
train_loader = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True,
num_workers=args.prefetch, pin_memory=True)
test_loader = torch.utils.data.DataLoader(
test_data, batch_size=args.test_bs, shuffle=False,
num_workers=args.prefetch, pin_memory=True)
# Create model
net = ConvNet()
start_epoch = 0
# Restore model if desired
if args.load != '':
for i in range(1000 - 1, -1, -1):
model_name = os.path.join(args.load, calib_indicator + 'baseline_epoch_' + str(i) + '.pt')
if os.path.isfile(model_name):
net.load_state_dict(torch.load(model_name))
print('Model restored! Epoch:', i)
start_epoch = i + 1
break
if start_epoch == 0:
assert False, "could not resume"
import torch
# from models.neuralnet import NeuralNet
from models.convnet import ConvNet
import torch.nn as nn
if __name__ == '__main__':
classifier = ConvNet(
3, [['conv2d', '15,3,2'], ['conv2dt', '35,3,2'], ['conv2d', '55,3,2'],
['conv2dt', '50,3,2']],
[[nn.MaxPool2d(
(2, 2)), nn.BatchNorm2d(15),
nn.ReLU()], 'skip',
nn.ReLU(),
nn.ReLU()])
print(classifier(torch.randn(1, 3, 100, 100)).shape)
def create_model(model_name, num_classes=1000, pretrained=False, **kwargs):
if 'test_time_pool' in kwargs:
test_time_pool = kwargs.pop('test_time_pool')
else:
test_time_pool = True
if 'extra' in kwargs:
extra = kwargs.pop('extra')
else:
extra = True
if model_name == 'dpn68':
model = dpn68(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool)
elif model_name == 'dpn68b':
model = dpn68b(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool)
elif model_name == 'dpn92':
model = dpn92(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool,
extra=extra)
elif model_name == 'dpn92_re':
model = dpn92_re(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool,
extra=extra)
elif model_name == 'dpn98':
model = dpn98(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool)
elif model_name == 'dpn131':
model = dpn131(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool)
elif model_name == 'dpn107':
model = dpn107(num_classes=num_classes,
pretrained=pretrained,
test_time_pool=test_time_pool)
elif model_name == 'resnet18':
model = resnet18(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet18_sigmoid':
model = resnet18_sigmoid(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet34':
model = resnet34(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet50':
model = resnet50(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet50_bestfitting':
model = resnet50_bestfitting(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet50_sigmoid':
model = resnet50_sigmoid(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet101':
model = resnet101(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet152':
model = resnet152(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'resnet152_sigmoid':
model = resnet152_sigmoid(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'densenet121':
model = densenet121(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'densenet161':
model = densenet161(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'densenet169':
model = densenet169(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'densenet201':
model = densenet201(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'inception_v3':
model = inception_v3(num_classes=num_classes,
pretrained=pretrained,
transform_input=False,
**kwargs)
elif model_name == 'vgg16_bn':
model = vgg16_bn(num_classes=num_classes,
pretrained=pretrained,
**kwargs)
elif model_name == 'ConvNet_sigmoid':
model = ConvNet()
else:
assert False, "Unknown model architecture (%s)" % model_name
return model