def main():
logging.basicConfig(filename='log/experiment_spatial.log',
level=logging.INFO)
logging.info("Start training image network: {}".format(
time.asctime(time.localtime(time.time()))))
ctx = mx.gpu(0)
classes_labels, train_videos_classes, test_videos_classes = get_ucf101_split(
ucf.split_dir, ucf.split_id)
#videos = list(test_videos_classes.keys())
#sample_videos= random.sample(videos, 500)
#test_videos_classes_samples = {}
#for video in sample_videos:
# test_videos_classes_samples[video] = test_videos_classes[video]
cm = ConvNet(model_params=resnet_50,
data_params=ucf.image,
train_params=train_image,
test_params=test_image,
train_videos_classes=train_videos_classes,
test_videos_classes=test_videos_classes,
classes_labels=classes_labels,
num_classes=ucf.num_classes,
ctx=ctx,
mode='spatial')
cm.train()
return
def main(args):
args.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('device: {}'.format(args.device))
# create model
model = ConvNet(cfg.NUM_CLASSES).to(args.device)
# define loss function (criterion)
criterion = nn.CrossEntropyLoss().to(args.device)
# load checkpoint
if args.model_weight:
if os.path.isfile(args.model_weight):
print("=> loading checkpoint '{}'".format(args.model_weight))
checkpoint = torch.load(args.model_weight, map_location=args.device)
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}'".format(args.model_weight))
else:
print("=> no checkpoint found at '{}'".format(args.model_weight))
# Data loading code
test_dataset = ImageFolder(cfg.TEST_PATH, mode='test')
test_loader = torch.utils.data.DataLoader(
test_dataset,
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
# Evaluate on test dataset
validate(test_loader, model, criterion, args)
class Testing:
def __init__(self, trainedModel, pathTest, batchSize, imageSize):
super(Testing, self).__init__()
self.model = ConvNet(10)
state_dict = torch.load(trainedModel)
self.model.load_state_dict(state_dict)
self.pathTest = pathTest
self.imageSize = imageSize
self.batchSize = batchSize
self.data_size = calculate_data_size(self.pathTest)
self.data_loader = run_loader('test', self.pathTest, self.batchSize, self.imageSize, shuffle=False)
self.test()
def test(self):
self.model.eval()
acc = 0
y_hat = []
y_true = []
for X, y in tqdm(self.data_loader):
out = self.model(X)
predictions = torch.argmax(out, 1)
acc += torch.sum(predictions == y).item()
y_hat.append(predictions)
y_true.append(y)
y_hat = torch.cat(y_hat)
y_true = torch.cat(y_true)
acc = acc / self.data_size
print(acc)
print(classification_report(y_hat, y_true))
def main():
args = get_arguments()
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
face_dataset = ImageData(root_dir=args.input_dir,\
transform=transforms.Compose([PreprocessData(args.scale_size, args.crop_size)]))
dataloader = DataLoader(face_dataset,
batch_size=args.batch_size,
shuffle=True)
########### setup network ##############
net = ConvNet(3, args.K).to(device)
#----------Weight Initialization---------------
def init_weights(m):
if type(m) == nn.Conv2d:
nn.init.normal_(m.weight, 0, 0.02)
net.apply(init_weights)
#---------------------------------------------
summary(net, (3, 128, 128))
# for name, param in net.named_parameters():
# if param.requires_grad:
# print(name, param.data.size(), type(param))
optimizer = torch.optim.Adam(net.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay)
# #########################################
print()
train(dataloader, net, optimizer, args)
def __init__(self, args, actor_id, q_trace, learner):
self.seed = args.seed
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.q_trace = q_trace
self.learner = learner
self.env = gym.make(args.env)
self.env_state = self.env.reset()
self.n_act = self.env.action_space.n
self.n_state = self.env.observation_space.shape[0]
self.n_atom = args.atom
self.net = ConvNet(self.n_state, self.n_act, self.n_atom).to(self.device)
# パラメータ
self.v_min = args.v_min
self.v_max = args.v_max
self.dz = float(self.v_max - self.v_min) / (self.n_atom - 1)
self.z = [self.v_min + i * self.dz for i in range(self.n_atom)]
# self.z = np.linspace(self.v_min, self.v_max, self.n_atom)
self.value_range = torch.FloatTensor(self.z).to(self.device) # (N_ATOM)
self.step_num = args.step_num
self.eps_greedy = 0.4 ** (1 + actor_id * 7 / (args.n_actors - 1)) \
if args.n_actors > 1 else 0.4
self.n_episodes = 0
self.n_steps = 0
def main(opt):
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
train_set, val_set, test_set = load_dataset(opt)
train_loader = data_utils.DataLoader(train_set,
batch_size=opt.batch_size,
shuffle=True,
num_workers=4)
val_loader = data_utils.DataLoader(val_set,
batch_size=opt.batch_size,
shuffle=True,
num_workers=4)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=opt.test_batch_size,
shuffle=True,
num_workers=4)
model = ConvNet().to(device)
# optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)
# optimizer = optim.SGD(model.parameters(), lr=1e-2)
optimizer = optim.Adam(model.parameters(), lr=opt.lr)
train_loss_list, train_acc_list = [], []
val_loss_list, val_acc_list = [], []
for epoch in range(1, opt.epochs + 1):
loss, acc = train(model, device, train_loader, optimizer, epoch, opt)
train_loss_list.append(loss)
train_acc_list.append(acc)
# print(train_loss_list[-1], train_acc_list[-1])
if (epoch % opt.val_intervals == 0):
loss, acc = test(model, device, val_loader, 'Validation')
val_loss_list.append(loss)
val_acc_list.append(acc)
# print(val_loss_list[-1], val_acc_list[-1])
test(model, device, test_loader, 'Test')
plt.figure(1)
plt.plot(train_loss_list)
plt.plot(val_loss_list)
plt.ylabel("loss")
plt.xlabel("epochs")
plt.legend(['Train', 'Validation'])
plt.title('Loss Plot')
plt.savefig("loss.png")
plt.show()
plt.figure(2)
plt.plot(train_acc_list)
plt.plot(val_acc_list)
plt.ylabel("accuarcy")
plt.xlabel("epochs")
plt.legend(['Train', 'Validation'])
plt.title('Accuracy Plot')
plt.savefig("accuracy.png")
plt.show()
def main():
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("Using device: ", device)
model = ConvNet().to(device)
torch.save(model.state_dict(), FILE)
print("Finished saving model.")
def main():
logging.basicConfig(filename='log/experiment_spatial.log', level=logging.INFO)
logging.info("Start testing image network: {}".format(time.asctime(time.localtime(time.time()))))
ctx = mx.gpu(0)
classes_labels, train_videos_classes, test_videos_classes = get_ucf101_split(ucf.split_dir, ucf.split_id)
cm = ConvNet(model_params=resnet_50, data_params=ucf.image, train_params=train_image, test_params=test_image,
train_videos_classes=train_videos_classes, test_videos_classes=test_videos_classes,
classes_labels=classes_labels, num_classes=ucf.num_classes, ctx=ctx)
cm.test_dataset_evaluation()
return
def __init__(self, trainedModel, pathTest, batchSize, imageSize):
super(Testing, self).__init__()
self.model = ConvNet(10)
state_dict = torch.load(trainedModel)
self.model.load_state_dict(state_dict)
self.pathTest = pathTest
self.imageSize = imageSize
self.batchSize = batchSize
self.data_size = calculate_data_size(self.pathTest)
self.data_loader = run_loader('test', self.pathTest, self.batchSize, self.imageSize, shuffle=False)
self.test()
def __init__(self, epoch, learningRate, batchSize, imageSize, L2Rate, trainPath):
super(Training, self).__init__()
self.epoch = epoch
self.learningRate = learningRate
self.batchSize = batchSize
self.imageSize = imageSize
self.L2Rate = L2Rate
self.trainPath = trainPath
self.data_size = calculate_data_size(self.trainPath)
self.num_batches = self.data_size // batchSize
self.data_loader = run_loader('train', trainPath, batchSize, imageSize, shuffle=True)
self.model = ConvNet(10)
self.train()
def model_fn(model_dir):
"""Load the PyTorch model from the `model_dir` directory."""
print("Loading model.")
# First, load the parameters used to create the model.
model_info = {}
model_info_path = os.path.join(model_dir, 'model_info.pth')
with open(model_info_path, 'rb') as f:
model_info = torch.load(f)
print("model_info: {}".format(model_info))
# Determine the device and construct the model.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = torch.nn.DataParallel(ConvNet(model_info["hidden_dim"], model_info["output_dim"]))
# Load the stored model parameters.
model_path = os.path.join(model_dir, 'model.pth')
with open(model_path, 'rb') as f:
model.load_state_dict(torch.load(f))
# Load the saved word_dict.
model.to(device).eval()
print("Done loading model.")
return model
def main():
if torch.cuda.is_available():
torch.set_default_tensor_type('torch.cuda.FloatTensor')
parser = argparse.ArgumentParser(prog='train',
description="""Script to train the DOA estimation system""")
parser.add_argument("--input", "-i", default="data", help="Directory where data and labels are", type=str)
parser.add_argument("--savedir", "-s", default=".", help="Directory to write results", type=str)
parser.add_argument("--rate", "-r", type=float, default=None, help="Choose a learning rate, default to sweep")
parser.add_argument("--batchsize", "-b", type=int, default=None, help="Choose a batchsize, default to sweep")
parser.add_argument("--epochs", "-e", type=int, default=10, help="Number of epochs")
parser.add_argument("--dropout", "-dp", type=float, default=0., help="Specify dropout rate")
# parser.add_argument("--input_dropout", "-id", type=float, default=0., help="Specify input dropout rate")
# parser.add_argument("--conv_dropout", "-cd", type=float, default=0., help="Specify conv dropout rate (applied at all layers)")
# parser.add_argument("--lstm_dropout", "-ld", type=float, default=0., help="Specify lstm dropout rate (applied to lstm output)")
parser.add_argument("--model", "-m", type=str, choices=["CNN", "CRNN"], required=True, help="Choose network model")
parser.add_argument("--outputformulation", "-of", type=str, choices=["Reg", "Class"], required=True, help="Choose output formulation")
parser.add_argument("--lstmout", "-lo", type=str, choices=[LSTM_FULL, LSTM_FIRST, LSTM_LAST], required=False, default=LSTM_FULL, help="Choose what to use from LSTM ouput")
args = parser.parse_args()
# dropouts = Dropouts(args.input_dropout, args.conv_dropout, args.lstm_dropout)
dropouts = Dropouts(args.dropout, args.dropout, args.dropout)
rates = [1e-5, 1e-7, 1e-3, 1e-9, 1e-1] if not args.rate else [args.rate]
batches = [128, 32, 64] if not args.batchsize else [args.batchsize]
for learning_rate in rates:
for batch_size in batches:
# dir to store the experiment files
results_dir = os.path.join(args.savedir, \
"results" + '_{}'.format(args.model) + '_{}'.format(args.outputformulation) + \
'_lr{}'.format(learning_rate) + '_bs{}'.format(batch_size) + '_drop{}'.format(args.dropout))
print('writing results to {}'.format(results_dir))
doa_classes = None
if args.outputformulation == "Reg":
loss = nn.MSELoss(reduction='sum')
output_dimension = 3
elif args.outputformulation == "Class":
loss = nn.CrossEntropyLoss(reduction="sum")
doa_classes = DoaClasses()
output_dimension = len(doa_classes.classes)
if args.model == "CNN":
model_choice = ConvNet(device, dropouts, output_dimension, doa_classes).to(device)
elif args.model == "CRNN":
model_choice = CRNN(device, dropouts, output_dimension, doa_classes, args.lstmout).to(device)
config = Config(data_folder=args.input,\
learning_rate=learning_rate,\
batch_size=batch_size,\
num_epochs=args.epochs,\
test_to_all_ratio=0.1,\
results_dir=results_dir,\
model=model_choice,\
loss_criterion=loss,\
doa_classes=doa_classes,\
lstm_output=args.lstmout,\
shuffle=True)
doa_train(config)
def main(args):
# append extension to filename if needed
try:
assert args.submission_name[-4:] == '.csv'
except AssertionError:
args.submission_name += '.csv'
# load necessaries from checkpoint
check = torch.load(args.checkpoint)
model_name = check['model']
state_dict = check['state_dict']
# enable cuda if available and desired
args.use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if args.use_cuda else "cpu")
# torch dataset and dataloader
mnist_transform = transforms.Normalize((0.1307, ), (0.3081, ))
dataset = KaggleMNIST(args.data_folder,
train=False,
transform=mnist_transform)
loader = DataLoader(dataset, batch_size=len(dataset))
# construct checkpoint's corresponding model type
if model_name == 'linear':
model = Softmax().to(device)
elif model_name == 'neuralnet':
model = TwoLayer().to(device)
else:
model = ConvNet().to(device)
# load trained weights into model
model.load_state_dict(state_dict)
# make predictions with trained model on test data
# loader has only one element (e.g. batch), so we don't need a loop
# this won't work for large datasets; use a loop in those cases
imgs, _ = next(iter(loader))
imgs = imgs.to(device)
logits = model(imgs)
_, preds = torch.max(
logits,
dim=1) # returns max prob and argmax (e.g. corresponding class)
# when dim is supplied
# construct numpy array with two columns: ids, digit predictions
# we'll use that array to create our text file using the np.savetxt function
ids = (np.arange(len(preds)) + 1).reshape(-1, 1)
preds = preds.view(-1, 1).numpy()
submission = np.concatenate((ids, preds), axis=1)
# writing submisison array to text file with proper formatting
np.savetxt(args.data_folder + args.submission_name,
submission,
fmt='%1.1i',
delimiter=',',
header='ImageId,Label',
comments='')
def main():
X_tr, y_tr, X_te, y_te = load_data()
X_tr, y_tr = X_tr[:1024], y_tr[:1024]
X_te, y_te = X_te[:128], y_te[:128]
if args.model == 'cnn':
model = ConvNet()
model_save_path = config.CNN_MODEL_PATH
else:
model = CapsuleNet()
model_save_path = config.CAPSULE_MODEL_PATH
model.to(device)
optimizer = Adam(model.parameters())
train_loss = []
train_accuracy = []
best_acc = 0.0
for epoch in range(10):
print(("Epoch %d " + "-" * 70) % (epoch + 1))
loss = train(model, optimizer, X_tr, y_tr)
train_loss.append(loss)
acc = test(model, X_tr, y_tr, "Train")
train_accuracy.append(acc)
if acc > best_acc:
best_acc = acc
torch.save(model.state_dict(), model_save_path)
pickle.dump((train_loss, train_accuracy), \
open('result/' + args.model + '_train.p', 'wb'))
def __init__(self, config, cont=None):
super(Wrapper, self).__init__()
with open(config, 'r') as f:
config = json.load(f)
self.config = config
self.best_path = str(self.config['model']['model_save_path'] +
self.config['name'] + '_model_best.pt')
self.model = ConvNet(config['model'])
self.continuing = False
if cont is not None:
print('loading in weights')
self.load_model(cont)
self.continuing = True
self.cuda = torch.cuda.is_available()
if self.cuda:
print('using cuda')
self.model.cuda()
def inference_model(network,lstm_out,out_format,model_path):
doa_classes = DoaClasses()
if out_format == "cartesian":
out_dim = 3
elif out_format == "class":
out_dim = len(doa_classes.classes)
if network == "CNN":
model = ConvNet(device, Dropouts(0,0,0), out_dim, doa_classes)
elif network == "CRNN":
model = CRNN(device, Dropouts(0,0,0), out_dim, doa_classes, lstm_out)
model.load_state_dict(torch.load(model_path,map_location=device))
model.eval()
model.to(device)
return model,doa_classes
def __init__(self, args, q_batch):
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.q_batch = q_batch
self.update_count = 0
self.gamma = args.gamma
self.batch_size = args.batch_size
self.env_eval = gym.make(args.env)
self.n_act = self.env_eval.action_space.n
self.n_state = self.env_eval.observation_space.shape[0]
self.n_quant = args.quant
self.target_net_update_freq = args.target_net_update_freq
self.net = ConvNet(self.n_state, self.n_act,
self.n_quant).to(self.device)
self.target_net = ConvNet(self.n_state, self.n_act,
self.n_quant).to(self.device)
self.optimizer = optim.Adam(self.net.parameters(), lr=args.lr)
def __init__(self, args, actor_id, q_trace, learner):
self.actor_id = actor_id
self.seed = args.seed
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.q_trace = q_trace
self.learner = learner
self.env = gym.make(args.env)
self.env_state = self.env.reset()
self.n_act = self.env.action_space.n
self.n_state = self.env.observation_space.shape[0]
self.n_quant = args.quant
self.net = ConvNet(self.n_state, self.n_act, self.n_quant).to(self.device)
self.eps_greedy = 0.4 ** (1 + actor_id * 7 / (args.n_actors - 1)) \
if args.n_actors > 1 else 0.4
self.n_episodes = 0
self.n_steps = 0
class Training:
def __init__(self, epoch, learningRate, batchSize, imageSize, L2Rate, trainPath):
super(Training, self).__init__()
self.epoch = epoch
self.learningRate = learningRate
self.batchSize = batchSize
self.imageSize = imageSize
self.L2Rate = L2Rate
self.trainPath = trainPath
self.data_size = calculate_data_size(self.trainPath)
self.num_batches = self.data_size // batchSize
self.data_loader = run_loader('train', trainPath, batchSize, imageSize, shuffle=True)
self.model = ConvNet(10)
self.train()
def train(self):
self.model.train()
crossentropy = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learningRate, weight_decay=self.L2Rate)
for epoch in range(self.epoch):
epoch_loss = 0
epoch_acc = 0
for X, y in tqdm(self.data_loader):
optimizer.zero_grad()
out = self.model(X)
loss = crossentropy(out, y)
loss.backward()
optimizer.step()
epoch_loss += loss.item() # makes it to python float
predictions = torch.argmax(out, 1)
epoch_acc += torch.sum(predictions == y).item()
epoch_loss = epoch_loss / self.num_batches
epoch_acc = epoch_acc / self.data_size
print(f"Epoch {epoch}:", "ACC:", epoch_acc, "LOSS:", epoch_loss)
torch.save(self.model.state_dict(), f"Trained/Model_{epoch}.model")
def __init__(self, args, q_batch):
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.q_batch = q_batch
self.learn_step_counter = 0
self.gamma = args.gamma
self.batch_size = args.batch_size
self.env = gym.make(args.env)
self.n_act = self.env.action_space.n
self.n_state = self.env.observation_space.shape[0]
self.n_atom = args.atom
self.v_min = args.v_min
self.v_max = args.v_max
self.dz = (self.v_max - self.v_min) / (self.n_atom - 1)
self.z = [self.v_min + i * self.dz for i in range(self.n_atom)]
self.z_space = torch.FloatTensor(self.z).to(self.device)
self.net = ConvNet(self.n_state, self.n_act, self.n_atom).to(self.device)
self.target_net = ConvNet(self.n_state, self.n_act, self.n_atom).to(self.device)
self.optimizer = optim.Adam(self.net.parameters(), lr=args.lr)
def run_inference(model_dir=None, model_path=None, test_data_path=None):
hp_file = model_dir + 'hp.pkl'
f = open(hp_file, "rb")
hp = pickle.load(f)
model = ConvNet().to(device)
ckpt = torch.load(model_dir + model_path, map_location=device)
model.load_state_dict(ckpt['net'])
batch_size = 100
dataset = TestDataGenerator(test_data_path)
iterator = DataLoader(dataset=dataset,
batch_size=batch_size,
num_workers=hp['num_workers'],
pin_memory=True,
shuffle=False,
drop_last=True)
prediction = np.zeros(10000)
with torch.no_grad():
for i, img in enumerate(iterator):
print(i)
img = img.to(device, dtype=torch.float)
logits = model(img)
logits = logits.detach().cpu().numpy()
pred_label = np.argmax(logits, axis=1)
pred_label = pred_label + 1 # class numbers 1 - 10, predicted labels 0 - 9
prediction[i * batch_size:i * batch_size + batch_size] = pred_label
test_image_names = os.listdir(test_data_path)
for i, name in enumerate(test_image_names):
test_image_names[i] = test_image_names[i][0:5]
with open(model_dir + 'predictions.csv', 'w') as f:
writer = csv.writer(f, delimiter=',')
writer.writerow(['Id', 'Category'])
for i in range(len(test_image_names)):
writer.writerow([test_image_names[i], int(prediction[i])])
def create_model(formulation, model):
dropouts = Dropouts(0, 0, 0)
doa_classes = None
if formulation == "Reg":
loss = nn.MSELoss(reduction='sum')
output_dimension = 3
elif formulation == "Class":
loss = nn.CrossEntropyLoss(reduction="sum")
doa_classes = DoaClasses()
output_dimension = len(doa_classes.classes)
if model == "CNN":
model_choice = ConvNet(device, dropouts, output_dimension, doa_classes).to(device)
elif model == "CRNN":
model_choice = CRNN(device, dropouts, output_dimension, doa_classes, "Full").to(device)
return model_choice
def normal_train(args, loader_train, loader_test, dtype):
model = ConvNet()
model = model.type(dtype)
model.train()
loss_f = nn.CrossEntropyLoss()
SCHEDULE_EPOCHS = [15]
learning_rate = 0.01
for num_epochs in SCHEDULE_EPOCHS:
print('\nTraining %d epochs with learning rate %.4f' %
(num_epochs, learning_rate))
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
for epoch in range(num_epochs):
print('\nTraining epoch %d / %d ...\n' % (epoch + 1, num_epochs))
# print(model.training)
for i, (X_, y_) in enumerate(loader_train):
X = Variable(X_.type(dtype), requires_grad=False)
y = Variable(y_.type(dtype), requires_grad=False).long()
preds = model(X)
loss = loss_f(preds, y)
if (i + 1) % args.print_every == 0:
print('Batch %d done, loss = %.7f' % (i + 1, loss.item()))
optimizer.zero_grad()
loss.backward()
optimizer.step()
print('Batch %d done, loss = %.7f' % (i + 1, loss.item()))
test(model, loader_test, dtype)
learning_rate *= 0.1
return model
def main():
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("Using device: ", device)
model = ConvNet().to(device)
try:
model.load_state_dict(torch.load(FILE))
print("Finished loading model.")
model.eval()
except IOError:
print("Failed to load model. Model might not exist.")
return
print("Print Network Parameters:")
for param in model.parameters():
print(param)
print("Print model state dict: ", model.state_dict())
with torch.no_grad():
print("Perform inference/testing here...")
def black_box_function(lr_pow):
learning_rate = 10.0**lr_pow
results_dir = os.path.join(
savedir, "results" + '_{}'.format(modelname) +
'_{}'.format(args.outputformulation) + '_lr{}'.format(learning_rate) +
'_bs{}'.format(batch_size) + '_drop{}'.format(dropout))
print('writing results to {}'.format(results_dir))
dropouts = Dropouts(dropout, dropout, dropout)
doa_classes = None
if outputformulation == "Reg":
loss = nn.MSELoss(reduction='sum')
output_dimension = 3
elif outputformulation == "Class":
loss = nn.CrossEntropyLoss()
doa_classes = DoaClasses()
output_dimension = len(doa_classes.classes)
if modelname == "CNN":
model_choice = ConvNet(device, dropouts, output_dimension,
doa_classes).to(device)
elif modelname == "CRNN":
model_choice = CRNN(device, dropouts, output_dimension, doa_classes,
lstmout).to(device)
config = TrainConfig() \
.set_data_folder(inputdir) \
.set_learning_rate(learning_rate) \
.set_batch_size(batch_size) \
.set_num_epochs(epochs) \
.set_test_to_all_ratio(0.1) \
.set_results_dir(results_dir) \
.set_model(model_choice) \
.set_loss_criterion(loss) \
.set_doa_classes(doa_classes) \
.set_lstm_output(lstmout)
# negative sign for minimization
return -doa_train(config)
def unrolled(args, loader_train, loader_test, dtype):
model = ConvNet()
model = model.type(dtype)
model.train()
SCHEDULE_EPOCHS = [50, 50]
learning_rate = 5e-4
for num_epochs in SCHEDULE_EPOCHS:
print('\nTraining %d epochs with learning rate %.7f' %
(num_epochs, learning_rate))
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
for epoch in range(num_epochs):
print('\nTraining epoch %d / %d ...\n' % (epoch + 1, num_epochs))
# print(model.training)
for i, (X_, y_) in enumerate(loader_train):
X = Variable(X_.type(dtype), requires_grad=False)
y = Variable(y_.type(dtype), requires_grad=False)
loss = cw_train_unrolled(model, X, y, dtype)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (i + 1) % args.print_every == 0:
print('Batch %d done, loss = %.7f' % (i + 1, loss.item()))
test(model, loader_test, dtype)
print('Batch %d done, loss = %.7f' % (i + 1, loss.item()))
learning_rate *= 0.1
return model
shuffle=True,
pin_memory=True)
val_loader = DataLoader(AudioDataset(
path=os.path.join("audio", "validation"),
sample_rate=config["sample_rate"],
n_mels=config["n_mels"],
n_fft=config["n_fft"],
win_length=config["win_length"],
hop_length=config["hop_length"],
),
batch_size=config["batch_size"],
shuffle=True,
pin_memory=True)
# Initialize model, loss function, optimizers, and lr scheduler
model = ConvNet(base=4)
model.to(device)
loss_fn = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=config["learning_rate"])
lr_scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=10,
gamma=0.1)
# Initialize wandb
wandb.init(project="torch", config=config)
wandb.watch(model, log="all")
# Start training
for epoch in range(1, config["n_epochs"] + 1):
print(f"Epoch {epoch}/{config['n_epochs']}")
start_time = time.time()
def pred_prob(arg_path, field_path, pth_path, doc, device=torch.device('cpu')):
# Load args
# with open(arg_path) as f:
# args = json.load(f)['args']
arg_path = os.path.join(
'https://raw.githubusercontent.com/qianyingw/rob-pome/master/rob-app',
arg_path)
with urllib.request.urlopen(arg_path) as url:
args = json.loads(url.read().decode())['args']
# Load TEXT field
field_url = os.path.join(
'https://github.com/qianyingw/rob-pome/raw/master/rob-app', field_path)
field_path = wget.download(field_url)
with open(field_path, "rb") as fin:
TEXT = dill.load(fin)
os.remove(field_path)
unk_idx = TEXT.vocab.stoi[TEXT.unk_token] # 0
pad_idx = TEXT.vocab.stoi[TEXT.pad_token] # 1
# Load model
if args['net_type'] == 'cnn':
sizes = args['filter_sizes'].split(',')
sizes = [int(s) for s in sizes]
model = ConvNet(vocab_size=args['max_vocab_size'] + 2,
embedding_dim=args['embed_dim'],
n_filters=args['num_filters'],
filter_sizes=sizes,
output_dim=2,
dropout=args['dropout'],
pad_idx=pad_idx,
embed_trainable=args['embed_trainable'],
batch_norm=args['batch_norm'])
if args['net_type'] == 'attn':
model = AttnNet(vocab_size=args['max_vocab_size'] + 2,
embedding_dim=args['embed_dim'],
rnn_hidden_dim=args['rnn_hidden_dim'],
rnn_num_layers=args['rnn_num_layers'],
output_dim=2,
bidirection=args['bidirection'],
rnn_cell_type=args['rnn_cell_type'],
dropout=args['dropout'],
pad_idx=pad_idx,
embed_trainable=args['embed_trainable'],
batch_norm=args['batch_norm'],
output_attn=False)
# Load checkpoint
pth_url = os.path.join(
'https://github.com/qianyingw/rob-pome/raw/master/rob-app', pth_path)
pth_path = wget.download(pth_url)
checkpoint = torch.load(pth_path, map_location=device)
os.remove(pth_path)
state_dict = checkpoint['state_dict']
model.load_state_dict(state_dict, strict=False)
model.cpu()
# Load pre-trained embedding
pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)
model.embedding.weight.data[unk_idx] = torch.zeros(
args['embed_dim']) # Zero the initial weights for <unk> tokens
model.embedding.weight.data[pad_idx] = torch.zeros(
args['embed_dim']) # Zero the initial weights for <pad> tokens
# Tokenization
tokens = [tok.text.lower() for tok in nlp.tokenizer(doc)]
idx = [TEXT.vocab.stoi[t] for t in tokens]
while len(idx) < args['max_token_len']:
idx = idx + [1] * args['max_token_len']
if len(idx) > args['max_token_len']:
idx = idx[:args['max_token_len']]
# Prediction
model.eval()
doc_tensor = torch.LongTensor(idx).to(device)
doc_tensor = doc_tensor.unsqueeze(
1) # bec AttnNet input shape is [seq_len, batch_size]
probs = model(doc_tensor)
probs = probs.data.cpu().numpy()[0]
# print("Prob of RoB reported: {:.4f}".format(probs[1]))
return probs[1]
import torch
import numpy as np
from sklearn import metrics
import matplotlib.pyplot as plt
from data import TwoClassCifar10
from model import ConvNet, LogisticRegression
from config import Config as config
test_dataset = TwoClassCifar10(config.root, train=False)
conv_net = ConvNet(config.input_channel, 2)
lr_model = LogisticRegression(config.cifar10_input_size)
conv_net.load_state_dict(torch.load("model/2020428163925_0.719000.pth"))
lr_model.load_state_dict(torch.load("model/2020428163951_0.589000.pth"))
conv_preds = []
lr_preds = []
targets = []
with torch.no_grad():
for image, label in test_dataset:
image.unsqueeze_(0)
conv_pred = conv_net(image)
lr_pred = lr_model(image)
conv_pred = torch.max(torch.softmax(conv_pred, dim=1),
dim=1)[0].squeeze()
lr_pred = torch.sigmoid(lr_pred).squeeze()
conv_preds.append(conv_pred.item())
lr_preds.append(lr_pred.item())
targets.append(label)
fpr, tpr, thresholds = metrics.roc_curve(targets, conv_preds)
def main(argv):
cuda = FLAGS.cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda and not FLAGS.convex else "cpu")
log_freq = 20
total_steps = max(FLAGS.max_steps,
int(FLAGS.epoches / FLAGS.q)) #total steps
#if(FLAGS.SGN==2):
# total_steps*=4
if (FLAGS.dataset == 'mnist'):
train_tuple = MNIST_data().train()
test_tuple = MNIST_data().test()
if (FLAGS.convex):
train_tuple = (train_tuple[0].reshape(-1, 784), train_tuple[1])
test_tuple = (test_tuple[0].reshape(-1, 784), test_tuple[1])
model = Logistic(FLAGS.dataset).to(device)
else:
#train_tuple=(train_tuple[0].reshape(-1, 784), train_tuple[1])
#test_tuple=(test_tuple[0].reshape(-1, 784), test_tuple[1])
model = ConvNet().to(device)
elif (FLAGS.dataset == 'covertype'):
train_tuple = Covertype_data.train()
test_tuple = Covertype_data.test()
if (FLAGS.convex):
model = Logistic(FLAGS.dataset).to(device)
else:
model = Linear().to(device)
if (FLAGS.momentum == 0):
optimizer = optim.SGD(model.parameters(), lr=FLAGS.lr, momentum=0)
elif (FLAGS.momentum > 0 and FLAGS.momentum <= 1):
if (FLAGS.momentum == 1):
FLAGS.momentum = 0.5
optimizer = optim.SGD(model.parameters(),
lr=FLAGS.lr,
momentum=FLAGS.momentum)
else:
optimizer = optim.Adam(model.parameters())
if (FLAGS.delta == -1):
FLAGS.delta = 1. / (train_tuple[0].shape[0]**2)
diff = 0
if (FLAGS.delta != 0 and FLAGS.epoches != -1):
if (FLAGS.auto_sigma == 0):
FLAGS.sigma = get_sigma(FLAGS.q, FLAGS.epoches, FLAGS.epsilon,
FLAGS.delta)
elif (FLAGS.auto_sigma == 1):
FLAGS.SGN = 0
FLAGS.sigma = get_sigma(FLAGS.q, FLAGS.epoches, FLAGS.epsilon,
FLAGS.delta)
FLAGS.sigma *= 2
#FLAGS.sigma=20
#recording information of this experiment instance
experiment_info = 'Dataset: %r \nSampling probability: %r \nDelta: %r \nConvex: %r \nClip_bound: %r \nSigma: %r\nMomentum: %r\nAuto_sigma: %d\nSGN: %d \nEpoches: %d \nEpsilon: %r \n' % (
FLAGS.dataset, FLAGS.q, FLAGS.delta, FLAGS.convex, FLAGS.clip_bound,
FLAGS.sigma, FLAGS.momentum, FLAGS.auto_sigma, FLAGS.SGN,
FLAGS.epoches, FLAGS.epsilon)
logging(experiment_info, 'w')
total_privacy_l = [
0.
] * 128 #tracking alpha at different orders [1,128], can be converted to (epsilon,delta)-differential privacy
epsilons = [0.5, 1., 2.0]
deltas = [0., 0., 2.0] #one delta for one epsilon
log_array = []
norm_list = []
for t in range(1, total_steps + 1):
#print(FLAGS.sigma, 'here')
#get the gradients, notice the optimizer.step() is ran outside the train function.
total_privacy_l = train(model, device, train_tuple, optimizer, diff,
total_privacy_l, t, norm_list)
if (FLAGS.delta > 0 and FLAGS.delta < 1): #training privately
all_failed = True
for i, eps in enumerate(epsilons):
if (deltas[i] > FLAGS.delta
): #discarding the epsilon we already failed
continue
#use rdp_accountant to get delta for given epsilon
if_update_delta, order = _compute_delta(
range(2, 2 + 128), total_privacy_l, eps)
#print(if_update_delta, 'hereheee')
if (
if_update_delta > FLAGS.delta
): #record the final model satisfies (eps,deltas[i])-differential privacy
accuracy = test(model, device, test_tuple, t)
info = 'For epislon %r, delta %r we get accuracy: %r%% at step %r\n' % (
eps, deltas[i], accuracy, t)
deltas[i] = 1. #abort current epsilon
logging(info)
print(info)
else:
deltas[i] = if_update_delta #update delta
all_failed = False #still got at least one epsilon not failed
if (not all_failed):
optimizer.step()
else:
info = 'failed at all given epsilon, exiting\n'
print(info)
logging(info)
exit()
else: #training no privately
optimizer.step()
if (t % log_freq == 0):
#aa=1
accuracy = test(model, device, test_tuple, t)
log_array.append(copy.deepcopy([t, accuracy, epsilons, deltas]))
np.save('logs/log%d.npy' % FLAGS.experiment_id,
np.array(log_array, dtype=object))
