def main():
torch.manual_seed(43)
criterion = nn.CrossEntropyLoss()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
train_loader, test_loader = data_loaders()
model = FNN_ID(750).to(device)
optimizer = optim.Adam(model.parameters())
train(model, device, train_loader, criterion, optimizer)
test(model, device, test_loader, criterion)
save(model)
def train_it():
"""
Function for triggering training of the model
"""
print("request received: training the model")
results = train(
"Hello! This is the result of training from Flask backend!")
resp = jsonify(results)
resp.headers.add("Access-Control-Allow-Origin", "*")
return resp
def main():
# user inputs from command line
in_arg = get_input_args()
# load and process data into training, validation and test data sets
trainloader, validationloader, testloader, train_data = load_and_transform(
in_arg.data_dir)
# load pre-trained nn and build classifier with user inputs (loss criterion & optimizer are fixed) to create the model
model, criterion, optimizer = build_classifier(in_arg.arch,
in_arg.in_features,
in_arg.hidden_layers,
in_arg.output_size,
in_arg.learning_rate)
# Train the model
trained_model = train(in_arg.epochs, trainloader, validationloader,
optimizer, model, criterion, in_arg.gpu)
# saving the model
save_model(trained_model, optimizer, in_arg.saving_dir, in_arg.arch,
in_arg.learning_rate, in_arg.epochs, train_data)
def main(net, dataloader, device, config):
train_loader, test_loader = dataloader[0], dataloader[1]
optimizer = optim.SGD(net.parameters(),
lr=config.lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
scheduler = optim.lr_scheduler.ExponentialLR(optimizer, 0.95)
crit = get_criterion(config)
crit = crit.to(device)
if not os.path.isdir(config.weight):
os.makedirs(config.weight)
checkpoint = os.path.join(config.weight, config.model + '.pth')
best_acc = 0
for epoch in range(config.epoch):
logging.info(f'LR: {scheduler.get_lr()}')
########## TRAIN ##########
start = time()
net.train()
loss_train = train(config, net, device, train_loader, crit, optimizer,
epoch)
end = time() - start
logging.info(
f'=> Epoch[{epoch}], Average train Loss: {loss_train:.3f}, Tot Time: {end:.3f}'
)
########## TEST ###########
start = time()
net.eval()
acc = test(config, net, device, test_loader, epoch)
end = time() - start
logging.info(
f'=> Epoch[{epoch}], Final accuracy: {acc:.3f}, Tot Time: {end:.3f}\n'
)
if acc > best_acc:
torch.save(net.state_dict(), checkpoint)
logging.info(f'Saving best model checkpoint at {checkpoint}')
best_acc = acc
scheduler.step()
logging.info(f'Best test accuracy: {best_acc}')
def command_line(dictionary):
while True:
command = input('Enter your command: ')
if command == 'exit':
return 'exit'
elif command == 'move':
functions.move()
elif command == 'help':
for command, description in list(commands_list.items()):
print(command + ' = ' + description)
elif command == 'add':
functions.add()
return functions.update()
elif command == 'remove':
functions.remove()
elif command == 'print':
continue
elif command == 'show':
functions.show()
elif command == 'train':
return functions.train(dictionary)
baseline = baselineNet(nObs).to(DEVICE) if BASELINE else None
saver.addObj(baseline, "baseline", True,
DEVICE) if saver is not None else None
optbaseline = optimizerMaker("adam", LEARNING_RATE)
# Training policy
print("Training policy\n -------------")
LOGR.logr("Begining training")
accRewards, var, steps = train(envmaker,
policy,
optpolicy,
baseline=baseline,
optBaselineMaker=optbaseline,
device=DEVICE,
iterations=ITERS,
testFreq=TEST_FREQ,
maxEpisodeLength=MAX_EPISODE_LENGTH,
gamma=GAMMA,
batchSize=BATCH_SIZE,
pBatchMem=PBM,
nWorkers=1,
saver=saver,
learningRate=LEARNING_RATE)
LOGR.logr("Trainnig Done")
saver.saveAll() if saver is not None else None
LOGR.logr("Objects saved")
graphResults(accRewards, var, meanRnd, varRnd, ITERS, TEST_FREQ, GRAPH_MOD)
if PLAY: playTest(env, policy, NAME, 200, 20)
LOGR.logr("Experiment done")
for i_batch in range(nb_batches - 1):
sentences_batch = sentences[i_batch *
glob.batch_size:(i_batch + 1) *
glob.batch_size]
# xlm_batch = XLM_pre.create_batch(sentences_batch)
# xlm_batch = xlm_batch.detach() # do not train the pretrained XLM weights
for s in sentences_batch:
actual_tags = torch.tensor(
[glob.POS_DICT[w.pos] for w in s.words],
device=glob.device)
# indexes = sentence_to_word_index(dico, s)
train(Net, s, actual_tags, optimizer)
batch_time = time.clock() - start_time
print(round(100 * (i_batch + 1) / nb_batches, 2), end="% ")
remaining_time = batch_time / (
(i_batch + 1) / nb_batches) - batch_time
remaining_time *= glob.epochs
h = int(remaining_time // 3600)
m = int(remaining_time // 60 - (remaining_time // 3600) * 60)
s = int(remaining_time % 60)
print("Remaining : {0}h{1}m{2}s".format(h, m, s))
sys.stdout.flush()
print('.')
except KeyboardInterrupt:
def main():
args = cfg.parse_args()
random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net = Generator(bottom_width=args.bottom_width,
gf_dim=args.gf_dim,
latent_dim=args.latent_dim).cuda()
dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda()
gen_net.apply(weights_init)
dis_net.apply(weights_init)
initial_gen_net_weight = torch.load(os.path.join(args.init_path,
'initial_gen_net.pth'),
map_location="cpu")
initial_dis_net_weight = torch.load(os.path.join(args.init_path,
'initial_dis_net.pth'),
map_location="cpu")
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
exp_str = args.dir
args.load_path = os.path.join('output', exp_str, 'pth',
'epoch{}.pth'.format(args.load_epoch))
# state dict:
assert os.path.exists(args.load_path)
checkpoint = torch.load(args.load_path)
print('=> loaded checkpoint %s' % args.load_path)
state_dict = checkpoint['generator']
gen_net = load_subnet(args, state_dict, initial_gen_net_weight).cuda()
avg_gen_net = deepcopy(gen_net)
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
else:
raise NotImplementedError('no fid stat for %s' % args.dataset.lower())
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
np.random.seed(args.random_seed)
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
start_epoch = 0
best_fid = 1e4
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
#logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch))
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
gen_avg_param = copy_params(gen_net)
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict)
logger.info(
'Inception score: %.4f, FID score: %.4f || @ epoch %d.' %
(inception_score, fid_score, epoch))
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net.load_state_dict(gen_net.state_dict())
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
def main():
args = parse_args()
args.pretrain = False
root_path = 'exps/exp_{}'.format(args.exp)
if not os.path.exists(root_path):
os.mkdir(root_path)
os.mkdir(os.path.join(root_path, "log"))
os.mkdir(os.path.join(root_path, "model"))
base_lr = args.lr # base learning rate
train_dataset, val_dataset = build_dataset(args.dataset, args.data_root,
args.train_list)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
num_workers=args.num_workers,
pin_memory=True)
model = VNet(args.n_channels, args.n_classes).cuda()
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=0.0005)
#scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.7)
model = torch.nn.DataParallel(model)
model.train()
if args.resume is None:
assert os.path.exists(args.load_path)
state_dict = model.state_dict()
print("Loading weights...")
pretrain_state_dict = torch.load(args.load_path,
map_location="cpu")['state_dict']
for k in list(pretrain_state_dict.keys()):
if k not in state_dict:
del pretrain_state_dict[k]
model.load_state_dict(pretrain_state_dict)
print("Loaded weights")
else:
print("Resuming from {}".format(args.resume))
checkpoint = torch.load(args.resume, map_location="cpu")
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
model.load_state_dict(checkpoint['state_dict'])
logger = Logger(root_path)
saver = Saver(root_path)
for epoch in range(args.start_epoch, args.epochs):
train(model, train_loader, optimizer, logger, args, epoch)
validate(model, val_loader, optimizer, logger, saver, args, epoch)
adjust_learning_rate(args, optimizer, epoch)
valid_loss2 = []
valid_acc2 = []
train_loss3 = []
train_acc3 = []
valid_loss3 = []
valid_acc3 = []
train_loss4 = []
train_acc4 = []
valid_loss4 = []
valid_acc4 = []
for epoch in range(1, nb_epochs + 1):
train_l, train_a = train(resnet2, train_loader, learning_rate, criterion,
epoch, batch_log, device)
train_l, train_a = test(resnet2, train_loader, criterion, epoch, batch_log,
device)
train_loss2.append(train_l)
train_acc2.append(train_a)
valid_l, valid_a = test(resnet2, valid_loader, criterion, epoch, batch_log,
device)
valid_loss2.append(valid_l)
valid_acc2.append(valid_a)
with open("compareResnet_log2.txt", "w") as output:
output.write("ResNet\t size=16\t ratio=2^(-1/3)\t nratio=6 \n")
output.write(str(train_loss2))
output.write(str(train_acc2))
output.write(str(valid_loss2))
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda()
dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda()
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
avg_gen_net = deepcopy(gen_net)
avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = copy_params(avg_gen_net)
del avg_gen_net
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict)
logger.info(
f'Inception score: {inception_score}, FID score: {fid_score} || @ epoch {epoch}.'
)
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net = deepcopy(gen_net)
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
del avg_gen_net
data[3].append(float(time.time() - init_time))
functions.us(us_dur, us_pin)
data[3].append(float(time.time() - init_time))
print(time.time() - init_time)
else:
if i in probe_trials:
time.sleep(timing[i])
data[3].append(float(time.time() - init_time))
functions.cs(cs_dur, cs_pin)
data[3].append(float(time.time() - init_time))
print(time.time() - init_time)
continue
else:
time.sleep(timing[i])
data[3].append(float(time.time() - init_time))
functions.train(cs_dur, us_dur, isi, cs_pin, us_pin)
data[3].append(float(time.time() - init_time))
print(time.time() - init_time)
# %%
#writing stuff
for i in range(0, len(data[0])):
with open(data[0][i] + ".csv", 'w') as myfile:
write = csv.writer(myfile, quoting=csv.QUOTE_ALL)
write.writerow(data[i + 1])
# %%
(data[1][5] - data[1][4]) * 1000
# %%
def main():
#define supported models
allowed_models = [
'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152'
]
#Set up argument parser for console input
parser = argparse.ArgumentParser(description='Train NN')
parser.add_argument('data_dir',
help='directory containing sub-folders with data')
parser.add_argument('--save_dir',
help='directory for saving checkpoint',
default='checkpoints')
parser.add_argument('--arch',
help='pre-trained model architecture',
default='resnet18',
choices=allowed_models)
parser.add_argument('--learning_rate',
help='learning rate during learning',
type=float,
default=0.01)
parser.add_argument('--dropout',
help='dropout during learning',
type=float,
default=0.05)
parser.add_argument('--hidden_units',
help='List of number of nodes in hidden layers',
nargs='+',
type=int,
default=[256, 128])
parser.add_argument('--epochs',
help='Number of epochs for training',
default=3,
type=int)
parser.add_argument('--gpu', help='Enable GPU', action='store_true')
args = parser.parse_args()
# Describe directories relative to working directory
data_dir = args.data_dir
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'
save_dir = args.save_dir
# Set variables for console input arguments
model_arch = args.arch
model_hidden_units = args.hidden_units
learning_rate = args.learning_rate
drop = args.dropout
#Testing area
print('Data directory: ' + data_dir)
print('hidden units: ' + str(args.hidden_units))
print('Save directory: ' + save_dir)
print('Architecture: ' + args.arch)
#create save directory if not existing
fu.create_directory(save_dir)
# Loading Pre-Trained model dependent on console input arch
model = models.__getattribute__(model_arch)(pretrained=True)
# Freeze parameters so we don't backprop through them
for param in model.parameters():
param.requires_grad = False
# Create the network, define the criterion and optimizer
model.fc = fu.Network(model.fc.in_features, 102, model_hidden_units, drop)
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.fc.parameters(), lr=learning_rate)
device = torch.device(
'cuda' if torch.cuda.is_available() and args.gpu == True else 'cpu')
print('Device is: ', device)
epochs = args.epochs
print_every = 50
running_loss = 0
steps = 0
train_loader, test_loader, valid_loader, train_data, test_data, valid_data = load_transform.load_transform(
data_dir, train_dir, valid_dir, test_dir)
fu.train(device, model, epochs, criterion, optimizer, print_every,
train_loader, test_loader, valid_loader)
fu.save_checkpoint(model, model_arch, epochs, criterion, optimizer,
train_data, save_dir)
return model, test_loader, criterion
def predict_review(trie):
review_typed = ''
while True:
next_chars = stdin.readline().rstrip('\r\n')
if len(next_chars) == 0:
return
review_typed += next_chars
preds = f.predict(review_typed, trie)
try:
print('{}{}{}{}{}'.format(preds[0], sep, preds[1], sep, preds[2]))
stdout.flush()
except Exception:
print(
' {} {} '.format(sep, sep)
) # in the event that anything weird happens, pass three blank predictions
stdout.flush()
trie = f.train(para.training_sample_size, para.training_file)
while True:
try:
print('Predicting review.', file=stderr)
predict_review(trie)
except EOFError:
sys.exit(0)
except Exception as e:
print(e, file=stderr)
sys.exit(1)
def main(index, args):
device = xm.xla_device()
gen_net = Generator(args).to(device)
dis_net = Discriminator(args).to(device)
enc_net = Encoder(args).to(device)
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
enc_net.apply(weights_init)
ae_recon_optimizer = torch.optim.Adam(
itertools.chain(enc_net.parameters(), gen_net.parameters()),
args.ae_recon_lr, (args.beta1, args.beta2))
ae_reg_optimizer = torch.optim.Adam(
itertools.chain(enc_net.parameters(), gen_net.parameters()),
args.ae_reg_lr, (args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(dis_net.parameters(), args.d_lr,
(args.beta1, args.beta2))
gen_optimizer = torch.optim.Adam(gen_net.parameters(), args.g_lr,
(args.beta1, args.beta2))
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
valid_loader = dataset.valid
para_loader = pl.ParallelLoader(train_loader, [device])
fid_stat = str(pathlib.Path(
__file__).parent.absolute()) + '/fid_stat/fid_stat_cifar10_test.npz'
if not os.path.exists(fid_stat):
download_stat_cifar10_test()
is_best = True
args.num_epochs = np.ceil(args.num_iter / len(train_loader))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0,
args.num_iter / 2, args.num_iter)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0,
args.num_iter / 2, args.num_iter)
ae_recon_scheduler = LinearLrDecay(ae_recon_optimizer, args.ae_recon_lr, 0,
args.num_iter / 2, args.num_iter)
ae_reg_scheduler = LinearLrDecay(ae_reg_optimizer, args.ae_reg_lr, 0,
args.num_iter / 2, args.num_iter)
# initial
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
enc_net.load_state_dict(checkpoint['enc_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
ae_recon_optimizer.load_state_dict(checkpoint['ae_recon_optimizer'])
ae_reg_optimizer.load_state_dict(checkpoint['ae_reg_optimizer'])
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
logs_dir = str(pathlib.Path(__file__).parent.parent) + '/logs'
args.path_helper = set_log_dir(logs_dir, args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.num_epochs)),
desc='total progress'):
lr_schedulers = (gen_scheduler, dis_scheduler, ae_recon_scheduler,
ae_reg_scheduler)
train(device, args, gen_net, dis_net, enc_net, gen_optimizer,
dis_optimizer, ae_recon_optimizer, ae_reg_optimizer, para_loader,
epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == args.num_epochs - 1:
fid_score = validate(args, fid_stat, gen_net, writer_dict,
valid_loader)
logger.info(f'FID score: {fid_score} || @ epoch {epoch}.')
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
save_checkpoint(
{
'epoch': epoch + 1,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'enc_state_dict': enc_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'ae_recon_optimizer': ae_recon_optimizer.state_dict(),
'ae_reg_optimizer': ae_reg_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
data = getData(ifPlotData=True)
## 3.2
A = rg.regressAData()
B = rg.regressBData()
v = rg.validateData()
X_train = A[0]
Y_train = A[1]
X_test = B[0]
Y_test = B[1]
X_val = v[0]
Y_val = v[1]
M_list = [1,2,5,10]
lambda_list = [1e-5,1e-2,0.1,1]
mod_A = f.model_eval(X_train,Y_train,M_list,lambda_list)
mod_B = f.model_select(X_val,Y_val,M_list,lambda_list, mod_A)
weights, errors, grid = f.train(B, v, M_list, lambda_list)
sse = fun.SSE(X_test,Y_test,1)
w = weights[0,0]
sse(w)
fun.graph_reg(X_train,Y_train,1,w)
w, RSS_final = eval(B[0],B[1], weights[1,0], M = 2, l = 1e-7, plot = True)
siallcnn.to(device)
criterion = nn.CrossEntropyLoss()
all_optimizer = optim.Adam(allcnn.parameters(), lr=learning_rate)
siall_optimizer = optim.Adam(siallcnn.parameters(), lr=learning_rate)
siall_train_loss = []
siall_train_acc = []
siall_test_loss = []
siall_test_acc = []
siall_test_loss_sc = []
siall_test_acc_sc = []
for epoch in range(1, nb_epochs + 1):
train_l, train_a = train(siallcnn, train_loader, siall_optimizer,
criterion, epoch, batch_log, device)
test_l, test_a = test(siallcnn, test_loader, criterion, epoch, batch_log,
device)
siall_train_loss.append(train_l)
siall_train_acc.append(train_a)
siall_test_loss.append(test_l)
siall_test_acc.append(test_a)
test_l_sc, test_a_sc = test(siallcnn, test_loader_sc, criterion, epoch,
batch_log, device)
siall_test_loss_sc.append(test_l_sc)
siall_test_acc_sc.append(test_a_sc)
with open("si_allcnn_log_scaled.txt", "w") as output:
output.write(str(siall_train_loss))
output.write(str(siall_train_acc))
output.write(str(siall_test_loss))
def main(args):
logging.info(args)
writer = SummaryWriter(args.save_dir)
best_prec1, best_ata, cln_best_prec1 = 0, 0, 0
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
cudnn.benchmark = True
setup_seed(args.seed)
model = get_network_func(args.norm_module)
model.cuda()
start_epoch = 0
if args.resume:
print('resume from checkpoint')
checkpoint = torch.load(os.path.join(args.save_dir, 'model.pt'))
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
# dataset process
train_datasets = datasets.CIFAR10(root=args.data,
train=True,
transform=transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(32, 4),
transforms.ToTensor()
]),
download=True)
test_dataset = datasets.CIFAR10(root=args.data,
train=False,
transform=transforms.Compose(
[transforms.ToTensor()]),
download=True)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=2,
pin_memory=True)
valid_size = 0.1
indices = list(range(len(train_datasets)))
split = int(np.floor(valid_size * len(train_datasets)))
np.random.seed(args.seed)
np.random.shuffle(indices)
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = torch.utils.data.DataLoader(train_datasets,
batch_size=args.batch_size,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(train_datasets,
batch_size=args.batch_size,
sampler=valid_sampler)
decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=decreasing_lr,
gamma=0.1)
print('starting adv training')
all_result = {}
train_acc = []
ta0 = []
ata0 = []
ta1 = []
ata1 = []
test_ta0 = []
test_ata0 = []
test_ta1 = []
test_ata1 = []
if os.path.exists(args.save_dir) is not True:
os.mkdir(args.save_dir)
for epoch in tqdm(range(args.epochs)):
if epoch < start_epoch:
scheduler.step()
continue
print(optimizer.state_dict()['param_groups'][0]['lr'])
acc, train_loss = train(args, train_loader, model, criterion,
optimizer, epoch)
writer.add_scalar('acc/train_acc', acc, epoch)
writer.add_scalar('loss/train_loss', train_loss, epoch)
# evaluate on validation set
tacc_clean, tloss_clean = validate(args, val_loader, model, criterion,
0)
writer.add_scalar('acc/tacc_clean', tacc_clean, epoch)
writer.add_scalar('loss/tloss_clean', tloss_clean, epoch)
atacc_clean, atloss_clean = validate_adv(args, val_loader, model,
criterion, 0)
writer.add_scalar('acc/atacc_clean', atacc_clean, epoch)
writer.add_scalar('loss/atloss_clean', atloss_clean, epoch)
tacc_adv, tloss_adv = validate(args, val_loader, model, criterion, 1)
writer.add_scalar('acc/tacc_adv', tacc_adv, epoch)
writer.add_scalar('loss/tloss_adv', tloss_adv, epoch)
atacc_adv, atloss_adv = validate_adv(args, val_loader, model,
criterion, 1)
writer.add_scalar('acc/atacc_adv', atacc_adv, epoch)
writer.add_scalar('loss/atloss_adv', atloss_adv, epoch)
# evaluate on test set
# clean branch
test_tacc_clean, test_tloss_clean = validate(args, test_loader, model,
criterion, 0)
writer.add_scalar('acc/test_tacc_clean', test_tacc_clean, epoch)
writer.add_scalar('loss/test_tloss_clean', test_tloss_clean, epoch)
test_atacc_clean, test_atloss_clean = validate_adv(
args, test_loader, model, criterion, 0)
writer.add_scalar('acc/test_atacc_clean', test_atacc_clean, epoch)
writer.add_scalar('loss/test_atloss_clean', test_atloss_clean, epoch)
# adv branch
test_tacc_adv, test_tloss_adv = validate(args, test_loader, model,
criterion, 1)
writer.add_scalar('acc/test_tacc_adv', test_tacc_adv, epoch)
writer.add_scalar('loss/test_tloss_adv', test_tloss_adv, epoch)
test_atacc_adv, test_atloss_adv = validate_adv(args, test_loader,
model, criterion, 1)
writer.add_scalar('acc/test_atacc_adv', test_atacc_adv, epoch)
writer.add_scalar('loss/test_atloss_adv', test_atloss_adv, epoch)
scheduler.step()
train_acc.append(acc)
ta0.append(tacc_clean)
ata0.append(atacc_clean)
ta1.append(tacc_adv)
ata1.append(atacc_adv)
test_ta0.append(test_tacc_clean)
test_ata0.append(test_atacc_clean)
test_ta1.append(test_tacc_adv)
test_ata1.append(test_atacc_adv)
# remember best prec@1 and save checkpoint
is_best = tacc_adv > best_prec1
best_prec1 = max(tacc_adv, best_prec1)
ata_is_best = atacc_adv > best_ata
best_ata = max(atacc_adv, best_ata)
cln_is_best = tacc_clean > cln_best_prec1
cln_best_prec1 = max(tacc_clean, cln_best_prec1)
if is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'best_model.pt'))
if cln_is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'clean_best_model.pt'))
if ata_is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'ata_best_model.pt'))
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'model.pt'))
if epoch and epoch % 10 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, f'model_epoch{epoch}.pt'))
all_result['train'] = train_acc
all_result['test_ta0'] = test_ta0
all_result['test_ata0'] = test_ata0
all_result['test_ta1'] = test_ta1
all_result['test_ata1'] = test_ata1
all_result['ta0'] = ta0
all_result['ata0'] = ata0
all_result['ta1'] = ta1
all_result['ata1'] = ata1
pickle.dump(all_result,
open(os.path.join(args.save_dir, 'result_c.pkl'), 'wb'))
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
# Setting up the resnet model
if args.pretrained:
resnet = torchvision.models.resnet50(pretrained=True, progress=True)
else:
resnet = torchvision.models.resnet50(pretrained=False, progress=True)
num_features = resnet.fc.in_features
resnet.fc = nn.Linear(num_features, args.num_classes)
resnet = resnet.cuda()
# Setting up the optimizer
if args.optimizer == 'sgd':
optimizer = optim.SGD(resnet.parameters(),
lr=args.lr,
weight_decay=1e-4)
elif args.optimizer == 'sgd_momentum':
optimizer = optim.SGD(resnet.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=1e-4)
elif args.optimizer == 'adam':
optimizer = optim.Adam(
filter(lambda p: p.requires_grad, resnet.parameters()), args.g_lr,
(args.beta1, args.beta2))
else:
optimizer = None
assert optimizer != None
criterion = nn.CrossEntropyLoss()
if args.percentage == 1.0:
train_data, val_data, test_data = get_train_validation_test_data(
args.train_csv_path, args.train_img_path, args.val_csv_path,
args.val_img_path, args.test_csv_path, args.test_img_path)
else:
train_data = get_label_unlabel_dataset(args.train_csv_path,
args.train_img_path,
args.percentage)
_, val_data, test_data = get_train_validation_test_data(
args.train_csv_path, args.train_img_path, args.val_csv_path,
args.val_img_path, args.test_csv_path, args.test_img_path)
train_loader = DataLoader(train_data,
batch_size=args.train_batch_size,
shuffle=True,
drop_last=True,
num_workers=args.num_workers)
val_loader = DataLoader(val_data,
batch_size=args.eval_batch_size,
shuffle=True,
drop_last=True,
num_workers=args.num_workers)
test_loader = DataLoader(test_data,
batch_size=args.eval_batch_size,
shuffle=True,
drop_last=True,
num_workers=args.num_workers)
print('Training Datasize:', len(train_data))
start_epoch = 0
best_acc1 = 0
best_acc2 = 0
best_acc3 = 0
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint_last.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
resnet.load_state_dict(checkpoint['resnet_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
start = time.time()
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
best_curr_acc1, best_curr_acc2, best_curr_acc3 = train(
args, resnet, optimizer, criterion, train_loader, val_loader,
epoch, writer_dict, best_acc1, best_acc2, best_acc3)
best_acc1, best_acc2, best_acc3 = best_curr_acc1, best_curr_acc2, best_curr_acc3
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
val_acc = get_val_acc(val_loader, resnet)
logger.info(f'Validation Accuracy {val_acc} || @ epoch {epoch}.')
save_checkpoint(
{
'epoch': epoch + 1,
'resnet_state_dict': resnet.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'path_helper': args.path_helper
},
False,
False,
False,
args.path_helper['ckpt_path'],
filename='checkpoint_last.pth')
end = time.time()
final_val_acc = get_val_acc(val_loader, resnet)
final_test_acc = get_val_acc(test_loader, resnet)
time_elapsed = end - start
print('\n Final Validation Accuracy:', final_val_acc.data,
'\n Final Test Accuracy:', final_test_acc.data, '\n Time Elapsed:',
time_elapsed, 'seconds.')
def main():
# for reproduciblity
random_seed = 2020
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed(random_seed)
cudnn.benchmark = config.CUDNN.BENCHMARK
torch.backends.cudnn.deterministic = config.CUDNN.DETERMINISTIC
torch.backends.cudnn.enabled = config.CUDNN.ENABLED
args = parse_args()
reset_config(config, args)
# model loading
model = get_pose_net(config, is_train=True)
# model = model.half()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
logging.info(f"Training on CUDA: {torch.cuda.is_available()}")
model = model.to("cuda" if torch.cuda.is_available() else "cpu")
# Data loading process
train_dataset = coco(
config,
config.DATASET.ROOT,
config.DATASET.TRAIN_SET,
is_train=True,
is_eval=False,
transform=tfms.RandomErasing(p=0.8, scale=(0.5, 0.5)),
)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.TRAIN.BATCH_SIZE,
shuffle=True,
drop_last=True)
valid_dataset = coco(config,
config.DATASET.ROOT,
config.DATASET.TEST_SET,
is_train=False,
is_eval=True,
transform=None)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=config.TEST.BATCH_SIZE,
shuffle=False,
drop_last=False)
start_epoch = config.TRAIN.BEGIN_EPOCH
optimizer = torch.optim.Adam(model.parameters(),
lr=5e-4,
weight_decay=1e-5,
eps=1e-5)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[90, 120],
gamma=0.1)
losschecker = BestLossChecker()
if config.MODEL.CHECKPOINT is not None:
info = load_checkpoint(config.MODEL.CHECKPOINT)
if info is not None:
start_epoch, model_dic, optim_dic, sched_dic = info
try:
model.load_state_dict(model_dic)
logging.info('Model Loaded.')
optimizer.load_state_dict(optim_dic)
logging.info('Optimizer Loaded.')
if sched_dic is not None:
scheduler.load_state_dict(sched_dic)
else:
scheduler.last_epoch = start_epoch
scheduler.optimizer.load_state_dict(optim_dic)
logging.info('Scheduler Loaded.')
logging.info('All Weights Loaded...\n')
except Exception as e:
start_epoch = config.TRAIN.BEGIN_EPOCH
logging.info('Model shape is different. Plz check.')
logging.info('Starts with init weights...\n')
end = time.time()
logging.info('Training Ready\n')
for epoch in range(start_epoch, config.TRAIN.END_EPOCH):
if epoch == 10:
config.TEST.FLIP_TEST = True
if epoch % 5 == 0 and epoch != 0:
result = evaluate(config, model, valid_loader)
if epoch % 100 == 0 and epoch != 0:
with open(f'{config.result_dir}/data.json', 'w') as f:
json.dump(result, f)
os.makedirs(config.result_dir, exist_ok=True)
valid_dataset.keypoint_eval(result, config.result_dir + '/pred')
valid_dataset.keypoint_eval(
'./data/input_pose_path/keypoints_valid2017_results.json',
config.result_dir + '/ori/')
end = time.time()
losses = train(config,
epoch=epoch,
loader=train_loader,
model=model,
optimizer=optimizer)
total_loss, hm_loss, coord_loss = losses
is_best = losschecker.update(epoch, total_loss, hm_loss, coord_loss)
try:
state_dict = model.module.state_dict()
except Exception as e:
state_dict = model.state_dict()
save_checkpoint(
{
'epoch': epoch,
'model': get_model_name(config),
'state_dict': state_dict,
'optimizer': optimizer.state_dict(),
}, is_best, "./weights_2")
scheduler.step()
spent = time.time() - end
hour = int(spent // 3600)
min = int((spent - hour * 3600) // 60)
second = (spent - hour * 3600 - min * 60)
logging.info(
f"Epoch {epoch} taken {hour:d}h{min:2d}m {second:2.3f}s\n")
end = time.time()
logging.info('Reading train set')
ncs = read_ncs(args.p2tc)
logging.info('Creating vector for training instances')
X, Y = get_vectors(ncs, gensim_w2v_model)
if model_config.poly_degree > 1:
X = get_poly_features(X, model_config.poly_degree)
logging.info('Creating batches')
in_batches, tar_batches = create_batch(X, Y, model_config.batch_size)
logging.info('Creating the regression model')
model, optimizer, criterion = build_model(X, Y)
logging.info('Training')
train(in_batches, tar_batches, model, model_config.nb_epochs, optimizer,
criterion)
logging.info('Calculating regression-based scores')
reg_score = regression_score(eval_ncs, gensim_w2v_model, model)
write_score(eval_ncs, reg_score, args.p2out + 'reg_scores.csv')
print('Spearman rho bet. inv human score and regression',
scipy.stats.spearmanr(reg_score, eval_scores_inv))
print('Spearman rho bet. inv human score and additive score',
scipy.stats.spearmanr(additive_score, eval_scores_inv))
if args.rank == 'true':
print('Ranking based on regression model: ',
rank_with_score(eval_ncs, reg_score))
print('Ranking based on additive model: ',
rank_with_score(eval_ncs, additive_score))
from sklearn import tree
from variables import *
from functions import train,make_conversions,output_data,test
# train the program
name,Pclass,Sex,SibSp,Parch,Embarked,Survived = train()
make_conversions(Pclass,Sex,SibSp,Parch,Embarked)
# test the program, this recieved the inputs from the test set
name,Pclass,Sex,SibSp,Parch,Embarked = test()
make_conversions(Pclass,Sex,SibSp,Parch,Embarked)
# fill x with all possible parameters for each person
for i in range(0,len(name)):
z.append(pclass_conv[i])
z.append(sex_conv[i])
z.append(sibsp_conv[i])
z.append(parch_conv[i])
z.append(embarked_conv[i])
x.append(z)
z = []
# apply to sklearns function
y = Survived
clf = tree.DecisionTreeClassifier()
clf = clf.fit(x,y)
# output a csv to compare results to the original training dataset
output_data(name,Pclass,Sex,SibSp,Parch,Survived,Embarked,clf)
resnet.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(resnet.parameters(), lr = learning_rate)
all_train_loss=[]
all_train_acc = []
all_test_loss = []
all_test_acc = []
dynamics = []
for epoch in range(1, nb_epochs + 1):
train_l, train_a = train(resnet, train_loader, optimizer, criterion, epoch, batch_log, device)
train_l, train_a = test(resnet, train_loader, criterion, epoch, batch_log, device)
all_train_loss.append(train_l)
all_train_acc.append(train_a)
test_l, test_a = test(resnet, test_loader_sc, criterion, epoch, batch_log, device)
all_test_loss.append(test_l)
all_test_acc.append(test_a)
dynamics.append({
"epoch": epoch,
"train_loss": train_l,
"train_acc": train_a,
"test_loss": test_l,
"test_acc": test_a
})
dest='dropout',
action='store',
type=int,
default=0.05)
# parser.add_argument('--lr' , dest='lr' , action ='store' , type = int, default= 0.001)
parser.add_argument('--save_directory',
dest='save_directory',
action='store',
default='./checkpoint.pth')
parser = parser.parse_args()
epochs = parser.epochs
lr = parser.lr
structure = parser.structure
dropout = parser.dropout
hidden_layer1 = parser.hidden_layer1
hidden_layer2 = parser.hidden_layer2
power = parser.gpu
train_loaders, valid_loaders, test_loaders = functions.read_data(data_dir)
## load the model
model, optimizer, criterion = functions.model_setup(structure, epochs, dropout,
hidden_layer1,
hidden_layer2, lr, power)
## train the model
functions.train(model, epochs, criterion, optimizer, train_loaders,
valid_loaders, power)
## save the model
functions.save_checkpoint(path, hidden_layer1, hidden_layer2, dropout, lr,
epochs)
#!/usr/bin/env python3
import sys
import numpy as np
from functions import train, test
#Load in files
training_file = sys.argv[1]
testing_file = sys.argv[2]
#Convert to 1D array if necessary
train_test = [np.loadtxt(training_file), np.loadtxt(testing_file)]
for i in range(0, 2):
if len(train_test[i].shape) < 2:
train_test[i] = np.array([train_test[i]])
tree = []
train(train_test[0], tree, 1)
#indices = np.argsort(tree,axis=0)
#for i in indices[:,0]:
#print(tree[i])
test(train_test[1], tree)
models = [
kanazawa(f_in, ratio, nratio, srange=0),
kanazawa(f_in, ratio, nratio, srange),
kanazawa_big(f_in, ratio, nratio, srange=0),
kanazawa(f_in, ratio=2**(1 / 3), nratio=6, srange=0),
kanazawa(f_in, ratio=2**(1 / 3), nratio=6, srange=2)
]
for m, model in enumerate(models):
print("trial {}, model {}".format(ii, m))
model.to(device)
model_log = open("mnist_trained_model_{}_{}_k.pickle".format(m, ii),
"wb")
for epoch in range(1, nb_epochs + 1):
train_l, train_a = train(model, train_loader, learning_rate,
criterion, epoch, batch_log, device)
train_l, train_a = test(model, train_loader, criterion, epoch,
batch_log, device)
dyn = {"epoch": epoch, "train_loss": train_l, "train_acc": train_a}
pickle.dump(dyn, model_log)
pickle.dump(model, model_log)
model_log.close()
#lists of last test loss and acc for each scale with model ii
s_test_loss = []
s_test_acc = []
for s in scales:
test_transf = transforms.Compose([
transforms.Resize(40),
RandomRescale(size=40, scales=(s, s), sampling="uniform"),
transforms.ToTensor(),
def main():
args = parse_args()
update_config(cfg, args)
# cudnn related setting
cudnn.benchmark = cfg.CUDNN.BENCHMARK
torch.backends.cudnn.deterministic = cfg.CUDNN.DETERMINISTIC
torch.backends.cudnn.enabled = cfg.CUDNN.ENABLED
# Set the random seed manually for reproducibility.
np.random.seed(cfg.SEED)
torch.manual_seed(cfg.SEED)
torch.cuda.manual_seed_all(cfg.SEED)
# Loss
criterion = CrossEntropyLoss(cfg.MODEL.NUM_CLASSES).cuda()
# model and optimizer
print(f"Definining network with {cfg.MODEL.LAYERS} layers...")
model = Network(cfg.MODEL.INIT_CHANNELS, cfg.MODEL.NUM_CLASSES, cfg.MODEL.LAYERS, criterion, primitives_2,
drop_path_prob=cfg.TRAIN.DROPPATH_PROB)
model = model.cuda()
# weight params
arch_params = list(map(id, model.arch_parameters()))
weight_params = filter(lambda p: id(p) not in arch_params,
model.parameters())
# Optimizer
optimizer = optim.Adam(
weight_params,
lr=cfg.TRAIN.LR
)
# resume && make log dir and logger
if args.load_path and os.path.exists(args.load_path):
checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint_best.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
# load checkpoint
begin_epoch = checkpoint['epoch']
last_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
best_acc1 = checkpoint['best_acc1']
optimizer.load_state_dict(checkpoint['optimizer'])
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info("=> loaded checkpoint '{}'".format(checkpoint_file))
else:
exp_name = args.cfg.split('/')[-1].split('.')[0]
args.path_helper = set_path('logs_search', exp_name)
logger = create_logger(args.path_helper['log_path'])
begin_epoch = cfg.TRAIN.BEGIN_EPOCH
best_acc1 = 0.0
last_epoch = -1
logger.info(args)
logger.info(cfg)
# copy model file
this_dir = os.path.dirname(__file__)
shutil.copy2(
os.path.join(this_dir, 'models', cfg.MODEL.NAME + '.py'),
args.path_helper['ckpt_path'])
# Datasets and dataloaders
# The toy dataset is downloaded with 10 items for each partition. Remove the sample_size parameters to use the full toy dataset
asv_train, asv_dev, asv_eval = asv_toys(sample_size=10)
train_dataset = asv_train #MNIST('mydata', transform=totensor, train=True, download=True)
val_dataset = asv_dev #MNIST('mydata', transform=totensor, train=False, download=True)
train_loader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=cfg.TRAIN.BATCH_SIZE,
num_workers=cfg.DATASET.NUM_WORKERS,
pin_memory=True,
shuffle=True,
drop_last=True,
)
print(f'search.py: Train loader of {len(train_loader)} batches')
print(f'Tot train set: {len(train_dataset)}')
val_loader = torch.utils.data.DataLoader(
dataset=val_dataset,
batch_size=cfg.TRAIN.BATCH_SIZE,
num_workers=cfg.DATASET.NUM_WORKERS,
pin_memory=True,
shuffle=True,
drop_last=True,
)
print(f'search.py: Val loader of {len(val_loader)} batches')
print(f'Tot val set {len(val_dataset)}')
test_dataset = asv_eval #MNIST('mydata', transform=totensor, train=False, download=True)
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=cfg.TRAIN.BATCH_SIZE,
num_workers=cfg.DATASET.NUM_WORKERS,
pin_memory=True,
shuffle=True,
drop_last=True,
)
# training setting
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': begin_epoch * len(train_loader),
'valid_global_steps': begin_epoch // cfg.VAL_FREQ,
}
# training loop
architect = Architect(model, cfg)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, cfg.TRAIN.END_EPOCH, cfg.TRAIN.LR_MIN,
last_epoch=last_epoch
)
for epoch in tqdm(range(begin_epoch, cfg.TRAIN.END_EPOCH), desc='search progress'):
model.train()
genotype = model.genotype()
logger.info('genotype = %s', genotype)
if cfg.TRAIN.DROPPATH_PROB != 0:
model.drop_path_prob = cfg.TRAIN.DROPPATH_PROB * epoch / (cfg.TRAIN.END_EPOCH - 1)
train(cfg, model, optimizer, train_loader, val_loader, criterion, architect, epoch, writer_dict)
if epoch % cfg.VAL_FREQ == 0:
# get threshold and evaluate on validation set
acc = validate_identification(cfg, model, test_loader, criterion)
# remember best acc@1 and save checkpoint
is_best = acc > best_acc1
best_acc1 = max(acc, best_acc1)
# save
logger.info('=> saving checkpoint to {}'.format(args.path_helper['ckpt_path']))
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optimizer.state_dict(),
'arch': model.arch_parameters(),
'genotype': genotype,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'], 'checkpoint_{}.pth'.format(epoch))
lr_scheduler.step(epoch)
from functions import load_data, train, evaluate, plot_result
from keras.models import Sequential
from keras.layers import Dense, Dropout, LeakyReLU
(x_train, y_train), (x_test, y_test) = load_data()
model = Sequential()
model.add(Dense(784, activation='relu', input_shape=x_train.shape[1:]))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dropout(rate=0.25))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(10, activation='softmax'))
model.compile(
loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy']
)
history = train(model, x_train, y_train)
plot_result(history)
evaluate(model, x_test, y_test)
test_iter = gdata.DataLoader(
gdata.ArrayDataset(*d2l.preprocess_imdb(test_data, vocab)), batch_size)
## 4.指定参数并加载模型
embed_size, num_hiddens, num_layers, ctx = 100, 200, 2, d2l.try_all_gpus()
net = BiRNN(vocab, embed_size, num_hiddens, num_layers) #实例化一个双向RNN
net.initialize(init.Xavier(), ctx=ctx) #对模型进行初始化
## 4.1 加载词向量预训练集
print("加载预训练的词向量，若电脑没有预训练数据集将会自动从网上下载")
glove_embedding = text.embedding.create(
'glove', pretrained_file_name='glove.6B.100d.txt',
vocabulary=vocab) #其中的100指的是词向量长度为100
print("加载与训练的词向量 ok")
## 4.2 初试化模型参数
net.embedding.weight.set_data(glove_embedding.idx_to_vec)
net.embedding.collect_params().setattr('grad_req', 'null')
## 5. 指定损失函数下降速度，训练轮数，并训练模型
lr, num_epochs = 0.01, 5
trainer = gluon.Trainer(net.collect_params(), 'adam',
{'learning_rate': lr})
loss = gloss.SoftmaxCrossEntropyLoss()
loss_list, train_acc_list, test_acc_list, time_list = d2l.train(
train_iter, test_iter, net, loss, trainer, ctx, num_epochs)
## 5.2 打印损失函数下降图
plt.plot(loss_list) # 损失函数下降数组
plt.legend("loss") # 可省略，图像上的字（可认为标题）设置loss
plt.show() # 显示图像
# 6. 调用predict进行测试
test_str = "this movie is so great" # 测试一条语句看看是
print("测试语句判断结果是：", predict(net, vocab, test_str))
import functions import classes pic_file = "../data/monalisa.jpg" s_o_m = functions.initialize_som() image = classes.Dataset(pic_file) functions.train(image,s_o_m) functions.save_som(s_o_m,pic_file) functions.draw(s_o_m) functions.reproduce(image,s_o_m,image.size[0],image.size[1]) # import after database updated with new SOM import db_check db_check.similar_som()
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
torch.cuda.manual_seed_all(args.random_seed)
np.random.seed(args.random_seed)
random.seed(args.random_seed)
torch.backends.cudnn.deterministic = True
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# epoch number for dis_net
dataset = datasets.ImageDataset(args, cur_img_size=8)
train_loader = dataset.train
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter / len(train_loader))
else:
args.max_iter = args.max_epoch * len(train_loader)
args.max_epoch = args.max_epoch * args.n_critic
# import network
gen_net = eval('models.' + args.gen_model + '.Generator')(args=args).cuda()
dis_net = eval('models.' + args.dis_model +
'.Discriminator')(args=args).cuda()
gen_net.set_arch(args.arch, cur_stage=2)
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform_(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
gpu_ids = [i for i in range(int(torch.cuda.device_count()))]
gen_net = torch.nn.DataParallel(gen_net.to("cuda:0"), device_ids=gpu_ids)
dis_net = torch.nn.DataParallel(dis_net.to("cuda:0"), device_ids=gpu_ids)
gen_net.module.cur_stage = 0
dis_net.module.cur_stage = 0
gen_net.module.alpha = 1.
dis_net.module.alpha = 1.
# set optimizer
if args.optimizer == "adam":
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
elif args.optimizer == "adamw":
gen_optimizer = AdamW(filter(lambda p: p.requires_grad,
gen_net.parameters()),
args.g_lr,
weight_decay=args.wd)
dis_optimizer = AdamW(filter(lambda p: p.requires_grad,
dis_net.parameters()),
args.g_lr,
weight_decay=args.wd)
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
elif args.fid_stat is not None:
fid_stat = args.fid_stat
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (64, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path)
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
# avg_gen_net = deepcopy(gen_net)
# avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = checkpoint['gen_avg_param']
# del avg_gen_net
cur_stage = cur_stages(start_epoch, args)
gen_net.module.cur_stage = cur_stage
dis_net.module.cur_stage = cur_stage
gen_net.module.alpha = 1.
dis_net.module.alpha = 1.
args.path_helper = checkpoint['path_helper']
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
def return_states():
states = {}
states['epoch'] = epoch
states['best_fid'] = best_fid_score
states['gen_state_dict'] = gen_net.state_dict()
states['dis_state_dict'] = dis_net.state_dict()
states['gen_optimizer'] = gen_optimizer.state_dict()
states['dis_optimizer'] = dis_optimizer.state_dict()
states['gen_avg_param'] = gen_avg_param
states['path_helper'] = args.path_helper
return states
# train loop
for epoch in range(start_epoch + 1, args.max_epoch):
train(
args,
gen_net,
dis_net,
gen_optimizer,
dis_optimizer,
gen_avg_param,
train_loader,
epoch,
writer_dict,
fixed_z,
)
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
fid_score = validate(
args,
fixed_z,
fid_stat,
epoch,
gen_net,
writer_dict,
)
logger.info(f'FID score: {fid_score} || @ epoch {epoch}.')
load_params(gen_net, backup_param)
is_best = False
if epoch == 1 or fid_score < best_fid_score:
best_fid_score = fid_score
is_best = True
if is_best or epoch % 1 == 0:
states = return_states()
save_checkpoint(states,
is_best,
args.path_helper['ckpt_path'],
filename=f'checkpoint_epoch_{epoch}.pth')
