def use_cuda(enabled, device_id=0):
"""Verifies if CUDA is available and sets default device to be device_id."""
if not enabled:
return None
assert torch.cuda.is_available(), 'CUDA is not available'
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.set_device(device_id)
return device_id
def tensors_default_to(host):
"""
Context manager to temporarily use Cpu or Cuda tensors in PyTorch.
:param str host: Either "cuda" or "cpu".
"""
assert host in ('cpu', 'cuda'), host
old_module, name = torch.Tensor().type().rsplit('.', 1)
new_module = 'torch.cuda' if host == 'cuda' else 'torch'
torch.set_default_tensor_type('{}.{}'.format(new_module, name))
try:
yield
finally:
torch.set_default_tensor_type('{}.{}'.format(old_module, name))
def initialize_pipeline(nlp, docs, golds, config, device):
nlp.add_pipe(nlp.create_pipe("tagger"))
nlp.add_pipe(nlp.create_pipe("parser"))
if config.multitask_tag:
nlp.parser.add_multitask_objective("tag")
if config.multitask_sent:
nlp.parser.add_multitask_objective("sent_start")
for gold in golds:
for tag in gold.tags:
if tag is not None:
nlp.tagger.add_label(tag)
if torch is not None and device != -1:
torch.set_default_tensor_type("torch.cuda.FloatTensor")
optimizer = nlp.begin_training(
lambda: golds_to_gold_tuples(docs, golds),
device=device,
subword_features=config.subword_features,
conv_depth=config.conv_depth,
bilstm_depth=config.bilstm_depth,
)
if config.pretrained_tok2vec:
_load_pretrained_tok2vec(nlp, config.pretrained_tok2vec)
return optimizer
eps= args.eps #1e-9
factor = args.factor #1
warmup= args.warmup #400
batchSize = args.batchSize #30
epochCount = args.epochCount #20
n_layers = args.n_layers #1
d_model_global = args.d_model_global #512
d_ff_global = args.d_ff_global #2048
h_global = args.h_global #8
dropout_global = args.dropout_global #0.1
sequence_length = args.sequence_length
# https://discuss.pytorch.org/t/global-gpu-flag/17195
use_gpu = lambda x=True: torch.set_default_tensor_type(torch.cuda.FloatTensor
if torch.cuda.is_available() and x
else torch.FloatTensor)
use_gpu()
class EncoderDecoder(nn.Module):
"""
A standard Encoder-Decoder architecture. Base for this and many
other models.
"""
def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):
super(EncoderDecoder, self).__init__()
self.encoder = encoder
self.decoder = decoder
self.src_embed = src_embed
def main():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=str, help="input trace")
parser.add_argument("starting_cache", type=int, help="starting cache size")
parser.add_argument("size_increment",
type=int,
help="increment in cache size")
parser.add_argument("max_cache", type=int, help="max cache size")
parser.add_argument("cache_descriptor",
type=str,
help="descriptor to construct cache object")
parser.add_argument("output", help="output file name", type=str)
parser.add_argument("-log",
"--log_file_prefix",
help="log files prefix to write intermediate results",
type=str,
default=None)
parser.add_argument(
"-css",
"--cold_start_skip",
help="number of requests to skip when evaluating hit rate",
type=int,
default=0)
parser.add_argument("-fc",
"--force_cpu",
help="force cpu execution for PyTorch",
action="store_true")
parser.add_argument("-sm",
"--size_meta",
help="add size to metadata for NN",
action="store_true")
parser.add_argument("-dm",
"--daytime_meta",
help="add daytime to metadata for NN",
action="store_true")
args = parser.parse_args()
with tqdm(total=args.max_cache, desc="Sizes processed") as pbar:
cur_size = args.starting_cache
with open(args.output, 'w') as f:
while cur_size <= args.max_cache:
desc = None
if args.cache_descriptor is not None:
with open(args.cache_descriptor) as f_desc:
desc = json.load(f_desc)
nn = None
if desc["nn_location"] != "":
with open(desc["nn_location"], "rb") as unpickle_file:
nn = pickle.load(unpickle_file)
if not args.force_cpu and torch.cuda.is_available():
print("Running on: GPU")
torch.set_default_tensor_type("torch.cuda.FloatTensor")
else:
print("Running on: CPU")
torch.set_default_tensor_type("torch.FloatTensor")
online = (desc["online_learning"] == "True")
cache = FeedforwardNNCacheFullTorch(
cur_size, nn, int(desc["counter_num"]),
float(desc["time_window"]),
int(desc["update_sample_size"]), online,
float(desc["cf_coef"]), float(desc["learning_rate"]),
int(desc["batch_size"]))
hit_rate = eval_cache_hit(
cache, args.input, args.cold_start_skip,
args.log_file_prefix + "_{}.log".format(cur_size))
f.write(f"{cur_size} {hit_rate}\n")
f.flush()
cur_size += args.size_increment
pbar.update(args.size_increment)
def test_with_epipolar_lines():
"""Unit test you will create for your RANSAC implementation.
It should take no arguments and it does not need to return anything,
but it **must** display the images when run.
Use the code in the jupyter notebook as an example for how to open the
image files and perform the necessary operations on them in our workflow.
Remember the steps are Harris, SIFT, match features, RANSAC fundamental matrix.
Display the proposed correspondences, the true inlier correspondences
found by RANSAC, and most importantly the epipolar lines in both of your images.
It should be clear that the epipolar lines intersect where the second image
was taken, and the true point correspondences should indeed be good matches.
"""
##############################
# TODO: Student code goes here
from feature_matching.SIFTNet import get_siftnet_features
from feature_matching.utils import load_image, PIL_resize, rgb2gray
import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
# Rushmore
image1 = load_image('../data/test_c.jpg')
image2 = load_image('../data/test_d.jpg')
scale_factor = 0.5
image1 = PIL_resize(image1, (int(
image1.shape[1] * scale_factor), int(image1.shape[0] * scale_factor)))
image2 = PIL_resize(image2, (int(
image2.shape[1] * scale_factor), int(image2.shape[0] * scale_factor)))
image1_bw = rgb2gray(image1)
image2_bw = rgb2gray(image2)
#convert images to tensor
tensor_type = torch.FloatTensor
torch.set_default_tensor_type(tensor_type)
to_tensor = transforms.ToTensor()
image_input1 = to_tensor(image1_bw).unsqueeze(0)
image_input2 = to_tensor(image2_bw).unsqueeze(0)
from feature_matching.HarrisNet import get_interest_points
from feature_matching.utils import show_interest_points
x1, y1, _ = get_interest_points(image_input1.float())
x2, y2, _ = get_interest_points(image_input2.float())
x1, x2 = x1.detach().numpy(), x2.detach().numpy()
y1, y2 = y1.detach().numpy(), y2.detach().numpy()
print('{:d} corners in image 1, {:d} corners in image 2'.format(
len(x1), len(x2)))
image1_features = get_siftnet_features(image_input1, x1, y1)
image2_features = get_siftnet_features(image_input2, x2, y2)
from feature_matching.student_feature_matching import match_features
matches, confidences = match_features(image1_features, image2_features, x1,
y1, x2, y2)
print('{:d} matches from {:d} corners'.format(len(matches), len(x1)))
from feature_matching.utils import show_correspondence_circles, show_correspondence_lines
# num_pts_to_visualize = len(matches)
num_pts_to_visualize = 100
c2 = show_correspondence_lines(image1, image2,
x1[matches[:num_pts_to_visualize, 0]],
y1[matches[:num_pts_to_visualize, 0]],
x2[matches[:num_pts_to_visualize, 1]],
y2[matches[:num_pts_to_visualize, 1]])
plt.figure()
plt.title('Proposed Matches')
plt.imshow(c2)
from proj3_code.ransac import ransac_fundamental_matrix
# print(image1_features.shape, image2_features.shape)
num_features = min([len(image1_features), len(image2_features)])
x0s = np.zeros((len(matches), 2))
x1s = np.zeros((len(matches), 2))
x0s[:, 0] = x1[matches[:, 0]]
x0s[:, 1] = y1[matches[:, 0]]
x1s[:, 0] = x2[matches[:, 1]]
x1s[:, 1] = y2[matches[:, 1]]
# print(image1_pts.shape)
F, matches_x0, matches_x1 = ransac_fundamental_matrix(x0s, x1s)
print(F)
# print(matches_x0)
# print(matches_x1)
from proj3_code.utils import draw_epipolar_lines
# Draw the epipolar lines on the images and corresponding matches
match_image = show_correspondence_lines(
image1, image2, matches_x0[:num_pts_to_visualize,
0], matches_x0[:num_pts_to_visualize, 1],
matches_x1[:num_pts_to_visualize, 0], matches_x1[:num_pts_to_visualize,
1])
plt.figure()
plt.title('True Matches')
plt.imshow(match_image)
draw_epipolar_lines(F, image1, image2, matches_x0, matches_x1)
def main():
import argparse
parser = argparse.ArgumentParser(
description="Pytorch Image CNN training from Configure Files")
parser.add_argument(
'--config_file',
required=True,
help="This scripts only accepts parameters from Json files")
input_args = parser.parse_args()
config_file = input_args.config_file
args = parse_config(config_file)
if args.name is None:
args.name = get_stem(config_file)
torch.set_default_tensor_type('torch.FloatTensor')
best_prec1 = 0
args.script_name = get_stem(__file__)
current_time_str = get_date_str()
if args.save_directory is None:
save_directory = get_dir(
os.path.join(project_root, 'ckpts', '{:s}'.format(args.name),
'{:s}-{:s}'.format(args.ID, current_time_str)))
else:
save_directory = get_dir(
os.path.join(project_root, 'ckpts', args.save_directory))
print("Save to {}".format(save_directory))
log_file = os.path.join(save_directory,
'log-{0}.txt'.format(current_time_str))
logger = log_utils.get_logger(log_file)
log_utils.print_config(vars(args), logger)
print_func = logger.info
print_func('ConfigFile: {}'.format(config_file))
args.log_file = log_file
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
if args.gpu is not None:
warnings.warn('You have chosen a specific GPU. This will completely '
'disable data parallelism.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
if args.pretrained:
print_func("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True,
num_classes=args.num_classes)
else:
print_func("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=False,
num_classes=args.num_classes)
if args.freeze:
model = CNN_utils.freeze_all_except_fc(model)
if args.gpu is not None:
model = model.cuda(args.gpu)
elif args.distributed:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
else:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
# # Update: here
# config = {'loss': {'type': 'simpleCrossEntropyLoss', 'args': {'param': None}}}
# criterion = get_instance(loss_funcs, 'loss', config)
# criterion = criterion.cuda(args.gpu)
criterion = nn.CrossEntropyLoss(ignore_index=-1).cuda(args.gpu)
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.lr_schedule:
print_func("Using scheduled learning rate")
scheduler = lr_scheduler.MultiStepLR(
optimizer, [int(i) for i in args.lr_schedule.split(',')],
gamma=0.1)
else:
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=args.lr_patience)
# optimizer = torch.optim.SGD(model.parameters(), args.lr,
# momentum=args.momentum,
# weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print_func("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
import collections
if isinstance(checkpoint, collections.OrderedDict):
load_state_dict(model,
checkpoint,
exclude_layers=['fc.weight', 'fc.bias'])
else:
load_state_dict(
model,
checkpoint['state_dict'],
exclude_layers=['module.fc.weight', 'module.fc.bias'])
print_func("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print_func("=> no checkpoint found at '{}'".format(args.resume))
return
else:
print_func(
"=> This script is for fine-tuning only, please double check '{}'".
format(args.resume))
print_func("Now using randomly initialized parameters!")
cudnn.benchmark = True
model_total_params = sum(p.numel() for p in model.parameters())
model_grad_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print_func("Total Parameters: {0}\t Gradient Parameters: {1}".format(
model_total_params, model_grad_params))
# Data loading code
val_dataset = get_instance(custom_datasets,
'{0}_val'.format(args.dataloader), args)
if val_dataset is None:
val_loader = None
else:
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
collate_fn=none_collate)
if args.evaluate:
validate(val_loader, model, criterion, args, print_func)
return
else:
train_dataset = get_instance(custom_datasets,
'{0}_train'.format(args.dataloader), args)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler,
collate_fn=none_collate)
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
if args.lr_schedule:
# CNN_utils.adjust_learning_rate(optimizer, epoch, args.lr)
scheduler.step()
current_lr = optimizer.param_groups[0]['lr']
print_func("Epoch: [{}], learning rate: {}".format(epoch, current_lr))
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch, args,
print_func)
# evaluate on validation set
if val_loader:
prec1, val_loss = validate(val_loader, model, criterion, args,
print_func)
else:
prec1 = 0
val_loss = 0
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
print("Current best performance: {}\t{:.3f}".format(epoch, best_prec1))
CNN_utils.save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
file_directory=save_directory,
epoch=epoch)
if not args.lr_schedule:
scheduler.step(val_loss)
import torch
import torch.nn as nn
import numpy as np
import pandas as pd
import d2lzh_pytorch as d2l
torch.set_default_tensor_type(torch.FloatTensor)
# 读取数据集
train_data = pd.read_csv('../../test/sources/data/kaggle_house/train.csv')
test_data = pd.read_csv('../../test/sources/data/kaggle_house/test.csv')
# 共1460个样本和80个特征
print(train_data.shape) # 输出 (1460, 81)
print(train_data.iloc[0:4, [0, 1, 2, 3, -3, -2, -1]]) # 后面的数组是指定输出的列
all_features = pd.concat(
(train_data.iloc[:, 1:-1], test_data.iloc[:, 1:])) # 第一列是id，不需要，训练集最后一个是标签
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
all_features[numeric_features] = all_features[numeric_features].apply(
lambda x: (x - x.mean()) / (x.std()))
# 标准化后，每个数值特征的均值变为0，所以可以直接用0来替换缺失值
all_features[numeric_features] = all_features[numeric_features].fillna(0)
# 将离散数值转变成指示特征
all_features = pd.get_dummies(all_features, dummy_na=True)
print(all_features.shape) # (2919,331) --> 将字段的枚举类型单独作为一特征，所列多了
# 转Numpy格式
def main():
print(CP_R + "e-Lab Segmentation Training Script" + CP_C)
#################################################################
# Initialization step
torch.manual_seed(args.seed)
torch.set_default_tensor_type('torch.FloatTensor')
#################################################################
# Acquire dataset loader object
# Normalization factor based on ResNet stats
prep_data = transforms.Compose([
#transforms.RandomCrop(900),
transforms.Resize(args.img_size, 0),
#transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
prep_target = transforms.Compose([
#transforms.RandomCrop(900),
transforms.Resize(args.img_size, 0),
#transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
if args.dataset == 'cs':
import data.segmented_data as segmented_data
print("{}Cityscapes dataset in use{}!!!".format(CP_G, CP_C))
else:
print("{}Invalid data-loader{}".format(CP_R, CP_C))
# Training data loader
data_obj_train = segmented_data.SegmentedData(root=args.datapath,
mode='train',
transform=prep_data,
target_transform=prep_target)
data_loader_train = DataLoader(data_obj_train,
batch_size=args.bs,
shuffle=True,
num_workers=args.workers)
data_len_train = len(data_obj_train)
# Testing data loader
data_obj_test = segmented_data.SegmentedData(root=args.datapath,
mode='test',
transform=prep_data,
target_transform=prep_target)
data_loader_test = DataLoader(data_obj_test,
batch_size=args.bs,
shuffle=True,
num_workers=args.workers)
data_len_test = len(data_obj_test)
n_classes = len(data_obj_train.class_name())
#################################################################
# Load model
print('{}{:=<80}{}'.format(CP_R, '', CP_C))
print('{}Models will be saved in: {}{}'.format(CP_Y, CP_C, str(args.save)))
if not os.path.exists(str(args.save)):
os.mkdir(str(args.save))
if args.saveAll:
if not os.path.exists(str(args.save) + '/all'):
os.mkdir(str(args.save) + '/all')
epoch = 0
if args.resume:
# Load previous model state
checkpoint = torch.load(args.save + '/model_resume.pt')
epoch = checkpoint['epoch']
model = checkpoint['model_def']
model.load_state_dict(checkpoint['state_dict'])
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
optimizer.load_stat_dict(checkpoint['optim_state'])
print('{}Loaded model from previous checkpoint epoch # {}()'.format(
CP_G, CP_C, epoch))
else:
# Load fresh model definition
if args.model == 'linknet':
# Save model definiton script
call(["cp", "./models/linknet.py", args.save])
from models.linknet import LinkNet
model = LinkNet(n_classes)
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
# Criterion
model = torch.nn.DataParallel(model,
device_ids=range(torch.cuda.device_count()))
model.cuda()
criterion = nn.NLLLoss2d()
# Save arguements used for training
args_log = open(args.save + '/args.log', 'w')
for k in args.__dict__:
args_log.write(k + ' : ' + str(args.__dict__[k]) + '\n')
args_log.close()
error_log = list()
prev_iou = 10000
# Setup Metrics
metrics = runningScore(n_classes)
train = Train(model, data_loader_train, optimizer, criterion, args.lr,
args.wd, args.visdom)
test = Test(model, data_loader_test, criterion, metrics, args.visdom)
while epoch <= args.maxepoch:
train_error = train.forward()
test_error, test_score, conf_matrix, class_iou = test.forward()
mean_iou = test_score['Mean IoU']
print('{}{:-<80}{}'.format(CP_R, '', CP_C))
print('{}Epoch #: {}{:03}'.format(CP_B, CP_C, epoch))
print(
'{}Training Error: {}{:.6f} | {}Testing Error: {}{:.6f} |{}Mean IoU: {}{:.6f}'
.format(CP_B, CP_C, train_error, CP_B, CP_C, test_error, CP_G,
CP_C, mean_iou))
error_log.append((train_error, test_error, mean_iou))
# Save weights and model definition
prev_error = save_model(
{
'epoch': epoch,
'model_def': LinkNet,
'state_dict': model.state_dict(),
'optim_state': optimizer.state_dict(),
}, mean_iou, prev_iou, test_score, conf_matrix, class_iou,
args.save, args.saveAll)
epoch += 1
logger = open(args.save + '/error.log', 'w')
logger.write('{:10} {:10}'.format('Train Error', 'Test Error'))
logger.write('\n{:-<20}'.format(''))
for total_error in error_log:
logger.write('\n{:.6f} {:.6f} {:.6f}'.format(total_error[0],
total_error[1],
total_error[2]))
logger.close()
def main(seed=0,
n_train=60000,
n_test=10000,
inhib=250,
kernel_size=(16, ),
stride=(2, ),
time=100,
n_filters=25,
crop=0,
lr=1e-2,
lr_decay=0.99,
dt=1,
theta_plus=0.05,
theta_decay=1e-7,
intensity=5,
norm=0.2,
progress_interval=10,
update_interval=250,
train=True,
plot=False,
gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [
seed, kernel_size, stride, n_filters, crop, lr, lr_decay, n_train,
inhib, time, dt, theta_plus, theta_decay, intensity, norm,
progress_interval, update_interval
]
model_name = '_'.join([str(x) for x in params])
if not train:
test_params = [
seed, kernel_size, stride, n_filters, crop, lr, lr_decay, n_train,
n_test, inhib, time, dt, theta_plus, theta_decay, intensity, norm,
progress_interval, update_interval
]
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
side_length = 28 - crop * 2
n_inpt = side_length**2
n_examples = n_train if train else n_test
n_classes = 10
# Build network.
if train:
network = LocallyConnectedNetwork(
n_inpt=n_inpt,
input_shape=[side_length, side_length],
kernel_size=kernel_size,
stride=stride,
n_filters=n_filters,
inh=inhib,
dt=dt,
nu=[0, lr],
theta_plus=theta_plus,
theta_decay=theta_decay,
wmin=0.0,
wmax=1.0,
norm=norm)
else:
network = load_network(os.path.join(params_path, model_name + '.pt'))
network.connections['X', 'Y'].update_rule = NoOp(
connection=network.connections['X', 'Y'],
nu=network.connections['X', 'Y'].nu)
network.layers['Y'].theta_decay = 0
network.layers['Y'].theta_plus = 0
conv_size = network.connections['X', 'Y'].conv_size
locations = network.connections['X', 'Y'].locations
conv_prod = int(np.prod(conv_size))
n_neurons = n_filters * conv_prod
# Voltage recording for excitatory and inhibitory layers.
voltage_monitor = Monitor(network.layers['Y'], ['v'], time=time)
network.add_monitor(voltage_monitor, name='output_voltage')
# Load MNIST data.
dataset = MNIST(path=data_path, download=True)
if train:
images, labels = dataset.get_train()
else:
images, labels = dataset.get_test()
images *= intensity
images = images[:, crop:-crop, crop:-crop]
# Record spikes during the simulation.
spike_record = torch.zeros(update_interval, time, n_neurons)
full_spike_record = torch.zeros(n_examples, n_neurons).long()
# Neuron assignments and spike proportions.
if train:
assignments = -torch.ones_like(torch.Tensor(n_neurons))
proportions = torch.zeros_like(torch.Tensor(n_neurons, n_classes))
rates = torch.zeros_like(torch.Tensor(n_neurons, n_classes))
ngram_scores = {}
else:
path = os.path.join(params_path,
'_'.join(['auxiliary', model_name]) + '.pt')
assignments, proportions, rates, ngram_scores = torch.load(
open(path, 'rb'))
if train:
best_accuracy = 0
# Sequence of accuracy estimates.
curves = {'all': [], 'proportion': [], 'ngram': []}
predictions = {scheme: torch.Tensor().long() for scheme in curves.keys()}
spikes = {}
for layer in set(network.layers):
spikes[layer] = Monitor(network.layers[layer],
state_vars=['s'],
time=time)
network.add_monitor(spikes[layer], name=f'{layer}_spikes')
# Train the network.
if train:
print('\nBegin training.\n')
else:
print('\nBegin test.\n')
spike_ims = None
spike_axes = None
weights_im = None
start = t()
for i in range(n_examples):
if i % progress_interval == 0:
print(f'Progress: {i} / {n_examples} ({t() - start:.4f} seconds)')
start = t()
if i % update_interval == 0 and i > 0:
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(images) - update_interval:i %
len(images)]
# Update and print accuracy evaluations.
curves, preds = update_curves(curves,
current_labels,
n_classes,
spike_record=spike_record,
assignments=assignments,
proportions=proportions,
ngram_scores=ngram_scores,
n=2)
print_results(curves)
for scheme in preds:
predictions[scheme] = torch.cat(
[predictions[scheme], preds[scheme]], -1)
if train:
# Decay learning rate.
network.connections['X', 'Y'].update_rule.nu[1] *= lr_decay
# Save accuracy curves to disk.
to_write = ['train'] + params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save((curves, update_interval, n_examples),
open(os.path.join(curves_path, f), 'wb'))
if any([x[-1] > best_accuracy for x in curves.values()]):
print(
'New best accuracy! Saving network parameters to disk.'
)
# Save network to disk.
network.save(os.path.join(params_path, model_name + '.pt'))
path = os.path.join(
params_path,
'_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores),
open(path, 'wb'))
best_accuracy = max([x[-1] for x in curves.values()])
# Assign labels to excitatory layer neurons.
assignments, proportions, rates = assign_labels(
spike_record, current_labels, n_classes, rates)
# Compute ngram scores.
ngram_scores = update_ngram_scores(spike_record,
current_labels, n_classes,
2, ngram_scores)
print()
# Get next input sample.
image = images[i % len(images)].contiguous().view(-1)
sample = poisson(datum=image, time=time, dt=dt)
inpts = {'X': sample}
# Run the network on the input.
network.run(inpts=inpts, time=time)
retries = 0
while spikes['Y'].get('s').sum() < 5 and retries < 3:
retries += 1
image *= 2
sample = poisson(datum=image, time=time, dt=dt)
inpts = {'X': sample}
network.run(inpts=inpts, time=time)
# Add to spikes recording.
spike_record[i % update_interval] = spikes['Y'].get('s').t()
full_spike_record[i] = spikes['Y'].get('s').t().sum(0).long()
# Optionally plot various simulation information.
if plot:
_spikes = {
'X': spikes['X'].get('s').view(side_length**2, time),
'Y': spikes['Y'].get('s').view(n_filters * conv_prod, time)
}
spike_ims, spike_axes = plot_spikes(spikes=_spikes,
ims=spike_ims,
axes=spike_axes)
weights_im = plot_locally_connected_weights(
network.connections[('X', 'Y')].w,
n_filters,
kernel_size,
conv_size,
locations,
side_length,
im=weights_im)
plt.pause(1e-8)
network.reset_() # Reset state variables.
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.4f} seconds)')
i += 1
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(images) - update_interval:i %
len(images)]
if labels.numel() > n_examples:
labels = labels[:n_examples]
else:
while labels.numel() < n_examples:
if 2 * labels.numel() > n_examples:
labels = torch.cat(
[labels, labels[:n_examples - labels.numel()]])
else:
labels = torch.cat([labels, labels])
# Update and print accuracy evaluations.
curves, preds = update_curves(curves,
current_labels,
n_classes,
spike_record=spike_record,
assignments=assignments,
proportions=proportions,
ngram_scores=ngram_scores,
n=2)
print_results(curves)
for scheme in preds:
predictions[scheme] = torch.cat([predictions[scheme], preds[scheme]],
-1)
if train:
if any([x[-1] > best_accuracy for x in curves.values()]):
print('New best accuracy! Saving network parameters to disk.')
# Save network to disk.
network.save(os.path.join(params_path, model_name + '.pt'))
path = os.path.join(params_path,
'_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores),
open(path, 'wb'))
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')
print('Average accuracies:\n')
for scheme in curves.keys():
print('\t%s: %.2f' % (scheme, float(np.mean(curves[scheme]))))
# Save accuracy curves to disk.
if train:
to_write = ['train'] + params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save((curves, update_interval, n_examples),
open(os.path.join(curves_path, f), 'wb'))
# Save results to disk.
results = [
np.mean(curves['all']),
np.mean(curves['proportion']),
np.mean(curves['ngram']),
np.max(curves['all']),
np.max(curves['proportion']),
np.max(curves['ngram'])
]
to_write = params + results if train else test_params + results
to_write = [str(x) for x in to_write]
if train:
name = 'train.csv'
else:
name = 'test.csv'
if not os.path.isfile(os.path.join(results_path, name)):
with open(os.path.join(results_path, name), 'w') as f:
if train:
f.write(
'random_seed,kernel_size,stride,n_filters,crop,lr,lr_decay,n_train,inhib,time,timestep,theta_plus,'
'theta_decay,intensity,norm,progress_interval,update_interval,mean_all_activity,'
'mean_proportion_weighting,mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n'
)
else:
f.write(
'random_seed,kernel_size,stride,n_filters,crop,lr,lr_decay,n_train,n_test,inhib,time,timestep,'
'theta_plus,theta_decay,intensity,norm,progress_interval,update_interval,mean_all_activity,'
'mean_proportion_weighting,mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n'
)
with open(os.path.join(results_path, name), 'a') as f:
f.write(','.join(to_write) + '\n')
# Compute confusion matrices and save them to disk.
confusions = {}
for scheme in predictions:
confusions[scheme] = confusion_matrix(labels, predictions[scheme])
to_write = ['train'] + params if train else ['test'] + test_params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save(confusions, os.path.join(confusion_path, f))
# Save full spike record to disk.
torch.save(full_spike_record, os.path.join(spikes_path, f))
def train(self, rank, start_time, return_dict):
device = torch.device("cuda:" + str(rank))
print('Running on device: ', device)
torch.cuda.set_device(device)
torch.set_default_tensor_type(torch.FloatTensor)
writer = None
if not self.args.cross_validate_hp:
writer = SummaryWriter(logdir=os.path.join(self.save_dir, 'logs'))
# posting parameters
param_string = ""
for k, v in vars(self.args).items():
param_string += ' ' * 10 + k + ': ' + str(v) + '\n'
writer.add_text("params", param_string)
self.setup(rank, self.args.num_processes)
if self.cfg.MC_DQL:
transition = namedtuple('Transition', ('episode'))
else:
transition = namedtuple(
'Transition',
('state', 'action', 'reward', 'next_state', 'done'))
memory = TransitionData_ts(capacity=self.args.t_max,
storage_object=transition)
env = SpGcnEnv(self.args,
device,
writer=writer,
writer_counter=self.global_writer_quality_count,
win_event_counter=self.global_win_event_count)
# Create shared network
# model = GcnEdgeAC_1(self.cfg, self.args.n_raw_channels, self.args.n_embedding_features, 1, device, writer=writer)
model = GcnEdgeAC(self.cfg, self.args, device, writer=writer)
# model = GcnEdgeAC(self.cfg, self.args.n_raw_channels, self.args.n_embedding_features, 1, device, writer=writer)
model.cuda(device)
shared_model = DDP(model,
device_ids=[model.device],
find_unused_parameters=True)
# dloader = DataLoader(MultiDiscSpGraphDsetBalanced(no_suppix=False, create=False), batch_size=1, shuffle=True, pin_memory=True,
# num_workers=0)
dloader = DataLoader(SpgDset(),
batch_size=self.cfg.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
# Create optimizer for shared network parameters with shared statistics
# optimizer = CstmAdam(shared_model.parameters(), lr=self.args.lr, betas=self.args.Adam_betas,
# weight_decay=self.args.Adam_weight_decay)
######################
self.action_range = 1
self.device = torch.device(device)
self.discount = 0.5
self.critic_tau = self.cfg.critic_tau
self.actor_update_frequency = self.cfg.actor_update_frequency
self.critic_target_update_frequency = self.cfg.critic_target_update_frequency
self.batch_size = self.cfg.batch_size
self.log_alpha = torch.tensor(np.log(self.cfg.init_temperature)).to(
self.device)
self.log_alpha.requires_grad = True
# set target entropy to -|A|
######################
# optimizers
OptimizerContainer = namedtuple('OptimizerContainer',
('actor', 'critic', 'temperature'))
actor_optimizer = torch.optim.Adam(
shared_model.module.actor.parameters(),
lr=self.cfg.actor_lr,
betas=self.cfg.actor_betas)
critic_optimizer = torch.optim.Adam(
shared_model.module.critic.parameters(),
lr=self.cfg.critic_lr,
betas=self.cfg.critic_betas)
temp_optimizer = torch.optim.Adam([self.log_alpha],
lr=self.cfg.alpha_lr,
betas=self.cfg.alpha_betas)
optimizers = OptimizerContainer(actor_optimizer, critic_optimizer,
temp_optimizer)
for param in model.fe_ext.parameters():
param.requires_grad = False
if self.args.model_name != "":
shared_model.load_state_dict(
torch.load(os.path.join(self.save_dir, self.args.model_name)))
elif self.args.model_fe_name != "":
shared_model.module.fe_ext.load_state_dict(
torch.load(os.path.join(self.save_dir,
self.args.model_fe_name)))
elif self.args.fe_extr_warmup:
print('loaded fe extractor')
shared_model.module.fe_ext.load_state_dict(
torch.load(os.path.join(self.save_dir, 'agent_model_fe_extr')))
dist.barrier()
if not self.args.test_score_only:
quality = self.args.stop_qual_scaling + self.args.stop_qual_offset
best_quality = np.inf
while self.global_count.value() <= self.args.T_max:
if self.global_count.value() == 78:
a = 1
self.update_env_data(env, dloader, device)
# waff_dis = torch.softmax(env.edge_features[:, 0].squeeze() + 1e-30, dim=0)
# waff_dis = torch.softmax(env.gt_edge_weights + 0.5, dim=0)
# waff_dis = torch.softmax(torch.ones_like(env.b_sg_gt_edge_weights), dim=0)
# loss_weight = torch.softmax(env.b_sg_gt_edge_weights + 1, dim=0)
env.reset()
# self.target_entropy = - float(env.gt_edge_weights.shape[0])
self.target_entropy = -self.args.s_subgraph
env.stop_quality = self.stop_qual_rule.apply(
self.global_count.value(), quality)
if self.cfg.temperature_regulation == 'follow_quality':
self.alpha = self.eps_rule.apply(self.global_count.value(),
quality)
print(self.alpha.item())
with open(os.path.join(self.save_dir,
'runtime_cfg.yaml')) as info:
args_dict = yaml.full_load(info)
if args_dict is not None:
if 'safe_model' in args_dict:
self.args.safe_model = args_dict['safe_model']
args_dict['safe_model'] = False
if 'add_noise' in args_dict:
self.args.add_noise = args_dict['add_noise']
if 'critic_lr' in args_dict and args_dict[
'critic_lr'] != self.cfg.critic_lr:
self.cfg.critic_lr = args_dict['critic_lr']
adjust_learning_rate(critic_optimizer,
self.cfg.critic_lr)
if 'actor_lr' in args_dict and args_dict[
'actor_lr'] != self.cfg.actor_lr:
self.cfg.actor_lr = args_dict['actor_lr']
adjust_learning_rate(actor_optimizer,
self.cfg.actor_lr)
if 'alpha_lr' in args_dict and args_dict[
'alpha_lr'] != self.cfg.alpha_lr:
self.cfg.alpha_lr = args_dict['alpha_lr']
adjust_learning_rate(temp_optimizer,
self.cfg.alpha_lr)
with open(os.path.join(self.save_dir, 'runtime_cfg.yaml'),
"w") as info:
yaml.dump(args_dict, info)
if self.args.safe_model:
best_quality = quality
if rank == 0:
if self.args.model_name_dest != "":
torch.save(
shared_model.state_dict(),
os.path.join(self.save_dir,
self.args.model_name_dest))
else:
torch.save(
shared_model.state_dict(),
os.path.join(self.save_dir, 'agent_model'))
state = env.get_state()
while not env.done:
# Calculate policy and values
post_input = True if (
self.global_count.value() +
1) % 15 == 0 and env.counter == 0 else False
round_n = env.counter
# sample action for data collection
distr = None
if self.global_count.value() < self.cfg.num_seed_steps:
action = torch.rand_like(env.sg_current_edge_weights)
else:
distr, _, _, action = self.agent_forward(
env,
shared_model,
state,
grad=False,
post_input=post_input)
logg_dict = {'temperature': self.alpha.item()}
if distr is not None:
logg_dict['mean_loc'] = distr.loc.mean().item()
logg_dict['mean_scale'] = distr.scale.mean().item()
if self.global_count.value(
) >= self.cfg.num_seed_steps and memory.is_full():
self._step(memory,
optimizers,
env,
shared_model,
self.global_count.value(),
writer=writer)
self.global_writer_loss_count.increment()
next_state, reward, quality = env.execute_action(
action, logg_dict)
if self.args.add_noise:
noise = torch.randn_like(reward) * self.alpha.item()
reward = reward + noise
memory.push(self.state_to_cpu(state), action.cpu(),
reward.cpu(), self.state_to_cpu(next_state),
env.done)
# Train the network
# self._step(memory, shared_model, env, optimizer, loss_weight, off_policy=True, writer=writer)
# reward = self.args.reward_clip and min(max(reward, -1), 1) or reward # Optionally clamp rewards
# done = done or episode_length >= self.args.max_episode_length # Stop episodes at a max length
state = next_state
self.global_count.increment()
if "self_reg" in self.args.eps_rule and quality <= 2:
break
dist.barrier()
if rank == 0:
if not self.args.cross_validate_hp and not self.args.test_score_only and not self.args.no_save:
# pass
if self.args.model_name_dest != "":
torch.save(
shared_model.state_dict(),
os.path.join(self.save_dir, self.args.model_name_dest))
print('saved')
else:
torch.save(shared_model.state_dict(),
os.path.join(self.save_dir, 'agent_model'))
self.cleanup()
def train_model(args):
# Constant
tol = 1e-12
warm_start = True
bias = False # without bias outer_lr can be bigger (much faster convergence)
train_log_interval = 100
val_log_interval = 1
# Basic Setting
seed = 0
torch.manual_seed(seed)
np.random.seed(seed)
cuda = True and torch.cuda.is_available()
default_tensor_str = 'torch.cuda.FloatTensor' if cuda else 'torch.FloatTensor'
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
torch.set_default_tensor_type(default_tensor_str)
#torch.multiprocessing.set_start_method('forkserver')
# Functions
def frnp(x): return torch.from_numpy(x).cuda().float() if cuda else torch.from_numpy(x).float()
def tonp(x, cuda=cuda): return x.detach().cpu().numpy() if cuda else x.detach().numpy()
def train_loss(params, hparams, data):
x_mb, y_mb = data
# print(x_mb.size()) = torch.Size([5657, 130107])
out = out_f(x_mb, params)
return F.cross_entropy(out, y_mb) + reg_f(params, *hparams)
def val_loss(opt_params, hparams):
x_mb, y_mb = next(val_iterator)
# print(x_mb.size()) = torch.Size([5657, 130107])
out = out_f(x_mb, opt_params[:len(parameters)])
val_loss = F.cross_entropy(out, y_mb)
pred = out.argmax(dim=1, keepdim=True) # get the index of the max log-probability
acc = pred.eq(y_mb.view_as(pred)).sum().item() / len(y_mb)
val_losses.append(tonp(val_loss))
val_accs.append(acc)
return val_loss
def reg_f(params, l2_reg_params, l1_reg_params=None):
r = 0.5 * ((params[0] ** 2) * torch.exp(l2_reg_params.unsqueeze(1) * ones_dxc)).mean()
if l1_reg_params is not None:
r += (params[0].abs() * torch.exp(l1_reg_params.unsqueeze(1) * ones_dxc)).mean()
return r
def out_f(x, params):
out = x @ params[0]
out += params[1] if len(params) == 2 else 0
return out
def eval(params, x, y):
out = out_f(x, params)
loss = F.cross_entropy(out, y)
pred = out.argmax(dim=1, keepdim=True) # get the index of the max log-probability
acc = pred.eq(y.view_as(pred)).sum().item() / len(y)
return loss, acc
# load twentynews and preprocess
val_size_ratio = 0.5
X, y = fetch_20newsgroups_vectorized(subset='train', return_X_y=True,
#remove=('headers', 'footers', 'quotes')
)
x_test, y_test = fetch_20newsgroups_vectorized(subset='test', return_X_y=True,
#remove=('headers', 'footers', 'quotes')
)
x_train, x_val, y_train, y_val = train_test_split(X, y, stratify=y, test_size=val_size_ratio)
train_samples, n_features = x_train.shape
test_samples, n_features = x_test.shape
val_samples, n_features = x_val.shape
n_classes = np.unique(y_train).shape[0]
# train_samples=5657, val_samples=5657, test_samples=7532, n_features=130107, n_classes=20
print('Dataset 20newsgroup, train_samples=%i, val_samples=%i, test_samples=%i, n_features=%i, n_classes=%i'
% (train_samples, val_samples, test_samples, n_features, n_classes))
ys = [frnp(y_train).long(), frnp(y_val).long(), frnp(y_test).long()]
xs = [x_train, x_val, x_test]
if cuda:
xs = [from_sparse(x).cuda() for x in xs]
else:
xs = [from_sparse(x) for x in xs]
# x_train.size() = torch.Size([5657, 130107])
# y_train.size() = torch.Size([5657])
x_train, x_val, x_test = xs
y_train, y_val, y_test = ys
# torch.DataLoader has problems with sparse tensor on GPU
iterators, train_list, val_list = [], [], []
# For minibatch method, we build the list to store the splited tensor
if args.alg == 'minibatch':
for bs, x, y in [(len(y_train), x_train, y_train), (len(y_val), x_val, y_val)]:
iterators.append(CustomTensorIterator([x, y], batch_size=args.batch_size, shuffle=True, **kwargs))
train_iterator, val_iterator = iterators
for _ in range(train_samples // args.batch_size+1):
train_list.append(next(train_iterator))
for _ in range(val_samples // args.val_size+1):
val_list.append(next(val_iterator))
train_list_len, val_list_len = len(train_list), len(val_list)
# set up another train_iterator & val_iterator to make sure train_list and val_list are full
iterators = []
for bs, x, y in [(len(y_train), x_train, y_train), (len(y_val), x_val, y_val)]:
iterators.append(repeat([x, y]))
train_iterator, val_iterator = iterators
else:
for bs, x, y in [(len(y_train), x_train, y_train), (len(y_val), x_val, y_val)]:
iterators.append(repeat([x, y]))
train_iterator, val_iterator = iterators
# Initialize parameters
l2_reg_params = torch.zeros(n_features).requires_grad_(True) # one hp per feature
l1_reg_params = (0.*torch.ones(1)).requires_grad_(True) # one l1 hp only (best when really low)
#l2_reg_params = (-20.*torch.ones(1)).requires_grad_(True) # one l2 hp only (best when really low)
#l1_reg_params = (-1.*torch.ones(n_features)).requires_grad_(True)
hparams = [l2_reg_params]
# hparams: the outer variables (or hyperparameters)
ones_dxc = torch.ones(n_features, n_classes)
outer_opt = torch.optim.SGD(lr=args.outer_lr, momentum=args.outer_mu, params=hparams)
# outer_opt = torch.optim.Adam(lr=0.01, params=hparams)
params_history = []
val_losses, val_accs = [], []
test_losses, test_accs = [], []
w = torch.zeros(n_features, n_classes).requires_grad_(True)
parameters = [w]
# params_history: the inner iterates (from first to last)
if bias:
b = torch.zeros(n_classes).requires_grad_(True)
parameters.append(b)
if args.inner_mu > 0:
#inner_opt = hg.Momentum(train_loss, inner_lr, inner_mu, data_or_iter=train_iterator)
inner_opt = hg.HeavyBall(train_loss, args.inner_lr, args.inner_mu, data_or_iter=train_iterator)
else:
inner_opt = hg.GradientDescent(train_loss, args.inner_lr, data_or_iter=train_iterator)
inner_opt_cg = hg.GradientDescent(train_loss, 1., data_or_iter=train_iterator)
total_time = 0
loss_acc_time_results = np.zeros((args.n_steps+1, 3))
test_loss, test_acc = eval(parameters, x_test, y_test)
loss_acc_time_results[0, 0] = test_loss
loss_acc_time_results[0, 1] = test_acc
loss_acc_time_results[0, 2] = 0.0
for o_step in range(args.n_steps):
train_index_list = torch.randperm(train_list_len)
val_index_list = torch.randperm(val_list_len)
start_time = time.time()
if args.alg == 'minibatch':
inner_losses = []
for t in range(args.T):
loss_train = train_loss(parameters, hparams, train_list[train_index_list[t%train_list_len]])
inner_grad = torch.autograd.grad(loss_train, parameters)
parameters[0] = parameters[0] - args.inner_lr*inner_grad[0]
inner_losses.append(loss_train)
if t % train_log_interval == 0 or t == args.T-1:
print('t={} loss: {}'.format(t, inner_losses[-1]))
# Gy_gradient
Gy_gradient = gradient_gy(args, parameters, val_list[val_index_list[0]], hparams)
Gy_gradient = torch.reshape(Gy_gradient, [-1])
G_gradient = torch.reshape(parameters[0], [-1]) - args.eta*Gy_gradient
# Fy_gradient
Fy_gradient = gradient_fy(args, parameters, val_list[val_index_list[1%val_list_len]])
v_0 = torch.unsqueeze(torch.reshape(Fy_gradient, [-1]), 1).detach()
# Hessian
z_list = []
v_Q = args.eta*v_0
for q in range(args.K):
Jacobian = torch.matmul(G_gradient, v_0)
v_new = torch.autograd.grad(Jacobian, parameters, retain_graph=True)[0]
v_0 = torch.unsqueeze(torch.reshape(v_new, [-1]), 1).detach()
z_list.append(v_0)
v_Q = v_Q+torch.sum(torch.stack(z_list), dim=0)
# Gyx_gradient
Gy_gradient = gradient_gy(args, parameters, val_list[val_index_list[2%val_list_len]], hparams)
Gy_gradient = torch.reshape(Gy_gradient, [-1])
Gyx_gradient = torch.autograd.grad(torch.matmul(Gy_gradient, v_Q.detach()), hparams)[0]
hparams[0] = hparams[0] - args.outer_lr*(-Gyx_gradient)
final_params = parameters
for p, new_p in zip(parameters, final_params[:len(parameters)]):
if warm_start:
p.data = new_p
else:
p.data = torch.zeros_like(p)
val_loss(final_params, hparams)
else:
inner_losses = []
if params_history:
params_history = [params_history[-1]]
else:
params_history = [inner_opt.get_opt_params(parameters)]
for t in range(args.T):
params_history.append(inner_opt(params_history[-1], hparams, create_graph=False))
inner_losses.append(inner_opt.curr_loss)
if t % train_log_interval == 0 or t == args.T-1:
print('t={} loss: {}'.format(t, inner_losses[-1]))
final_params = params_history[-1]
outer_opt.zero_grad()
if args.alg == 'reverse':
hg.reverse(params_history[-args.K-1:], hparams, [inner_opt]*args.K, val_loss)
elif args.alg == 'fixed_point':
hg.fixed_point(final_params, hparams, args.K, inner_opt, val_loss, stochastic=False, tol=tol)
elif args.alg == 'neuman':
hg.neumann(final_params, hparams, args.K, inner_opt, val_loss, tol=tol)
elif args.alg == 'CG':
hg.CG(final_params[:len(parameters)], hparams, args.K, inner_opt_cg, val_loss, stochastic=False, tol=tol)
outer_opt.step()
for p, new_p in zip(parameters, final_params[:len(parameters)]):
if warm_start:
p.data = new_p
else:
p.data = torch.zeros_like(p)
iter_time = time.time() - start_time
total_time += iter_time
if o_step % val_log_interval == 0 or o_step == args.T-1:
test_loss, test_acc = eval(final_params[:len(parameters)], x_test, y_test)
loss_acc_time_results[o_step+1, 0] = test_loss
loss_acc_time_results[o_step+1, 1] = test_acc
loss_acc_time_results[o_step+1, 2] = total_time
print('o_step={} ({:.2e}s) Val loss: {:.4e}, Val Acc: {:.2f}%'.format(o_step, iter_time, val_losses[-1],
100*val_accs[-1]))
print(' Test loss: {:.4e}, Test Acc: {:.2f}%'.format(test_loss, 100*test_acc))
print(' l2_hp norm: {:.4e}'.format(torch.norm(hparams[0])))
if len(hparams) == 2:
print(' l1_hp : ', torch.norm(hparams[1]))
file_name = 'results.npy'
file_addr = os.path.join(args.save_folder, file_name)
with open(file_addr, 'wb') as f:
np.save(f, loss_acc_time_results)
print(loss_acc_time_results)
print('HPO ended in {:.2e} seconds\n'.format(total_time))
def main():
batchsize = 3
train_data = myDataLoader.MyDataset(txt='data/train_pic.txt',
transform=transforms.ToTensor())
train_data_load = torch.utils.data.DataLoader(train_data,
batch_size=batchsize,
shuffle=True,
num_workers=0)
if torch.cuda.is_available():
net = ResNet18().double().cuda()
else:
net = ResNet18().double()
print(net)
criterion = nn.MSELoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
# train
print("training begin")
for epoch in range(300):
start = time.time()
running_loss = 0
for i, data in enumerate(train_data_load, 0):
# print (inputs,labels)
image, label = data
if torch.cuda.is_available():
# print('GPU2')
image = image.double().cuda()
label = label.double().cuda()
else:
image = image.double()
label = label.double()
image = Variable(image)
label = Variable(label)
# imshow(torchvision.utils.make_grid(image))
# plt.show()
# print (label)
optimizer.zero_grad()
outputs = net(image)
# print (outputs)
loss = criterion(outputs, label)
if torch.isnan(loss):
break
# print(outputs)
# print(label)
# print(loss)
torch.set_default_tensor_type(torch.FloatTensor)
loss.backward()
optimizer.step()
running_loss += loss.data
# if i % 5 == 1 :
if i % 100 == 99:
end = time.time()
print('[epoch %d,imgs %5d] loss: %.7f time: %0.3f s' %
(epoch + 1, (i + 1) * batchsize, running_loss / 100,
(end - start)))
with open("train_log.txt", 'a+') as f1:
line = '[epoch %d,imgs %5d] loss: %.7f time: %0.3f s' % (
epoch + 1, (i + 1) * batchsize, running_loss / 100,
(end - start)) + '\n'
f1.writelines(line)
start = time.time()
running_loss = 0
if epoch % 100 == 99:
torch.save(net, "net/epoch%d" % (epoch + 1) + 'resnet.pkl')
print('model has saved')
print("finish training")
torch.save(net, 'net/resnet.pkl') # 保存整个网络
def transfer_codes(config):
torch.set_default_tensor_type('torch.cuda.FloatTensor')
N_per_transfer = 8
N_transfer = 10
parser = config_parser()
args = parser.parse_args(['--config', config])
render_kwargs_train, render_kwargs_test, _, _, optimizer, styles = create_nerf(
args, return_styles=True)
nfs = [[args.blender_near, args.blender_far]
for _ in range(N_per_transfer)]
os.makedirs(f'{args.expname}/transfer_codes', exist_ok=True)
with torch.no_grad():
for i1 in range(N_transfer):
for i2 in tqdm(range(N_transfer)):
if i1 == i2:
continue
s1, s2 = styles[i1].unsqueeze(dim=0), styles[i2].unsqueeze(
dim=0)
poses_1, hwfs_1 = get_poses_hwfs(args,
i1,
styles.shape[0],
num_canvases=N_per_transfer)
poses_2, hwfs_2 = get_poses_hwfs(args,
i2,
styles.shape[0],
num_canvases=N_per_transfer)
rgb1 = render_path(poses_1,
s1,
hwfs_1,
4096,
render_kwargs_test,
nfs=nfs,
verbose=False)[0][0]
rgb2 = render_path(poses_2,
s2,
hwfs_2,
4096,
render_kwargs_test,
nfs=nfs,
verbose=False)[0][0]
utils.save_image(
torch.tensor(rgb1).permute(2, 0, 1),
f'{args.expname}/transfer_codes/{i1}.png')
utils.save_image(
torch.tensor(rgb2).permute(2, 0, 1),
f'{args.expname}/transfer_codes/{i2}.png')
take_color = torch.cat([s1[:, :32], s2[:, 32:]], dim=1)
take_shape = torch.cat([s2[:, :32], s1[:, 32:]], dim=1)
# i1 with shape from i2 is the same as i2 color from i1, so don't duplicate
color_from = render_path(poses_1,
take_color.repeat(
(N_per_transfer, 1)),
hwfs_1,
4096,
render_kwargs_test,
nfs=nfs,
verbose=False)[0]
shape_from = render_path(poses_2,
take_shape.repeat(
(N_per_transfer, 1)),
hwfs_2,
4096,
render_kwargs_test,
nfs=nfs,
verbose=False)[0]
utils.save_image(
torch.tensor(color_from).permute(0, 3, 1, 2),
f'{args.expname}/transfer_codes/{i1}_color_from_{i2}.png')
utils.save_image(
torch.tensor(shape_from).permute(0, 3, 1, 2),
f'{args.expname}/transfer_codes/{i1}_shape_from_{i2}.png')
from pyro.infer import SVI, Trace_ELBO
from pyro.optim import Adam
import glob
import matplotlib.pyplot as plt
from functools import partial
import pandas as pd
import numpy as np
import gaussian_1V_M1 as cretin_nn
import pca_data_1V as pca_data_hu
# logging.basicConfig(level=logging.DEBUG)
USE_GPU = torch.cuda.is_available()
device = torch.device('cuda' if USE_GPU else 'cpu')
torch.set_default_tensor_type('torch.cuda.FloatTensor' if USE_GPU else 'torch.FloatTensor')
if os.environ['HOSTNAME'] == 'fractal':
PARAM_FILES = '/hdd/bdhammel/checkpoints/bayes/*.params'
else:
PARAM_FILES = '/usr/WS1/hammel1/proj/checkpoints/bayes/*.params'
if os.environ['HOSTNAME'] == 'fractal':
DATA_FILES = '/hdd/bdhammel/checkpoints/bayes/*.npy'
else:
DATA_FILES = '/usr/WS1/hammel1/proj/checkpoints/bayes/*.npy'
experiment_id = '2019-09-11T16:39:29.282283'
RESTORE_PATH = f'/usr/WS1/hammel1/proj/checkpoints/bayes/{experiment_id}'
def main(
width=128,
depth=1,
vector_length=128,
min_batch_size=16,
max_batch_size=16,
learn_rate=0.001,
momentum=0.9,
dropout=0.5,
dropout_decay=1e-4,
nb_epoch=20,
L2=1e-6,
):
using_gpu = prefer_gpu()
if using_gpu:
torch.set_default_tensor_type("torch.cuda.FloatTensor")
cfg = dict(locals())
print(cfg)
train_data, check_data, nr_tag = ancora_pos_tags()
train_data = list(train_data)
check_data = list(check_data)
extracter = FeatureExtracter("es", attrs=[LOWER, SHAPE, PREFIX, SUFFIX])
with Model.define_operators({"**": clone, ">>": chain, "+": add, "|": concatenate}):
lower_case = HashEmbed(width, 100, column=0)
shape = HashEmbed(width // 2, 200, column=1)
prefix = HashEmbed(width // 2, 100, column=2)
suffix = HashEmbed(width // 2, 100, column=3)
model = (
with_flatten(
(lower_case | shape | prefix | suffix) >> Maxout(width, pieces=3)
)
>> PyTorchBiLSTM(width, width, depth)
>> with_flatten(Softmax(nr_tag))
)
train_X, train_y = preprocess(model.ops, extracter, train_data, nr_tag)
dev_X, dev_y = preprocess(model.ops, extracter, check_data, nr_tag)
n_train = float(sum(len(x) for x in train_X))
global epoch_train_acc
with model.begin_training(train_X[:10], train_y[:10], **cfg) as (
trainer,
optimizer,
):
trainer.each_epoch.append(track_progress(**locals()))
trainer.batch_size = min_batch_size
batch_size = float(min_batch_size)
for X, y in trainer.iterate(train_X, train_y):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
gradient = [yh[i] - y[i] for i in range(len(yh))]
backprop(gradient, optimizer)
trainer.batch_size = min(int(batch_size), max_batch_size)
batch_size *= 1.001
print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
with open("/tmp/model.pickle", "wb") as file_:
pickle.dump(model, file_)
from __future__ import absolute_import, division, print_function
import argparse
import os
import numpy as np
import torch
import pyro
import pyro.optim as optim
import pyro.poutine as poutine
import wget
from pyro.distributions import Gamma, Poisson
from pyro.infer import SVI, Trace_ELBO
torch.set_default_tensor_type('torch.FloatTensor')
pyro.enable_validation(True)
pyro.util.set_rng_seed(0)
class SparseGammaDEF(object):
def __init__(self):
# define the sizes of the layers in the deep exponential family
self.top_width = 100
self.mid_width = 40
self.bottom_width = 15
self.image_size = 64 * 64
# define hyperpaameters that control the prior
self.alpha_z = torch.tensor(0.1)
self.beta_z = torch.tensor(0.1)
self.alpha_w = torch.tensor(0.1)
def neural_net(training_labels, training_spectra, validation_labels, validation_spectra,\
num_neurons = 300, num_steps=1e5, learning_rate=1e-4, batch_size=512,\
num_features = 64*5, mask_size=11, num_pixel=7214):
'''
Training neural networks to emulate spectral models
training_labels has the dimension of [# training spectra, # stellar labels]
training_spectra has the dimension of [# training spectra, # wavelength pixels]
The validation set is used to independently evaluate how well the neural networks
are emulating the spectra. If the networks overfit the spectral variation, while
the loss function will continue to improve for the training set, but the validation
set should show a worsen loss function.
The training is designed in a way that it always returns the best neural networks
before the networks start to overfit (gauged by the validation set).
num_steps = how many steps to train until convergence.
1e5 is good for the specific NN architecture and learning I used by default,
but bigger networks take more steps, and decreasing the learning rate will
also change this. You can get a sense of how many steps are needed for a new
NN architecture by plotting the loss function evaluated on both the training set
and a validation set as a function of step number. It should plateau once the NN
has converged.
learning_rate = step size to take for gradient descent
This is also tunable, but 1e-4 seems to work well for most use cases. Again,
diagnose with a validation set if you change this.
num_features is the number of features before the deconvolutional layers; it only
applies if ResNet is used. For the simple multi-layer perceptron model, this parameter
is not used. We truncate the predicted model if the output number of pixels is
larger than what is needed. In the current default model, the output is ~8500 pixels
in the case where the number of pixels is > 8500, increase the number of features, and
tweak the ResNet model accordingly
batch_size = the batch size for training the neural networks during the stochastic
gradient descent. A larger batch_size reduces the stochasticity, but it might also
risk to stuck in a local minimum
returns:
training loss and validation loss
'''
# try to run on cuda
try:
dtype = torch.cuda.FloatTensor
torch.set_default_tensor_type('torch.cuda.FloatTensor')
except:
dtype = torch.FloatTensor
torch.set_default_tensor_type('torch.FloatTensor')
# scale the labels, optimizing neural networks is easier if the labels are more normalized
x_max = np.max(training_labels, axis=0)
x_min = np.min(training_labels, axis=0)
x = (training_labels - x_min) / (x_max - x_min) - 0.5
x_valid = (validation_labels - x_min) / (x_max - x_min) - 0.5
# save scaling relation
np.savez("NN_scaling.npz", x_min=x_min, x_max=x_max)
# dimension of the input
dim_in = x.shape[1]
#--------------------------------------------------------------------------------------------
# assume L2 loss
loss_fn = torch.nn.L1Loss(reduction='mean')
# make pytorch variables
x = Variable(torch.from_numpy(x)).type(dtype)
y = Variable(torch.from_numpy(training_spectra),
requires_grad=False).type(dtype)
x_valid = Variable(torch.from_numpy(x_valid)).type(dtype)
y_valid = Variable(torch.from_numpy(validation_spectra),
requires_grad=False).type(dtype)
# initiate Payne and optimizer
model = Payne_model(dim_in, num_neurons, num_features, mask_size,
num_pixel)
try:
model.cuda()
except:
model.to('cpu')
model.train()
# we adopt rectified Adam for the optimization
optimizer = radam.RAdam(
[p for p in model.parameters() if p.requires_grad == True],
lr=learning_rate)
#--------------------------------------------------------------------------------------------
# break into batches
nsamples = x.shape[0]
nbatches = nsamples // batch_size
nsamples_valid = x_valid.shape[0]
nbatches_valid = nsamples_valid // batch_size
# initiate counter
current_loss = np.inf
training_loss = []
validation_loss = []
#-------------------------------------------------------------------------------------------------------
# train the network
for e in range(int(num_steps)):
# randomly permute the data
perm = torch.randperm(nsamples)
try:
perm = perm.cuda()
except:
perm = perm.to('cpu')
# for each batch, calculate the gradient with respect to the loss
for i in range(nbatches):
idx = perm[i * batch_size:(i + 1) * batch_size]
y_pred = model(x[idx])
loss = loss_fn(y_pred, y[idx]) * 1e4
optimizer.zero_grad()
loss.backward(retain_graph=False)
optimizer.step()
# the average loss
if e % 100 == 0:
# randomly permute the data
perm_valid = torch.randperm(nsamples_valid)
try:
perm_valid = perm_valid.cuda()
except:
pass
loss_valid = 0
for j in range(nbatches_valid):
idx = perm_valid[j * batch_size:(j + 1) * batch_size]
y_pred_valid = model(x_valid[idx])
loss_valid += loss_fn(y_pred_valid, y_valid[idx]) * 1e4
loss_valid /= nbatches_valid
print('iter %s:' % e, 'training loss = %.3f' % loss,\
'validation loss = %.3f' % loss_valid)
loss_data = loss.detach().data.item()
loss_valid_data = loss_valid.detach().data.item()
training_loss.append(loss_data)
validation_loss.append(loss_valid_data)
# record the weights and biases if the validation loss improves
if loss_valid_data < current_loss:
current_loss = loss_valid_data
state_dict = model.state_dict()
for k, v in state_dict.items():
state_dict[k] = v.cpu()
torch.save(state_dict, 'NN_normalized_spectra.pt')
np.savez("training_loss.npz",\
training_loss = training_loss,\
validation_loss = validation_loss)
#--------------------------------------------------------------------------------------------
# save the final training loss
np.savez("training_loss.npz",\
training_loss = training_loss,\
validation_loss = validation_loss)
return
import torch.nn as nn
# from art.classifiers import PyTorchClassifier
import foolbox as fb
from cached_property import cached_property
from tda.devices import device
from tda.models.layers import (
Layer,
SoftMaxLayer,
)
from tda.rootpath import model_dir
from tda.tda_logging import get_logger
from tda.precision import default_tensor_type
torch.set_default_tensor_type(default_tensor_type)
logger = get_logger("Architecture")
#################
# Architectures #
#################
logger = get_logger("Architecture")
class Architecture(nn.Module):
def __init__(
self,
layers: List[Layer],
preprocess: Callable = None,
layer_links: List[Tuple[int, int]] = None,
name: str = "",
def main():
#create model
best_prec1 = 0
# Dataset
#idxs = np.load('SubResampleIndex.npy')
Dataset_train = H5Dataset(root='data', mode='training', DataType='sen2')
Dataloader_train = data.DataLoader(Dataset_train,
args.batch_size,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
'''
Dataloader_train = data.DataLoader(Dataset_train, args.batch_size,
num_workers = args.num_workers,
shuffle = True, pin_memory = True)
'''
Dataset_validation = H5Dataset(root=args.dataset_root,
mode='validation',
DataType='sen2')
Dataloader_validation = data.DataLoader(Dataset_validation,
batch_size=1,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.set_device(0)
if args.basenet == 'ResNeXt':
model = Sen2ResNeXt(num_classes=args.class_num,
depth=11,
cardinality=16)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
if args.basenet == 'SimpleNetSen2':
model = SimpleNetSen2(args.class_num)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
elif args.basenet == 'se_resnet50':
model = se_resnet50(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
elif args.basenet == 'se_resnet50_shallow':
model = se_resnet50_shallow(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
elif args.basenet == 'nasnetamobile':
model = nasnetamobile(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
elif args.basenet == 'pnasnet':
model = pnasnet5large(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
if args.resume:
model.load_state_dict(torch.load(args.resume))
else:
state_dict = torch.load('pnasnet5large-bf079911.pth')
state_dict.pop('last_linear.bias')
state_dict.pop('last_linear.weight')
model.load_state_dict(state_dict, strict=False)
init.xavier_uniform_(model.last_linear.weight.data)
model.last_linear.bias.data.zero_()
elif args.basenet == 'se_resnet101':
model = se_resnet101(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
if args.resume:
model.load_state_dict(torch.load(args.resume))
else:
state_dict = torch.load('se_resnet101-7e38fcc6.pth')
state_dict.pop('last_linear.bias')
state_dict.pop('last_linear.weight')
model.load_state_dict(state_dict, strict=False)
init.xavier_uniform_(model.last_linear.weight.data)
model.last_linear.bias.data.zero_()
elif args.basenet == 'se_resnext101_32x4d':
model = se_resnext101_32x4d(args.class_num, None)
#net = Networktorch.nn.DataParallel(Network, device_ids=[0])
cudnn.benchmark = True
if args.resume:
model.load_state_dict(torch.load(args.resume))
else:
state_dict = torch.load('se_resnext101_32x4d-3b2fe3d8.pth')
state_dict.pop('last_linear.bias')
state_dict.pop('last_linear.weight')
model.load_state_dict(state_dict, strict=False)
init.xavier_uniform_(model.last_linear.weight.data)
model.last_linear.bias.data.zero_()
model = model.cuda()
cudnn.benchmark = True
'''
weights = [ 9.73934491, 2.02034301, 1.55741015, 5.70558317, 2.99272419,
1.39866818, 15.09911288, 1.25512384, 3.63361307, 4.12907813,
1.1505058 , 5.18803868, 5.38559738, 1.1929091 , 20.63503344,
6.24955685, 1. ]
'''
#weights = np.load('Precision_CM.npy')
#weights = 1/torch.diag(torch.FloatTensor(weights))weight = weights
criterion = nn.CrossEntropyLoss().cuda()
#criterion = nn.CosineEmbeddingLoss().cuda()
Optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
torch.save(
model.state_dict(),
'weights/lr8e-3_bs8_sen2_' + args.basenet + '/' + 'LCZ42_SGD' + '.pth')
for epoch in range(args.start_epoch, args.epochs):
#adjust_learning_rate(Optimizer, epoch)
# train for one epoch
train(Dataloader_train, model, criterion, Optimizer, epoch,
Dataloader_validation
) #train(Dataloader_train, Network, criterion, Optimizer, epoch)
def main():
#create model
best_prec1 = 0
idxs = np.load('data/MulResampleIndex.npy')
Dataset_train = H5DatasetSiaResample(root=args.dataset_root,
mode='training',
indices=idxs,
symmetrize=True)
Dataloader_train = data.DataLoader(Dataset_train,
args.batch_size,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
idxs_val_fortrain = np.load('data/MulResampleIndex_forval.npy')
Dataset_val = H5DatasetSiaResample(root=args.dataset_root,
mode='validation',
indices=idxs_val_fortrain,
symmetrize=True)
Dataloader_validation_fortrain = data.DataLoader(
Dataset_val,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
#idxs_val = np.load('data/rand_tiers_1_val.npy')
Dataset_val = H5DatasetSia(root=args.dataset_root,
mode='validation_copy',
symmetrize=False)
Dataloader_validation = data.DataLoader(Dataset_val,
batch_size=1,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
'''
idxs_val = np.load('data/MulResampleIndex.npy')
Dataset_train = H5DatasetSoftAnno(root = args.dataset_root, mode = 'training', indices = idxs_val)
Dataloader_train = data.DataLoader(Dataset_train, args.batch_size,
num_workers = args.num_workers,
shuffle = True, pin_memory = True)
Dataset_val = H5DatasetSia(root = args.dataset_root, mode = 'validation')
Dataloader_validation = data.DataLoader(Dataset_val, batch_size = 1,
num_workers = args.num_workers,
shuffle = True, pin_memory = True)
idxs_val = np.load('data/MulResampleIndex_forval.npy')
Dataset_validation = H5DatasetSoftAnno(root = args.dataset_root, mode = 'validation', indices = idxs_val)
Dataloader_validation_fortrain = data.DataLoader(Dataset_validation, batch_size = args.batch_size,
num_workers = args.num_workers,
shuffle = True, pin_memory = True)
'''
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.set_device(0)
model = NetworkFactory.ConsturctNetwork(args.basenet, args.resume).cuda()
model = model.cuda()
cudnn.benchmark = True
#weights = torch.FloatTensor(weights)
if args.MultiLabel == None:
#criterion = nn.CrossEntropyLoss().cuda()#criterion = DistillationLoss()
criterion = FocalLoss(gamma=2, alpha=0.25)
else:
criterion = nn.MultiLabelMarginLoss().cuda()
Optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
#torch.save(model.state_dict(), 'weights/bs32_8cat10channel_NonNeste_'+ args.basenet +'/'+ 'LCZ42_SGD' + '.pth')
prefix = 'weights/FL_WarmUp_Cosine_AllBalanced_bs32_8cat10channel_'
for epoch in range(args.start_epoch, args.epochs):
if epoch > 1:
adjust_learning_rate(Optimizer,
epoch,
mode='Cosine',
decay=math.sqrt(0.5))
# train for one epoch
if (epoch % 2 == 0):
train(Dataloader_train, model, criterion, Optimizer, epoch,
Dataloader_validation, args.AddNoise)
else:
train(Dataloader_validation_fortrain, model, criterion, Optimizer,
epoch, Dataloader_validation, args.AddNoise)
def __init__(self):
args = get_args()
self.writer = SummaryWriter(
log_dir='runs3a_224/' + 'lrpm0_' + str(args.lrpm0) + '_' +
'lrpm1_' + str(args.lrpm1) + '_' + 'layers0_' + str(args.layers0) +
'_' + 'layers1_' + str(args.layers1) + '_' + 'bs_' +
str(args.batch_size) + '_' + 'epochs_' + str(args.epochs) + '_' +
'cells' + str(args.cells) + '_' + args.criti + '_' + current_time)
self.modelfile='likelihood_models/'+'model224_3a' + 'lrpm0_' + str(args.lrpm0)+'_' \
+ 'lrpm1_' + str(args.lrpm1)+'_' \
+ 'layers0_' + str(args.layers0)+'_' \
+ 'layers1_' + str(args.layers1)+'_' \
+ 'bs_' + str(args.batch_size)+'_' \
+ 'cells_' + str(args.cells)+'_' \
+ args.criti \
+'_'+current_time
self.batch_size = args.batch_size
self.grid_rows = 11
self.map_rows = 224
self.thresh = Variable(torch.Tensor([args.thresh]))
self.layers0 = args.layers0
self.layers1 = args.layers1
self.epochs = args.epochs
self.criti = args.criti
self.cells = args.cells
self.likeli_model_l0 = perceptual_conv_real_224_l0(self.layers0)
self.likeli_model_l1 = perceptual_conv_real_224_l1(self.layers1)
args.gpu = True
if args.gpu == True and torch.cuda.is_available():
self.device = torch.device("cuda")
torch.set_default_tensor_type(torch.cuda.FloatTensor)
else:
self.device = torch.device("cpu")
torch.set_default_tensor_type(torch.FloatTensor)
if self.device == torch.device(
"cuda") and torch.cuda.device_count() > 1:
print("Use", torch.cuda.device_count(), 'GPUs')
self.likeli_model_l0 = nn.DataParallel(self.likeli_model_l0).cuda()
self.likeli_model_l1 = nn.DataParallel(self.likeli_model_l1).cuda()
else:
print("Use CPU")
self.optimizer0 = torch.optim.Adam(list(
self.likeli_model_l0.parameters()),
lr=args.lrpm0)
self.optimizer1 = torch.optim.Adam(
list(self.likeli_model_l1.parameters()) +
list(self.likeli_model_l0.parameters()),
lr=args.lrpm1)
if self.criti == 'mse':
self.criterion = nn.MSELoss()
elif self.criti == 'kld':
self.criterion = nn.KLDivLoss()
self.args = args
self.start_time = time.time()
#if self.args.gpu == True and torch.cuda.is_available():
if (self.args.gpu) and torch.cuda.is_available():
self.device = torch.device("cuda")
torch.set_default_tensor_type(torch.cuda.FloatTensor)
else:
self.device = torch.device("cpu")
torch.set_default_tensor_type(torch.FloatTensor)
self.grid_rows, self.grid_cols, self.grid_dirs = 11, 11, 4
self.map_rows, self.map_cols = 224, 224
self.max_scan_range = 3.5
self.min_scan_range = 0.1
self.map_2d = np.load('/home/sai/tb3-anl/dal/env_map.npy')
self.map_2d = (255 - self.map_2d) / 255
self.taken = np.arange(self.map_2d.size)[self.map_2d.flatten() == 1]
# self.laser_1d = None
self.xlim = (-self.args.xlim, self.args.xlim)
self.ylim = (-self.args.xlim, self.args.xlim)
self.longest = float(self.grid_dirs / 2 + self.grid_rows - 1 +
self.grid_cols -
1) #longest possible manhattan distance
self.cell_size = (self.xlim[1] - self.xlim[0]) / self.grid_rows
self.heading_resol = 2 * np.pi / self.grid_dirs
self.collision = False
self.scans_over_map = np.zeros((self.grid_rows, self.grid_cols, 360))
self.scans_over_map_high = np.zeros(
(self.map_rows, self.map_cols, 360))
self.scan_2d = np.zeros((self.map_rows, self.map_cols))
self.likelihood = torch.ones(
(self.grid_dirs, self.grid_rows, self.grid_cols),
device=torch.device(self.device))
self.likelihood = self.likelihood / self.likelihood.sum()
self.gt_likelihood = np.ones(
(self.grid_dirs, self.grid_rows, self.grid_cols))
self.gt_likelihood_high = np.ones(
(self.grid_dirs, self.map_rows, self.map_cols))
self.gt_likelihood_unnormalized = np.ones(
(self.grid_dirs, self.grid_rows, self.grid_cols))
self.gt_likelihood_unnormalized_high = np.ones(
(self.grid_dirs, self.grid_rows, self.grid_cols))
self.turtle_loc = np.zeros((self.map_rows, self.map_cols))
# what to do
# current pose: where the robot really is. motion incurs errors in pose
self.current_pose = Pose2d(0, 0, 0)
self.goal_pose = Pose2d(0, 0, 0)
self.last_pose = Pose2d(0, 0, 0)
self.perturbed_goal_pose = Pose2d(0, 0, 0)
self.start_pose = Pose2d(0, 0, 0)
#grid pose
self.true_grid = Grid(head=0, row=0, col=0)
self.bel_grid = Grid(head=0, row=0, col=0)
parser.add_argument('--exp', dest='exp', type=int, default=1)
args = parser.parse_args()
assert (not args.video is None or not args.img is None)
if args.UHD and args.scale == 1.0:
args.scale = 0.5
assert args.scale in [0.25, 0.5, 1.0, 2.0, 4.0]
if not args.img is None:
args.png = True
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.set_grad_enabled(False)
if torch.cuda.is_available():
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
if (args.fp16):
torch.set_default_tensor_type(torch.cuda.HalfTensor)
try:
from model.RIFE_HDv2 import Model
model = Model()
model.load_model(args.modelDir, -1)
print("Loaded v2.x HD model.")
except:
from model.RIFE_HD import Model
model = Model()
model.load_model(args.modelDir, -1)
print("Loaded v1.x HD model")
model.eval()
model.device()
if not args.video is None:
from distributions.funnel_cp import V_funnel_cp
from distributions.funnel_ncp import V_funnel_ncp
import numpy
from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict
from adapt_util.tune_param_classes.tune_param_setting_util import *
from experiments.experiment_obj import tuneinput_class
from experiments.experiment_obj import experiment, experiment_setting_dict
import torch
#from experiments.correctdist_experiments.prototype import check_mean_var
seedid = 2
numpy.random.seed(seedid)
torch.manual_seed(seedid)
precision_type = 'torch.DoubleTensor'
#precision_type = 'torch.FloatTensor'
torch.set_default_tensor_type(precision_type)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=100,
num_chains=1,
num_cpu=1,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=10,
is_float=False,
isstore_to_disk=False)
input_dict = {
"v_fun": [V_funnel_cp],
"epsilon": [0.1],
"alpha": [1e6],
"second_order": [True],
"evolve_L": [10],
def __init__(self, name="neural-ner-model", model_file=None, **kwargs):
self.params = {
"vocab_size":20000, # size of the (token) vocabulary
"trainable_word_embeddings":True, # Whether to tune the word embeddings or keep them fixed
"word_emb_transform_dim":0, # Dimension of the dense layer transforming the word embeddings (0 to skip)
"char_embedding_dim":48, # Dimension for the character embeddings (0 to skip the layer)
"normalise_chars":False, # Whether to normalise the characters in the tokens
"max_token_length":32, # Maximum number of characters per token (for the character embeddings)
"char_lstm_dim":48, # Dimension for the character-level biLSTM (0 to skip the layer)
"use_roberta_embeddings":False, # Whether to also include roberta embeddings
"nb_convo_layers":0, # Number of token-level convolutional layers (0 for no layers)
"token_kernel_size":5, # Kernel size for the convolutional layers
"token_filter_dim":128, # Filter size for the convolutional layers
"token_lstm_dim":128, # Dimension for the token-level biLSTM (0 to skip the layer)
"dense_dim":0, # Size of the dense layer (after convolutions and LSTMs)
"use_crf":False, # Whether to use a CRF as final layer (0 to skip the layer)
"dropout":0.3, # Dropout ratio (on all embeddings)
"optimiser":"Adam", # Optimisation algorithm
"epoch_length":50000, # number of training examples per epoch
"nb_epochs":5, # number of epochs
"lr":0.001, # learning rate
"batch_size":1, # Number of documents per batch
"gpu":0 # GPU index
}
self.params.update(kwargs)
self.name = name
# Setting up the GPU
if self.params["gpu"] is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = str(self.params["gpu"])
# gpu = tf.config.experimental.list_physical_devices('GPU')[self.params["gpu"]]
# tf.config.experimental.set_memory_growth(gpu, True)
# We load the Spacy standard model
self.nlp = spacy.load("en_core_web_md")
# If we need to reload an existing model
if model_file is not None:
f = h5py.File(model_file, 'r')
self.name = f.attrs["name"]
self.indices = json.loads(f.attrs['indices'])
self.char_indices = json.loads(f.attrs['char_indices'])
self.label_indices = json.loads(f.attrs['label_indices'])
self.label_indices_inverted = {i:l for l,i in self.label_indices.items()}
self.params.update(json.loads(f.attrs['params']))
self.params.update(kwargs)
self.model = tf.keras.models.load_model(model_file)
f.close()
else:
# Token indices
# CONVENTION: 0 is padding value and 1 for indices of unknown tokens
self.indices = {x.norm_:(x.rank+2) for x in self.nlp.vocab if x.has_vector
and x.rank < (self.params["vocab_size"]-2)}
# Character indices
self.char_indices = {c:i+2 for i, c in enumerate(CHARACTERS)}
# Output label indices
self.label_indices = {"O":0}
self.label_indices_inverted = {0:"O"}
for label in LABELS:
for biluo in "BILU":
index = len(self.label_indices)
full_label = "%s-%s"%(biluo, label)
self.label_indices[full_label] = index
self.label_indices_inverted[index] = full_label
# Empty model for now
self.model = None
if self.params["use_roberta_embeddings"]:
if self.params["gpu"] is not None:
is_using_gpu = spacy.prefer_gpu()
if is_using_gpu:
print("Using GPU for RoBERTa...")
torch.set_default_tensor_type("torch.cuda.FloatTensor")
self.roberta = spacy.load("en_trf_robertabase_lg")
def __init__(self, config):
self.config = config
if torch.cuda.is_available():
self.use_cuda = True
torch.set_default_tensor_type('torch.cuda.FloatTensor')
else:
self.use_cuda = False
torch.set_default_tensor_type('torch.FloatTensor')
self.nz = config.nz
self.optimizer = config.optimizer
self.resl = 2 # we start from 2^2 = 4
self.lr = config.lr
self.eps_drift = config.eps_drift
self.smoothing = config.smoothing
self.max_resl = config.max_resl
self.trns_tick = config.trns_tick
self.stab_tick = config.stab_tick
self.TICK = config.TICK
self.globalIter = 0
self.globalTick = 0
self.kimgs = 0
self.stack = 0
self.epoch = 0
self.fadein = {'gen': None, 'dis': None}
self.complete = {'gen': 0, 'dis': 0}
self.phase = 'init'
self.flag_flush_gen = False
self.flag_flush_dis = False
self.flag_add_noise = self.config.flag_add_noise
self.flag_add_drift = self.config.flag_add_drift
# network and cirterion
self.G = net.Generator(config)
self.D = net.Discriminator(config)
print('Generator structure: ')
print(self.G.model)
print('Discriminator structure: ')
print(self.D.model)
self.mse = torch.nn.MSELoss()
if self.use_cuda:
self.mse = self.mse.cuda()
torch.cuda.manual_seed(config.random_seed)
if config.n_gpu == 1:
self.G = torch.nn.DataParallel(self.G).cuda(device=0)
self.D = torch.nn.DataParallel(self.D).cuda(device=0)
else:
gpus = []
for i in range(config.n_gpu):
gpus.append(i)
self.G = torch.nn.DataParallel(self.G, device_ids=gpus).cuda()
self.D = torch.nn.DataParallel(self.D, device_ids=gpus).cuda()
# define tensors, ship model to cuda, and get dataloader.
self.renew_everything()
# tensorboard
self.use_tb = config.use_tb
if self.use_tb:
self.tb = tensorboard.tf_recorder()
from easydict import EasyDict as ED
from torch_ecg.cfg import Cfg
from torch_ecg.utils.utils_nn import compute_deconv_output_shape, compute_module_size
from torch_ecg.utils.misc import dict_to_str
from torch_ecg.models.nets import (
Activations,
Conv_Bn_Activation,
MultiConv,
DownSample,
ZeroPadding,
GlobalContextBlock,
)
if Cfg.torch_dtype.lower() == "double":
torch.set_default_tensor_type(torch.DoubleTensor)
__all__ = [
"ECG_YOLO",
]
class ResNetGCBlock(nn.Module):
""" NOT finished, NOT checked,
ResNet (basic, not bottleneck) block with global context
References:
-----------
[1] entry 0436 of CPSC2019
[2] https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
import torch
import torch.optim as optim
import torch.nn as nn
from torch.autograd import Variable
from torch.distributions.kl import kl_divergence
from agents.TRPO.trpo import trpo_step
from agents.TRPO.utils import *
####################3
from models.agent import StochasticPolicy, Value
from utils import device
####################
torch.set_default_tensor_type('torch.DoubleTensor')
def update_params(batch, policy_net, value_net, gamma, tau, l2_reg, max_kl,
damping, entropy_coef):
rewards = torch.Tensor(batch.reward)
masks = torch.Tensor(batch.mask)
actions = torch.Tensor(batch.action)
states = torch.Tensor(batch.state)
values = value_net(Variable(states))
returns = torch.Tensor(actions.size(0), 1)
deltas = torch.Tensor(actions.size(0), 1)
advantages = torch.Tensor(actions.size(0), 1)
parser.add_argument('--gamma',
default=0.1,
type=float,
help='Gamma update for SGD')
parser.add_argument('--visdom',
default=False,
type=str2bool,
help='Use visdom for loss visualization')
parser.add_argument('--save_folder',
default='weights/',
help='Directory for saving checkpoint models')
args = parser.parse_args()
if torch.cuda.is_available():
if args.cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if not args.cuda:
print("WARNING: It looks like you have a CUDA device, but aren't " +
"using CUDA.\nRun with --cuda for optimal training speed.")
torch.set_default_tensor_type('torch.FloatTensor')
else:
torch.set_default_tensor_type('torch.FloatTensor')
if not os.path.exists(args.save_folder):
os.mkdir(args.save_folder)
def train():
if args.dataset == 'COCO':
if args.dataset_root == VOC_ROOT:
if not os.path.exists(COCO_ROOT):
def main():
logger.auto_set_dir()
global args
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser()
parser.add_argument('--dataroot', default='/home/hutao/lab/pytorchgo/example/ROAD/data',
help='Path to source dataset')
parser.add_argument('--batchSize', type=int, default=1, help='input batch size')
parser.add_argument('--max_epoch', type=int, default=max_epoch, help='Number of training iterations')
parser.add_argument('--optimizer', type=str, default='Adam', help='Optimizer to use | SGD, Adam')
parser.add_argument('--lr', type=float, default=base_lr, help='learning rate')
parser.add_argument('--momentum', type=float, default=0.99, help='Momentum for SGD')
parser.add_argument('--beta1', type=float, default=0.9, help='beta1 for adam. default=0.5')
parser.add_argument('--weight_decay', type=float, default=0.0005, help='Weight decay')
parser.add_argument('--model', type=str, default='vgg16')
parser.add_argument('--gpu', type=int, default=2)
args = parser.parse_args()
print(args)
gpu = args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
cuda = torch.cuda.is_available()
torch.manual_seed(1337)
if cuda:
logger.info("random seed 1337")
torch.cuda.manual_seed(1337)
# Defining data loaders
kwargs = {'num_workers': 4, 'pin_memory': True, 'drop_last': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(
torchfcn.datasets.SYNTHIA('SYNTHIA', args.dataroot, split='train', transform=True, image_size=image_size),
batch_size=args.batchSize, shuffle=True, **kwargs)
val_loader = torch.utils.data.DataLoader(
torchfcn.datasets.CityScapes('cityscapes', args.dataroot, split='val', transform=True, image_size=image_size),
batch_size=1, shuffle=False)
target_loader = torch.utils.data.DataLoader(
torchfcn.datasets.CityScapes('cityscapes', args.dataroot, split='train', transform=True, image_size=image_size),
batch_size=args.batchSize, shuffle=True)
if cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if args.model == "vgg16":
model = origin_model = torchfcn.models.Seg_model(n_class=class_num)
vgg16 = torchfcn.models.VGG16(pretrained=True)
model.copy_params_from_vgg16(vgg16)
model_fix = torchfcn.models.Seg_model(n_class=class_num)
model_fix.copy_params_from_vgg16(vgg16)
elif args.model == "deeplabv2": # TODO may have problem!
model = origin_model = torchfcn.models.Res_Deeplab(num_classes=class_num, image_size=image_size)
saved_state_dict = model_zoo.load_url(Deeplabv2_restore_from)
new_params = model.state_dict().copy()
for i in saved_state_dict:
# Scale.layer5.conv2d_list.3.weight
i_parts = i.split('.')
# print i_parts
if not class_num == 19 or not i_parts[1] == 'layer5':
new_params['.'.join(i_parts[1:])] = saved_state_dict[i]
# print i_parts
model.load_state_dict(new_params)
model_fix = torchfcn.models.Res_Deeplab(num_classes=class_num, image_size=image_size)
model_fix.load_state_dict(new_params)
else:
raise ValueError("only support vgg16, deeplabv2!")
for param in model_fix.parameters():
param.requires_grad = False
netD = torchfcn.models.Domain_classifer_forAdapSegNet(n_class=class_num)
netD.apply(weights_init)
model_summary([model, netD])
if cuda:
model = model.cuda()
netD = netD.cuda()
# Defining optimizer
if args.optimizer == 'SGD':
raise ValueError("SGD is not prepared well..")
optim = torch.optim.SGD(
[
{'params': get_parameters(model, bias=False)},
{'params': get_parameters(model, bias=True),
'lr': args.lr * 2,
'weight_decay': args.weight_decay},
],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optimizer == 'Adam':
if args.model == "vgg16":
optim = torch.optim.Adam(
[
{'params': get_parameters(model, bias=False),
'weight_decay': args.weight_decay},
{'params': get_parameters(model, bias=True),
'lr': args.lr * 2,
'weight_decay': args.weight_decay},
],
lr=args.lr,
betas=(args.beta1, 0.999))
elif args.model == "deeplabv2":
optim = torch.optim.Adam(
origin_model.optim_parameters(args.lr),
lr=args.lr,
betas=(args.beta1, 0.999),
weight_decay=args.weight_decay)
else:
raise
else:
raise ValueError('Invalid optmizer argument. Has to be SGD or Adam')
optimD = torch.optim.Adam(netD.parameters(), lr=dis_lr, weight_decay=args.weight_decay, betas=(0.7, 0.999))
optimizer_summary([optim, optimD])
trainer = MyTrainer_ROAD(
cuda=cuda,
model=model,
model_fix=model_fix,
netD=netD,
optimizer=optim,
optimizerD=optimD,
train_loader=train_loader,
target_loader=target_loader,
val_loader=val_loader,
batch_size=args.batchSize,
image_size=image_size,
loss_print_interval=LOSS_PRINT_INTERVAL
)
trainer.epoch = 0
trainer.iteration = 0
trainer.train()
def run_inference_ss_vae(args):
"""
run inference for SS-VAE
:param args: arguments for SS-VAE
:return: None
"""
if args.use_cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if args.seed is not None:
set_seed(args.seed, args.use_cuda)
viz = None
if args.visualize:
from visdom import Visdom
viz = Visdom()
mkdir_p("./vae_results")
# batch_size: number of images (and labels) to be considered in a batch
ss_vae = SSVAE(z_dim=args.z_dim,
hidden_layers=args.hidden_layers,
epsilon_scale=args.epsilon_scale,
use_cuda=args.use_cuda,
aux_loss_multiplier=args.aux_loss_multiplier)
# setup the optimizer
adam_params = {"lr": args.learning_rate, "betas": (args.beta_1, 0.999)}
optimizer = Adam(adam_params)
# set up the loss(es) for inference setting the enum_discrete parameter builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
loss_basic = SVI(ss_vae.model, ss_vae.guide, optimizer, loss="ELBO", enum_discrete=args.enum_discrete)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
if args.aux_loss:
loss_aux = SVI(ss_vae.model_classify, ss_vae.guide_classify, optimizer, loss="ELBO")
losses.append(loss_aux)
try:
# setup the logger if a filename is provided
logger = None if args.logfile is None else open(args.logfile, "w")
data_loaders = setup_data_loaders(MNISTCached, args.use_cuda, args.batch_size, sup_num=args.sup_num)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
periodic_interval_batches = int(MNISTCached.train_data_size / (1.0 * args.sup_num))
# number of unsupervised examples
unsup_num = MNISTCached.train_data_size - args.sup_num
# initializing local variables to maintain the best validation accuracy
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# run inference for a certain number of epochs
for i in range(0, args.num_epochs):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = \
run_inference_for_epoch(data_loaders, losses, periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / args.sup_num, epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / unsup_num, epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
str_print = "{} epoch: avg losses {}".format(i, "{} {}".format(str_loss_sup, str_loss_unsup))
validation_accuracy = get_accuracy(data_loaders["valid"], ss_vae.classifier, args.batch_size)
str_print += " validation accuracy {}".format(validation_accuracy)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(data_loaders["test"], ss_vae.classifier, args.batch_size)
str_print += " test accuracy {}".format(test_accuracy)
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < validation_accuracy:
best_valid_acc = validation_accuracy
corresponding_test_acc = test_accuracy
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(data_loaders["test"], ss_vae.classifier, args.batch_size)
print_and_log(logger, "best validation accuracy {} corresponding testing accuracy {} "
"last testing accuracy {}".format(best_valid_acc, corresponding_test_acc, final_test_accuracy))
# visualize the conditional samples
visualize(ss_vae, viz, data_loaders["test"])
finally:
# close the logger file object if we opened it earlier
if args.logfile is not None:
logger.close()
def glimpse(
dataset: str = typer.Option(
partial(get_default, "dataset"), help="Dataset name", prompt="Dataset name"
),
P: int = typer.Option(
partial(get_default, "P"),
"--aoi-size",
"-P",
help="AOI image size - number of pixels along the axis",
prompt="AOI image size - number of pixels along the axis",
),
frame_range: bool = typer.Option(
partial(get_default, "frame-range"),
help="Specify frame range.",
prompt="Specify frame range?",
callback=deactivate_prompts,
),
frame_start: Optional[int] = typer.Option(
partial(get_default, "frame-start"),
help="Starting frame.",
prompt="Starting frame",
),
frame_end: Optional[int] = typer.Option(
partial(get_default, "frame-end"),
help="Ending frame.",
prompt="Ending frame",
),
num_channels: int = typer.Option(
partial(get_default, "num-channels"),
"--num-channels",
"-C",
help="Number of color channels",
prompt="Number of color channels",
),
use_offtarget: bool = typer.Option(
partial(get_default, "use-offtarget"),
help="Use off-target AOI locations.",
prompt="Use off-target AOI locations?",
),
name: Optional[List[str]] = typer.Option(partial(get_default, "name")),
glimpse_folder: Optional[List[str]] = typer.Option(
partial(get_default, "glimpse-folder")
),
ontarget_aoiinfo: Optional[List[str]] = typer.Option(
partial(get_default, "ontarget-aoiinfo")
),
offtarget_aoiinfo: Optional[List[str]] = typer.Option(
partial(get_default, "offtarget-aoiinfo")
),
driftlist: Optional[List[str]] = typer.Option(partial(get_default, "driftlist")),
overwrite: bool = typer.Option(
True,
"--overwrite",
"-w",
help="Overwrite defaults values.",
prompt="Overwrite defaults values?",
),
no_input: bool = typer.Option(
False,
"--no-input",
help="Disable interactive prompt.",
is_eager=True,
callback=deactivate_prompts,
),
labels: bool = typer.Option(
False,
"--labels",
"-l",
help="Add on-target binding labels.",
),
progress_bar=None,
):
"""
Extract AOIs from raw glimpse images.
Analyzing data acquired with Glimpse and pre-processed with the imscroll program
will require the following files:
* image data in glimpse format and header file\n
* aoiinfo file designating the locations of target molecules (on-target AOIs) in the binder channel\n
* (optional) aoiinfo file designating the off-target control locations (off-target AOIs) in the binder channel\n
* driftlist file recording the stage movement that took place during the experiment
"""
from tapqir.imscroll import read_glimpse
torch.set_default_tensor_type(torch.FloatTensor)
global DEFAULTS
cd = DEFAULTS["cd"]
if progress_bar is None:
progress_bar = tqdm
# fill in default values
DEFAULTS["dataset"] = dataset
DEFAULTS["P"] = P
DEFAULTS["num-channels"] = num_channels
DEFAULTS["frame-range"] = frame_range
DEFAULTS["frame-start"] = frame_start
DEFAULTS["frame-end"] = frame_end
DEFAULTS["use-offtarget"] = use_offtarget
DEFAULTS["labels"] = labels
# inputs descriptions
desc = {}
desc["name"] = "Channel name"
desc["glimpse-folder"] = "Header/glimpse folder"
desc["driftlist"] = "Driftlist file"
desc["ontarget-aoiinfo"] = "Target molecule locations file"
desc["offtarget-aoiinfo"] = "Off-target control locations file"
desc["ontarget-labels"] = "On-target AOI binding labels"
desc["offtarget-labels"] = "Off-target AOI binding labels"
keys = [
"name",
"glimpse-folder",
"ontarget-aoiinfo",
"offtarget-aoiinfo",
"driftlist",
]
flags = [name, glimpse_folder, ontarget_aoiinfo, offtarget_aoiinfo, driftlist]
for c in range(num_channels):
if len(DEFAULTS["channels"]) < c + 1:
DEFAULTS["channels"].append({})
for flag, key in zip(flags, keys):
DEFAULTS["channels"][c][key] = flag[c] if c < len(flag) else None
# interactive input
if not no_input:
for c in range(num_channels):
if labels:
keys += ["ontarget-labels", "offtarget-labels"]
typer.echo(f"\nINPUTS FOR CHANNEL #{c}\n")
for key in keys:
if key == "offtarget-aoiinfo" and not use_offtarget:
continue
DEFAULTS["channels"][c][key] = typer.prompt(
desc[key], default=DEFAULTS["channels"][c][key]
)
DEFAULTS = dict(DEFAULTS)
for c in range(num_channels):
DEFAULTS["channels"][c] = dict(DEFAULTS["channels"][c])
if overwrite:
with open(cd / ".tapqir" / "config.yaml", "w") as cfg_file:
yaml.dump(
{key: value for key, value in DEFAULTS.items() if key != "cd"},
cfg_file,
sort_keys=False,
)
typer.echo("Extracting AOIs ...")
read_glimpse(
path=cd,
progress_bar=progress_bar,
**DEFAULTS,
)
typer.echo("Extracting AOIs: Done")
from functools import wraps
from itertools import product
from copy import deepcopy
from numbers import Number
import __main__
import errno
import torch
import torch.cuda
from torch.autograd import Variable
from torch._six import string_classes
import torch.backends.cudnn
torch.set_default_tensor_type('torch.DoubleTensor')
torch.backends.cudnn.disable_global_flags()
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('--seed', type=int, default=1234)
parser.add_argument('--accept', action='store_true')
args, remaining = parser.parse_known_args()
SEED = args.seed
ACCEPT = args.accept
UNITTEST_ARGS = [sys.argv[0]] + remaining
torch.manual_seed(SEED)
def run_tests():
unittest.main(argv=UNITTEST_ARGS)
def main(args):
if args.cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
logging.info('Loading data')
data = poly.load_data(poly.JSB_CHORALES)
logging.info('-' * 40)
model = models[args.model]
logging.info('Training {} on {} sequences'.format(
model.__name__, len(data['train']['sequences'])))
sequences = data['train']['sequences']
lengths = data['train']['sequence_lengths']
# find all the notes that are present at least once in the training set
present_notes = ((sequences == 1).sum(0).sum(0) > 0)
# remove notes that are never played (we remove 37/88 notes)
sequences = sequences[..., present_notes]
if args.truncate:
lengths = lengths.clamp(max=args.truncate)
sequences = sequences[:, :args.truncate]
num_observations = float(lengths.sum())
pyro.set_rng_seed(args.seed)
pyro.clear_param_store()
# We'll train using MAP Baum-Welch, i.e. MAP estimation while marginalizing
# out the hidden state x. This is accomplished via an automatic guide that
# learns point estimates of all of our conditional probability tables,
# named probs_*.
guide = AutoDelta(
poutine.block(model,
expose_fn=lambda msg: msg["name"].startswith("probs_")))
# To help debug our tensor shapes, let's print the shape of each site's
# distribution, value, and log_prob tensor. Note this information is
# automatically printed on most errors inside SVI.
if args.print_shapes:
first_available_dim = -2 if model is model_0 else -3
guide_trace = poutine.trace(guide).get_trace(
sequences, lengths, args=args, batch_size=args.batch_size)
model_trace = poutine.trace(
poutine.replay(poutine.enum(model, first_available_dim),
guide_trace)).get_trace(sequences,
lengths,
args=args,
batch_size=args.batch_size)
logging.info(model_trace.format_shapes())
# Enumeration requires a TraceEnum elbo and declaring the max_plate_nesting.
# All of our models have two plates: "data" and "tones".
optim = Adam({'lr': args.learning_rate})
if args.tmc:
if args.jit:
raise NotImplementedError(
"jit support not yet added for TraceTMC_ELBO")
elbo = TraceTMC_ELBO(max_plate_nesting=1 if model is model_0 else 2)
tmc_model = poutine.infer_config(model, lambda msg: {
"num_samples": args.tmc_num_samples,
"expand": False
} if msg["infer"].get("enumerate", None) == "parallel" else {}
) # noqa: E501
svi = SVI(tmc_model, guide, optim, elbo)
else:
Elbo = JitTraceEnum_ELBO if args.jit else TraceEnum_ELBO
elbo = Elbo(max_plate_nesting=1 if model is model_0 else 2,
strict_enumeration_warning=(model is not model_7),
jit_options={"time_compilation": args.time_compilation})
svi = SVI(model, guide, optim, elbo)
# We'll train on small minibatches.
logging.info('Step\tLoss')
for step in range(args.num_steps):
loss = svi.step(sequences,
lengths,
args=args,
batch_size=args.batch_size)
logging.info('{: >5d}\t{}'.format(step, loss / num_observations))
if args.jit and args.time_compilation:
logging.debug('time to compile: {} s.'.format(
elbo._differentiable_loss.compile_time))
# We evaluate on the entire training dataset,
# excluding the prior term so our results are comparable across models.
train_loss = elbo.loss(model,
guide,
sequences,
lengths,
args,
include_prior=False)
logging.info('training loss = {}'.format(train_loss / num_observations))
# Finally we evaluate on the test dataset.
logging.info('-' * 40)
logging.info('Evaluating on {} test sequences'.format(
len(data['test']['sequences'])))
sequences = data['test']['sequences'][..., present_notes]
lengths = data['test']['sequence_lengths']
if args.truncate:
lengths = lengths.clamp(max=args.truncate)
num_observations = float(lengths.sum())
# note that since we removed unseen notes above (to make the problem a bit easier and for
# numerical stability) this test loss may not be directly comparable to numbers
# reported on this dataset elsewhere.
test_loss = elbo.loss(model,
guide,
sequences,
lengths,
args=args,
include_prior=False)
logging.info('test loss = {}'.format(test_loss / num_observations))
# We expect models with higher capacity to perform better,
# but eventually overfit to the training set.
capacity = sum(
value.reshape(-1).size(0) for value in pyro.get_param_store().values())
logging.info('{} capacity = {} parameters'.format(model.__name__,
capacity))
# Owner(s): ["module: onnx"]
import functools
import os
import unittest
import sys
import torch
import torch.autograd.function as function
pytorch_test_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(-1, pytorch_test_dir)
from torch.testing._internal.common_utils import * # noqa: F401,F403
torch.set_default_tensor_type("torch.FloatTensor")
BATCH_SIZE = 2
RNN_BATCH_SIZE = 7
RNN_SEQUENCE_LENGTH = 11
RNN_INPUT_SIZE = 5
RNN_HIDDEN_SIZE = 3
def _skipper(condition, reason):
def decorator(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
if condition():
raise unittest.SkipTest(reason)
return f(*args, **kwargs)
def FastStyleTransfer(unique_id, iterations, content_img, style_img,
export_img):
SEED = 1081
np.random.seed(SEED)
torch.manual_seed(SEED)
if torch.cuda.is_available():
torch.cuda.manual_seed(SEED)
kwargs = {'num_workers': 4, 'pin_memory': True}
else:
kwargs = {}
IMAGE_SIZE = 224
BATCH_SIZE = 4
DATASET = "./fast_neural_style_transfer/coco_2017" # Downloaded from http://images.cocodataset.org/zips/val2017.zip
transform = transforms.Compose([
transforms.Resize(IMAGE_SIZE),
transforms.CenterCrop(IMAGE_SIZE),
transforms.ToTensor(),
tensor_normalizer()
])
# http://pytorch.org/docs/master/torchvision/datasets.html#imagefolder
train_dataset = datasets.ImageFolder(DATASET, transform)
# http://pytorch.org/docs/master/data.html#torch.utils.data.DataLoader
train_loader = DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
**kwargs)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
with torch.no_grad():
loss_network = LossNetwork()
loss_network.to(device)
loss_network.eval()
STYLE_IMAGE = style_img # "./fast-neural-style-transfer/style_images/mosaic.jpg" # style arg
# STYLE_IMAGE = "./style_images/candy.jpg"
style_img = Image.open(STYLE_IMAGE).convert('RGB')
with torch.no_grad():
style_img_tensor = transforms.Compose([
# transforms.Resize(IMAGE_SIZE* 2),
transforms.ToTensor(),
tensor_normalizer()
])(style_img).unsqueeze(0)
# assert np.sum(style_img - recover_image(style_img_tensor.numpy())[0].astype(np.uint8)) < 3 * style_img_tensor.size()[2] * style_img_tensor.size()[3]
style_img_tensor = style_img_tensor.to(device)
# Style image
#plt.imshow(recover_image(style_img_tensor.cpu().numpy())[0])
# http://pytorch.org/docs/master/notes/autograd.html#volatile
with torch.no_grad():
style_loss_features = loss_network(style_img_tensor)
gram_style = [gram_matrix(y) for y in style_loss_features]
for i in range(len(style_loss_features)):
tmp = style_loss_features[i].cpu().numpy()
print(i, np.mean(tmp), np.std(tmp))
for i in range(len(style_loss_features)):
print(i, gram_style[i].numel(), gram_style[i].size())
def save_debug_image(tensor_orig, tensor_transformed, tensor_with_noise,
filename):
assert tensor_orig.size() == tensor_transformed.size()
result = Image.fromarray(
recover_image(tensor_transformed.cpu().numpy())[0])
noise = Image.fromarray(
recover_image(tensor_with_noise.cpu().numpy())[0])
orig = Image.fromarray(recover_image(tensor_orig.cpu().numpy())[0])
new_im = Image.new('RGB', (result.size[0] * 3 + 10, result.size[1]))
new_im.paste(orig, (0, 0))
new_im.paste(result, (result.size[0] + 5, 0))
new_im.paste(noise, (result.size[0] * 2 + 10, 0))
new_im.save(filename)
transformer = TransformerNet()
mse_loss = torch.nn.MSELoss()
# l1_loss = torch.nn.L1Loss()
transformer.to(device)
torch.set_default_tensor_type('torch.FloatTensor')
def train(steps, base_steps=0):
transformer.train()
count = 0
agg_content_loss = 0.
agg_style_loss = 0.
agg_reg_loss = 0.
agg_stable_loss = 0.
while True:
print(0)
for x, _ in train_loader:
print(0)
count += 1
optimizer.zero_grad()
x = x.to(device)
y = transformer(x)
with torch.no_grad():
mask = torch.bernoulli(
torch.ones_like(x, device=device, dtype=torch.float) *
NOISE_P)
noise = torch.normal(
torch.zeros_like(x),
torch.ones_like(x, device=device, dtype=torch.float) *
NOISE_STD).clamp(-1, 1)
# print((noise * mask).sum())
y_noise = transformer(x + noise * mask)
print(0)
with torch.no_grad():
xc = x.detach()
features_xc = loss_network(xc)
features_y = loss_network(y)
print(0)
with torch.no_grad():
f_xc_c = features_xc[2].detach()
content_loss = CONTENT_WEIGHT * mse_loss(features_y[2], f_xc_c)
print(0)
reg_loss = REGULARIZATION * (
torch.sum(torch.abs(y[:, :, :, :-1] - y[:, :, :, 1:])) +
torch.sum(torch.abs(y[:, :, :-1, :] - y[:, :, 1:, :])))
print(0)
style_loss = 0.
for l, weight in enumerate(STYLE_WEIGHTS):
gram_s = gram_style[l]
gram_y = gram_matrix(features_y[l])
style_loss += float(weight) * mse_loss(
gram_y, gram_s.expand_as(gram_y))
stability_loss = NOISE_WEIGHT * mse_loss(
y_noise.view(-1),
y.view(-1).detach())
print(0)
total_loss = content_loss + style_loss + reg_loss + stability_loss
total_loss.backward()
optimizer.step()
agg_content_loss += content_loss
agg_style_loss += style_loss
agg_reg_loss += reg_loss
agg_stable_loss += stability_loss
print(0)
if count % LOG_INTERVAL == 0:
mesg = "{} [{}/{}] content: {:.2f} style: {:.2f} reg: {:.2f} stable: {:.2f} total: {:.6f}".format(
time.ctime(), count, steps,
agg_content_loss / LOG_INTERVAL,
agg_style_loss / LOG_INTERVAL,
agg_reg_loss / LOG_INTERVAL,
agg_stable_loss / LOG_INTERVAL,
(agg_content_loss + agg_style_loss + agg_reg_loss +
agg_stable_loss) / LOG_INTERVAL)
print(mesg)
agg_content_loss = 0.
agg_style_loss = 0.
agg_reg_loss = 0.
agg_stable_loss = 0.
transformer.eval()
y = transformer(x)
save_debug_image(
x, y.detach(), y_noise.detach(),
"./fast_neural_style_transfer/debug/{}.png".format(
base_steps + count))
transformer.train()
print(0)
if count >= steps:
return
CONTENT_WEIGHT = 1
STYLE_WEIGHTS = np.array([1e-1, 1, 1e1, 5, 1e1]) * 5e3
REGULARIZATION = 1e-6
NOISE_P = 0.2
NOISE_STD = 0.35
NOISE_WEIGHT = 10 * 2
LOG_INTERVAL = 50
LR = 1e-3
optimizer = Adam(transformer.parameters(), LR)
print("Started Training...")
train(iterations, 0) # iteration arg
print("Finished Training...")
save_model_path = "./fast_neural_style_transfer/models/" + str(
unique_id) + "_weights.pth"
torch.save(transformer.state_dict(), save_model_path)
import glob
fnames = glob.glob(DATASET + r"/*/*")
transformer = transformer.eval()
img = Image.open(content_img).convert('RGB')
transform = transforms.Compose(
[transforms.Resize(512),
transforms.ToTensor(),
tensor_normalizer()])
img_tensor = transform(img).unsqueeze(0)
print(img_tensor.size())
if torch.cuda.is_available():
img_tensor = img_tensor.cuda()
img_output = transformer(Variable(img_tensor, volatile=True))
# Original content image
#plt.imshow(recover_image(img_tensor.cpu().numpy())[0])
# Content + Style
#plt.imshow(recover_image(img_output.data.cpu().numpy())[0])
output_img = Image.fromarray(
recover_image(img_output.data.cpu().numpy())[0])
output_img.save(export_img)
import numbers
import os
import unittest
import warnings
from copy import deepcopy
from itertools import product
import numpy as np
import pytest
import torch
import torch.cuda
from numpy.testing import assert_allclose
from pytest import approx
from torch.autograd import Variable
torch.set_default_tensor_type(os.environ.get('PYRO_TENSOR_TYPE', 'torch.DoubleTensor'))
"""
Contains test utilities for assertions, approximate comparison (of tensors and other objects).
Code has been largely adapted from pytorch/test/common.py
Source: https://github.com/pytorch/pytorch/blob/master/test/common.py
"""
TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
RESOURCE_DIR = os.path.join(TESTS_DIR, 'resources')
EXAMPLES_DIR = os.path.join(os.path.dirname(TESTS_DIR), 'examples')
def suppress_warnings(fn):
def wrapper(*args, **kwargs):
def cross_val(args):
torch.set_default_tensor_type('torch.DoubleTensor')
allele_list_9 = [
'HLA-A*02:01', 'HLA-A*03:01', 'HLA-A*11:01', 'HLA-A*02:03',
'HLA-B*15:01', 'HLA-A*31:01', 'HLA-A*01:01', 'HLA-B*07:02',
'HLA-A*26:01', 'HLA-A*02:06', 'HLA-A*68:02', 'HLA-B*08:01',
'HLA-B*58:01', 'HLA-B*40:01', 'HLA-B*27:05', 'HLA-A*30:01',
'HLA-A*69:01', 'HLA-B*57:01', 'HLA-B*35:01', 'HLA-A*02:02',
'HLA-A*24:02', 'HLA-B*18:01', 'HLA-B*51:01', 'HLA-A*29:02',
'HLA-A*68:01', 'HLA-A*33:01', 'HLA-A*23:01'
]
allele_list_10 = [
'HLA-A*02:01', 'HLA-A*03:01', 'HLA-A*11:01', 'HLA-A*68:01',
'HLA-A*31:01', 'HLA-A*02:06', 'HLA-A*68:02', 'HLA-A*02:03',
'HLA-A*33:01', 'HLA-A*02:02'
]
if not os.path.exists(args.savedir):
os.mkdir(args.savedir)
logFileLoc = args.savedir + os.sep + args.testFile
if os.path.isfile(logFileLoc):
logger = open(logFileLoc, 'a')
logger.write("%s\t%s\t\t\t%s\t\t\t%s\t\t\t%s\n" %
('Length', 'Allele', 'Pearson', 'AUC', 'SRCC'))
logger.flush()
else:
logger = open(logFileLoc, 'w')
logger.write("%s\t%s\t\t\t%s\t\t\t%s\t\t\t%s\n" %
('Length', 'Allele', 'Pearson', 'AUC', 'SRCC'))
logger.flush()
for length in [10, 9]:
if length == 9:
allele_list = allele_list_9
elif length == 10:
allele_list = allele_list_10
else:
print("Invalid Length")
exit(0)
for allele in allele_list: #[9,10]
model_dir = args.savedir + os.sep + 'best_model' + os.sep + allele
if not os.path.isdir(model_dir):
os.makedirs(model_dir)
data_dict = pickle.load(
open(
args.data_dir + os.sep + 'pickle_' + str(length) + os.sep +
allele.replace('*', '.').replace(':', '_') + '.p', 'rb'))
print('test on allele: ' + data_dict['allele'])
if not length == data_dict['sequ_length']:
print('length error')
exit()
encode_channel = data_dict['channel_encode']
meas = data_dict['label']
bind = []
for i in meas:
i = (-1) * math.log10(i)
bind.append(i)
sequ, label = encode_channel, bind
if (len(sequ) > 0):
sequ_ori, label_ori = sequ, label
output_list = []
label_list = []
fold_num = 0
kf = KFold(n_splits=5, shuffle=True, random_state=42)
for train_set, test_set in kf.split(sequ_ori, label_ori):
fold_num += 1
train_sequ, test_sequ, train_label, test_label = [sequ_ori[i] for i in train_set], [sequ_ori[i] for i in test_set],\
[label_ori[i] for i in train_set], [label_ori[i] for i in test_set]
test_data_load = torch.utils.data.DataLoader(
myDataLoader.MyDataset(test_sequ, test_label),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
model = net.ResNetC1()
if args.onGPU == True:
#model = torch.nn.DataParallel(model, device_ids=[0,1,2,3]).cuda()
model = model.cuda()
criteria = MSELoss()
if args.onGPU == True:
criteria = criteria.cuda()
best_model_dict = torch.load(model_dir + os.sep + allele +
'_' + str(length) + '_' +
str(fold_num) + '.pth')
model.load_state_dict(best_model_dict)
_, _, output, label = val(args, test_data_load, model,
criteria)
output_list.extend(output)
label_list.extend(label)
IC_output_list = [
math.pow(10, (-1) * value) for value in output_list
]
IC_label_list = [
math.pow(10, (-1) * value) for value in label_list
]
bi_output_list = [
1 if ic < 500 else 0 for ic in IC_output_list
]
bi_label_list = [1 if ic < 500 else 0 for ic in IC_label_list]
pearson = pearsonr(output_list, label_list)
auc = roc_auc_score(bi_label_list, bi_output_list)
srcc = spearmanr(label_list, output_list)
logger.write("%s\t%s\t\t%.4f\t\t\t%.4f\t\t\t%.4f\n" %
(length, allele, pearson[0], auc, srcc[0]))
logger.flush()
prediction = args.savedir + os.sep + args.predict
if os.path.exists(prediction):
append_write = 'a' # append if already exists
else:
append_write = 'w'
true_value = open(prediction, append_write)
true_value.write("%s\n" % (allele))
for i in range(int(len(output_list) / 10)):
true_value.write("%.4f\t%.4f\n" %
(label_list[i], output_list[i]))
true_value.flush()
logger.close()
