def __init__(self, out_mlp_dims, emb_dim=22, dropout=0, n_neighbors=20):
super().__init__()
self.n_neighbors = n_neighbors
self.bn1 = nn.BatchNorm2d(64)
self.bn2 = nn.BatchNorm2d(64)
self.bn3 = nn.BatchNorm2d(128)
self.bn4 = nn.BatchNorm2d(256)
self.bn5 = nn.BatchNorm1d(512)
self.conv1 = nn.Sequential(
nn.Conv2d(12, 64, kernel_size=1, bias=False), self.bn1,
nn.LeakyReLU(negative_slope=0.2))
self.conv2 = nn.Sequential(
nn.Conv2d(64 * 2, 64, kernel_size=1, bias=False), self.bn2,
nn.LeakyReLU(negative_slope=0.2))
self.conv3 = nn.Sequential(
nn.Conv2d(64 * 2, 128, kernel_size=1, bias=False), self.bn3,
nn.LeakyReLU(negative_slope=0.2))
self.conv4 = nn.Sequential(
nn.Conv2d(128 * 2, 256, kernel_size=1, bias=False), self.bn4,
nn.LeakyReLU(negative_slope=0.2))
self.conv5 = nn.Sequential(
nn.Conv1d(512, 512, kernel_size=1, bias=False), self.bn5,
nn.LeakyReLU(negative_slope=0.2))
self.out_mlp = MLP(512, out_mlp_dims, emb_dim, torch.nn.GELU())
def __init__(self,
input_dim,
hidden_dims,
nonlinearity,
context_dim=0,
event_dim=-1,
scale_fn_type='exp',
eps=1E-8,
split_dim=None):
"""Implements affine coupling transform according to https://arxiv.org/abs/1605.08803 with choice between exponential and sigmoid scaling"""
super().__init__()
self.event_dim = event_dim
self.input_dim = input_dim
if split_dim == None:
self.split_dim = input_dim // 2
else:
self.split_dim = split_dim
self.context_dim = context_dim
out_dim = (self.input_dim - self.split_dim) * 2
self.nn = MLP(self.split_dim + context_dim,
hidden_dims,
out_dim,
nonlinearity,
residual=True)
self.scale_fn_type = scale_fn_type
if self.scale_fn_type == 'exp':
self.scale_fn = lambda x: torch.exp(x)
elif self.scale_fn_type == 'sigmoid':
self.scale_fn = lambda x: (2 * torch.sigmoid(x) - 1) * (1 - eps
) + 1
else:
raise Exception('Invalid scale_fn_type')
def __init__(self, input_dim, hidden_dims, nonlinearity, num_bins, context_dim=0, event_dim=-1):
super().__init__()
self.event_dim = event_dim
self.input_dim = input_dim
self.split_dim = input_dim//2
self.context_dim = context_dim
self.num_bins = num_bins
out_dim = (self.num_bins*3 + 1)*self.split_dim
self.nn = MLP(self.split_dim + context_dim, hidden_dims,
out_dim, nonlinearity, residual=True)
def __init__(self,
image_size,
image_channels,
classes,
fc_layers=3,
fc_units=1000):
# Configurations.
super().__init__()
self.classes = classes
self.label = "Classifier"
self.fc_layers = fc_layers
# Attributes that need to be set before training.
self.optimizer = None
self.multi_head = None #--> <list> with for each task its class-IDs
# Check whether there is at least 1 fc-layer.
if fc_layers < 1:
raise ValueError(
"The classifier needs to have at least 1 fully-connected layer."
)
######------SPECIFY MODEL------######
# Flatten image to 2D-tensor.
self.flatten = modules.Flatten()
# Fully connected hidden layers.
self.fcE = MLP(input_size=image_channels * image_size**2,
output_size=fc_units,
layers=fc_layers - 1,
hid_size=fc_units)
self.mlp_output_size = fc_units if fc_layers > 1 else image_channels * image_size**2
# Classifier.
self.classifier = fc_layer(self.mlp_output_size,
classes,
nl='none',
bias=True)
def __init__(self,input_dim,hidden_dims,nonlinearity,context_dim=0,event_dim=-1,algo='original',eps_expm=1e-8):
super().__init__()
self.event_dim = event_dim
self.input_dim = input_dim
self.scale = nn.Parameter(torch.ones(1)/8)
self.shift = nn.Parameter(torch.zeros(1))
self.rescale = nn.Parameter(torch.ones(1))
self.reshift = nn.Parameter(torch.zeros(1))
self.split_dim = input_dim//2
self.algo = algo
self.context_dim = context_dim
self.eps_expm = eps_expm
out_dim = (self.input_dim - self.split_dim)**2 + self.input_dim - self.split_dim
self.nn = MLP(self.split_dim +context_dim,hidden_dims,out_dim,nonlinearity,residual=True)
def create_model(args, input_size, num_classes, sample_batch=None):
if args.nn == 'mlp':
model = MLP(input_size=input_size, hidden_size=args.hidden_size, num_layers=args.num_layers,
weight_norm=args.weight_norm, num_classes=num_classes, init=args.init,
sample_batch=sample_batch)
elif args.nn == 'cnn':
model = CNN(input_size=input_size, hidden_size=args.hidden_size, num_layers=args.num_layers,
weight_norm=args.weight_norm, num_classes=num_classes, init=args.init,
sample_batch=sample_batch)
elif args.nn == 'resnet':
model = ResNet(input_size=input_size, hidden_size=args.hidden_size, num_classes=num_classes,
weight_norm=args.weight_norm, batch_norm=args.batch_norm, init=args.init,
num_blocks=args.num_blocks, sample_batch=sample_batch,
init_extra_param=args.init_extra_param)
elif args.nn == 'wrn':
model = WideResNet(input_size=input_size, num_blocks=args.wrn_n, num_classes=num_classes,
weight_norm=args.weight_norm, batch_norm=args.batch_norm, init=args.init,
sample_batch=sample_batch, k=args.wrn_k, reduced_memory=args.wrn_reduced_memory,
init_extra_param=args.init_extra_param)
else:
raise NotImplementedError
return model
class Classifier(nn.Module):
'''Model for classifying images.'''
def __init__(self,
image_size,
image_channels,
classes,
fc_layers=3,
fc_units=1000):
# Configurations.
super().__init__()
self.classes = classes
self.label = "Classifier"
self.fc_layers = fc_layers
# Attributes that need to be set before training.
self.optimizer = None
self.multi_head = None #--> <list> with for each task its class-IDs
# Check whether there is at least 1 fc-layer.
if fc_layers < 1:
raise ValueError(
"The classifier needs to have at least 1 fully-connected layer."
)
######------SPECIFY MODEL------######
# Flatten image to 2D-tensor.
self.flatten = modules.Flatten()
# Fully connected hidden layers.
self.fcE = MLP(input_size=image_channels * image_size**2,
output_size=fc_units,
layers=fc_layers - 1,
hid_size=fc_units)
self.mlp_output_size = fc_units if fc_layers > 1 else image_channels * image_size**2
# Classifier.
self.classifier = fc_layer(self.mlp_output_size,
classes,
nl='none',
bias=True)
def _device(self):
return next(self.parameters()).device
def _is_on_cuda(self):
return next(self.parameters()).is_cuda
def list_init_layers(self):
'''Return list of modules whose parameters could be initialized differently (i.e., conv- or fc-layers).'''
list = []
list += self.fcE.list_init_layers()
list += self.classifier.list_init_layers()
return list
@property
def name(self):
if self.fc_layers > 1:
return "{}_c{}".format(self.fcE.name, self.classes)
else:
return "i{}_c{}".format(self.mlp_output_size, self.classes)
def forward(self, x, task=None):
# Run forward pass of model.
final_features = self.fcE(self.flatten(x))
logits = self.classifier(final_features)
# If using multi-headed output layer, select the correct "output-head" depending on provided task.
# --> [task]: <int> task-ID (if all samples in [x] from same task) -OR-
# <list>/<tensor> with task-IDs of all samples in [x]
if task is None or self.multi_head is None:
return logits
elif type(task) == int:
return logits[:, self.multi_head[task - 1]]
else:
task_indeces = []
for i in task:
task_indeces.append(self.multi_head[i - 1])
return logits.gather(dim=1,
index=torch.LongTensor(task_indeces).to(
self._device()))
def feature_extractor(self, images):
return self.fcE(self.flatten(images))
def train_a_batch(self, x, y, task=None):
'''Train model for one batch (`x`, `y`, `task`).
Args:
x (tensor): batch of inputs
y (tensor): batch of corresponding labels
task (int, list/tensor or None): taskID of entire batch (if int) or of each sample in batch (if list/tensor)
'''
# Set model to training-mode
self.train()
# Reset optimizer
self.optimizer.zero_grad()
# Run model
y_hat = self(x, task=task)
# Calculate training-precision
_, predicted = torch.max(y_hat, 1)
correct = (y == predicted)
precision = None if y is None else correct.sum().item() / x.size(0)
# Get "gradients" required for updates (i.e. the "delta-Ws" or "desired updates")
# -calculate prediction loss
loss = F.cross_entropy(input=y_hat, target=y, reduction='mean')
# -backpropagate errors
loss.backward()
# Take optimization-step
self.optimizer.step()
# Return the dictionary with training-loss and training-precision
return {
'loss': loss.item(),
'precision': precision if precision is not None else 0.,
}