def __init__(self, out_mlp_dims, emb_dim=22, dropout=0, n_neighbors=20): super().__init__() self.n_neighbors = n_neighbors self.bn1 = nn.BatchNorm2d(64) self.bn2 = nn.BatchNorm2d(64) self.bn3 = nn.BatchNorm2d(128) self.bn4 = nn.BatchNorm2d(256) self.bn5 = nn.BatchNorm1d(512) self.conv1 = nn.Sequential( nn.Conv2d(12, 64, kernel_size=1, bias=False), self.bn1, nn.LeakyReLU(negative_slope=0.2)) self.conv2 = nn.Sequential( nn.Conv2d(64 * 2, 64, kernel_size=1, bias=False), self.bn2, nn.LeakyReLU(negative_slope=0.2)) self.conv3 = nn.Sequential( nn.Conv2d(64 * 2, 128, kernel_size=1, bias=False), self.bn3, nn.LeakyReLU(negative_slope=0.2)) self.conv4 = nn.Sequential( nn.Conv2d(128 * 2, 256, kernel_size=1, bias=False), self.bn4, nn.LeakyReLU(negative_slope=0.2)) self.conv5 = nn.Sequential( nn.Conv1d(512, 512, kernel_size=1, bias=False), self.bn5, nn.LeakyReLU(negative_slope=0.2)) self.out_mlp = MLP(512, out_mlp_dims, emb_dim, torch.nn.GELU())
def __init__(self, input_dim, hidden_dims, nonlinearity, context_dim=0, event_dim=-1, scale_fn_type='exp', eps=1E-8, split_dim=None): """Implements affine coupling transform according to https://arxiv.org/abs/1605.08803 with choice between exponential and sigmoid scaling""" super().__init__() self.event_dim = event_dim self.input_dim = input_dim if split_dim == None: self.split_dim = input_dim // 2 else: self.split_dim = split_dim self.context_dim = context_dim out_dim = (self.input_dim - self.split_dim) * 2 self.nn = MLP(self.split_dim + context_dim, hidden_dims, out_dim, nonlinearity, residual=True) self.scale_fn_type = scale_fn_type if self.scale_fn_type == 'exp': self.scale_fn = lambda x: torch.exp(x) elif self.scale_fn_type == 'sigmoid': self.scale_fn = lambda x: (2 * torch.sigmoid(x) - 1) * (1 - eps ) + 1 else: raise Exception('Invalid scale_fn_type')
def __init__(self, input_dim, hidden_dims, nonlinearity, num_bins, context_dim=0, event_dim=-1): super().__init__() self.event_dim = event_dim self.input_dim = input_dim self.split_dim = input_dim//2 self.context_dim = context_dim self.num_bins = num_bins out_dim = (self.num_bins*3 + 1)*self.split_dim self.nn = MLP(self.split_dim + context_dim, hidden_dims, out_dim, nonlinearity, residual=True)
def __init__(self, image_size, image_channels, classes, fc_layers=3, fc_units=1000): # Configurations. super().__init__() self.classes = classes self.label = "Classifier" self.fc_layers = fc_layers # Attributes that need to be set before training. self.optimizer = None self.multi_head = None #--> <list> with for each task its class-IDs # Check whether there is at least 1 fc-layer. if fc_layers < 1: raise ValueError( "The classifier needs to have at least 1 fully-connected layer." ) ######------SPECIFY MODEL------###### # Flatten image to 2D-tensor. self.flatten = modules.Flatten() # Fully connected hidden layers. self.fcE = MLP(input_size=image_channels * image_size**2, output_size=fc_units, layers=fc_layers - 1, hid_size=fc_units) self.mlp_output_size = fc_units if fc_layers > 1 else image_channels * image_size**2 # Classifier. self.classifier = fc_layer(self.mlp_output_size, classes, nl='none', bias=True)
def __init__(self,input_dim,hidden_dims,nonlinearity,context_dim=0,event_dim=-1,algo='original',eps_expm=1e-8): super().__init__() self.event_dim = event_dim self.input_dim = input_dim self.scale = nn.Parameter(torch.ones(1)/8) self.shift = nn.Parameter(torch.zeros(1)) self.rescale = nn.Parameter(torch.ones(1)) self.reshift = nn.Parameter(torch.zeros(1)) self.split_dim = input_dim//2 self.algo = algo self.context_dim = context_dim self.eps_expm = eps_expm out_dim = (self.input_dim - self.split_dim)**2 + self.input_dim - self.split_dim self.nn = MLP(self.split_dim +context_dim,hidden_dims,out_dim,nonlinearity,residual=True)
def create_model(args, input_size, num_classes, sample_batch=None): if args.nn == 'mlp': model = MLP(input_size=input_size, hidden_size=args.hidden_size, num_layers=args.num_layers, weight_norm=args.weight_norm, num_classes=num_classes, init=args.init, sample_batch=sample_batch) elif args.nn == 'cnn': model = CNN(input_size=input_size, hidden_size=args.hidden_size, num_layers=args.num_layers, weight_norm=args.weight_norm, num_classes=num_classes, init=args.init, sample_batch=sample_batch) elif args.nn == 'resnet': model = ResNet(input_size=input_size, hidden_size=args.hidden_size, num_classes=num_classes, weight_norm=args.weight_norm, batch_norm=args.batch_norm, init=args.init, num_blocks=args.num_blocks, sample_batch=sample_batch, init_extra_param=args.init_extra_param) elif args.nn == 'wrn': model = WideResNet(input_size=input_size, num_blocks=args.wrn_n, num_classes=num_classes, weight_norm=args.weight_norm, batch_norm=args.batch_norm, init=args.init, sample_batch=sample_batch, k=args.wrn_k, reduced_memory=args.wrn_reduced_memory, init_extra_param=args.init_extra_param) else: raise NotImplementedError return model
class Classifier(nn.Module): '''Model for classifying images.''' def __init__(self, image_size, image_channels, classes, fc_layers=3, fc_units=1000): # Configurations. super().__init__() self.classes = classes self.label = "Classifier" self.fc_layers = fc_layers # Attributes that need to be set before training. self.optimizer = None self.multi_head = None #--> <list> with for each task its class-IDs # Check whether there is at least 1 fc-layer. if fc_layers < 1: raise ValueError( "The classifier needs to have at least 1 fully-connected layer." ) ######------SPECIFY MODEL------###### # Flatten image to 2D-tensor. self.flatten = modules.Flatten() # Fully connected hidden layers. self.fcE = MLP(input_size=image_channels * image_size**2, output_size=fc_units, layers=fc_layers - 1, hid_size=fc_units) self.mlp_output_size = fc_units if fc_layers > 1 else image_channels * image_size**2 # Classifier. self.classifier = fc_layer(self.mlp_output_size, classes, nl='none', bias=True) def _device(self): return next(self.parameters()).device def _is_on_cuda(self): return next(self.parameters()).is_cuda def list_init_layers(self): '''Return list of modules whose parameters could be initialized differently (i.e., conv- or fc-layers).''' list = [] list += self.fcE.list_init_layers() list += self.classifier.list_init_layers() return list @property def name(self): if self.fc_layers > 1: return "{}_c{}".format(self.fcE.name, self.classes) else: return "i{}_c{}".format(self.mlp_output_size, self.classes) def forward(self, x, task=None): # Run forward pass of model. final_features = self.fcE(self.flatten(x)) logits = self.classifier(final_features) # If using multi-headed output layer, select the correct "output-head" depending on provided task. # --> [task]: <int> task-ID (if all samples in [x] from same task) -OR- # <list>/<tensor> with task-IDs of all samples in [x] if task is None or self.multi_head is None: return logits elif type(task) == int: return logits[:, self.multi_head[task - 1]] else: task_indeces = [] for i in task: task_indeces.append(self.multi_head[i - 1]) return logits.gather(dim=1, index=torch.LongTensor(task_indeces).to( self._device())) def feature_extractor(self, images): return self.fcE(self.flatten(images)) def train_a_batch(self, x, y, task=None): '''Train model for one batch (`x`, `y`, `task`). Args: x (tensor): batch of inputs y (tensor): batch of corresponding labels task (int, list/tensor or None): taskID of entire batch (if int) or of each sample in batch (if list/tensor) ''' # Set model to training-mode self.train() # Reset optimizer self.optimizer.zero_grad() # Run model y_hat = self(x, task=task) # Calculate training-precision _, predicted = torch.max(y_hat, 1) correct = (y == predicted) precision = None if y is None else correct.sum().item() / x.size(0) # Get "gradients" required for updates (i.e. the "delta-Ws" or "desired updates") # -calculate prediction loss loss = F.cross_entropy(input=y_hat, target=y, reduction='mean') # -backpropagate errors loss.backward() # Take optimization-step self.optimizer.step() # Return the dictionary with training-loss and training-precision return { 'loss': loss.item(), 'precision': precision if precision is not None else 0., }