def __init__(self, in_size, out_size, nl=nn.ReLU(), drop=0., bias=True, excitability=False, excit_buffer=False, batch_norm=False, gated=False): super().__init__() if drop > 0: self.dropout = nn.Dropout(drop) self.linear = em.LinearExcitability(in_size, out_size, bias=False if batch_norm else bias, excitability=excitability, excit_buffer=excit_buffer) if batch_norm: self.bn = nn.BatchNorm1d(out_size) if gated: self.gate = nn.Linear(in_size, out_size) self.sigmoid = nn.Sigmoid() if isinstance(nl, nn.Module): self.nl = nl elif not nl == "none": self.nl = nn.ReLU() if nl == "relu" else ( nn.LeakyReLU() if nl == "leakyrelu" else utils.Identity())
def __init__(self, num_class): super(Jigsaw, self).__init__() self.feature = torchvision.models.resnet18(pretrained=False) self.feature_dim = self.feature.fc.in_features self.feature.fc = utils.Identity() # discard last fc layer self.classifier = nn.Linear(self.feature_dim, num_class) self.ceLoss = nn.CrossEntropyLoss()
def set_activation(self, nl): if isinstance(nl, nn.Module): self.nl = nl elif nl == "relu": self.nl = nn.ReLU() elif nl == "leakyrelu": self.nl = nn.LeakyReLU() else: self.nl = utils.Identity()
def __init__(self, in_size, out_size, drop=0., bias=True, excitability=False, excit_buffer=False, batch_norm=False, nl="relu"): super().__init__() self.dropout = nn.Dropout(drop) self.linear = em.LinearExcitability(in_size, out_size, bias=False if batch_norm else bias, excitability=excitability, excit_buffer=excit_buffer) self.bn = nn.BatchNorm1d(out_size) if batch_norm else utils.Identity() self.nl = nn.ReLU() if nl == "relu" else ( nn.LeakyReLU() if nl == "leakyrelu" else utils.Identity())
def __init__(self, num_class): super(ImprintJigsaw, self).__init__() self.num_class = num_class self.feature = torchvision.models.resnet18(pretrained=False) self.feature_dim = self.feature.fc.in_features self.feature.fc = utils.Identity() # discard last fc layer self.scale = nn.Parameter(torch.FloatTensor([10])) self.classifier = nn.Linear(self.feature_dim, num_class, bias=False) self.ceLoss = nn.CrossEntropyLoss()
def __init__(self, input_size=1000, output_size=10, layers=2, hid_size=1000, hid_smooth=None, size_per_layer=None, drop=0, batch_norm=True, nl="relu", bias=True, excitability=False, excit_buffer=False, gated=False, output='normal'): '''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_smooth], last=[output]. [input_size] # of inputs [output_size] # of units in final layer [layers] # of layers [hid_size] # of units in each hidden layer [hid_smooth] if None, all hidden layers have [hid_size] units, else # of units linearly in-/decreases s.t. final hidden layer has [hid_smooth] units (if only 1 hidden layer, it has [hid_size] units) [size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs) --> overwrites [input_size], [output_size], [layers], [hid_size] and [hid_smooth] [drop] % of each layer's inputs that is randomly set to zero during training [batch_norm] <bool>; if True, batch-normalization is applied to each layer [nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none") [gated] <bool>; if True, each linear layer has an additional learnable gate [output] <str>; if - "normal", final layer is same as all others - "BCE", final layer has sigmoid non-linearity''' super().__init__() self.output = output # get sizes of all layers if size_per_layer is None: hidden_sizes = [] if layers > 1: if (hid_smooth is not None): hidden_sizes = [ int(x) for x in np.linspace( hid_size, hid_smooth, num=layers - 1) ] else: hidden_sizes = [ int(x) for x in np.repeat(hid_size, layers - 1) ] size_per_layer = [input_size] + hidden_sizes + [output_size] self.layers = len(size_per_layer) - 1 # set label for this module # -determine "non-default options"-label nd_label = "{drop}{bias}{exc}{bn}{nl}{gate}{out}".format( drop="" if drop == 0 else "-drop{}".format(drop), bias="" if bias else "-noBias", exc="-exc" if excitability else "", bn="-bn" if batch_norm else "", nl="-lr" if nl == "leakyrelu" else "", gate="-gated" if gated else "", out="" if output == "normal" else "-{}".format(output), ) # -set label self.label = "MLP({}{})".format(size_per_layer, nd_label) if self.layers > 0 else "" # set layers for lay_id in range(1, self.layers + 1): # number of units of this layer's input and output in_size = size_per_layer[lay_id - 1] out_size = size_per_layer[lay_id] # define and set the fully connected layer if lay_id == self.layers and output in ("logistic", "gaussian"): layer = fc_layer_split( in_size, out_size, bias=bias, excitability=excitability, excit_buffer=excit_buffer, drop=drop, batch_norm=False, gated=gated, nl_mean=nn.Sigmoid() if output == "logistic" else utils.Identity(), nl_logvar=nn.Hardtanh(min_val=-4.5, max_val=0.) if output == "logistic" else utils.Identity(), ) else: layer = fc_layer( in_size, out_size, bias=bias, excitability=excitability, excit_buffer=excit_buffer, drop=drop, batch_norm=False if (lay_id == self.layers and not output == "normal") else batch_norm, gated=gated, nl=nn.Sigmoid() if (lay_id == self.layers and not output == "normal") else nl, ) setattr(self, 'fcLayer{}'.format(lay_id), layer) # if no layers, add "identity"-module to indicate in this module's representation nothing happens if self.layers < 1: self.noLayers = utils.Identity()
def __init__(self, num_classes=10, input_size=1000, output_size=10, layers=2, hid_size=1000, hid_smooth=None, size_per_layer=None, drop=0, batch_norm=True, nl="relu", bias=True, excitability=False, excit_buffer=False, output='normal', fixed_mask=True, mask_prob=0.8, only_first=False, with_skip=False): '''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_smooth], last=[output]. [num_classes] # of classes [input_size] # of inputs [output_size] # of output units [layers] # of layers [hid_size] # of units in each hidden layer [hid_smooth] if None, all hidden layers have [hid_size] units, else # of units linearly in-/decreases s.t. final hidden layer has [hid_smooth] units (if only 1 hidden layer, it has [hid_size] units) [size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs) --> overwrites [input_size], [output_size], [layers], [hid_size] and [hid_smooth] [drop] % of each layer's inputs that is randomly set to zero during training [batch_norm] <bool>; if True, batch-normalization is applied to each layer [nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none") [output] <str>; if - "normal", final layer is same as all others - "none", final layer has no non-linearity - "sigmoid", final layer has sigmoid non-linearity EBM-related parameters [fixed_mask] <bool>; whether to use fixed masks instead of learnable gates [mask_prop] <float>; probability of each node being gated for particular class (if using `fixed_mask`) [only_first] <bool>; whether learnable gate is only used for first layer (only if not using `fixed_mask`) NOTE: if set to ``False``, all layers must have same number of units! [with_skip] <bool>; whehter there should be a skip-connection around the learnable gate ''' super().__init__() self.output = output self.fixed_mask = fixed_mask self.only_first = only_first self.num_classes = num_classes self.with_skip = with_skip # get sizes of all layers if size_per_layer is None: hidden_sizes = [] if layers > 1: if (hid_smooth is not None): hidden_sizes = [ int(x) for x in np.linspace( hid_size, hid_smooth, num=layers - 1) ] else: hidden_sizes = [ int(x) for x in np.repeat(hid_size, layers - 1) ] size_per_layer = [input_size] + hidden_sizes + [output_size] self.layers = len(size_per_layer) - 1 self.output_size = size_per_layer[-1] # set label for this module # -determine "non-default options"-label nd_label = "{drop}{bias}{exc}{bn}{nl}".format( drop="" if drop == 0 else "d{}".format(drop), bias="" if bias else "n", exc="e" if excitability else "", bn="b" if batch_norm else "", nl="l" if nl == "leakyrelu" else "", ) nd_label = "{}{}".format( "" if nd_label == "" else "-{}".format(nd_label), "" if output == "normal" else "-{}".format(output)) # -set label size_statement = "" for i in size_per_layer: size_statement += "{}{}".format( "-" if size_statement == "" else "x", i) self.label = "EBM{}{}{}{}".format( "fm{}".format(mask_prob) if fixed_mask else ("sk" if with_skip else ""), "-of" if only_first else "", size_statement, nd_label) if self.layers > 0 else "" # set layers for lay_id in range(1, self.layers + 1): # number of units of this layer's input and output in_size = size_per_layer[lay_id - 1] out_size = size_per_layer[lay_id] # embedding of y if not fixed_mask: self.goal_ebm = nn.Embedding(self.num_classes, size_per_layer[1]) # define and set the fully connected layer if fixed_mask and (lay_id == 1 or not self.only_first): layer = fc_layer_fixed_gates( in_size, out_size, bias=bias, excitability=excitability, excit_buffer=excit_buffer, drop=drop, batch_norm=False if (lay_id == self.layers and not output == "normal") else batch_norm, nl=nn.Sigmoid() if (lay_id == self.layers and not output == "normal") else nl, gate_size=num_classes, gating_prop=mask_prob, ) else: layer = fc_layer( in_size, out_size, bias=bias, excitability=excitability, excit_buffer=excit_buffer, drop=drop, batch_norm=False if (lay_id == self.layers and not output == "normal") else batch_norm, nl=nn.Sigmoid() if (lay_id == self.layers and not output == "normal") else nl, ) setattr(self, 'fcLayer{}'.format(lay_id), layer) # if no layers, add "identity"-module to indicate in this module's representation nothing happens if self.layers < 1: self.noLayers = utils.Identity()
def __init__(self, input_size=1000, output_size=10, layers=2, hid_size=1000, size_per_layer=None, drop=0, batch_norm=True, nl="relu", bias=True, excitability=False, excit_buffer=False, final_nl=True): '''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_size], last=[output]. [input_size] # of inputs [output_size] # of units in final layer [layers] # of layers [hid_size] # of units in each hidden layer [size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs) --> overwrites [input_size], [output_size], [layers] and [hid_size] [drop] % of each layer's inputs that is randomly set to zero during training [batch_norm] <bool>; if True, batch-normalization is applied to each layer [nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none") [final_nl] <bool>; if False, final layer has no non-linearity and batchnorm''' super().__init__() # get sizes of all layers if size_per_layer is None: hidden_sizes = [] if layers > 1: hidden_sizes = [ int(x) for x in np.repeat(hid_size, layers - 1) ] size_per_layer = [input_size] + hidden_sizes + [output_size] self.layers = len(size_per_layer) - 1 # set label for this module # -determine "non-default options"-label nd_label = "{drop}{bias}{exc}{bn}{nl}{fnl}".format( drop="" if drop == 0 else "-drop_{}".format(drop), bias="" if bias else "-noBias", exc="-exc" if excitability else "", bn="-bn" if batch_norm else "", nl="-lr" if nl == "leakyrelu" else "", fnl="-nfnl" if not final_nl else "", ) # -set label self.label = "MLP({}{})".format(size_per_layer, nd_label) if self.layers > 0 else "" # set layers for lay_id in range(1, self.layers + 1): # number of units of this layer's input and output in_size = size_per_layer[lay_id - 1] out_size = size_per_layer[lay_id] # define and set the fully connected layer layer = fc_layer( in_size, out_size, bias=bias, excitability=excitability, excit_buffer=excit_buffer, drop=drop, batch_norm=False if (lay_id == self.layers and not final_nl) else batch_norm, nl="no" if (lay_id == self.layers and not final_nl) else nl) setattr(self, 'fcLayer{}'.format(lay_id), layer) # if no layers, add "identity"-module to indicate in this module's representation nothing happens if self.layers < 1: self.noLayers = utils.Identity()