def __init__(self,
in_size,
out_size,
nl=nn.ReLU(),
drop=0.,
bias=True,
excitability=False,
excit_buffer=False,
batch_norm=False,
gated=False):
super().__init__()
if drop > 0:
self.dropout = nn.Dropout(drop)
self.linear = em.LinearExcitability(in_size,
out_size,
bias=False if batch_norm else bias,
excitability=excitability,
excit_buffer=excit_buffer)
if batch_norm:
self.bn = nn.BatchNorm1d(out_size)
if gated:
self.gate = nn.Linear(in_size, out_size)
self.sigmoid = nn.Sigmoid()
if isinstance(nl, nn.Module):
self.nl = nl
elif not nl == "none":
self.nl = nn.ReLU() if nl == "relu" else (
nn.LeakyReLU() if nl == "leakyrelu" else utils.Identity())
def __init__(self, num_class):
super(Jigsaw, self).__init__()
self.feature = torchvision.models.resnet18(pretrained=False)
self.feature_dim = self.feature.fc.in_features
self.feature.fc = utils.Identity() # discard last fc layer
self.classifier = nn.Linear(self.feature_dim, num_class)
self.ceLoss = nn.CrossEntropyLoss()
def set_activation(self, nl):
if isinstance(nl, nn.Module):
self.nl = nl
elif nl == "relu":
self.nl = nn.ReLU()
elif nl == "leakyrelu":
self.nl = nn.LeakyReLU()
else:
self.nl = utils.Identity()
def __init__(self,
in_size,
out_size,
drop=0.,
bias=True,
excitability=False,
excit_buffer=False,
batch_norm=False,
nl="relu"):
super().__init__()
self.dropout = nn.Dropout(drop)
self.linear = em.LinearExcitability(in_size,
out_size,
bias=False if batch_norm else bias,
excitability=excitability,
excit_buffer=excit_buffer)
self.bn = nn.BatchNorm1d(out_size) if batch_norm else utils.Identity()
self.nl = nn.ReLU() if nl == "relu" else (
nn.LeakyReLU() if nl == "leakyrelu" else utils.Identity())
def __init__(self, num_class):
super(ImprintJigsaw, self).__init__()
self.num_class = num_class
self.feature = torchvision.models.resnet18(pretrained=False)
self.feature_dim = self.feature.fc.in_features
self.feature.fc = utils.Identity() # discard last fc layer
self.scale = nn.Parameter(torch.FloatTensor([10]))
self.classifier = nn.Linear(self.feature_dim, num_class, bias=False)
self.ceLoss = nn.CrossEntropyLoss()
def __init__(self,
input_size=1000,
output_size=10,
layers=2,
hid_size=1000,
hid_smooth=None,
size_per_layer=None,
drop=0,
batch_norm=True,
nl="relu",
bias=True,
excitability=False,
excit_buffer=False,
gated=False,
output='normal'):
'''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_smooth], last=[output].
[input_size] # of inputs
[output_size] # of units in final layer
[layers] # of layers
[hid_size] # of units in each hidden layer
[hid_smooth] if None, all hidden layers have [hid_size] units, else # of units linearly in-/decreases s.t.
final hidden layer has [hid_smooth] units (if only 1 hidden layer, it has [hid_size] units)
[size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs)
--> overwrites [input_size], [output_size], [layers], [hid_size] and [hid_smooth]
[drop] % of each layer's inputs that is randomly set to zero during training
[batch_norm] <bool>; if True, batch-normalization is applied to each layer
[nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none")
[gated] <bool>; if True, each linear layer has an additional learnable gate
[output] <str>; if - "normal", final layer is same as all others
- "BCE", final layer has sigmoid non-linearity'''
super().__init__()
self.output = output
# get sizes of all layers
if size_per_layer is None:
hidden_sizes = []
if layers > 1:
if (hid_smooth is not None):
hidden_sizes = [
int(x) for x in np.linspace(
hid_size, hid_smooth, num=layers - 1)
]
else:
hidden_sizes = [
int(x) for x in np.repeat(hid_size, layers - 1)
]
size_per_layer = [input_size] + hidden_sizes + [output_size]
self.layers = len(size_per_layer) - 1
# set label for this module
# -determine "non-default options"-label
nd_label = "{drop}{bias}{exc}{bn}{nl}{gate}{out}".format(
drop="" if drop == 0 else "-drop{}".format(drop),
bias="" if bias else "-noBias",
exc="-exc" if excitability else "",
bn="-bn" if batch_norm else "",
nl="-lr" if nl == "leakyrelu" else "",
gate="-gated" if gated else "",
out="" if output == "normal" else "-{}".format(output),
)
# -set label
self.label = "MLP({}{})".format(size_per_layer,
nd_label) if self.layers > 0 else ""
# set layers
for lay_id in range(1, self.layers + 1):
# number of units of this layer's input and output
in_size = size_per_layer[lay_id - 1]
out_size = size_per_layer[lay_id]
# define and set the fully connected layer
if lay_id == self.layers and output in ("logistic", "gaussian"):
layer = fc_layer_split(
in_size,
out_size,
bias=bias,
excitability=excitability,
excit_buffer=excit_buffer,
drop=drop,
batch_norm=False,
gated=gated,
nl_mean=nn.Sigmoid()
if output == "logistic" else utils.Identity(),
nl_logvar=nn.Hardtanh(min_val=-4.5, max_val=0.)
if output == "logistic" else utils.Identity(),
)
else:
layer = fc_layer(
in_size,
out_size,
bias=bias,
excitability=excitability,
excit_buffer=excit_buffer,
drop=drop,
batch_norm=False if
(lay_id == self.layers
and not output == "normal") else batch_norm,
gated=gated,
nl=nn.Sigmoid() if
(lay_id == self.layers and not output == "normal") else nl,
)
setattr(self, 'fcLayer{}'.format(lay_id), layer)
# if no layers, add "identity"-module to indicate in this module's representation nothing happens
if self.layers < 1:
self.noLayers = utils.Identity()
def __init__(self,
num_classes=10,
input_size=1000,
output_size=10,
layers=2,
hid_size=1000,
hid_smooth=None,
size_per_layer=None,
drop=0,
batch_norm=True,
nl="relu",
bias=True,
excitability=False,
excit_buffer=False,
output='normal',
fixed_mask=True,
mask_prob=0.8,
only_first=False,
with_skip=False):
'''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_smooth], last=[output].
[num_classes] # of classes
[input_size] # of inputs
[output_size] # of output units
[layers] # of layers
[hid_size] # of units in each hidden layer
[hid_smooth] if None, all hidden layers have [hid_size] units, else # of units linearly in-/decreases s.t.
final hidden layer has [hid_smooth] units (if only 1 hidden layer, it has [hid_size] units)
[size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs)
--> overwrites [input_size], [output_size], [layers], [hid_size] and [hid_smooth]
[drop] % of each layer's inputs that is randomly set to zero during training
[batch_norm] <bool>; if True, batch-normalization is applied to each layer
[nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none")
[output] <str>; if - "normal", final layer is same as all others
- "none", final layer has no non-linearity
- "sigmoid", final layer has sigmoid non-linearity
EBM-related parameters
[fixed_mask] <bool>; whether to use fixed masks instead of learnable gates
[mask_prop] <float>; probability of each node being gated for particular class (if using `fixed_mask`)
[only_first] <bool>; whether learnable gate is only used for first layer (only if not using `fixed_mask`)
NOTE: if set to ``False``, all layers must have same number of units!
[with_skip] <bool>; whehter there should be a skip-connection around the learnable gate
'''
super().__init__()
self.output = output
self.fixed_mask = fixed_mask
self.only_first = only_first
self.num_classes = num_classes
self.with_skip = with_skip
# get sizes of all layers
if size_per_layer is None:
hidden_sizes = []
if layers > 1:
if (hid_smooth is not None):
hidden_sizes = [
int(x) for x in np.linspace(
hid_size, hid_smooth, num=layers - 1)
]
else:
hidden_sizes = [
int(x) for x in np.repeat(hid_size, layers - 1)
]
size_per_layer = [input_size] + hidden_sizes + [output_size]
self.layers = len(size_per_layer) - 1
self.output_size = size_per_layer[-1]
# set label for this module
# -determine "non-default options"-label
nd_label = "{drop}{bias}{exc}{bn}{nl}".format(
drop="" if drop == 0 else "d{}".format(drop),
bias="" if bias else "n",
exc="e" if excitability else "",
bn="b" if batch_norm else "",
nl="l" if nl == "leakyrelu" else "",
)
nd_label = "{}{}".format(
"" if nd_label == "" else "-{}".format(nd_label),
"" if output == "normal" else "-{}".format(output))
# -set label
size_statement = ""
for i in size_per_layer:
size_statement += "{}{}".format(
"-" if size_statement == "" else "x", i)
self.label = "EBM{}{}{}{}".format(
"fm{}".format(mask_prob) if fixed_mask else
("sk" if with_skip else ""), "-of" if only_first else "",
size_statement, nd_label) if self.layers > 0 else ""
# set layers
for lay_id in range(1, self.layers + 1):
# number of units of this layer's input and output
in_size = size_per_layer[lay_id - 1]
out_size = size_per_layer[lay_id]
# embedding of y
if not fixed_mask:
self.goal_ebm = nn.Embedding(self.num_classes,
size_per_layer[1])
# define and set the fully connected layer
if fixed_mask and (lay_id == 1 or not self.only_first):
layer = fc_layer_fixed_gates(
in_size,
out_size,
bias=bias,
excitability=excitability,
excit_buffer=excit_buffer,
drop=drop,
batch_norm=False if
(lay_id == self.layers
and not output == "normal") else batch_norm,
nl=nn.Sigmoid() if
(lay_id == self.layers and not output == "normal") else nl,
gate_size=num_classes,
gating_prop=mask_prob,
)
else:
layer = fc_layer(
in_size,
out_size,
bias=bias,
excitability=excitability,
excit_buffer=excit_buffer,
drop=drop,
batch_norm=False if
(lay_id == self.layers
and not output == "normal") else batch_norm,
nl=nn.Sigmoid() if
(lay_id == self.layers and not output == "normal") else nl,
)
setattr(self, 'fcLayer{}'.format(lay_id), layer)
# if no layers, add "identity"-module to indicate in this module's representation nothing happens
if self.layers < 1:
self.noLayers = utils.Identity()
def __init__(self,
input_size=1000,
output_size=10,
layers=2,
hid_size=1000,
size_per_layer=None,
drop=0,
batch_norm=True,
nl="relu",
bias=True,
excitability=False,
excit_buffer=False,
final_nl=True):
'''sizes: 0th=[input], 1st=[hid_size], ..., 1st-to-last=[hid_size], last=[output].
[input_size] # of inputs
[output_size] # of units in final layer
[layers] # of layers
[hid_size] # of units in each hidden layer
[size_per_layer] None or <list> with for each layer number of units (1st element = number of inputs)
--> overwrites [input_size], [output_size], [layers] and [hid_size]
[drop] % of each layer's inputs that is randomly set to zero during training
[batch_norm] <bool>; if True, batch-normalization is applied to each layer
[nl] <str>; type of non-linearity to be used (options: "relu", "leakyrelu", "none")
[final_nl] <bool>; if False, final layer has no non-linearity and batchnorm'''
super().__init__()
# get sizes of all layers
if size_per_layer is None:
hidden_sizes = []
if layers > 1:
hidden_sizes = [
int(x) for x in np.repeat(hid_size, layers - 1)
]
size_per_layer = [input_size] + hidden_sizes + [output_size]
self.layers = len(size_per_layer) - 1
# set label for this module
# -determine "non-default options"-label
nd_label = "{drop}{bias}{exc}{bn}{nl}{fnl}".format(
drop="" if drop == 0 else "-drop_{}".format(drop),
bias="" if bias else "-noBias",
exc="-exc" if excitability else "",
bn="-bn" if batch_norm else "",
nl="-lr" if nl == "leakyrelu" else "",
fnl="-nfnl" if not final_nl else "",
)
# -set label
self.label = "MLP({}{})".format(size_per_layer,
nd_label) if self.layers > 0 else ""
# set layers
for lay_id in range(1, self.layers + 1):
# number of units of this layer's input and output
in_size = size_per_layer[lay_id - 1]
out_size = size_per_layer[lay_id]
# define and set the fully connected layer
layer = fc_layer(
in_size,
out_size,
bias=bias,
excitability=excitability,
excit_buffer=excit_buffer,
drop=drop,
batch_norm=False if
(lay_id == self.layers and not final_nl) else batch_norm,
nl="no" if (lay_id == self.layers and not final_nl) else nl)
setattr(self, 'fcLayer{}'.format(lay_id), layer)
# if no layers, add "identity"-module to indicate in this module's representation nothing happens
if self.layers < 1:
self.noLayers = utils.Identity()
