def __init__(self, dim=100, AF=None, BN=True):
super(ResidualBlock_FFNNs, self).__init__()
self.ffnns = nn.Sequential()
layer_1 = nn.Linear(dim, dim)
nr_init(layer_1.weight)
self.ffnns.add_module('L_1', layer_1)
if BN:
self.ffnns.add_module(
'BN_1',
nn.BatchNorm1d(dim,
momentum=1.0,
affine=True,
track_running_stats=False))
self.ffnns.add_module('ACT_1', AF)
layer_2 = nn.Linear(dim, dim)
nr_init(layer_2.weight)
self.ffnns.add_module('L_2', layer_2)
if BN:
self.ffnns.add_module(
'BN_2',
nn.BatchNorm1d(dim,
momentum=1.0,
affine=True,
track_running_stats=False))
self.AF = AF
def ini_ffnns(self, num_features=None, h_dim=100, out_dim=1, num_layers=3, HD_AF='R', HN_AF='R', TL_AF='S', apply_tl_af=True):
head_AF, hidden_AF, tail_AF = get_AF(HD_AF), get_AF(HN_AF), get_AF(TL_AF) #configurable activation functions
ffnns = nn.Sequential()
if 1 == num_layers:
nr_h1 = nn.Linear(num_features, out_dim) # 1st layer
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
if apply_tl_af:
ffnns.add_module('ACT_1', tail_AF)
else:
nr_h1 = nn.Linear(num_features, h_dim) # 1st layer
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
ffnns.add_module('ACT_1', head_AF)
if num_layers > 2: # middle layers if needed
for i in range(2, num_layers):
ffnns.add_module('_'.join(['DR', str(i)]), nn.Dropout(0.01))
nr_hi = nn.Linear(h_dim, h_dim)
nr_init(nr_hi.weight)
ffnns.add_module('_'.join(['L', str(i)]), nr_hi)
ffnns.add_module('_'.join(['ACT', str(i)]), hidden_AF)
nr_hn = nn.Linear(h_dim, out_dim) #relevance prediction layer
nr_init(nr_hn.weight)
ffnns.add_module('_'.join(['L', str(num_layers)]), nr_hn)
if apply_tl_af:
ffnns.add_module('_'.join(['ACT', str(num_layers)]), tail_AF)
return ffnns
def __init__(self, dim=100, AF=None):
super(ResidualBlock_FFNNs, self).__init__()
self.ffnns = nn.Sequential()
layer_1 = nn.Linear(dim, dim)
nr_init(layer_1.weight)
self.ffnns.add_module('L_1', layer_1)
self.ffnns.add_module('ACT_1', get_AF(af_str=AF))
layer_2 = nn.Linear(dim, dim)
nr_init(layer_2.weight)
self.ffnns.add_module('L_2', layer_2)
self.AF = get_AF(af_str=AF)
def ini_ffnns(self,
input_dim=None,
h_dim=None,
out_dim=1,
num_layers=None,
hd_af=None,
hn_af=None,
tl_af=None,
dropout_rate=None,
apply_tl_af=None):
head_AF, hidden_AF, tail_AF = activation(hd_af), activation(
hn_af), activation(tl_af)
ffnns = nn.Sequential()
if 1 == num_layers:
nr_h1 = nn.Linear(input_dim, out_dim) # Input layer
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
if apply_tl_af:
ffnns.add_module('ACT_1', tail_AF)
else:
nr_h1 = nn.Linear(input_dim, h_dim)
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
ffnns.add_module('ACT_1', head_AF)
if num_layers > 2: # Hidden layer
for i in range(2, num_layers):
h_dim_half = h_dim / 2
ffnns.add_module('_'.join(['DR', str(i)]),
nn.Dropout(dropout_rate))
nr_hi = nn.Linear(h_dim, int(h_dim_half))
nr_init(nr_hi.weight)
ffnns.add_module('_'.join(['L', str(i)]), nr_hi)
ffnns.add_module('_'.join(['ACT', str(i)]), hidden_AF)
h_dim = int(h_dim_half)
nr_hn = nn.Linear(int(h_dim_half), out_dim) #Output layer
nr_init(nr_hn.weight)
ffnns.add_module('_'.join(['L', str(num_layers)]), nr_hn)
if apply_tl_af:
ffnns.add_module('_'.join(['ACT', str(num_layers)]), tail_AF)
return ffnns
def ini_ffnns(self,
num_features=None,
h_dim=100,
out_dim=1,
num_layers=3,
HD_AF='R',
HN_AF='R',
TL_AF='S',
apply_tl_af=True,
BN=True,
RD=False,
FBN=False):
'''
Initialization of a feed-forward neural network
:param num_features: the number of dimensions of the input layer todo in_dims
:param h_dim: the number of dimensions of a hidden layer
:param out_dim: the number of dimensions of the output layer todo out_dims
:param num_layers: the number of layers
:param HD_AF: the activation function for the first layer
:param HN_AF: the activation function for the hidden layer(s)
:param TL_AF:
:param apply_tl_af: the activation function for the output layer
:param BN: batch normalization
:param RD: each hidden layer is implemented as a residual block
:param FBN: perform batch normalization over raw input features enabling learnable normalization
:return:
'''
head_AF, hidden_AF = get_AF(HD_AF), get_AF(
HN_AF) # configurable activation functions
tail_AF = get_AF(TL_AF) if apply_tl_af else None
ffnns = nn.Sequential()
if 1 == num_layers:
# using in-build batch-normalization layer to normalize raw features, enabling learnable normalization
if FBN:
ffnns.add_module(
'FeatureBN',
nn.BatchNorm1d(num_features=num_features,
momentum=1.0,
affine=True,
track_running_stats=False))
nr_h1 = nn.Linear(num_features, out_dim) # 1st layer
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
if BN: # before applying activation
bn_1 = nn.BatchNorm1d(
out_dim,
momentum=1.0,
affine=True,
track_running_stats=False
) # normalize the result w.r.t. each neural unit before activation
ffnns.add_module('BN_1', bn_1)
if apply_tl_af:
ffnns.add_module('ACT_1', tail_AF)
else:
# using in-build batch-normalization layer to normalize raw features, enabling learnable normalization
if FBN:
ffnns.add_module(
'FeatureBN',
nn.BatchNorm1d(num_features=num_features,
momentum=1.0,
affine=True,
track_running_stats=False))
nr_h1 = nn.Linear(num_features, h_dim) # 1st layer
nr_init(nr_h1.weight)
ffnns.add_module('L_1', nr_h1)
if BN: # before applying activation
bn1 = nn.BatchNorm1d(h_dim,
momentum=1.0,
affine=True,
track_running_stats=False)
ffnns.add_module('BN_1', bn1)
ffnns.add_module('ACT_1', head_AF)
if num_layers > 2: # middle layers if needed
if RD:
for i in range(2, num_layers):
ffnns.add_module(
'_'.join(['RD', str(i)]),
ResidualBlock_FFNNs(dim=h_dim, AF=hidden_AF,
BN=BN))
else:
for i in range(2, num_layers):
ffnns.add_module('_'.join(['DR', str(i)]),
nn.Dropout(0.01))
nr_hi = nn.Linear(h_dim, h_dim)
nr_init(nr_hi.weight)
ffnns.add_module('_'.join(['L', str(i)]), nr_hi)
if BN: # before applying activation
bn_i = nn.BatchNorm1d(h_dim,
momentum=1.0,
affine=True,
track_running_stats=False)
ffnns.add_module('_'.join(['BN', str(i)]), bn_i)
ffnns.add_module('_'.join(['ACT', str(i)]), hidden_AF)
nr_hn = nn.Linear(h_dim, out_dim) # relevance prediction layer
nr_init(nr_hn.weight)
ffnns.add_module('_'.join(['L', str(num_layers)]), nr_hn)
if BN: # before applying activation
ffnns.add_module(
'_'.join(['BN', str(num_layers)]),
nn.BatchNorm1d(out_dim,
momentum=1.0,
affine=True,
track_running_stats=False))
if apply_tl_af:
ffnns.add_module('_'.join(['ACT', str(num_layers)]), tail_AF)
# print(ffnns)
# for w in ffnns.parameters():
# print(w.data)
return ffnns
def get_stacked_FFNet(ff_dims=None,
AF=None,
TL_AF=None,
apply_tl_af=False,
dropout=0.1,
BN=True,
bn_type=None,
bn_affine=False,
device='cpu',
split_penultimate_layer=False):
'''
Generate one stacked feed-forward network.
'''
# '2' refers to the simplest case: num_features, out_dim
assert ff_dims is not None and len(ff_dims) >= 2
ff_net = nn.Sequential()
num_layers = len(ff_dims)
if num_layers > 2:
for i in range(1, num_layers - 1):
prior_dim, ff_i_dim = ff_dims[i - 1], ff_dims[i]
ff_net.add_module('_'.join(['dr', str(i)]), nn.Dropout(dropout))
nr_hi = nn.Linear(prior_dim, ff_i_dim)
nr_init(nr_hi.weight)
ff_net.add_module('_'.join(['ff', str(i + 1)]), nr_hi)
if BN: # before applying activation
if 'BN' == bn_type:
bn_i = LTRBatchNorm(ff_i_dim,
momentum=0.1,
affine=bn_affine,
track_running_stats=False)
elif 'BN2' == bn_type:
bn_i = LTRBatchNorm2(ff_i_dim,
momentum=0.1,
affine=bn_affine,
device=device)
else:
raise NotImplementedError
ff_net.add_module('_'.join(['bn', str(i + 1)]), bn_i)
ff_net.add_module('_'.join(['act', str(i + 1)]), get_AF(AF))
# last layer
penultimate_dim, out_dim = ff_dims[-2], ff_dims[-1]
nr_hn = nn.Linear(penultimate_dim, out_dim)
nr_init(nr_hn.weight)
if split_penultimate_layer:
tail_net = nn.Sequential()
tail_net.add_module('_'.join(['ff', str(num_layers)]), nr_hn)
if apply_tl_af:
if BN: # before applying activation
if 'BN' == bn_type:
tail_net.add_module(
'_'.join(['bn', str(num_layers)]),
LTRBatchNorm(out_dim,
momentum=0.1,
affine=bn_affine,
track_running_stats=False))
elif 'BN2' == bn_type:
tail_net.add_module(
'_'.join(['bn', str(num_layers)]),
LTRBatchNorm2(out_dim,
momentum=0.1,
affine=bn_affine,
device=device))
else:
raise NotImplementedError
tail_net.add_module('_'.join(['act', str(num_layers)]),
get_AF(TL_AF))
return ff_net, tail_net
else:
ff_net.add_module('_'.join(['ff', str(num_layers)]), nr_hn)
if apply_tl_af:
if BN: # before applying activation
if 'BN' == bn_type:
ff_net.add_module(
'_'.join(['bn', str(num_layers)]),
LTRBatchNorm(out_dim,
momentum=0.1,
affine=bn_affine,
track_running_stats=False))
elif 'BN2' == bn_type:
ff_net.add_module(
'_'.join(['bn', str(num_layers)]),
LTRBatchNorm2(out_dim,
momentum=0.1,
affine=bn_affine,
device=device))
else:
raise NotImplementedError
ff_net.add_module('_'.join(['act', str(num_layers)]),
get_AF(TL_AF))
return ff_net