def __init__(self,
indim,
outdim,
bias=True,
gate_activation="sigmoid",
activation="tanh",
recurrent_batch_norm=None):
super(_LSTMCell, self).__init__()
self.indim, self.outdim, self.use_bias = indim, outdim, bias
self.gate_activation, self.activation = gate_activation, activation # sigm, tanh
self.hdim = self.outdim
if self.activation == "crelu":
self.hdim = self.hdim // 2
self._rbn_on = recurrent_batch_norm
self._rbn_gates = self._rbn_on == "full"
self.gates = PackedRNUGates(self.indim + self.outdim,
[(self.outdim, self.gate_activation),
(self.outdim, self.gate_activation),
(self.outdim, self.gate_activation),
(self.hdim, None)],
use_bias=self.use_bias,
rec_bn=self._rbn_gates)
self.activation_fn = q.name2fn(activation)()
self.reset_parameters()
def __init__(self, indim, outgatespecs, use_bias=True, rec_bn=False):
super(PackedRNUGates, self).__init__()
self.indim = indim
self.outdim = 0
self.outgates = []
self.use_bias = use_bias
for outgatespec in outgatespecs:
gateoutdim = outgatespec[0]
gateact = outgatespec[1]
gateact_fn = q.name2fn(gateact)()
self.outgates.append(
((self.outdim, self.outdim + gateoutdim), gateact_fn))
self.outdim += gateoutdim
self.W = nn.Parameter(torch.FloatTensor(self.indim, self.outdim))
if self.use_bias:
self.b = nn.Parameter(torch.FloatTensor(
1,
self.outdim,
))
else:
self.register_parameter("b", None)
self.rec_bn = rec_bn
self._rbn = None
if self.rec_bn is True:
self._rbn = q.SeqBatchNorm1d(self.outdim)
self.reset_parameters()
def __init__(self, d_hid, d_inner_hid, activation="relu", dropout=0.1):
super(PositionwiseForward, self).__init__()
self.w_1 = nn.Conv1d(d_hid, d_inner_hid, 1) # position-wise
self.w_2 = nn.Conv1d(d_inner_hid, d_hid, 1) # position-wise
self.layer_norm = q.LayerNormalization(d_hid)
self.dropout = nn.Dropout(dropout)
self.activation_fn = q.name2fn(activation)()
def __init__(self, ldim, rdim, aggdim, activation="tanh", use_bias=True):
super(ForwardDistance, self).__init__()
self.lblock = nn.Linear(ldim, aggdim, bias=use_bias)
self.rblock = nn.Linear(rdim, aggdim, bias=use_bias)
self.activation = q.name2fn(activation)()
self.agg = nn.Parameter(torch.FloatTensor(aggdim))
self.reset_parameters()
def __init__(self,
indim,
outdim,
hdim=None,
activation="sigmoid",
use_bias=True):
super(RNUGate, self).__init__()
self.indim, self.outdim, self.activation, self.use_bias = indim, outdim, activation, use_bias
self.activation_fn = q.name2fn(self.activation)()
self.W = nn.Parameter(torch.FloatTensor(self.indim, self.outdim))
udim = self.outdim if hdim is None else hdim
self.U = nn.Parameter(torch.FloatTensor(udim, self.outdim))
if self.use_bias:
self.b = nn.Parameter(torch.FloatTensor(1, self.outdim))
else:
self.register_parameter("b", None)
self.reset_parameters()
def __init__(self,
dim,
bias=True,
gate_activation="sigmoid",
activation="tanh"):
super(_SRUCell, self).__init__()
self.indim, self.outdim, self.use_bias = dim, dim, bias
self.gate_activation, self.activation = gate_activation, activation
self.hdim = self.outdim
if self.activation == "crelu":
self.hdim = self.hdim // 2 # TODO
self.gates = PackedRNUGates(self.indim,
[(self.outdim, self.gate_activation),
(self.outdim, self.gate_activation),
(self.outdim, None)])
self.activation_fn = q.name2fn(activation)()
self.reset_parameters()
def __init__(self, ldim, rdim, aggdim, activation="tanh", use_bias=False):
super(TrilinearDistance, self).__init__()
self.block = nn.Bilinear(ldim, rdim, aggdim, bias=use_bias)
self.activation = q.name2fn(activation)()
self.agg = nn.Parameter(torch.FloatTensor(aggdim))
self.reset_parameters()
def __init__(self, indim, outdim, activation="tanh", use_bias=True):
super(Forward, self).__init__()
self.lin = nn.Linear(indim, outdim, bias=use_bias)
self.activation = q.name2fn(activation)()
self.reset_parameters()