def __init__(self, params):
assert_param(param=params, field_type=int, field="input_dim")
super().__init__()
self.params = params
self.div_term = torch.exp(
torch.arange(0, params['input_dim'], 2) *
-(math.log(10000.0) / params['input_dim'])).requires_grad_(False)
def __init__(self, params):
super().__init__()
assert_param(param=params, field='head_num', field_type=int)
assert_param(param=params, field='input_dim', field_type=int)
self.params = params
self.attention = MultiHeadAttention({
'head_num': params['head_num'],
'input_dim': params['input_dim']
})
self.attention_weight = None
def __init__(self, params):
super().__init__()
assert_param(param=params, field_type=int, field="input_dim")
assert_param(param=params, field_type=int, field="hidden_dim")
if "dropout" not in params:
params["dropout"] = 0.0
input_dim, hidden_dim, dropout = params["input_dim"], params[
"hidden_dim"], params["dropout"]
self.linear1 = nn.Linear(input_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, input_dim)
self.dropout = nn.Dropout(dropout)
def __init__(self, params):
super().__init__()
assert type(params) in [dict, str], "Invalid Parameter Type!"
if type(params) is str:
params = configContainer.fetch_config(params)
assert_param(param=params, field='input_dim', field_type=int)
assert_param(param=params, field='hidden_dim', field_type=int)
assert_param(param=params, field='layer_number_1', field_type=int)
assert_param(param=params, field='layer_number_2', field_type=int)
if "dropout" not in params:
params["dropout"] = 0.0
if "share_embedding" not in params:
params["share_embedding"] = True
self.params = default_params
self.params.update(params)
self.transformer1 = TransformerCore({
"head_num": self.params["head_num"],
"input_dim": self.params["input_dim"],
"decoder_layer_num": self.params["layer_number_1"],
"hidden_dim": self.params["hidden_dim"],
"share_parameter": self.params["share_parameter"],
"causal": self.params["causal"]
})
self.transformer2 = TransformerCore({
"head_num": self.params["head_num"],
"input_dim": self.params["input_dim"],
"decoder_layer_num": self.params["layer_number_2"],
"hidden_dim": self.params["hidden_dim"],
"share_parameter": self.params["share_parameter"],
"causal": self.params["causal"]
})
def __init__(self, params):
assert type(params) in [dict, str], "Invalid Parameter Type!"
if type(params) is str:
params = configContainer.fetch_config(params)
if "share_embedding" not in params:
params["share_embedding"] = True
self.params = params
super().__init__(params)
assert_param(param=params, field='input_dim', field_type=int)
assert_param(param=params, field='target_vocab', field_type=Vocabulary)
self.out = nn.Linear(params['input_dim'], params['target_vocab'].word_num)
def __init__(self, params):
super().__init__()
assert type(params) in [dict, str], "Invalid Parameter Type!"
if type(params) is str:
params = configContainer.fetch_config(params)
assert_param(param=params, field='input_dim', field_type=int)
assert_param(param=params, field='hidden_dim', field_type=int)
self.params = default_params
self.params.update(params)
if not self.params["share_parameter"]:
self.encoder = TransformerEncoder(self.params)
self.decoder = TransformerDecoder(self.params)
else:
self.encoder = self.decoder = TransformerDecoder(self.params)
def __init__(self, params):
super().__init__()
assert_param(param=params, field='head_num', field_type=int)
assert_param(param=params, field='input_dim', field_type=int)
assert params['input_dim'] % params['head_num'] == 0
if 'dropout' not in params:
params['dropout'] = 0
self.params = params
self.params['hidden_dim'] = params['input_dim'] // params['head_num']
self.query_head = nn.Linear(params['input_dim'], params['input_dim'])
self.key_head = nn.Linear(params['input_dim'], params['input_dim'])
self.value_head = nn.Linear(params['input_dim'], params['input_dim'])
self.attention = ScaledDotProductAttention()
self.dropout = nn.Dropout(p=params['dropout'])
self.out = nn.Linear(params['input_dim'], params['input_dim'])
self.attention_weight = None
def __init__(self, params):
assert_param(param=params, field_type=int, field="input_dim")
if "max_len" not in params:
params["max_len"] = 5000
super().__init__()
# Compute the positional encodings once in log space.
pos_encoding = torch.zeros(params['max_len'], params['input_dim'])
position = torch.arange(0., params['max_len']).unsqueeze(1)
div_term = torch.exp(
torch.arange(0., params['input_dim'], 2) *
-(math.log(10000.0) / params['input_dim']))
pos_encoding[:, 0::2] = torch.sin(position * div_term)
pos_encoding[:, 1::2] = torch.cos(position * div_term)
pos_encoding = pos_encoding.unsqueeze(
0).clone().detach().requires_grad_(False)
self.register_buffer('pos_encoding', pos_encoding)
def __init__(self, params):
super().__init__()
# self.dropout = nn.Dropout(dropout)
assert_param(params, "input_dim", int)
assert_param(params, "memory_dim", int)
assert_param(params, "hidden_dim", int)
self.linear_query = nn.Linear(params["input_dim"],
params["hidden_dim"],
bias=True)
self.linear_memory = nn.Linear(params["memory_dim"],
params["hidden_dim"],
bias=False)
self.linear_final = nn.Linear(params["hidden_dim"], 1, bias=False)
self.attention_weight = None
def __init__(self, params):
assert type(params) in [dict, str], "Invalid Parameter Type!"
if type(params) is str:
params = configContainer.fetch_config(params)
assert_param(param=params, field='hidden_dim', field_type=int)
assert_param(param=params, field='use_attention', field_type=bool)
if params['use_attention']:
assert_param(param=params, field='memory_dim', field_type=int)
assert_param(param=params, field='batch_size', field_type=int)
assert_param(param=params, field='input_dim', field_type=int)
assert_param(param=params, field='is_bidirectional', field_type=bool)
super().__init__()
params.update(default_params)
self.params = params
self.rnn = nn.GRU(input_size=self.params['input_dim'],
hidden_size=self.params['hidden_dim'],
num_layers=self.params['num_layers'],
dropout=self.params['rnn_drop_prob'],
batch_first=True,
bidirectional=self.params['is_bidirectional'])
self.attention = TanhAttention({
'input_dim': self.params['input_dim'],
'memory_dim': self.params['memory_dim'],
'target_dim': self.params['hidden_dim']
}) if self.params['use_attention'] else None
self.mixture = nn.Linear(self.params['hidden_dim'], self.params['hidden_dim']) \
if not self.params['use_attention'] else None
self.rnn.flatten_parameters()