def __init__(self,
d_input: int,
d_channel: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
dropout: float = 0.3,
pe: bool = False):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_input = d_input
self._d_channel = d_channel
self._d_model = d_model
self._pe = pe
self.layers_encoding = nn.ModuleList(
[Encoder(d_model, q, v, h, dropout=dropout) for _ in range(N)])
# self.layers_decoding = nn.ModuleList([Decoder(d_model,
# q,
# v,
# h,
# dropout=dropout) for _ in range(N)])
self._embedding = nn.Linear(self._d_channel, d_model)
self._linear = nn.Linear(d_model * d_input, d_output)
def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None,
activation='nothing'):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
self.softmax = nn.Softmax(dim=1)
self.activation = activation
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
self.activation_functions = { # for last number
'sigmoid': torch.sigmoid,
'relu': torch.relu,
'tanh': torch.tanh,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)
def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: str = 'chunk',
pe: str = None,
pe_period: int = 24):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
self._pe_period = pe_period
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)
self.name = 'transformer'
def __init__(self, # training.ipynbでは
d_input: int, # 37(datasetのカラム数)
d_model: int, # 64 Lattent dim
d_output: int, # 8 (datasetのoutput側のカラム数)
q: int, # 8
v: int, # 8
h: int, # 4
N: int, # 4 tencoder-decoderの重ね回数
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)])
self.layers_decoding = nn.ModuleList([Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.')
def __init__(self,
d_input: int,
d_channel: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
dropout: float = 0.3,
pe: bool = False,
mask: bool = False):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_input = d_input
self._d_channel = d_channel
self._d_model = d_model
self._pe = pe
self._h = h
self._q = q
self.layers_encoding_1 = nn.ModuleList([
Encoder(d_model, q, v, h, dropout=dropout, mask=mask)
for _ in range(N)
])
self.layers_encoding_2 = nn.ModuleList([
Encoder(d_model, q, v, h, dropout=dropout, mask=True)
for _ in range(N)
])
# self.layers_decoding = nn.ModuleList([Decoder(d_model,
# q,
# v,
# h,
# dropout=dropout) for _ in range(N)])
self._embedding_channel = nn.Linear(self._d_channel, d_model)
self._embedding_input = nn.Linear(self._d_input, d_model)
self._linear = nn.Linear(d_model * d_input + d_model * d_channel,
d_output)
self._layerNorm = nn.LayerNorm(d_model * d_input + d_model * d_channel)
# self._gate = nn.Linear(d_model * d_input + d_model * d_channel,2)
self._gate = nn.Linear(d_model * d_input + d_model * d_channel, 2)
# self._W_q = nn.Linear(d_model * d_input + d_model * d_channel,q * h)
# self._W_k = nn.Linear(d_model * d_input + d_model * d_channel,q * h)
# self._W_v = nn.Linear(d_model * d_input + d_model * d_channel,v * h)
self._attention_linear = nn.Linear(
d_model * d_input + d_model * d_channel,
8 * d_input + 8 * d_channel)
# self._linear_avg = nn.Linear(d_model+d_model, d_output)
self._W_q = nn.Linear(1, q * h)
self._W_k = nn.Linear(1, q * h)
self._W_v = nn.Linear(1, v * h)
# Output linear function
# self._W_o = nn.Linear(v * h, d_model * d_input + d_model * d_channel)
self._W_o = nn.Linear(v * h, 1)
self._dropout = nn.Dropout(p=dropout)
def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N_e: int,
N_d: int,
n_months: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None,
n_lag_convs=3,
e_days=28 * 4,
d_days=28):
super().__init__()
self._d_model = d_model
self._inp_dropout = nn.Dropout(0.1)
self.emb_layers = nn.ModuleList(
[nn.Embedding(x, y) for x, y in emb_dims])
self._lag_per_time = nn.Linear(63, n_lag_convs)
self._e_inp_norm = nn.LayerNorm(d_input + n_lag_convs)
self._d_inp_norm = nn.LayerNorm(d_input + n_lag_convs - 1)
self._embedding = nn.Linear(d_input, d_model)
self.enc_embedding = nn.Linear(d_input + n_lag_convs, d_model)
self.dec_embedding = nn.Linear(d_input - 1 + n_lag_convs, d_model)
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N_e)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
n_months,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N_d)
])
self.hist_covs = ConvModule(e_days)
out_dim = 237
self.out = nn.Sequential(
nn.Linear(out_dim, out_dim // 2),
nn.LeakyReLU(),
nn.LayerNorm(out_dim // 2),
nn.Linear(out_dim // 2, d_output),
)
# self.out = nn.Linear(out_dim, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)