def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None,
activation='nothing'):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
self.softmax = nn.Softmax(dim=1)
self.activation = activation
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
self.activation_functions = { # for last number
'sigmoid': torch.sigmoid,
'relu': torch.relu,
'tanh': torch.tanh,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)
def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: str = 'chunk',
pe: str = None,
pe_period: int = 24):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)
])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
self._pe_period = pe_period
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)
self.name = 'transformer'
def __init__(self, # training.ipynbでは
d_input: int, # 37(datasetのカラム数)
d_model: int, # 64 Lattent dim
d_output: int, # 8 (datasetのoutput側のカラム数)
q: int, # 8
v: int, # 8
h: int, # 4
N: int, # 4 tencoder-decoderの重ね回数
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None):
"""Create transformer structure from Encoder and Decoder blocks."""
super().__init__()
self._d_model = d_model
self.layers_encoding = nn.ModuleList([Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)])
self.layers_decoding = nn.ModuleList([Decoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N)])
self._embedding = nn.Linear(d_input, d_model)
self._linear = nn.Linear(d_model, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.')
def __init__(self,
d_input: int,
d_model: int,
d_output: int,
q: int,
v: int,
h: int,
N_e: int,
N_d: int,
n_months: int,
attention_size: int = None,
dropout: float = 0.3,
chunk_mode: bool = True,
pe: str = None,
n_lag_convs=3,
e_days=28 * 4,
d_days=28):
super().__init__()
self._d_model = d_model
self._inp_dropout = nn.Dropout(0.1)
self.emb_layers = nn.ModuleList(
[nn.Embedding(x, y) for x, y in emb_dims])
self._lag_per_time = nn.Linear(63, n_lag_convs)
self._e_inp_norm = nn.LayerNorm(d_input + n_lag_convs)
self._d_inp_norm = nn.LayerNorm(d_input + n_lag_convs - 1)
self._embedding = nn.Linear(d_input, d_model)
self.enc_embedding = nn.Linear(d_input + n_lag_convs, d_model)
self.dec_embedding = nn.Linear(d_input - 1 + n_lag_convs, d_model)
self.layers_encoding = nn.ModuleList([
Encoder(d_model,
q,
v,
h,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N_e)
])
self.layers_decoding = nn.ModuleList([
Decoder(d_model,
q,
v,
h,
n_months,
attention_size=attention_size,
dropout=dropout,
chunk_mode=chunk_mode) for _ in range(N_d)
])
self.hist_covs = ConvModule(e_days)
out_dim = 237
self.out = nn.Sequential(
nn.Linear(out_dim, out_dim // 2),
nn.LeakyReLU(),
nn.LayerNorm(out_dim // 2),
nn.Linear(out_dim // 2, d_output),
)
# self.out = nn.Linear(out_dim, d_output)
pe_functions = {
'original': generate_original_PE,
'regular': generate_regular_PE,
}
if pe in pe_functions.keys():
self._generate_PE = pe_functions[pe]
elif pe is None:
self._generate_PE = None
else:
raise NameError(
f'PE "{pe}" not understood. Must be one of {", ".join(pe_functions.keys())} or None.'
)