def __init__(self, number_time_series: int, seq_length: int = 48, output_seq_len: int = None, d_model: int = 128, n_heads: int = 8, dropout=0.1, forward_dim=2048, sigmoid=False): """ Full transformer model """ super().__init__() if output_seq_len is None: output_seq_len = seq_length self.out_seq_len = output_seq_len self.mask = generate_square_subsequent_mask(seq_length) self.dense_shape = torch.nn.Linear(number_time_series, d_model) self.pe = SimplePositionalEncoding(d_model) self.transformer = Transformer(d_model, nhead=n_heads) self.final_layer = torch.nn.Linear(d_model, 1) self.sequence_size = seq_length self.tgt_mask = generate_square_subsequent_mask(output_seq_len) self.sigmoid = None if sigmoid: self.sigmoid = torch.nn.Sigmoid()
def greedy_decode(model, src: torch.Tensor, max_len: int, real_target: torch.Tensor, unsqueeze_dim=1, output_len=1, device='cpu', multi_targets=1, probabilistic=False, scaler=None): """ Mechanism to sequentially decode the model :src The Historical time series values :real_target The real values (they should be masked), however if you want can include known real values. :returns torch.Tensor """ src = src.float() real_target = real_target.float() if hasattr(model, "mask"): src_mask = model.mask memory = model.encode_sequence(src, src_mask) # Get last element of src array to forecast from ys = src[:, -1, :].unsqueeze(unsqueeze_dim) for i in range(max_len): mask = generate_square_subsequent_mask(i + 1).to(device) with torch.no_grad(): out = model.decode_seq(memory, Variable(ys), Variable(mask), i + 1) real_target[:, i, 0] = out[:, i] src = torch.cat((src, real_target[:, i, :].unsqueeze(1)), 1) ys = torch.cat((ys, real_target[:, i, :].unsqueeze(1)), 1) memory = model.encode_sequence(src[:, i + 1:, :], src_mask) return ys[:, 1:, :]
def __init__( self, seq_length: int, output_seq_length: int, n_time_series: int, d_model=128, output_dim=1, n_layers_encoder=6, forward_dim=2048, dropout=0.1, use_mask=False, meta_data=None, final_act=None, n_heads=8): """ Uses a number of encoder layers with simple linear decoder layer. """ super().__init__() self.dense_shape = torch.nn.Linear(n_time_series, d_model) self.pe = SimplePositionalEncoding(d_model) encoder_layer = TransformerEncoderLayer(d_model, 8, forward_dim, dropout) encoder_norm = LayerNorm(d_model) self.transformer_enc = TransformerEncoder(encoder_layer, n_layers_encoder, encoder_norm) self.output_dim_layer = torch.nn.Linear(d_model, output_dim) self.output_seq_length = output_seq_length self.out_length_lay = torch.nn.Linear(seq_length, output_seq_length) self.mask = generate_square_subsequent_mask(seq_length) self.out_dim = output_dim self.mask_it = use_mask self.final_act = None if final_act: self.final_act = activation_dict[final_act] if meta_data: self.meta_merger = MergingModel(meta_data["method"], meta_data["params"])
def __init__(self, number_time_series: int, seq_length: int = 48, output_seq_len: int = None, d_model: int = 128, n_heads: int = 8, dropout=0.1, forward_dim=2048, sigmoid=False): """A full transformer model :param number_time_series: The total number of time series present (e.g. n_feature_time_series + n_targets) :type number_time_series: int :param seq_length: The length of your input sequence, defaults to 48 :type seq_length: int, optional :param output_seq_len: The length of your output sequence, defaults to None :type output_seq_len: int, optional :param d_model: The dimensions of your model, defaults to 128 :type d_model: int, optional :param n_heads: The number of heads in each encoder/decoder block, defaults to 8 :type n_heads: int, optional :param dropout: The fraction of dropout you wish to apply during training, defaults to 0.1 (currently not functional) :type dropout: float, optional :param forward_dim: Currently not functional, defaults to 2048 :type forward_dim: int, optional :param sigmoid: Whether to apply a sigmoid activation to the final layer (useful for binary classification), defaults to False :type sigmoid: bool, optional """ super().__init__() if output_seq_len is None: output_seq_len = seq_length self.out_seq_len = output_seq_len self.mask = generate_square_subsequent_mask(seq_length) self.dense_shape = torch.nn.Linear(number_time_series, d_model) self.pe = SimplePositionalEncoding(d_model) self.transformer = Transformer(d_model, nhead=n_heads) self.final_layer = torch.nn.Linear(d_model, 1) self.sequence_size = seq_length self.tgt_mask = generate_square_subsequent_mask(output_seq_len) self.sigmoid = None if sigmoid: self.sigmoid = torch.nn.Sigmoid()
def __init__(self, seq_length: int, output_seq_length: int, n_time_series: int, d_model=128, output_dim=1, n_layers_encoder=6, forward_dim=2048, dropout=0.1, use_mask=False, meta_data=None, final_act=None, squashed_embedding=False, n_heads=8): """Uses a number of encoder layers with simple linear decoder layer. :param seq_length: The number of historical time-steps fed into the model in each forward pass. :type seq_length: int :param output_seq_length: The number of forecasted time-steps outputted by the model. :type output_seq_length: int :param n_time_series: The total number of time series present (targets + features) :type n_time_series: int :param d_model: The embedding dim of the mode, defaults to 128 :type d_model: int, optional :param output_dim: The output dimension (should correspond to n_targets), defaults to 1 :type output_dim: int, optional :param n_layers_encoder: The number of encoder layers, defaults to 6 :type n_layers_encoder: int, optional :param forward_dim: The forward embedding dim, defaults to 2048 :type forward_dim: int, optional :param dropout: How much dropout to use, defaults to 0.1 :type dropout: float, optional :param use_mask: Whether to use subsquent sequence mask during training, defaults to False :type use_mask: bool, optional :param meta_data: Whether to use static meta-data, defaults to None :type meta_data: str, optional :param final_act: Whether to use a final activation function, defaults to None :type final_act: str, optional :param squashed_embedding: Whether to create a one 1-D time embedding, defaults to False :type squashed_embedding: bool, optional :param n_heads: [description], defaults to 8 :type n_heads: int, optional """ super().__init__() self.dense_shape = torch.nn.Linear(n_time_series, d_model) self.pe = SimplePositionalEncoding(d_model) encoder_layer = TransformerEncoderLayer(d_model, 8, forward_dim, dropout) encoder_norm = LayerNorm(d_model) self.transformer_enc = TransformerEncoder(encoder_layer, n_layers_encoder, encoder_norm) self.output_dim_layer = torch.nn.Linear(d_model, output_dim) self.output_seq_length = output_seq_length self.out_length_lay = torch.nn.Linear(seq_length, output_seq_length) self.mask = generate_square_subsequent_mask(seq_length) self.out_dim = output_dim self.mask_it = use_mask self.final_act = None self.squashed = None if final_act: self.final_act = activation_dict[final_act] if meta_data: self.meta_merger = MergingModel(meta_data["method"], meta_data["params"]) if squashed_embedding: self.squashed = torch.nn.Linear(seq_length, 1) self.unsquashed = torch.nn.Linear(1, seq_length)