def __init__( self, input_size: int, num_layers: int, num_cells: int, cell_type: str, history_length: int, context_length: int, prediction_length: int, distr_output: DistributionOutput, control_output: DistributionOutput, dropout_rate: float, cardinality: List[int], embedding_dimension: List[int], lags_seq: List[int], scaling: bool = True, dtype: np.dtype = np.float32, ) -> None: super().__init__() self.num_layers = num_layers self.num_cells = num_cells self.cell_type = cell_type self.history_length = history_length self.context_length = context_length self.prediction_length = prediction_length self.dropout_rate = dropout_rate self.cardinality = cardinality self.embedding_dimension = embedding_dimension self.num_cat = len(cardinality) self.scaling = scaling self.dtype = dtype self.lags_seq = lags_seq self.distr_output = distr_output self.control_output = control_output rnn = {"LSTM": nn.LSTM, "GRU": nn.GRU}[self.cell_type] self.rnn = rnn( input_size=input_size, hidden_size=num_cells, num_layers=num_layers, dropout=dropout_rate, batch_first=True, ) self.target_shape = distr_output.event_shape self.proj_distr_args = distr_output.get_args_proj(num_cells + 1) self.proj_control_args = control_output.get_args_proj(num_cells) self.embedder = FeatureEmbedder(cardinalities=cardinality, embedding_dims=embedding_dimension) if scaling: self.scaler = MeanScaler(keepdim=True) self.control_scaler = NOPScaler(keepdim=True) else: self.scaler = NOPScaler(keepdim=True) self.control_scaler = NOPScaler(keepdim=True)
def learn_distribution( distr_output: DistributionOutput, samples: torch.Tensor, init_biases: List[np.ndarray] = None, num_epochs: int = 5, learning_rate: float = 1e-2, ): arg_proj = distr_output.get_args_proj(in_features=1) if init_biases is not None: for param, bias in zip(arg_proj.proj, init_biases): nn.init.constant_(param.bias, bias) dummy_data = torch.ones((len(samples), 1, 1)) dataset = TensorDataset(dummy_data, samples) train_data = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) optimizer = SGD(arg_proj.parameters(), lr=learning_rate) for e in range(num_epochs): cumulative_loss = 0 num_batches = 0 for i, (data, sample_label) in enumerate(train_data): optimizer.zero_grad() distr_args = arg_proj(data) distr = distr_output.distribution(distr_args) loss = -distr.log_prob(sample_label).mean() loss.backward() #clip_grad_norm_(arg_proj.parameters(), 10.0) optimizer.step() num_batches += 1 cumulative_loss += loss.item() print("Epoch %s, loss: %s" % (e, cumulative_loss / num_batches)) sampling_dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) i, (data, sample_label) = next(enumerate(sampling_dataloader)) distr_args = arg_proj(data) distr = distr_output.distribution(distr_args) samples = distr.sample((NUM_SAMPLES, )) with torch.no_grad(): percentile_90 = distr.quantile_function(torch.ones((1, 1, 1)), torch.ones((1, 1)) * 0.9) percentile_10 = distr.quantile_function(torch.ones((1, 1, 1)), torch.ones((1, 1)) * 0.1) return samples.mean(), samples.std(), percentile_10.squeeze( ), percentile_90.squeeze()
def maximum_likelihood_estimate_sgd( distr_output: DistributionOutput, samples: torch.Tensor, init_biases: List[np.ndarray] = None, num_epochs: PositiveInt = PositiveInt(5), learning_rate: PositiveFloat = PositiveFloat(1e-2), ): arg_proj = distr_output.get_args_proj(in_features=1) if init_biases is not None: for param, bias in zip(arg_proj.proj, init_biases): nn.init.constant_(param.bias, bias) dummy_data = torch.ones((len(samples), 1)) dataset = TensorDataset(dummy_data, samples) train_data = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True) optimizer = SGD(arg_proj.parameters(), lr=learning_rate) for e in range(num_epochs): cumulative_loss = 0 num_batches = 0 for i, (data, sample_label) in enumerate(train_data): optimizer.zero_grad() distr_args = arg_proj(data) distr = distr_output.distribution(distr_args) loss = -distr.log_prob(sample_label).mean() loss.backward() clip_grad_norm_(arg_proj.parameters(), 10.0) optimizer.step() num_batches += 1 cumulative_loss += loss.item() print("Epoch %s, loss: %s" % (e, cumulative_loss / num_batches)) if len(distr_args[0].shape) == 1: return [ param.detach().numpy() for param in arg_proj(torch.ones((1, 1))) ] return [ param[0].detach().numpy() for param in arg_proj(torch.ones((1, 1))) ]
def __init__( self, freq: str, context_length: int, prediction_length: int, num_feat_dynamic_real: int, num_feat_static_real: int, num_feat_static_cat: int, cardinality: List[int], embedding_dimension: Optional[List[int]] = None, num_layers: int = 2, hidden_size: int = 40, dropout_rate: float = 0.1, distr_output: DistributionOutput = StudentTOutput(), lags_seq: Optional[List[int]] = None, scaling: bool = True, num_parallel_samples: int = 100, ) -> None: super().__init__() self.context_length = context_length self.prediction_length = prediction_length self.distr_output = distr_output self.param_proj = distr_output.get_args_proj(hidden_size) self.target_shape = distr_output.event_shape self.num_feat_dynamic_real = num_feat_dynamic_real self.num_feat_static_cat = num_feat_static_cat self.num_feat_static_real = num_feat_static_real self.embedding_dimension = ( embedding_dimension if embedding_dimension is not None or cardinality is None else [min(50, (cat + 1) // 2) for cat in cardinality] ) self.lags_seq = lags_seq or get_lags_for_frequency(freq_str=freq) self.num_parallel_samples = num_parallel_samples self.history_length = self.context_length + max(self.lags_seq) self.embedder = FeatureEmbedder( cardinalities=cardinality, embedding_dims=self.embedding_dimension, ) if scaling: self.scaler = MeanScaler(dim=1, keepdim=True) else: self.scaler = NOPScaler(dim=1, keepdim=True) self.lagged_rnn = LaggedLSTM( input_size=1, # TODO fix features_size=self._number_of_features, num_layers=num_layers, hidden_size=hidden_size, dropout_rate=dropout_rate, lags_seq=[lag - 1 for lag in self.lags_seq], )
def __init__( self, input_size: int, d_model: int, num_heads: int, act_type: str, dropout_rate: float, dim_feedforward_scale: int, num_encoder_layers: int, num_decoder_layers: int, history_length: int, context_length: int, prediction_length: int, distr_output: DistributionOutput, cardinality: List[int], embedding_dimension: List[int], lags_seq: List[int], scaling: bool = True, **kwargs, ) -> None: super().__init__(**kwargs) self.history_length = history_length self.context_length = context_length self.prediction_length = prediction_length self.scaling = scaling self.cardinality = cardinality self.embedding_dimension = embedding_dimension self.distr_output = distr_output assert len( set(lags_seq)) == len(lags_seq), "no duplicated lags allowed!" lags_seq.sort() self.lags_seq = lags_seq self.target_shape = distr_output.event_shape # [B, T, input_size] -> [B, T, d_model] self.encoder_input = nn.Linear(input_size, d_model) self.decoder_input = nn.Linear(input_size, d_model) # [B, T, d_model] where d_model / num_heads is int self.transformer = nn.Transformer( d_model=d_model, nhead=num_heads, num_encoder_layers=num_encoder_layers, num_decoder_layers=num_decoder_layers, dim_feedforward=dim_feedforward_scale * d_model, dropout=dropout_rate, activation=act_type, ) self.proj_dist_args = distr_output.get_args_proj(d_model) self.embedder = FeatureEmbedder( cardinalities=cardinality, embedding_dims=embedding_dimension, ) if scaling: self.scaler = MeanScaler(keepdim=True) else: self.scaler = NOPScaler(keepdim=True) # mask self.register_buffer( "tgt_mask", self.transformer.generate_square_subsequent_mask( prediction_length), )
def __init__( self, input_size: int, num_layers: int, num_cells: int, cell_type: str, history_length: int, context_length: int, prediction_length: int, distr_output: DistributionOutput, dropout_rate: float, lags_seq: List[int], target_dim: int, cardinality: List[int], embedding_dimension: List[int], scaling: bool = True, **kwargs, ) -> None: super().__init__(**kwargs) self.num_layers = num_layers self.num_cells = num_cells self.cell_type = cell_type self.history_length = history_length self.context_length = context_length self.prediction_length = prediction_length self.dropout_rate = dropout_rate self.cardinality = cardinality self.embedding_dimension = embedding_dimension self.num_cat = len(cardinality) self.target_dim = target_dim self.scaling = scaling self.target_dim_sample = target_dim assert len( set(lags_seq)) == len(lags_seq), "no duplicated lags allowed!" lags_seq.sort() self.lags_seq = lags_seq self.distr_output = distr_output self.target_dim = target_dim rnn = {"LSTM": nn.LSTM, "GRU": nn.GRU}[self.cell_type] self.rnn = rnn( input_size=input_size, hidden_size=num_cells, num_layers=num_layers, dropout=dropout_rate, batch_first=True, ) self.target_shape = distr_output.event_shape self.proj_dist_args = distr_output.get_args_proj(num_cells) self.embed = FeatureEmbedder(cardinalities=cardinality, embedding_dims=embedding_dimension) if scaling: self.scaler = MeanScaler(keepdim=True) else: self.scaler = NOPScaler(keepdim=True)