def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = "parametric_softplus", # Transformer units_rnn: Optional[int] = 16, layers_rnn: Optional[int] = 1, num_heads: Optional[int] = 1, activation_rnn: Optional[str] = "relu", dropout_rnn: Optional[float] = 0., attn_activation: Optional[str] = "softmax", # Other params mc_prop_est: Optional[float] = 1., emb_dim: Optional[int] = 4, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(SelfAttentionCmDecoder, self).__init__(name="selfattention-cm", input_size=encoding_size(encoding=encoding, emb_dim=emb_dim), mc_prop_est=mc_prop_est, emb_dim=emb_dim, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) self.mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp) self.mlp1 = MLP(units=[1], activations=None, constraint="nonneg", dropout_rates=None, input_shape=1, activation_final=None, use_bias=False)
def __init__( self, # MLP args units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = None, # Other args emb_dim: Optional[int] = 1, embedding_constraint: Optional[str] = None, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(MLPVariableEncoder, self).__init__( name="mlp-variable", output_size=units_mlp[-1], emb_dim=emb_dim, embedding_constraint=embedding_constraint, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) self.mlp = MLP( units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp)
def __init__(self, name: str, input_size: Optional[int] = None, emb_dim: Optional[int] = 1, embedding_constraint: Optional[str] = None, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(VariableHistoryDecoder, self).__init__(name=name, input_size=input_size, marks=marks, **kwargs) self.emb_dim = emb_dim self.encoding = encoding self.time_encoding = time_encoding self.embedding_constraint = embedding_constraint self.encoding_size = encoding_size(encoding=self.encoding, emb_dim=self.emb_dim) self.embedding = None if encoding in [ "marks_only", "concatenate", "temporal_with_labels", "learnable_with_labels" ]: self.embedding = MLP(units=[self.emb_dim], activations=None, constraint=self.embedding_constraint, dropout_rates=0, input_shape=self.marks, activation_final=None, use_bias=False) self.temporal_enc = None if encoding in ["temporal", "temporal_with_labels"]: self.temporal_enc = SinusoidalEncoding(emb_dim=self.emb_dim, scaling=temporal_scaling) elif encoding in ["learnable", "learnable_with_labels"]: self.temporal_enc = MLP(units=[self.emb_dim], activations=None, constraint=self.embedding_constraint, dropout_rates=0, input_shape=1, activation_final=None)
def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = "parametric_softplus", # Transformer units_rnn: Optional[int] = 16, layers_rnn: Optional[int] = 1, n_heads: Optional[int] = 1, activation_rnn: Optional[str] = "relu", dropout_rnn: Optional[float] = 0., attn_activation: Optional[str] = "softmax", constraint_rnn: Optional[str] = None, # Other params mc_prop_est: Optional[float] = 1., emb_dim: Optional[int] = 2, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(SelfAttentionMCDecoder, self).__init__(name="selfattention-mc", input_size=encoding_size(encoding=encoding, emb_dim=emb_dim), mc_prop_est=mc_prop_est, emb_dim=emb_dim, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) decoder_layer = TransformerDecoderLayer( d_model=self.encoding_size, nhead=n_heads, dim_feedforward=units_rnn, dropout=dropout_rnn, activation=activation_rnn, attn_activation=attn_activation, constraint=constraint_rnn, normalisation="layernorm") self.transformer_decoder = TransformerDecoderNetwork( decoder_layer=decoder_layer, num_layers=layers_rnn) self.mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp) self.n_heads = n_heads
def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = None, # Transformer units_rnn: Optional[int] = 16, layers_rnn: Optional[int] = 1, n_heads: Optional[int] = 1, activation_rnn: Optional[str] = "relu", dropout_rnn: Optional[float] = 0., attn_activation: Optional[str] = "softmax", # Other allow_window_attention: Optional[bool] = False, emb_dim: Optional[int] = 2, embedding_constraint: Optional[str] = None, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(SelfAttentionEncoder, self).__init__(name="selfattention", output_size=units_mlp[-1], emb_dim=emb_dim, embedding_constraint=embedding_constraint, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) self.src_mask = None self.allow_window_attention = allow_window_attention encoder_layer = TransformerEncoderLayer( d_model=self.encoding_size, nhead=n_heads, dim_feedforward=units_rnn, dropout=dropout_rnn, activation=activation_rnn, attn_activation=attn_activation) self.transformer_encoder = TransformerEncoderNetwork( encoder_layer=encoder_layer, num_layers=layers_rnn) self.mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp)
def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = "nonneg", activation_final_mlp: Optional[str] = "parametric_softplus", # Other params model_log_cm: Optional[bool] = False, do_zero_subtraction: Optional[bool] = True, emb_dim: Optional[int] = 2, encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): if constraint_mlp is None: print("Warning! MLP decoder is unconstrained. Setting to `nonneg`") constraint_mlp = "nonneg" enc_size = encoding_size(encoding=encoding, emb_dim=emb_dim) input_size = units_mlp[0] - enc_size super(MLPCmDecoder, self).__init__(name="mlp-cm", do_zero_subtraction=do_zero_subtraction, model_log_cm=model_log_cm, input_size=input_size, emb_dim=emb_dim, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) self.mlp = MLP( units=units_mlp[1:], activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, # units_mlp in this class also provides the input dimensionality # of the mlp input_shape=units_mlp[0], activation_final=activation_final_mlp)
def __init__( self, # RNN args units_rnn: int, # MLP args units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = "parametric_softplus", # Other params mc_prop_est: Optional[float] = 1., input_size: Optional[int] = None, emb_dim: Optional[int] = 1, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): super(NeuralHawkesDecoder, self).__init__(name="neural-hawkes", mc_prop_est=mc_prop_est, input_size=input_size, emb_dim=emb_dim, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) # Parameters self.weight_ih = nn.Parameter(th.Tensor(units_rnn, units_rnn * 7)) self.weight_hh = nn.Parameter(th.Tensor(units_rnn, units_rnn * 7)) self.bias = nn.Parameter(th.Tensor(units_rnn * 7)) self.mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp) self.units_rnn = units_rnn self.reset_parameters()
def __init__(self, units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = None, history_size: Optional[int] = 2, marks: Optional[int] = 1, **kwargs): mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=history_size, activation_final=activation_final_mlp) super(MLPFixedEncoder, self).__init__(name="mlp-fixed", net=mlp, output_size=units_mlp[-1], history_size=history_size, marks=marks, **kwargs)
def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = "parametric_softplus", # Other params mc_prop_est: Optional[float] = 1., emb_dim: Optional[int] = 2, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): enc_size = encoding_size(encoding=encoding, emb_dim=emb_dim) input_size = units_mlp[0] - enc_size if len(units_mlp) < 2: raise ValueError("Units of length at least 2 need to be specified") super(MLPMCDecoder, self).__init__( name="mlp-mc", input_size=input_size, mc_prop_est=mc_prop_est, emb_dim=emb_dim, temporal_scaling=temporal_scaling, encoding=encoding, time_encoding=time_encoding, marks=marks, **kwargs) self.mlp = MLP( units=units_mlp[1:], activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, # units_mlp in this class also provides the input dimensionality # of the mlp input_shape=units_mlp[0], activation_final=activation_final_mlp)
def __init__( self, # MLP units_mlp: List[int], activation_mlp: Optional[str] = "relu", dropout_mlp: Optional[float] = 0., constraint_mlp: Optional[str] = None, activation_final_mlp: Optional[str] = "parametric_softplus", # Transformer units_rnn: Optional[int] = 16, layers_rnn: Optional[int] = 1, n_heads: Optional[int] = 1, activation_rnn: Optional[str] = "relu", dropout_rnn: Optional[float] = 0., attn_activation: Optional[str] = "softmax", constraint_rnn: Optional[str] = None, # Other params do_zero_subtraction: Optional[bool] = True, model_log_cm: Optional[bool] = False, mc_prop_est: Optional[float] = 1., emb_dim: Optional[int] = 4, temporal_scaling: Optional[float] = 1., encoding: Optional[str] = "times_only", time_encoding: Optional[str] = "relative", marks: Optional[int] = 1, **kwargs): if constraint_rnn is None: print("Warning! SA decoder is unconstrained. Setting to `nonneg`") constraint_rnn = "nonneg" if constraint_mlp is None: print("Warning! MLP decoder is unconstrained. Setting to `nonneg`") constraint_mlp = "nonneg" super(SelfAttentionCmDecoder, self).__init__(name="selfattention-cm", do_zero_subtraction=do_zero_subtraction, model_log_cm=model_log_cm, input_size=encoding_size(encoding=encoding, emb_dim=emb_dim), mc_prop_est=mc_prop_est, emb_dim=emb_dim, temporal_scaling=temporal_scaling, encoding=encoding, marks=marks, **kwargs) decoder_layer = TransformerDecoderLayer( d_model=self.encoding_size, nhead=n_heads, dim_feedforward=units_rnn, dropout=dropout_rnn, activation=activation_rnn, attn_activation=attn_activation, constraint=constraint_rnn, normalisation="layernorm_with_running_stats") self.transformer_decoder = TransformerDecoderNetwork( decoder_layer=decoder_layer, num_layers=layers_rnn) self.mlp = MLP(units=units_mlp, activations=activation_mlp, constraint=constraint_mlp, dropout_rates=dropout_mlp, input_shape=self.encoding_size, activation_final=activation_final_mlp) self.n_heads = n_heads
prev_time = t - events prev_time[prev_time < 0] = x_max + 1. prev_time = th.argmin(prev_time) prev_time = events[prev_time] prev_times[i] = prev_time tau = x_train - prev_times activations, constraint, activation_final = "relu", None, None if cumulative: (activations, constraint, activation_final) = "gumbel", "nonneg", "parametric_softplus" mlp = MLP( units=units, input_shape=1, activations=activations, constraint=constraint, activation_final=activation_final) optimiser = th.optim.Adam(params=mlp.parameters(), lr=1.e-3) mse = torch.nn.MSELoss() tau_r = tau.reshape(-1, 1) y_train_r = y_train.reshape(-1, 1) if cumulative: tau_r.requires_grad = True for i in range(epochs): optimiser.zero_grad() y_pred = mlp(tau_r)