def tcn(config: BaseConfig, trial: optuna.trial.Trial) -> pl.LightningModule: """Returns a tunable PyTorch lightning tcn module. Args: config (BaseConfig): the hard-coded configuration. trial (optuna.Trial): optuna trial. Returns: pl.LightningModule: a lightning module. """ training_config = get_training_config( lr=trial.suggest_loguniform('lr', 1e-3, 1e-0), weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1), max_epochs=config.MAX_EPOCHS) tcn = TemporalConvNet(training_config=training_config, num_inputs=config.NUM_INPUTS, num_outputs=config.NUM_OUTPUTS, num_hidden=trial.suggest_int('num_hidden', 1, 4), kernel_size=trial.suggest_int('kernel_size', 2, 4), num_layers=trial.suggest_int('num_layers', 1, 2), dropout=trial.suggest_float('dropout', 0.1, 0.3)) return tcn
def __init__(self, **kwargs): super(TCN, self).__init__() self.input_size = kwargs['input_channels'] self.wavelet = kwargs['wavelet'] self.input_length = kwargs['input_length'] output_size = kwargs['output_size'] kernel_size = kwargs['kernel_size'] dropout = kwargs['dropout'] num_channels = kwargs['channel_lst'] wavelet_output_size = kwargs['wavelet_output_size'] num_channels = [int(x) for x in num_channels.split(',')] linear_size = num_channels[-1] if self.wavelet: self.input_size = self.input_size//2 wvlt_size = self.input_length * self.input_size // 2 self.linear_wavelet = nn.Linear(wvlt_size, wavelet_output_size) linear_size += 2 * wavelet_output_size self.tcn = TemporalConvNet( self.input_size, num_channels, kernel_size=kernel_size, dropout=dropout) self.input_bn = nn.BatchNorm1d(linear_size) self.linear = nn.Linear(linear_size, output_size)
def feedforward(config: BaseConfig, trial: optuna.trial.Trial) -> pl.LightningModule: """Returns a tunable PyTorch lightning feedforward module. Args: config (BaseConfig): the hard-coded configuration. trial (optuna.Trial): optuna trial. Returns: pl.LightningModule: a lightning module. """ model = FeedForward(num_inputs=config.NUM_INPUTS, num_outputs=config.NUM_OUTPUTS, num_hidden=trial.suggest_int('num_hidden', 1, 4), num_layers=trial.suggest_int('num_layers', 1, 2), dropout=trial.suggest_float('dropout', 0.0, 0.5), activation=trial.suggest_categorical( 'activation', ['relu', 'none'])) training_config = get_training_config( lr=trial.suggest_loguniform('lr', 1e-3, 1e-0), weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1), max_epochs=config.MAX_EPOCHS) pl_model = TemporalConvNet(training_config=training_config, lr=trial.suggest_loguniform('lr', 1e-3, 1e-0), weight_decay=trial.suggest_loguniform( 'weight_decay', 1e-5, 1e-1), max_epochs=config.MAX_EPOCHS) return pl_model
def __init__(self, out_dim, v_hdim, cnn_fdim, no_cnn=False, frame_shape=(3, 64, 64), mlp_dim=(300, 200), cnn_type='resnet', v_net_type='lstm', v_net_param=None, cnn_rs=True, causal=False, device=None): super().__init__() self.out_dim = out_dim self.cnn_fdim = cnn_fdim self.v_hdim = v_hdim self.no_cnn = no_cnn self.cnn_type = cnn_type self.frame_shape = frame_shape self.device = device if no_cnn: self.cnn = None else: self.frame_shape = (1, 32, 32, 64) """ only for ResNet based models """ if v_net_param is None: v_net_param = {} spec = v_net_param.get('spec', 'resnet18') self.cnn = P2PsfNet(cnn_fdim, device=self.device, running_stats=cnn_rs, spec=spec) self.v_net_type = v_net_type if v_net_type == 'lstm': self.v_net = RNN(cnn_fdim, v_hdim, v_net_type, bi_dir=not causal) elif v_net_type == 'tcn': if v_net_param is None: v_net_param = {} tcn_size = v_net_param.get('size', [64, 128]) dropout = v_net_param.get('dropout', 0.2) kernel_size = v_net_param.get('kernel_size', 3) assert tcn_size[-1] == v_hdim self.v_net = TemporalConvNet(cnn_fdim, tcn_size, kernel_size=kernel_size, dropout=dropout, causal=causal) if self.v_net_type is 'no_lstm': self.mlp = MLP(self.cnn_fdim, mlp_dim, 'relu') else: self.mlp = MLP(v_hdim, mlp_dim, 'relu') self.linear = nn.Linear(self.mlp.out_dim, out_dim)
def __init__(self, num_features, bottleneck_channels=128, hidden_channels=256, skip_channels=128, kernel_size=3, num_blocks=3, num_layers=8, dilated=True, separable=True, causal=True, nonlinear='prelu', norm=True, mask_nonlinear='sigmoid', n_sources=2, eps=EPS): super().__init__() self.num_features, self.n_sources = num_features, n_sources self.norm1d = choose_layer_norm(num_features, causal=causal, eps=eps) self.bottleneck_conv1d = nn.Conv1d(num_features, bottleneck_channels, kernel_size=1, stride=1) self.tcn = TemporalConvNet(bottleneck_channels, hidden_channels=hidden_channels, skip_channels=skip_channels, kernel_size=kernel_size, num_blocks=num_blocks, num_layers=num_layers, dilated=dilated, separable=separable, causal=causal, nonlinear=nonlinear, norm=norm) self.prelu = nn.PReLU() self.mask_conv1d = nn.Conv1d(skip_channels, n_sources * num_features, kernel_size=1, stride=1) if mask_nonlinear == 'sigmoid': self.mask_nonlinear = nn.Sigmoid() elif mask_nonlinear == 'softmax': self.mask_nonlinear = nn.Softmax(dim=1) else: raise ValueError("Cannot support {}".format(mask_nonlinear))
def __init__(self): super(Classifier, self).__init__() self.add_module( 'resnets', nn.ModuleList( map( lambda it: nn.Sequential( ), # ResNet1d(**it, activation=activation), profile.resnets))) self.add_module( 'tcns', nn.ModuleList( map( lambda it: TemporalConvNet(**it, activation=activation), profile.tcns))) self.add_module( 'flatten', nn.Sequential(SelfAttention(), nn.Flatten(), Unsqueeze(1), nn.AdaptiveAvgPool1d(profile.pool_size), Squeeze())) self.add_module( 'linears', nn.Sequential(*tuple( map( lambda idx: nn.Sequential( nn.Linear( profile.pool_size if idx == 0 else profile.linear_features[ idx - 1], profile.num_classes if idx == len(profile.linear_features) - 1 else profile.linear_features[idx], True), activation(True) if idx != len(profile.linear_features) - 1 else nn.Sequential()), range(len(profile.linear_features))))) if len(profile.linear_features) > 0 else nn.Linear( profile.pool_size, profile.num_classes, True)) self.init_weights()
def __init__(self, out_dim, v_hdim, cnn_fdim, no_cnn=False, frame_shape=(3, 224, 224), mlp_dim=(300, 200), cnn_type='resnet', v_net_type='lstm', v_net_param=None, causal=False): super().__init__() self.out_dim = out_dim self.cnn_fdim = cnn_fdim self.v_hdim = v_hdim self.no_cnn = no_cnn self.frame_shape = frame_shape if no_cnn: self.cnn = None elif cnn_type == 'resnet': self.cnn = ResNet(cnn_fdim) elif cnn_type == 'mobile': self.cnn = MobileNet(cnn_fdim) self.v_net_type = v_net_type if v_net_type == 'lstm': self.v_net = RNN(cnn_fdim, v_hdim, v_net_type, bi_dir=not causal) elif v_net_type == 'tcn': if v_net_param is None: v_net_param = {} tcn_size = v_net_param.get('size', [64, 128]) dropout = v_net_param.get('dropout', 0.2) kernel_size = v_net_param.get('kernel_size', 3) assert tcn_size[-1] == v_hdim self.v_net = TemporalConvNet(cnn_fdim, tcn_size, kernel_size=kernel_size, dropout=dropout, causal=causal) self.mlp = MLP(v_hdim, mlp_dim, 'relu') self.linear = nn.Linear(self.mlp.out_dim, out_dim)
def __init__(self, enc_in, dec_in, c_out, seq_len, out_len, d_model=120, n_heads=8, e_layers=3, d_layers=2, d_ff=512, dropout=0.0, embed='fixed', data='Ali_00', activation='gelu', device=torch.device('cuda:0')): super(Model, self).__init__() self.pred_len = out_len self.tcn = TemporalConvNet(d_model, [d_model, d_model, d_model], kernel_size=2, dropout=dropout) # Encoding self.enc_embedding = DataEmbedding(enc_in, d_model, dropout) self.dec_embedding = DataEmbedding(dec_in, d_model, dropout) # Attention Attn = FullAttention # Encoder self.encoder = Encoder([ EncoderLayer(AttentionLayer(Attn(False, attention_dropout=dropout), d_model, n_heads), d_model, d_ff, dropout=dropout, activation=activation) for l in range(e_layers) ], [ConvLayer(d_model) for l in range(e_layers - 1)], tcn_layers=self.tcn, norm_layer=torch.nn.LayerNorm(d_model)) self.hidden = d_model * 12 self.predict = nn.Linear(self.hidden, 24, bias=None) self.d_model = d_model
def __init__(self, cnn_feat_dim, state_dim, v_hdim=128, v_margin=10, v_net_type='lstm', v_net_param=None, s_hdim=None, s_net_type='id', dynamic_v=False): super().__init__() s_hdim = state_dim if s_hdim is None else s_hdim self.mode = 'test' self.cnn_feat_dim = cnn_feat_dim self.state_dim = state_dim self.v_net_type = v_net_type self.v_hdim = v_hdim self.v_margin = v_margin self.s_net_type = s_net_type self.s_hdim = s_hdim self.dynamic_v = dynamic_v self.out_dim = v_hdim + s_hdim if v_net_type == 'lstm': self.v_net = RNN(cnn_feat_dim, v_hdim, v_net_type, bi_dir=False) elif v_net_type == 'tcn': if v_net_param is None: v_net_param = {} tcn_size = v_net_param.get('size', [64, 128]) dropout = v_net_param.get('dropout', 0.2) kernel_size = v_net_param.get('kernel_size', 3) assert tcn_size[-1] == v_hdim self.v_net = TemporalConvNet(cnn_feat_dim, tcn_size, kernel_size=kernel_size, dropout=dropout, causal=True) if s_net_type == 'lstm': self.s_net = RNN(state_dim, s_hdim, s_net_type, bi_dir=False) self.v_out = None self.t = 0 # training only self.indices = None self.s_scatter_indices = None self.s_gather_indices = None self.v_gather_indices = None self.cnn_feat_ctx = None self.num_episode = None self.max_episode_len = None self.set_mode('test')
def __init__(self, cnn_feat_dim, v_hdim=128, v_margin=10, v_net_type='lstm', v_net_param=None, causal=False): super().__init__() self.mode = 'test' self.cnn_feat_dim = cnn_feat_dim self.v_net_type = v_net_type self.v_hdim = v_hdim self.v_margin = v_margin if v_net_type == 'lstm': self.v_net = RNN(cnn_feat_dim, v_hdim, v_net_type, bi_dir=not causal) elif v_net_type == 'tcn': if v_net_param is None: v_net_param = {} tcn_size = v_net_param.get('size', [64, 128]) dropout = v_net_param.get('dropout', 0.2) kernel_size = v_net_param.get('kernel_size', 3) assert tcn_size[-1] == v_hdim self.v_net = TemporalConvNet(cnn_feat_dim, tcn_size, kernel_size=kernel_size, dropout=dropout, causal=causal) self.v_out = None self.t = 0 # training only self.indices = None self.scatter_indices = None self.gather_indices = None self.cnn_feat_ctx = None