Пример #1
0
    def __init__(self,
                 act_fn,
                 combinator,
                 neurons,
                 normalize=None,
                 init='random',
                 alpha_dropout=None,
                 hr_test=None):
        super(MIX, self).__init__()
        self.combinator = combinator  # name of the combinator, e.g. "Linear"
        self.act_fn = act_fn  # basic activation function to be used, e.g. "Tanh, Sigmoid"
        self.normalize = normalize  # normalize alpha, e.g. with a Sigmoid
        self.neurons = neurons  # number of neurons of the layer
        self.alpha_dropout = alpha_dropout  # apply a dropout on alpha (only for MLP_ATT)
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        self.act_module = {
            'relu': nn.ReLU(),  # dictionary containing useful functions
            'sigmoid': nn.Sigmoid(),
            'tanh': nn.Tanh(),
            'antirelu': Antirelu(),
            'identity': Identity(),
            'softmax': nn.Softmax(dim=-1)
        }
        self.hr_test = hr_test
        # TODO: assert hr_test != False implies combinator=='MLP_ATT_b'
        if combinator == 'Linear':  # 3 different alpha initialization for the Linear combinator
            assert init in ['normal', 'uniform', 'random'
                            ], "init must be 'normal','uniform','random'"
            if init == 'normal':  # sample taken from a gaussian N(0,1)
                self.alpha = nn.Parameter(torch.randn(neurons, len(act_fn)),
                                          requires_grad=True)
            elif init == 'uniform':  # same init for each alpha, equal to 1/(num of act_fn)
                self.alpha = nn.Parameter(torch.ones(neurons, len(act_fn)) /
                                          len(act_fn),
                                          requires_grad=True)
            elif init == 'random':  # sample alpha in a uniform interval
                self.alpha = nn.Parameter(torch.FloatTensor(
                    neurons, len(act_fn)).uniform_(-0.5, 0.5),
                                          requires_grad=True)

        elif combinator in MLP_list + ATT_list:  # create a list of MLP
            self.MLP_list = nn.ModuleList([
                MLP(combinator).to(self.device) for _ in range(neurons)
            ]).to(self.device)
            if combinator == 'MLP_ATT_b':
                self.beta = nn.Parameter(torch.FloatTensor(neurons).uniform_(
                    -0.5, 0.5),
                                         requires_grad=True).to(self.device)

        elif combinator == 'Hybrid':
            self.MLP_list = nn.ModuleList([])
            for i in range(neurons // 3):
                self.MLP_list.extend([self.act_module['relu']])
                self.MLP_list.extend([self.act_module['relu']])
                self.MLP_list.extend([MLP('MLP1')])
        elif combinator == 'MLPr':  # MLPr is a mix of MLP1, MLP2
            self.MLP_list = nn.ModuleList([])
            for i in range(neurons // 2):
                self.MLP_list.extend([MLP('MLP1')])
                self.MLP_list.extend([MLP('MLP2')])
Пример #2
0
    def __init__(self, args):
        super(Model, self).__init__()

        self.args = args
        # the embedding layer
        self.word_embed = nn.Embedding(num_embeddings=args.n_words,
                                       embedding_dim=args.n_embed)
        if args.feat == 'char':
            self.feat_embed = CHAR_LSTM(n_chars=args.n_feats,
                                        n_embed=args.n_char_embed,
                                        n_out=args.n_embed)
        elif args.feat == 'bert':
            self.feat_embed = BertEmbedding(model=args.bert_model,
                                            n_layers=args.n_bert_layers,
                                            n_out=args.n_embed)
        else:
            self.feat_embed = nn.Embedding(num_embeddings=args.n_feats,
                                           embedding_dim=args.n_embed)
        self.embed_dropout = IndependentDropout(p=args.embed_dropout)

        # the word-lstm layer
        self.lstm = BiLSTM(input_size=args.n_embed*2,
                           hidden_size=args.n_lstm_hidden,
                           num_layers=args.n_lstm_layers,
                           dropout=args.lstm_dropout)
        self.lstm_dropout = SharedDropout(p=args.lstm_dropout)

        # the MLP layers
        self.mlp_arc_h = MLP(n_in=args.n_lstm_hidden*2,
                             n_hidden=args.n_mlp_arc,
                             dropout=args.mlp_dropout)
        self.mlp_arc_d = MLP(n_in=args.n_lstm_hidden*2,
                             n_hidden=args.n_mlp_arc,
                             dropout=args.mlp_dropout)
        self.mlp_rel_h = MLP(n_in=args.n_lstm_hidden*2,
                             n_hidden=args.n_mlp_rel,
                             dropout=args.mlp_dropout)
        self.mlp_rel_d = MLP(n_in=args.n_lstm_hidden*2,
                             n_hidden=args.n_mlp_rel,
                             dropout=args.mlp_dropout)

        # the Biaffine layers
        self.arc_attn = Biaffine(n_in=args.n_mlp_arc,
                                 bias_x=True,
                                 bias_y=False)
        self.rel_attn = Biaffine(n_in=args.n_mlp_rel,
                                 n_out=args.n_rels,
                                 bias_x=True,
                                 bias_y=True)
        self.pad_index = args.pad_index
        self.unk_index = args.unk_index
Пример #3
0
    def __init__(self,
                 in_dim,
                 out_dim,
                 aggregator='softmax',
                 beta=1.0,
                 learn_beta=False,
                 p=1.0,
                 learn_p=False,
                 msg_norm=False,
                 learn_msg_scale=False,
                 mlp_layers=1,
                 eps=1e-7):
        super(GENConv, self).__init__()

        self.aggr = aggregator
        self.eps = eps

        channels = [in_dim]
        for _ in range(mlp_layers - 1):
            channels.append(in_dim * 2)
        channels.append(out_dim)

        self.mlp = MLP(channels)
        self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None

        self.beta = nn.Parameter(
            torch.Tensor([beta]), requires_grad=True
        ) if learn_beta and self.aggr == 'softmax' else beta
        self.p = nn.Parameter(torch.Tensor([p]),
                              requires_grad=True) if learn_p else p

        self.edge_encoder = BondEncoder(in_dim)
Пример #4
0
    def __init__(self, input_dim, hps, use_global=False):
        super(Generator, self).__init__()
        dim = hps.gen_dim
        style_dim = hps.gen_style_dim
        n_downsample = hps.gen_n_downsample
        n_res = hps.gen_n_res
        activ = hps.gen_activ
        pad_type = hps.gen_pad_type
        mlp_dim = hps.gen_mlp_dim

        # style encoder
        self.enc_style = StyleEncoder(4, input_dim, dim, style_dim, norm='none', activ=activ, pad_type=pad_type)

        # content encoder
        if use_global:
            self.enc_content = ContentEncoder(n_downsample, n_res, input_dim, dim, 'global_in', activ,
                                              pad_type=pad_type)
            self.dec = Decoder(n_downsample, n_res, self.enc_content.output_dim, input_dim, res_norm='global_adain',
                               activ=activ, pad_type=pad_type, use_global=use_global)
        else:
            self.enc_content = ContentEncoder(n_downsample, n_res, input_dim, dim, 'in', activ, pad_type=pad_type)
            self.dec = Decoder(n_downsample, n_res, self.enc_content.output_dim, input_dim, res_norm='adain',
                               activ=activ, pad_type=pad_type, use_global=use_global)

        # MLP to generate AdaIN parameters
        self.mlp = MLP(style_dim, self.get_num_adain_params(self.dec), mlp_dim, 3, norm='none', activ=activ)
Пример #5
0
    def __init__(self, word_vocab_size, word_emb_dim, pos_vocab_size,
                 pos_emb_dim, emb_dropout, lstm_hidden, lstm_depth,
                 lstm_dropout, arc_hidden, arc_depth, arc_dropout,
                 arc_activation, lab_hidden, lab_depth, lab_dropout,
                 lab_activation, n_labels):
        super(BiAffineParser, self).__init__()

        # Embeddings
        self.word_embedding = TimeDistributed(
            nn.Embedding(word_vocab_size, word_emb_dim, padding_idx=0))
        self.pos_embedding = TimeDistributed(
            nn.Embedding(pos_vocab_size, pos_emb_dim, padding_idx=0))
        self.emb_dropout = nn.Dropout(p=emb_dropout)

        # LSTM
        lstm_input = word_emb_dim + pos_emb_dim
        self.lstm = nn.LSTM(input_size=lstm_input,
                            hidden_size=lstm_hidden,
                            batch_first=True,
                            dropout=lstm_dropout,
                            bidirectional=True)

        # MLPs
        self.arc_mlp_h = TimeDistributed(
            MLP(lstm_hidden * 2, arc_hidden, arc_depth, arc_activation,
                arc_dropout))
        self.arc_mlp_d = TimeDistributed(
            MLP(lstm_hidden * 2, arc_hidden, arc_depth, arc_activation,
                arc_dropout))

        self.lab_mlp_h = TimeDistributed(
            MLP(lstm_hidden * 2, lab_hidden, lab_depth, lab_activation,
                lab_dropout))
        self.lab_mlp_d = TimeDistributed(
            MLP(lstm_hidden * 2, lab_hidden, lab_depth, lab_activation,
                lab_dropout))

        # BiAffine layers
        self.arc_attn = BiAffineAttn(arc_hidden,
                                     1,
                                     bias_head=False,
                                     bias_dep=True)
        self.lab_attn = BiAffineAttn(lab_hidden,
                                     n_labels,
                                     bias_head=True,
                                     bias_dep=True)
Пример #6
0
    def __init__(self, num_atoms, v_dim, e_dim, num_edge_types):
        super().__init__()

        self.num_atoms = num_atoms
        self.num_edge_types = num_edge_types
        self.mlp_v = MLP(n_in=e_dim, n_hid=64, n_out=v_dim)
        self.mlp_e = nn.ModuleList([
            MLP(n_in=v_dim * 2, n_hid=64, n_out=e_dim)
            for _ in range(num_edge_types)
        ])
        off_diag = np.ones([num_atoms, num_atoms]) - np.eye(num_atoms)

        rel_rec = np.array(encode_onehot(np.where(off_diag)[1]),
                           dtype=np.float32)
        rel_send = np.array(encode_onehot(np.where(off_diag)[0]),
                            dtype=np.float32)
        self.rel_rec = torch.FloatTensor(rel_rec).cuda()
        self.rel_send = torch.FloatTensor(rel_send).cuda()
Пример #7
0
    def __init__(self, num_atoms, v_dim, e_dim):
        super().__init__()

        self.num_atoms = num_atoms
        self.mlp_v = MLP(n_in=e_dim, n_hid=32,
                         n_out=v_dim)  #nn.Sequential(nn.Linear(e_dim, 16),
        #              nn.Tanh(),
        #              nn.Linear(16, v_dim))
        self.mlp_e = MLP(n_in=v_dim * 2, n_hid=32,
                         n_out=e_dim)  #nn.Sequential(nn.Linear(v_dim*2, 16),
        #              nn.Tanh(),
        #              nn.Linear(16, e_dim))
        off_diag = np.ones([num_atoms, num_atoms]) - np.eye(num_atoms)

        rel_rec = np.array(encode_onehot(np.where(off_diag)[1]),
                           dtype=np.float32)
        rel_send = np.array(encode_onehot(np.where(off_diag)[0]),
                            dtype=np.float32)
        self.rel_rec = torch.FloatTensor(rel_rec).cuda()
        self.rel_send = torch.FloatTensor(rel_send).cuda()
    def __init__(self,
                 sr=SAMPLE_RATE,
                 rnn_channels=512,
                 out_size_in_seconds=SAMPLE_LENGTH_IN_SECONDS,
                 n_oscillators=101,
                 filter_size=256,
                 z_size=16):
        """
        Construct a DDSPDecoder

        Args:
            sr (int, optional): Sample rate. Defaults to SAMPLE_RATE.
            rnn_channels (int, optional): Number of RNN hidden states. Defaults
                to 512.
            out_size_in_seconds (float, optional): Length of the output in
                seconds. Defaults to SAMPLE_LENGTH_IN_SECONDS.
            n_oscillators (int, optional): Number of oscillators in the
                harmonic oscillator bank. Defaults to 101.
            filter_size (int, optional): Length of the FIR filter impulse
                response. Defaults to 256.
            z_size (int, optional): Size of the latent dimension. Defaults to
                16.
        """
        super().__init__()
        self.out_size = int(out_size_in_seconds * sr)
        self.f0_mlp = MLP(in_size=1)
        self.loudness_mlp = MLP(in_size=1)
        self.z_mlp = MLP(in_size=z_size)

        self.gru = nn.GRU(input_size=rnn_channels * 3,
                          hidden_size=rnn_channels)

        self.final_mlp = MLP(in_size=rnn_channels * 3)

        self.H_out = nn.Linear(rnn_channels, filter_size // 2 + 1)
        self.amp_out = nn.Linear(rnn_channels, n_oscillators)

        self.output_activation_H = nn.Sigmoid()  # ScaledSigmoid()
        self.output_activation_amp = nn.Sigmoid()  # ScaledSigmoid()
Пример #9
0
 def __init__(self, useritem_embeds, model, hidden_size, bidirectional,
              input_size, layer_num):
     nn.Module.__init__(self)
     self.useritem_embeds = useritem_embeds
     self.model = model
     self.lstm = nn.LSTM(input_size=input_size,
                         hidden_size=hidden_size,
                         batch_first=False,
                         bidirectional=bidirectional)
     self.support_proj = MLP(input_size=2 * input_size,
                             hidden_layers=[(input_size, True, 0.2)],
                             normalization='layer_norm',
                             activation='relu')
     self.bidirectional = bidirectional
     self.layer_num = layer_num
     if bidirectional:
         hidden_size *= 2
     self.user_attn = nn.Linear(hidden_size, layer_num * input_size * 2)
     self.item_attn = nn.Linear(hidden_size, layer_num * input_size * 2)
Пример #10
0
    def __init__(self,
                 dataset,
                 in_dim,
                 out_dim,
                 aggregator='softmax',
                 beta=1.0,
                 learn_beta=False,
                 p=1.0,
                 learn_p=False,
                 msg_norm=False,
                 learn_msg_scale=False,
                 norm='batch',
                 mlp_layers=1,
                 eps=1e-7):
        super(GENConv, self).__init__()

        self.aggr = aggregator
        self.eps = eps

        channels = [in_dim]
        for i in range(mlp_layers - 1):
            channels.append(in_dim * 2)
        channels.append(out_dim)

        self.mlp = MLP(channels, norm=norm)
        self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None

        self.beta = nn.Parameter(
            torch.Tensor([beta]), requires_grad=True
        ) if learn_beta and self.aggr == 'softmax' else beta
        self.p = nn.Parameter(torch.Tensor([p]),
                              requires_grad=True) if learn_p else p

        if dataset == 'ogbg-molhiv':
            self.edge_encoder = BondEncoder(in_dim)
        elif dataset == 'ogbg-ppa':
            self.edge_encoder = nn.Linear(in_dim, in_dim)
        else:
            raise ValueError(f'Dataset {dataset} is not supported.')
Пример #11
0
    def __init__(self,
                 num_steps,
                 x_size,
                 window_size,
                 z_what_size,
                 rnn_hidden_size,
                 encoder_net=[],
                 decoder_net=[],
                 predict_net=[],
                 embed_net=None,
                 bl_predict_net=[],
                 non_linearity='ReLU',
                 decoder_output_bias=None,
                 decoder_output_use_sigmoid=False,
                 use_masking=True,
                 use_baselines=True,
                 baseline_scalar=None,
                 scale_prior_mean=3.0,
                 scale_prior_sd=0.1,
                 pos_prior_mean=0.0,
                 pos_prior_sd=1.0,
                 likelihood_sd=0.3,
                 use_cuda=False):

        super(AIR, self).__init__()

        self.num_steps = num_steps
        self.x_size = x_size
        self.window_size = window_size
        self.z_what_size = z_what_size
        self.rnn_hidden_size = rnn_hidden_size
        self.use_masking = use_masking
        self.use_baselines = use_baselines
        self.baseline_scalar = baseline_scalar
        self.likelihood_sd = likelihood_sd
        self.use_cuda = use_cuda
        prototype = torch.tensor(0.).cuda() if use_cuda else torch.tensor(0.)
        self.options = dict(dtype=prototype.dtype, device=prototype.device)

        self.z_pres_size = 1
        self.z_where_size = 3
        # By making these parameters they will be moved to the gpu
        # when necessary. (They are not registered with pyro for
        # optimization.)
        self.z_where_loc_prior = nn.Parameter(torch.FloatTensor(
            [scale_prior_mean, pos_prior_mean, pos_prior_mean]),
                                              requires_grad=False)
        self.z_where_scale_prior = nn.Parameter(torch.FloatTensor(
            [scale_prior_sd, pos_prior_sd, pos_prior_sd]),
                                                requires_grad=False)

        # Create nn modules.
        rnn_input_size = x_size**2 if embed_net is None else embed_net[-1]
        rnn_input_size += self.z_where_size + z_what_size + self.z_pres_size
        nl = getattr(nn, non_linearity)

        self.rnn = nn.LSTMCell(rnn_input_size, rnn_hidden_size)
        self.encode = Encoder(window_size**2, encoder_net, z_what_size, nl)
        self.decode = Decoder(window_size**2, decoder_net, z_what_size,
                              decoder_output_bias, decoder_output_use_sigmoid,
                              nl)
        self.predict = Predict(rnn_hidden_size, predict_net, self.z_pres_size,
                               self.z_where_size, nl)
        self.embed = Identity() if embed_net is None else MLP(
            x_size**2, embed_net, nl, True)

        self.bl_rnn = nn.LSTMCell(rnn_input_size, rnn_hidden_size)
        self.bl_predict = MLP(rnn_hidden_size, bl_predict_net + [1], nl)
        self.bl_embed = Identity() if embed_net is None else MLP(
            x_size**2, embed_net, nl, True)

        # Create parameters.
        self.h_init = nn.Parameter(torch.zeros(1, rnn_hidden_size))
        self.c_init = nn.Parameter(torch.zeros(1, rnn_hidden_size))
        self.bl_h_init = nn.Parameter(torch.zeros(1, rnn_hidden_size))
        self.bl_c_init = nn.Parameter(torch.zeros(1, rnn_hidden_size))
        self.z_where_init = nn.Parameter(torch.zeros(1, self.z_where_size))
        self.z_what_init = nn.Parameter(torch.zeros(1, self.z_what_size))

        if use_cuda:
            self.cuda()
Пример #12
0
    def __init__(self,
                 num_steps,
                 x_size,
                 window_size,
                 z_what_size,
                 rnn_hidden_size,
                 encoder_net=[],
                 decoder_net=[],
                 predict_net=[],
                 embed_net=None,
                 bl_predict_net=[],
                 non_linearity='ReLU',
                 decoder_output_bias=None,
                 decoder_output_use_sigmoid=False,
                 use_masking=True,
                 use_baselines=True,
                 baseline_scalar=None,
                 fudge_z_pres=False,
                 use_cuda=False):

        super(AIR, self).__init__()

        self.num_steps = num_steps
        self.x_size = x_size
        self.window_size = window_size
        self.z_what_size = z_what_size
        self.rnn_hidden_size = rnn_hidden_size
        self.use_masking = use_masking and not fudge_z_pres
        self.use_baselines = use_baselines and not fudge_z_pres
        self.baseline_scalar = baseline_scalar
        self.fudge_z_pres = fudge_z_pres
        self.use_cuda = use_cuda

        self.z_pres_size = 1
        self.z_where_size = 3
        # By making these parameters they will be moved to the gpu
        # when necessary. (They are not registered with pyro for
        # optimization.)
        self.z_where_mu_prior = nn.Parameter(torch.FloatTensor([3.0, 0, 0]),
                                             requires_grad=False)
        self.z_where_sigma_prior = nn.Parameter(torch.FloatTensor([0.1, 1, 1]),
                                                requires_grad=False)

        # Create nn modules.
        rnn_input_size = x_size**2 if embed_net is None else embed_net[-1]
        rnn_input_size += self.z_where_size + z_what_size + self.z_pres_size
        nl = getattr(nn, non_linearity)

        self.rnn = nn.LSTMCell(rnn_input_size, rnn_hidden_size)
        self.encode = Encoder(window_size**2, encoder_net, z_what_size, nl)
        self.decode = Decoder(window_size**2, decoder_net, z_what_size,
                              decoder_output_bias, decoder_output_use_sigmoid,
                              nl)
        self.predict = Predict(rnn_hidden_size, predict_net, self.z_pres_size,
                               self.z_where_size, nl)
        self.embed = Identity() if embed_net is None else MLP(
            x_size**2, embed_net, nl, True)

        self.bl_rnn = nn.LSTMCell(rnn_input_size, rnn_hidden_size)
        self.bl_predict = MLP(rnn_hidden_size, bl_predict_net + [1], nl)
        self.bl_embed = Identity() if embed_net is None else MLP(
            x_size**2, embed_net, nl, True)

        # Create parameters.
        self.h_init = zeros(1, rnn_hidden_size)
        self.c_init = zeros(1, rnn_hidden_size)
        self.bl_h_init = zeros(1, rnn_hidden_size)
        self.bl_c_init = zeros(1, rnn_hidden_size)
        self.z_where_init = zeros(1, self.z_where_size)
        self.z_what_init = zeros(1, self.z_what_size)

        if use_cuda:
            self.cuda()
    def __init__(
        self,
        token_embeddings: TokenEmbeddings,
        relations_dictionary: Dictionary,
        lstm_hidden_size: int = 400,
        n_mlp_arc: int = 500,
        n_mlp_rel: int = 100,
        n_lstm_layers: int = 3,
        mlp_dropout: float = .33,
        lstm_dropout: float = 0.2,
        relearn_embeddings: bool = True,
        beta: float = 1.0,
        pickle_module: str = "pickle",
    ):

        super(BiAffineParser, self).__init__()
        self.token_embeddings = token_embeddings
        self.beta = beta
        self.relations_dictionary: Dictionary = relations_dictionary
        self.relearn_embeddings = relearn_embeddings
        self.lstm_hidden_size = lstm_hidden_size
        self.n_mlp_arc = n_mlp_arc
        self.n_mlp_rel = n_mlp_rel
        self.n_lstm_layers = n_lstm_layers
        self.lstm_dropout = lstm_dropout
        self.mlp_dropout = mlp_dropout
        lstm_input_dim: int = self.token_embeddings.embedding_length

        if self.relations_dictionary:
            self.embedding2nn = torch.nn.Linear(lstm_input_dim, lstm_input_dim)

        self.lstm = BiLSTM(input_size=self.lstm_input_dim,
                           hidden_size=self.lstm_hidden_size,
                           num_layers=self.n_lstm_layers,
                           dropout=self.lstm_dropout)

        self.mlp_arc_h = MLP(n_in=self.lstm_hidden_size * 2,
                             n_hidden=self.n_mlp_arc,
                             dropout=self.mlp_dropout)
        self.mlp_arc_d = MLP(n_in=self.lstm_hidden_size * 2,
                             n_hidden=self.n_mlp_arc,
                             dropout=self.mlp_dropout)
        self.mlp_rel_h = MLP(n_in=self.lstm_hidden_size * 2,
                             n_hidden=self.n_mlp_rel,
                             dropout=self.mlp_dropout)
        self.mlp_rel_d = MLP(n_in=self.lstm_hidden_size * 2,
                             n_hidden=self.n_mlp_rel,
                             dropout=self.mlp_dropout)

        # the Biaffine layers
        self.arc_attn = Biaffine(n_in=self.n_mlp_arc,
                                 bias_x=True,
                                 bias_y=False)

        self.rel_attn = Biaffine(n_in=self.n_mlp_rel,
                                 n_out=len(relations_dictionary),
                                 bias_x=True,
                                 bias_y=True)

        self.loss_function = torch.nn.CrossEntropyLoss()
        self.to(flair.device)