def get_transformer(ff_dim: int, n_layers: int, n_heads: int, dropout_prob: float): """ Creates a new transformer and tokenizer using the given parameters :param ff_dim: :param n_layers: :param n_heads: :param dropout_prob: :return: """ # Load english model with 25k word-pieces tokenizer = BPEmb(lang='en', dim=300, vs=25000) # Extract the embeddings and add a randomly initialized embedding for our extra [PAD] token pretrained_embeddings = np.concatenate( [tokenizer.emb.vectors, np.zeros(shape=(1, 300))], axis=0) # Extract the vocab and add an extra [PAD] token vocabulary = tokenizer.emb.index2word + ['[PAD]'] tokenizer.pad_token_id = len(vocabulary) - 1 model = TransformerClassifier(torch.tensor(pretrained_embeddings).type( torch.FloatTensor), ff_dim=ff_dim, d_model=300, n_heads=n_heads, n_layers=n_layers, dropout_prob=dropout_prob).to(device) return model, tokenizer
def get_cnn(in_channels, out_channels, kernel_heights, stride, padding, dropout_prob): """ Creates a new CNN and tokenizer using the given parameters :return: """ # Load english model with 25k word-pieces tokenizer = BPEmb(lang='en', dim=300, vs=25000) # Extract the embeddings and add a randomly initialized embedding for our extra [PAD] token pretrained_embeddings = np.concatenate([tokenizer.emb.vectors, np.zeros(shape=(1, 300))], axis=0) # Extract the vocab and add an extra [PAD] token vocabulary = tokenizer.emb.index2word + ['[PAD]'] tokenizer.pad_token_id = len(vocabulary) - 1 model = CNN( torch.tensor(pretrained_embeddings).type(torch.FloatTensor), n_labels=2, in_channels=in_channels, out_channels=out_channels, kernel_heights=kernel_heights, stride=stride, padding=padding, dropout=dropout_prob ).to(device) return model, tokenizer