def get_transformer(ff_dim: int, n_layers: int, n_heads: int,
dropout_prob: float):
"""
Creates a new transformer and tokenizer using the given parameters
:param ff_dim:
:param n_layers:
:param n_heads:
:param dropout_prob:
:return:
"""
# Load english model with 25k word-pieces
tokenizer = BPEmb(lang='en', dim=300, vs=25000)
# Extract the embeddings and add a randomly initialized embedding for our extra [PAD] token
pretrained_embeddings = np.concatenate(
[tokenizer.emb.vectors,
np.zeros(shape=(1, 300))], axis=0)
# Extract the vocab and add an extra [PAD] token
vocabulary = tokenizer.emb.index2word + ['[PAD]']
tokenizer.pad_token_id = len(vocabulary) - 1
model = TransformerClassifier(torch.tensor(pretrained_embeddings).type(
torch.FloatTensor),
ff_dim=ff_dim,
d_model=300,
n_heads=n_heads,
n_layers=n_layers,
dropout_prob=dropout_prob).to(device)
return model, tokenizer
def get_cnn(in_channels, out_channels, kernel_heights, stride, padding, dropout_prob):
"""
Creates a new CNN and tokenizer using the given parameters
:return:
"""
# Load english model with 25k word-pieces
tokenizer = BPEmb(lang='en', dim=300, vs=25000)
# Extract the embeddings and add a randomly initialized embedding for our extra [PAD] token
pretrained_embeddings = np.concatenate([tokenizer.emb.vectors, np.zeros(shape=(1, 300))], axis=0)
# Extract the vocab and add an extra [PAD] token
vocabulary = tokenizer.emb.index2word + ['[PAD]']
tokenizer.pad_token_id = len(vocabulary) - 1
model = CNN(
torch.tensor(pretrained_embeddings).type(torch.FloatTensor),
n_labels=2,
in_channels=in_channels,
out_channels=out_channels,
kernel_heights=kernel_heights,
stride=stride,
padding=padding,
dropout=dropout_prob
).to(device)
return model, tokenizer
