def main(width=100,
depth=4,
vector_length=64,
min_batch_size=1,
max_batch_size=32,
learn_rate=0.001,
momentum=0.9,
dropout=0.5,
dropout_decay=1e-4,
nb_epoch=20,
L2=1e-6):
cfg = dict(locals())
print(cfg)
if cupy is not None:
print("Using GPU")
Model.ops = CupyOps()
train_data, check_data, nr_tag = ancora_pos_tags()
extracter = FeatureExtracter('es', attrs=[LOWER, SHAPE, PREFIX, SUFFIX])
Model.lsuv = True
with Model.define_operators({
'**': clone,
'>>': chain,
'+': add,
'|': concatenate
}):
lower_case = HashEmbed(width, 100, column=0)
shape = HashEmbed(width // 2, 200, column=1)
prefix = HashEmbed(width // 2, 100, column=2)
suffix = HashEmbed(width // 2, 100, column=3)
model = (with_flatten(
(lower_case | shape | prefix | suffix) >> Maxout(width, pieces=3)
>> Residual(ExtractWindow(nW=1) >> Maxout(width, pieces=3))**depth
>> Softmax(nr_tag),
pad=depth))
train_X, train_y = preprocess(model.ops, extracter, train_data, nr_tag)
dev_X, dev_y = preprocess(model.ops, extracter, check_data, nr_tag)
n_train = float(sum(len(x) for x in train_X))
global epoch_train_acc
with model.begin_training(train_X[:5000], train_y[:5000],
**cfg) as (trainer, optimizer):
trainer.each_epoch.append(track_progress(**locals()))
trainer.batch_size = min_batch_size
batch_size = float(min_batch_size)
for X, y in trainer.iterate(train_X, train_y):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
gradient = [yh[i] - y[i] for i in range(len(yh))]
backprop(gradient, optimizer)
trainer.batch_size = min(int(batch_size), max_batch_size)
batch_size *= 1.001
with model.use_params(trainer.optimizer.averages):
print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
with open('/tmp/model.pickle', 'wb') as file_:
pickle.dump(model, file_)
def build_model(nr_class, width, depth, conv_depth, **kwargs):
with Model.define_operators({"|": concatenate, ">>": chain, "**": clone}):
embed = (HashEmbed(width, 5000, column=1)
| StaticVectors("spacy_pretrained_vectors", width, column=5)
| HashEmbed(width // 2, 750, column=2)
| HashEmbed(width // 2, 750, column=3)
| HashEmbed(width // 2, 750, column=4)) >> LN(Maxout(width))
sent2vec = (flatten_add_lengths >> with_getitem(
0,
embed >> Residual(ExtractWindow(nW=1) >> LN(Maxout(width)))**
conv_depth,
) >> ParametricAttention(width) >> Pooling(sum_pool) >> Residual(
LN(Maxout(width)))**depth)
model = (
foreach(sent2vec, drop_factor=2.0) >> flatten_add_lengths
# This block would allow the model to learn some cross-sentence
# features. It's not useful on this problem. It might make more
# sense to use a BiLSTM here, following Liang et al (2016).
# >> with_getitem(0,
# Residual(ExtractWindow(nW=1) >> LN(Maxout(width))) ** conv_depth
# )
>> ParametricAttention(width, hard=False) >> Pooling(sum_pool) >>
Residual(LN(Maxout(width)))**depth >> Softmax(nr_class))
model.lsuv = False
return model
def Tok2Vec(width, embed_size, **kwargs):
pretrained_vectors = kwargs.get("pretrained_vectors", None)
cnn_maxout_pieces = kwargs.get("cnn_maxout_pieces", 3)
subword_features = kwargs.get("subword_features", True)
conv_depth = kwargs.get("conv_depth", 4)
bilstm_depth = kwargs.get("bilstm_depth", 0)
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
with Model.define_operators(
{">>": chain, "|": concatenate, "**": clone, "+": add, "*": reapply}
):
norm = HashEmbed(width, embed_size, column=cols.index(NORM), name="embed_norm")
if subword_features:
prefix = HashEmbed(
width, embed_size // 2, column=cols.index(PREFIX), name="embed_prefix"
)
suffix = HashEmbed(
width, embed_size // 2, column=cols.index(SUFFIX), name="embed_suffix"
)
shape = HashEmbed(
width, embed_size // 2, column=cols.index(SHAPE), name="embed_shape"
)
else:
prefix, suffix, shape = (None, None, None)
if pretrained_vectors is not None:
glove = StaticVectors(pretrained_vectors, width, column=cols.index(ID))
if subword_features:
embed = uniqued(
(glove | norm | prefix | suffix | shape)
>> LN(Maxout(width, width * 5, pieces=3)),
column=cols.index(ORTH),
)
else:
embed = uniqued(
(glove | norm) >> LN(Maxout(width, width * 2, pieces=3)),
column=cols.index(ORTH),
)
elif subword_features:
embed = uniqued(
(norm | prefix | suffix | shape)
>> LN(Maxout(width, width * 4, pieces=3)),
column=cols.index(ORTH),
)
else:
embed = norm
convolution = Residual(
ExtractWindow(nW=1)
>> LN(Maxout(width, width * 3, pieces=cnn_maxout_pieces))
)
tok2vec = FeatureExtracter(cols) >> with_flatten(
embed >> convolution ** conv_depth, pad=conv_depth
)
if bilstm_depth >= 1:
tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth)
# Work around thinc API limitations :(. TODO: Revise in Thinc 7
tok2vec.nO = width
tok2vec.embed = embed
return tok2vec
def MishWindowEncoder(config):
from thinc.v2v import Mish
nO = config["width"]
nW = config["window_size"]
depth = config["depth"]
cnn = chain(ExtractWindow(nW=nW), LayerNorm(Mish(nO, nO * ((nW * 2) + 1))))
model = clone(Residual(cnn), depth)
model.nO = nO
return model
def build_text_classifier(nr_class, width=64, **cfg):
nr_vector = cfg.get('nr_vector', 5000)
pretrained_dims = cfg.get('pretrained_dims', 0)
with Model.define_operators({
'>>': chain,
'+': add,
'|': concatenate,
'**': clone
}):
if cfg.get('low_data') and pretrained_dims:
model = (SpacyVectors >> flatten_add_lengths >> with_getitem(
0, Affine(width, pretrained_dims)) >>
ParametricAttention(width) >> Pooling(sum_pool) >>
Residual(ReLu(width, width))**2 >> zero_init(
Affine(nr_class, width, drop_factor=0.0)) >> logistic)
return model
lower = HashEmbed(width, nr_vector, column=1)
prefix = HashEmbed(width // 2, nr_vector, column=2)
suffix = HashEmbed(width // 2, nr_vector, column=3)
shape = HashEmbed(width // 2, nr_vector, column=4)
trained_vectors = (FeatureExtracter(
[ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID]) >> with_flatten(
uniqued((lower | prefix | suffix | shape) >> LN(
Maxout(width, width + (width // 2) * 3)),
column=0)))
if pretrained_dims:
static_vectors = (
SpacyVectors >> with_flatten(Affine(width, pretrained_dims)))
# TODO Make concatenate support lists
vectors = concatenate_lists(trained_vectors, static_vectors)
vectors_width = width * 2
else:
vectors = trained_vectors
vectors_width = width
static_vectors = None
cnn_model = (
vectors >> with_flatten(
LN(Maxout(width, vectors_width)) >> Residual(
(ExtractWindow(nW=1) >> LN(Maxout(width, width * 3))))**2,
pad=2) >> flatten_add_lengths >> ParametricAttention(width) >>
Pooling(sum_pool) >> Residual(zero_init(Maxout(width, width))) >>
zero_init(Affine(nr_class, width, drop_factor=0.0)))
linear_model = (
_preprocess_doc >> LinearModel(nr_class, drop_factor=0.))
model = ((linear_model | cnn_model) >> zero_init(
Affine(nr_class, nr_class * 2, drop_factor=0.0)) >> logistic)
model.nO = nr_class
model.lsuv = False
return model
def MaxoutWindowEncoder(config):
nO = config["width"]
nW = config["window_size"]
nP = config["pieces"]
depth = config["depth"]
cnn = chain(ExtractWindow(nW=nW),
LayerNorm(Maxout(nO, nO * ((nW * 2) + 1), pieces=nP)))
model = clone(Residual(cnn), depth)
model.nO = nO
model.receptive_field = nW * depth
return model
def Tok2Vec(width, embed_size, **kwargs):
pretrained_vectors = kwargs.get('pretrained_vectors', None)
cnn_maxout_pieces = kwargs.get('cnn_maxout_pieces', 2)
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
with Model.define_operators({
'>>': chain,
'|': concatenate,
'**': clone,
'+': add,
'*': reapply
}):
norm = HashEmbed(width,
embed_size,
column=cols.index(NORM),
name='embed_norm')
prefix = HashEmbed(width,
embed_size // 2,
column=cols.index(PREFIX),
name='embed_prefix')
suffix = HashEmbed(width,
embed_size // 2,
column=cols.index(SUFFIX),
name='embed_suffix')
shape = HashEmbed(width,
embed_size // 2,
column=cols.index(SHAPE),
name='embed_shape')
if pretrained_vectors is not None:
glove = StaticVectors(pretrained_vectors,
width,
column=cols.index(ID))
embed = uniqued((glove | norm | prefix | suffix | shape) >> LN(
Maxout(width, width * 5, pieces=3)),
column=cols.index(ORTH))
else:
embed = uniqued((norm | prefix | suffix | shape) >> LN(
Maxout(width, width * 4, pieces=3)),
column=cols.index(ORTH))
convolution = Residual(
ExtractWindow(
nW=1) >> LN(Maxout(width, width *
3, pieces=cnn_maxout_pieces)))
tok2vec = (FeatureExtracter(cols) >> with_flatten(
embed >> convolution**4, pad=4))
# Work around thinc API limitations :(. TODO: Revise in Thinc 7
tok2vec.nO = width
tok2vec.embed = embed
return tok2vec
def my_tok_to_vec(width, embed_size, pretrained_vectors, **kwargs):
# Circular imports :(
from spacy._ml import PyTorchBiLSTM
cnn_maxout_pieces = kwargs.get("cnn_maxout_pieces", 3)
conv_depth = kwargs.get("conv_depth", 4)
bilstm_depth = kwargs.get("bilstm_depth", 0)
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
storage = []
with Model.define_operators({">>": chain, "|": concatenate, "**": clone}):
# norm = HashEmbed(width, embed_size, column=cols.index(NORM), name="embed_norm")
# prefix = HashEmbed(
# width, embed_size // 2, column=cols.index(PREFIX), name="embed_prefix"
# )
# suffix = HashEmbed(
# width, embed_size // 2, column=cols.index(SUFFIX), name="embed_suffix"
# )
shape = HashEmbed(
width, embed_size // 2, column=cols.index(SHAPE), name="embed_shape"
)
glove = Vectors(storage, pretrained_vectors, width, column=cols.index(NORM), )
vec_width = glove.nV
embed = uniqued(
(glove | shape)
>> LN(Maxout(width, width + vec_width, pieces=3)),
column=cols.index(ORTH),
)
convolution = Residual(
ExtractWindow(nW=1)
>> LN(Maxout(width, width * 3, pieces=cnn_maxout_pieces))
)
tok2vec = SaveDoc(storage) >> FeatureExtracter(cols) >> with_flatten(
embed >> convolution ** conv_depth, pad=conv_depth
)
if bilstm_depth >= 1:
tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth)
# Work around thinc API limitations :(. TODO: Revise in Thinc 7
tok2vec.nO = width
tok2vec.embed = embed
return tok2vec
def build_model(nr_class, width, depth, conv_depth, **kwargs):
with Model.define_operators({'|': concatenate, '>>': chain, '**': clone}):
embed = ((HashEmbed(width, 5000, column=1)
| StaticVectors('spacy_pretrained_vectors', width, column=5)
| HashEmbed(width // 2, 750, column=2)
| HashEmbed(width // 2, 750, column=3)
| HashEmbed(width // 2, 750, column=4)) >> LN(Maxout(width)))
sent2vec = (flatten_add_lengths >> with_getitem(
0, embed >> Residual(ExtractWindow(nW=1) >> LN(Maxout(width)))**
conv_depth) >> ParametricAttention(width) >> Pooling(sum_pool) >>
Residual(LN(Maxout(width)))**depth)
model = (foreach(sent2vec, drop_factor=2.0) >> flatten_add_lengths >>
ParametricAttention(width, hard=False) >> Pooling(sum_pool) >>
Residual(LN(Maxout(width)))**depth >> Softmax(nr_class))
model.lsuv = False
return model
def build_text_classifier(nr_class, width=64, **cfg):
depth = cfg.get("depth", 2)
nr_vector = cfg.get("nr_vector", 5000)
pretrained_dims = cfg.get("pretrained_dims", 0)
with Model.define_operators({
">>": chain,
"+": add,
"|": concatenate,
"**": clone
}):
if cfg.get("low_data") and pretrained_dims:
model = (SpacyVectors >> flatten_add_lengths >> with_getitem(
0, Affine(width, pretrained_dims)) >>
ParametricAttention(width) >> Pooling(sum_pool) >>
Residual(ReLu(width, width))**2 >> zero_init(
Affine(nr_class, width, drop_factor=0.0)) >> logistic)
return model
lower = HashEmbed(width, nr_vector, column=1)
prefix = HashEmbed(width // 2, nr_vector, column=2)
suffix = HashEmbed(width // 2, nr_vector, column=3)
shape = HashEmbed(width // 2, nr_vector, column=4)
trained_vectors = FeatureExtracter(
[ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID]) >> with_flatten(
uniqued(
(lower | prefix | suffix | shape) >> LN(
Maxout(width, width + (width // 2) * 3)),
column=0,
))
if pretrained_dims:
static_vectors = SpacyVectors >> with_flatten(
Affine(width, pretrained_dims))
# TODO Make concatenate support lists
vectors = concatenate_lists(trained_vectors, static_vectors)
vectors_width = width * 2
else:
vectors = trained_vectors
vectors_width = width
static_vectors = None
tok2vec = vectors >> with_flatten(
LN(Maxout(width, vectors_width)) >> Residual(
(ExtractWindow(nW=1) >> LN(Maxout(width, width * 3))))**depth,
pad=depth,
)
cnn_model = (
tok2vec >> flatten_add_lengths >> ParametricAttention(width) >>
Pooling(sum_pool) >> Residual(zero_init(Maxout(width, width))) >>
zero_init(Affine(nr_class, width, drop_factor=0.0)))
linear_model = build_bow_text_classifier(nr_class,
ngram_size=cfg.get(
"ngram_size", 1),
exclusive_classes=False)
if cfg.get("exclusive_classes"):
output_layer = Softmax(nr_class, nr_class * 2)
else:
output_layer = (zero_init(
Affine(nr_class, nr_class * 2, drop_factor=0.0)) >> logistic)
model = (linear_model | cnn_model) >> output_layer
model.tok2vec = chain(tok2vec, flatten)
model.nO = nr_class
model.lsuv = False
return model
def main(dataset='quora',
width=50,
depth=2,
min_batch_size=1,
max_batch_size=512,
dropout=0.2,
dropout_decay=0.0,
pooling="mean+max",
nb_epoch=5,
pieces=3,
L2=0.0,
use_gpu=False,
out_loc=None,
quiet=False,
job_id=None,
ws_api_url=None,
rest_api_url=None):
global CTX
if job_id is not None:
CTX = neptune.Context()
width = CTX.params.width
L2 = CTX.params.L2
nb_epoch = CTX.params.nb_epoch
depth = CTX.params.depth
max_batch_size = CTX.params.max_batch_size
cfg = dict(locals())
if out_loc:
out_loc = Path(out_loc)
if not out_loc.parent.exists():
raise IOError("Can't open output location: %s" % out_loc)
print(cfg)
if pooling == 'mean+max':
pool_layer = Pooling(mean_pool, max_pool)
elif pooling == "mean":
pool_layer = mean_pool
elif pooling == "max":
pool_layer = max_pool
else:
raise ValueError("Unrecognised pooling", pooling)
print("Load spaCy")
nlp = get_spacy('en')
if use_gpu:
Model.ops = CupyOps()
print("Construct model")
# Bind operators for the scope of the block:
# * chain (>>): Compose models in a 'feed forward' style,
# i.e. chain(f, g)(x) -> g(f(x))
# * clone (**): Create n copies of a model, and chain them, i.e.
# (f ** 3)(x) -> f''(f'(f(x))), where f, f' and f'' have distinct weights.
# * concatenate (|): Merge the outputs of two models into a single vector,
# i.e. (f|g)(x) -> hstack(f(x), g(x))
Model.lsuv = True
#Model.ops = CupyOps()
with Model.define_operators({
'>>': chain,
'**': clone,
'|': concatenate,
'+': add
}):
mwe_encode = ExtractWindow(nW=1) >> BN(
Maxout(width, drop_factor=0.0, pieces=pieces))
sent2vec = ( # List[spacy.token.Doc]{B}
flatten_add_lengths # : (ids{T}, lengths{B})
>> with_getitem(
0,
#(StaticVectors('en', width)
HashEmbed(width, 3000)
#+ HashEmbed(width, 3000))
#>> Residual(mwe_encode ** 2)
) # : word_ids{T}
>> Pooling(mean_pool, max_pool)
#>> Residual(BN(Maxout(width*2, pieces=pieces), nO=width*2)**2)
>> Maxout(width * 2, pieces=pieces, drop_factor=0.0) >> logistic)
model = Siamese(sent2vec, CauchySimilarity(width * 2))
print("Read and parse data: %s" % dataset)
if dataset == 'quora':
train, dev = datasets.quora_questions()
elif dataset == 'snli':
train, dev = datasets.snli()
elif dataset == 'stackxc':
train, dev = datasets.stack_exchange()
elif dataset in ('quora+snli', 'snli+quora'):
train, dev = datasets.quora_questions()
train2, dev2 = datasets.snli()
train.extend(train2)
dev.extend(dev2)
else:
raise ValueError("Unknown dataset: %s" % dataset)
get_ids = get_word_ids(Model.ops)
train_X, train_y = preprocess(model.ops, nlp, train, get_ids)
dev_X, dev_y = preprocess(model.ops, nlp, dev, get_ids)
with model.begin_training(train_X[:10000], train_y[:10000],
**cfg) as (trainer, optimizer):
# Pass a callback to print progress. Give it all the local scope,
# because why not?
trainer.each_epoch.append(track_progress(**locals()))
trainer.batch_size = min_batch_size
batch_size = float(min_batch_size)
print("Accuracy before training", model.evaluate_logloss(dev_X, dev_y))
print("Train")
global epoch_train_acc
n_iter = 0
for X, y in trainer.iterate(train_X, train_y, progress_bar=not quiet):
# Slightly useful trick: Decay the dropout as training proceeds.
yh, backprop = model.begin_update(X, drop=trainer.dropout)
assert yh.shape == y.shape, (yh.shape, y.shape)
assert (yh >= 0.).all(), yh
train_acc = ((yh >= 0.5) == (y >= 0.5)).sum()
loss = model.ops.xp.abs(yh - y).mean()
epoch_train_acc += train_acc
backprop(yh - y, optimizer)
n_iter += 1
# Slightly useful trick: start with low batch size, accelerate.
trainer.batch_size = min(int(batch_size), max_batch_size)
batch_size *= 1.001
if out_loc:
out_loc = Path(out_loc)
print('Saving to', out_loc)
with out_loc.open('wb') as file_:
pickle.dump(model, file_, -1)
def Tok2Vec(width, embed_size, **kwargs):
# Circular imports :(
from .._ml import CharacterEmbed
from .._ml import PyTorchBiLSTM
pretrained_vectors = kwargs.get("pretrained_vectors", None)
cnn_maxout_pieces = kwargs.get("cnn_maxout_pieces", 3)
subword_features = kwargs.get("subword_features", True)
char_embed = kwargs.get("char_embed", False)
if char_embed:
subword_features = False
conv_depth = kwargs.get("conv_depth", 4)
bilstm_depth = kwargs.get("bilstm_depth", 0)
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
with Model.define_operators({">>": chain, "|": concatenate, "**": clone}):
norm = HashEmbed(width,
embed_size,
column=cols.index(NORM),
name="embed_norm",
seed=6)
if subword_features:
prefix = HashEmbed(width,
embed_size // 2,
column=cols.index(PREFIX),
name="embed_prefix",
seed=7)
suffix = HashEmbed(width,
embed_size // 2,
column=cols.index(SUFFIX),
name="embed_suffix",
seed=8)
shape = HashEmbed(width,
embed_size // 2,
column=cols.index(SHAPE),
name="embed_shape",
seed=9)
else:
prefix, suffix, shape = (None, None, None)
if pretrained_vectors is not None:
glove = StaticVectors(pretrained_vectors,
width,
column=cols.index(ID))
if subword_features:
embed = uniqued(
(glove | norm | prefix | suffix | shape) >> LN(
Maxout(width, width * 5, pieces=3)),
column=cols.index(ORTH),
)
elif char_embed:
embed = concatenate_lists(
CharacterEmbed(nM=64, nC=8),
FeatureExtracter(cols) >> with_flatten(glove),
)
reduce_dimensions = LN(
Maxout(width, 64 * 8 + width, pieces=cnn_maxout_pieces))
else:
embed = uniqued(
(glove | norm) >> LN(Maxout(width, width * 2, pieces=3)),
column=cols.index(ORTH),
)
elif subword_features:
embed = uniqued(
(norm | prefix | suffix | shape) >> LN(
Maxout(width, width * 4, pieces=3)),
column=cols.index(ORTH),
)
elif char_embed:
embed = concatenate_lists(
CharacterEmbed(nM=64, nC=8),
FeatureExtracter(cols) >> with_flatten(norm),
)
reduce_dimensions = LN(
Maxout(width, 64 * 8 + width, pieces=cnn_maxout_pieces))
else:
embed = norm
convolution = Residual(
ExtractWindow(
nW=1) >> LN(Maxout(width, width *
3, pieces=cnn_maxout_pieces)))
if char_embed:
tok2vec = embed >> with_flatten(
reduce_dimensions >> convolution**conv_depth, pad=conv_depth)
else:
tok2vec = FeatureExtracter(cols) >> with_flatten(
embed >> convolution**conv_depth, pad=conv_depth)
if bilstm_depth >= 1:
tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth)
# Work around thinc API limitations :(. TODO: Revise in Thinc 7
tok2vec.nO = width
tok2vec.embed = embed
return tok2vec