def main(depth=2, width=512, nb_epoch=30):
prefer_gpu()
# Configuration here isn't especially good. But, for demo..
with Model.define_operators({"**": clone, ">>": chain}):
model = ReLu(width) >> ReLu(width) >> Softmax()
train_data, dev_data, _ = datasets.mnist()
train_X, train_y = model.ops.unzip(train_data)
dev_X, dev_y = model.ops.unzip(dev_data)
dev_y = to_categorical(dev_y)
with model.begin_training(train_X, train_y, L2=1e-6) as (trainer, optimizer):
epoch_loss = [0.0]
def report_progress():
with model.use_params(optimizer.averages):
print(epoch_loss[-1], model.evaluate(dev_X, dev_y), trainer.dropout)
epoch_loss.append(0.0)
trainer.each_epoch.append(report_progress)
trainer.nb_epoch = nb_epoch
trainer.dropout = 0.3
trainer.batch_size = 128
trainer.dropout_decay = 0.0
train_X = model.ops.asarray(train_X, dtype="float32")
y_onehot = to_categorical(train_y)
for X, y in trainer.iterate(train_X, y_onehot):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
loss = ((yh - y) ** 2.0).sum() / y.shape[0]
backprop(yh - y, optimizer)
epoch_loss[-1] += loss
with model.use_params(optimizer.averages):
print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y))
with open("out.pickle", "wb") as file_:
pickle.dump(model, file_, -1)
def main(width=32, nr_vector=1000):
train_data, check_data, nr_tag = ancora_pos_tags(encode_words=True)
model = with_flatten(
chain(HashEmbed(width, 1000), ReLu(width, width), ReLu(width, width),
Softmax(nr_tag, width)))
train_X, train_y = zip(*train_data)
dev_X, dev_y = zip(*check_data)
train_y = [to_categorical(y, nb_classes=nr_tag) for y in train_y]
dev_y = [to_categorical(y, nb_classes=nr_tag) for y in dev_y]
with model.begin_training(train_X, train_y) as (trainer, optimizer):
trainer.each_epoch.append(lambda: print(model.evaluate(dev_X, dev_y)))
for X, y in trainer.iterate(train_X, train_y):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
backprop([yh[i] - y[i] for i in range(len(yh))], optimizer)
with model.use_params(optimizer.averages):
print(model.evaluate(dev_X, dev_y))
def build_text_classifier(nr_class, width=64, **cfg):
nr_vector = cfg.get('nr_vector', 5000)
pretrained_dims = cfg.get('pretrained_dims', 0)
with Model.define_operators({
'>>': chain,
'+': add,
'|': concatenate,
'**': clone
}):
if cfg.get('low_data') and pretrained_dims:
model = (SpacyVectors >> flatten_add_lengths >> with_getitem(
0, Affine(width, pretrained_dims)) >>
ParametricAttention(width) >> Pooling(sum_pool) >>
Residual(ReLu(width, width))**2 >> zero_init(
Affine(nr_class, width, drop_factor=0.0)) >> logistic)
return model
lower = HashEmbed(width, nr_vector, column=1)
prefix = HashEmbed(width // 2, nr_vector, column=2)
suffix = HashEmbed(width // 2, nr_vector, column=3)
shape = HashEmbed(width // 2, nr_vector, column=4)
trained_vectors = (FeatureExtracter(
[ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID]) >> with_flatten(
uniqued((lower | prefix | suffix | shape) >> LN(
Maxout(width, width + (width // 2) * 3)),
column=0)))
if pretrained_dims:
static_vectors = (
SpacyVectors >> with_flatten(Affine(width, pretrained_dims)))
# TODO Make concatenate support lists
vectors = concatenate_lists(trained_vectors, static_vectors)
vectors_width = width * 2
else:
vectors = trained_vectors
vectors_width = width
static_vectors = None
cnn_model = (
vectors >> with_flatten(
LN(Maxout(width, vectors_width)) >> Residual(
(ExtractWindow(nW=1) >> LN(Maxout(width, width * 3))))**2,
pad=2) >> flatten_add_lengths >> ParametricAttention(width) >>
Pooling(sum_pool) >> Residual(zero_init(Maxout(width, width))) >>
zero_init(Affine(nr_class, width, drop_factor=0.0)))
linear_model = (
_preprocess_doc >> LinearModel(nr_class, drop_factor=0.))
model = ((linear_model | cnn_model) >> zero_init(
Affine(nr_class, nr_class * 2, drop_factor=0.0)) >> logistic)
model.nO = nr_class
model.lsuv = False
return model
def main(depth=2, width=512, nb_epoch=30):
if CupyOps.xp != None:
Model.ops = CupyOps()
Model.Ops = CupyOps
# Configuration here isn't especially good. But, for demo..
with Model.define_operators({'**': clone, '>>': chain}):
model = ReLu(width) >> ReLu(width) >> Softmax()
train_data, dev_data, _ = datasets.mnist()
train_X, train_y = model.ops.unzip(train_data)
dev_X, dev_y = model.ops.unzip(dev_data)
dev_y = to_categorical(dev_y)
with model.begin_training(train_X, train_y,
L2=1e-6) as (trainer, optimizer):
epoch_loss = [0.]
def report_progress():
with model.use_params(optimizer.averages):
print(epoch_loss[-1], model.evaluate(dev_X, dev_y),
trainer.dropout)
epoch_loss.append(0.)
trainer.each_epoch.append(report_progress)
trainer.nb_epoch = nb_epoch
trainer.dropout = 0.3
trainer.batch_size = 128
trainer.dropout_decay = 0.0
train_X = model.ops.asarray(train_X, dtype='float32')
y_onehot = to_categorical(train_y)
for X, y in trainer.iterate(train_X, y_onehot):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
loss = ((yh - y)**2.).sum() / y.shape[0]
backprop(yh - y, optimizer)
epoch_loss[-1] += loss
with model.use_params(optimizer.averages):
print('Avg dev.: %.3f' % model.evaluate(dev_X, dev_y))
with open('out.pickle', 'wb') as file_:
pickle.dump(model, file_, -1)
def build_text_classifier(nr_class, width=64, **cfg):
depth = cfg.get("depth", 2)
nr_vector = cfg.get("nr_vector", 5000)
pretrained_dims = cfg.get("pretrained_dims", 0)
with Model.define_operators({
">>": chain,
"+": add,
"|": concatenate,
"**": clone
}):
if cfg.get("low_data") and pretrained_dims:
model = (SpacyVectors >> flatten_add_lengths >> with_getitem(
0, Affine(width, pretrained_dims)) >>
ParametricAttention(width) >> Pooling(sum_pool) >>
Residual(ReLu(width, width))**2 >> zero_init(
Affine(nr_class, width, drop_factor=0.0)) >> logistic)
return model
lower = HashEmbed(width, nr_vector, column=1)
prefix = HashEmbed(width // 2, nr_vector, column=2)
suffix = HashEmbed(width // 2, nr_vector, column=3)
shape = HashEmbed(width // 2, nr_vector, column=4)
trained_vectors = FeatureExtracter(
[ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID]) >> with_flatten(
uniqued(
(lower | prefix | suffix | shape) >> LN(
Maxout(width, width + (width // 2) * 3)),
column=0,
))
if pretrained_dims:
static_vectors = SpacyVectors >> with_flatten(
Affine(width, pretrained_dims))
# TODO Make concatenate support lists
vectors = concatenate_lists(trained_vectors, static_vectors)
vectors_width = width * 2
else:
vectors = trained_vectors
vectors_width = width
static_vectors = None
tok2vec = vectors >> with_flatten(
LN(Maxout(width, vectors_width)) >> Residual(
(ExtractWindow(nW=1) >> LN(Maxout(width, width * 3))))**depth,
pad=depth,
)
cnn_model = (
tok2vec >> flatten_add_lengths >> ParametricAttention(width) >>
Pooling(sum_pool) >> Residual(zero_init(Maxout(width, width))) >>
zero_init(Affine(nr_class, width, drop_factor=0.0)))
linear_model = build_bow_text_classifier(nr_class,
ngram_size=cfg.get(
"ngram_size", 1),
exclusive_classes=False)
if cfg.get("exclusive_classes"):
output_layer = Softmax(nr_class, nr_class * 2)
else:
output_layer = (zero_init(
Affine(nr_class, nr_class * 2, drop_factor=0.0)) >> logistic)
model = (linear_model | cnn_model) >> output_layer
model.tok2vec = chain(tok2vec, flatten)
model.nO = nr_class
model.lsuv = False
return model