def __init__(self, nS=12, nM=768, nH=12, nO=2, device='cpu'):
Model.__init__(self)
self.nM = nM
self.nO = nO
self.nS = nS
self.enc = clone(EncoderLayer(nM=nM, nH=nH, device=device), nS)
self.affine = Affine(nI=nM, nO=nM)
self.softmax = Softmax(nI=nM, nO=nO)
self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device))
self.slicer = PyTorchWrapper(PytorchSlicer())
self.device = device
self.layers_ = [self.enc]
def __init__(self, nM=300, nH=6, device='cpu'):
Model.__init__(self)
self.y_attn = MultiHeadedAttention(nM=nM, nH=nH)
self.x_attn = MultiHeadedAttention(nM=nM, nH=nH)
self.norm = PyTorchWrapper(PytorchLayerNorm(nM, device=device))
self.ffd = PositionwiseFeedForward(nM, 4 * nM)
self.layers_ = [self.norm, self.y_attn, self.x_attn, self.ffd]
def __init__(self, nM=300, nH=6):
self.nH = nH
self.nM = nM # model size: the length of the embeddings
self.nD = nM // nH
self.get_queries = with_reshape(Affine(nM, nM))
self.get_keys = with_reshape(Affine(nM, nM))
self.get_values = with_reshape(Affine(nM, nM))
self.get_output = with_reshape(Affine(nM, nM))
self._layers = [self.get_queries, self.get_keys, self.get_values, self.get_output]
self._softmax = PyTorchWrapper(nn.Softmax(dim=-1))
''' mask conf '''
i_grad = [1, 0]
o_xp = None
b_map = None
ret_x = [0]
conf = [i_grad, o_xp, b_map, ret_x]
self._mask = PyTorchWrapper(PytorchMaskScores(), conf=conf)
def main(length=1000, nO=32, nI=32):
pt_model = nn.Linear(nI, nO)
optimizer = torch.optim.Adam(pt_model.parameters())
model = PyTorchWrapper(pt_model)
X = numpy.ones((length, nI), dtype='f')
y = 1. / X
for i in range(10):
yh, get_dX = model.begin_update(X)
dY = (yh - y) / len(y)
dX = get_dX(dY)
def test_wrapper(nN=2, nI=3, nO=4):
if PyTorchWrapper is None:
return
model = PyTorchWrapper(torch.nn.Linear(nI, nO))
sgd = SGD(model.ops, 0.001)
X = numpy.zeros((nN, nI), dtype="f")
X += numpy.random.uniform(size=X.size).reshape(X.shape)
Y = numpy.zeros((nN, nO), dtype="f")
Yh, get_dX = model.begin_update(X)
assert Yh.shape == (nN, nO)
dYh = (Yh - Y) / Yh.shape[0]
dX = get_dX(dYh, sgd=sgd)
assert dX.shape == (nN, nI)
check_learns_zero_output(model, sgd, X, Y)
def main(length=1000, nO=32, nI=32):
if CupyOps.xp is not None:
print("Use GPU")
Model.ops = CupyOps()
Model.Ops = CupyOps
torch.set_default_tensor_type("torch.cuda.FloatTensor")
pt_model = nn.Linear(nI, nO)
optimizer = torch.optim.Adam(pt_model.parameters()) # noqa: F841
model = PyTorchWrapper(pt_model)
X = Model.ops.xp.ones((length, nI), dtype="f")
y = 1.0 / X
for i in range(10):
yh, get_dX = model.begin_update(X)
dY = (yh - y) / len(y)
dX = get_dX(dY) # noqa: F841
def main(depth=2, width=512, nb_epoch=30):
prefer_gpu()
torch.set_num_threads(1)
train_data, dev_data, _ = datasets.mnist()
train_X, train_y = Model.ops.unzip(train_data)
dev_X, dev_y = Model.ops.unzip(dev_data)
dev_y = to_categorical(dev_y)
model = PyTorchWrapper(
PyTorchFeedForward(
depth=depth,
width=width,
input_size=train_X.shape[1],
output_size=dev_y.shape[1],
))
with model.begin_training(train_X, train_y,
L2=1e-6) as (trainer, optimizer):
epoch_loss = [0.0]
def report_progress():
# with model.use_params(optimizer.averages):
print(epoch_loss[-1], model.evaluate(dev_X, dev_y),
trainer.dropout)
epoch_loss.append(0.0)
trainer.each_epoch.append(report_progress)
trainer.nb_epoch = nb_epoch
trainer.dropout = 0.3
trainer.batch_size = 128
trainer.dropout_decay = 0.0
train_X = model.ops.asarray(train_X, dtype="float32")
y_onehot = to_categorical(train_y)
for X, y in trainer.iterate(train_X, y_onehot):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
loss = ((yh - y)**2.0).sum() / y.shape[0]
backprop(yh - y, optimizer)
epoch_loss[-1] += loss
with model.use_params(optimizer.averages):
print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y))
with open("out.pickle", "wb") as file_:
pickle.dump(model, file_, -1)
def main(length=1000, nO=32, nI=32):
''' Driver function '''
if CupyOps.xp != None:
print("Use GPU")
Model.ops = CupyOps()
Model.Ops = CupyOps
torch.set_default_tensor_type('torch.cuda.FloatTensor')
else:
print("GPU not available. Running on CPU.")
pt_model = nn.Linear(nI, nO)
optimizer = torch.optim.Adam(pt_model.parameters()) # noqa: F841
model = PyTorchWrapper(pt_model)
X = Model.ops.xp.ones((length, nI), dtype="f")
y = 1.0 / X
for i in range(10):
yh, get_dX = model.begin_update(X)
dY = (yh - y) / len(y)
dX = get_dX(dY) # noqa: F841
def __init__(self, nS=1, nH=6, nM=300, nTGT=10000, device='cpu'):
'''
EncoderDecoder consists of an encoder stack, a decoder stack and an
output layer which is a linear + softmax.
Parameters explanation:
nS: the number of encoders/decoders in the stack
nH: the number of heads in the multiheaded attention
nM: the token's embedding size
nTGT: the number of unique words in output vocabulary
'''
Model.__init__(self)
self.nS = nS
self.nH = nH
self.nM = nM
self.nTGT = nTGT
self.device = device
self.enc = Encoder(nM=nM, nH=nH, device=device, nS=nS)
self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device))
self.dec = clone(DecoderLayer(nM=nM, nH=nH, device=device), nS)
self.proj = with_reshape(Softmax(nO=nTGT, nI=nM))
self._layers = [self.enc, self.dec, self.proj]
def batch_train_increment(dataset, input_model=None, output_model=None, lang='en',
factor=1, dropout=0.2, n_iter=1, batch_size=10,
eval_id=None, eval_split=None, long_text=False, silent=False,shuffle=False,gpu_id = None):
"""
Batch train a new text classification model from annotations. Prodigy will
export the best result to the output directory, and include a JSONL file of
the training and evaluation examples. You can either supply a dataset ID
containing the evaluation data, or choose to split off a percentage of
examples for evaluation.
"""
#log("RECIPE: Starting recipe textcat.batch-train", locals())
if(gpu_id):
spacy.util.use_gpu(gpu_id)
if(n_iter ==1):
print("one pass mode")
print("batch_size",batch_size)
print(factor,type(factor))
DB = connect()
print_ = get_print(silent)
random.seed(0)
if input_model is not None:
nlp = spacy.load(input_model, disable=['ner'])
print_('\nLoaded model {}'.format(input_model))
else:
print("build your customized model")
nlp = spacy.load('en_core_web_lg')
pt_model = FastText(vocab_size=684831, emb_dim = 300)
pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data))
model = PyTorchWrapper(pt_model)
#textcat = TextCategorizer(nlp.vocab,model)
textcat = Loss_TextCategorizer(nlp.vocab,model)
nlp.add_pipe(textcat)
examples = DB.get_dataset(dataset)
labels = {eg['label'] for eg in examples}
labels = list(sorted(labels))
print(labels)
model = TextClassifier(nlp, labels, long_text=long_text,
low_data=len(examples) < 1000)
if shuffle:
print("it's shuffling")
random.shuffle(examples)
else:
print("it's not shuffling")
if eval_id:
evals = DB.get_dataset(eval_id)
print_("Loaded {} evaluation examples from '{}'"
.format(len(evals), eval_id))
else:
examples, evals, eval_split = split_evals(examples, eval_split)
print_("Using {}% of examples ({}) for evaluation"
.format(round(eval_split * 100), len(evals)))
if shuffle:
random.shuffle(examples)
examples = examples[:int(len(examples) * factor)]
print_(printers.trainconf(dropout, n_iter, batch_size, factor,
len(examples)))
if len(evals) > 0:
print_(printers.tc_update_header())
# best_acc = {'accuracy': 0}
# best_model = None
if long_text:
examples = list(split_sentences(nlp, examples, min_length=False))
batch_idx = 0
start_time = datetime.now()
for batch in cytoolz.partition_all(batch_size,
tqdm.tqdm(examples, leave=False)):
batch = list(batch)
for i in range(n_iter):
loss = model.update(batch, revise=False, drop=dropout)
if len(evals) > 0:
#print("optimizer averages",model.optimizer.averages)
with nlp.use_params(model.optimizer.averages):
acc = model.evaluate(tqdm.tqdm(evals, leave=False))
#print_(printers.tc_update(i, loss, acc))
end_time = datetime.now() -start_time
print('Time:[{0} seconds], Epoch: [{1}/{2}], batch: [{3}/{4}], Loss:{5}, Accuracy:{6}'.format(
end_time.seconds,i+1, n_iter, batch_idx+1, len(examples)//batch_size, loss, acc['accuracy']))
batch_idx += 1
return acc
def textcat_al(dataset, spacy_model,source=None, label='', api=None, patterns=None,
loader=None, long_text=False, exclude=None):
"""
Collect the best possible training data for a text classification model
with the model in the loop. Based on your annotations, Prodigy will decide
which questions to ask next.
"""
# logDB = setup_mongo('activelearning')
#nlp = spacy.load('/home/ysun/pytorchprodigy/')
if(spacy_model is not None):
if(type(spacy_model) == str):
print("Load model ",spacy_model)
nlp=spacy.load(spacy_model, disable=['ner', 'parser'])
model = TextClassifier(nlp, label, long_text=long_text)
else:
model = spacy_model
else:
print("build your customized model")
nlp = spacy.load('en_core_web_lg')
#pt_model = nn.Linear(100,1)
#pt_model = LSTMSentiment(embedding_dim = 100, hidden_dim =100, vocab_size=259136, label_size=2, batch_size=3, dropout=0.5)
pt_model = FastText_test(vocab_size=684831, emb_dim = 300)
pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data))
model = PyTorchWrapper(pt_model)
textcat = Loss_TextCategorizer(nlp.vocab,model)
nlp.add_pipe(textcat)
model = TextClassifier(nlp, label, long_text=long_text)
stream = get_stream(source,input_key = 'text')
if patterns is None:
predict = model
update = model.update
else:
matcher = PatternMatcher(model.nlp, prior_correct=5.,
prior_incorrect=5., label_span=False,
label_task=True)
matcher = matcher.from_disk(patterns)
#log("RECIPE: Created PatternMatcher and loaded in patterns", patterns)
# Combine the textcat model with the PatternMatcher to annotate both
# match results and predictions, and update both models.
predict, update = combine_models(model, matcher)
# Rank the stream. Note this is continuous, as model() is a generator.
# As we call model.update(), the ranking of examples changes.
stream = test_stream(stream,predict)
def updateDB(answers):
model.update(answers)
#print("update model")
#for eg in answers:
# print(eg)
#for score,eg in model(answers):
# eg["update_score"] = score
# print("new",score)
#print(answers)
def on_exit():
print("on_exit")
return model
return {
'view_id': 'classification',
'dataset': dataset,
'stream': stream,
'exclude': exclude,
'update': updateDB,
'on_exit': on_exit,
'config': {'labels': model.labels,'batch_size':1}
}
def batch_train(dataset, input_model=None, output_model=None, lang='en',
factor=1, dropout=0.2, n_iter=10, batch_size=10,
eval_id=None, eval_split=None, long_text=False, silent=False,shuffle=False):
"""
Batch train a new text classification model from annotations. Prodigy will
export the best result to the output directory, and include a JSONL file of
the training and evaluation examples. You can either supply a dataset ID
containing the evaluation data, or choose to split off a percentage of
examples for evaluation.
"""
#log("RECIPE: Starting recipe textcat.batch-train", locals())
print("batch_size",batch_size)
print(factor,type(factor))
DB = connect()
print_ = get_print(silent)
random.seed(0)
if input_model is not None:
nlp = spacy.load(input_model, disable=['ner'])
print_('\nLoaded model {}'.format(input_model))
else:
print("build your customized model")
nlp = spacy.load('en_core_web_lg')
pt_model = FastText(vocab_size=684831, emb_dim = 300)
pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data))
model = PyTorchWrapper(pt_model)
textcat = TextCategorizer(nlp.vocab,model)
nlp.add_pipe(textcat)
#pt_model = LSTMSentiment(embedding_dim = 100, hidden_dim =100, vocab_size=259136, label_size=2, batch_size=3, dropout=0.5)
#model = PyTorchWrapper(pt_model)
#nlp = spacy.load('/home/ysun/pytorchprodigy/')
#textcat = TextCategorizer(nlp.vocab,model)
#nlp.add_pipe(textcat)
examples = DB.get_dataset(dataset)
labels = {eg['label'] for eg in examples}
labels = list(sorted(labels))
print(labels)
model = TextClassifier(nlp, labels, long_text=long_text,
low_data=len(examples) < 1000)
#log('RECIPE: Initialised TextClassifier with model {}'
# .format(input_model), model.nlp.meta)
if shuffle:
print("it's shuffling")
random.shuffle(examples)
else:
print("it's not shuffling")
if eval_id:
evals = DB.get_dataset(eval_id)
print_("Loaded {} evaluation examples from '{}'"
.format(len(evals), eval_id))
else:
examples, evals, eval_split = split_evals(examples, eval_split)
print_("Using {}% of examples ({}) for evaluation"
.format(round(eval_split * 100), len(evals)))
if shuffle:
random.shuffle(examples)
examples = examples[:int(len(examples) * factor)]
print_(printers.trainconf(dropout, n_iter, batch_size, factor,
len(examples)))
if len(evals) > 0:
print_(printers.tc_update_header())
best_acc = {'accuracy': 0}
best_model = None
if long_text:
examples = list(split_sentences(nlp, examples, min_length=False))
for i in range(n_iter):
loss = 0.
random.shuffle(examples)
for batch in cytoolz.partition_all(batch_size,
tqdm.tqdm(examples, leave=False)):
batch = list(batch)
loss += model.update(batch, revise=False, drop=dropout)
if len(evals) > 0:
with nlp.use_params(model.optimizer.averages):
acc = model.evaluate(tqdm.tqdm(evals, leave=False))
if acc['accuracy'] > best_acc['accuracy']:
best_acc = dict(acc)
best_model = nlp.to_bytes()
print_(printers.tc_update(i, loss, acc))
if len(evals) > 0:
print_(printers.tc_result(best_acc))
if output_model is not None:
if best_model is not None:
nlp = nlp.from_bytes(best_model)
msg = export_model_data(output_model, nlp, examples, evals)
print_(msg)
return best_acc['accuracy']
def __init__(self, nM=300, nH=6, nS=6, device='cpu'):
Model.__init__(self)
self.stack = clone(EncoderLayer(nM=nM, nH=nH, device=device), nS)
self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device))