def __init__(self, nS=12, nM=768, nH=12, nO=2, device='cpu'): Model.__init__(self) self.nM = nM self.nO = nO self.nS = nS self.enc = clone(EncoderLayer(nM=nM, nH=nH, device=device), nS) self.affine = Affine(nI=nM, nO=nM) self.softmax = Softmax(nI=nM, nO=nO) self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device)) self.slicer = PyTorchWrapper(PytorchSlicer()) self.device = device self.layers_ = [self.enc]
def __init__(self, nM=300, nH=6, device='cpu'): Model.__init__(self) self.y_attn = MultiHeadedAttention(nM=nM, nH=nH) self.x_attn = MultiHeadedAttention(nM=nM, nH=nH) self.norm = PyTorchWrapper(PytorchLayerNorm(nM, device=device)) self.ffd = PositionwiseFeedForward(nM, 4 * nM) self.layers_ = [self.norm, self.y_attn, self.x_attn, self.ffd]
def __init__(self, nM=300, nH=6): self.nH = nH self.nM = nM # model size: the length of the embeddings self.nD = nM // nH self.get_queries = with_reshape(Affine(nM, nM)) self.get_keys = with_reshape(Affine(nM, nM)) self.get_values = with_reshape(Affine(nM, nM)) self.get_output = with_reshape(Affine(nM, nM)) self._layers = [self.get_queries, self.get_keys, self.get_values, self.get_output] self._softmax = PyTorchWrapper(nn.Softmax(dim=-1)) ''' mask conf ''' i_grad = [1, 0] o_xp = None b_map = None ret_x = [0] conf = [i_grad, o_xp, b_map, ret_x] self._mask = PyTorchWrapper(PytorchMaskScores(), conf=conf)
def main(length=1000, nO=32, nI=32): pt_model = nn.Linear(nI, nO) optimizer = torch.optim.Adam(pt_model.parameters()) model = PyTorchWrapper(pt_model) X = numpy.ones((length, nI), dtype='f') y = 1. / X for i in range(10): yh, get_dX = model.begin_update(X) dY = (yh - y) / len(y) dX = get_dX(dY)
def test_wrapper(nN=2, nI=3, nO=4): if PyTorchWrapper is None: return model = PyTorchWrapper(torch.nn.Linear(nI, nO)) sgd = SGD(model.ops, 0.001) X = numpy.zeros((nN, nI), dtype="f") X += numpy.random.uniform(size=X.size).reshape(X.shape) Y = numpy.zeros((nN, nO), dtype="f") Yh, get_dX = model.begin_update(X) assert Yh.shape == (nN, nO) dYh = (Yh - Y) / Yh.shape[0] dX = get_dX(dYh, sgd=sgd) assert dX.shape == (nN, nI) check_learns_zero_output(model, sgd, X, Y)
def main(length=1000, nO=32, nI=32): if CupyOps.xp is not None: print("Use GPU") Model.ops = CupyOps() Model.Ops = CupyOps torch.set_default_tensor_type("torch.cuda.FloatTensor") pt_model = nn.Linear(nI, nO) optimizer = torch.optim.Adam(pt_model.parameters()) # noqa: F841 model = PyTorchWrapper(pt_model) X = Model.ops.xp.ones((length, nI), dtype="f") y = 1.0 / X for i in range(10): yh, get_dX = model.begin_update(X) dY = (yh - y) / len(y) dX = get_dX(dY) # noqa: F841
def main(depth=2, width=512, nb_epoch=30): prefer_gpu() torch.set_num_threads(1) train_data, dev_data, _ = datasets.mnist() train_X, train_y = Model.ops.unzip(train_data) dev_X, dev_y = Model.ops.unzip(dev_data) dev_y = to_categorical(dev_y) model = PyTorchWrapper( PyTorchFeedForward( depth=depth, width=width, input_size=train_X.shape[1], output_size=dev_y.shape[1], )) with model.begin_training(train_X, train_y, L2=1e-6) as (trainer, optimizer): epoch_loss = [0.0] def report_progress(): # with model.use_params(optimizer.averages): print(epoch_loss[-1], model.evaluate(dev_X, dev_y), trainer.dropout) epoch_loss.append(0.0) trainer.each_epoch.append(report_progress) trainer.nb_epoch = nb_epoch trainer.dropout = 0.3 trainer.batch_size = 128 trainer.dropout_decay = 0.0 train_X = model.ops.asarray(train_X, dtype="float32") y_onehot = to_categorical(train_y) for X, y in trainer.iterate(train_X, y_onehot): yh, backprop = model.begin_update(X, drop=trainer.dropout) loss = ((yh - y)**2.0).sum() / y.shape[0] backprop(yh - y, optimizer) epoch_loss[-1] += loss with model.use_params(optimizer.averages): print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y)) with open("out.pickle", "wb") as file_: pickle.dump(model, file_, -1)
def main(length=1000, nO=32, nI=32): ''' Driver function ''' if CupyOps.xp != None: print("Use GPU") Model.ops = CupyOps() Model.Ops = CupyOps torch.set_default_tensor_type('torch.cuda.FloatTensor') else: print("GPU not available. Running on CPU.") pt_model = nn.Linear(nI, nO) optimizer = torch.optim.Adam(pt_model.parameters()) # noqa: F841 model = PyTorchWrapper(pt_model) X = Model.ops.xp.ones((length, nI), dtype="f") y = 1.0 / X for i in range(10): yh, get_dX = model.begin_update(X) dY = (yh - y) / len(y) dX = get_dX(dY) # noqa: F841
def __init__(self, nS=1, nH=6, nM=300, nTGT=10000, device='cpu'): ''' EncoderDecoder consists of an encoder stack, a decoder stack and an output layer which is a linear + softmax. Parameters explanation: nS: the number of encoders/decoders in the stack nH: the number of heads in the multiheaded attention nM: the token's embedding size nTGT: the number of unique words in output vocabulary ''' Model.__init__(self) self.nS = nS self.nH = nH self.nM = nM self.nTGT = nTGT self.device = device self.enc = Encoder(nM=nM, nH=nH, device=device, nS=nS) self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device)) self.dec = clone(DecoderLayer(nM=nM, nH=nH, device=device), nS) self.proj = with_reshape(Softmax(nO=nTGT, nI=nM)) self._layers = [self.enc, self.dec, self.proj]
def batch_train_increment(dataset, input_model=None, output_model=None, lang='en', factor=1, dropout=0.2, n_iter=1, batch_size=10, eval_id=None, eval_split=None, long_text=False, silent=False,shuffle=False,gpu_id = None): """ Batch train a new text classification model from annotations. Prodigy will export the best result to the output directory, and include a JSONL file of the training and evaluation examples. You can either supply a dataset ID containing the evaluation data, or choose to split off a percentage of examples for evaluation. """ #log("RECIPE: Starting recipe textcat.batch-train", locals()) if(gpu_id): spacy.util.use_gpu(gpu_id) if(n_iter ==1): print("one pass mode") print("batch_size",batch_size) print(factor,type(factor)) DB = connect() print_ = get_print(silent) random.seed(0) if input_model is not None: nlp = spacy.load(input_model, disable=['ner']) print_('\nLoaded model {}'.format(input_model)) else: print("build your customized model") nlp = spacy.load('en_core_web_lg') pt_model = FastText(vocab_size=684831, emb_dim = 300) pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data)) model = PyTorchWrapper(pt_model) #textcat = TextCategorizer(nlp.vocab,model) textcat = Loss_TextCategorizer(nlp.vocab,model) nlp.add_pipe(textcat) examples = DB.get_dataset(dataset) labels = {eg['label'] for eg in examples} labels = list(sorted(labels)) print(labels) model = TextClassifier(nlp, labels, long_text=long_text, low_data=len(examples) < 1000) if shuffle: print("it's shuffling") random.shuffle(examples) else: print("it's not shuffling") if eval_id: evals = DB.get_dataset(eval_id) print_("Loaded {} evaluation examples from '{}'" .format(len(evals), eval_id)) else: examples, evals, eval_split = split_evals(examples, eval_split) print_("Using {}% of examples ({}) for evaluation" .format(round(eval_split * 100), len(evals))) if shuffle: random.shuffle(examples) examples = examples[:int(len(examples) * factor)] print_(printers.trainconf(dropout, n_iter, batch_size, factor, len(examples))) if len(evals) > 0: print_(printers.tc_update_header()) # best_acc = {'accuracy': 0} # best_model = None if long_text: examples = list(split_sentences(nlp, examples, min_length=False)) batch_idx = 0 start_time = datetime.now() for batch in cytoolz.partition_all(batch_size, tqdm.tqdm(examples, leave=False)): batch = list(batch) for i in range(n_iter): loss = model.update(batch, revise=False, drop=dropout) if len(evals) > 0: #print("optimizer averages",model.optimizer.averages) with nlp.use_params(model.optimizer.averages): acc = model.evaluate(tqdm.tqdm(evals, leave=False)) #print_(printers.tc_update(i, loss, acc)) end_time = datetime.now() -start_time print('Time:[{0} seconds], Epoch: [{1}/{2}], batch: [{3}/{4}], Loss:{5}, Accuracy:{6}'.format( end_time.seconds,i+1, n_iter, batch_idx+1, len(examples)//batch_size, loss, acc['accuracy'])) batch_idx += 1 return acc
def textcat_al(dataset, spacy_model,source=None, label='', api=None, patterns=None, loader=None, long_text=False, exclude=None): """ Collect the best possible training data for a text classification model with the model in the loop. Based on your annotations, Prodigy will decide which questions to ask next. """ # logDB = setup_mongo('activelearning') #nlp = spacy.load('/home/ysun/pytorchprodigy/') if(spacy_model is not None): if(type(spacy_model) == str): print("Load model ",spacy_model) nlp=spacy.load(spacy_model, disable=['ner', 'parser']) model = TextClassifier(nlp, label, long_text=long_text) else: model = spacy_model else: print("build your customized model") nlp = spacy.load('en_core_web_lg') #pt_model = nn.Linear(100,1) #pt_model = LSTMSentiment(embedding_dim = 100, hidden_dim =100, vocab_size=259136, label_size=2, batch_size=3, dropout=0.5) pt_model = FastText_test(vocab_size=684831, emb_dim = 300) pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data)) model = PyTorchWrapper(pt_model) textcat = Loss_TextCategorizer(nlp.vocab,model) nlp.add_pipe(textcat) model = TextClassifier(nlp, label, long_text=long_text) stream = get_stream(source,input_key = 'text') if patterns is None: predict = model update = model.update else: matcher = PatternMatcher(model.nlp, prior_correct=5., prior_incorrect=5., label_span=False, label_task=True) matcher = matcher.from_disk(patterns) #log("RECIPE: Created PatternMatcher and loaded in patterns", patterns) # Combine the textcat model with the PatternMatcher to annotate both # match results and predictions, and update both models. predict, update = combine_models(model, matcher) # Rank the stream. Note this is continuous, as model() is a generator. # As we call model.update(), the ranking of examples changes. stream = test_stream(stream,predict) def updateDB(answers): model.update(answers) #print("update model") #for eg in answers: # print(eg) #for score,eg in model(answers): # eg["update_score"] = score # print("new",score) #print(answers) def on_exit(): print("on_exit") return model return { 'view_id': 'classification', 'dataset': dataset, 'stream': stream, 'exclude': exclude, 'update': updateDB, 'on_exit': on_exit, 'config': {'labels': model.labels,'batch_size':1} }
def batch_train(dataset, input_model=None, output_model=None, lang='en', factor=1, dropout=0.2, n_iter=10, batch_size=10, eval_id=None, eval_split=None, long_text=False, silent=False,shuffle=False): """ Batch train a new text classification model from annotations. Prodigy will export the best result to the output directory, and include a JSONL file of the training and evaluation examples. You can either supply a dataset ID containing the evaluation data, or choose to split off a percentage of examples for evaluation. """ #log("RECIPE: Starting recipe textcat.batch-train", locals()) print("batch_size",batch_size) print(factor,type(factor)) DB = connect() print_ = get_print(silent) random.seed(0) if input_model is not None: nlp = spacy.load(input_model, disable=['ner']) print_('\nLoaded model {}'.format(input_model)) else: print("build your customized model") nlp = spacy.load('en_core_web_lg') pt_model = FastText(vocab_size=684831, emb_dim = 300) pt_model.embeds.weight.data.copy_(torch.from_numpy(nlp.vocab.vectors.data)) model = PyTorchWrapper(pt_model) textcat = TextCategorizer(nlp.vocab,model) nlp.add_pipe(textcat) #pt_model = LSTMSentiment(embedding_dim = 100, hidden_dim =100, vocab_size=259136, label_size=2, batch_size=3, dropout=0.5) #model = PyTorchWrapper(pt_model) #nlp = spacy.load('/home/ysun/pytorchprodigy/') #textcat = TextCategorizer(nlp.vocab,model) #nlp.add_pipe(textcat) examples = DB.get_dataset(dataset) labels = {eg['label'] for eg in examples} labels = list(sorted(labels)) print(labels) model = TextClassifier(nlp, labels, long_text=long_text, low_data=len(examples) < 1000) #log('RECIPE: Initialised TextClassifier with model {}' # .format(input_model), model.nlp.meta) if shuffle: print("it's shuffling") random.shuffle(examples) else: print("it's not shuffling") if eval_id: evals = DB.get_dataset(eval_id) print_("Loaded {} evaluation examples from '{}'" .format(len(evals), eval_id)) else: examples, evals, eval_split = split_evals(examples, eval_split) print_("Using {}% of examples ({}) for evaluation" .format(round(eval_split * 100), len(evals))) if shuffle: random.shuffle(examples) examples = examples[:int(len(examples) * factor)] print_(printers.trainconf(dropout, n_iter, batch_size, factor, len(examples))) if len(evals) > 0: print_(printers.tc_update_header()) best_acc = {'accuracy': 0} best_model = None if long_text: examples = list(split_sentences(nlp, examples, min_length=False)) for i in range(n_iter): loss = 0. random.shuffle(examples) for batch in cytoolz.partition_all(batch_size, tqdm.tqdm(examples, leave=False)): batch = list(batch) loss += model.update(batch, revise=False, drop=dropout) if len(evals) > 0: with nlp.use_params(model.optimizer.averages): acc = model.evaluate(tqdm.tqdm(evals, leave=False)) if acc['accuracy'] > best_acc['accuracy']: best_acc = dict(acc) best_model = nlp.to_bytes() print_(printers.tc_update(i, loss, acc)) if len(evals) > 0: print_(printers.tc_result(best_acc)) if output_model is not None: if best_model is not None: nlp = nlp.from_bytes(best_model) msg = export_model_data(output_model, nlp, examples, evals) print_(msg) return best_acc['accuracy']
def __init__(self, nM=300, nH=6, nS=6, device='cpu'): Model.__init__(self) self.stack = clone(EncoderLayer(nM=nM, nH=nH, device=device), nS) self.norm = PyTorchWrapper(PytorchLayerNorm(nM=nM, device=device))