def graph_input_sts(si0, si1, y, f0=None, f1=None): """ Produce Keras task specification from vocab-vectorized sentences. """ import pysts.loader as loader gr = {'si0': si0, 'si1': si1, 'classes': loader.sts_labels2categorical(y)} if f0 is not None: gr['f0'] = f0 gr['f1'] = f1 return gr
def graph_input_sts(si0, si1, y, f0=None, f1=None, s0=None, s1=None): """ Produce Keras task specification from vocab-vectorized sentences. """ import pysts.loader as loader gr = {'si0': si0, 'si1': si1, 'classes': loader.sts_labels2categorical(y)} if f0 is not None: gr['f0'] = f0 gr['f1'] = f1 if s0 is not None: # This is useful for non-neural baselines gr['s0'] = s0 gr['s1'] = s1 return gr
def predict(self, gr): scores = [] for i in range(len(gr['s0'])): s0 = [self._norm(w) for w in gr['s0'][i]] s1 = [self._norm(w) for w in gr['s1'][i]] scores.append([self._score(s0, s1)]) scores = np.array(scores) if self.output == 'score': return {'score': scores} elif self.output == 'binary': # XXX: we should tune the threshold to maximize accuracy scores0 = scores - np.min(scores) return {'score': scores0 * 5. / np.max(scores0)} elif self.output == 'classes': scores0 = scores - np.min(scores) return {'classes': loader.sts_labels2categorical(scores0 * 5. / np.max(scores0))}
def predict(self, gr): scores = [] for i in range(len(gr['s0'])): s0 = [self._norm(w) for w in gr['s0'][i]] s1 = [self._norm(w) for w in gr['s1'][i]] scores.append([self._score(s0, s1)]) scores = np.array(scores) if self.output == 'score': return {'score': scores} elif self.output == 'binary': # XXX: we should tune the threshold to maximize accuracy scores0 = scores - np.min(scores) return {'score': scores0 * 5. / np.max(scores0)} elif self.output == 'classes': scores0 = scores - np.min(scores) return { 'classes': loader.sts_labels2categorical(scores0 * 5. / np.max(scores0)) }
parser = argparse.ArgumentParser(description="Benchmark yu1412 on semantic relatedness regression-classification (sts)") parser.add_argument("-N", help="GloVe dim", type=int, default=300) parser.add_argument("--sick", help="whether to run on SICK2014 inst. of sts2012-14/15 dataset", type=int, default=0) args = parser.parse_args() glove = emb.GloVe(N=args.N) if args.sick == 1: Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014) Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014) else: Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*') Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*') model = prep_model(glove) model.compile(loss={'classes': 'categorical_crossentropy'}, optimizer='adam') model.fit({'e0': Xtrain[0], 'e1': Xtrain[1], 'classes': loader.sts_labels2categorical(ytrain)}, batch_size=20, nb_epoch=100, validation_data={'e0': Xtest[0], 'e1': Xtest[1], 'classes': loader.sts_labels2categorical(ytest)}) ev.eval_sts(model.predict({'e0': Xtrain[0], 'e1': Xtrain[1]})['classes'], ytrain, 'Train') ev.eval_sts(model.predict({'e0': Xtest[0], 'e1': Xtest[1]})['classes'], ytest, 'Test') """ Performance tuning (100 iters) on sick2014: * Just elementwise-mul: 4500/4500 [==============================] - 0s - loss: 1.2332 - acc: 0.5016 - val_loss: 1.1926 - val_acc: 0.4770 Train Pearson: 0.721662 Train Spearman: 0.593711 Train MSE: 0.529364
loader.load_sick2014) Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014) else: Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*') Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*') model = prep_model(glove) model.compile(loss={'classes': 'categorical_crossentropy'}, optimizer='adam') model.fit( { 'e0': Xtrain[0], 'e1': Xtrain[1], 'classes': loader.sts_labels2categorical(ytrain) }, batch_size=20, nb_epoch=100, validation_data={ 'e0': Xtest[0], 'e1': Xtest[1], 'classes': loader.sts_labels2categorical(ytest) }) ev.eval_sts( model.predict({ 'e0': Xtrain[0], 'e1': Xtrain[1] })['classes'], ytrain, 'Train') ev.eval_sts( model.predict({
model1.add(RepeatVector(6)) # [nclass] model = Sequential() model.add(Merge([model0, model1], mode='dot', dot_axes=([2], [2]))) model.add(Flatten()) # 6x6 matrix with cross-activations -> 36 vector model.add(Dense(6, W_regularizer=l2(l2reg))) # 36 vector -> 6 vector, ugh model.add(Activation('softmax')) return model if __name__ == "__main__": parser = argparse.ArgumentParser(description="Benchmark yu1412 on semantic relatedness regression-classification (sts)") parser.add_argument("-N", help="GloVe dim", type=int, default=300) parser.add_argument("--sick", help="whether to run on SICK2014 inst. of sts2012-14/15 dataset", type=int, default=0) args = parser.parse_args() glove = emb.GloVe(N=args.N) if args.sick == 1: Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014) Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014) else: Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*') Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*') model = prep_model(glove) model.compile(loss='categorical_crossentropy', optimizer='adam') model.fit(Xtrain, loader.sts_labels2categorical(ytrain), batch_size=80, nb_epoch=200, show_accuracy=True, validation_data=(Xtest, loader.sts_labels2categorical(ytest))) ev.eval_sts(model.predict_proba(Xtrain), ytrain, 'Train') ev.eval_sts(model.predict_proba(Xtest), ytest, 'Test')