def get_data():
with open(data_dir + 'texts.pkl', 'rb') as f:
texts = pickle.load(f)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts[:25000])
sequences = tokenizer.texts_to_sequences(texts)
# word_index = tokenizer.word_index
# sequences = []
# for i in range(50000):
# t = []
# tokens = texts[i].lower().split(' ')
# for j in range(len(tokens)):
# index = word_index.get(tokens[j], 0)
# if index < num_words:
# t.append(index)
# else:
# t.append(0)
# sequences.append(t)
x = pad_sequences(sequences, maxlen=max_len)
x_train = x[:25000]
x_test = x[25000:]
y_train = np.zeros((25000, ), dtype=np.float32)
y_test = np.zeros((25000, ), dtype=np.float32)
y_train[12500:25000] = np.ones((12500, ), dtype=np.float32)
y_test[12500:25000] = np.ones((12500, ), dtype=np.float32)
return x_train, y_train, x_test, y_test
def get_input():
with open(root_dir + "temp/ag/extract_data/texts.pkl", 'rb') as f:
texts = pickle.load(f)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.fit_on_texts(texts[:num_train])
print('there are %d words' % (len(tokenizer.word_index)))
sequences = tokenizer.texts_to_sequences(texts)
# print('average length is %d'%(np.sum([len(s) for s in sequences])/len(sequences)))
new_text = []
for seq in sequences:
t = []
for i in range(len(seq)):
t.append('%d' % (seq[i]))
for i in range(len(seq) - 1):
t.append('%d_%d' % (seq[i], seq[i + 1]))
# for i in range(len(seq) - 2):
# t.append('%d_%d_%d' % (seq[i], seq[i + 1], seq[i + 2]))
new_text.append(' '.join(t))
tokenizer2 = Tokenizer(num_words=num_ngram)
tokenizer2.filters = ''
tokenizer2.fit_on_texts(new_text[:num_train])
sequences2 = tokenizer2.texts_to_sequences(new_text)
print('there are %d ngrams' % (len(tokenizer2.word_index)))
x = pad_sequences(sequences2, maxlen=max_len)
x_train = x[:num_train]
x_test = x[num_train:]
y = np.load(root_dir + 'temp/ag/extract_data/label.npy')
y_train = to_categorical(y[:num_train])
y_test = to_categorical(y[num_train:])
return x_train, y_train, x_test, y_test
def prepare_train():
print("prepare training data")
f = codecs.open('../../../temp/imdb/keras_code/utils/texts.pkl', 'rb')
text1 = pickle.load(f)
text1 = text1[:25000]
f.close()
f = codecs.open('../../../temp/imdb/keras_code/utils/texts_unsup.pkl',
'rb')
text2 = pickle.load(f)
f.close()
texts = text1 + text2
tokenizer = Tokenizer(num_words=vocab_size)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts)
sequence = tokenizer.texts_to_sequences(texts)
sequence_pad = pad_sequences(sequence,
maxlen=MAX_DOCUMENT_LENGTH + 1,
dtype=np.int32,
padding='post',
truncating='post')
seq_len = []
for i in range(len(sequence)):
r = len(sequence[i])
if r < MAX_DOCUMENT_LENGTH:
seq_len.append(r)
else:
seq_len.append(MAX_DOCUMENT_LENGTH)
x_1 = sequence_pad[:, :-1]
y_ = sequence_pad[:, 1:]
return x_1, seq_len, y_
def prepare_train():
print("prepare training data")
f = codecs.open('../temp/texts.pkl', 'rb')
text1 = pickle.load(f)
text1 = text1[:25000]
f.close()
f = codecs.open('../temp/texts_unsup.pkl', 'rb')
text2 = pickle.load(f)
f.close()
texts = text1 + text2
tokenizer = Tokenizer(num_words=vocab_size)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts)
sequence = tokenizer.texts_to_sequences(texts)
sequence_pad = pad_sequences(sequence,
maxlen=MAX_DOCUMENT_LENGTH + 1,
dtype=np.int32,
padding='post',
truncating='post')
x_1 = sequence_pad[:, :-1]
x_2 = np.arange(num_train)
x_2 = np.reshape(x_2, (num_train, 1))
y_ = sequence_pad[:, 1:]
return x_1, x_2, y_
def preserve_index_word():
num_words = 30000
max_len = 500
num_train = 25000
num_test = 25000
f = codecs.open('../temp/texts.pkl', 'rb')
texts = pickle.load(f)
f.close()
newText = []
for sentence in texts:
t = []
words = WordPunctTokenizer().tokenize(sentence)
for word in words:
if word.isalpha():
t.append(word)
newText.append(' '.join(t))
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(newText[:num_train])
sequences = tokenizer.texts_to_sequences(newText)
word_index = tokenizer.word_index
index_word = {}
for word, index in word_index.items():
index_word[index] = word
# f = codecs.open('../temp/index_word.pkl', 'wb')
# pickle.dump(index_word, f, 1)
# f.close()
return index_word, sequences
def prepare_data():
global num_ngram
print("prepare training data")
with open('../../../temp/imdb/keras_code/utils/texts.pkl', 'rb') as f:
texts = pickle.load(f)[:25000]
with open('../../../temp/imdb/keras_code/utils/texts_unsup.pkl', 'rb') as f:
texts += pickle.load(f)
with open('/home/yan/my_datasets/glove/words.pkl', 'rb') as f:
glove_words = pickle.load(f)
glove_words_set = set(glove_words)
for i, s in enumerate(texts):
new_s = []
for token in s.split(' '):
if token in glove_words_set:
new_s.append(token)
texts[i] = ' '.join(new_s)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
embeddings_index = {}
wordX = np.load('/home/yan/my_datasets/glove/embedding.300d.npy')
for i in range(len(glove_words)):
embeddings_index[glove_words[i]] = wordX[i, :]
embedding_matrix = np.zeros((num_words, 300))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None and i < num_words:
embedding_matrix[i] = embedding_vector
all_ngram = set()
for seq in sequences:
for i in range(len(seq) - 2):
all_ngram.add(tuple(seq[i:i + 3]))
num_ngram = len(all_ngram)
all_ngram=list(all_ngram)
print('there are %d ngrams' % (num_ngram))
x = np.zeros((num_ngram * 3, 3), dtype=np.int32)
y = np.zeros((num_ngram * 3, 1), dtype=np.int32)
index = 0
for i, seq in enumerate(all_ngram):
for id in seq:
x[index] = seq
y[index] = id
index += 1
indice = np.arange(num_ngram * 3)
np.random.shuffle(indice)
x = x[indice]
y = y[indice]
print(x.shape,y.shape)
return x, y, embedding_matrix
def prepare_data():
print("prepare training data")
global embedding_matrix
f = codecs.open('../../../temp/imdb/keras_code/utils/texts.pkl', 'rb')
text = pickle.load(f)
f.close()
tokenizer = Tokenizer(num_words=vocab_size)
tokenizer.filters = ''
tokenizer.fit_on_texts(text)
sequence = tokenizer.texts_to_sequences(text)
sequence_pad = pad_sequences(sequence,
maxlen=MAX_DOCUMENT_LENGTH,
dtype=np.int32,
padding='post',
truncating='post')
# prepare pretrained word2vec vectors
# word_index = tokenizer.word_index
# embeddings_index = {}
# wordX = np.load('/media/yan/winD/docker/word2vec/words/word2vec.npy')
# f = codecs.open('/media/yan/winD/docker/word2vec/words/wordsInWord2vec.pkl', 'rb')
# allwords = pickle.load(f)
# f.close()
# for i in range(3000000):
# embeddings_index[''.join(allwords[i])] = wordX[i, :]
# embedding_matrix = np.zeros((vocab_size, 300))
# for word, i in word_index.items():
# embedding_vector = embeddings_index.get(word)
# if embedding_vector is not None and i < vocab_size:
# embedding_matrix[i] = embedding_vector
seq_len = []
for i in range(len(sequence)):
r = len(sequence[i])
if r < MAX_DOCUMENT_LENGTH:
seq_len.append(r)
else:
seq_len.append(MAX_DOCUMENT_LENGTH)
x_train = sequence_pad[:25000]
x_test = sequence_pad[25000:]
x_train_len = np.asarray(seq_len[:25000])
x_test_len = np.asarray(seq_len[25000:])
y_train = np.zeros((25000, ), dtype=np.int8)
y_test = np.zeros((25000, ), dtype=np.int8)
y_train[12500:25000] = np.ones((12500, ), dtype=np.int8)
y_test[12500:25000] = np.ones((12500, ), dtype=np.int8)
indice = np.arange(25000)
np.random.shuffle(indice)
x_train = x_train[indice]
x_test = x_test[indice]
y_train = y_train[indice]
y_test = y_test[indice]
x_train_len = x_train_len[indice]
x_test_len = x_test_len[indice]
return x_train, x_test, y_train, y_test, x_train_len, x_test_len
def prepare_data():
print("prepare training data")
with open('../../../temp/imdb/keras_code/utils/texts.pkl', 'rb') as f:
text1 = pickle.load(f)
with open('../../../temp/imdb/keras_code/utils/texts_unsup.pkl',
'rb') as f:
texts = text1[:25000] + pickle.load(f)
with open('/home/yan/my_datasets/glove/words.pkl', 'rb') as f:
glove_words = pickle.load(f)
glove_words_set = set(glove_words)
for i, s in enumerate(texts):
new_s = []
for token in s.split(' '):
if token in glove_words_set:
new_s.append(token)
texts[i] = ' '.join(new_s)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(text1)
sequences = pad_sequences(sequences, maxlen=max_len)
word_index = tokenizer.word_index
embeddings_index = {}
wordX = np.load('/home/yan/my_datasets/glove/embedding.300d.npy')
for i in range(len(glove_words)):
embeddings_index[glove_words[i]] = wordX[i, :]
embedding_matrix = np.zeros((num_words, 300))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None and i < num_words:
embedding_matrix[i] = embedding_vector
x = np.zeros((50000, (max_len - 2) * 3), dtype=np.int32)
for i, seq in enumerate(sequences):
for j in range(len(seq) - 2):
x[i, j:j + 3] = seq[j:j + 3]
x_train = x[:25000]
x_test = x[25000:]
y_train = np.zeros((25000, ), dtype=np.float32)
y_test = np.zeros((25000, ), dtype=np.float32)
y_train[12500:25000] = np.ones((12500, ), dtype=np.float32)
y_test[12500:25000] = np.ones((12500, ), dtype=np.float32)
indice = np.arange(25000)
np.random.shuffle(indice)
x_train = x_train[indice]
x_test = x_test[indice]
y_train = y_train[indice]
y_test = y_test[indice]
return x_train, x_test, y_train, y_test, embedding_matrix
def load_data(path, vocab_size, maxlen, filters, ids=False, tknzr=None):
'''
Read corpus file, tokenize words and encode to sequences
'''
# Read file and append end anf begin of sentence tags
print(' Reading data file')
f = open(path,'r')
text = []
if ids: # Open ids file
# Change file extension to .ids
name = ''.join(path.split('.')[:-1]) + 'ids'
idfile = open(name)
idname = ''
for line in f:
text.append(BOS +' '+line[:-1]+' ' +EOS)
if ids: # Add context separator
read = idfile.readline()
if read != idname:
idname = read
text.append('<EOC>')
f.close()
# Create vocabulary
if tknzr is None:
print(' Generating vocabulary')
tknzr = Tokenizer(num_words=vocab_size, lower=False, oov_token=OOV)
if not filters:
tknzr.filters = ''
else:
tknzr.filters = tknzr.filters.replace('<','') #need keep tags
tknzr.filters = tknzr.filters.replace('>','')
tknzr.fit_on_texts(text)
print(' Word2idx len:',len(tknzr.word_index))
# Create one_hot vectors
print(' Creating one-hot vectors')
data = tokenize(tknzr, text, maxlen)
return data, tknzr
def get_input():
with open(root_dir + "temp/imdb/keras_code/utils/texts.pkl", 'rb') as f:
texts = pickle.load(f)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters=''
tokenizer.fit_on_texts(texts[:25000])
print('there are %d words' % (len(tokenizer.word_index)))
# word_index=tokenizer.word_index
# index_word=dict([(index,word) for word, index in word_index.items()])
sequences = tokenizer.texts_to_sequences(texts)
new_text=[]
for seq in sequences:
t=[]
for i in range(len(seq)):
t.append('%d'%(seq[i]))
for i in range(len(seq)-1):
t.append('%d_%d'%(seq[i],seq[i+1]))
for i in range(len(seq)-2):
t.append('%d_%d_%d'%(seq[i],seq[i+1],seq[i+2]))
new_text.append(' '.join(t))
tokenizer2=Tokenizer(num_words=num_ngram)
tokenizer2.filters=''
tokenizer2.fit_on_texts(new_text[:25000])
sequences2=tokenizer2.texts_to_sequences(new_text)
print('there are %d ngrams'%(len(tokenizer2.word_index)))
x = pad_sequences(sequences2, maxlen=max_len)
x_train = x[:25000]
x_test = x[25000:]
y_train = np.zeros((25000,), dtype=np.float32)
y_test = np.zeros((25000,), dtype=np.float32)
y_train[12500:25000] = np.ones((12500,), dtype=np.float32)
y_test[12500:25000] = np.ones((12500,), dtype=np.float32)
# print(x_train[0])
return x_train, y_train, x_test, y_test
def tokenize(x_train,
y_train,
x_test,
y_test,
max_nb_words=400000,
max_sequence_length=26):
"""This function does the following:
Apply tokenization on headlines
Apply tokenization on holdout headlines
(input for both: MAX_NB_WORDS, MAX_SEQUENCE_LENGTH)
"""
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import np_utils
import numpy as np
print("Initializing Keras tokenizer, max. unique words allowed = %d" %
max_nb_words)
tokenizer = Tokenizer(num_words=max_nb_words)
tokenizer.filters = '#$%&()*+,-./:;<=>@[\\]^_`{|}~\t\n'
tokenizer.fit_on_texts(x_train)
print("Applying tokenizer on training corpus...")
x_train = tokenizer.texts_to_sequences(x_train)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
print('Padding training sequences to max length %d' % max_sequence_length)
x_train = pad_sequences(x_train, maxlen=max_sequence_length)
y_train = np_utils.to_categorical(np.asarray(y_train))
print('Shape of training x_train tensor:', x_train.shape)
print('Shape of training y_train tensor:', y_train.shape)
""" Here starts a section dedicated to tokenizing new headlines on which the model has
not been trained and which may thus contain words that are not in the vocabulary.
These words will be removed from the new headline and a tokenization will be made
on the sentence as if it does not contain those words.
"""
print("Applying tokenizer on holdout corpus...")
x_test = tokenizer.texts_to_sequences(x_test)
print('Padding holdout sequences to max length %d' % max_sequence_length)
x_test = pad_sequences(x_test, maxlen=max_sequence_length)
y_test = np_utils.to_categorical(np.asarray(y_test))
print('Shape of holdout x_test tensor:', x_test.shape)
print('Shape of holdout y_test tensor:', y_test.shape)
return (x_train, y_train, x_test, y_test, tokenizer)
def __init__(self,
num_words,
max_len,
filters='',
truncing='post',
padding='pre'):
self.num_words = num_words
self.max_len = max_len
self.truncing = truncing
self.padding = padding
f = codecs.open('../../../temp/imdb/keras_code/utils/texts.pkl', 'rb')
self.texts = pickle.load(f)
f.close()
tokenizer = Tokenizer(num_words=num_words)
if filters is not None:
tokenizer.filters = filters
tokenizer.fit_on_texts(self.texts[:25000])
self.tokenizer = tokenizer
import numpy as np
import keras
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Activation, Conv1D, GlobalMaxPooling1D, Input, Embedding, \
GlobalAveragePooling1D, MaxPooling2D, AveragePooling1D
num_words = 30000
max_len = 600
f = codecs.open('../temp/texts.pkl', 'rb')
texts = pickle.load(f)
f.close()
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts[0:25000])
# sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
sequences = []
for i in range(50000):
t = []
tokens = texts[i].lower().split(' ')
for j in range(len(tokens)):
index = word_index.get(tokens[j], 0)
if index < num_words:
t.append(index)
else:
t.append(0)
sequences.append(t)
def main():
f = FileIO(os.path.join(FLAGS.buckets, "texts.pkl"), mode='r+')
texts = pickle.load(f)
f.close()
tokenizer = Tokenizer(nb_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts[0:25000])
# print(texts[0])
# sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
sequences = []
for i in range(50000):
t = []
tokens = texts[i].lower().split(' ')
for j in range(len(tokens)):
index = word_index.get(tokens[j], 0)
if index < num_words:
t.append(index)
else:
t.append(0)
sequences.append(t)
print('Found %s unique tokens.' % len(word_index))
data1 = pad_sequences(sequences[0:25000], maxlen=max_len)
data2 = pad_sequences(sequences[25000:50000], maxlen=max_len)
Ytrain = np.zeros((25000, ), dtype=np.float32)
Ytest = np.zeros((25000, ), dtype=np.float32)
Ytrain[12500:25000] = np.ones((12500, ), dtype=np.float32)
Ytest[12500:25000] = np.ones((12500, ), dtype=np.float32)
Xtrain1 = np.zeros((25000, (max_len - 2) * 3), dtype=np.int)
Xtest1 = np.zeros((25000, (max_len - 2) * 3), dtype=np.int)
for i in range(25000):
for j in range(max_len - 2):
Xtrain1[i, j * 3] = data1[i, j]
Xtrain1[i, j * 3 + 1] = data1[i][j + 1] + num_words
Xtrain1[i, j * 3 + 2] = data1[i][j + 2] + num_words * 2
for i in range(25000):
for j in range(max_len - 2):
Xtest1[i, j * 3] = data2[i, j]
Xtest1[i, j * 3 + 1] = data2[i][j + 1] + num_words
Xtest1[i, j * 3 + 2] = data2[i][j + 2] + num_words * 2
Xtrain2 = np.zeros((25000, (max_len - 1) * 2), dtype=np.int)
Xtest2 = np.zeros((25000, (max_len - 1) * 2), dtype=np.int)
for i in range(25000):
for j in range(max_len - 1):
Xtrain2[i, j * 2] = data1[i, j]
Xtrain2[i, j * 2 + 1] = data1[i][j + 1] + num_words
for i in range(25000):
for j in range(max_len - 1):
Xtest2[i, j * 2] = data2[i, j]
Xtest2[i, j * 2 + 1] = data2[i][j + 1] + num_words
indice1 = np.arange(25000)
np.random.shuffle(indice1)
Xtrain1 = Xtrain1[indice1]
Xtrain2 = Xtrain2[indice1]
Ytrain = Ytrain[indice1]
indice2 = np.arange(25000)
np.random.shuffle(indice2)
Xtest1 = Xtest1[indice2]
Xtest2 = Xtest2[indice2]
Ytest = Ytest[indice2]
print('begin to build model ...')
input1 = Input(shape=((max_len - 2) * 3, ))
embedding1 = Embedding(num_words * 3,
embedding_dimension,
input_length=(max_len - 2) * 3,
init='orthogonal')(input1)
x = AveragePooling1D(pool_length=3)(embedding1)
x = GlobalMaxPooling1D()(x)
input2 = Input(shape=((max_len - 1) * 2, ))
embedding2 = Embedding(num_words * 2,
embedding_dimension,
input_length=(max_len - 1) * 2,
init='orthogonal')(input2)
y = AveragePooling1D(pool_length=2, stride=2)(embedding2)
y = GlobalMaxPooling1D()(y)
z = merge([x, y], mode='concat')
# model.add(Dropout(0.5))
output = Dense(1, activation='sigmoid')(z)
model = Model(input=[input1, input2], output=output)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
model.fit([Xtrain1, Xtrain2],
Ytrain,
batch_size=32,
nb_epoch=20,
verbose=2,
validation_data=([Xtest1, Xtest2], Ytest))
def prepare_data():
global tokenizer
print("prepare training data")
with FileIO(os.path.join(FLAGS.buckets, 'imdb/texts.pkl'), 'r+') as f:
texts = pickle.load(f)[:25000]
with FileIO(os.path.join(FLAGS.buckets, 'imdb/texts_unsup.pkl'),
'r+') as f:
texts += pickle.load(f)
with FileIO(os.path.join(FLAGS.buckets, 'glove/words.pkl'), 'r+') as f:
glove_words = pickle.load(f)
glove_words_set = set(glove_words)
for i, s in enumerate(texts):
new_s = []
for token in s.split(' '):
if token in glove_words_set:
new_s.append(token)
texts[i] = ' '.join(new_s)
tokenizer = Tokenizer(num_words=num_words)
tokenizer.filters = ''
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
embeddings_index = {}
wordX = np.load(
FileIO(os.path.join(FLAGS.buckets, "glove/embedding.300d.npy"),
mode='r+'))
for i in range(len(glove_words)):
embeddings_index[glove_words[i]] = wordX[i, :]
embedding_matrix = np.zeros((num_words, 300))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None and i < num_words:
embedding_matrix[i] = embedding_vector
ngram_text = []
for seq in sequences:
s = [str(i) for i in seq]
ngram_sentence = []
for i in range(len(seq) - 2):
ngram_sentence.append('_'.join(s[i:i + 3]))
ngram_text.append(' '.join(ngram_sentence))
tokenizer2 = Tokenizer(num_words=num_ngram)
tokenizer2.filters = ''
tokenizer2.fit_on_texts(ngram_text)
ngram_index = tokenizer2.word_index
frequent_ngram = sorted(ngram_index.items(),
key=lambda k: k[1])[:num_ngram]
print('there are %d ngrams' % (len(ngram_index)))
x = np.zeros((num_ngram * 3, 1), dtype=np.int32)
y = np.zeros((num_ngram * 3, 1), dtype=np.int32)
index = 0
for i, seq in enumerate(frequent_ngram):
word_ids = seq[0].split('_')
for id in word_ids:
x[index] = i
y[index] = int(id)
index += 1
indice = np.arange(num_ngram * 3)
np.random.shuffle(indice)
x = x[indice]
y = y[indice]
pickle.dump(
word_index,
FileIO(os.path.join(FLAGS.buckets, "word_index.pkl"), mode='w+'))
pickle.dump(
ngram_index,
FileIO(os.path.join(FLAGS.buckets, "ngram_index.pkl"), mode='w+'))
return x, y, embedding_matrix
def main():
global ngram
f = FileIO(os.path.join(FLAGS.buckets, "texts.pkl"), mode='r+')
texts = pickle.load(f)
f.close()
tokenizer = Tokenizer(nb_words=num_words)
tokenizer.filters=''
tokenizer.fit_on_texts(texts[0:25000])
sequences = tokenizer.texts_to_sequences(texts)
# word_index = tokenizer.word_index
# sequences = []
# for i in range(50000):
# t = []
# tokens = texts[i].lower().split(' ')
# for j in range(len(tokens)):
# index = word_index.get(tokens[j], 0)
# if index < num_words:
# t.append(index)
# else:
# t.append(0)
# sequences.append(t)
# print('Found %s unique tokens.' % len(word_index))
data1 = pad_sequences(sequences[0:25000], maxlen=max_len)
data2 = pad_sequences(sequences[25000:50000], maxlen=max_len)
Ytrain = np.zeros((25000,), dtype=np.float32)
Ytest = np.zeros((25000,), dtype=np.float32)
Ytrain[12500:25000] = np.ones((12500,), dtype=np.float32)
Ytest[12500:25000] = np.ones((12500,), dtype=np.float32)
Xtrain = np.zeros((25000, (max_len - ngram + 1) * ngram), dtype=np.int)
Xtest = np.zeros((25000, (max_len - ngram + 1) * ngram), dtype=np.int)
id_range = np.arange(max_len - ngram + 1)
for i in range(ngram):
Xtrain[:, id_range * ngram + i] = data1[:, id_range + i] + num_words * i
Xtest[:, id_range * ngram + i] = data2[:, id_range + i] + num_words * i
print('begin to build model ...')
main_input = Input(shape=((max_len - ngram + 1) * ngram,))
# embedding1 = Embedding(num_words * ngram, word_dim, embeddings_initializer=keras.initializers.Orthogonal())(main_input)
embedding1 = Embedding(num_words * ngram, word_dim)(main_input)
x = AveragePooling1D(pool_size=ngram)(embedding1)
x = GlobalMaxPooling1D()(x)
weight = np.ones((word_dim, 1), dtype=np.float)
weight[int(word_dim / 2):] = -1 * np.ones([int(word_dim / 2), 1], dtype=np.float)
output = Dense(1,
weights=[weight, np.zeros([1])],
trainable=False,
activation='sigmoid')(x)
model = Model(input=main_input, output=output)
model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])
model.fit([Xtrain], Ytrain,
batch_size=32,
shuffle=True,
nb_epoch=15,
verbose=2,
validation_data=([Xtest], Ytest))