def __init__(
self,
sequences: List[List[int]],
num_words: int,
window_size: int = 5,
batch_size: int = 32,
):
super().__init__(sequences, window_size, batch_size)
self._num_words = num_words
self._sampling_table = sequence.make_sampling_table(size=num_words)
def create_dataset(text, vocab, num_words, window_size, negative_samples):
data = vocab.texts_to_sequences([text]).pop()
sampling_table = make_sampling_table(num_words)
couples, labels = skipgrams(data, num_words,
window_size=window_size,
negative_samples=negative_samples,
sampling_table=sampling_table)
word_target, word_context = zip(*couples)
word_target = np.reshape(word_target, (-1, 1))
word_context = np.reshape(word_context, (-1, 1))
labels = np.asfarray(labels)
return [word_target, word_context], labels
def training_data_generator(text_encoded,
window_size=4,
negative_samples=1.0,
batch_docs=50):
"""
For given encoded text, return 3 np.array:
words, contexts, labels
Do not pair the w and its context cross different documents.
input:
text_encoded: list of list of int, each list of int is the numerical encoding of the doc
window_size: int, define the context
negative_samples: float, how much negative sampling you need, normally 1.0
batch_docs: int, number of docs for which it generates one return
return:
words: list of int, the numerical encoding of the central words
contexts: list of int, the numerical encoding of the context words
labels: list of int, 1 or 0
hint:
1. You can use skipgrams method from keras
2. For training purpose, words and contexts needs to be 2D array, with shape (N, 1),
but labels is 1D array, with shape (N, )
3. The output can be very big, you SHOULD using generator
"""
"""
Write your code here
"""
sampling_table = make_sampling_table(VOCAB_SIZE)
loc = list(range(len(text_encoded)))
random.shuffle(loc)
for j in loc[:batch_docs]:
couples, label = skipgrams(text_encoded[j],
VOCAB_SIZE,
window_size=window_size,
sampling_table=sampling_table,
negative_samples=negative_samples,
shuffle=True)
if len(couples) > 0:
target, context_ = zip(*couples)
target = np.array(target, dtype="int32")
context_ = np.array(context_, dtype="int32")
yield target.tolist(), context_.tolist(), label
else:
continue
def downsample_skipgrams(ids, vocab_size, subsample=1e-3, window=2, neg=2):
w = []
y = []
sampling_table = make_sampling_table(vocab_size, sampling_factor=subsample)
span = 2 * window + 1
targets = ids[window::span]
pairs, labels = skipgrams(ids,
vocabulary_size=vocab_size,
window_size=np.random.randint(window - 1) + 1,
negative_samples=neg,
sampling_table=sampling_table,
shuffle=True)
for (t, c), l in zip(pairs, labels):
if t in targets:
w.append([t, c])
y.append(l)
return w, y
class Word2VecDataGenerator(DataGenerator):
def __init__(self, config, language):
self.language = language
super(Word2VecDataGenerator, self).__init__(config)
def _generate(self, case=None):
vocabulary_size = self.config["params"]["vocab_size"]
window_size = self.config["params"]["word2vec"]["window_size"]
batch_size = self.config["hyper_params"]["word2vec"]["batch_size"]
sampling_table = sequence.make_sampling_table(vocabulary_size, 0.1)
while True:
pairs = []
labels = []
indexes = np.arange(len(self.sentences))
np.random.shuffle(indexes)
for i in indexes:
p, l = sequence.skipgrams(self.sentences[i],
vocabulary_size,
window_size=window_size,
sampling_table=sampling_table)
pairs.extend(p)
labels.extend(l)
if len(pairs) >= batch_size:
output_pairs = pairs[:batch_size]
output_labels = labels[:batch_size]
pairs = pairs[batch_size:]
labels = labels[batch_size:]
output_targets, output_contexts = zip(*output_pairs)
output_targets = np.array(output_targets, dtype=np.int32)
output_contexts = np.array(output_contexts, dtype=np.int32)
output_labels = np.array(output_labels, dtype=np.float32)
yield (output_targets, output_contexts), output_labels
def create_skipgrams(cls,
normalized_doc,
vocabulary_size=10000,
ratio=3.0):
'''
Create the skipgrams to be trained on the model
normalized_doc: Normalized document nested array of mapped sentences.
vocabulary_size: Size of the given vocabulary data has been compiled against.
ratio: Negative to Positive sampling ratio.
'''
# Used for generating the sampling_table argument for skipgrams. sampling_table[i] is the
# probability of sampling the word i-th most common word in a dataset (more common words
# should be sampled less frequently, for balance).
sampling_table = sequence.make_sampling_table(vocabulary_size)
# Create Skipgrams with Keras
# This function transforms a sequence of word indexes (list of integers) into tuples of
# words of the form:
# * (word, word in the same window), with label 1 (positive samples).
# * (word, random word from the vocabulary), with label 0 (negative samples).
# Flatten normalized document
data = list(itertools.chain.from_iterable(normalized_doc))
couples, labels = skipgrams(data,
vocabulary_size,
negative_samples=ratio,
sampling_table=sampling_table)
# Split couples into target and context
word_target, word_context = zip(*couples)
# Convert to Numpy array, ! rank 1 array!
word_target = np.array(word_target, dtype="int32")
word_context = np.array(word_context, dtype="int32")
return word_target, word_context, labels
vector_dim = 128 # dimensionality of the embedding vector
epochs = 10 ** 6 # one million epochs
negative_samples = 5
#validation constants
valid_size = 16
valid_window = 100
#valid_examples = np.random.choice(valid_window, valid_size, replace=False)
#['WAT', 'SWKS', 'PAYX', 'CSCO', 'RF', 'NKE', 'INTC', 'ALL', 'GS', 'AVGO', 'HUM', 'NEE', 'T', 'CL', 'DVA', 'AMGN']
custom_examples = ['CSCO', 'NKE', 'INTC', 'GS', 'T', 'TSLA', 'AAPL', 'PAYX']
valid_examples = [tick_to_idx[tick] for tick in custom_examples]
#picks {16} of the first {100} words for validation
#may need to replace this with some pure play companies like TSLA compared to Ford
#sampling table for negative examples
sampling_table = sequence.make_sampling_table(vocab_size + 1) #+1 due to index zero being skipped
#function to create skipgrams per ETF
targets, contexts, labels = [], [], []
for etf in etf_names:
tokens = np.array([tick_to_idx[tick] for tick in df.loc[df['ETF'] == etf, 'Ticker'].values])
etf_couples, etf_labels = skipgrams(
tokens,
vocab_size,
window_size = window_size,
negative_samples = negative_samples,
sampling_table = sampling_table)
etf_targets, etf_contexts = zip(*etf_couples) #separate into target and contexts by etf
targets.append(np.array(etf_targets))
contexts.append(np.array(etf_contexts))
labels.append(np.array(etf_labels))
def word2vec(self, WINDOW_SZ, EMBEDDING_DIM, W2V_EPOCHS):
print("WORD2VEC...")
valid_size = 20 #random word set to evaluate similarity
valid_window = 500 #pick samples in 500 most common words
valid_examples = np.random.choice(valid_window,
valid_size,
replace=False)
vocab_size = self.vocab_size
## skipgram set up
sampling_table = sequence.make_sampling_table(vocab_size,
sampling_factor=0.01)
skipgrams = [
sequence.skipgrams(tweet,
vocab_size,
window_size=WINDOW_SZ,
sampling_table=sampling_table)
for tweet in self.x_train
]
couples, labels = skipgrams[0][0], skipgrams[0][1]
word_target, word_context = zip(*couples)
word_target = np.array(word_target, dtype="int32")
word_context = np.array(word_context, dtype="int32")
## Functional API model
#input layers take in target and context word as ints
input_target = layers.Input((1, ))
input_context = layers.Input((1, ))
#embedding layer then transpose vectors to take dot prod
embedding = Embedding(vocab_size,
EMBEDDING_DIM,
input_length=1,
name='embedding')
target = embedding(input_target)
target = Reshape((EMBEDDING_DIM, 1))(target)
context = embedding(input_context)
context = Reshape((EMBEDDING_DIM, 1))(context)
#cosine similarity to be used in validation model
similarity = Dot(axes=0, normalize=True)
#dot product layers to measure similarity
dot_product = Dot(axes=1)([target, context])
dot_product = Reshape((1, ))(dot_product)
#sigmoid output layer
output = Dense(1, activation='sigmoid')(dot_product)
model = Model(inputs=[input_target, input_context], outputs=output)
model.compile(loss='binary_crossentropy', optimizer='rmsprop')
#cosine similarity to be used in validation model
similarity = Dot(axes=1, normalize=True)([target, context])
validation_model = Model(inputs=[input_target, input_context],
outputs=similarity)
reversed_word_index = self.reversed_word_index
## Helper class for validating Word2Vec while training
class SimilarityCallback:
def run_sim(self):
for i in range(valid_size):
valid_word = reversed_word_index[valid_examples[i]]
top_k = 8 #num of nearest neighbors
sim = self._get_sim(valid_examples[i])
nearest = (-sim).argsort()[1:top_k + 1]
log_str = 'Nearest to %s:' % valid_word
for k in range(top_k):
close_word = reversed_word_index[nearest[k]]
log_str = '%s %s,' % (log_str, close_word)
print(log_str)
@staticmethod
def _get_sim(valid_word_idx):
sim = np.zeros((vocab_size, ))
in_arr1 = np.zeros((1, ))
in_arr2 = np.zeros((1, ))
for i in range(vocab_size):
in_arr1[0, ] = valid_word_idx
in_arr2[0, ] = i
out = validation_model.predict_on_batch([in_arr1, in_arr2])
sim[i] = out
return sim
sim_cb = SimilarityCallback()
arr_1 = np.zeros((1, ))
arr_2 = np.zeros((1, ))
arr_3 = np.zeros((1, ))
## Train network
for cnt in range(W2V_EPOCHS):
idx = np.random.randint(0, len(labels) - 1)
arr_1[0, ] = word_target[idx]
arr_2[0, ] = word_context[idx]
arr_3[0, ] = labels[idx]
loss = model.train_on_batch([arr_1, arr_2], arr_3)
# Every 100 epochs print loss
if cnt % 100 == 0:
print("Iteration {}, loss={}".format(cnt, loss))
# Every 500 run similarity test on validation data
if cnt % 500 == 0:
sim_cb.run_sim()
tokenizer = tensorflow.keras.preprocessing.text.Tokenizer(num_words=vocab_size,
oov_token="<OOV>")
tokenizer.fit_on_texts(training_sentences)
words = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(training_sentences)
padded = tensorflow.keras.preprocessing.sequence.pad_sequences(
sequences, maxlen=120, truncating="post")
word_target_final = []
word_context_final = []
couples_final = []
labels_final = []
for i in range(1, int(len(padded) / 100)):
sampling_table = sequence.make_sampling_table(vocab_size)
couples, labels = sequence.skipgrams(padded[i], vocab_size, window_size=2)
word_target, word_context = zip(*couples)
word_target = np.array(word_target, dtype="int32")
word_context = np.array(word_context, dtype="int32")
labels_final.append(labels)
word_target_final.append(word_target)
word_context_final.append(word_context)
input_target = tensorflow.keras.layers.Input((1, ))
input_context = tensorflow.keras.layers.Input((1, ))
embedding = tensorflow.keras.layers.Embedding(vocab_size,
vector_dim,
input_length=1,
name='embedding')