def gen_test_tokenize(self, kwargs):
self.assertEqual(["asdf", "asdb"], fastText.tokenize("asdf asdb"))
self.assertEqual(["asdf"], fastText.tokenize("asdf"))
self.assertEqual([fastText.EOS], fastText.tokenize("\n"))
self.assertEqual(["asdf", fastText.EOS], fastText.tokenize("asdf\n"))
self.assertEqual([], fastText.tokenize(""))
self.assertEqual([], fastText.tokenize(" "))
# An empty string is not a token (it's just whitespace)
# So the minimum length must be 1
words = get_random_words(100, 1, 20)
self.assertEqual(words, fastText.tokenize(" ".join(words)))
def gen_test_tokenize(self, kwargs):
self.assertEqual(["asdf", "asdb"], fastText.tokenize("asdf asdb"))
self.assertEqual(["asdf"], fastText.tokenize("asdf"))
self.assertEqual([fastText.EOS], fastText.tokenize("\n"))
self.assertEqual(["asdf", fastText.EOS], fastText.tokenize("asdf\n"))
self.assertEqual([], fastText.tokenize(""))
self.assertEqual([], fastText.tokenize(" "))
# An empty string is not a token (it's just whitespace)
# So the minimum length must be 1
words = get_random_words(100, 1, 20)
self.assertEqual(words, fastText.tokenize(" ".join(words)))
def cleanData():
data = pd.read_csv("data.csv", sep=";")
data["text"] = data["text"].apply(str)
data["text"] = data["text"].apply(lambda x: x.lower())
# data["text"] = data["text"].apply(lambda x: re.sub("(\xc3\x84|ä)", "ae", x))
# data["text"] = data["text"].apply(lambda x: re.sub("(\xc3\x9c|ü)", "ue", x))
# data["text"] = data["text"].apply(lambda x: re.sub("(\xc3\xb6|ö)", "oe", x))
##see for tokenization rules: https://github.com/facebookresearch/fastText/blob/master/python/README.md
tokenizer = lambda x: " ".join(fastText.tokenize(x))
data["text"] = data["text"].apply(tokenizer)
##delete any non-alpha character
data["text"] = data["text"].apply(lambda x: re.sub("[^A-Za-z ]", "", x))
##delete if text empty
data = data[data["text"]!=""]
data.to_csv("data.csv", sep=";", index=False, encoding="utf-8", quoting=csv.QUOTE_ALL)
def _data_generator(self):
for i in range(len(self.X_train // self.opt['batch_size'])):
X_sample = self.X_train[i * self.opt['batch_size']:i *
self.opt['batch_size'] +
self.opt['batch_size']]
y_sample = self.y_train[i * self.opt['batch_size']:i *
self.opt['batch_size'] +
self.opt['batch_size']]
X_sample = [[
self.ft.get_word_vector(word)
for word in fastText.tokenize(sent)
] for sent in X_sample]
yield X_sample, y_sample
def get_word_vectors(text):
"""
Tokenize text and return fastText word vectors.
Requires a fastText model to be loaded (see :func:`load_fasttext_model`)
:param text: input text
:return: word vector matrix
"""
try:
matrix = [
_fasttext_model.get_word_vector(w)
for w in fastText.tokenize(util.normalize_message_text(text))
]
except Exception as e:
logger.error('Failed to tokenize line: {}'.format(e))
matrix = [get_word_vector('')]
if len(matrix) == 0:
matrix = [get_word_vector('')]
return np.array(matrix)
def get_word_vector(data, model):
t1 = time.time()
print("Reading")
with open(data, 'r') as f:
tokens = tokenize(f.read())
t2 = time.time()
print("Read TIME: " + str(t2 - t1))
print("Read NUM : " + str(len(tokens)))
f = load_model(model)
# This is not equivalent to piping the data into
# print-word-vector, because the data is tokenized
# first.
t3 = time.time()
i = 0
for t in tokens:
f.get_word_vector(t)
i += 1
if i % 10000 == 0:
sys.stderr.write("\ri: " + str(float(i / len(tokens))))
sys.stderr.flush()
t4 = time.time()
print("\nVectoring: " + str(t4 - t3))
def get_word_vector(data, model):
t1 = time.time()
print("Reading")
with open(data, 'r') as f:
tokens = tokenize(f.read())
t2 = time.time()
print("Read TIME: " + str(t2 - t1))
print("Read NUM : " + str(len(tokens)))
f = load_model(model)
# This is not equivalent to piping the data into
# print-word-vector, because the data is tokenized
# first.
t3 = time.time()
i = 0
for t in tokens:
f.get_word_vector(t)
i += 1
if i % 10000 == 0:
sys.stderr.write("\ri: " + str(float(i / len(tokens))))
sys.stderr.flush()
t4 = time.time()
print("\nVectoring: " + str(t4 - t3))
songs = pd.read_csv('../dataset/dataset.csv')
print("Loading the embedding model for querying...")
model = ft.load_model("./embeddings_lyrics.bin")
dim = model.get_dimension()
print("Loading the embeddings for songs to compare...")
embedding_of_lyrics = pd.read_csv('../database/song_embedings.csv').values[1:,
1:]
print("Making the KDTree tree...")
tree = KDTree(embedding_of_lyrics)
while (True):
query = input("Enter your query : ")
tokens = ft.tokenize(query)
#print(tokens)
# embedding matrix for the query
query_embedding_matrix = []
for x in tokens:
query_embedding_matrix.append(model.get_word_vector(x))
# embedding of the query - using avg method with fastText
embedding_of_query = model.get_word_vector(query)
# finding the num_search_results closest songs
# # finding the index
dist, ind = tree.query([embedding_of_query], k=num_search_results)
print(ind)
def test_tokenize(self):
train, _, _ = self.build_paths("fil9", "rw/rw.txt", "fil9")
with open(train, 'r') as f:
_ = tokenize(f.read())
def test_tokenize(self):
train, _, _ = self.build_paths("fil9", "rw/rw.txt", "fil9")
with open(train, 'r') as f:
_ = tokenize(f.read())