def test_loaded_polyglot_embeddings(self):
data = pickle_call('data/embeddings/polyglot-en.pkl')
dl = DataLoader(embeddings_initial='Polyglot',
embedding_loading='in_dict')
dl.load('data/pickles/')
dl.get_all_and_dump('data/pickles/')
all_true = None
for i in range(len(data[0])):
term = data[0][i]
embedding = data[1][i]
if term in dl.embedding.vocab_dict:
position = dl.embedding.vocab_dict[term]
stored_embedding = dl.embedding.embeddings.weight[
position].data.numpy()
if all_true is None:
all_true = np.array_equal(embedding, stored_embedding)
else:
all_true = all_true and np.array_equal(
embedding, stored_embedding)
self.assertTrue(all_true)
def split_train_dev_test(path, nr_train, nr_dev, nr_test):
"""
Here we split the data sets based on the defined number of data points
:param path: path of the original data set
:param nr_train: number of data points to train on
:param nr_dev: number of data points to validate on
:param nr_test: number of data points to test on
:return:
"""
# retrieve the data set from file
data = pickle_call(path)
#shuffle it
random.shuffle(data)
# split it based on the defined numbers
train = data[:nr_train]
dev = data[nr_train:nr_train + nr_dev]
test = data[nr_train + nr_dev:nr_train + nr_dev + nr_test]
# dump it to file
pickle_dump(path + "_train", train)
pickle_dump(path + "_dev", dev)
pickle_dump(path + "_test", test)
def load_top_k(self, K, preload=False):
"""
Option for loading strategy: Only load top k of embeddings assuming that they are ordered in frequency
:param K: Number of top k embeddings to be retrieved
:return:embeddings matrix as numpy
"""
# This embedding dataset does not have PAD UNK START and END tokens pretrained that is why we initialize them
# ourselves and only load K - 4 embeddings
# we load the token dictionary from file
token_dict = pickle_call(self.path)
# to load only the top most frequent words, we sort them
token_list = [[token, counts] for token, counts in token_dict.items()]
token_list = sorted(token_list, key=lambda x: -x[1])
# because we also need to add special embeddings, we reduce K by 4
K = K - 4
# if K is defined larger than what is available, set K to the maximum length
K = min(K, len(token_list))
embeddings = []
# add all the special embeddings and define the one-hot-encoded positions
self.add_term("<S>", preload=preload)
embedding = np.zeros(K+4, dtype=np.float64)
embedding[0] = 1.0
embeddings.append(embedding)
self.add_term("</S>", preload=preload)
embedding = np.zeros(K+4, dtype=np.float64)
embedding[1] = 1.0
embeddings.append(embedding)
self.add_term("<PAD>", preload=preload)
embedding = np.zeros(K+4, dtype=np.float64)
embedding[2] = 1.0
embeddings.append(embedding)
self.add_term("<UNK>", preload=preload)
embedding = np.zeros(K+4, dtype=np.float64)
embedding[3] = 1.0
embeddings.append(embedding)
# then loop through the sorted tokens and also define the one-hot-encoded vectors
for k in range(K):
token = token_list[k][0]
self.add_term(token, preload=preload)
embedding = np.zeros(K + 4, dtype=np.float64)
embedding[k+4] = 1.0
embeddings.append(embedding)
# define the embedding size
self.embedding_size = K + 4
# return the embeddings
return np.array(embeddings)
def load(self, file_name):
"""
load all data from file if we have it
:param file_name: path and filename where it should be stored
:return:
"""
dump_dict = pickle_call(file_name)
if dump_dict is not None:
self.vocab_size = dump_dict['vocab_size']
self.vocab_list = dump_dict['vocab_list']
self.vocab_dict = dump_dict['vocab_dict']
self.embeddings = dump_dict['embeddings']
return True
return False
def load_data(self,
data_type='train',
directory=None,
initialize_term=False):
"""
This function loads all the data defined by the data set name
:param data_type: 'train', 'dev', 'test'
:param directory: DEPRECATED
:param initialize_term:
:return:
"""
if self.name == 'Language_Text_100':
path = '../data/data_set/data_set_100_' + data_type
if not os.path.isfile(path):
path = 'data/data_set/data_set_100_' + data_type
if not os.path.isfile(path):
raise Exception("tokens dont exist")
if self.name == 'Language_Text_1000':
path = '../data/data_set/data_set_1000_' + data_type
if not os.path.isfile(path):
path = 'data/data_set/data_set_1000_' + data_type
if not os.path.isfile(path):
raise Exception("tokens dont exist")
if self.name == 'Language_Text_10000':
path = '../data/data_set/data_set_10000_' + data_type
if not os.path.isfile(path):
path = 'data/data_set/data_set_10000_' + data_type
if not os.path.isfile(path):
raise Exception("tokens dont exist")
# retrieve the data set from disc
self.data_sets[data_type] = pickle_call(path)
# loop through every sentence and encode it using the embedding object
for elem in self.data_sets[data_type]:
line = elem['sentence']
elem['sentence_positions'] = self.embeddings.encode_sentence(
line, initialize=initialize_term, count_up=True)
return self.data_sets[data_type]
def __init__(self):
"""
This class calls the FastText data
"""
# Load the super class
super(PolyglotEmbeddings, self).__init__()
# Name of the embedding dataset
self.name = 'Polyglot'
path = '../data/embeddings/polyglot-en.pkl'
if not os.path.isfile(path):
path = 'data/embeddings/polyglot-en.pkl'
if not os.path.isfile(path):
raise Exception(
"please load Polyglot Embeddings from http://bit.ly/19bSoAS and store in data/embeddings/"
)
self.path = path
self.poly_data = pickle_call(self.path)
def load(self, file_name):
dump_dict = pickle_call(file_name)
if dump_dict is not None:
self.data_sets = dump_dict
return True
return False