def load_result(self, sim_name, dataset_name):
"""
This function loads the result of a similarity metric for a specific dataset
:param sim_name: the name similarity metric
:param dataset_name: the name of word similarity dataset
:return: cor relation score and rating scores generated by similarity metric
"""
data = FileIO.read_list_file('dataset/wordsim/results/%s-%s.txt' % (dataset_name, sim_name))
data = list(map(float, data))
return data[0], data[1:]
def load_result(self, sim_name, dataset_name):
"""
This function loads the result of a similarity metric for a specific dataset
:param sim_name: the name similarity metric
:param dataset_name: the name of word similarity dataset
:return: cor relation score and rating scores generated by similarity metric
"""
data = FileIO.read_list_file('eval/word_similarity/results/%s-%s.txt' % (dataset_name, sim_name))
data = map(float, data)
return data[0], data[1:]
def load_dataset(self, dataset_name):
"""
This function loads the word similarity dataset
:param dataset_name: the file name of word similarity dataset
:return: word pairs and huamn ratings
"""
data = FileIO.read_list_file('dataset/wordsim/%s.txt' % dataset_name)
#print "dataset ", dataset_name, " ", len(data), " word pairs"
word_pairs = map(lambda x: (x.split()[0], x.split()[1]), data)
human = list(map(float, map(lambda x: x.split()[2], data)))
return word_pairs, human
def load_dataset(self, dataset_name):
"""
This function loads the word similarity dataset
:param dataset_name: the file name of word similarity dataset
:return: word pairs and huamn ratings
"""
data = FileIO.read_list_file('eval/word_similarity/%s.txt' % dataset_name)
#print "dataset ", dataset_name, " ", len(data), " word pairs"
word_pairs = map(lambda x: (x.split()[0], x.split()[1]), data)
human = map(float, map(lambda x: x.split()[2], data))
return word_pairs, human
def load_dataset(self, dataset_file):
"""
Generate sentence pairs.
:param dataset_file: dataset file
:return: sentence pairs
"""
data = FileIO.read_list_file(dataset_file)
data = [d.strip() for d in data]
corpus = []
for d in data:
item = d.split('\t')
corpus.append((item[0], item[1]))
return corpus
def load_dataset(self, dataset_file):
"""
Generate sentence pairs.
:param dataset_file: dataset file
:return: sentence pairs
"""
data = FileIO.read_list_file(dataset_file)
data = [d.strip() for d in data]
corpus = []
for d in data:
item = d.split('\t')
corpus.append((item[0], item[1]))
return corpus
def load_stopwords(self, filename):
data = FileIO.read_list_file(FileIO.filename(filename))
data = [d.split() for d in data[1:]] # skip first line, in case more than one word per line
data = list(itertools.chain.from_iterable(data))
return data
def load_stopwords(self, filename):
data = FileIO.read_list_file(FileIO.filename(filename))
data = [d.split() for d in data[1:]] # skip first line, in case more than one word per line
data = list(itertools.chain.from_iterable(data))
return data
