def __init__(self, data_model):
"""
Iniital Setting
:param data_model: Setting Slack Model. Slack Model has the a lot of paramater
"""
self.slack_channel = data_model.slack_channel
self.data = ""
self.parameter = data_model.parameter_dict
self.model_name = "../model_word_match/ChainerDialogue"
self.generation_limit = 200
"""
We confirm channel number
https://api.slack.com/methods/channels.list
"""
self.chan = data_model.chan
self.usr = data_model.user
self.mecab_dict = data_model.mecab_dict
self.Mecab = MeCab.Tagger("-Owakati -d %s" % self.mecab_dict)
XP.set_library(False, 0)
self.XP = XP
wn_summary_list = APP_ROOT + '/../Data/wn_total_summary_51519_limit05_out_put_list.txt'
self.input_module = InputFileCython(wn_summary_list)
self.input_module.input_special_format_file()
file_list = self.input_module.get_file_data()
self.class_word_vector = self.__make_class_word_vector(file_list)
self.sqlite_twitter_summary = SqliteTwitterSummary(
self.class_word_vector)
self.word_class_dict = self.sqlite_twitter_summary.make_class_word_dict(
)
self.word_class = ""
self.multi_train_execute = ExecuteAttentionDialogue()
self.elastic_search = GetAnswer()
def setUp(self):
"""
setting initial paramater
Args:
data: test file name
split_module: setting the split_module instance
"""
wiki_vector_file_name = APP_ROOT + '/../../Data/jawiki_vector/jawiki_vector.txt'
self.input_module = InputFileCython(wiki_vector_file_name)
def get_file_data(self, file):
"""
get file data
:param file: summary word net data
:return:
"""
self.input_module = InputFileCython(file)
self.input_module.input_special_format_file()
return self.input_module.get_file_data()
def test_summary_class(self):
"""
test make summary dict
"""
file_list = self.input_module.get_file_data()
wn_average_vector_list = APP_ROOT + '/../../Data/wn_average_vector_list.txt'
self.input_module = InputFileCython(wn_average_vector_list)
self.input_module.input_special_format_file()
vector_list = self.input_module.get_file_data()
class_word_vector = {}
class_average_vector = {}
for file in file_list:
self.input_module = InputFileCython(APP_ROOT +
"/../../Data/wn_summary/" +
file.strip())
self.input_module.input_special_format_file()
if file.strip() not in class_word_vector:
word_list = re.sub(
"\]|\[|\'", "",
self.input_module.get_file_data()[0].strip())
class_word_vector.update(
{file.strip().replace(".txt", ""): word_list.split(",")})
for vector in vector_list:
self.input_module = InputFileCython(
APP_ROOT + "/../../Data/wn_summary_multi/" + vector.strip())
self.input_module.input_special_format_file()
vector_list = []
if vector.strip() not in class_average_vector:
for value in self.input_module.get_file_data():
value = re.sub("\]|\[|\'", "", value.strip())
[
vector_list.append(each_value)
for each_value in value.split(" ") if each_value != ""
]
vector_list = list(map(float, vector_list))
class_average_vector.update({
vector.strip().replace(".txt_vector.txt", ""):
vector_list
})
class_summary_cosine_similarity_cython = ClassSummaryCosineSimilarityCython(
class_word_vector, class_average_vector)
class_summary_cosine_similarity_cython.summary_class_use_cosine_similarity(
)
def setUp(self):
"""
setting initial paramater
Args:
data: test file name
split_module: setting the split_module instance
"""
wn_summary_list = APP_ROOT + '/../../Data/wn_summary_list.txt'
self.input_module = InputFileCython(wn_summary_list)
self.input_module.input_special_format_file()
def __initial_setting_vector(self):
# Wiki vector dict
wiki_vector_file_name = APP_ROOT + '/../Data/jawiki_vector/jawiki_vector_delete_first.txt'
self.input_module = InputFileCython(wiki_vector_file_name)
self.input_module.input_fast_large_file()
self.wiki_vector = self.input_module.get_vector()
# Make average vector dict
wiki_average_vector_file_name_list = APP_ROOT + '/../Data/wn_summary_multi_vector_list.txt'
self.input_module = InputFileCython(wiki_average_vector_file_name_list)
self.input_module.input_special_format_file()
summary_vector_file_list = self.input_module.get_file_data()
for file in summary_vector_file_list:
read_file = APP_ROOT + "/../Data/wn_summary_multi_vector/" + file
self.input_module = InputFileCython(read_file)
self.input_module.input_file_str_list()
summary_vector_file_list = self.input_module.get_file_data()
class_name = file.replace("_summary.txt_vector.txt", "")
if class_name not in self.class_average_vector:
self.class_average_vector.update(
{class_name: summary_vector_file_list})
self.class_summary = ClassSummary("", self.wiki_vector, "")
self.cosine_similarity = ClassCosineSimilarity("", "")
def test_summary_class(self):
"""
test make summary dict
"""
file_list = self.input_module.get_file_data()
OUT_PUT_PATH = APP_ROOT + "/../../Data/wn_total_summary_51519_limit05_out_put/"
for file in file_list:
self.input_module = InputFileCython(
APP_ROOT + "/../../Data/wn_total_summary_51519_limit05/" +
file.strip())
self.input_module.input_special_format_file()
self.class_summary_exclude.exclude_data(
OUT_PUT_PATH, file.strip(), self.input_module.get_file_data())
def test_summary_class(self):
"""
test make summary dict
"""
self.input_module.input_fast_large_file()
wiki_vector = self.input_module.get_vector()
wn_summary_list = APP_ROOT + '/../../Data/wn_summary_list.txt'
self.input_module = InputFileCython(wn_summary_list)
self.input_module.input_special_format_file()
file_list = self.input_module.get_file_data()
count = 0
class_word_vector = {}
class_average_vector = {}
for file in file_list:
self.input_module = InputFileCython(APP_ROOT +
"/../../Data/wn_summary/" +
file.strip())
self.input_module.input_special_format_file()
if count == 0:
class_summary = ClassSummaryCython(
file.strip(), self.input_module.get_file_data(),
wiki_vector)
else:
class_summary = ClassSummaryCython(
file.strip(), self.input_module.get_file_data(),
wiki_vector, class_word_vector, class_average_vector)
class_word_vector, class_average_vector = class_summary.summary_class(
)
fo = open(
APP_ROOT + "/../../Data/test/" + file.strip() + "_vector.txt",
'w')
sys.stdout = fo
print(class_average_vector[file.strip()])
fo.close()
sys.stdout = sys.__stdout__
class_summary_cosine_similarity_cython = ClassSummaryCosineSimilarityCython(
class_average_vector)
class_summary_cosine_similarity_cython.summary_class_use_cosine_similarity(
)
def test_summary_class(self):
"""
test make summary dict
"""
file_list = self.input_module.get_file_data()
class_word_vector = {}
for file in file_list:
self.input_module = InputFileCython(APP_ROOT + "/../../Data/wn_total_summary_51519_limit05_out_put/" + file.strip())
self.input_module.input_special_format_file()
if file.strip() not in class_word_vector:
word_list = (list(map(lambda x:x.strip(), self.input_module.get_file_data())))
class_word_vector.update({file.strip().replace("_summary.txt", ""): word_list})
sqlite_twitter_cython = SqliteTwitterSummaryCython(class_word_vector)
sqlite_twitter_cython.call_sql()
def __make_class_word_vector(self, file_list):
"""
Make class word vector dict
:param file_list:
:return:
"""
class_word_vector = {}
for file in file_list:
self.input_module = InputFileCython(
APP_ROOT +
"/../Data/wn_total_summary_51519_limit05_out_put//" +
file.strip())
self.input_module.input_special_format_file()
if file.strip() not in class_word_vector:
word_list = (list(
map(lambda x: x.strip(),
self.input_module.get_file_data())))
class_word_vector.update(
{file.strip().replace("_summary.txt", ""): word_list})
return class_word_vector
if __name__ == '__main__':
"""
args
-r: setting word net link list
Example:
'/../../Data/wn_summary_list.txt'
"""
parser = argparse.ArgumentParser()
parser.add_argument('--read_word_net_list_file',
'-r',
default='',
help='set word net list file')
args = parser.parse_args()
# Word Net File
wn_summary_split_list = APP_ROOT + "/../../Data/" + args.read_word_net_list_file
input_module = InputFileCython(wn_summary_split_list)
input_module.input_special_format_file()
file_list = input_module.get_file_data()
# Wiki Vector
wiki_vector_file_name = APP_ROOT + '/../../Data/jawiki_vector/jawiki_vector.txt'
input_module = InputFileCython(wiki_vector_file_name)
input_module.input_fast_large_file()
wiki_vector = input_module.get_vector()
producerConsumer = ProducerConsumerClassSummary()
multi_thread_producer_crawl_instance = threading.Thread(
target=producerConsumer.producer_run, args=([file_list]))
multi_thread_consumer_crawl_instance = threading.Thread(
target=producerConsumer.consumer_run, args=([wiki_vector]))
multi_thread_producer_crawl_instance.start()
multi_thread_consumer_crawl_instance.start()