Ejemplo n.º 1
0
 def __init__(self, data_model):
     """
     Iniital Setting
     :param data_model: Setting Slack Model. Slack Model has the a lot of paramater
     """
     self.slack_channel = data_model.slack_channel
     self.data = ""
     self.parameter = data_model.parameter_dict
     self.model_name = "../model_word_match/ChainerDialogue"
     self.generation_limit = 200
     """
     We confirm channel number
     https://api.slack.com/methods/channels.list
     """
     self.chan = data_model.chan
     self.usr = data_model.user
     self.mecab_dict = data_model.mecab_dict
     self.Mecab = MeCab.Tagger("-Owakati -d %s" % self.mecab_dict)
     XP.set_library(False, 0)
     self.XP = XP
     wn_summary_list = APP_ROOT + '/../Data/wn_total_summary_51519_limit05_out_put_list.txt'
     self.input_module = InputFileCython(wn_summary_list)
     self.input_module.input_special_format_file()
     file_list = self.input_module.get_file_data()
     self.class_word_vector = self.__make_class_word_vector(file_list)
     self.sqlite_twitter_summary = SqliteTwitterSummary(
         self.class_word_vector)
     self.word_class_dict = self.sqlite_twitter_summary.make_class_word_dict(
     )
     self.word_class = ""
     self.multi_train_execute = ExecuteAttentionDialogue()
     self.elastic_search = GetAnswer()
Ejemplo n.º 2
0
 def setUp(self):
     """
     setting initial paramater
     Args:
         data: test file name
         split_module: setting the split_module instance
     """
     wiki_vector_file_name = APP_ROOT + '/../../Data/jawiki_vector/jawiki_vector.txt'
     self.input_module = InputFileCython(wiki_vector_file_name)
 def get_file_data(self, file):
     """
     get file data
     :param file: summary word net data
     :return:
     """
     self.input_module = InputFileCython(file)
     self.input_module.input_special_format_file()
     return self.input_module.get_file_data()
 def test_summary_class(self):
     """
     test make summary dict
     """
     file_list = self.input_module.get_file_data()
     wn_average_vector_list = APP_ROOT + '/../../Data/wn_average_vector_list.txt'
     self.input_module = InputFileCython(wn_average_vector_list)
     self.input_module.input_special_format_file()
     vector_list = self.input_module.get_file_data()
     class_word_vector = {}
     class_average_vector = {}
     for file in file_list:
         self.input_module = InputFileCython(APP_ROOT +
                                             "/../../Data/wn_summary/" +
                                             file.strip())
         self.input_module.input_special_format_file()
         if file.strip() not in class_word_vector:
             word_list = re.sub(
                 "\]|\[|\'", "",
                 self.input_module.get_file_data()[0].strip())
             class_word_vector.update(
                 {file.strip().replace(".txt", ""): word_list.split(",")})
     for vector in vector_list:
         self.input_module = InputFileCython(
             APP_ROOT + "/../../Data/wn_summary_multi/" + vector.strip())
         self.input_module.input_special_format_file()
         vector_list = []
         if vector.strip() not in class_average_vector:
             for value in self.input_module.get_file_data():
                 value = re.sub("\]|\[|\'", "", value.strip())
                 [
                     vector_list.append(each_value)
                     for each_value in value.split(" ") if each_value != ""
                 ]
                 vector_list = list(map(float, vector_list))
             class_average_vector.update({
                 vector.strip().replace(".txt_vector.txt", ""):
                 vector_list
             })
     class_summary_cosine_similarity_cython = ClassSummaryCosineSimilarityCython(
         class_word_vector, class_average_vector)
     class_summary_cosine_similarity_cython.summary_class_use_cosine_similarity(
     )
 def setUp(self):
     """
     setting initial paramater
     Args:
         data: test file name
         split_module: setting the split_module instance
     """
     wn_summary_list = APP_ROOT + '/../../Data/wn_summary_list.txt'
     self.input_module = InputFileCython(wn_summary_list)
     self.input_module.input_special_format_file()
Ejemplo n.º 6
0
 def __initial_setting_vector(self):
     # Wiki vector dict
     wiki_vector_file_name = APP_ROOT + '/../Data/jawiki_vector/jawiki_vector_delete_first.txt'
     self.input_module = InputFileCython(wiki_vector_file_name)
     self.input_module.input_fast_large_file()
     self.wiki_vector = self.input_module.get_vector()
     # Make average vector dict
     wiki_average_vector_file_name_list = APP_ROOT + '/../Data/wn_summary_multi_vector_list.txt'
     self.input_module = InputFileCython(wiki_average_vector_file_name_list)
     self.input_module.input_special_format_file()
     summary_vector_file_list = self.input_module.get_file_data()
     for file in summary_vector_file_list:
         read_file = APP_ROOT + "/../Data/wn_summary_multi_vector/" + file
         self.input_module = InputFileCython(read_file)
         self.input_module.input_file_str_list()
         summary_vector_file_list = self.input_module.get_file_data()
         class_name = file.replace("_summary.txt_vector.txt", "")
         if class_name not in self.class_average_vector:
             self.class_average_vector.update(
                 {class_name: summary_vector_file_list})
     self.class_summary = ClassSummary("", self.wiki_vector, "")
     self.cosine_similarity = ClassCosineSimilarity("", "")
 def test_summary_class(self):
     """
     test make summary dict
     """
     file_list = self.input_module.get_file_data()
     OUT_PUT_PATH = APP_ROOT + "/../../Data/wn_total_summary_51519_limit05_out_put/"
     for file in file_list:
         self.input_module = InputFileCython(
             APP_ROOT + "/../../Data/wn_total_summary_51519_limit05/" +
             file.strip())
         self.input_module.input_special_format_file()
         self.class_summary_exclude.exclude_data(
             OUT_PUT_PATH, file.strip(), self.input_module.get_file_data())
Ejemplo n.º 8
0
 def test_summary_class(self):
     """
     test make summary dict
     """
     self.input_module.input_fast_large_file()
     wiki_vector = self.input_module.get_vector()
     wn_summary_list = APP_ROOT + '/../../Data/wn_summary_list.txt'
     self.input_module = InputFileCython(wn_summary_list)
     self.input_module.input_special_format_file()
     file_list = self.input_module.get_file_data()
     count = 0
     class_word_vector = {}
     class_average_vector = {}
     for file in file_list:
         self.input_module = InputFileCython(APP_ROOT +
                                             "/../../Data/wn_summary/" +
                                             file.strip())
         self.input_module.input_special_format_file()
         if count == 0:
             class_summary = ClassSummaryCython(
                 file.strip(), self.input_module.get_file_data(),
                 wiki_vector)
         else:
             class_summary = ClassSummaryCython(
                 file.strip(), self.input_module.get_file_data(),
                 wiki_vector, class_word_vector, class_average_vector)
         class_word_vector, class_average_vector = class_summary.summary_class(
         )
         fo = open(
             APP_ROOT + "/../../Data/test/" + file.strip() + "_vector.txt",
             'w')
         sys.stdout = fo
         print(class_average_vector[file.strip()])
         fo.close()
         sys.stdout = sys.__stdout__
     class_summary_cosine_similarity_cython = ClassSummaryCosineSimilarityCython(
         class_average_vector)
     class_summary_cosine_similarity_cython.summary_class_use_cosine_similarity(
     )
 def test_summary_class(self):
     """
     test make summary dict
     """
     file_list = self.input_module.get_file_data()
     class_word_vector = {}
     for file in file_list:
         self.input_module = InputFileCython(APP_ROOT + "/../../Data/wn_total_summary_51519_limit05_out_put/" + file.strip())
         self.input_module.input_special_format_file()
         if file.strip() not in class_word_vector:
             word_list = (list(map(lambda x:x.strip(), self.input_module.get_file_data())))
             class_word_vector.update({file.strip().replace("_summary.txt", ""): word_list})
     sqlite_twitter_cython = SqliteTwitterSummaryCython(class_word_vector)
     sqlite_twitter_cython.call_sql()
Ejemplo n.º 10
0
 def __make_class_word_vector(self, file_list):
     """
     Make class word vector dict
     :param file_list:
     :return:
     """
     class_word_vector = {}
     for file in file_list:
         self.input_module = InputFileCython(
             APP_ROOT +
             "/../Data/wn_total_summary_51519_limit05_out_put//" +
             file.strip())
         self.input_module.input_special_format_file()
         if file.strip() not in class_word_vector:
             word_list = (list(
                 map(lambda x: x.strip(),
                     self.input_module.get_file_data())))
             class_word_vector.update(
                 {file.strip().replace("_summary.txt", ""): word_list})
     return class_word_vector
if __name__ == '__main__':
    """
    args
       -r: setting word net link list
           Example:
               '/../../Data/wn_summary_list.txt'
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--read_word_net_list_file',
                        '-r',
                        default='',
                        help='set word net list file')
    args = parser.parse_args()
    #  Word Net File
    wn_summary_split_list = APP_ROOT + "/../../Data/" + args.read_word_net_list_file
    input_module = InputFileCython(wn_summary_split_list)
    input_module.input_special_format_file()
    file_list = input_module.get_file_data()
    # Wiki Vector
    wiki_vector_file_name = APP_ROOT + '/../../Data/jawiki_vector/jawiki_vector.txt'
    input_module = InputFileCython(wiki_vector_file_name)
    input_module.input_fast_large_file()
    wiki_vector = input_module.get_vector()

    producerConsumer = ProducerConsumerClassSummary()
    multi_thread_producer_crawl_instance = threading.Thread(
        target=producerConsumer.producer_run, args=([file_list]))
    multi_thread_consumer_crawl_instance = threading.Thread(
        target=producerConsumer.consumer_run, args=([wiki_vector]))
    multi_thread_producer_crawl_instance.start()
    multi_thread_consumer_crawl_instance.start()