def stanford_parser(sentences, proc_id=0):
    # print('Im stanford parsing process'+proc_id)
    if os.path.exists('data/.tmp'+proc_id):
        shutil.rmtree('data/.tmp'+proc_id)
    os.makedirs('data/.tmp'+proc_id)  # create a temperal dir for parsing large paragraphs

    for sent_id in sentences.keys():
        f = io.open(os.path.join('data/.tmp'+proc_id, sent_id), 'w', -1, 'utf-8')
        f.write(sentences[sent_id])
        f.close()
    parsing_result = list(batch_parse('data/.tmp'+proc_id,
                                      os.path.join(os.environ['CN_SF_PATH'],
                                                   'externals/stanford-corenlp-full-2014-08-27/'),
                                      properties="StanfordCoreNLP-chinese.properties",
                                      ))
    result = dict()
    for r in parsing_result:
        result[r['file_name']] = r['sentences'][0]

    return result
def stanford_parser(sentences, proc_id=0):
    # print('Im stanford parsing process'+proc_id)
    if os.path.exists('data/.tmp'+proc_id):
        shutil.rmtree('data/.tmp'+proc_id)
    os.makedirs('data/.tmp'+proc_id)  # create a temperal dir for parsing large paragraphs

    for sent_id in sentences.keys():
        f = io.open(os.path.join('data/.tmp'+proc_id, sent_id), 'w', -1, 'utf-8')
        f.write(sentences[sent_id])
        f.close()
    stanford_parser_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                       '../../externals/stanford-corenlp-full-2014-08-27/')
    parsing_result = list(batch_parse('data/.tmp'+proc_id,
                                      stanford_parser_dir,
                                      properties=os.path.join(stanford_parser_dir, "StanfordCoreNLP-chinese.properties")
                                      ))
    result = dict()
    for r in parsing_result:
        result[r['file_name']] = r['sentences'][0]

    return result
    def evidence_extaction(self):
        # ************* batch segment long article ************* #
        start = time.time()
        if os.path.exists('data/.tmp/'):
            shutil.rmtree('data/.tmp')
        os.makedirs('data/.tmp/')  # create a temperal dir for parsing large paragraphs
        for doc_id in self.cleaned_docs:
            f = io.open(os.path.join('data/.tmp', doc_id), 'w', -1, 'utf-8')
            f.write(self.cleaned_docs[doc_id])
            f.close()

        # run stanford segmenter
        stanford_nlp_dir = os.path.join(self.CN_SF_PATH,
                                        'externals/stanford-corenlp-full-2014-08-27/')
        segmenter_result = list(batch_parse('data/.tmp/',
                                            stanford_nlp_dir,
                                            properties=os.path.join(stanford_nlp_dir,
                                                                    "StanfordCoreNLP-chinese.Segmenter.properties")
                                            ))
        for r in segmenter_result:
            self.segmented_docs[r['file_name']] = r['sentences']
        print('segmenting time cost '+str(time.time()-start))

        # cpickle for development
        # cPickle.dump(self.segmented_docs, open('data/segmented_docs.pkl', 'wb'))
        # self.segmented_docs = cPickle.load(open('data/segmented_docs.pkl', 'rb'))

        # ************* select evidence ************* #
        sent_to_parse = dict()

        self.evidence = OrderedDict()
        for query in self.queries:
            print('\textracting ' + query.name)

            evidences = OrderedDict()  # {slot_type: sentence_parsed_result}
            for doc_id in self.query_docs[query.id].keys():
                seg_result = self.segmented_docs[doc_id]
                for i in xrange(len(seg_result)):  # sentence is stanford standard format output
                    sentence = seg_result[i]
                    sent_id = '|'.join([doc_id, str(i)])
                    # if sentence is too long or too short, it carries less dependency information
                    if len(sentence['words']) > 130 or len(sentence['words']) < 3:
                        continue

                    sent_text = ''.join(sentence['text'])

                    # *************** check if this sentence is an evidence ******************** #
                    # ============== common case ============= #
                    seg_sent_text = sentence['text']  # list of tokens
                    seg_sent_text = [jianfan.ftoj(w) for w in seg_sent_text]

                    # here joining s['text'] list will overcome segmentation errors
                    if query.name not in ''.join(seg_sent_text):
                        continue

                    triggers = self.triggers[query.entity_type]

                    if query.entity_type == 'PER':
                        slot_types = self.PER_SLOT_TYPE
                    elif query.entity_type == 'ORG':
                        slot_types = self.ORG_SLOT_TYPE

                    for slot_type in slot_types:
                        if slot_type not in evidences.keys():
                            evidences[slot_type] = []
                        for t in triggers[slot_type]:
                            # compare triggers to words by segmentation, might affected by segmentation errors
                            if t not in seg_sent_text:
                                continue
                            evidences[slot_type].append(Evidence(doc_id, query.id, t, sent_text, sent_id))
                            sent_to_parse[sent_id] = sent_text  # add sentence and do parallel parsing later.

                    # ============== special case ============== #
                    if query.entity_type == 'PER':
                        evidences['per:alternate_names'].append(Evidence(doc_id, query.id, '',
                                                                         sent_text, sent_id, sentence))

                    if query.entity_type == 'ORG':
                        # for org:alternate_names, the article contains the query is evidence, for pattern match
                        evidences['org:alternate_names'].append(Evidence(doc_id, query.id, '',
                                                                         sent_text, sent_id, sentence))

                        # for org:XXX_headquarters, the article contains the query is evidence, for pattern match
                        evidences['org:country_of_headquarters'].append((Evidence(doc_id, query.id, '',
                                                                                  sent_text, sent_id, sentence)))
                        evidences['org:stateorprovince_of_headquarters'].append((Evidence(doc_id, query.id, '',
                                                                                          sent_text, sent_id, sentence)))
                        evidences['org:city_of_headquarters'].append((Evidence(doc_id, query.id, '',
                                                                               sent_text, sent_id, sentence)))

            self.evidence[query.id] = evidences

        # *************** parallel parsing ****************** #
        def chunkIt(seq, num):
            avg = len(seq) / float(num)
            out = []
            last = 0.0

            while last < len(seq):
                out.append(seq[int(last):int(last + avg)])
                last += avg

            return out

        # run stanford parser in multiprocessing
        process_num = multiprocessing.cpu_count() / 2 if multiprocessing.cpu_count() / 2 < 10 else 10
        p = multiprocessing.Pool(processes=process_num)
        chunked_sent = [dict(item) for item in chunkIt(sent_to_parse.items(), process_num)]
        mp_result = [p.apply_async(stanford_parser,
                                   args=(chunked_sent[i], str(i))) for i in range(process_num)]
        mp_result = [p.get() for p in mp_result]
        sent_parsing_result = {}
        for r in mp_result:
            sent_parsing_result.update(r)

        # cpickle for development
        # cPickle.dump(sent_parsing_result, open('data/sent_parsing_result.pkl', 'wb'))
        # sent_parsing_result = cPickle.load(open('data/sent_parsing_result.pkl', 'rb'))

        # updating evidences
        for q_id in self.evidence.keys():
            evidences = self.evidence[q_id]
            for slot_type in evidences.keys():
                for e in evidences[slot_type]:
                    if not e.trigger:
                        continue
                    e.parse_result = sent_parsing_result[e.sent_id]

        # *************** correct segmenter error ******************** #
        china_province_city = cPickle.load(open('data/dict/china_province_city.pkl', 'rb'))
        province_city_list = []
        for p in china_province_city:
            province_city_list += [p['name']]
            for c in p['sub']:
                province_city_list += [c['name']]
                if p['type'] == 0:
                    continue
                for d in c['sub']:
                    province_city_list += [d['name']]

        for q_id in self.evidence.keys():
            for slot_type in self.evidence[q_id]:
                for i in xrange(len(self.evidence[q_id][slot_type])):
                    self.evidence[q_id][slot_type][i] = self.correct_evidence(self.find_query(q_id).name,
                                                                              self.evidence[q_id][slot_type][i])
                    for p_or_c in province_city_list:
                        if len(p_or_c) > 2 and p_or_c in \
                                ''.join(self.evidence[q_id][slot_type][i].parse_result['text']):
                            self.evidence[q_id][slot_type][i] = \
                                self.correct_evidence(p_or_c, self.evidence[q_id][slot_type][i])

        print('Done')