def get_court2nd(self, content): """获取法院""" court = '' content = unicode(content) res = toolsutil.re_findone(self.court_pattern2nd, content) if not res: res = toolsutil.re_findone(self.court_pattern2nd, content) if res: court = res return court
def get_money_list(self, judge_content): '''获取涉案最大金额''' tmp_max_money_list = toolsutil.my_split(judge_content, [',', ',', '。']) ret_list = toolsutil.re_findall(self.money_regex, unicode(judge_content)) money_list = [] for row_content in tmp_max_money_list: ret_chs = toolsutil.re_findone(self.money_regex_chs, unicode(row_content)) if ret_chs: chs_money = self.money_parser.trans_chs_money(ret_chs) money_list.append(float(chs_money[0])) if ret_list: for ret in ret_list: digit_money = self.money_parser.transfer_money(ret) money_list.append(float(digit_money[0])) if money_list == []: ret_list2 = toolsutil.re_findall(self.money_regex_last, unicode(judge_content)) if ret_list2: for ret in ret_list2: digit_money = self.money_parser.transfer_money(ret) money_list.append(float(digit_money[0])) if money_list != []: max_money = max(money_list) sum_money = sum(money_list) else: max_money = 0 sum_money = 0 return max_money, sum_money
def format_extract_data(self, extract_data, topic_id): '''解析数据''' entity_data = copy.deepcopy(extract_data) tmp_max_money = extract_data.get("duty", "") tmp_max_money = unicode(tmp_max_money.replace(" ", "")) tmp_max_money_list = toolsutil.my_split(tmp_max_money, [',', ',', '。']) ret_list = toolsutil.re_findall(self.money_regex, tmp_max_money) money_list = [] for row_content in tmp_max_money_list: ret_chs = toolsutil.re_findone(self.money_regex_chs, unicode(row_content)) if ret_chs: chs_money = self.parser_tool.money_parser.trans_chs_money( ret_chs) money_list.append(float(chs_money[0])) if ret_list: for ret in ret_list: digit_money = self.parser_tool.money_parser.transfer_money(ret) money_list.append(float(digit_money[0])) if money_list == []: ret_list2 = toolsutil.re_findall(self.money_regex_last, tmp_max_money) if ret_list2: for ret in ret_list2: digit_money = self.parser_tool.money_parser.transfer_money( ret) money_list.append(float(digit_money[0])) if money_list == []: ret3 = toolsutil.re_findone(self.money_regex3, tmp_max_money) if ret3 and ret3 == tmp_max_money: money_list.append(float(ret3)) if money_list != []: max_money = max(money_list) sum_money = sum(money_list) else: max_money = 0 sum_money = 0 court = entity_data.get("court", "") province = self.parser_tool.province_parser.get_province(court) entity_data["max_money"] = max_money entity_data["sum_money"] = sum_money entity_data["province"] = province return entity_data
def get_court(self, content): '''获取法院''' court = self.parser_tool.court_parser.get_court(content) if not court: ret = toolsutil.re_findone(self.court_regex, content) if ret: court = ret else: court = "" return court
def format_extract_data(self, extract_data, topic_id): '''实体解析抽取数据''' entity_data = copy.deepcopy(extract_data) if entity_data.has_key("code"): code_content = entity_data.get("code") ret = toolsutil.re_findone(self.public_sector_regex, code_content) if ret: entity_data["public_sector"] = self.public_sector_dict.get(ret) if entity_data.has_key("info"): entity_data.pop("info") for item in extract_data["info"]: for key, value in item.items(): base_key = key[:3] base_value = "value" + key[3:] if base_key == "key" and item.has_key(base_value): key = item[key].encode("utf8") value = item[base_value].encode("utf8") key_values = [(key, value)] key_pars = key.split("\t") value_pars = value.split("\t") if len(key_pars) > 1 and len(key_pars) == len( value_pars): index = 0 while index < len(key_pars): key_values.append( (key_pars[index], value_pars[index])) index += 1 for key, value in key_values: for key_conf, value_conf in self.mapping_conf.items( ): if key_conf in key: if unicode(key_conf) == u"注册资本": src_money = key + value value, unit = self.parser_tool.money_parser.transfer_money( src_money) entity_data[value_conf] = value break return entity_data
def get_court_place(self, content): '''获取开庭地点''' content = unicode(content).replace(" ", "") content_list = toolsutil.my_split(content, self.seps) court_place = self.parser_tool.court_place_parser.get_court_place( content) if not court_place: for row_content in content_list: court_place_list = toolsutil.re_findone( self.court_place_regex, unicode(row_content)) if court_place_list: for item in court_place_list: if item: court_place = item break if court_place and len(court_place) < self.court_place_len: for replace_str in ktgg_conf.court_place_replace_str_list: court_place = court_place.replace(replace_str, '') break return court_place
def format_litigant(self, input_plaintiff_list, input_defendant_list, replace_str_list): '''格式化当事人''' plaintiff_list = [] defendant_list = [] for defendant in input_defendant_list: defendant = re.sub(u'(\S+)', '', unicode(defendant)) for replace_str in replace_str_list: defendant = defendant.replace(replace_str, "") if len(unicode(defendant)) >= self.min_litigant_len and len( unicode(defendant)) <= self.max_litigant_len: defendant_list.append(defendant) for plaintiff in input_plaintiff_list: plaintiff = re.sub(u'(\S+)', '', unicode(plaintiff)) for replace_str in replace_str_list: plaintiff = plaintiff.replace(replace_str, "") if len(unicode(plaintiff)) >= self.min_litigant_len and len( unicode(plaintiff)) <= self.max_litigant_len: plaintiff_list.append(plaintiff) plaintiff_list = [x for x in plaintiff_list if x] defendant_list = [ x for x in defendant_list if x not in plaintiff_list and x ] new_defendant_list = [] for defendant in defendant_list: if toolsutil.re_findone( self.case_id_regex, defendant) or defendant in plaintiff_list or not defendant: continue if u'法院' in defendant: continue defendant = defendant.replace('(', '(').replace(')', ')') defendant = re.sub(u'(\S+)', '', defendant) new_defendant_list.append(defendant) return plaintiff_list, new_defendant_list
def before_parser(self, extract_data): '''解析预处理''' province = extract_data.get('province', '') content = extract_data.get('content', '') self.log.info("%sktgg_common_parser" % province) extract_data_list = [] if province in [u'河南']: extract_data["content"] = extract_data.get("title", "") extract_data_list.append(extract_data) elif province in [u"海南", u"辽宁"]: if unicode(content).find(u'基本案情') != -1: content = unicode(content) court_time = toolsutil.re_findone( re.compile(u'开庭时间[::]{0,1}(\S+)'), content) if court_time: extract_data["court_time"] = toolsutil.norm_date_time( court_time) court_place = toolsutil.re_findone( re.compile(u'开庭地点[::]{0,1}(\S+庭)'), content) if court_place: extract_data["court_place"] = court_place case_id_content = toolsutil.re_findone( re.compile(u'案号[::]{0,1}(\S+)'), content) if case_id_content: case_id = self.parser_tool.caseid_parser.get_case_id( case_id_content) extract_data["case_id"] = case_id court = toolsutil.re_findone(re.compile(u'法院[::]{0,1}(\S+)'), content) if court: extract_data["court"] = court judge_data = toolsutil.re_findone( re.compile(u'审判长[::]{0,1}(\S+)|主办人[::]{1,2}(\S+)'), content) judge = "" if judge_data: judge = self.get_value_from_tuple(judge_data) if judge: extract_data["judge"] = judge plaintiffs_data = toolsutil.re_findone( re.compile( u'原告人[::]{0,1}(\S+)|公诉人[::]{1,2}(\S+)|原告[::]{1,2}(\S+)' ), content) plaintiffs = "" if plaintiffs_data: plaintiffs = self.get_value_from_tuple(plaintiffs_data) if plaintiffs: extract_data["plaintiffs"] = plaintiffs defendants_data = toolsutil.re_findone( re.compile(u'被告人[::]{0,1}(\S+)|被告[::]{1,2}(\S+)'), content) defendants = "" if defendants_data: defendants = self.get_value_from_tuple(defendants_data) if defendants: extract_data["defendants"] = defendants content = toolsutil.re_findone( re.compile(u'基本案情[::]{0,1}(\S+)'), content) if content: extract_data["content"] = content extract_data_list.append(extract_data) else: extract_data_list.append(extract_data) else: extract_data_list.append(extract_data) return extract_data_list
def get_data_from_content(self, extract_data): '''抽取数据中有内容''' src_content = extract_data.get("content") content = unicode(src_content).replace(" ", "") content_list = toolsutil.my_split(content, self.seps) content = self.norm_content(content) #1 解析案由 case_cause_list = [] if not extract_data.has_key("case_cause"): case_cause_list = self.parser_tool.case_cause_parser.get_case_causes( content) case_cause = ','.join(case_cause_list) extract_data["case_cause"] = case_cause else: extract_data["case_cause"] = extract_data["case_cause"].replace( u"一案", "") case_cause_list.append(extract_data["case_cause"]) #2 解析案号 if not extract_data.has_key("case_id"): case_id = self.parser_tool.caseid_parser.get_case_id(content) extract_data["case_id"] = case_id #3 解析法院 if not extract_data.has_key("court"): extract_data["court"] = self.get_court(content) else: court_list = toolsutil.my_split(extract_data.get("court", ""), [':', ':']) if len(court_list) > 0: extract_data["court"] = court_list[-1] #4 解析法官 if not extract_data.has_key("judge"): judge = "" src_content = unicode(src_content) tmp_content_list = toolsutil.my_split(src_content, ['\r\n', '\r', '\n']) for row_content in tmp_content_list: judge = toolsutil.re_findone(self.judge_regex, unicode(row_content)) if judge: judge_list = toolsutil.my_split(judge, [' ', ',', ':']) judge = ','.join(judge_list) break else: judge = "" extract_data["judge"] = judge #5 解析开庭时间 if not extract_data.has_key("court_time"): for row_content in content_list: court_time = toolsutil.re_findone(self.court_time_regex, unicode(row_content)) if court_time: for week in ktgg_conf.week_day_list: court_time = court_time.replace(week, " ") else: court_time = "" extract_data["court_time"] = court_time break #6 解析开庭地点 if not extract_data.has_key("court_place"): extract_data["court_place"] = self.get_court_place(content) #7 解析当事人/原告/被告 #---获取当事人内容 litigants = plaintiffs = defendants = '' find_flag = False src_content = src_content.replace(" ", "") tmp_content_list = toolsutil.my_split(src_content, ['\r', '\n', '。']) for litigant_regex in self.litigant_regex_list: for row_content in tmp_content_list: litigants = toolsutil.re_findone(litigant_regex, unicode(row_content)) if litigants: find_flag = True break if find_flag: break else: litigants = "" if litigants == "": litigants = content #---通过当事人内容获取原告被告 plaintiffs, defendants, litigants = self.get_plaintiff_defendant( litigants, extract_data.get("case_id", ""), extract_data.get("court", ""), case_cause_list) if litigants: litigant_list, litigants, plaintiff_list, defendant_list = self.format_litigants( plaintiffs, defendants, litigants) else: litigant_list = plaintiff_list = defendant_list = [] for item in self.strip_list: extract_data['content'] = extract_data.get('content').replace( item, ' ') return self.get_entity_data(extract_data, litigant_list, litigants, plaintiff_list, defendant_list)
def format_extract_data(self, extract_data, topic_id): '''实体解析抽取数据''' is_baseinfo_page = False entity_data = {} if extract_data: entity_data = copy.deepcopy(extract_data) for key in gsxx_conf.gsxx_key_list: if entity_data.has_key(key) and entity_data.get(key) == None: entity_data.pop(key) if entity_data.has_key("base_info"): is_baseinfo_page = True entity_data.pop("base_info") for item in extract_data["base_info"]: for key, value in item.items(): base_key = key[:3] base_value = "value" + key[3:] if base_key == "key" and item.has_key(base_value): key = item[key].encode("utf8") if item.get(base_value) == None: continue value = item.get(base_value, "").encode("utf8") key_values = [(key, value)] key_pars = key.split("\t") value_pars = value.split("\t") if len(key_pars) > 1 and len(key_pars) == len( value_pars): index = 0 while index < len(key_pars): key_values.append( (key_pars[index], value_pars[index])) index += 1 for key, value in key_values: value = value.strip() if key in self.mapping_conf: if not entity_data.has_key( self.mapping_conf[key]): entity_data[ self.mapping_conf[key]] = value if self.mapping_conf[key] == 'code': if value != None and len(value) >= 18: entity_data[ "unified_social_credit_code"] = value else: entity_data[ "registered_code"] = value break company = entity_data.get("company", "") # Check whether this page is the base info page # F**k this code a thousand times! if entity_data.has_key("unified_social_credit_code") or \ entity_data.has_key("registered_code") or \ entity_data.has_key("code"): is_baseinfo_page = True if company == "": self.log.error("base info without company " + json.dumps(entity_data)) if entity_data.has_key("shareholder_information"): shareholder_information = entity_data.get( "shareholder_information") new_shareholder_information = [] for each in shareholder_information: if each.has_key("subscription_detail") and each.get( "subscription_detail") != None: each["subscription_detail"] = self.deal_data( each.get("subscription_detail", []), ["subscription_amount"]) each["subscription_detail"] = self.deal_time( each.get("subscription_detail", []), [ "subscription_time", "subscription_publish_time" ]) if each.has_key("paied_detail" ) and each.get("paied_detail") != None: each["paied_detail"] = self.deal_data( each.get("paied_detail", []), ["paied_amount"]) each["paied_detail"] = self.deal_time( each.get("paied_detail", []), ["paied_time", "paied_publish_time"]) new_shareholder_information.append(each) entity_data["shareholder_information"] = self.deal_data( new_shareholder_information, ["subscription_amount", "paied_amount"]) if entity_data.has_key("contributor_information"): shareholder_information = entity_data.get( "contributor_information") new_shareholder_information = [] for each in shareholder_information: if each.has_key("subscription_detail") and each.get( "subscription_detail") != None: each["subscription_detail"] = self.deal_data( each.get("subscription_detail", []), ["subscription_amount"]) each["subscription_detail"] = self.deal_time( each.get("subscription_detail", []), [ "subscription_time", "subscription_publish_time" ]) if each.has_key("paied_detail" ) and each.get("paied_detail") != None: each["paied_detail"] = self.deal_data( each.get("paied_detail", []), ["paied_amount"]) each["paied_detail"] = self.deal_time( each.get("paied_detail", []), ["paied_time", "paied_publish_time"]) new_shareholder_information.append(each) entity_data["contributor_information"] = self.deal_data( new_shareholder_information, ["subscription_amount", "paied_amount"]) if entity_data.has_key("code"): value = entity_data["code"] # entity_data.pop("code") if len(value) == 18: entity_data["unified_social_credit_code"] = value else: entity_data["registered_code"] = value src_registered_capital = entity_data.get( 'src_registered_capital') if not src_registered_capital: src_registered_capital = entity_data.get( 'registered_capital') if src_registered_capital: entity_data[ 'src_registered_capital'] = src_registered_capital registered_capital, registered_capital_unit = self.parser_tool.money_parser.transfer_money( src_registered_capital) entity_data["registered_capital"] = registered_capital entity_data[ "registered_capital_unit"] = registered_capital_unit if entity_data.has_key("period_from"): start_time = self.parser_tool.date_parser.get_date_list( entity_data["period_from"]) entity_data.pop("period_from") if entity_data.has_key("period_to"): if entity_data.get("period_to") == None: entity_data.pop("period_to") else: end_time = self.parser_tool.date_parser.get_date_list( entity_data["period_to"]) entity_data["period"] = toolsutil.norm_date( start_time) + u"至" + toolsutil.norm_date( end_time) entity_data.pop("period_to") else: entity_data["period"] = toolsutil.norm_date( start_time) + u"至" if entity_data.has_key("period"): period = entity_data.get("period", "") ret = toolsutil.re_findone(self.period_regex, period) if ret and len(ret) == 2: start_time = toolsutil.norm_date(ret[0]) end_time = toolsutil.norm_date(ret[1]) period = start_time + u"至" + end_time else: ret2 = toolsutil.re_findone(self.period_regex2, period) if ret2: start_time = toolsutil.norm_date(ret2) period = start_time + u"至" else: period = u"--" entity_data["period"] = period if company != "": province, city = self.cal_province_city(entity_data) entity_data["province"] = entity_data.get( "province") if entity_data.get( "province") else province entity_data["city"] = entity_data.get( "city") if entity_data.get("city") else city if not self.filter_company(company): entity_data["delete"] = 1 entity_data['company'] = company.replace('(', '(').replace( ')', ')') if entity_data.has_key("changerecords"): changerecords_list = [] used_name_list = [] for item in entity_data["changerecords"]: change_item = item.get("change_item", "") change_item = unicode(change_item) if change_item in gsxx_conf.used_name_change_item_list: after_name = unicode(item.get("after_content", "")) befor_name = unicode(item.get("before_content", "")) checked_after_name = self.check_name( company, after_name) checked_befor_name = self.check_name( company, befor_name) if checked_after_name: used_name_list.append(checked_after_name) if checked_befor_name: used_name_list.append(checked_befor_name) change_date = item.get("change_date", "") if isinstance(change_date, basestring): change_date = toolsutil.norm_date_time( self.parser_tool.date_parser.get_date_list( item.get("change_date", ""))) else: change_date = str(change_date) ret = toolsutil.re_find_one(u'\d+', change_date) if len(change_date) > 10 and ret == change_date: tmp = int(change_date[:-3]) data_value = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(tmp)) change_date = data_value item["change_date"] = change_date changerecords_list.append(item) used_name_list = list(set(used_name_list)) entity_data["used_name_list"] = used_name_list entity_data["changerecords"] = changerecords_list if entity_data.has_key("invested_companies"): invested_companies_list = [] for single in entity_data["invested_companies"]: invest_amount, invest_amount_unit = self.parser_tool.money_parser.transfer_money( single.get("invest_amount", "")) single["invest_amount"] = invest_amount single["invest_amount_unit"] = invest_amount_unit invested_companies_list.append(single) entity_data["invested_companies"] = invested_companies_list if entity_data.has_key("investor_change"): entity_data["investor_change"] = self.deal_time( entity_data.get("investor_change", []), ["change_date"]) if entity_data.has_key("business_status"): entity_data["business_status"] = entity_data.get( "business_status", "").replace(',', ',') if entity_data.has_key("registered_code"): value = entity_data.get("registered_code", "") if len(value.strip()) >= 18: entity_data["registered_code"] = "" else: entity_data["registered_code"] = "" if not entity_data.get("industry"): entity_data[ "industry"] = self.parser_tool.industry_parser.predict( company) for key, value in entity_data.items(): if value is None or (isinstance(value, basestring) and value.strip() == ''): del entity_data[key] return entity_data
def get_parser_data(self, content, bulletin_type): '''获取实体信息,当事人,原告,被告,公告类型''' plaintiff_list = [] defendant_list = [] norm_content = unicode(content).replace(" ", "") content_list = toolsutil.my_split(norm_content, [',', ',', '。', '\r\n', '\t']) find_flag = False #1 获取原告 for rowcontent in content_list: for plaintiff_regex in self.plaintiff_regex_list: ret = toolsutil.re_findone(plaintiff_regex, unicode(rowcontent)) if ret: plaintiff_list = toolsutil.my_split( ret, self.litiants_seps) #print "原告:",plaintiff_regex.pattern,','.join(plaintiff_list) find_flag = True break if find_flag: break #2 获取被告 if unicode(bulletin_type) in self.bulletin_type_list: find_flag = False for rowcontent in content_list: for defendant_regex in self.defendant_regex_list: ret = toolsutil.re_findone(defendant_regex, unicode(rowcontent)) if ret: if u'你' in unicode(ret): defendant_list = toolsutil.my_split( content_list[0], self.litiants_seps) else: defendant_list = toolsutil.my_split( ret, self.litiants_seps) if plaintiff_list == []: plaintiff_list = defendant_list defendant_list = [] # print "被告:", defendant_regex.pattern, ','.join(defendant_list) # print "原告:", ','.join(plaintiff_list) find_flag = True break for defendant_pattern in fygg_conf.defendant_pattern_list: ret = toolsutil.re_find_one(defendant_pattern, unicode(rowcontent)) if ret: defendant_list = toolsutil.my_split( ret, self.litiants_seps) find_flag = True break if find_flag: break plaintiff_list, defendant_list = self.format_litigant( plaintiff_list, defendant_list, fygg_conf.litigant_replace_str_list) else: content_list = toolsutil.my_split(norm_content, ['。', '\r\n', '\t', ',']) for rowcontent in content_list: tmp_list = re.split(':|:|;', rowcontent) if len(tmp_list) == 2: defendant_list = toolsutil.my_split( tmp_list[0], self.litiants_seps) replace_str_list = fygg_conf.defendant_keyword_list + fygg_conf.plaintiff_keyword_list plaintiff_list, defendant_list = self.format_litigant( plaintiff_list, defendant_list, replace_str_list) break info = { "plaintiff_list": plaintiff_list, "defendant_list": defendant_list, "bulletin_type": bulletin_type, } return info