def get_money(self, content, esm_index, content_length): money_list = [] money_ret = esm_index.query(content) if money_ret: find_flag = False for ret in money_ret: pos = ret[0][1] relate_content = content[pos:pos + content_length] money = toolsutil.re_find_one('\d+.\d+万', relate_content) if money: money, unit = MoneyParser().transfer_money(money) money = unicode(money) money_list.append((money, unit)) find_flag = True if not find_flag: for ret in money_ret: find_flag2 = False pos = ret[0][1] relate_content = content[pos:pos + content_length] if '万元' in relate_content: find_flag2 = True money_yuan = toolsutil.re_find_one('\d+.\d+', relate_content) if money_yuan: if find_flag2: money_yuan = money_yuan + '万' money_yuan, unit = MoneyParser().transfer_money( money_yuan) money_yuan = unicode(money_yuan) money_list.append((money_yuan, unit)) return money_list
def format_extract_data(self, extract_data, topic_id): '''实体解析抽取数据''' entity_data = copy.deepcopy(extract_data) tmp_code = extract_data.get("code", "").strip() code = toolsutil.re_find_one("\d+", tmp_code) if not code: code = "" self.log.warning("cann't extract code from [%s]" % tmp_code) publish_time = extract_data.get("publish_time") try: tmp_publish_time = int(publish_time[:-3]) publish_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(tmp_publish_time)) except: pass parser_info = {} if entity_data.has_key("content"): content = entity_data.get("content", "").strip() notice_id = self.get_notice_id(content) abstract = content[0:256] parser_info["notice_id"] = notice_id parser_info["abstract"] = abstract entity_data["notice_id"] = parser_info.get("notice_id") entity_data["abstract"] = parser_info.get("abstract") entity_data["publish_time"] = publish_time entity_data["code"] = code return entity_data
def get_bulletin_date(self, content): '''从正文中抽取公告日期''' content = unicode(content)[:100] print content date_ret = toolsutil.re_find_one('\d{4}-\d{1,2}-\d{1,2}', content) if date_ret: return toolsutil.norm_date_time(date_ret) else: return ""
def get_court(self, content): """获取法院""" court = '' content = unicode(content) content_list = toolsutil.my_split(content, self.seps) for row_content in content_list: if not row_content: continue row_content = unicode(row_content) ret = toolsutil.re_find_one(self.court_regex, row_content) if ret and len(ret) <= self.max_court_length: court = ret break if court == '': court = self.get_court2nd(content) return court
def get_region_from_phone(self, content): content_ret = self.phone_index.query(content) region_count_map = {} if content_ret: for ret in content_ret: pos = ret[0][0] phone_content = content[pos:pos + self.content_length] phone_num = toolsutil.re_find_one('\d+|\d+-\d+', phone_content) if phone_num: region = self.province_parser.get_region_from_phonenum(phone_num) if region: if region not in region_count_map.keys(): region_count_map[region] = 0 region_count_map[region] += 1 for k, v in sorted(region_count_map.items(), lambda x, y: cmp(x[1], y[1]), reverse=True): return k return ''
def get_litigant(self, keyword_list, content_list): pattern_list = [] result_list = [] for ret in keyword_list: for end in self.wenshu_conf.company_end_list: pattern = ret[1] + '(\S+' + end + ')' pattern_list.append(pattern) pattern = ret[1] + '(\S+)' pattern_list.append(pattern) pattern_list = list(set(pattern_list)) for content in content_list: for pattern in pattern_list: ret = toolsutil.re_find_one(unicode(pattern), content) if ret: if len(unicode(ret)) < self.company_length_limit: result_list.append(ret) result_list = self.norm_entity_list(result_list) return result_list
def get_last_table(self, last_table_str): '''获取对应主题最近备份表''' self.collection_names = self.db.db.collection_names() last_table_map = {} for collection in self.collection_names: if last_table_str in collection: tmp_list = collection.split("_") if len(tmp_list) > 2: time_str = tmp_list[-2] + " " + tmp_list[-1] if not toolsutil.re_find_one( "\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}", time_str): continue time_second = time.mktime( time.strptime(time_str, '%Y-%m-%d %H:%M:%S')) last_table_map[collection] = time_second for key, value in sorted(last_table_map.items(), lambda x, y: cmp(x[1], y[1]), reverse=True): return key return ""
def deal_time(self, src_data_list, key_list): new_data_list = [] deal_flag = False for iter_data in src_data_list: for key in key_list: try: time_value = str(iter_data[key]) ret = toolsutil.re_find_one(u'\d+', time_value) if len(time_value) > 10 and ret == time_value: tmp = int(time_value[:-3]) data_value = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(tmp)) iter_data[key] = data_value deal_flag = True else: iter_data[key] = toolsutil.norm_date_time( iter_data[key]) deal_flag = True except: pass new_data_list.append(iter_data) return new_data_list
def format_extract_data(self, extract_data, topic_id): '''实体解析抽取数据''' is_baseinfo_page = False entity_data = {} if extract_data: entity_data = copy.deepcopy(extract_data) for key in gsxx_conf.gsxx_key_list: if entity_data.has_key(key) and entity_data.get(key) == None: entity_data.pop(key) if entity_data.has_key("base_info"): is_baseinfo_page = True entity_data.pop("base_info") for item in extract_data["base_info"]: for key, value in item.items(): base_key = key[:3] base_value = "value" + key[3:] if base_key == "key" and item.has_key(base_value): key = item[key].encode("utf8") if item.get(base_value) == None: continue value = item.get(base_value, "").encode("utf8") key_values = [(key, value)] key_pars = key.split("\t") value_pars = value.split("\t") if len(key_pars) > 1 and len(key_pars) == len( value_pars): index = 0 while index < len(key_pars): key_values.append( (key_pars[index], value_pars[index])) index += 1 for key, value in key_values: value = value.strip() if key in self.mapping_conf: if not entity_data.has_key( self.mapping_conf[key]): entity_data[ self.mapping_conf[key]] = value if self.mapping_conf[key] == 'code': if value != None and len(value) >= 18: entity_data[ "unified_social_credit_code"] = value else: entity_data[ "registered_code"] = value break company = entity_data.get("company", "") # Check whether this page is the base info page # F**k this code a thousand times! if entity_data.has_key("unified_social_credit_code") or \ entity_data.has_key("registered_code") or \ entity_data.has_key("code"): is_baseinfo_page = True if company == "": self.log.error("base info without company " + json.dumps(entity_data)) if entity_data.has_key("shareholder_information"): shareholder_information = entity_data.get( "shareholder_information") new_shareholder_information = [] for each in shareholder_information: if each.has_key("subscription_detail") and each.get( "subscription_detail") != None: each["subscription_detail"] = self.deal_data( each.get("subscription_detail", []), ["subscription_amount"]) each["subscription_detail"] = self.deal_time( each.get("subscription_detail", []), [ "subscription_time", "subscription_publish_time" ]) if each.has_key("paied_detail" ) and each.get("paied_detail") != None: each["paied_detail"] = self.deal_data( each.get("paied_detail", []), ["paied_amount"]) each["paied_detail"] = self.deal_time( each.get("paied_detail", []), ["paied_time", "paied_publish_time"]) new_shareholder_information.append(each) entity_data["shareholder_information"] = self.deal_data( new_shareholder_information, ["subscription_amount", "paied_amount"]) if entity_data.has_key("contributor_information"): shareholder_information = entity_data.get( "contributor_information") new_shareholder_information = [] for each in shareholder_information: if each.has_key("subscription_detail") and each.get( "subscription_detail") != None: each["subscription_detail"] = self.deal_data( each.get("subscription_detail", []), ["subscription_amount"]) each["subscription_detail"] = self.deal_time( each.get("subscription_detail", []), [ "subscription_time", "subscription_publish_time" ]) if each.has_key("paied_detail" ) and each.get("paied_detail") != None: each["paied_detail"] = self.deal_data( each.get("paied_detail", []), ["paied_amount"]) each["paied_detail"] = self.deal_time( each.get("paied_detail", []), ["paied_time", "paied_publish_time"]) new_shareholder_information.append(each) entity_data["contributor_information"] = self.deal_data( new_shareholder_information, ["subscription_amount", "paied_amount"]) if entity_data.has_key("code"): value = entity_data["code"] # entity_data.pop("code") if len(value) == 18: entity_data["unified_social_credit_code"] = value else: entity_data["registered_code"] = value src_registered_capital = entity_data.get( 'src_registered_capital') if not src_registered_capital: src_registered_capital = entity_data.get( 'registered_capital') if src_registered_capital: entity_data[ 'src_registered_capital'] = src_registered_capital registered_capital, registered_capital_unit = self.parser_tool.money_parser.transfer_money( src_registered_capital) entity_data["registered_capital"] = registered_capital entity_data[ "registered_capital_unit"] = registered_capital_unit if entity_data.has_key("period_from"): start_time = self.parser_tool.date_parser.get_date_list( entity_data["period_from"]) entity_data.pop("period_from") if entity_data.has_key("period_to"): if entity_data.get("period_to") == None: entity_data.pop("period_to") else: end_time = self.parser_tool.date_parser.get_date_list( entity_data["period_to"]) entity_data["period"] = toolsutil.norm_date( start_time) + u"至" + toolsutil.norm_date( end_time) entity_data.pop("period_to") else: entity_data["period"] = toolsutil.norm_date( start_time) + u"至" if entity_data.has_key("period"): period = entity_data.get("period", "") ret = toolsutil.re_findone(self.period_regex, period) if ret and len(ret) == 2: start_time = toolsutil.norm_date(ret[0]) end_time = toolsutil.norm_date(ret[1]) period = start_time + u"至" + end_time else: ret2 = toolsutil.re_findone(self.period_regex2, period) if ret2: start_time = toolsutil.norm_date(ret2) period = start_time + u"至" else: period = u"--" entity_data["period"] = period if company != "": province, city = self.cal_province_city(entity_data) entity_data["province"] = entity_data.get( "province") if entity_data.get( "province") else province entity_data["city"] = entity_data.get( "city") if entity_data.get("city") else city if not self.filter_company(company): entity_data["delete"] = 1 entity_data['company'] = company.replace('(', '(').replace( ')', ')') if entity_data.has_key("changerecords"): changerecords_list = [] used_name_list = [] for item in entity_data["changerecords"]: change_item = item.get("change_item", "") change_item = unicode(change_item) if change_item in gsxx_conf.used_name_change_item_list: after_name = unicode(item.get("after_content", "")) befor_name = unicode(item.get("before_content", "")) checked_after_name = self.check_name( company, after_name) checked_befor_name = self.check_name( company, befor_name) if checked_after_name: used_name_list.append(checked_after_name) if checked_befor_name: used_name_list.append(checked_befor_name) change_date = item.get("change_date", "") if isinstance(change_date, basestring): change_date = toolsutil.norm_date_time( self.parser_tool.date_parser.get_date_list( item.get("change_date", ""))) else: change_date = str(change_date) ret = toolsutil.re_find_one(u'\d+', change_date) if len(change_date) > 10 and ret == change_date: tmp = int(change_date[:-3]) data_value = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(tmp)) change_date = data_value item["change_date"] = change_date changerecords_list.append(item) used_name_list = list(set(used_name_list)) entity_data["used_name_list"] = used_name_list entity_data["changerecords"] = changerecords_list if entity_data.has_key("invested_companies"): invested_companies_list = [] for single in entity_data["invested_companies"]: invest_amount, invest_amount_unit = self.parser_tool.money_parser.transfer_money( single.get("invest_amount", "")) single["invest_amount"] = invest_amount single["invest_amount_unit"] = invest_amount_unit invested_companies_list.append(single) entity_data["invested_companies"] = invested_companies_list if entity_data.has_key("investor_change"): entity_data["investor_change"] = self.deal_time( entity_data.get("investor_change", []), ["change_date"]) if entity_data.has_key("business_status"): entity_data["business_status"] = entity_data.get( "business_status", "").replace(',', ',') if entity_data.has_key("registered_code"): value = entity_data.get("registered_code", "") if len(value.strip()) >= 18: entity_data["registered_code"] = "" else: entity_data["registered_code"] = "" if not entity_data.get("industry"): entity_data[ "industry"] = self.parser_tool.industry_parser.predict( company) for key, value in entity_data.items(): if value is None or (isinstance(value, basestring) and value.strip() == ''): del entity_data[key] return entity_data
def do_parser(self, content, title): '''获取公司''' # 1 根据关键字查找公司名 houxuanzhongbiao_company_list = self.get_company_list( content, self.houxuanzhongbiao_company_index, bid_conf.zhongbiao_company_end_list, self.zhongbiao_offset) zhongbiao_company_list = self.get_company_list( content, self.zhongbiao_company_index, bid_conf.zhongbiao_company_end_list, self.zhongbiao_offset) zhaobiao_company_list = self.get_company_list( content, self.zhaobiao_company_index, bid_conf.zhaobiao_company_end_list, self.zhaobiao_offset) agent_company_list = self.get_company_list( content, self.agent_company_index, bid_conf.agent_company_end_list, self.agent_offset) zhaobiao_company = None if zhaobiao_company_list: if len(zhaobiao_company_list) >= 2: find_flag = False for item in zhaobiao_company_list: if u'公司' not in unicode(item): zhaobiao_company = item find_flag = True break if not find_flag: zhaobiao_company = zhaobiao_company_list[0] else: zhaobiao_company = zhaobiao_company_list[0] agent = '' if agent_company_list: agent = agent_company_list[0] # 2 若招标或者代理公司未找到,则匹配(\S*)受(\S+)委托 content = unicode(content) ret = toolsutil.re_find_one(u'(\S*)受(\S+)委托', content) if ret: tmp_agent = self.company_parser.get_one_company( ret[0], bid_conf.agent_company_end_list) tmp_zhaobiao_company = self.company_parser.get_one_company( ret[1], bid_conf.zhaobiao_company_end_list) agent = tmp_agent if tmp_agent else agent zhaobiao_company = tmp_zhaobiao_company if tmp_zhaobiao_company else zhaobiao_company # 3 若招标公司还未找到,则从标题中寻找 if not zhaobiao_company: zhaobiao_company = self.company_parser.get_one_company( title, bid_conf.zhaobiao_company_end_list) result_data = self.filter_company_list(zhaobiao_company, zhongbiao_company_list, houxuanzhongbiao_company_list, agent) result_data["zhaobiao"] = unicode(result_data["zhaobiao"]) result_data["agent"] = unicode(result_data["agent"]) for i in range(len(result_data["zhongbiao"])): result_data["zhongbiao"][i] = unicode(result_data["zhongbiao"][i]) for i in range(len(result_data["houxuan_zhongbiao"])): result_data["houxuan_zhongbiao"][i] = unicode( result_data["houxuan_zhongbiao"][i]) return result_data
def deal_time(self,src_data_list,key_list): new_data_list = [] deal_flag = False for iter_data in src_data_list: for key in key_list: try: time_value = str(iter_data[key]) ret = toolsutil.re_find_one(u'\d+',time_value) if len(time_value) > 10 and ret == time_value: tmp = int(time_value[:-3]) data_value = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(tmp)) iter_data[key] = data_value deal_flag = True else: iter_data[key] = toolsutil.norm_date_time(iter_data[key]) deal_flag = True except: pass new_data_list.append(iter_data) return new_data_list # if __name__ == '__main__': # import sys # sys.path.append('../../') # topic_id = 136 # import pytoml # from conf import get_config # from bdp.i_crawler.i_extractor.ttypes import BaseInfo, CrawlInfo, ExtractInfo, PageParseInfo # with open('../../entity.toml', 'rb') as config: # config = pytoml.load(config) # conf = get_config(config) # import common # from entity_extractor_route import EntityExtractorRoute # route = EntityExtractorRoute() # topic_info = route.all_topics.get(topic_id, None) # obj = AnnualReportsExtractor(topic_info, common.log) # extract_data = { # "base_info": [ # { # "key": "统一社会信用代码/注册号", # "value": "911400001123101349" # }, # { # "key": "企业名称", # "value": "华晋焦煤有限责任公司" # }, # { # "key": "企业通信地址", # "value": "山西省吕梁市离市区久安路57号" # }, # { # "key": "邮政编码", # "value": "033000" # }, # { # "key": "企业联系电话", # "value": "0358-8296368" # }, # { # "key": "企业电子邮箱", # "value": "*****@*****.**" # }, # { # "key": "从业人数", # "value": "企业选择不公示" # }, # { # "key": "企业经营状态", # "value": "开业" # }, # { # "key": "是否有网站或网店", # "value": "是" # }, # { # "key": "有限责任公司本年度是否发生股东股权转让", # "value": "否" # }, # { # "key": "是否有投资信息或购买其他公司股权", # "value": "有" # }, # { # "key": "对外提供保证担保信息", # "value": "否" # } # ], # "enterprise_asset_status_information": [ # { # "key": "资产总额", # "value": "企业选择不公示" # }, # { # "key": "所得者权益合计", # "value": "企业选择不公示" # }, # { # "key": "营业总收入", # "value": "企业选择不公示" # }, # { # "key": "利润总额", # "value": "企业选择不公示" # }, # { # "key": "营业总收入中主营业务收入", # "value": "企业选择不公示" # }, # { # "key": "净利润", # "value": "企业选择不公示" # }, # { # "key": "纳税总额", # "value": "企业选择不公示" # }, # { # "key": "负债总额", # "value": "企业选择不公示" # } # ], # "invested_companies": [ # { # "company_name": "华晋煤层气综合利用有限责任公司", # "registered_code": "140000110106177" # }, # { # "company_name": "山西华晋吉宁煤业有限责任公司", # "registered_code": "140000105974138" # }, # { # "company_name": "山西华晋明珠煤业有限责任公司", # "registered_code": "140000206970138" # }, # { # "company_name": "山西焦煤华晋寨圪塔能源有限责任公司", # "registered_code": "141000000074580" # }, # { # "company_name": "石太铁路客运专线有限责任公司", # "registered_code": "140100103043161" # }, # { # "company_name": "山西汾河焦煤股份有限公司", # "registered_code": "140000100099469" # }, # { # "company_name": "山西焦煤集团汾河物业管理有限公司", # "registered_code": "140100103047124" # }, # { # "company_name": "山西焦煤集团房地产开发有限公司", # "registered_code": "140100103020695" # }, # { # "company_name": "山西焦煤交通能源投资有限公司", # "registered_code": "140000110111179" # } # ], # "province": "山西", # "shareholder_information": [ # { # "paied_amount": "42354.798018", # "paied_time": "1204646400000", # "paied_type": "货币", # "shareholder_name": "山西焦煤集团有限责任公司", # "subscription_amount": "42354.798018", # "subscription_time": "1204646400000", # "subscription_type": "货币" # }, # { # "paied_amount": "40693.825547", # "paied_time": "2017年6月4日", # "paied_type": "货币", # "shareholder_name": "中国中煤能源股份有限公司", # "subscription_amount": "40693.825547", # "subscription_time": "1204646400000", # "subscription_type": "货币" # } # ], # "websites": [ # { # "name": "华晋焦煤有限责任公司", # "site": "http://www.sx.xinhuanet.com/qyzx/hjjm/", # "type": "网站" # } # ], # "year": "2015年度" # } # src_url = "www.baidu.com" # data = json.dumps(extract_data) # extract_info = ExtractInfo(ex_status=2, extract_data=data) # base_info = BaseInfo(url=src_url) # parser_info = PageParseInfo(base_info=base_info, extract_info=extract_info) # entity_data = obj.entity_extract(parser_info, extract_data) # #data = obj.after_extract(src_url, entity_data, extract_data) # print data # for key, value in entity_data.items(): # if isinstance(value, list): # for i in value: # print key, ":", i # elif isinstance(value, dict): # for key2, value2 in value.items(): # print key2, ":", value2 # else: # print key, ":", value
def get_parser_data(self, content, bulletin_type): '''获取实体信息,当事人,原告,被告,公告类型''' plaintiff_list = [] defendant_list = [] norm_content = unicode(content).replace(" ", "") content_list = toolsutil.my_split(norm_content, [',', ',', '。', '\r\n', '\t']) find_flag = False #1 获取原告 for rowcontent in content_list: for plaintiff_regex in self.plaintiff_regex_list: ret = toolsutil.re_findone(plaintiff_regex, unicode(rowcontent)) if ret: plaintiff_list = toolsutil.my_split( ret, self.litiants_seps) #print "原告:",plaintiff_regex.pattern,','.join(plaintiff_list) find_flag = True break if find_flag: break #2 获取被告 if unicode(bulletin_type) in self.bulletin_type_list: find_flag = False for rowcontent in content_list: for defendant_regex in self.defendant_regex_list: ret = toolsutil.re_findone(defendant_regex, unicode(rowcontent)) if ret: if u'你' in unicode(ret): defendant_list = toolsutil.my_split( content_list[0], self.litiants_seps) else: defendant_list = toolsutil.my_split( ret, self.litiants_seps) if plaintiff_list == []: plaintiff_list = defendant_list defendant_list = [] # print "被告:", defendant_regex.pattern, ','.join(defendant_list) # print "原告:", ','.join(plaintiff_list) find_flag = True break for defendant_pattern in fygg_conf.defendant_pattern_list: ret = toolsutil.re_find_one(defendant_pattern, unicode(rowcontent)) if ret: defendant_list = toolsutil.my_split( ret, self.litiants_seps) find_flag = True break if find_flag: break plaintiff_list, defendant_list = self.format_litigant( plaintiff_list, defendant_list, fygg_conf.litigant_replace_str_list) else: content_list = toolsutil.my_split(norm_content, ['。', '\r\n', '\t', ',']) for rowcontent in content_list: tmp_list = re.split(':|:|;', rowcontent) if len(tmp_list) == 2: defendant_list = toolsutil.my_split( tmp_list[0], self.litiants_seps) replace_str_list = fygg_conf.defendant_keyword_list + fygg_conf.plaintiff_keyword_list plaintiff_list, defendant_list = self.format_litigant( plaintiff_list, defendant_list, replace_str_list) break info = { "plaintiff_list": plaintiff_list, "defendant_list": defendant_list, "bulletin_type": bulletin_type, } return info
def profit_format_extract_data(self, extract_data): '''利润表解析抽取数据''' month = year = publish_time = code = '' entity_data = copy.deepcopy(extract_data) if entity_data.has_key('year_month'): year_month = entity_data.get('year_month', '') for year_mode in self.year_mode_list: if year_mode in year_month: year = year_mode break for key, value in self.month_mode_map.items(): if key in year_month: month = value break for key in self.quarter_map.keys(): if key in year_month: month = key break if month and year: publish_time = year + month if entity_data.has_key('code'): code = toolsutil.re_find_one('\d+', entity_data.get('code', '')) if code: entity_data["code"] = code if entity_data.has_key('money_unit'): money_unit = entity_data.get("money_unit", "") found = False for money_type in self.money_type_list: if money_type in unicode(money_unit): entity_data["money_unit"] = money_type found = True break if not found: entity_data["money_unit"] = u'元' caibao_type = self.quarter_map.get(month,"") title = publish_time + caibao_type + u'利润表' data_info = {} if entity_data.has_key("info"): entity_data.pop("info") for item in extract_data["info"]: for key, value in item.items(): base_key = key[:3] base_value = "value" + key[3:] if base_key == "key" and item.has_key(base_value): key = item[key].encode("utf8") value = item[base_value].encode("utf8") key_values = [(key, value)] key_pars = key.split("\t") value_pars = value.split("\t") if len(key_pars) > 1 and len(key_pars) == len(value_pars): index = 0 while index < len(key_pars): key_values.append((key_pars[index], value_pars[index])) index += 1 for key, value in key_values: key = key.strip().replace(" ", "") if toolsutil.re_find_one(u'\d+\.\d+', key): continue data_info[key] = value for key_conf, value_conf in self.profit_mapping_conf.items(): if key_conf in key: entity_data[value_conf] = value break entity_data["data_info"] = data_info entity_data["title"] = title entity_data["caibao_type"] = caibao_type entity_data["publish_time"] = publish_time return entity_data