def get_dict_table_items(self, table_tag): """获得字典类型的表格的结构 """ table_items = {} for tr in table_tag.find_all('tr'): if tr.find('th') and tr.find('td'): ths = tr.find_all('th') tds = tr.find_all('td') for index, td in enumerate(tds): table_items[CrawlerUtils.get_raw_text_in_bstag(td).strip('{}').replace('PAPERS_', '')]\ = CrawlerUtils.get_raw_text_in_bstag(ths[index]) return table_items
def parse_ind_comm_pub_reg_modify_table(self, bs_table, table_name, page): """解析工商公示信息-注册信息-变更信息表格,由于含有详情页,需要单独处理 """ tbody = bs_table.find('tbody') if tbody: columns = self.get_columns_of_record_table(bs_table, page) column_size = len(columns) item_array = [] for tr in tbody.find_all('tr'): if tr.find('td'): col_count = 0 item = {} for td in tr.find_all('td'): if td.find('a'): #try to retrieve detail link from page next_url = self.get_detail_link(td.find('a'), page) #has detail link if next_url: detail_page = self.crawler.crawl_page_by_url( next_url).content detail_soup = BeautifulSoup( detail_page, 'html.parser') before_modify_table = detail_soup.body.find_all( 'table')[1] table_data = self.parse_table( before_modify_table, 'before_modify', detail_page) item[columns[col_count][0]] = self.parse_table( before_modify_table, 'before_modify', detail_page) col_count += 1 after_modify_table = detail_soup.body.find_all( 'table')[2] item[columns[col_count][0]] = self.parse_table( after_modify_table, 'after_modify', detail_page) else: item[columns[col_count] [0]] = CrawlerUtils.get_raw_text_in_bstag( td) else: item[columns[col_count] [0]] = CrawlerUtils.get_raw_text_in_bstag(td) col_count += 1 if col_count == column_size: item_array.append(item.copy()) col_count = 0 return item_array
def get_list_table_items(self, table_tag): """获取记录类型的表格的结构 """ table_items = {} if len(table_tag.find_all('tr')) != 3: print 'abnormal list table skeleton, table_tag = ', table_tag return table_items ths = table_tag.find_all('tr')[1].find_all('th') tds = table_tag.find_all('tr')[2].find_all('td') if len(ths) != len(tds): print 'abnormal list table skeleton, table_tag = ', table_tag return table_items for index, td in enumerate(tds): table_items[ CrawlerUtils.get_raw_text_in_bstag(td).strip('{}').replace( 'PAPERS_', '')] = CrawlerUtils.get_raw_text_in_bstag(ths[index]) return table_items
def parse_list_table_without_sub_list(self, records_tag, table_name, page, columns): """提取没有子列的记录形式的表 Args: records_tag: 表的记录标签,用beautiful soup 从html中提取出来的 table_name: 表名 page: 原始的html页面 columns: 表的表头结构 Returns: item_array: 提取出的表数据,列表形式,列表中的元素为一个python字典 """ item_array = [] for tr in records_tag.find_all('tr'): col_count = 0 item = {} for td in tr.find_all('td', recursive=False): if td.find('a'): # try to retrieve detail link from page next_url = self.get_detail_link(td.find('a'), page) # has detail link if next_url: detail_page = self.crawler.crawl_page_by_url(next_url) if table_name == 'ent_pub_ent_annual_report': page_data = self.parse_ent_pub_annual_report_page( detail_page) item[u'报送年度'] = CrawlerUtils.get_raw_text_in_bstag( td) item[ u'详情'] = page_data # this may be a detail page data elif table_name == 'ind_comm_pub_reg_shareholder': page_data = self.parse_ind_comm_pub_shareholder_detail_page( detail_page) item[u'详情'] = {u"投资人及出资信息": page_data} else: page_data = self.parse_page( detail_page, table_name + '_detail') item[columns[col_count][ 0]] = page_data # this may be a detail page data else: # item[columns[col_count]] = CrawlerUtils.get_raw_text_in_bstag(td) item[columns[col_count][0]] = self.get_column_data( columns[col_count][1], td) else: item[columns[col_count][0]] = self.get_column_data( columns[col_count][1], td) col_count += 1 if item: item_array.append(item) return item_array
def parse_annual_report_shareholder_info(self, page): """解析年报信息中的投资人信息 需要单独处理 """ shareholder_info = [] record_columns = [ u'股东', u'认缴出资额', u'认缴出资时间', u'认缴出资方式', u'实缴出资额', u'实缴出资时间', u'实缴出资方式' ] json_obj = json.loads(page) for record in json_obj.get('items'): if not record.get('D1'): continue result = {} soup = BeautifulSoup(record.get('D1'), 'html.parser') tds = soup.find_all('td') if not tds: continue for index, column in enumerate(record_columns): result[column] = CrawlerUtils.get_raw_text_in_bstag(tds[index]) shareholder_info.append(result) return shareholder_info
def get_year_of_annual_report(page): soup = BeautifulSoup(page, 'html.parser') t = soup.body.find('table') return CrawlerUtils.get_raw_text_in_bstag(t.find('tr'))
def parse_annual_report_skeleton(self, page): """解析 企业年报页面结构 """ #企业基本信息 soup = BeautifulSoup(page, 'html.parser') annual_report_table_items = {} tag = soup.find('div', attrs={'id': 'qyjbxx'}) if not tag: print 'parse annual report skeleton failed, do not find qyjbxx table' return table = tag.find('table', attrs={'class': 'detailsList'}) if table: ent_basic_info_table = {} for tr in table.find_all('tr'): if tr.find('th') and tr.find('td'): ths = tr.find_all('th') tds = tr.find_all('td') for index, td in enumerate(tds): ent_basic_info_table[td.get( 'id')] = CrawlerUtils.get_raw_text_in_bstag( ths[index]) self.parse_table_items[ 'annual_report_ent_basic_info'] = ent_basic_info_table #网站或网店信息 table = soup.find('table', attrs={ 'id': 'web', 'name': 'applicationList1TAB' }) if table: self.parse_table_items[ 'annual_report_web_info'] = self.get_list_table_items(table) #股东及出资信息 table = soup.find('table', attrs={ 'id': 'touziren', 'name': 'applicationList4TAB' }) if table: shareholder_info_table = {} #对外投资信息 table = soup.find('table', attrs={ 'id': 'duiwaitouzi', 'name': 'applicationList3TAB' }) if table: self.parse_table_items[ 'annual_report_investment_abord_info'] = self.get_list_table_items( table) #企业资产状况信息 for table in soup.find_all('table'): tr = table.find('tr') if tr and tr.find('th') and tr.find('th').text == u'企业资产状况信息': ent_property_info_table = {} for tr in table.find_all('tr'): if tr.find('th') and tr.find('td'): ths = tr.find_all('th') tds = tr.find_all('td') for index, td in enumerate(tds): ent_property_info_table[td.get( 'id')] = CrawlerUtils.get_raw_text_in_bstag( ths[index]) self.parse_table_items[ 'annual_report_ent_property_info'] = ent_property_info_table break #对外提供担保信息 table = soup.find('table', attrs={ 'id': 'duiwaidanbao', 'name': 'applicationList6TAB' }) if table: self.parse_table_items[ 'annual_report_external_guarantee_info'] = self.get_list_table_items( table) #股权变更信息 table = soup.find('table', attrs={ 'id': 'guquanchange', 'name': 'applicationList5TAB' }) if table: self.parse_table_items[ 'annual_report_equity_modify_info'] = self.get_list_table_items( table) #修改记录 table = soup.find('table', attrs={ 'id': 'modifyRecord', 'name': 'applicationList2TAB' }) if table: self.parse_table_items[ 'annual_report_modify_record'] = self.get_list_table_items( table) self.annual_report_skeleton_built = True