def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) data = json.loads(response.text) html = data.get("contentViewsMap", "").get("rightData1", "") tree = etree.HTML(html) tr_list = tree.xpath(".//tr") title_list = [] value_list = [] for tr in tr_list[1:]: tds = tr.xpath(".//td") title_list += [''.join(td.xpath("string()")).strip() for td in tds[::2]] value_list += [''.join(td.xpath("string()")).strip() for td in tds[1::2]] data_dict = dict(zip(title_list, value_list)) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = self.convert_time(map_field(data_dict)) yield item self.logger1.info('{} save successfully'.format(response.url)) except Exception as e: self.logger1.warning("Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) tr_list = response.xpath("//div[@class='fl ml20 mb10 mt10 f_yh']//table//tr") if tr_list: data_dict = {} for tr in tr_list: title = clean_all_space(''.join(tr.xpath(".//th").xpath("string(.)").extract()).replace(":", "").replace(":", "")) value = clean_all_space(''.join(tr.xpath(".//td").xpath("string(.)").extract())) data_dict.update({title: value}) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' res_dict = self.convert_time(map_field(data_dict)) if "xzcf" in self.name: if "license_status" in res_dict.keys(): res_dict["punish_status"] = res_dict.pop("license_status", "") item['_parsed_data'] = res_dict yield item self.logger1.info('{} save successfully'.format(response.url)) else: self.logger1.info('retry {}'.format(response.url)) yield Request(response.url, callback=self.parse_detail, errback=self.error_parse) except Exception as e: self.logger1.warning("Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) tr_list = response.xpath("//table[@class='table_normal1']//tr") data_dict = {} for tr in tr_list: title = ''.join( tr.xpath(".//th").xpath("string()").extract()).strip() value = ''.join( tr.xpath(".//td").xpath("string()").extract()).strip() data_dict.update({title: value}) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' res_dict = self.convert_time(map_field(data_dict)) if "xzcf" in self.name and "license_status" in res_dict.keys(): res_dict["punish_status"] = res_dict.pop("license_status", "") item['_parsed_data'] = res_dict yield item self.logger1.info('{} save successfully'.format(response.url)) except Exception as e: self.logger1.warning( "Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) li_list = response.xpath("//div[@class='warp']//ul//li") data_dict = {} for li in li_list: data_str = clean_all_space(''.join( li.xpath("string()").extract()).replace(":", ":")) title = data_str.split(":", 1)[0] value = ''.join( li.xpath(".//span").xpath("string()").extract()).strip() data_dict.update({title: value}) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = self.convert_time(map_field(data_dict)) yield item self.logger1.info('{} save successfully'.format(response.url)) except Exception as e: self.logger1.warning( "Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def handle_result(self, response, result_dict): item = self.result_item_assembler(response) item['_parsed_data'] = map_field(result_dict) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['bbd_params'] = '' return item
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: if "error" in response.url: return self.logger1.info('start to parse {}'.format(response.url)) tr_list = response.xpath("//div[@class='display_con']//table//tr[position()>1]") if tr_list: data_dict = {} title_list = [] value_list = [] for tr in tr_list: tds = tr.xpath(".//td") title_list += [clean_all_space(''.join(td.xpath("string(.)").extract())) for td in tds[::2]] value_list += [clean_all_space(''.join(td.xpath("string(.)").extract())) for td in tds[1::2]] data_dict.update(dict(zip(title_list, value_list))) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' result_dict = map_field(data_dict) if "xzcf" in self.bbd_table: result_dict["punish_status"] = result_dict.pop("license_status", "") item['_parsed_data'] = self.convert_time(result_dict) yield item self.logger1.info('{} save successfully'.format(response.url)) else: self.logger1.info('retry {}'.format(response.url)) yield Request(response.url, callback=self.parse_detail, errback=self.error_parse) except Exception as e: self.logger1.warning("Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) tr_list = response.xpath("//table[@class='infor']//tr") if tr_list: data_dict = {} result = {} for tr in tr_list: title = ''.join( tr.xpath(".//td[@class='name']/text()").extract() ).strip().replace(':', '').replace(':', '').strip() value = ''.join( tr.xpath(".//td[last()]").xpath( "string()").extract()).strip() if title in ['处罚类别', '处罚类型']: if '处罚类别1' in data_dict.keys(): title = '处罚类别2' else: title = '处罚类别1' data_dict.update({title: value}) data_dict[self.data_status] = data_dict.pop('当前状态', '') result.update(map_field(data_dict)) for key, value in result.items(): if 'date' in key: if value.strip().isdigit(): val = '{}-{}-{} 00:00:00'.format( value[:4], value[4:6], value[6:]) elif ':' in value: val = '-'.join( re.findall( r'(\d+)', value.split(' ')[0])) + ' {}'.format( value.split(' ')[1]) else: val = '-'.join(re.findall(r'(\d+)', value) ) + ' 00:00:00' if value else '' result[key] = val item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = result yield item self.logger1.info('{} save successfully'.format(response.url)) else: self.logger1.info("retry {}".format(response.url)) yield Request(response.url, callback=self.parse_detail, errback=self.error_parse) except Exception as e: self.logger1.warning( "Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) detail = json.loads(response.body.decode())[0] if detail: result = {} if 'xkid=' in response.url: result['license_code'] = detail.get('xkwsh', '') result['case_name'] = detail.get('xmmc', '') result['approval_category'] = detail.get('splb', '') result['license_content'] = detail.get('xknr', '') result['company_name'] = detail.get('xzxdr', '') result['license_start_date'] = detail.get( 'xkjdrq', '').replace('/', '-') + ' 00:00:00' result['license_end_date'] = detail.get( 'xkjzrq', '').replace('/', '-') + ' 00:00:00' result['license_org'] = detail.get('xkjg', '') else: result['punish_code'] = detail.get('cfwsh', '') result['case_name'] = detail.get('cfmc', '') result['punish_category_one'] = detail.get('cflb', '') result['punish_type'] = detail.get('cfsy', '') result['punish_basis'] = detail.get('cfyj', '') result['company_name'] = detail.get('xzxdr', '') result['public_date'] = detail.get('cfjdrq', '').replace( '/', '-') + ' 00:00:00' result['punish_org'] = detail.get('cfjguan', '') item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = result yield item self.logger1.info('{} save successfully'.format(response.url)) else: count = response.meta['count'] + 1 if count < 10: self.logger1.info( 'The page has no content,try! {}'.format( response.url)) yield Request(response.url, callback=self.parse_detail, errback=self.error_parse, priority=6, meta={'count': count}) else: self.logger1.info( 'The page has no content,discard! {}'.format( response.url)) except Exception as e: self.logger1.warning( "Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def handle_result(self, response, result_dict, info_id): item = self.result_item_assembler(response) item['_parsed_data'] = result_dict item['bbd_html'] = '' item[ 'bbd_url'] = response.url + '?CategoryNum=014013&InfoID={}'.format( info_id) item['_id'] = calc_str_md5(info_id) return item
def handle_result(self, response, result_dict): item = ParsedItem() self.common_item_assembler(response, item) item['_parsed_data'] = map_field(result_dict) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['bbd_type'] = "credit_bj" item['rowkey'] = gen_rowkey(item, keys=('do_time', 'bbd_type')) item['bbd_params'] = '' return item
def handle_result(self, response, result_dict): result_dict = map_field(result_dict) if 'xzcf' in self.name and 'license_status' in result_dict: result_dict['punish_status'] = result_dict.pop( 'license_status', '') item = self.result_item_assembler(response) item['_parsed_data'] = result_dict item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['bbd_params'] = '' return item
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse {}'.format(response.url)) tr_list = response.xpath("//table[@class='table table-hover']//tr") if tr_list: data_dict = {} result = {} for tr in tr_list: title = ''.join(tr.xpath(".//th").xpath("string()").extract()).strip() value = ''.join(tr.xpath(".//td").xpath("string()").extract()).strip() if title in ['处罚类别', '处罚类型']: if '处罚类别1' in data_dict.keys(): title = '处罚类别2' else: title = '处罚类别1' data_dict.update({title: value}) result.update(map_field(data_dict)) for key, value in result.items(): if 'date' in key: if value.strip().isdigit(): val = '{}-{}-{} 00:00:00'.format(value[:4], value[4:6], value[6:]) elif ':' in value: val = '-'.join(re.findall(r'(\d+)', value.split(' ')[0])) + ' {}'.format(value.split(' ')[1]) else: val = '-'.join(re.findall(r'(\d+)', value)) + ' 00:00:00' if value else '' result[key] = val if 'punish_basis' in result and 'punish_code' not in result: result['punish_code'] = result.pop('license_code', '') item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = result yield item self.logger1.info('{} save successfully'.format(response.url)) else: count = response.meta['count'] + 1 if count < 10: self.logger1.info('The page has no content,try! {}'.format(response.url)) yield Request(response.url, callback=self.parse_detail, errback=self.error_parse, priority=6, meta={'count': count}) else: self.logger1.info('The page has no content,discard! {}'.format(response.url)) except Exception as e: self.logger1.warning("Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))
def handle_result(self, response, result_dict): result_dict = map_field(result_dict) if 'xzcf' in self.name and 'license_status' in result_dict: result_dict['punish_status'] = result_dict.pop( 'license_status', '') item = ParsedItem() self.common_item_assembler(response, item) item['_parsed_data'] = result_dict item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['bbd_params'] = '' item['bbd_type'] = self.name.split('__')[-1][:-7] item['rowkey'] = gen_rowkey(item, keys=('do_time', 'bbd_type')) return item
def parse_detail(self, response): """ parse detail page :param response: :return: """ try: self.logger1.info('start to parse detail content {}'.format( response.url)) cmp_name = ''.join( response.xpath( './/td[@class="listf2"]//text()').extract()).strip() punish_rst = ''.join( response.xpath('//td[@class="xzcf_jds"]//text()').extract()) cont_type = ''.join( response.xpath('//td[@width="270"]//text()').extract()) tr_list = response.xpath('//table[@class="xzcf_bg"]//tr') special = tr_list.pop(1) key_lst = [ ''.join(item.xpath('string(.)').extract()).strip(u':') for item in tr_list.xpath('.//td[2]') ] val_lst = [ ''.join(item.xpath('string(.)').extract()).strip() for item in tr_list.xpath('.//td[3]') ] special_dic = self.deal_special(special) detail_dic = dict(zip(key_lst, val_lst)) detail_dic.update(special_dic) if u'行政处罚' in cont_type: key = 'punish_content' else: key = 'license_content' detail_dic.update({'case_name': cmp_name, key: punish_rst}) result = map_field(detail_dic) item = self.result_item_assembler(response) item['_id'] = calc_str_md5(response.url) item['bbd_html'] = '' item['_parsed_data'] = result yield item self.logger1.info('{} save successfully'.format(response.url)) except Exception as e: self.logger1.warning( "Exception on save detail page {} {} {}".format( response.url, traceback.format_exc(), e))