class Spider(MainSpider): def __init__(self): self.task_id = "qinghai" self.site_name = "青海法院网" MainSpider.__init__(self, task_id=self.task_id) self.http = HttpRequest(self.task_id, self.site_name) self.headers = headers self.http.set_charset("gbk") def parse(self): form = { "p": "1", "LocationID": "0700000000", "sub": "" } url = "http://qhfy.chinacourt.org/fygg/index.php" log.info("开始抓取=============={}".format(self.site_name)) log.info("开始抓取=============={},第{}页".format(self.site_name, (form['p']))) self.http.http_requst(url, "post", data=form, headers=self.headers) if self.http.res_code() == 200: html_data = self.http.parse_html() object_list, total_page = self.parse_html(html_data) log.info("开始存储=============={},第{}页".format(self.site_name, (form['p']))) # 将对象列表插入数据库 self.mysql_client.session_insert_list(object_list) # 提交 self.mysql_client.session_commit() for i in range(2, int(total_page)+1): try: form["p"] = i log.info("开始抓取=============={},第{}页".format(self.site_name, (form['p']))) self.http.http_session(url, "post", data=form, headers=self.headers) if self.http.res_code() == 200: html_data = self.http.parse_html() object_list, total_page = self.parse_html(html_data) log.info("开始存储=============={},第{}页".format(self.site_name, (form['p']))) # 将对象列表插入数据库 self.mysql_client.session_insert_list(object_list) # 提交 self.mysql_client.session_commit() else: SpiderException("抓取{},第{}页异常".format(self.site_name, ( form['p'])), self.task_id, url, self.site_name) # except Exception: # 捕获异常 m = traceback.format_exc() SpiderException(m, self.task_id, url, self.site_name) # 目前为测试状态,只抓取前两页内容,正式上线前将break删掉 break else: SpiderException("抓取{},第{}页异常".format(self.site_name, (form['p'])), self.task_id, url, self.site_name) # 关闭数据库链接 self.mysql_client.session_close() log.info("抓取{}结束".format(self.site_name)) def added_parse(self): pass def parse_html(self, html): # 解析html doc = pq(html) # print(doc("td.td_pagebar").text()) total_page = "".join(re.findall("共\s.*\s页", doc("td.td_pagebar").text())).replace( "共", "").replace("页", "").strip() lis = doc('td.td_line').items() object_list = list() for x in lis: if "开庭" in x.text(): self.http.http_session("http://qhfy.chinacourt.org" + x('a').attr.href, "get", headers=self.headers) htm = self.http.parse_html() doc = pq(htm) content = doc item = dict() item["taskid"] = self.task_id item["release_date"] = "".join(re.findall("\d{4}-\d{2}-\d{2}", content("p").text())) item["title"] = x.text() t_way = self.task_id + str(time.time()) + '.txt' item["bulletin_way"] = t_way item["court_y"] = "".join(re.findall(".{2,10}人民法院", content('span.detail_content').text())) item["court_t"] = "".join(re.findall("(在.{2,10}公开)", content('span.detail_content').text()) ).replace("在", "").replace("公开", "").replace("依法", "") # item["start_court_t"] = "".join(re.findall("\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}", x('a').attr.title)) item["court_part"] = "".join(re.findall("(在.{2,10}公开)", content('span.detail_content').text()) ).replace("在", "").replace("公开", "").replace("依法", "") item["site_name"] = self.site_name # print(item) if eval(item["release_date"].replace("-", "")) > eval("20180101"): file_out(t_way, str(htm)) # 将item字典映射成对象 b = BulletinCourt(**item) object_list.append(b) # 返回对象列表和总页数 return object_list, total_page
class Spider(MainSpider): site_name = '黑龙江法院网' def __init__(self, taskid): MainSpider.__init__(self, task_id=taskid) self.http = HttpRequest(taskid, self.site_name) self.url = 'http://www.hljcourt.gov.cn/ktgg/index.php?p={page}&st={start}&et={end}' self.taskid = taskid def parse(self): log.info('开始抓取黑龙江法院网第{page}页信息'.format(page='1')) ts = datetime.date.today() tm = datetime.date.today() + datetime.timedelta(days=365) self.http.http_session(self.url.format(page='1', start=str(ts), end=str(tm)), 'get', headers=self.http.headers) self.http.set_charset('gb2312') r = self.http.parse_html() print(r) log.info('解析抓取黑龙江法院网第{page}页信息'.format(page='1')) p_list = self.parse_list(r) b_list = list() for p in p_list: try: d_url = 'http://www.hljcourt.gov.cn/ktgg/' + p['det_url'] log.info('开始抓取黑龙江法院网第{page},第{strip}条信息'.format( page='1', strip=str(p_list.index(p) + 1))) self.http.http_session(d_url, 'get', headers=self.http.headers) det_mess = self.http.parse_html() log.info('解析黑龙江法院网第{page},第{strip}条信息'.format( page='1', strip=str(p_list.index(p) + 1))) self.parse_info(det_mess, p) t_way = self.taskid + str(time.time()) + '.txt' file_out(t_way, p['html']) p['bulletin_way'] = t_way p.pop('det_url') p.pop('html') p['taskid'] = self.taskid b = BulletinCourt(**p) b_list.append(b) except Exception: m = traceback.format_exc() SpiderException(m, self.taskid, self.site_name, d_url) break log.info('存储黑龙江法院网第{page}页数据'.format(page='1')) self.mysql_client.session_insert_list(b_list) self.mysql_client.session_commit() p_total = self.page_total(r) for total in range(2, p_total): try: log.info('开始抓取黑龙江法院网第{page}页信息'.format(page=str(total))) self.http.http_session(self.url.format(page=str(total), start=str(ts), end=str(tm)), 'get', headers=self.http.headers) r = self.http.parse_html() log.info('解析黑龙江法院网第{page}页信息'.format(page=str(total))) p_list = self.parse_list(r) b_list = list() for p in p_list: try: d_url = 'http://www.hljcourt.gov.cn/ktgg/' + p[ 'det_url'] log.info('开始抓取黑龙江法院网第{page},第{strip}条信息'.format( page=str(total), strip=str(p_list.index(p) + 1))) self.http.http_session(d_url, 'get', headers=self.http.headers) det_mess = self.http.parse_html() log.info('解析黑龙江法院网第{page},第{strip}条信息'.format( page=str(total), strip=str(p_list.index(p) + 1))) self.parse_info(det_mess, p) t_way = self.taskid + str(time.time()) + '.txt' file_out(t_way, p['html']) p['bulletin_way'] = t_way p.pop('det_url') p.pop('html') p['taskid'] = self.taskid b = BulletinCourt(**p) b_list.append(b) except Exception: m = traceback.format_exc() SpiderException(m, self.taskid, self.site_name, d_url) log.info('存储黑龙江法院网第{page}页数据'.format(page=str(total))) self.mysql_client.session_insert_list(b_list) self.mysql_client.session_commit() except Exception: m = traceback.format_exc() SpiderException( m, self.taskid, self.site_name, self.url.format(end=str(tm), start=str(ts), page=str(total))) self.mysql_client.session_close() log.info('抓取黑龙江法院网结束') def added_parse(self): pass def parse_list(self, r): info_list = list() doc = pq(r) tb = doc('table tbody').children('tr').children('td') k = int(tb.size() / 5) for i in range(0, k): item = dict() title = tb.eq(i * 5 + 1).text() court_num = tb.eq(i * 5 + 2).text() court_part = tb.eq(i * 5 + 3).text() start_court_t = tb.eq(i * 5 + 4).text() det_url = tb.eq(i * 5 + 1).children('div').children('a').attr('href') item['title'] = title item['court_num'] = court_num item['court_part'] = court_part item['start_court_t'] = start_court_t item['det_url'] = det_url info_list.append(item) return info_list def parse_info(self, rs, item): doc = pq(rs) ct = doc('div.ggnr') h2 = ct('h2').text() h3 = ct('h3').text() p = ct('p').text() t1 = ct('div.text-01').text() t2 = ct('div.text-02').text() html = h2 + '\r\n' + h3 + '\r\n' + p + '\r\n' + t1 + '\r\n' + t2 item['html'] = html item['court_y'] = h2 def page_total(self, res): try: k = int( re.search('共\d*页', res).group().replace('共', '').replace('页', '')) return k except Exception: m = traceback.format_exc() SpiderException(m, self.taskid, self.site_name, '解析总页数异常') return 0
class Spider(MainSpider): def __init__(self): self.task_id = "sichuan" self.site_name = "四川法院司法公开网" MainSpider.__init__(self, task_id=self.task_id) self.http = HttpRequest(self.task_id, self.site_name) self.headers = headers def parse(self): form = {"ah": "", "page": "1", "fydm": "51", "limit": "9", "nd": ""} url = "http://111.230.134.78:8081/sdgl/app/sdggsd_list" log.info("开始抓取=============={}".format(self.site_name)) log.info("开始抓取=============={},第{}页".format(self.site_name, str(form['page']))) self.http.set_charset("unicode") self.http.http_session(url, "post", data=form, headers=self.headers) if self.http.res_code() == 200: json_data = self.http.parse_json() object_list = self.parse_list(json_data, form) log.info("开始存储=============={},第{}页".format( self.site_name, str(form['page']))) self.mysql_client.session_insert_list(object_list) self.mysql_client.session_commit() total_page = self.get_total_page(json_data) for i in range(2, total_page + 1): try: form["page"] = i log.info("开始抓取=============={},第{}页".format( self.site_name, i)) self.http.http_session(url, "post", data=form, headers=self.headers) if self.http.res_code() == 200: json_data = self.http.parse_json() object_list = self.parse_list(json_data, form) log.info("开始存储=============={},第{}页".format( self.site_name, str(form['page']))) self.mysql_client.session_insert_list(object_list) self.mysql_client.session_commit() else: SpiderException( "抓取json{},第{}页异常".format(self.site_name, str(form['page'])), self.task_id, url, self.site_name) except Exception: m = traceback.format_exc() SpiderException(m, self.task_id, url, self.site_name) # 目前为测试状态,只抓取前两页内容,正式上线前将break删掉 # break self.mysql_client.session_close() else: SpiderException( "抓取json{},第{}页异常".format(self.site_name, str(form['page'])), self.task_id, url, self.site_name) log.info("抓取{}结束".format(self.site_name)) def added_parse(self): pass def parse_list(self, json_data, form): # 解析获取到的json log.info("开始解析{}第{}页".format(self.site_name, (form['page']))) object_list = list() case_list = json_data["data"] for case in case_list: if "开庭公告" in html.unescape(case["ggbt"]): item = dict() item["release_date"] = case["clsj"] formdata = { "ggsdid": "{}".format(str(case['ggsdid'])), "ssfy": "{}".format(str(case['fydm'])) } ur = "http://111.230.134.78:8081/sdgl/app/getGgsdInfo.do" self.http.http_session(ur, "post", data=formdata, headers=self.headers) json = self.http.parse_json()["data"] item["taskid"] = self.task_id item["release_date"] = html.unescape(json.get("CLSJ")) item["title"] = html.unescape(json.get("GGBT")) item["court_y"] = get_content(json.get("SSFYMC")) # 法院 content = html.unescape(json.get("GGNR")) t_way = self.task_id + str(time.time()) + '.txt' file_out(t_way, str(content)) item["court_t"] = "".join(re.findall("法院.{1,10}庭", content)).replace( "法院", "") item["court_num"] = html.unescape(json.get("AH")) # 案号 item["trial_cause"] = html.unescape( json.get("CBRXM").strip()) # 审判人员 item['bulletin_way'] = t_way item["site_name"] = self.site_name b = BulletinCourt(**item) object_list.append(b) return object_list