def __init__(self, json_restore_path=None): headers = { #'Connetion': 'Keep-Alive', 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', "User-Agent": get_user_agent() } self.CR = CaptchaRecognition("hebei") self.requests = requests.Session() self.requests.headers.update(headers) adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) self.requests.mount('http://', adapter) self.ents = {} self.json_dict = {} self.json_restore_path = json_restore_path self.csrf = "" #验证码图片的存储路径 self.path_captcha = self.json_restore_path + '/hebei/ckcode.jpeg' #html数据的存储路径 self.html_restore_path = self.json_restore_path + '/hebei/' self.proxies = get_proxy('hebei') self.timeout = (30, 20)
class HunanCrawler(ZongjuCrawler): """湖南爬虫 """ #html数据的存储路径 html_restore_path = settings.json_restore_path + '/hunan/' #验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/hunan/ckcode.jpg' code_cracker = CaptchaRecognition('hunan') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = {'host': 'http://www.hnaic.net.cn/visit/category/a/hnaicalllist', 'official_site': 'http://gsxt.hnaic.gov.cn/notice/search/ent_info_list', 'get_checkcode': 'http://gsxt.hnaic.gov.cn/notice/captcha?preset=', 'post_checkcode': 'http://gsxt.hnaic.gov.cn/notice/search/popup_captcha', 'get_info_entry': 'http://gsxt.hnaic.gov.cn/notice/search/ent_info_list', # 获得企业入口 'open_info_entry': 'http://gsxt.hnaic.gov.cn/notice/notice/view?', # 获得企业信息页面的url,通过指定不同的tab=1-4来选择不同的内容(工商公示,企业公示...) 'open_detail_info_entry': '', } def __init__(self, json_restore_path): ZongjuCrawler.__init__(self, json_restore_path) self.json_restore_path = json_restore_path self.parser = HunanParser(self)
def __init__(self, *args, **kwargs): self.ckcode_image_path = settings.json_restore_path + '/anhui/ckcode.jpg' self.code_cracker = CaptchaRecognition('qinghai') if not os.path.exists(os.path.dirname(self.ckcode_image_path)): os.makedirs(os.path.dirname(self.ckcode_image_path)) self.urls = { 'eareName': 'http://www.ahcredit.gov.cn', 'search': 'http://www.ahcredit.gov.cn/search.jspx', 'checkCheckNo': 'http://www.ahcredit.gov.cn/checkCheckNo.jspx', 'searchList': 'http://www.ahcredit.gov.cn/searchList.jspx', 'validateCode': 'http://www.ahcredit.gov.cn/validateCode.jspx?type=0&id=0.22788021906613765', 'QueryInvList': 'http://www.ahcredit.gov.cn/QueryInvList.jspx?', 'queryInvDetailAction': 'http://www.ahcredit.gov.cn/queryInvDetailAction.jspx?', 'businessPublicity': 'http://www.ahcredit.gov.cn/businessPublicity.jspx?', 'enterprisePublicity': 'http://www.ahcredit.gov.cn/enterprisePublicity.jspx?', 'otherDepartment': 'http://www.ahcredit.gov.cn/otherDepartment.jspx?', 'justiceAssistance': 'http://www.ahcredit.gov.cn/justiceAssistance.jspx?' } self.timeout = 30 self.result_json = {} self.result_json_list = [] pass
class FujianCrawler(ZongjuCrawler): """福建爬虫 """ #html数据的存储路径 html_restore_path = settings.json_restore_path + '/fujian/' #验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/fujian/ckcode.jpg' code_cracker = CaptchaRecognition('fujian') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'http://www.fjaic.gov.cn/', 'official_site': 'http://wsgs.fjaic.gov.cn/creditpub/home', 'get_checkcode': 'http://wsgs.fjaic.gov.cn/creditpub/captcha?preset=math-01', 'post_checkcode': 'http://wsgs.fjaic.gov.cn/creditpub/security/verify_captcha', 'get_info_entry': 'http://wsgs.fjaic.gov.cn/creditpub/search/ent_info_list', 'open_info_entry': 'http://wsgs.fjaic.gov.cn/creditpub/notice/view?', #获得企业信息页面的url,通过指定不同的tab=1-4来选择不同的内容(工商公示,企业公示...) 'open_detail_info_entry': '', } def __init__(self, json_restore_path): ZongjuCrawler.__init__(self, json_restore_path) self.json_restore_path = json_restore_path self.parser = FujianParser(self)
def setUp(self): unittest.TestCase.setUp(self) from CaptchaRecognition import CaptchaRecognition self.crawler = ChongqingClawer('./enterprise_crawler/chongqing.json') self.parser = self.crawler.parser ChongqingClawer.code_cracker = CaptchaRecognition('chongqing') self.crawler.json_dict = {} self.crawler.ent_number = '500232000003942'
class HubeiCrawler(HeilongjiangClawer): """湖北爬虫 """ #html数据的存储路径 html_restore_path = settings.json_restore_path + '/hubei/' #验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/hubei/ckcode.jpg' code_cracker = CaptchaRecognition('hubei') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'www.hljaic.gov.cn', 'get_checkcode': 'http://xyjg.egs.gov.cn/ECPS_HB/validateCode.jspx?type=0', 'post_checkcode': 'http://xyjg.egs.gov.cn/ECPS_HB/checkCheckNo.jspx', 'get_info_entry': 'http://xyjg.egs.gov.cn/ECPS_HB/searchList.jspx', 'ind_comm_pub_skeleton': 'http://xyjg.egs.gov.cn/ECPS_HB/businessPublicity.jspx?id=', 'ent_pub_skeleton': 'http://xyjg.egs.gov.cn/ECPS_HB/enterprisePublicity.jspx?id=', 'other_dept_pub_skeleton': 'http://xyjg.egs.gov.cn/ECPS_HB/otherDepartment.jspx?id=', 'judical_assist_skeleton': 'http://xyjg.egs.gov.cn/ECPS_HB/justiceAssistance.jspx?id=', 'ind_comm_pub_reg_shareholder': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryInvList.jspx?', # 股东信息 'ind_comm_pub_reg_modify': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryAltList.jspx?', # 变更信息翻页 'ind_comm_pub_arch_key_persons': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryMemList.jspx?', # 主要人员信息翻页 'ind_comm_pub_spot_check': 'http://xyjg.egs.gov.cn/ECPS_HB/QuerySpotCheckList.jspx?', # 抽样检查信息翻页 'ind_comm_pub_movable_property_reg': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryMortList.jspx?', # 动产抵押登记信息翻页 'ind_comm_pub_business_exception': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryExcList.jspx?', # 经营异常信息 'ind_comm_pub_equity_ownership_reg': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryPledgeList.jspx?', # 股权出质登记信息翻页 'ind_comm_pub_arch_branch': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryChildList.jspx?', # 分支机构信息 'shareholder_detail': 'http://xyjg.egs.gov.cn/ECPS_HB/queryInvDetailAction.jspx?id=', # 投资人详情 'movable_property_reg_detail': 'http://xyjg.egs.gov.cn/ECPS_HB/mortInfoDetail.jspx?id=', # 动产抵押登记详情 'annual_report': 'http://xyjg.egs.gov.cn/ECPS_HB/QueryYearExamineDetail.jspx?id=', # 企业年报详情 } def __init__(self, json_restore_path): HeilongjiangClawer.__init__(self, json_restore_path) self.json_restore_path = json_restore_path self.parser = HubeiParser(self)
def __init__(self, json_restore_path): self.CR = CaptchaRecognition("hebei") self.requests = requests.Session() self.requests.headers.update(headers) self.ents = [] self.json_restore_path = json_restore_path self.csrf = "" #验证码图片的存储路径 self.path_captcha = settings.json_restore_path + '/hebei/ckcode.jpeg' #html数据的存储路径 self.html_restore_path = settings.json_restore_path + '/hebei/'
def __init__(self, json_restore_path): self.CR = CaptchaRecognition("guangdong") self.requests = requests.Session() self.requests.headers.update(headers) self.ents = [] self.main_host = "" self.json_dict = {} self.json_restore_path = json_restore_path self.html_restore_path = settings.json_restore_path + '/hainan/' #验证码图片的存储路径 self.path_captcha = settings.json_restore_path + '/hainan/ckcode.png'
def __init__(self, json_restore_path=None): self.html_search = None self.html_showInfo = None self.Captcha = None self.CR = CaptchaRecognition("guangdong") self.requests = requests.Session() self.requests.headers.update(headers) self.ents = [] self.main_host = "" self.json_dict = {} self.json_restore_path = json_restore_path self.dir_restore_path = settings.json_restore_path + '/guangdong/' #self.json_restore_path = settings.json_restore_path + '/guangdong.json' #验证码图片的存储路径 self.path_captcha = settings.json_restore_path + '/guangdong/ckcode.jpg'
def __init__(self, json_restore_path): """ 初始化函数 Args: json_restore_path: json文件的存储路径,所有重庆的企业,应该写入同一个文件,因此在多线程爬取时设置相同的路径。同时, 需要在写入文件的时候加锁 Returns: """ # json 数据集 # POST self.json_restore_path = json_restore_path if os.path.exists(self.json_restore_path) is False: os.makedirs(self.json_restore_path, 0775) self.parser = ChongqingParser(self) self.credit_ticket = None #html数据的存储路径 self.html_restore_path = os.path.join(self.json_restore_path, "/chongqing/") if os.path.exists(self.html_restore_path) is False: os.makedirs(self.html_restore_path, 0775) #验证码图片的存储路径 self.ckcode_image_path = os.path.join(self.html_restore_path, 'ckcode.jpg') self.code_cracker = CaptchaRecognition("chongqing") self.ent_number = None # GET self.ckcode = None self.json_ent_info = None self.json_sfxzgdbg = None self.json_sfxz = None self.json_other_qlicinfo = None self.json_other_qpeninfo = None self.json_year_report = None self.json_year_report_detail = None self.json_year_daily_transinfo = None self.json_year_daily_invsub = None self.json_year_daily_peninfo = None self.json_year_daily_licinfo = None self.json_year_daily_pleinfo = None self.json_dict = {} self.json_restore_path = json_restore_path self.parser = ChongqingParser(self) self.reqst = requests.Session() self.reqst.headers.update({ 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0'})
def __init__(self, *args, **kwargs): """江苏工商公示信息网页爬虫初始化函数 Args: json_restore_path: json文件的存储路径,所有江苏的企业,应该写入同一个文件,因此在多线程爬取时设置相同的路径。同时, 需要在写入文件的时候加锁 Returns: """ self.ent_number = None #html数据的存储路径 self.html_restore_path = settings.json_restore_path + '/jiangsu/' # 验证码图片的存储路径 self.ckcode_image_path = settings.json_restore_path + '/jiangsu/ckcode.jpg' if not os.path.exists(os.path.dirname(self.ckcode_image_path)): os.makedirs(os.path.dirname(self.ckcode_image_path)) self.code_cracker = CaptchaRecognition('jiangsu') #多线程爬取时往最后的json文件中写时的加锁保护 self.write_file_mutex = threading.Lock() self.urls = { 'host': 'www.jsgsj.gov.cn', 'official_site': 'http://www.jsgsj.gov.cn:58888/province/', 'get_checkcode': 'http://www.jsgsj.gov.cn:58888/province/rand_img.jsp?type=7', 'post_checkcode': 'http://www.jsgsj.gov.cn:58888/province/infoQueryServlet.json?queryCinfo=true', 'ind_comm_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfor_gsRelease.jsp', 'ent_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qyRelease.jsp', 'other_dept_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qtbmRelease.jsp', 'judical_assist_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryJudicialAssistance.jsp', 'annual_report_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/reportCheck/company/cPublic.jsp', 'ci_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciEnter=true', 'common_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/commonServlet.json?commonEnter=true', 'nb_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/nbServlet.json?nbEnter=true', 'ci_detail': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciDetail=true' } self.result_json = {} self.result_json_list = []
def __init__(self, json_restore_path=None): self.html_showInfo = None self.Captcha = None self.CR = CaptchaRecognition("guangdong") self.requests = requests.Session() self.requests.headers.update(headers) adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) self.requests.mount('http://', adapter) self.ents = {} self.json_restore_path = json_restore_path self.dir_restore_path = self.json_restore_path + '/neimenggu/' #验证码图片的存储路径 self.path_captcha = self.json_restore_path + '/neimenggu/ckcode.jpg' self.timeout = (30, 20) proxies = get_proxy('neimenggu') if proxies: print proxies self.requests.proxies = proxies
def __init__(self, *args, **kwargs): # 验证码图片的存储路径 self.ckcode_image_path = settings.json_restore_path + '/zongju/ckcode.jpg' self.code_cracker = CaptchaRecognition('zongju') if not os.path.exists(os.path.dirname(self.ckcode_image_path)): os.makedirs(os.path.dirname(self.ckcode_image_path)) # 多线程爬取时往最后的json文件中写时的加锁保护 self.write_file_mutex = threading.Lock() self.timeout = 40 self.urls = { 'host': 'http://qyxy.baic.gov.cn', 'official_site': 'http://gsxt.saic.gov.cn/zjgs/', 'get_checkcode': 'http://gsxt.saic.gov.cn/zjgs/captcha?preset=', 'post_checkcode': 'http://gsxt.saic.gov.cn/zjgs/search/ent_info_list', # 'get_info_entry': 'http://gsxt.saic.gov.cn/zjgs/search/ent_info_list', # 获得企业入口 'open_info_entry': 'http://gsxt.saic.gov.cn/zjgs/notice/view?', # 获得企业信息页面的url,通过指定不同的tab=1-4来选择不同的内容(工商公示,企业公示...) 'open_detail_info_entry': '' }
def __init__(self, json_restore_path=None): """ 初始化函数 Args: json_restore_path: json文件的存储路径,所有重庆的企业,应该写入同一个文件,因此在多线程爬取时设置相同的路径。同时, 需要在写入文件的时候加锁 Returns: """ super(ChongqingCrawler, self).__init__() self.json_restore_path = json_restore_path #html数据的存储路径 self.html_restore_path = os.path.join(self.json_restore_path, "chongqing") #验证码图片的存储路径 self.ckcode_image_path = os.path.join(self.html_restore_path, 'ckcode.jpg') self.code_cracker = CaptchaRecognition("chongqing") self.parser = ChongqingParser(self) self.credit_ticket = None self.ent_number = None self.ents = {} # GET self.ckcode = None self.json_ent_info = None self.json_sfxzgdbg = None self.json_sfxz = None self.json_other_qlicinfo = None self.json_other_qpeninfo = None self.json_year_report = None self.json_year_report_detail = None self.json_year_daily_transinfo = None self.json_year_daily_invsub = None self.json_year_daily_peninfo = None self.json_year_daily_licinfo = None self.json_year_daily_pleinfo = None
def __init__(self, json_restore_path=None): self.cur_time = str(int(time.time() * 1000)) self.nbxh = None self.reqst = requests.Session() self.json_restore_path = json_restore_path self.html_restore_path = self.json_restore_path + '/guizhou/' self.ckcode_image_path = self.json_restore_path + '/guizhou/ckcode.jpg' self.code_cracker = CaptchaRecognition('guizhou') self.result_json_dict = {} self.reqst.headers.update({ 'Connection': "keep-alive", 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': get_user_agent() }) self.mydict = { 'eareName': 'http://www.ahcredit.gov.cn', 'search': 'http://gsxt.gzgs.gov.cn/', 'searchList': 'http://gsxt.gzgs.gov.cn/search!searchSczt.shtml', 'validateCode': 'http://gsxt.gzgs.gov.cn/search!generateCode.shtml?validTag=searchImageCode&' } self.one_dict = { u'基本信息': 'ind_comm_pub_reg_basic', u'股东信息': 'ind_comm_pub_reg_shareholder', u'发起人信息': 'ind_comm_pub_reg_shareholder', u'股东(发起人)信息': 'ind_comm_pub_reg_shareholder', u'变更信息': 'ind_comm_pub_reg_modify', u'主要人员信息': 'ind_comm_pub_arch_key_persons', u'分支机构信息': 'ind_comm_pub_arch_branch', u'清算信息': 'ind_comm_pub_arch_liquidation', u'动产抵押登记信息': 'ind_comm_pub_movable_property_reg', u'股权出置登记信息': 'ind_comm_pub_equity_ownership_reg', u'股权出质登记信息': 'ind_comm_pub_equity_ownership_reg', u'行政处罚信息': 'ind_comm_pub_administration_sanction', u'经营异常信息': 'ind_comm_pub_business_exception', u'严重违法信息': 'ind_comm_pub_serious_violate_law', u'抽查检查信息': 'ind_comm_pub_spot_check' } self.two_dict = { u'企业年报': 'ent_pub_ent_annual_report', u'企业投资人出资比例': 'ent_pub_shareholder_capital_contribution', u'股东(发起人)及出资信息': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息(币种与注册资本一致)': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息': 'ent_pub_shareholder_capital_contribution', u'股权变更信息': 'ent_pub_equity_change', u'行政许可信息': 'ent_pub_administration_license', u'知识产权出资登记': 'ent_pub_knowledge_property', u'知识产权出质登记信息': 'ent_pub_knowledge_property', u'行政处罚信息': 'ent_pub_administration_sanction', u'变更信息': 'ent_pub_shareholder_modify' } self.three_dict = { u'行政许可信息': 'other_dept_pub_administration_license', u'行政处罚信息': 'other_dept_pub_administration_sanction' } self.four_dict = { u'股权冻结信息': 'judical_assist_pub_equity_freeze', u'司法股权冻结信息': 'judical_assist_pub_equity_freeze', u'股东变更信息': 'judical_assist_pub_shareholder_modify', u'司法股东变更登记信息': 'judical_assist_pub_shareholder_modify' } self.result_json_dict = {}
def __init__(self, json_restore_path=None): self.pripid = None self.cur_time = str(int(time.time() * 1000)) self.reqst = requests.Session() self.reqst.headers.update(headers) adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) self.reqst.mount('http://', adapter) self.json_restore_path = json_restore_path self.ckcode_image_path = self.json_restore_path + '/sichuan/ckcode.jpg' #html数据的存储路径 self.html_restore_path = self.json_restore_path + '/sichuan/' self.code_cracker = CaptchaRecognition('sichuan') self.result_json_dict = {} self.json_list = [] proxies = get_proxy('shaanxi') if proxies: print proxies self.reqst.proxies = proxies self.timeout = (30, 20) self.ents = {} self.mydict = { 'eareName': 'http://www.ahcredit.gov.cn', 'search': 'http://gsxt.scaic.gov.cn/ztxy.do?method=index&random=', 'searchList': 'http://gsxt.scaic.gov.cn/ztxy.do?method=list&djjg=&random=', 'validateCode': 'http://gsxt.scaic.gov.cn/ztxy.do?method=createYzm' } self.one_dict = {u'基本信息': 'ind_comm_pub_reg_basic', u'股东信息': 'ind_comm_pub_reg_shareholder', u'发起人信息': 'ind_comm_pub_reg_shareholder', u'股东(发起人)信息': 'ind_comm_pub_reg_shareholder', u'变更信息': 'ind_comm_pub_reg_modify', u'主要人员信息': 'ind_comm_pub_arch_key_persons', u'分支机构信息': 'ind_comm_pub_arch_branch', u'清算信息': 'ind_comm_pub_arch_liquidation', u'动产抵押登记信息': 'ind_comm_pub_movable_property_reg', u'股权出置登记信息': 'ind_comm_pub_equity_ownership_reg', u'股权出质登记信息': 'ind_comm_pub_equity_ownership_reg', u'行政处罚信息': 'ind_comm_pub_administration_sanction', u'经营异常信息': 'ind_comm_pub_business_exception', u'严重违法信息': 'ind_comm_pub_serious_violate_law', u'抽查检查信息': 'ind_comm_pub_spot_check'} self.two_dict = { u'企业年报': 'ent_pub_ent_annual_report', u'企业投资人出资比例': 'ent_pub_shareholder_capital_contribution', u'股东(发起人)及出资信息': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息(币种与注册资本一致)': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息': 'ent_pub_shareholder_capital_contribution', u'股权变更信息': 'ent_pub_equity_change', u'行政许可信息': 'ent_pub_administration_license', u'知识产权出资登记': 'ent_pub_knowledge_property', u'知识产权出质登记信息': 'ent_pub_knowledge_property', u'行政处罚信息': 'ent_pub_administration_sanction', u'变更信息': 'ent_pub_shareholder_modify' } self.three_dict = {u'行政许可信息': 'other_dept_pub_administration_license', u'行政处罚信息': 'other_dept_pub_administration_sanction'} self.four_dict = {u'股权冻结信息': 'judical_assist_pub_equity_freeze', u'司法股权冻结信息': 'judical_assist_pub_equity_freeze', u'股东变更信息': 'judical_assist_pub_shareholder_modify', u'司法股东变更登记信息': 'judical_assist_pub_shareholder_modify'}
class BeijingCrawler(Crawler): """北京工商爬虫 """ code_cracker = CaptchaRecognition('beijing') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'http://qyxy.baic.gov.cn', 'official_site': 'http://qyxy.baic.gov.cn/beijing', 'get_checkcode': 'http://qyxy.baic.gov.cn', 'post_checkcode': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!checkCode.dhtml', 'open_info_entry': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!getBjQyList.dhtml', 'ind_comm_pub_reg_basic': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!openEntInfo.dhtml?', 'ind_comm_pub_reg_shareholder': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!tzrFrame.dhtml?', 'ind_comm_pub_reg_modify': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!biangengFrame.dhtml?', 'ind_comm_pub_arch_key_persons': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!zyryFrame.dhtml?', 'ind_comm_pub_arch_branch': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!fzjgFrame.dhtml?', 'ind_comm_pub_arch_liquidation': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!qsxxFrame.dhtml?', 'ind_comm_pub_movable_property_reg': 'http://qyxy.baic.gov.cn/gjjbjTab/gjjTabQueryCreditAction!dcdyFrame.dhtml?', 'ind_comm_pub_equity_ownership_reg': 'http://qyxy.baic.gov.cn/gdczdj/gdczdjAction!gdczdjFrame.dhtml?', 'ind_comm_pub_administration_sanction': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list.dhtml?', 'ind_comm_pub_business_exception': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_jyycxx.dhtml?', 'ind_comm_pub_serious_violate_law': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_yzwfxx.dhtml?', 'ind_comm_pub_spot_check': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_ccjcxx.dhtml?', 'ent_pub_ent_annual_report': 'http://qyxy.baic.gov.cn/qynb/entinfoAction!qyxx.dhtml?', 'ent_pub_shareholder_capital_contribution': 'http://qyxy.baic.gov.cn/gdcz/gdczAction!list_index.dhtml?', 'ent_pub_equity_change': 'http://qyxy.baic.gov.cn/gdgq/gdgqAction!gdgqzrxxFrame.dhtml?', 'ent_pub_administration_license': 'http://qyxy.baic.gov.cn/xzxk/xzxkAction!list_index.dhtml?', 'ent_pub_knowledge_property': 'http://qyxy.baic.gov.cn/zscqczdj/zscqczdjAction!list_index.dhtml?', 'ent_pub_administration_sanction': 'http://qyxy.baic.gov.cn/gdgq/gdgqAction!qyxzcfFrame.dhtml?', 'other_dept_pub_administration_license': 'http://qyxy.baic.gov.cn/qtbm/qtbmAction!list_xzxk.dhtml?', 'other_dept_pub_administration_sanction': 'http://qyxy.baic.gov.cn/qtbm/qtbmAction!list_xzcf.dhtml?', 'shareholder_detail': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!touzirenInfo.dhtml?' } def __init__(self, json_restore_path=None): self.json_restore_path = json_restore_path #html数据的存储路径 html_restore_path = self.json_restore_path + '/beijing/' #验证码图片的存储路径 ckcode_image_path = self.json_restore_path + '/beijing/ckcode.jpg' self.parser = BeijingParser(self) self.credit_ticket = None if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) self.timeout = 20 def run(self, ent_number): """爬取的主函数 """ self.ent_id = '' return Crawler.run(self, ent_number) def crawl_page_by_url(self, url): resp = None try: resp = self.reqst.get(url, timeout=self.timeout, proxies=self.proxies) except requests.exceptions.ConnectionError: self.proxies = Proxies().get_proxies() logging.error("get method self.proxies changed proxies = %s\n" % (self.proxies)) return self.crawl_page_by_url(url) except requests.exceptions.Timeout: self.timeout += 5 logging.error( "get method self.timeout plus timeout = %d, proxies= %s\n" % (self.timeout, self.proxies)) return self.crawl_page_by_url(url) except Exception as e: logging.error("Other exception occured!type e = %s, proxies=%s\n" % (type(e), self.proxies)) return resp def crawl_page_by_url_post(self, url, data): resp = None try: resp = self.reqst.post(url, data, timeout=self.timeout, proxies=self.proxies) except requests.exceptions.ConnectionError: self.proxies = Proxies().get_proxies() logging.error("post method self.proxies changed. proxies = %s\n" % (self.proxies)) return self.crawl_page_by_url_post(url, data) except requests.exceptions.Timeout: self.timeout += 5 logging.error( "post method self.timeout plus, timeout= %d, proxies= %s\n" % (self.timeout, self.proxies)) return self.crawl_page_by_url_post(url, data) except Exception as e: logging.error( "Other exception occured!type e = %s, proxies=%s \n" % (type(e), self.proxies)) return resp def crawl_check_page(self): """爬取验证码页面,包括获取验证码url,下载验证码图片,破解验证码并提交 """ resp = self.crawl_page_by_url(self.urls['official_site']) if resp.status_code != 200: logging.error('failed to get official site page!') return False count = 0 while count < 15: count += 1 ckcode = self.crack_checkcode() if not ckcode[1]: logging.error( 'failed to get crackcode result, fail count = %d' % (count)) continue post_data = { 'currentTimeMillis': self.time_stamp, 'credit_ticket': self.credit_ticket, 'checkcode': ckcode[1], 'keyword': self.ent_number } next_url = self.urls['post_checkcode'] resp = self.crawl_page_by_url_post(next_url, data=post_data) if resp.status_code != 200: logging.error( 'failed to get crackcode image by url %s, fail count = %d' % (next_url, count)) continue logging.error('crack code = %s, %s, response = %s' % (ckcode[0], ckcode[1], resp.content)) if resp.content == 'fail': logging.error( 'crack checkcode failed, response content = failed, total fail count = %d' % count) time.sleep(random.uniform(0.1, 2)) continue next_url = self.urls['open_info_entry'] resp = self.crawl_page_by_url_post(next_url, data=post_data) if resp.status_code != 200: logging.error( 'failed to open info entry by url %s, fail count = %d' % (next_url, count)) continue crack_result = self.parse_post_check_page(resp.content) if crack_result: return True else: logging.error('crack checkcode failed, total fail count = %d' % count) time.sleep(random.uniform(3, 5)) return False def crawl_ind_comm_pub_pages(self): """爬取工商公示信息页面 """ for item in ( 'ind_comm_pub_reg_basic', # 登记信息-基本信息 'ind_comm_pub_reg_shareholder', # 股东信息 'ind_comm_pub_reg_modify', 'ind_comm_pub_arch_key_persons', # 备案信息-主要人员信息 'ind_comm_pub_arch_branch', # 备案信息-分支机构信息 'ind_comm_pub_arch_liquidation', # 备案信息-清算信息 'ind_comm_pub_movable_property_reg', # 动产抵押登记信息 'ind_comm_pub_equity_ownership_reg', # 股权出置登记信息 'ind_comm_pub_administration_sanction', # 行政处罚信息 'ind_comm_pub_business_exception', # 经营异常信息 'ind_comm_pub_serious_violate_law', # 严重违法信息 'ind_comm_pub_spot_check' # 抽查检查信息 ): self.get_page_json_data(item, 1) time.sleep(random.uniform(0, 3)) def crawl_ent_pub_pages(self): """爬取企业公示信息页面 """ for item in ( 'ent_pub_ent_annual_report', 'ent_pub_shareholder_capital_contribution', #企业投资人出资比例 'ent_pub_equity_change', #股权变更信息 'ent_pub_administration_license', #行政许可信息 'ent_pub_knowledge_property', #知识产权出资登记 'ent_pub_administration_sanction' #行政许可信息 ): self.get_page_json_data(item, 2) time.sleep(random.uniform(0, 3)) def crawl_other_dept_pub_pages(self): """爬取其他部门公示信息页面 """ for item in ( 'other_dept_pub_administration_license', #行政许可信息 'other_dept_pub_administration_sanction' #行政处罚信息 ): self.get_page_json_data(item, 3) def crawl_judical_assist_pub_pages(self): """爬取司法协助信息页面 """ pass def get_page_json_data(self, page_name, page_type): """获得页面的解析后的json格式数据 Args: page_name: 页面名称 page_type: 页面类型, 1 工商公示页面, 2 企业公示页面, 3 其他部门公示页面 """ page = self.get_page(page_name, page_type) pages = self.get_all_pages_of_a_section(page, page_name) if len(pages) == 1: self.json_dict[page_name] = {} json_data = self.parser.parse_page(page, page_name) if json_data: self.json_dict[page_name] = json_data else: self.json_dict[page_name] = [] for p in pages: json_data = self.parser.parse_page(p, page_name) if json_data: self.json_dict[page_name] += json_data def get_checkcode_url(self): count = 0 while count < 5: count += 1 resp = self.crawl_page_by_url(self.urls['official_site']) time.sleep(random.uniform(1, 5)) if resp.status_code != 200: logging.error('failed to get crackcode url') continue response = resp.content soup = BeautifulSoup(response, 'html.parser') ckimg_src = soup.find_all('img', id='MzImgExpPwd')[0].get('src') ckimg_src = str(ckimg_src) re_checkcode_captcha = re.compile(r'/([\s\S]*)\?currentTimeMillis') # re_currenttime_millis=re.compile(r'/CheckCodeCaptcha\?currentTimeMillis=([\s\S]*)') checkcode_type = re_checkcode_captcha.findall(ckimg_src)[0] if checkcode_type == 'CheckCodeCaptcha': #parse the pre check page, get useful information self.parse_pre_check_page(response) checkcode_url = self.urls['get_checkcode'] + ckimg_src return checkcode_url # elif checkcode_type == 'CheckCodeYunSuan': logging.error( 'can not get CheckCodeCaptcha type of checkcode img, count times = %d \n' % (count)) return None def parse_post_check_page(self, page): """解析提交验证码之后的页面,获取必要的信息 """ if page == 'fail': logging.error('checkcode error!') # if senting_open: # senting_client.captureMessage('checkcode error!') return False soup = BeautifulSoup(page, 'html.parser') r = soup.find_all('a', { 'href': "#", 'onclick': re.compile(r'openEntInfo') }) ent = '' if r: ent = r[0]['onclick'] else: logging.error('fail to find openEntInfo') return False m = re.search(r'\'([\w]*)\'[ ,]+\'([\w]*)\'[ ,]+\'([\w]*)\'', ent) if m: self.ent_id = m.group(1) self.credit_ticket = m.group(3) r = soup.find_all( 'input', { 'type': "hidden", 'name': "currentTimeMillis", 'id': "currentTimeMillis" }) if r: self.time_stamp = r[0]['value'] else: logging.error('fail to get time stamp') return True def parse_pre_check_page(self, page): """解析提交验证码之前的页面 """ soup = BeautifulSoup(page, 'html.parser') ckimg_src = soup.find_all('img', id='MzImgExpPwd')[0].get('src') ckimg_src = str(ckimg_src) re_currenttime_millis = re.compile( r'/CheckCodeCaptcha\?currentTimeMillis=([\s\S]*)') self.credit_ticket = soup.find_all('input', id='credit_ticket')[0].get('value') self.time_stamp = re_currenttime_millis.findall(ckimg_src)[0] # self.time_stamp = self.generate_time_stamp() """ def crawl_page_by_url(self, url): resp = self.crawl_page_by_url(url) if resp.status_code != 200: logging.error('failed to crawl page by url' % url) return page = resp.content time.sleep(random.uniform(0.2, 1)) # if saveingtml: # CrawlerUtils.save_page_to_file(self.html_restore_path + 'detail.html', page) return page """ def get_all_pages_of_a_section(self, page, type, url=None): """获取页面上含有 上一页、下一页跳转链接的区域的所有的数据 Args: page: 已经爬取的页面 type: 页面类型 url: 该页面的url,默认为None,因为一般可以通过 type 从 BeijingCrawler.urls 中找到 Returns: pages: 所有页面的列表 """ if not page: return page soup = BeautifulSoup(page, 'html.parser') page_count = 0 page_size = 0 pages_data = [] pages_data.append(page) r1 = soup.find_all('input', {'type': 'hidden', 'id': 'pagescount'}) r2 = soup.find_all('input', { 'type': 'hidden', 'id': 'pageSize', 'name': 'pageSize' }) if r1 and r2: page_count = int(r1[0].get('value')) page_size = int(r2[0].get('value')) else: #只有一页 return pages_data if page_count <= 1: return pages_data if not url: next_url = self.urls[type].rstrip('?') else: next_url = url for p in range(1, page_count): post_data = { 'pageNos': str(p + 1), 'clear': '', 'pageNo': str(p), 'pageSize': str(page_size), 'ent_id': self.ent_id } try: resp = self.crawl_page_by_url_post(next_url, data=post_data) if resp.status_code != 200: logging.error('failed to get all page of a section') return pages_data page = resp.content time.sleep(random.uniform(0.2, 1)) except Exception as e: logging.error( 'open new tab page failed, url = %s, page_num = %d' % (next_url, p + 1)) page = None raise e finally: if page: pages_data.append(page) return pages_data def get_page(self, type, tab): """获取页面,为了简便,在url后面添加了所有可能用到的数据,即使有多余的参数也不影响 Args: tab: 访问页面时在url后面所用到的数据。1 工商公示信息, 2 企业公示信息, 3 其他部门公示信息 """ url = CrawlerUtils.add_params_to_url( self.urls[type], { 'entId': self.ent_id, 'ent_id': self.ent_id, 'entid': self.ent_id, 'credit_ticket': self.credit_ticket, 'entNo': self.ent_number, 'entName': '', 'timeStamp': self.generate_time_stamp(), 'clear': 'true', 'str': tab }) logging.error('get %s, url:\n%s\n' % (type, url)) resp = self.crawl_page_by_url(url) if resp.status_code != 200: logging.error('get page failed by url %s' % url) return page = resp.content time.sleep(random.uniform(0.2, 1)) return page def crack_checkcode(self): """破解验证码""" ckcode = ('', '') checkcode_url = self.get_checkcode_url() if checkcode_url == None: return ckcode resp = self.crawl_page_by_url(checkcode_url) if resp.status_code != 200: logging.error('failed to get checkcode img') return ckcode page = resp.content time.sleep(random.uniform(1, 2)) self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(page) if not self.code_cracker: logging.error('invalid code cracker\n') return ckcode try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.error('exception occured when crack checkcode') ckcode = ('', '') finally: pass self.write_file_mutex.release() return ckcode def generate_time_stamp(self): """生成时间戳 """ return int(time.time())
class HeilongjiangClawer(Crawler): """黑龙江工商公示信息网页爬虫 """ # html数据的存储路径 html_restore_path = settings.json_restore_path + '/heilongjiang/' # 验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/heilongjiang/ckcode.jpg' code_cracker = CaptchaRecognition('heilongjiang') # 多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'www.hljaic.gov.cn', 'get_checkcode': 'http://gsxt.hljaic.gov.cn/validateCode.jspx?type=0', 'post_checkcode': 'http://gsxt.hljaic.gov.cn/checkCheckNo.jspx', 'get_info_entry': 'http://gsxt.hljaic.gov.cn/searchList.jspx', 'ind_comm_pub_skeleton': 'http://gsxt.hljaic.gov.cn/businessPublicity.jspx?id=', 'ent_pub_skeleton': 'http://gsxt.hljaic.gov.cn/enterprisePublicity.jspx?id=', 'other_dept_pub_skeleton': 'http://gsxt.hljaic.gov.cn/otherDepartment.jspx?id=', 'judical_assist_skeleton': 'http://gsxt.hljaic.gov.cn/justiceAssistance.jspx?id=', 'ind_comm_pub_reg_shareholder': 'http://gsxt.hljaic.gov.cn/QueryInvList.jspx?', # 股东信息 'ind_comm_pub_reg_modify': 'http://gsxt.hljaic.gov.cn/QueryAltList.jspx?', # 变更信息翻页 'ind_comm_pub_arch_key_persons': 'http://gsxt.hljaic.gov.cn/QueryMemList.jspx?', # 主要人员信息翻页 'ind_comm_pub_spot_check': 'http://gsxt.hljaic.gov.cn/QuerySpotCheckList.jspx?', # 抽样检查信息翻页 'ind_comm_pub_movable_property_reg': 'http://gsxt.hljaic.gov.cn/QueryMortList.jspx?', # 动产抵押登记信息翻页 'ind_comm_pub_business_exception': 'http://gsxt.hljaic.gov.cn/QueryExcList.jspx?', # 经营异常信息 'shareholder_detail': 'http://gsxt.hljaic.gov.cn/queryInvDetailAction.jspx?id=', # 投资人详情 'movable_property_reg_detail': 'http://gsxt.hljaic.gov.cn/mortInfoDetail.jspx?id=', # 动产抵押登记详情 'annual_report': 'http://gsxt.hljaic.gov.cn/QueryYearExamineDetail.jspx?id=', # 企业年报详情 } def __init__(self, json_restore_path): self.json_restore_path = json_restore_path self.parser = HeilongjiangParser(self) self.img_count = 1 if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) def run(self, ent_number=0): """爬取的主函数 """ return Crawler.run(self, ent_number) # return super(HeilongjiangClawer, self).run(ent_number) def crawl_check_page(self): """爬取验证码页面,包括下载验证码图片以及破解验证码 :return true or false """ count = 0 while count < 10: ck_code = self.crack_check_code() data = {'checkNo': ck_code} resp = self.reqst.post(self.urls['post_checkcode'], data=data) if resp.status_code != 200: logging.error("crawl post check page failed!") count += 1 continue if resp.content[10] == 't': data = {'checkNo': ck_code, 'entName': self.ent_number} resp = self.reqst.post(self.urls['get_info_entry'], data=data) soup = BeautifulSoup(resp.text, "html5lib") div = soup.find("div", {"style": "height:500px;"}) a = div.find("a") if a: company_id = a["href"].split('?')[1] self.company_id = company_id.split("=")[1] return True else: return False else: logging.error("crawl post check page failed!") count += 1 continue return False def crack_check_code(self): """破解验证码 :return 破解后的验证码 """ resp = self.reqst.get(self.urls['get_checkcode']) if resp.status_code != 200: logging.error('failed to get get_checkcode') return None time.sleep(random.uniform(2, 4)) self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(resp.content) try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.warn('exception occured when crack checkcode') ckcode = ('', '') finally: pass self.write_file_mutex.release() return ckcode[1] def crawl_ind_comm_pub_pages(self): """爬取工商公示信息 """ url = "%s%s" % (self.urls['ind_comm_pub_skeleton'], self.company_id) resp = self.reqst.get(url) if resp.status_code != 200: logging.error('failed to get ind_comm_pub_skeleton') self.parser.parse_ind_comm_pub_pages(resp.content) def crawl_ent_pub_pages(self): """爬取企业公示信息 """ url = "%s%s" % (self.urls['ent_pub_skeleton'], self.company_id) resp = self.reqst.get(url) if resp.status_code != 200: logging.error('failed to get ent_pub_skeleton') self.parser.parse_ent_pub_pages(resp.content) def crawl_other_dept_pub_pages(self): """爬取其他部门公示信息 """ url = "%s%s" % (self.urls['other_dept_pub_skeleton'], self.company_id) resp = self.reqst.get(url) if resp.status_code != 200: logging.error('failed to get other_dept_pub_skeleton') self.parser.crawl_other_dept_pub_pages(resp.content) def crawl_judical_assist_pub_pages(self): """爬取司法协助信息 """ url = "%s%s" % (self.urls['judical_assist_skeleton'], self.company_id) resp = self.reqst.get(url) if resp.status_code != 200: logging.error('failed to get judical_assist_skeleton') self.parser.parse_judical_assist_pub_pages(resp.content)
class LiaoningCrawler(Crawler): """辽宁工商爬虫 , 集成Crawler基类 """ code_cracker = CaptchaRecognition('liaoning') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'http://www.lngs.gov.cn/ecdomain/framework/lngs/index.jsp', 'get_checkcode': 'http://gsxt.lngs.gov.cn/saicpub/commonsSC/loginDC/securityCode.action?', 'post_checkcode': 'http://gsxt.lngs.gov.cn/saicpub/entPublicitySC/entPublicityDC/lngsSearchFpc.action', } def __init__(self, json_restore_path=None): super(LiaoningCrawler, self).__init__() self.json_restore_path = json_restore_path #html数据的存储路径 self.html_restore_path = self.json_restore_path + '/liaoning/' #验证码图片的存储路径 self.ckcode_image_path = self.json_restore_path + '/liaoning/ckcode.jpg' self.parser = LiaoningParser(self) self.proxies = get_proxy('liaoning') self.timeout = (30, 20) def run(self, _ent): """爬取的主函数 """ print self.__class__.__name__ logging.error('crawl %s .', self.__class__.__name__) if self.proxies: print self.proxies self.reqst.proxies = self.proxies if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) return Crawler.run(self, _ent) def crawl_check_page(self): """爬取验证码页面,包括获取验证码url,下载验证码图片,破解验证码并提交 """ count = 0 while count < 30: count += 1 ckcode = self.crack_checkcode() logging.debug('crack code = %s, %s' % (ckcode[0], ckcode[1])) if ckcode == "": continue post_data = { 'authCode': ckcode[1], 'solrCondition': self._ent, } next_url = self.urls['post_checkcode'] resp = self.reqst.post(next_url, data=post_data) if resp.status_code != 200: logging.error( 'failed to get crackcode image by url %s, fail count = %d' % (next_url, count)) continue m = re.search(r'codevalidator= \"(.*)\"', resp.content) status = m.group(1) if status == "fail": continue else: # m = re.search(r'searchList_paging\[(.*)\]', resp.content) # if m.group(1) != "null": # js ='[' + m.group(1) +']' # results = json.loads(js) # for item in results: # if item['entname'] == self._ent or item['regno']== self._ent: # self.ents[self._ent] = # self.type = js["enttype"] # self.pripid = js['pripid'] # self.entname = js["entname"] # self.optstate = js["optstate"] # self.regno = js["regno"] if self.analyze_showInfo(resp.content): return True else: logging.error( 'crack checkcode failed, total fail count = %d' % (count)) time.sleep(random.uniform(1, 4)) return False def analyze_showInfo(self, page): """ 判断是否成功搜索页面 分析 展示页面, 获得搜索到的企业列表 """ m = re.search(r'searchList_paging.*\[(.*)\]', page) if m: if m.group(1) != "null": js = '[' + m.group(1) + ']' results = json.loads(js) onclick = "openDetail('%s', '%s', '%s', '%s', '%s', '%s')" Ent = {} count = 0 for item in results: count += 1 if item['entname'] == self._ent or item[ 'regno'] == self._ent: Ent.clear() Ent[item['regno']] = onclick % ( item['regno'], item['enttype'], item['pripid'], item['entname'], item['optstate'], 'undefined') break else: Ent[item['regno']] = onclick % ( item['regno'], item['enttype'], item['pripid'], item['entname'], item['optstate'], 'undefined') if count == 3: break self.ents = Ent return True return False else: return False def crack_checkcode(self): """破解验证码""" checkcode_url = self.urls['get_checkcode'] resp = self.reqst.get(checkcode_url, verify=False) if resp.status_code != 200: logging.warn('failed to get checkcode img') return page = resp.content self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(page) if not self.code_cracker: print 'invalid code cracker' return '' try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.warn('exception occured when crack checkcode') ckcode = ('', '') os.remove(self.ckcode_image_path) time.sleep(10) finally: pass self.write_file_mutex.release() f.close() return ckcode @exe_time def crawl_ind_comm_pub_pages(self, *args, **kwargs): """爬取工商公示信息 """ if not len(args): return url = args[0] results = re.findall('\'(.*?)\'', url) self.regno = results[0] self.type = results[1] self.pripid = results[2] self.entname = results[3] self.optstate = results[4] self.revdate = results[5] self.parser.parse_ind_comm_pub_pages() @exe_time def crawl_ent_pub_pages(self, *args, **kwargs): """爬取企业公示信息 """ self.parser.parse_ent_pub_pages() @exe_time def crawl_other_dept_pub_pages(self, *args, **kwargs): """爬取其他部门公示信息 """ self.parser.crawl_other_dept_pub_pages() @exe_time def crawl_judical_assist_pub_pages(self, *args, **kwargs): """爬取司法协助信息 """ self.parser.parse_judical_assist_pub_pages()
class LiaoningCrawler(Crawler): """辽宁工商爬虫 """ #html数据的存储路径 html_restore_path = settings.json_restore_path + '/liaoning/' #验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/liaoning/ckcode.jpg' code_cracker = CaptchaRecognition('liaoning') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'http://www.lngs.gov.cn/ecdomain/framework/lngs/index.jsp', 'get_checkcode': 'http://gsxt.lngs.gov.cn/saicpub/commonsSC/loginDC/securityCode.action?', 'post_checkcode': 'http://gsxt.lngs.gov.cn/saicpub/entPublicitySC/entPublicityDC/lngsSearchFpc.action', } def __init__(self, json_restore_path): self.json_restore_path = json_restore_path self.parser = LiaoningParser(self) self.img_count = 1 if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) def run(self, ent_number=0): """爬取的主函数 """ return Crawler.run(self, ent_number) def crawl_check_page(self): """爬取验证码页面,包括获取验证码url,下载验证码图片,破解验证码并提交 """ count = 0 while count < 30: count += 1 ckcode = self.crack_checkcode() logging.info('crack code = %s, %s' % (ckcode[0], ckcode[1])) if ckcode == "": continue post_data = { 'authCode': ckcode[1], 'solrCondition': self.ent_number, } next_url = self.urls['post_checkcode'] resp = self.reqst.post(next_url, data=post_data) if resp.status_code != 200: logging.error( 'failed to get crackcode image by url %s, fail count = %d' % (next_url, count)) continue # logging.info('crack code = %s, %s, response = %s' %(ckcode[0], ckcode[1], resp.content)) m = re.search(r'codevalidator= \"(.*)\"', resp.content) status = m.group(1) if status == "fail": continue else: m = re.search(r'searchList_paging\(\[(.*)\]', resp.content) if m.group(1) != "null": js = json.loads(m.group(1)) self.type = js["enttype"] self.pripid = js['pripid'] self.entname = js["entname"] self.optstate = js["optstate"] self.regno = js["regno"] return True else: logging.error( 'crack checkcode failed, total fail count = %d' % (count)) time.sleep(random.uniform(2, 4)) return False def crack_checkcode(self): """破解验证码""" checkcode_url = self.urls['get_checkcode'] resp = self.reqst.get(checkcode_url, verify=False) if resp.status_code != 200: logging.warn('failed to get checkcode img') return page = resp.content self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(page) if not self.code_cracker: print 'invalid code cracker' return '' try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.warn('exception occured when crack checkcode') ckcode = ('', '') os.remove(self.ckcode_image_path) time.sleep(10) finally: pass self.write_file_mutex.release() f.close() return ckcode def crawl_ind_comm_pub_pages(self): """爬取工商公示信息 """ self.parser.parse_ind_comm_pub_pages() def crawl_ent_pub_pages(self): """爬取企业公示信息 """ self.parser.parse_ent_pub_pages() def crawl_other_dept_pub_pages(self): """爬取其他部门公示信息 """ self.parser.crawl_other_dept_pub_pages() def crawl_judical_assist_pub_pages(self): """爬取司法协助信息 """ self.parser.parse_judical_assist_pub_pages()
class JiangsuCrawler(Crawler): """江苏工商公示信息网页爬虫 """ code_cracker = CaptchaRecognition('jiangsu') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'www.jsgsj.gov.cn', 'official_site': 'http://www.jsgsj.gov.cn:58888/province/', 'get_checkcode': 'http://www.jsgsj.gov.cn:58888/province/rand_img.jsp?type=7', 'post_checkcode': 'http://www.jsgsj.gov.cn:58888/province/infoQueryServlet.json?query_info=true', 'ind_comm_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfor_gsRelease.jsp', 'ent_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qyRelease.jsp', 'other_dept_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qtbmRelease.jsp', 'judical_assist_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryJudicialAssistance.jsp', 'annual_report_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/reportCheck/company/cPublic.jsp', 'ci_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciEnter=true', 'common_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/commonServlet.json?commonEnter=true', 'nb_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/nbServlet.json?nbEnter=true', 'ci_detail': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciDetail=true' } def __init__(self, json_restore_path=None): """ 初始化函数 Args: json_restore_path: json文件的存储路径,所有江苏的企业,应该写入同一个文件,因此在多线程爬取时设置相同的路径。同时, 需要在写入文件的时候加锁 Returns: """ super(JiangsuCrawler, self).__init__() self.json_restore_path = json_restore_path #html数据的存储路径 self.html_restore_path = self.json_restore_path + '/jiangsu/' #验证码图片的存储路径 self.ckcode_image_path = self.json_restore_path + '/jiangsu/ckcode.jpg' # self.proxies = {} proxies = get_proxy('jiangsu') if proxies: print proxies self.reqst.proxies = proxies self.timeout = (30, 20) self.parser = JiangsuParser(self) self.corp_org = '' self.corp_id = '' self.corp_seq_id = '' self.common_enter_post_data = {} self.ci_enter_post_data = {} self.nb_enter_post_data = {} self.post_info = { 'ind_comm_pub_reg_basic': { 'url_type': 'ci_enter', 'post_type': 'ci_enter', 'specificQuery': 'basicInfo' }, 'ind_comm_pub_reg_shareholder': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'investmentInfor' }, 'ind_comm_pub_reg_modify': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'biangeng' }, 'ind_comm_pub_arch_key_persons': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'personnelInformation' }, 'ind_comm_pub_arch_branch': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'branchOfficeInfor' }, #'ind_comm_pub_arch_liquadition': {'url_type': 'ci_enter', 'post_type': 'common_enter', 'specificQuery': 'qsfzr'}, 'ind_comm_pub_movable_property_reg': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'dongchan' }, 'ind_comm_pub_equity_ownership_reg': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'guquanchuzhi' }, 'ind_comm_pub_administration_sanction': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'chufa' }, 'ind_comm_pub_business_exception': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'abnormalInfor' }, #'ind_comm_pub_serious_violate_law': {'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xxx'}, 'ind_comm_pub_spot_check': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'checkup' }, 'ind_comm_pub_reg_shareholder_detail': { 'url_type': 'ci_detail', 'post_type': 'ci_detail', 'specificQuery': 'investorInfor' }, 'ent_pub_annual_report': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_report_list' }, 'annual_report_detail': { 'url_type': 'nb_enter', 'post_type': 'nb_enter' }, 'ent_pub_shareholder_capital_contribution': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_tzcz' }, 'ent_pub_administrative_license': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_xzxk' }, 'ent_pub_knowledge_property': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_zscq' }, 'ent_pub_administration_sanction': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_xzcf' }, 'other_dept_pub_administration_license': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xingzheng' }, 'other_dept_pub_administration_sanction': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xingzhengchufa' }, 'judical_assist_pub_equity_freeze': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'gqdjList' }, 'judical_assist_pub_shareholder_modify': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'gdbgList' } } def run(self, ent_number): # print self.__class__.__name__ # logging.error('crawl %s.', self.__class__.__name__) if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) self.ent_number = ent_number return Crawler.run(self, self.ent_number) #分析 展示页面, 获得搜索到的企业列表 def analyze_showInfo(self, page): Ent = {} soup = BeautifulSoup(page, "html5lib") dts = soup.find_all("dt") dds = soup.find_all('dd') if len(dts) != len(dds): return False count = 0 for i in xrange(len(dts)): dt = dts[i] profile = dds[i] count += 1 url = "" ent = "" name = "" link = dt.find('a') if link and link.has_attr('onclick'): url = link['onclick'] name = link.get_text().strip() if profile and profile.span: ent = profile.span.get_text().strip() if name == self.ent_number: Ent.clear() Ent[ent] = url break if count == 3: break Ent[ent] = url self.ents = Ent return True def crawl_check_page(self): """爬取验证码页面,包括下载验证码图片以及破解验证码 :return true or false """ resp = self.request_by_method('GET', self.urls['official_site'], timeout=self.timeout) if not resp: logging.error("crawl the first page page failed!\n") return False count = 0 while count < 15: count += 1 ckcode = self.crack_checkcode() if not ckcode[1]: logging.error("crawl checkcode failed! count number = %d\n" % (count)) continue data = {'name': self.ent_number, 'verifyCode': ckcode[1]} resp = self.request_by_method('POST', self.urls['post_checkcode'], data=data, timeout=self.timeout) # print resp results = json.loads(resp)[0] if results['INFO']: if self.analyze_showInfo(results['INFO']): return True else: if results['COUNT'] == '>> 没有符合查询条件的结果 <<': break logging.error( "crawl post check page failed! count number = %d\n" % (count)) print "crawl post check page failed! count number = %d\n" % (count) time.sleep(random.uniform(3, 5)) return False def get_page_data(self, page_name, real_post_data=None): """获取页面数据,通过页面名称,和post_data, 江苏的页面中几乎全部都是post方式来获取数据 """ url = self.urls[self.post_info[page_name].get('url_type')] logging.info('get %s, url:\n%s\n' % (page_name, url)) if real_post_data: return self.get_pages(url, real_post_data) if self.post_info[page_name].get('post_type') == 'ci_enter': self.ci_enter_post_data['specificQuery'] = self.post_info[ page_name].get('specificQuery') post_data = self.ci_enter_post_data elif self.post_info[page_name].get( 'post_type') == 'ci_enter_with_recordline': self.ci_enter_with_record_line_post_data[ 'specificQuery'] = self.post_info[page_name].get( 'specificQuery') post_data = self.ci_enter_with_record_line_post_data elif self.post_info[page_name].get('post_type') == 'common_enter': self.common_enter_post_data['propertiesName'] = self.post_info[ page_name].get('propertiesName') post_data = self.common_enter_post_data elif self.post_info[page_name].get('post_type') == 'ci_detail': self.ci_detail_post_data['specificQuery'] = self.post_info[ page_name].get('specificQuery') post_data = self.ci_detail_post_data elif self.post_info[page_name].get('post_type') == 'nb_enter': self.nb_enter_post_data['propertiesName'] = self.post_info[ page_name].get('propertiesName') post_data = self.nb_enter_post_data return self.get_pages(url, post_data) def crawl_ind_comm_pub_pages(self, url): """爬取工商公示信息 """ if not self.parse_post_check_page(url): return if not self.parser.ind_comm_pub_skeleton_built: page = self.crawl_skeleton_page('ind_comm_pub_skeleton') if not page: logging.error('crawl ind comm pub skeleton failed!') return False self.parser.parse_page('ind_comm_pub_skeleton', page) for item in ( 'ind_comm_pub_reg_basic', # 登记信息-基本信息 'ind_comm_pub_reg_shareholder', # 股东信息 'ind_comm_pub_reg_modify', 'ind_comm_pub_arch_key_persons', # 备案信息-主要人员信息 'ind_comm_pub_arch_branch', # 备案信息-分支机构信息 #'ind_comm_pub_arch_liquidation', # 备案信息-清算信息, 网页中没有 'ind_comm_pub_movable_property_reg', # 动产抵押登记信息 #'ind_comm_pub_equity_ownership_reg', # 股权出置登记信息 'ind_comm_pub_administration_sanction', # 行政处罚信息 #'ind_comm_pub_business_exception', # 经营异常信息 , 网页中不存在 #'ind_comm_pub_serious_violate_law', # 严重违法信息 'ind_comm_pub_spot_check'): # 抽查检查信息 page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_ent_pub_pages(self, url): """爬取企业公示信息 """ if not self.parser.ent_pub_skeleton_built: page = self.crawl_skeleton_page('ent_pub_skeleton') if not page: logging.error('crawl ent pub skeleton failed!') return False self.parser.parse_page('ent_pub_skeleton', page) if not self.parser.annual_report_skeleton_built: page = self.crawl_skeleton_page('annual_report_skeleton') if not page: logging.error('crawl annual report skeleton failed!') return False self.parser.parse_page('annual_report_skeleton', page) for item in ( 'ent_pub_annual_report', #'ent_pub_shareholder_capital_contribution', #企业投资人出资比例 #'ent_pub_equity_change', #股权变更信息 'ent_pub_administrative_license', #行政许可信息 'ent_pub_knowledge_property', #知识产权出资登记 #'ent_pub_administration_sanction' #行政许可信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_other_dept_pub_pages(self, url): """爬取其他部门公示信息 """ if not self.parser.other_dept_pub_skeleton_built: page = self.crawl_skeleton_page('other_dept_pub_skeleton') if not page: logging.error('crawl other dept pub skeleton failed!') return False self.parser.parse_page('other_dept_pub_skeleton', page) for item in ( 'other_dept_pub_administration_license', #行政许可信息 'other_dept_pub_administration_sanction' #行政处罚信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_judical_assist_pub_pages(self, url): """爬取司法协助信息 """ if not self.parser.judical_assist_pub_skeleton_built: page = self.crawl_skeleton_page('judical_assist_pub_skeleton') if not page: logging.error('crawl judical assist skeleton failed!') return False self.parser.parse_page('judical_assist_pub_skeleton', page) for item in ( 'judical_assist_pub_equity_freeze', #股权冻结信息 'judical_assist_pub_shareholder_modify' #股东变更信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def get_pages(self, url, post_data): """获取网页数据 Args: url: url地址 post_data: post方式获取数据,返回的如果是一个列表,则将列表的所有元素都获得才返回 Returns: """ resp = self.request_by_method('POST', url, data=post_data, timeout=self.timeout) if not resp: logging.error('get all pages of a section failed!') return else: json_obj = json.loads(resp) if type(json_obj) == dict and json_obj.get( 'total', None) and int(json_obj.get('total')) > 5: post_data['pageSize'] = json_obj.get('total') resp = self.request_by_method('POST', url, data=post_data, timeout=self.timeout) if not resp: logging.error('get all pages of a section failed!') return return resp def crawl_skeleton_page(self, name): """爬取网页表格的框架页面,在江苏的网页中, 工商公示信息, 企业公示信息,其他部门公示信息,司法协助信息 所有的tab页面中的表格结构都在一个最开始的页面中给出 """ url = self.urls[name] post_data = { 'org': self.corp_org, 'id': self.corp_id, 'seq_id': self.corp_seq_id, 'reg_no': self.ent_number, 'name': self.ent_number, 'containContextPath': 'ecipplatform', 'corp_name': self.ent_number } resp = self.request_by_method('POST', url, data=post_data, timeout=self.timeout) if not resp: logging.error('crawl %s page failed, error code.\n' % (name)) return False return resp def parse_post_check_page(self, link): """解析提交验证码之后的页面,提取所需要的信息,比如corp id等 Args: page: 提交验证码之后的页面 """ m = re.findall(r"\'.*?\'", link) # queryInfor('/ecipplatform/inner_ci/ci_queryCorpInfor_gsRelease.jsp','876','126075','69','320000000000192','','ecipplatform') if m: self.corp_org = m[1] self.corp_id = m[2] self.corp_seq_id = m[3] self.common_enter_post_data = { 'showRecordLine': '1', 'specificQuery': 'commonQuery', 'propertiesName': '', 'corp_org': self.corp_org, 'corp_id': self.corp_id, 'pageNo': '1', 'pageSize': '5' } self.ci_enter_post_data = { 'org': self.corp_org, 'id': self.corp_id, 'seq_id': self.corp_seq_id, 'specificQuery': '' } self.ci_enter_with_record_line_post_data = { 'CORP_ORG': self.corp_org, 'CORP_ID': self.corp_id, 'CORP_SEQ_ID': self.corp_seq_id, 'specificQuery': '', 'pageNo': '1', 'pageSize': '5', 'showRecordLine': '1' } self.ci_detail_post_data = { 'ORG': self.corp_org, 'ID': '', 'CORP_ORG': self.corp_org, 'CORP_ID': self.corp_id, 'SEQ_ID': '', 'REG_NO': self.ent_number, 'specificQuery': '' } self.nb_enter_post_data = { 'ID': '', 'REG_NO': self.ent_number, 'showRecordLine': '0', 'specificQuery': 'gs_pb', 'propertiesName': '', 'pageNo': '1', 'pageSize': '5', 'ADMIT_MAIN': '08' } return True return False def crack_checkcode(self): """破解验证码 :return 破解后的验证码 """ resp = self.request_by_method('GET', self.urls['get_checkcode'], timeout=self.timeout) if not resp: logging.error('Failed, exception occured when getting checkcode') return ('', '') time.sleep(random.uniform(2, 4)) self.write_file_mutex.acquire() ckcode = ('', '') with open(self.ckcode_image_path, 'wb') as f: f.write(resp) try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.error('exception occured when crack checkcode') ckcode = ('', '') self.write_file_mutex.release() return ckcode def request_by_method(self, method, url, *args, **kwargs): r = None try: r = self.reqst.request(method, url, *args, **kwargs) except requests.exceptions.Timeout as err: logging.error(u'Getting url: %s timeout. %s .' % (url, err.message)) return False except requests.exceptions.ConnectionError: logging.error(u"Getting url:%s connection error ." % (url)) return False except Exception as err: logging.error(u'Getting url: %s exception:%s . %s .' % (url, type(err), err.message)) return False if r.status_code != 200: logging.error( u"Something wrong when getting url:%s , status_code=%d", url, r.status_code) return False return r.content def get_annual_report_detail(self, report_year, report_id): """获取企业年报的详细信息 """ annual_report_detail = {} post_data = self.nb_enter_post_data post_data['ID'] = report_id post_data['showRecordLine'] = '0' post_data['OPERATE_TYPE'] = '2' post_data['propertiesName'] = 'query_basicInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'企业基本信息'] = self.parser.parse_page( 'annual_report_ent_basic_info', page_data) annual_report_detail[u'企业资产状况信息'] = self.parser.parse_page( 'annual_report_ent_property_info', page_data) post_data['showRecordLine'] = '1' post_data['propertiesName'] = 'query_websiteInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'网站或网店信息'] = self.parser.parse_page( 'annual_report_web_info', page_data) post_data['propertiesName'] = 'query_investInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'对外投资信息'] = self.parser.parse_page( 'annual_report_investment_abord_info', page_data) post_data['MAIN_ID'] = report_id post_data['OPERATE_TYPE'] = '1' post_data['TYPE'] = 'NZGS' post_data['ADMIT_MAIN'] = '08' post_data['propertiesName'] = 'query_stockInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'股东及出资信息'] = self.parser.parse_page( 'annual_report_shareholder_info', page_data) post_data['propertiesName'] = 'query_InformationSecurity' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'对外提供保证担保信息'] = self.parser.parse_page( 'annual_report_external_guarantee_info', page_data) post_data['propertiesName'] = 'query_RevisionRecord' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'修改记录'] = self.parser.parse_page( 'annual_report_modify_record', page_data) return annual_report_detail
def __init__(self, json_restore_path): # self.cur_time = str(int(time.time()*1000)) self.id = None self.reqst = requests.Session() self.json_restore_path = json_restore_path self.ckcode_image_path = settings.json_restore_path + '/guangxi/ckcode.jpg' self.code_cracker = CaptchaRecognition('guangxi') self.result_json_dict = {} self.reqst.headers.update({ 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0' }) self.mydict = { 'eareName': 'http://gxqyxygs.gov.cn', 'search': 'http://gxqyxygs.gov.cn/search.jspx', 'searchList': 'http://gxqyxygs.gov.cn/searchList.jspx', 'validateCode': 'http://gxqyxygs.gov.cn/validateCode.jspx?type=0&id=0.6145392225593206' } self.search_dict = { 'eareName': 'http://gxqyxygs.gov.cn', 'search': 'http://222.143.24.157/search.jspx', 'validateCode': 'http://222.143.24.157/validateCode.jspx?type=0&id=0.8720359673599201', 'searchList': 'http://222.143.24.157/searchList.jspx', 'businessPublicity': 'http://222.143.24.157/businessPublicity.jspx?', 'enterprisePublicity': 'http://222.143.24.157/enterprisePublicity.jspx?', 'otherDepartment': 'http://222.143.24.157/otherDepartment.jspx?', 'justiceAssistance': 'http://222.143.24.157/justiceAssistance.jspx?', 'next_head': 'http://gxqyxygs.gov.cn/Query' } self.one_dict = { u'基本信息': 'ind_comm_pub_reg_basic', u'股东信息': 'ind_comm_pub_reg_shareholder', u'发起人信息': 'ind_comm_pub_reg_shareholder', u'股东(发起人)信息': 'ind_comm_pub_reg_shareholder', u'变更信息': 'ind_comm_pub_reg_modify', u'主要人员信息': 'ind_comm_pub_arch_key_persons', u'分支机构信息': 'ind_comm_pub_arch_branch', u'清算信息': 'ind_comm_pub_arch_liquidation', u'动产抵押登记信息': 'ind_comm_pub_movable_property_reg', u'股权出置登记信息': 'ind_comm_pub_equity_ownership_reg', u'股权出质登记信息': 'ind_comm_pub_equity_ownership_reg', u'行政处罚信息': 'ind_comm_pub_administration_sanction', u'经营异常信息': 'ind_comm_pub_business_exception', u'严重违法信息': 'ind_comm_pub_serious_violate_law', u'抽查检查信息': 'ind_comm_pub_spot_check' } self.two_dict = { u'企业年报': 'ent_pub_ent_annual_report', u'企业投资人出资比例': 'ent_pub_shareholder_capital_contribution', u'股东(发起人)及出资信息': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息(币种与注册资本一致)': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息': 'ent_pub_shareholder_capital_contribution', u'股权变更信息': 'ent_pub_equity_change', u'行政许可信息': 'ent_pub_administration_license', u'知识产权出资登记': 'ent_pub_knowledge_property', u'知识产权出质登记信息': 'ent_pub_knowledge_property', u'行政处罚信息': 'ent_pub_administration_sanction', u'变更信息': 'ent_pub_shareholder_modify' } self.three_dict = { u'行政许可信息': 'other_dept_pub_administration_license', u'行政处罚信息': 'other_dept_pub_administration_sanction' } self.four_dict = { u'股权冻结信息': 'judical_assist_pub_equity_freeze', u'司法股权冻结信息': 'judical_assist_pub_equity_freeze', u'股东变更信息': 'judical_assist_pub_shareholder_modify', u'司法股东变更登记信息': 'judical_assist_pub_shareholder_modify' } self.result_json_dict = {}
def __init__(self, *args, **kwargs): #html数据的存储路径 self.html_restore_path = settings.json_restore_path + '/beijing/' #验证码图片的存储路径 self.ckcode_image_path = settings.json_restore_path + '/beijing/ckcode.jpg' self.code_cracker = CaptchaRecognition('beijing') if not os.path.exists(os.path.dirname(self.ckcode_image_path)): os.makedirs(os.path.dirname(self.ckcode_image_path)) #多线程爬取时往最后的json文件中写时的加锁保护 self.write_file_mutex = threading.Lock() # self.json_restore_path = settings.json_restore_path self.credit_ticket = None if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) self.timeout = 20 self.ent_id = '' self.urls = { 'host': 'http://qyxy.baic.gov.cn', 'official_site': 'http://qyxy.baic.gov.cn/beijing', 'get_checkcode': 'http://qyxy.baic.gov.cn', 'post_checkcode': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!checkCode.dhtml', 'open_info_entry': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!getBjQyList.dhtml', 'ind_comm_pub_reg_basic': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!openEntInfo.dhtml?', 'ind_comm_pub_reg_shareholder': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!tzrFrame.dhtml?', 'ind_comm_pub_reg_modify': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!biangengFrame.dhtml?', 'ind_comm_pub_arch_key_persons': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!zyryFrame.dhtml?', 'ind_comm_pub_arch_branch': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!fzjgFrame.dhtml?', 'ind_comm_pub_arch_liquidation': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!qsxxFrame.dhtml?', 'ind_comm_pub_movable_property_reg': 'http://qyxy.baic.gov.cn/gjjbjTab/gjjTabQueryCreditAction!dcdyFrame.dhtml?', 'ind_comm_pub_equity_ownership_reg': 'http://qyxy.baic.gov.cn/gdczdj/gdczdjAction!gdczdjFrame.dhtml?', 'ind_comm_pub_administration_sanction': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list.dhtml?', 'ind_comm_pub_business_exception': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_jyycxx.dhtml?', 'ind_comm_pub_serious_violate_law': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_yzwfxx.dhtml?', 'ind_comm_pub_spot_check': 'http://qyxy.baic.gov.cn/gsgs/gsxzcfAction!list_ccjcxx.dhtml?', 'ent_pub_ent_annual_report': 'http://qyxy.baic.gov.cn/qynb/entinfoAction!qyxx.dhtml?', 'ent_pub_shareholder_capital_contribution': 'http://qyxy.baic.gov.cn/gdcz/gdczAction!list_index.dhtml?', 'ent_pub_equity_change': 'http://qyxy.baic.gov.cn/gdgq/gdgqAction!gdgqzrxxFrame.dhtml?', 'ent_pub_administration_license': 'http://qyxy.baic.gov.cn/xzxk/xzxkAction!list_index.dhtml?', 'ent_pub_knowledge_property': 'http://qyxy.baic.gov.cn/zscqczdj/zscqczdjAction!list_index.dhtml?', 'ent_pub_administration_sanction': 'http://qyxy.baic.gov.cn/gdgq/gdgqAction!qyxzcfFrame.dhtml?', 'other_dept_pub_administration_license': 'http://qyxy.baic.gov.cn/qtbm/qtbmAction!list_xzxk.dhtml?', 'other_dept_pub_administration_sanction': 'http://qyxy.baic.gov.cn/qtbm/qtbmAction!list_xzcf.dhtml?', 'shareholder_detail': 'http://qyxy.baic.gov.cn/gjjbj/gjjQueryCreditAction!touzirenInfo.dhtml?', 'annual_report_detail': 'http://qyxy.baic.gov.cn/entPub/entPubAction!gdcz_bj.dhtml', 'annual_report_detail_for_fro': 'http://qyxy.baic.gov.cn/entPub/entPubAction!qydwdb_bj.dhtml', 'annual_report_detail_change': 'http://qyxy.baic.gov.cn/entPub/entPubAction!qybg_bj.dhtml', }
class ZongjuCrawler(Crawler): """总局工商爬虫 """ code_cracker = CaptchaRecognition('zongju') # 多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = {'host': 'http://qyxy.saic.gov.cn', 'official_site': 'http://gsxt.saic.gov.cn/zjgs/', 'get_checkcode': 'http://gsxt.saic.gov.cn/zjgs/captcha?preset=', 'post_checkcode': 'http://gsxt.saic.gov.cn/zjgs/security/verify_captcha', 'get_info_entry': 'http://gsxt.saic.gov.cn/zjgs/search/ent_info_list', # 获得企业入口 'open_info_entry': 'http://gsxt.saic.gov.cn/zjgs/notice/view?', # 获得企业信息页面的url,通过指定不同的tab=1-4来选择不同的内容(工商公示,企业公示...) } def __init__(self, json_restore_path=None): super(ZongjuCrawler, self).__init__() self.json_restore_path = json_restore_path # html数据的存储路径 self.html_restore_path = self.json_restore_path + '/zongju/' # 验证码图片的存储路径 self.ckcode_image_path = self.json_restore_path + '/zongju/ckcode.jpg' self.parser = ZongjuParser(self) self.proxies = get_proxy('beijing') self.timeout = (30, 20) def run(self, _ent): """爬取的主函数 """ # self.proxies = {'http':'http://123.121.30.123:8118'} if self.proxies: print self.proxies self.reqst.proxies = self.proxies if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) return Crawler.run(self, _ent) def crawl_check_page(self): """爬取验证码页面,包括获取验证码url,下载验证码图片,破解验证码并提交 """ count = 0 next_url = self.urls['official_site'] resp = self.reqst.get(next_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('failed to get official site') return False if not self.parse_pre_check_page(resp.content): logging.error('failed to parse pre check page') return False while count < 30: count += 1 ckcode = self.crack_checkcode() if not ckcode[1]: continue post_data = {'captcha': ckcode[1], 'session.token': self.session_token} next_url = self.urls['post_checkcode'] resp = self.reqst.post(next_url, data=post_data, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('failed to get crackcode image by url %s, fail count = %d' % (next_url, count)) continue logging.error('crack code = %s, %s, response = %s' % (ckcode[0], ckcode[1], resp.content)) if resp.content == '0': logging.error('crack checkcode failed!count = %d' % (count)) continue next_url = self.urls['get_info_entry'] post_data = { 'searchType': '1', 'captcha': ckcode[1], 'session.token': self.session_token, 'condition.keyword': self._ent } resp = self.reqst.post(next_url, data=post_data, timeout=self.timeout) if resp.status_code != 200: logging.error('faild to crawl url %s' % next_url) return False if self.parse_post_check_page(resp.content): return True logging.error('crack checkcode failed, total fail count = %d' % count) print('crack checkcode failed!count = %d' % (count)) time.sleep(random.uniform(1, 3)) return False @exe_time def crawl_ind_comm_pub_pages(self, *args, **kwargs): """爬取工商公示信息页面 在总局的网站中,工商公示信息在一个页面中返回。页面中包含了多个表格,调用 Parser的 parse_ind_comm_page进行解析 在 Parser的ind_comm_pub_page 中,访问并设置 crawler中的 json_dict。 """ if not len(args): return url = args[0] m = re.search(r'[/\w\.\?]+=([\w\.=]+)&.+', url) if m: self.uuid = m.group(1) next_url = self.urls['open_info_entry'] + 'uuid=' + self.uuid + '&tab=01' resp = self.reqst.get(next_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('get ind comm pub info failed!') return False self.parser.parse_ind_comm_pub_pages(resp.content) @exe_time def crawl_ent_pub_pages(self, *args, **kwargs): """爬取企业公示信息页面 """ if not len(args): return url = args[0] m = re.search(r'[/\w\.\?]+=([\w\.=]+)&.+', url) if m: self.uuid = m.group(1) next_url = self.urls['open_info_entry'] + 'uuid=' + self.uuid + '&tab=02' resp = self.reqst.get(next_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('get ent pub info failed!') return False self.parser.parse_ent_pub_pages(resp.content) @exe_time def crawl_other_dept_pub_pages(self, *args, **kwargs): """爬取其他部门公示信息页面 """ if not len(args): return url = args[0] m = re.search(r'[/\w\.\?]+=([\w\.=]+)&.+', url) if m: self.uuid = m.group(1) next_url = self.urls['open_info_entry'] + 'uuid=' + self.uuid + '&tab=03' resp = self.reqst.get(next_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('get other dept pub info failed!') return False self.parser.parse_other_dept_pub_pages(resp.content) @exe_time def crawl_judical_assist_pub_pages(self, *args, **kwargs): """爬取司法协助信息页面 """ if not len(args): return url = args[0] m = re.search(r'[/\w\.\?]+=([\w\.=]+)&.+', url) if m: self.uuid = m.group(1) next_url = self.urls['open_info_entry'] + 'uuid=' + self.uuid + '&tab=06' resp = self.reqst.get(next_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('get judical assist info failed!') return False self.parser.parse_judical_assist_pub_pages(resp.content) def parse_post_check_page(self, page): """解析提交验证码之后的页面,获取必要的信息 """ soup = BeautifulSoup(page, 'html5lib') divs = soup.find_all('div', attrs={'class': 'list-item'}) if divs: Ent = {} count = 0 for div in divs: count += 1 link = div.find('div', attrs={'class': 'link'}) profile = div.find('div', attrs={'class': 'profile'}) url = "" ent = "" if link and link.find('a') and link.find('a').has_attr('href'): url = link.find('a')['href'] if profile and profile.span: ent = profile.span.get_text().strip() name = link.find('a').get_text().strip() if name == self._ent: Ent.clear() Ent[ent] = url break if count == 3: break Ent[ent] = url self.ents = Ent return True else: return False # div_tag = soup.find('div', attrs={'class': 'link'}) # if not div_tag: # return False # open_info_url = div_tag.find('a').get('href') # m = re.search(r'[/\w\.\?]+=([\w\.=]+)&.+', open_info_url) # if m: # self.uuid = m.group(1) # return True # else: # return False def parse_pre_check_page(self, page): """解析提交验证码之前的页面 """ soup = BeautifulSoup(page, 'html.parser') input_tag = soup.find('input', attrs={'type': 'hidden', 'name': 'session.token'}) if input_tag: self.session_token = input_tag.get('value') return True return False def crawl_page_by_url(self, url): """通过url直接获取页面 """ resp = self.reqst.get(url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('failed to crawl page by url' % url) return page = resp.content time.sleep(random.uniform(0.2, 1)) # if saveingtml: # CrawlerUtils.save_page_to_file(self.html_restore_path + 'detail.html', page) return page def crack_checkcode(self): """破解验证码""" checkcode_url = self.urls['get_checkcode'] + '&ra=' + str(random.random()) ckcode = ('', '') resp = self.reqst.get(checkcode_url, timeout=self.timeout, verify=False) if resp.status_code != 200: logging.error('failed to get checkcode img') return ckcode page = resp.content time.sleep(random.uniform(1, 2)) self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(page) if not self.code_cracker: logging.error('invalid code cracker with ckcode= None') return ckcode try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.error('exception occured when crack checkcode') ckcode = ('', '') os.remove(self.ckcode_image_path) finally: pass self.write_file_mutex.release() return ckcode
class JiangsuCrawler(Crawler): """江苏工商公示信息网页爬虫 """ #html数据的存储路径 html_restore_path = settings.json_restore_path + '/jiangsu/' #验证码图片的存储路径 ckcode_image_path = settings.json_restore_path + '/jiangsu/ckcode.jpg' code_cracker = CaptchaRecognition('jiangsu') #多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'www.jsgsj.gov.cn', 'official_site': 'http://www.jsgsj.gov.cn:58888/province/', 'get_checkcode': 'http://www.jsgsj.gov.cn:58888/province/rand_img.jsp?type=7', 'post_checkcode': 'http://www.jsgsj.gov.cn:58888/province/infoQueryServlet.json?queryCinfo=true', 'ind_comm_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfor_gsRelease.jsp', 'ent_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qyRelease.jsp', 'other_dept_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryCorpInfo_qtbmRelease.jsp', 'judical_assist_pub_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/inner_ci/ci_queryJudicialAssistance.jsp', 'annual_report_skeleton': 'http://www.jsgsj.gov.cn:58888/ecipplatform/reportCheck/company/cPublic.jsp', 'ci_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciEnter=true', 'common_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/commonServlet.json?commonEnter=true', 'nb_enter': 'http://www.jsgsj.gov.cn:58888/ecipplatform/nbServlet.json?nbEnter=true', 'ci_detail': 'http://www.jsgsj.gov.cn:58888/ecipplatform/ciServlet.json?ciDetail=true' } def __init__(self, json_restore_path=None): """ 初始化函数 Args: json_restore_path: json文件的存储路径,所有江苏的企业,应该写入同一个文件,因此在多线程爬取时设置相同的路径。同时, 需要在写入文件的时候加锁 Returns: """ self.proxies = Proxies().get_proxies() self.json_restore_path = json_restore_path self.parser = JiangsuParser(self) self.reqst = requests.Session() self.reqst.headers.update({ 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0' }) self.corp_org = '' self.corp_id = '' self.corp_seq_id = '' self.common_enter_post_data = {} self.ci_enter_post_data = {} self.nb_enter_post_data = {} self.post_info = { 'ind_comm_pub_reg_basic': { 'url_type': 'ci_enter', 'post_type': 'ci_enter', 'specificQuery': 'basicInfo' }, 'ind_comm_pub_reg_shareholder': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'investmentInfor' }, 'ind_comm_pub_reg_modify': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'biangeng' }, 'ind_comm_pub_arch_key_persons': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'personnelInformation' }, 'ind_comm_pub_arch_branch': { 'url_type': 'ci_enter', 'post_type': 'ci_enter_with_recordline', 'specificQuery': 'branchOfficeInfor' }, #'ind_comm_pub_arch_liquadition': {'url_type': 'ci_enter', 'post_type': 'common_enter', 'specificQuery': 'qsfzr'}, 'ind_comm_pub_movable_property_reg': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'dongchan' }, 'ind_comm_pub_equity_ownership_reg': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'guquanchuzhi' }, 'ind_comm_pub_administration_sanction': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'chufa' }, 'ind_comm_pub_business_exception': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'abnormalInfor' }, #'ind_comm_pub_serious_violate_law': {'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xxx'}, 'ind_comm_pub_spot_check': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'checkup' }, 'ind_comm_pub_reg_shareholder_detail': { 'url_type': 'ci_detail', 'post_type': 'ci_detail', 'specificQuery': 'investorInfor' }, 'ent_pub_annual_report': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_report_list' }, 'annual_report_detail': { 'url_type': 'nb_enter', 'post_type': 'nb_enter' }, 'ent_pub_shareholder_capital_contribution': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_tzcz' }, 'ent_pub_administrative_license': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_xzxk' }, 'ent_pub_knowledge_property': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_zscq' }, 'ent_pub_administration_sanction': { 'url_type': 'nb_enter', 'post_type': 'nb_enter', 'propertiesName': 'query_xzcf' }, 'other_dept_pub_administration_license': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xingzheng' }, 'other_dept_pub_administration_sanction': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'xingzhengchufa' }, 'judical_assist_pub_equity_freeze': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'gqdjList' }, 'judical_assist_pub_shareholder_modify': { 'url_type': 'common_enter', 'post_type': 'common_enter', 'propertiesName': 'gdbgList' } } def run(self, ent_number=0): if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) return Crawler.run(self, ent_number) ''' self.ent_number = str(ent_number) #对每个企业都指定一个html的存储目录 self.html_restore_path = self.html_restore_path + self.ent_number + '/' if settings.save_html and not os.path.exists(self.html_restore_path): CrawlerUtils.make_dir(self.html_restore_path) self.json_dict = {} if not self.crawl_check_page(): settings.logger.error('crack check code failed, stop to crawl enterprise %s' % self.ent_number) return False self.crawl_ind_comm_pub_pages() self.crawl_ent_pub_pages() self.crawl_other_dept_pub_pages() self.crawl_judical_assist_pub_pub_pages() #采用多线程,在写入文件时需要注意加锁 self.write_file_mutex.acquire() CrawlerUtils.json_dump_to_file(self.json_restore_path, {self.ent_number: self.json_dict}) self.write_file_mutex.release() return True ''' def crawl_check_page(self): """爬取验证码页面,包括下载验证码图片以及破解验证码 :return true or false """ resp = self.crawl_page_by_url(self.urls['official_site']) if not resp: logging.error("crawl the first page page failed!\n") return False count = 0 while count < 15: count += 1 ckcode = self.crack_checkcode() if not ckcode[1]: logging.error("crawl checkcode failed! count number = %d\n" % (count)) continue data = {'name': self.ent_number, 'verifyCode': ckcode[1]} resp = self.crawl_page_by_url_post(self.urls['post_checkcode'], data=data) if resp.find("onclick") >= 0 and self.parse_post_check_page(resp): return True else: logging.error( "crawl post check page failed! count number = %d\n" % (count)) time.sleep(random.uniform(5, 8)) return False def get_page_data(self, page_name, real_post_data=None): """获取页面数据,通过页面名称,和post_data, 江苏的页面中几乎全部都是post方式来获取数据 """ url = self.urls[self.post_info[page_name].get('url_type')] logging.info('get %s, url:\n%s\n' % (page_name, url)) if real_post_data: return self.get_pages(url, real_post_data) if self.post_info[page_name].get('post_type') == 'ci_enter': self.ci_enter_post_data['specificQuery'] = self.post_info[ page_name].get('specificQuery') post_data = self.ci_enter_post_data elif self.post_info[page_name].get( 'post_type') == 'ci_enter_with_recordline': self.ci_enter_with_record_line_post_data[ 'specificQuery'] = self.post_info[page_name].get( 'specificQuery') post_data = self.ci_enter_with_record_line_post_data elif self.post_info[page_name].get('post_type') == 'common_enter': self.common_enter_post_data['propertiesName'] = self.post_info[ page_name].get('propertiesName') post_data = self.common_enter_post_data elif self.post_info[page_name].get('post_type') == 'ci_detail': self.ci_detail_post_data['specificQuery'] = self.post_info[ page_name].get('specificQuery') post_data = self.ci_detail_post_data elif self.post_info[page_name].get('post_type') == 'nb_enter': self.nb_enter_post_data['propertiesName'] = self.post_info[ page_name].get('propertiesName') post_data = self.nb_enter_post_data return self.get_pages(url, post_data) def crawl_ind_comm_pub_pages(self): """爬取工商公示信息 """ if not self.parser.ind_comm_pub_skeleton_built: page = self.crawl_skeleton_page('ind_comm_pub_skeleton') if not page: logging.error('crawl ind comm pub skeleton failed!') return False self.parser.parse_page('ind_comm_pub_skeleton', page) for item in ( 'ind_comm_pub_reg_basic', # 登记信息-基本信息 'ind_comm_pub_reg_shareholder', # 股东信息 'ind_comm_pub_reg_modify', 'ind_comm_pub_arch_key_persons', # 备案信息-主要人员信息 'ind_comm_pub_arch_branch', # 备案信息-分支机构信息 #'ind_comm_pub_arch_liquidation', # 备案信息-清算信息, 网页中没有 'ind_comm_pub_movable_property_reg', # 动产抵押登记信息 #'ind_comm_pub_equity_ownership_reg', # 股权出置登记信息 'ind_comm_pub_administration_sanction', # 行政处罚信息 #'ind_comm_pub_business_exception', # 经营异常信息 , 网页中不存在 #'ind_comm_pub_serious_violate_law', # 严重违法信息 'ind_comm_pub_spot_check'): # 抽查检查信息 page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_ent_pub_pages(self): """爬取企业公示信息 """ if not self.parser.ent_pub_skeleton_built: page = self.crawl_skeleton_page('ent_pub_skeleton') if not page: logging.error('crawl ent pub skeleton failed!') return False self.parser.parse_page('ent_pub_skeleton', page) if not self.parser.annual_report_skeleton_built: page = self.crawl_skeleton_page('annual_report_skeleton') if not page: logging.error('crawl annual report skeleton failed!') return False self.parser.parse_page('annual_report_skeleton', page) for item in ( 'ent_pub_annual_report', #'ent_pub_shareholder_capital_contribution', #企业投资人出资比例 #'ent_pub_equity_change', #股权变更信息 'ent_pub_administrative_license', #行政许可信息 'ent_pub_knowledge_property', #知识产权出资登记 #'ent_pub_administration_sanction' #行政许可信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_other_dept_pub_pages(self): """爬取其他部门公示信息 """ if not self.parser.other_dept_pub_skeleton_built: page = self.crawl_skeleton_page('other_dept_pub_skeleton') if not page: logging.error('crawl other dept pub skeleton failed!') return False self.parser.parse_page('other_dept_pub_skeleton', page) for item in ( 'other_dept_pub_administration_license', #行政许可信息 'other_dept_pub_administration_sanction' #行政处罚信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def crawl_judical_assist_pub_pub_pages(self): """爬取司法协助信息 """ if not self.parser.judical_assist_pub_skeleton_built: page = self.crawl_skeleton_page('judical_assist_pub_skeleton') if not page: logging.error('crawl judical assist skeleton failed!') return False self.parser.parse_page('judical_assist_pub_skeleton', page) for item in ( 'judical_assist_pub_equity_freeze', #股权冻结信息 'judical_assist_pub_shareholder_modify' #股东变更信息 ): page_data = self.get_page_data(item) self.json_dict[item] = self.parser.parse_page(item, page_data) def get_pages(self, url, post_data): """获取网页数据 Args: url: url地址 post_data: post方式获取数据,返回的如果是一个列表,则将列表的所有元素都获得才返回 Returns: """ resp = self.crawl_page_by_url_post(url, data=post_data) if not resp: logging.error('get all pages of a section failed!') return else: json_obj = json.loads(resp) if type(json_obj) == dict and json_obj.get( 'total', None) and int(json_obj.get('total')) > 5: post_data['pageSize'] = json_obj.get('total') resp = self.crawl_page_by_url_post(url, data=post_data) if not resp: logging.error('get all pages of a section failed!') return return resp def crawl_skeleton_page(self, name): """爬取网页表格的框架页面,在江苏的网页中, 工商公示信息, 企业公示信息,其他部门公示信息,司法协助信息 所有的tab页面中的表格结构都在一个最开始的页面中给出 """ url = self.urls[name] post_data = { 'org': self.corp_org, 'id': self.corp_id, 'seq_id': self.corp_seq_id, 'reg_no': self.ent_number, 'name': self.ent_number, 'containContextPath': 'ecipplatform', 'corp_name': self.ent_number } resp = self.crawl_page_by_url_post(url, data=post_data) if not resp: logging.error('crawl %s page failed, error code.\n' % (name)) return False return resp def parse_post_check_page(self, page): """解析提交验证码之后的页面,提取所需要的信息,比如corp id等 Args: page: 提交验证码之后的页面 """ m = re.search( r'onclick=\\\"\w+\(\'([\w\./]+)\',\'(\w*)\',\'(\w*)\',\'(\w*)\',\'(\w*)\',\'(\w*)\',\'(\w*)\'\)', page) if m: self.corp_org = m.group(2) self.corp_id = m.group(3) self.corp_seq_id = m.group(4) self.common_enter_post_data = { 'showRecordLine': '1', 'specificQuery': 'commonQuery', 'propertiesName': '', 'corp_org': self.corp_org, 'corp_id': self.corp_id, 'pageNo': '1', 'pageSize': '5' } self.ci_enter_post_data = { 'org': self.corp_org, 'id': self.corp_id, 'seq_id': self.corp_seq_id, 'specificQuery': '' } self.ci_enter_with_record_line_post_data = { 'CORP_ORG': self.corp_org, 'CORP_ID': self.corp_id, 'CORP_SEQ_ID': self.corp_seq_id, 'specificQuery': '', 'pageNo': '1', 'pageSize': '5', 'showRecordLine': '1' } self.ci_detail_post_data = { 'ORG': self.corp_org, 'ID': '', 'CORP_ORG': self.corp_org, 'CORP_ID': self.corp_id, 'SEQ_ID': '', 'REG_NO': self.ent_number, 'specificQuery': '' } self.nb_enter_post_data = { 'ID': '', 'REG_NO': self.ent_number, 'showRecordLine': '0', 'specificQuery': 'gs_pb', 'propertiesName': '', 'pageNo': '1', 'pageSize': '5', 'ADMIT_MAIN': '08' } return True return False def crack_checkcode(self): """破解验证码 :return 破解后的验证码 """ resp = self.crawl_page_by_url(self.urls['get_checkcode']) if not resp: logging.error('Failed, exception occured when getting checkcode') return ('', '') time.sleep(random.uniform(2, 4)) self.write_file_mutex.acquire() ckcode = ('', '') with open(self.ckcode_image_path, 'wb') as f: f.write(resp) try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.error('exception occured when crack checkcode') ckcode = ('', '') finally: pass self.write_file_mutex.release() return ckcode def crawl_page_by_url(self, url): """根据url直接爬取页面 """ try: resp = self.reqst.get(url, proxies=self.proxies) if resp.status_code != 200: logging.error('crawl page by url failed! url = %s' % url) page = resp.content time.sleep(random.uniform(0.2, 1)) # if saveingtml: # CrawlerUtils.save_page_to_file(self.html_restore_path + 'detail.html', page) return page except Exception as e: logging.error("crawl page by url exception %s" % (type(e))) return None def crawl_page_by_url_post(self, url, data): """ 根据url和post数据爬取页面 """ r = self.reqst.post(url, data, proxies=self.proxies) time.sleep(random.uniform(0.2, 1)) if r.status_code != 200: logging.error( u"Getting page by url with post:%s\n, return status %s\n" % (url, r.status_code)) return False return r.content def get_annual_report_detail(self, report_year, report_id): """获取企业年报的详细信息 """ annual_report_detail = {} post_data = self.nb_enter_post_data post_data['ID'] = report_id post_data['showRecordLine'] = '0' post_data['OPERATE_TYPE'] = '2' post_data['propertiesName'] = 'query_basicInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'企业基本信息'] = self.parser.parse_page( 'annual_report_ent_basic_info', page_data) annual_report_detail[u'企业资产状况信息'] = self.parser.parse_page( 'annual_report_ent_property_info', page_data) post_data['showRecordLine'] = '1' post_data['propertiesName'] = 'query_websiteInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'网站或网店信息'] = self.parser.parse_page( 'annual_report_web_info', page_data) post_data['propertiesName'] = 'query_investInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'对外投资信息'] = self.parser.parse_page( 'annual_report_investment_abord_info', page_data) post_data['MAIN_ID'] = report_id post_data['OPERATE_TYPE'] = '1' post_data['TYPE'] = 'NZGS' post_data['ADMIT_MAIN'] = '08' post_data['propertiesName'] = 'query_stockInfo' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'股东及出资信息'] = self.parser.parse_page( 'annual_report_shareholder_info', page_data) post_data['propertiesName'] = 'query_InformationSecurity' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'对外提供保证担保信息'] = self.parser.parse_page( 'annual_report_external_guarantee_info', page_data) post_data['propertiesName'] = 'query_RevisionRecord' page_data = self.get_page_data('annual_report_detail', post_data) annual_report_detail[u'修改记录'] = self.parser.parse_page( 'annual_report_modify_record', page_data) return annual_report_detail
def __init__(self, json_restore_path): self.id = None self.reqst = requests.Session() self.json_restore_path = json_restore_path self.ckcode_image_path = settings.json_restore_path + '/yunnan/ckcode.jpg' if not os.path.exists(os.path.dirname(self.ckcode_image_path)): os.makedirs(os.path.dirname(self.ckcode_image_path)) self.result_json_dict = {} self.code_cracker = CaptchaRecognition('yunnan') self.reqst.headers.update({ 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0' }) useproxy = UseProxy() is_use_proxy = useproxy.get_province_is_use_proxy(province='guangxi') if not is_use_proxy: self.proxies = [] else: proxy = Proxy() self.proxies = { 'http': 'http://' + random.choice(proxy.get_proxy(num=5, province='guangxi')), 'https': 'https://' + random.choice(proxy.get_proxy(num=5, province='guangxi')) } print 'self.proxies:', self.proxies # self.proxies = [] self.mydict = { 'eareName': 'http://www.ahcredit.gov.cn', 'search': 'http://gsxt.ynaic.gov.cn/notice/', 'searchList': 'http://gsxt.ynaic.gov.cn/notice/search/ent_info_list', 'validateCode': 'http://gsxt.ynaic.gov.cn/notice/captcha?preset=&ra=0.06570781518790503' } self.one_dict = { u'基本信息': 'ind_comm_pub_reg_basic', u'股东信息': 'ind_comm_pub_reg_shareholder', u'发起人信息': 'ind_comm_pub_reg_shareholder', u'股东(发起人)信息': 'ind_comm_pub_reg_shareholder', u'合伙人信息': 'ind_comm_pub_reg_shareholder', u'变更信息': 'ind_comm_pub_reg_modify', u'主要人员信息': 'ind_comm_pub_arch_key_persons', u'分支机构信息': 'ind_comm_pub_arch_branch', u'清算信息': 'ind_comm_pub_arch_liquidation', u'动产抵押登记信息': 'ind_comm_pub_movable_property_reg', u'股权出置登记信息': 'ind_comm_pub_equity_ownership_reg', u'股权出质登记信息': 'ind_comm_pub_equity_ownership_reg', u'行政处罚信息': 'ind_comm_pub_administration_sanction', u'经营异常信息': 'ind_comm_pub_business_exception', u'严重违法信息': 'ind_comm_pub_serious_violate_law', u'抽查检查信息': 'ind_comm_pub_spot_check' } self.two_dict = { u'企业年报': 'ent_pub_ent_annual_report', u'企业投资人出资比例': 'ent_pub_shareholder_capital_contribution', u'股东(发起人)及出资信息': 'ent_pub_shareholder_capital_contribution', u'股东及出资信息(币种与注册资本一致)': 'ent_pub_shareholder_capital_contribution', u'股权变更信息': 'ent_pub_equity_change', u'行政许可信息': 'ent_pub_administration_license', u'知识产权出资登记': 'ent_pub_knowledge_property', u'知识产权出质登记信息': 'ent_pub_knowledge_property', u'行政处罚信息': 'ent_pub_administration_sanction', u'变更信息': 'ent_pub_shareholder_modify' } self.three_dict = { u'行政许可信息': 'other_dept_pub_administration_license', u'行政处罚信息': 'other_dept_pub_administration_sanction' } self.four_dict = { u'股权冻结信息': 'judical_assist_pub_equity_freeze', u'司法股权冻结信息': 'judical_assist_pub_equity_freeze', u'股东变更信息': 'judical_assist_pub_shareholder_modify', u'司法股东变更登记信息': 'judical_assist_pub_shareholder_modify' } self.result_json_dict = {}
class HeilongjiangClawer(Crawler): """黑龙江工商公示信息网页爬虫 """ code_cracker = CaptchaRecognition('heilongjiang') # 多线程爬取时往最后的json文件中写时的加锁保护 write_file_mutex = threading.Lock() urls = { 'host': 'www.hljaic.gov.cn', 'get_checkcode': 'http://gsxt.hljaic.gov.cn/validateCode.jspx?type=0', 'post_checkcode': 'http://gsxt.hljaic.gov.cn/checkCheckNo.jspx', 'get_info_entry': 'http://gsxt.hljaic.gov.cn/searchList.jspx', 'ind_comm_pub_skeleton': 'http://gsxt.hljaic.gov.cn/businessPublicity.jspx?id=', 'ent_pub_skeleton': 'http://gsxt.hljaic.gov.cn/enterprisePublicity.jspx?id=', 'other_dept_pub_skeleton': 'http://gsxt.hljaic.gov.cn/otherDepartment.jspx?id=', 'judical_assist_skeleton': 'http://gsxt.hljaic.gov.cn/justiceAssistance.jspx?id=', 'ind_comm_pub_reg_shareholder': 'http://gsxt.hljaic.gov.cn/QueryInvList.jspx?', # 股东信息 'ind_comm_pub_reg_modify': 'http://gsxt.hljaic.gov.cn/QueryAltList.jspx?', # 变更信息翻页 'ind_comm_pub_arch_key_persons': 'http://gsxt.hljaic.gov.cn/QueryMemList.jspx?', # 主要人员信息翻页 'ind_comm_pub_spot_check': 'http://gsxt.hljaic.gov.cn/QuerySpotCheckList.jspx?', # 抽样检查信息翻页 'ind_comm_pub_movable_property_reg': 'http://gsxt.hljaic.gov.cn/QueryMortList.jspx?', # 动产抵押登记信息翻页 'ind_comm_pub_business_exception': 'http://gsxt.hljaic.gov.cn/QueryExcList.jspx?', # 经营异常信息 'ent_pub_administration_license': 'http://gsxt.hljaic.gov.cn/QueryLicenseRegList.jspx?', # 行政许可信息 'shareholder_detail': 'http://gsxt.hljaic.gov.cn/queryInvDetailAction.jspx?id=', # 投资人详情 'movable_property_reg_detail': 'http://gsxt.hljaic.gov.cn/mortInfoDetail.jspx?id=', # 动产抵押登记详情 'annual_report': 'http://gsxt.hljaic.gov.cn/QueryYearExamineDetail.jspx?id=', # 企业年报详情 } def __init__(self, json_restore_path=None): # Crawler.__init__(self) super(HeilongjiangClawer, self).__init__() self.json_restore_path = json_restore_path # html数据的存储路径 self.html_restore_path = self.json_restore_path + '/heilongjiang/' # 验证码图片的存储路径 self.ckcode_image_path = self.json_restore_path + '/heilongjiang/ckcode.jpg' self.parser = HeilongjiangParser(self) self.proxies = get_proxy('heilongjiang') self.timeout = (30, 20) def run(self, _ent): """爬取的主函数 """ if self.proxies: print self.proxies self.reqst.proxies = self.proxies if not os.path.exists(self.html_restore_path): os.makedirs(self.html_restore_path) return Crawler.run(self, _ent) def request_by_method(self, method, url, *args, **kwargs): r = None try: r = self.requests.request(method, url, *args, **kwargs) except requests.exceptions.Timeout as err: logging.error(u'Getting url: %s timeout. %s .' % (url, err.message)) return False except requests.exceptions.ConnectionError: logging.error(u"Getting url:%s connection error ." % (url)) return False except Exception as err: logging.error(u'Getting url: %s exception:%s . %s .' % (url, type(err), err.message)) return False if r.status_code == 307: print "I am coming ! 307" r = self.request_by_method(method, url, *args, **kwargs) return r.content if r.status_code != 200: logging.error( u"Something wrong when getting url:%s , status_code=%d", url, r.status_code) return False return r.content def analyze_showInfo(self, page): """ 判断是否成功搜索页面 分析 展示页面, 获得搜索到的企业列表 """ soup = BeautifulSoup(page, "html5lib") divs = soup.find_all("div", {"class": "list"}) if divs: Ent = {} count = 0 for div in divs: count += 1 url = "" ent = "" link = div.find('li') if link and link.find('a') and link.find('a').has_attr('href'): url = link.find('a')['href'] else: break profile = link.find_next_sibling() if profile and profile.span: ent = profile.span.get_text().strip() name = link.find('a').get_text().strip() if name == self._ent: Ent.clear() Ent[ent] = url break if count == 3: break Ent[ent] = url self.ents = Ent return True else: return False def crawl_check_page(self): """爬取验证码页面,包括下载验证码图片以及破解验证码 :return true or false """ count = 0 while count < 10: count += 1 ck_code = self.crack_checkcode() data = {'checkNo': ck_code} resp = self.reqst.post(self.urls['post_checkcode'], data=data, timeout=self.timeout) if resp.status_code != 200: logging.error("crawl post check page failed! count=%d ." % (count)) continue if resp.content[10] == 't': data = {'checkNo': ck_code, 'entName': self._ent} resp = self.reqst.post(self.urls['get_info_entry'], data=data, timeout=self.timeout) if self.analyze_showInfo(resp.text): return True else: logging.error( "crawl post check page failed! count = %d ." % (count)) else: logging.error("crawl post check page failed! count = %d ." % (count)) time.sleep(random.uniform(1, 3)) print "crawl post check page failed! count = %d ." % (count) return False def crack_checkcode(self): """破解验证码 :return 破解后的验证码 """ resp = self.reqst.get(self.urls['get_checkcode'], timeout=self.timeout) if resp.status_code != 200: logging.error('failed to get get_checkcode') return None time.sleep(random.uniform(2, 4)) self.write_file_mutex.acquire() with open(self.ckcode_image_path, 'wb') as f: f.write(resp.content) try: ckcode = self.code_cracker.predict_result(self.ckcode_image_path) except Exception as e: logging.warn('exception occured when crack checkcode') ckcode = ('', '') finally: pass self.write_file_mutex.release() return ckcode[1] @exe_time def crawl_ind_comm_pub_pages(self, *args, **kwargs): """爬取工商公示信息 """ if not len(args): return url = args[0] company_id = url.split('?')[1] self.company_id = company_id.split("=")[1] url = "%s%s" % (self.urls['ind_comm_pub_skeleton'], self.company_id) resp = self.reqst.get(url, timeout=self.timeout) if resp.status_code != 200: logging.error('failed to get ind_comm_pub_skeleton') self.parser.parse_ind_comm_pub_pages(resp.content) @exe_time def crawl_ent_pub_pages(self, *args, **kwargs): """爬取企业公示信息 """ if not len(args): return url = args[0] company_id = url.split('?')[1] self.company_id = company_id.split("=")[1] url = "%s%s" % (self.urls['ent_pub_skeleton'], self.company_id) resp = self.reqst.get(url, timeout=self.timeout) if resp.status_code != 200: logging.error('failed to get ent_pub_skeleton') self.parser.parse_ent_pub_pages(resp.content) @exe_time def crawl_other_dept_pub_pages(self, *args, **kwargs): """爬取其他部门公示信息 """ if not len(args): return url = args[0] company_id = url.split('?')[1] self.company_id = company_id.split("=")[1] url = "%s%s" % (self.urls['other_dept_pub_skeleton'], self.company_id) resp = self.reqst.get(url, timeout=self.timeout) if resp.status_code != 200: logging.error('failed to get other_dept_pub_skeleton') self.parser.crawl_other_dept_pub_pages(resp.content) @exe_time def crawl_judical_assist_pub_pages(self, *args, **kwargs): """爬取司法协助信息 """ if not len(args): return url = args[0] company_id = url.split('?')[1] self.company_id = company_id.split("=")[1] url = "%s%s" % (self.urls['judical_assist_skeleton'], self.company_id) resp = self.reqst.get(url, timeout=self.timeout) if resp.status_code != 200: logging.error('failed to get judical_assist_skeleton') self.parser.parse_judical_assist_pub_pages(resp.content)