def initAnnualReportPre(self, module_super):
module = Module(self.getWebHtml, u"获取年报年份列表")
module.module_id = "fetch_qynb_list"
module.appendUrl(
lambda qyid, company_zch, qylx:
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/qygs_ViewQynb.pt?qyid=%s&zch=%s&qylx=%s&num=0"
% (qyid, company_zch, qylx))
module.appendHeaders(
lambda ua, qylx, qyid, company_zch: {
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "gsxt.jxaic.gov.cn",
"Connection": "keep-alive"
})
def getRightUrl(html=None):
if not html:
return []
rs = re.findall(r'<a\s+href="(.*?\?.*?nbnd=\d{4}.*?)"', html, re.S)
rs = list(set(rs))
return ['http://gsxt.jxaic.gov.cn' + x for x in rs]
module.appendOutput(name="qynb_param_list",
type=OutputType.FUNCTION,
function=getRightUrl,
show_up=OutputParameterShowUpType.OPTIONAL)
module_super.appendSubModule(module, True)
def initQynbInfo(self, module_super):
module = Module(None, u"设置年份")
def saveNbyear(qynb_url):
self.value_dict['nb_name'] = re.findall(r'nbnd=(\d{4})', qynb_url,
re.S)[0]
module.appendOutput(type=OutputType.FUNCTION, function=saveNbyear)
module_super.appendSubModule(module, True)
module = Module(self.visitQynb, u"获取企业年报")
module.appendUrl(lambda qynb_url: qynb_url)
module.appendHeaders(
lambda ua, qyid, company_zch, qylx: {
"User-Agent":
ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language":
"zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding":
"gzip, deflate",
"Host":
"gsxt.jxaic.gov.cn",
"Referer":
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/qygs_ViewQynb.pt?qyid=%s&zch=%s&qylx=%s&num=0&showgdxx=true"
% (qyid, company_zch, qylx),
"Connection":
"keep-alive"
})
module.addEvent(Event(EventType.OUTPUT_NOT_SATISFIED, retry_times=5))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=5))
module.addSleep(Sleep(2))
module_super.appendSubModule(module, True)
def initPenaltyInfo(self, module_super):
module = Module(self.visitXzcf, u"获取行政处罚信息")
module.appendUrl(
lambda qyid, company_zch, qylx:
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/gsgs_viewXzcfxx.pt?qyid=%s&zch=%s&qylx=%s&num=1&showgdxx=true"
% (qyid, company_zch, qylx))
module.appendHeaders(
lambda ua, qylx, qyid, company_zch: {
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4',
'Accept-Encoding':
'gzip, deflate, sdch',
'Connection':
'keep-alive',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent':
ua,
'Referer':
'http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/ccjcgs_ccjcgsIndexDetail.pt?qylx=%s&qyid=%s&zch=%s&tabName=1'
% (qylx, qyid, company_zch),
'Host':
'gsxt.jxaic.gov.cn'
})
module_super.appendSubModule(module, True)
def initSearchList(self):
module = Module(self.visitSearchList,u"第三步_开始搜索公司列表")
module.appendUrl("http://aic.hainan.gov.cn:1888/searchList.jspx")
module.appendHeaders(
{
"Host": "aic.hainan.gov.cn:1888",
"Connection": "keep-alive",
"Cache-Control": "max-age=0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Origin": "http://aic.hainan.gov.cn:1888",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36",
"Content-Type": "application/x-www-form-urlencoded",
"Referer": "http://aic.hainan.gov.cn:1888/search.jspx",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.8"
}
)
module.appendWebMethod("post")
module.appendPostData(lambda yzm, company_key:{
"checkNo": yzm,
"entName": company_key
})
module.appendOutput("url_list", ".//div[@class='list']//a/@href", OutputType.LIST)
module.appendOutput("name_list", ".//div[@class='list']//a/text()", OutputType.LIST)
module.appendOutput(name="search_list", type=OutputType.FUNCTION, function=lambda url_list, name_list: zip(url_list, name_list))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=20, redo_module="hn_yzm_pic"))
module.addEvent(Event(EventType.OUTPUT_NOT_SATISFIED, retry_times=20, redo_module="hn_yzm_pic"))
module.addEvent(Event(EventType.ASSERT_FAILED, retry_times=0, assert_function=lambda :False if self.report.access_type == SeedAccessType.NON_COMPANY else True))
module.appendMiddleValueMonitor("search_list")
module.addSleep(Sleep(1))
self.module_manager.appendSubModule(module)
def initConfigValidateCode(self):
module = Module(self.visitValidateCode, u"获取验证码")
module.module_id = "init_validate_code"
module.appendUrl(
"http://gsxt.hljaic.gov.cn/validateCode.jspx?type=0&id=" +
str(random.random()))
module.appendHeaders({
"Host": "gsxt.hljaic.gov.cn",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; rv:37.0) Gecko/20100101 Firefox/37.0",
"Accept": "image/png,image/*;q=0.8,*/*;q=0.5",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Referer": "http://gsxt.hljaic.gov.cn/search.jspx"
})
module.addSleep(Sleep(3))
module.appendEncoding("utf-8")
def checkValidatecode(yzm):
if not yzm:
self.holder.logging.warning(u"获取验证码失败")
return False
return True
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=100,
assert_function=checkValidatecode))
self.module_manager.appendSubModule(module, True)
def initNianBao(self, module_super):
module = Module(self.getWebHtml, u"抓取公司的年报信息")
module.appendUrl(
lambda company_url: company_url.replace('tab=01', 'tab=02'))
module.appendHeaders({
'Host': 'www.sgs.gov.cn',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive'
})
def xpaths(html):
tree = etree.HTML(html)
_list = tree.xpath('.//*[@class="info m-bottom m-top"]/tr/td/a')
qynb_list = []
for ll in _list:
url = ''.join(ll.xpath('@href')).strip()
name = ''.join(ll.xpath('text()')).replace(u'年度报告', '')
if name != u'详情':
qynb_list.append([url, name])
return qynb_list
module.appendOutput(name='qynb_list',
type=OutputType.FUNCTION,
function=xpaths,
show_up=OutputParameterShowUpType.OPTIONAL)
module_super.appendSubModule(module, True)
def initNbiter(self, module_super):
iterator = Iterator("qynb_list", "nianb")
module = Module(None, u"获取公司年报", iterator)
module_super.appendSubModule(module, True)
sub_module = Module(self.visitQynb, u"获取年报详情")
def prepare(nianb):
mv_dict = dict()
mv_dict['nb_url'] = nianb[0]
mv_dict['nb_name'] = nianb[1]
return mv_dict
sub_module.appendInput(InputType.FUNCTION, input_value=prepare)
sub_module.appendUrl('nb_url')
sub_module.appendHeaders({
'Host': 'www.sgs.gov.cn',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive'
})
module.appendSubModule(sub_module)
def initChangeInfoPage(self, module_super):
iterator = Iterator("bgxx_pages", "page_no")
module = Module(None, "进入变更信息翻页", iterator)
module_super.appendSubModule(module)
sub_module = Module(self.visitBgxx, "获取变更翻页信息")
sub_module.appendUrl(
lambda qyid:
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/gsgs_viewDjxxBgxx.pt?qyid=%s"
% qyid)
sub_module.appendWebMethod("post")
sub_module.appendPostData(lambda page_no: {
'page': page_no,
'limit': 5,
'mark': 0
})
sub_module.appendHeaders({
'Host':
'gsxt.jxaic.gov.cn',
'Connection':
'keep-alive',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4'
})
module.appendSubModule(sub_module)
def initSubmitYzm(self):
module = Module(self.getJson, u"第二步_提交验证码验证")
module.appendUrl("http://aic.hainan.gov.cn:1888/checkCheckNo.jspx")
module.appendHeaders(
{
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.8",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Host": "aic.hainan.gov.cn:1888",
"Origin": "http://aic.hainan.gov.cn:1888",
"Proxy-Connection": "keep-alive",
"Referer": "http://aic.hainan.gov.cn:1888/search.jspx",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36",
"X-Requested-With": "XMLHttpRequest"
}
)
module.appendWebMethod("post")
module.appendPostData(lambda yzm: {"checkNo": yzm})
module.addSleep(Sleep(3))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=0, redo_module="hn_yzm_pic"))
# 验证码判断条件
def submitYzmResult(json=None):
if not json:
return False
if '{success:true}' not in json:
return False
return True
module.addEvent(Event(EventType.ASSERT_FAILED, assert_function=submitYzmResult, retry_times=0, redo_module="hn_yzm_pic"))
self.module_manager.appendSubModule(module)
def initConfigBaseInfo(self, module_super):
module = Module(self.visitJbxx, "基本信息")
def prepare(company_url):
query_ = {}
for qq in map(lambda x: x.split("="),
urlparse.urlparse(company_url).query.split("&")):
query_[qq[0]] = qq[1]
print query_
return query_
module.appendInput(InputType.FUNCTION, prepare)
module.appendUrl(
lambda qyid, zch, qylx:
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/gsgs_viewDjxx.pt?qyid=%s&zch=%s&qylx=%s&num=undefined&showgdxx=true"
% (qyid, zch, qylx))
module.appendHeaders({
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36',
'Host': 'gsxt.jxaic.gov.cn'
})
module_super.appendSubModule(module, True)
def initToken(self):
module = Module(self.getWebHtml, u"令牌获取")
module.module_id = "module_token"
module.appendUrl('https://www.sgs.gov.cn/notice/search/popup_captcha')
module.appendHeaders({
'Host': 'www.sgs.gov.cn',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; rv:40.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'Referer': 'https://www.sgs.gov.cn/notice/home'
})
module.appendEncoding("utf-8")
def getToken(html):
if not html or '\"session.token\": \"' not in html:
self.holder.logging.error(u'获取session.token失败！')
return None
token = re.search(r'\"session\.token\": \"(.*?)\"', html).group(1)
if not token:
self.holder.logging.error(u'提取token失败！')
self.holder.logging.info('token: %s' % token)
return token
module.appendOutput(name="token",
type=OutputType.FUNCTION,
function=getToken)
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=100))
module.addEvent(Event(EventType.OUTPUT_NOT_SATISFIED, retry_times=100))
self.module_manager.appendSubModule(module)
def initSearchPage(self, module_super):
module = Module(self.visitSearchList, u"搜索列表-翻页")
module.appendUrl('https://www.sgs.gov.cn/notice/search/ent_info_list')
module.appendHeaders({
'Host':
'www.sgs.gov.cn',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; rv:40.0) Gecko/20100101 Firefox/40.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language':
'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding':
'gzip, deflate',
'Referer':
'https://www.sgs.gov.cn/notice/home',
'Content-Type':
'application/x-www-form-urlencoded'
})
module.appendWebMethod("post")
module.appendPostData(
lambda token, company_key, page_no: {
'searchType': '1',
'captcha': 0,
'session.token': token,
'condition.keyword': company_key,
'condition.pageNo': page_no
})
module.appendOutput("search_list", './/div[@class="list-item"]',
OutputType.LIST)
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=2))
module_super.appendSubModule(module)
def initConfigHomePage(self):
module = Module(self.visitHomePage, u"首页")
module.module_id = "module_home_page"
module.appendUrl("http://211.141.74.198:8081/aiccips/")
module.appendHeaders({
'Connection':
'keep-alive',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4',
'Accept-Encoding':
'gzip, deflate, sdch',
'Cache-Control':
'max-age=0',
'Referer':
'http://211.141.74.198:8081/aiccips/',
'Host':
'211.141.74.198:8081',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:36.0) Gecko/20100101 Firefox/36.0'
})
module.appendCookie("cookie")
module.appendOutput("csrf", ".//input[@name='_csrf']/@value",
OutputType.LIST)
module.appendMiddleValueMonitor("csrf")
module.addSleep(Sleep(3))
self.module_manager.appendSubModule(module, True)
def initChangeInfoPage(self, module_super):
iterator = Iterator("bgxx_pages", "page_no")
module = Module(None, u"进入变更信息翻页", iterator)
module_super.appendSubModule(module)
sub_module = Module(self.visitBgxx, u"获取变更翻页信息")
sub_module.appendUrl(
lambda qyid:
"http://gsxt.jxaic.gov.cn/ECPS/ccjcgs/gsgs_viewDjxxBgxx.pt?qyid=%s"
% qyid)
sub_module.appendWebMethod("post")
sub_module.appendPostData(lambda page_no: {
'page': page_no,
'limit': 5,
'mark': 0
})
sub_module.appendHeaders(
lambda ua: {
'Host': 'gsxt.jxaic.gov.cn',
'Connection': 'keep-alive',
'User-Agent': ua,
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4'
})
module.appendSubModule(sub_module, True)
def initConfigValidateCode(self):
module = Module(self.visitValidateCode, u"获取验证码")
module.module_id = "init_validate_code"
module.appendUrl(
"http://www.nmgs.gov.cn:7001/aiccips/verify.html?random=" +
str(random.random()))
module.appendHeaders({
'Host':
'www.nmgs.gov.cn:7001',
'Connection':
'keep-alive',
'Accept':
'image/webp,*/*;q=0.8',
'Referer':
'http://www.nmgs.gov.cn:7001/aiccips/',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4'
})
module.addSleep(Sleep(3))
module.appendEncoding("utf-8")
def checkValidatecode(yzm):
if not yzm:
return False
return True
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=100,
assert_function=checkValidatecode))
self.module_manager.appendSubModule(module, True)
def initConfigValidateCode(self):
module = Module(self.visitValidateCode, u'获取验证码图片')
module.module_id = "check_validatecode"
module.appendUrl(
"http://gsxt.scaic.gov.cn/ztxy.do?method=createYzm&dt=" +
str(int(time.time())) + "&random=" + str(int(time.time())))
module.appendHeaders(
lambda ua: {
"Host": "gsxt.scaic.gov.cn",
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8",
"Accept-Encoding": "gzip, deflate, sdch",
"Connection": "keep-alive",
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': 1
})
#对验证码进行简单的断言
def assertYzm(yzm=None):
print 'Yzm , ', yzm
if isinstance(yzm, int):
return True
return True if yzm else False
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=100,
assert_function=assertYzm,
redo_module="check_validatecode"))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=100))
module.addSleep(Sleep(2))
self.module_manager.appendSubModule(module, True)
def initCompanyInfo(self, module_super):
module = Module(self.visitJbxx, u"基本信息")
module.appendUrl(lambda com : "http://218.57.139.24/pub/"+com[1])
# def pri_com(com, company_zch):
# print 'COMMMMMMoC ', com[0], com[1], company_zch
module.appendHeaders(lambda ua: {
"Host": "218.57.139.24",
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
'Referer':'http://218.57.139.24/'})
module.appendOutput("company_zch_list", '//table[1]/tr[2]/td[1]/text()', OutputType.LIST)
def setCompanyZch(company_zch_list=None):
self.value_dict['company_zch'] = company_zch_list[0].strip() if company_zch_list else None
module.appendOutput(type=OutputType.FUNCTION, function=setCompanyZch)
module.addEvent(Event(EventType.OUTPUT_NOT_SATISFIED, retry_times=5, redo_module='home_page'))
def getGdxxParms(html):
return re.findall(r'\,"recid":"(.+?)",', html, re.S) if html else []
module.appendOutput(name="recid_list", type=OutputType.FUNCTION, function=getGdxxParms, show_up=OutputParameterShowUpType.OPTIONAL) #提取股东详情的list
#module.appendOutput(type=OutputType.FUNCTION, function=pri_com)
module.addSleep(Sleep(2))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=5, redo_module='home_page'))
module_super.appendSubModule(module, True)
def checkCompanyName(self):
module = Module(self.getWebHtml, u'验证公司名称')
module.appendUrl(
"http://gsxt.scaic.gov.cn/keyword.do?method=keywordFilter&random="
+ str(int(time.time())))
module.appendHeaders(
lambda ua: {
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "gsxt.scaic.gov.cn",
"Origin": "http://gsxt.scaic.gov.cn",
"Referer": "http://gsxt.scaic.gov.cn/ztxy.do?method=index",
"Connection": "keep-alive"
})
module.appendPostData(lambda company_key: {'qymc': company_key})
module.appendWebMethod('post')
def assertRecode(html):
if self.report.access_type == SeedAccessType.NON_COMPANY:
self.report.access_type = SeedAccessType.ERROR
return True if html and html.strip() == '1' else False
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=100,
assert_function=assertRecode,
redo_module="check_validatecode"))
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=100))
module.addSleep(Sleep(3))
self.module_manager.appendSubModule(module, True)
def initShareHolderDetail(self, module_super):
iterator = Iterator("xh_pripid", "xh_prid")
module = Module(None, "进入股东详情", iterator)
module.module_id = "fetch_gdxq_info"
module_super.appendSubModule(module, True)
sub_module = Module(self.visitGdxq, u"获取股东翻页信息")
sub_module.appendUrl('http://gsxt.scaic.gov.cn/ztxy.do')
sub_module.appendHeaders(
lambda ua: {
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "gsxt.scaic.gov.cn",
"Origin": "http://gsxt.scaic.gov.cn",
"Referer": "http://gsxt.scaic.gov.cn/ztxy.do",
"Connection": "keep-alive"
})
sub_module.appendWebMethod("post")
sub_module.appendPostData(
lambda xh_prid: {
'maent.pripid': xh_prid[1],
'maent.entbigtype': xh_prid[0],
'random': str(int(time.time() * 1000)),
'method': 'tzrCzxxDetial',
'random': str(int(time.time() * 1000))
})
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=5))
module.addSleep(Sleep(2))
module.appendSubModule(sub_module, True)
def initPenaltyInfo(self, module_super):
module = Module(self.visitXzcf, u"获取行政处罚信息")
module.appendUrl("http://gsxt.scaic.gov.cn/ztxy.do")
module.appendHeaders(
lambda ua: {
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "gsxt.scaic.gov.cn",
"Origin": "http://gsxt.scaic.gov.cn",
"Referer": "http://gsxt.scaic.gov.cn/ztxy.do?method=index",
"Connection": "keep-alive"
})
module.appendWebMethod("post")
module.appendPostData(
lambda pripid, entbigtype: {
'czmk': 'czmk3',
'maent.entbigtype': entbigtype,
'maent.pripid': pripid,
'method': 'cfInfo',
'random': str(int(time.time() * 1000))
})
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=5))
module.addSleep(Sleep(2))
module_super.appendSubModule(module, True)
def initConfigBaseInfo(self, module_super):
module = Module(self.visitJbxx, u"基本信息")
module.appendPostData(
lambda pripid, entbigtype: {
'djjg': '',
'maent.entbigtype': entbigtype,
'maent.pripid': pripid,
'method': 'qyInfo',
'random': str(int(time.time() * 1000))
})
module.appendWebMethod("post")
module.appendUrl("http://gsxt.scaic.gov.cn/ztxy.do")
module.appendHeaders(
lambda ua: {
"User-Agent": ua,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "gsxt.scaic.gov.cn",
"Origin": "http://gsxt.scaic.gov.cn",
"Referer": "http://gsxt.scaic.gov.cn/ztxy.do?method=index",
"Connection": "keep-alive"
})
module.appendOutput("company_zch_list",
'//table[1]/tr[2]/td[1]/text()', OutputType.LIST)
module.addEvent(Event(EventType.OUTPUT_NOT_SATISFIED, retry_times=5))
def setCompanyZch(company_zch_list=None):
self.value_dict['company_zch'] = company_zch_list[
0] if company_zch_list else None
module.appendOutput(type=OutputType.FUNCTION, function=setCompanyZch)
#对公司名字和注册号码断言
def assertNameZch(company_name=None, company_zch=None):
if company_name and company_zch and (0 < len(company_name) <
100) and (0 < len(company_zch)
< 100):
#self.report.access_type = SeedAccessType.OK
return True
return False
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=5,
assert_function=assertNameZch))
#从基本信息页面中提取股东详情的参数，组成访问股东详情的post参数
def getXhPripid(html):
return re.findall(r'\s+onclick="showRyxx\(\'(.+?)\'\,\'(.+?)\'\)"',
html, re.S)
module.appendOutput(name='xh_pripid',
type=OutputType.FUNCTION,
function=getXhPripid,
show_up=OutputParameterShowUpType.OPTIONAL)
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=5))
module.addSleep(Sleep(2))
module_super.appendSubModule(module, True)
def initAnnualReportPre(self, module_super):
module = Module(self.getWebHtml, u"获取年报年份列表")
module.module_id = "fetch_qynb_list"
module.appendUrl(lambda com :"http://218.57.139.24/pub/qygsdetail/%s/%s"%(com[3], com[2]))
module.appendHeaders(lambda ua, com: {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Host": "218.57.139.24",
"Referer": "http://218.57.139.24/pub/"+com[1],
"Connection": "keep-alive"})
module.appendOutput("qynb_search_parms", '//table/tr/td/a', OutputType.LIST, show_up=OutputParameterShowUpType.OPTIONAL)
def getQynbParms(qynb_search_parms=None):
#print 'qynb_search_parms :', qynb_search_parms
if not qynb_search_parms:
return []
rt_list = []
for tag_a in qynb_search_parms:
href = tag_a.xpath('./@href')
nb_name = tag_a.xpath('./text()')
# print href, '---->' , nb_name
if href and nb_name:
nb_year = re.findall(r'.*?(\d{4}).+?', nb_name[0].strip(), re.S)
# print 'nb_year, ', nb_year
if nb_year:
rt_list.append((nb_year[0].strip(), href[0].strip()))
# print 'rt_lsit ', rt_list
return rt_list
module.appendOutput(name="qinb_param_list", type=OutputType.FUNCTION, function=getQynbParms, show_up=OutputParameterShowUpType.OPTIONAL)
module.addSleep(Sleep(2))
module_super.appendSubModule(module, True)
def initAnnualReport(self, module_super):
iterator = Iterator(seeds="nb_list",param_name="nb")
module=Module(None, u"遍历企业年报列表获取Url",iterator)
module_super.appendSubModule(module)
# self.initAnnualReportInfo(module)
#
# def initAnnualReportInfo(self, module_super):
# module = Module(self.visitQynb, u"获取企业年报详细信息")
sub_module = Module(self.visitQynb, u"获取企业年报详细信息")
def annual_convert(nb):
con_dict = dict()
con_dict["nb_url"] = "http://aic.hainan.gov.cn:1888%s" % ''.join(nb.xpath("@href"))
con_dict["nb_name"] = ''.join(nb.xpath("text()")).replace(u"年度报告", "")
return con_dict
sub_module.appendInput(InputType.FUNCTION, input_value=annual_convert)
sub_module.appendUrl("nb_url")
sub_module.appendHeaders(
lambda company_url:
{
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.8",
"Cache-Control": "max-age=0",
"Connection": "keep-alive",
"Host": "aic.hainan.gov.cn:1888",
"Referer": company_url,
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"
}
)
sub_module.addSleep(Sleep(2))
module.appendSubModule(sub_module)
def initCookie(self):
module = Module(self.getWebHtml, u"获取cookie")
module.module_id = "module_cookie"
module.appendUrl('http://211.141.74.198:8081/aiccips/')
module.appendHeaders(
lambda ua: {
'Connection': 'keep-alive',
'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4',
'Accept-Encoding': 'gzip, deflate, sdch',
'Cache-Control': 'max-age=0',
'Referer': 'http://211.141.74.198:8081/aiccips/',
'Host': '211.141.74.198:8081',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent': ua
})
def getCookie(html):
pattern = re.compile(r'\}\(([\s\S]*?\{\})\)\)', re.S)
result = pattern.search(html).group(1)
params = result.split(',')
if len(params) != 6:
raise Exception("cookie获取失败！")
strstr = 'var document = {};var window = {};document[\'cookie\'] = "";window[\'location\'] ={}; \
window[\'location\'][\'reload\'] = function(){};eval(function (p, a, c, k, e, r) { \
e = function(c) { \
return c.toString(a) \
}; \
if (!\'\'.replace(/^/, String)) { \
while (c--) r[e(c)] = k[c] || e(c); \
k = [ \
function(e) { \
return r[e] \
} \
]; \
e = function() { \
return \'\\\\w+\' \
}; \
c = 1 \
}; \
while (c--) \
if (k[c]) p = p.replace(new RegExp(\'\\\\b\' + e(c) + \'\\\\b\', \'g\'), k[c]); \
return p \
}(' + params[0] + ',' + params[1] + ',' + params[2] + ',' + params[
3] + ',' + params[4] + ',' + params[5] + ')); \
challenge();var a = document[\'cookie\'];'
with PyV8.JSContext() as se:
se.eval(strstr)
a = se.locals.a
cookie = a.split('=')[1].split(';')[0]
cookie_temp1 = dict({'ROBOTCOOKIEID': cookie})
return cookie_temp1
module.appendOutput(name="cookie",
type=OutputType.FUNCTION,
function=getCookie)
module.appendMiddleValueMonitor("cookie")
module.addSleep(Sleep(3))
self.module_manager.appendSubModule(module, True)
def getCmpnySereachList(self):
module = Module(self.visitSearchList, u"抓取公司列表")
module.module_id = "get_search_list"
module.appendUrl(
"http://www.nmgs.gov.cn:7001/aiccips/CheckEntContext/showInfo.html"
)
module.appendHeaders({
'Host':
'www.nmgs.gov.cn:7001',
'Connection':
'keep-alive',
'Cache-Control':
'max-age=0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Origin':
'http://www.nmgs.gov.cn:7001',
'Content-Type':
'application/x-www-form-urlencoded',
'Referer':
'http://www.nmgs.gov.cn:7001/aiccips/',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4'
})
module.appendWebMethod("post")
module.appendEncoding("utf-8")
module.appendPostData(lambda yzm, textfield: {
"code": yzm,
"textfield": textfield.replace(r"\n", "")
})
module.addSleep(Sleep(3))
module.appendOutput("url_list", ".//*[@class='list']/ul/li/a/@href",
OutputType.LIST)
module.appendOutput("name_list", ".//*[@class='list']/ul/li/a/text()",
OutputType.LIST)
module.appendOutput(
name="search_list",
type=OutputType.FUNCTION,
function=lambda url_list, name_list: zip(url_list, name_list))
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=0,
assert_function=lambda: False if self.report.access_type ==
SeedAccessType.NON_COMPANY else True))
module.addEvent(
Event(EventType.EXCEPTION_OCCURED,
retry_times=100,
redo_module="init_validate_code"))
module.addEvent(
Event(EventType.OUTPUT_NOT_SATISFIED,
retry_times=100,
redo_module="init_validate_code"))
self.module_manager.appendSubModule(module)
def checkValidateCode(self):
module = Module(self.getJson, u"检验验证码")
module.module_id = "check_validate_code"
module.appendUrl(
"http://www.nmgs.gov.cn:7001/aiccips/CheckEntContext/checkCode.html"
)
module.appendHeaders({
'Host':
'www.nmgs.gov.cn:7001',
'Connection':
'keep-alive',
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Origin':
'http://www.nmgs.gov.cn:7001',
'X-Requested-With':
'XMLHttpRequest',
'Content-Type':
'application/x-www-form-urlencoded; charset=UTF-8',
'Referer':
'http://www.nmgs.gov.cn:7001/aiccips/',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4'
})
module.appendWebMethod("post")
module.addSleep(Sleep(3))
module.appendEncoding("utf-8")
module.appendPostData(lambda yzm, company_key: {
"code": yzm,
"textfield": company_key
})
def checkValidatecode(web=None):
if not web:
return False
else:
pattern = re.compile(r'\"([\s\S]*?)\"')
flags = pattern.findall(str(web.body))
if (len(flags) != 4 or flags[2] != 'textfield') or (
flags[0] == 'flag' and flags[1] != str(1)):
self.holder.logging.warning(u"验证码校验失败!")
return False
else:
self.value_dict["textfield"] = flags[3].decode(
'raw_unicode_escape')
return True
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=100,
assert_function=checkValidatecode,
redo_module="init_validate_code"))
self.module_manager.appendSubModule(module, True)
def getCmpnySereachList(self):
"""
抓取公司列表
:output: url_list, name_list, search_list
:return:
"""
module = Module(self.visitSearchList, u"抓取公司列表")
module.module_id = "get_search_list"
module.appendUrl("http://218.95.241.36/searchList.jspx")
module.appendHeaders({
"Host": "218.95.241.36",
"Proxy-Connection": "keep-alive",
"Cache-Control": "max-age=0",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Origin": "http://218.95.241.36",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36",
"Content-Type": "application/x-www-form-urlencoded",
"Referer": "http://218.95.241.36/search.jspx",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.8"
})
module.appendWebMethod("post")
module.appendEncoding("utf-8")
module.appendPostData(lambda yzm, company_key: {
"checkNo": yzm,
"entName": company_key
})
module.addSleep(Sleep(3))
module.appendOutput("url_list", ".//*[@class='list']/ul/li/a/@href",
OutputType.LIST)
module.appendOutput("name_list", ".//*[@class='list']/ul/li/a/text()",
OutputType.LIST)
module.appendOutput(
name="search_list",
type=OutputType.FUNCTION,
function=lambda url_list, name_list: zip(url_list, name_list))
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=0,
assert_function=lambda: False if self.report.access_type ==
SeedAccessType.NON_COMPANY else True))
module.addEvent(
Event(EventType.EXCEPTION_OCCURED,
retry_times=100,
redo_module="init_validate_code"))
module.addEvent(
Event(EventType.OUTPUT_NOT_SATISFIED,
retry_times=100,
redo_module="init_validate_code"))
self.module_manager.appendSubModule(module)
def initConfigValidateCode(self):
module = Module(self.visitValidateCode, "验证码")
module.module_id = "module_validate_code"
module.appendUrl("http://gsxt.gdgs.gov.cn/aiccips/verify.html?random=" + str(random.random()))
module.appendHeaders(
{'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3', 'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive', 'Accept': 'image/png,image/*;q=0.8,*/*;q=0.5',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:37.0) Gecko/20100101 Firefox/37.0',
'Host': 'gsxt.gdgs.gov.cn', 'Referer': 'http://gsxt.gdgs.gov.cn/aiccips/'})
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=100))
self.module_manager.appendSubModule(module)
def getStockholderInfo(self, module_super):
"""
抓取翻页的股东信息
:param module_super:
:return:
"""
module = Module(self.visitGdxx, u"抓取股东信息")
module.module_id = "get_stockholder_info"
module.appendUrl(
lambda pno, company_id:
"http://218.95.241.36/QueryInvList.jspx?pno=%s&mainId=%s" %
(pno, company_id))
module.appendHeaders(
lambda company_id: {
'Host':
'218.95.241.36',
'Proxy-Connection':
'keep-alive',
'Cache-Control':
'max-age=0',
'User-Agent':
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0",
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Referer':
'http://218.95.241.36/businessPublicity.jspx?id=' + str(
company_id),
'Accept-Encoding':
'gzip, deflate',
"Accept-Language":
"zh-CN,zh;q=0.8"
})
module.appendEncoding("utf-8")
module.addSleep(Sleep(3))
def getGdxqList(html):
query_dict = dict()
try:
tree = etree.HTML(html)
query_dict["gdxq_list"] = tree.xpath(
".//*[@class='detailsList']/tr/td/a/@onclick")
except Exception as e:
self.holder.logging.warning(u"获取股东翻页中的股东详情列表失败: %s" % e)
query_dict = dict()
return query_dict
module.appendOutput(type=OutputType.FUNCTION,
function=getGdxqList,
show_up=OutputParameterShowUpType.OPTIONAL)
module.addEvent(Event(EventType.EXCEPTION_OCCURED, retry_times=50))
module_super.appendSubModule(module, True)
def initConfigSearchList(self):
module = Module(self.visitSearchList, u"搜索列表")
module.appendUrl('http://211.141.74.198:8081/aiccips/pub/indsearch')
module.appendHeaders({
'Connection':
'keep-alive',
'Accept-Language':
'en-US,en;q=0.8,zh-CN;q=0.6,zh-TW;q=0.4',
'Accept-Encoding':
'gzip, deflate',
'Cache-Control':
'max-age=0',
'Referer':
'http://211.141.74.198:8081/aiccips/',
'Host':
'211.141.74.198:8081',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:36.0) Gecko/20100101 Firefox/36.0'
})
module.appendWebMethod("post")
module.appendPostData(
lambda csrf, yzm, company_key: {
'kw': company_key,
'_csrf': csrf[-1], # 参数不为空由首页输出模块保证，且此参数为必选参数，故未做判断直接使用
'secode': getMd5WithString(yzm)
})
module.appendCookie("cookie")
module.appendOutput("url_list", ".//*[@class='list']/ul/li/a/@href",
OutputType.LIST)
module.appendOutput("name_list", ".//*[@class='list']/ul/li/a/text()",
OutputType.LIST)
module.appendOutput(
name="search_list",
type=OutputType.FUNCTION,
function=lambda url_list, name_list: zip(url_list, name_list))
module.addEvent(
Event(EventType.EXCEPTION_OCCURED,
retry_times=100,
redo_module="module_cookie"))
module.addEvent(
Event(EventType.OUTPUT_NOT_SATISFIED,
retry_times=100,
redo_module="module_cookie"))
module.addEvent(
Event(EventType.ASSERT_FAILED,
retry_times=0,
assert_function=lambda: False if self.report.access_type ==
SeedAccessType.NON_COMPANY else True))
module.addSleep(Sleep(3))
self.module_manager.appendSubModule(module)
