def test_splash_form_request():
req = SplashFormRequest('http://example.com', formdata={'foo': 'bar'})
assert req.method == 'POST'
assert req.body == b'foo=bar'
assert req.meta['splash']['args']['url'] == 'http://example.com'
req = SplashFormRequest('http://example.com',
method='GET',
formdata={'foo': 'bar'},
endpoint='execute')
assert req.method == 'GET'
assert req.body == b''
assert req.url == req.meta['splash']['args']['url'] ==\
'http://example.com?foo=bar'
assert req.meta['splash']['endpoint'] == 'execute'
def parse(self, response):
if not self.credentials():
self.logger.error('add the credentials first!')
return
yield SplashFormRequest.from_response(
response,
formdata={
'txtUser': self.txtUser,
'txtPassword': self.txtPassword,
'txtDependencia': self.txtDependencia,
'ComboTipoDep': self.ComboTipoDep,
'btnAceptar': self.btnAceptar
},
callback=self.parse_login,
endpoint='execute',
cache_args=['lua_source'],
session_id='dummy',
args={
'html':
1,
'lua_source':
self.getFileContentAsString(self.lua_dir + 'init.lua'),
'wait':
5
})
def start_requests(self):
# 请求方式
# print(self.start_urls)
for url in self.start_urls:
if not self.dynamic:
if self.method.lower() == 'post':
request = FormRequest(url=url,
formdata=self.form,
headers=self.headers,
cookies=self.cookies,
callback=self.parse_first,
dont_filter=True)
else:
request = Request(url=url,
headers=self.headers,
cookies=self.cookies,
callback=self.parse_first)
else:
if self.method.lower() == 'post':
request = SplashFormRequest(url=url,
formdata=self.form,
callback=self.parse_first,
dont_filter=False,
args=self.args_data)
else:
request = SplashRequest(url,
callback=self.parse_first,
endpoint='execute',
args=self.args_data)
yield request
def parse_subject(self, response):
selector = Selector(text=response.body)
subjectList = selector.xpath('//li')
fieldId = response.meta['fieldId']
if len(subjectList):
for subject in subjectList:
item = subjectItem()
subjectId = subject.xpath('@id').extract()[0]
subjectName = subject.xpath('./text()').extract()[0]
subjectName = re.sub('\ue6a2', '', subjectName)
item['id'] = subjectId
item['name'] = subjectName
item['fieldId'] = fieldId
yield item
for subject in subjectList:
subjectId = subject.xpath('@id').extract()[0]
yield SplashFormRequest(self.base_major_url,
formdata={'method': 'subCategoryXk',
'key': subjectId},
callback=self.parse_major,
meta={'subjectId': subjectId}
)
def login_me(self, response):
RequestVerificationToken = response.selector.xpath(
"//form[@id='login-form']//input[@name='__RequestVerificationToken']/@value"
).get("")
# get the Captcha's options
sitekey = response.selector.xpath(
"//div[@class='g-recaptcha']/@data-sitekey").get("")
gcaptcha_txt = self.solve_captcha(sitekey, response.url)
if not gcaptcha_txt:
return
frm_data = {
'Empresa': '1',
'Email': self.e_mail,
'Senha': self.senha,
'g-recaptcha-response': gcaptcha_txt,
'__RequestVerificationToken': RequestVerificationToken
}
print(frm_data)
yield SplashFormRequest(self.start_url,
formdata=frm_data,
callback=self.get_main_page,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={'lua_source': script_10_sec_wait},
dont_filter=True)
def login_me(self, response):
form_inputs = response.selector.xpath(
"//form[.//input[@value='Entrar']]//input")
frm_data = {}
for inpt in form_inputs:
inpt_name = inpt.xpath("./@name").get("")
inpt_val = inpt.xpath("./@value").get("")
if "urlRedirectLogin" in inpt_name:
inpt_val = "/wps/portal/portaldetran/cidadao/infracoes/servicos/consultaMultas"
elif "modalMensagem" in inpt_name:
inpt_val = "Para realizar a pesquisa de débitos e restrições de veículos do proprietário,<br /> responda algumas perguntas, acesse com seu CPF e senha ou cadastre-se abaixo:"
elif "numeroLogin" in inpt_name:
inpt_val = self.cpf_cnpj
elif "senhaLogin" in inpt_name:
inpt_val = self.senha
frm_data.update({inpt_name: inpt_val})
url = "http://www.detran.sp.gov.br" + frm_data['javax.faces.encodedURL']
# yield FormRequest(url, formdata=frm_data, callback=self.set_renavam, dont_filter=True)
yield SplashFormRequest(url,
formdata=frm_data,
callback=self.set_renavam,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={'lua_source': script_10_sec_wait},
dont_filter=True)
def parse_field(self, response):
selector = Selector(text=response.body)
fieldList = selector.xpath('//li')
degreeId = response.meta['degreeId']
if len(fieldList):
for field in fieldList:
item = fieldItem()
fieldId = field.xpath('@id').extract()[0]
fieldName = field.xpath('./text()').extract()[0]
fieldName = re.sub('\ue6a2', '', fieldName)
item['id'] = fieldId
item['name'] = fieldName
item['degreeId'] = degreeId
yield item
for field in fieldList:
fieldId = field.xpath('@id').extract()[0]
# self.logger.debug(degreeId)
yield SplashFormRequest(self.base_major_url,
formdata={'method': 'subCategoryMl',
'key': fieldId},
callback=self.parse_subject,
meta={'fieldId': fieldId}
)
def parse_degree(self, response):
soup = BeautifulSoup(response.body, 'lxml')
container = soup.find('div', attrs={'class': 'zyk-list'})
degreeContainer = container.find('ul', attrs={'class': 'zyk-cc-ul'})
degreeList = degreeContainer.findAll('li')
for degree in degreeList:
item = degreeItem()
degreeId = degree.attrs['id']
degreeName = degree.text
degreeName = re.sub('\ue6a2', '', degreeName)
item['id'] = degreeId
item['name'] = degreeName
yield item
for degree in degreeList:
degreeId = degree.attrs['id']
# self.logger.debug(degreeId)
yield SplashFormRequest(self.base_major_url,
formdata={'method': 'subCategoryMl',
'key': degreeId},
callback=self.parse_field,
meta={'degreeId': degreeId}
)
def make_request_from_data(self, data):
"""
:param data: redis_key中的数据
:return: 生成 scrapy请求
"""
scheduled = ScheduledRequest(
**json.loads(bytes_to_str(data, self.redis_encoding)))
callback, dont_filter = self.get_callback(scheduled.callback)
if not callable(callback):
raise OSError(f"{scheduled.callback}没有指定回调函数")
params = {
'url': scheduled.url,
'method': scheduled.method,
'meta': scheduled.meta,
'dont_filter': dont_filter,
'callback': callback
}
if 'splash' in scheduled.meta:
wait = scheduled.meta.get('splash').get('wait', 2)
images = scheduled.meta.get('splash').get('images', 0) # 默认不下载图片
params['args'] = {'wait': wait, 'images': images}
if scheduled.method == "POST":
return SplashFormRequest(formdata=scheduled.body, **params)
else:
return SplashRequest(**params)
if scheduled.method == "POST":
return FormRequest(formdata=scheduled.body, **params)
else:
return Request(**params)
def preparation(self, resp):
form = resp.xpath(u'//*[@id="aspnetForm"]')
form_id = form.xpath(u"./@id").extract_first()
formdata = {
# u"__EVENTARGUMENT": u"saveToWindow=format:html;",
u"__EVENTARGUMENT":
u"saveToWindow=format:csv;",
u"__EVENTTARGET":
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportViewer",
u"ctl00_ctl00_ctl00_MainContent_MainContent_ReportOutputPlaceHolder_uxTabControl_uxReportToolbar_Menu_ITCNT5_PageNumber_VI":
u"1",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT5$PageNumber$DDD$L":
u"1",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT5$PageNumber":
u"1",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT6$PageCount":
u"25",
# u"ctl00_ctl00_ctl00_MainContent_MainContent_ReportOutputPlaceHolder_uxTabControl_uxReportToolbar_Menu_ITCNT11_SaveFormat_VI": u"html",
# u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT11$SaveFormat$DDD$L": u"html",
# u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT11$SaveFormat": u"Html",
u"ctl00_ctl00_ctl00_MainContent_MainContent_ReportOutputPlaceHolder_uxTabControl_uxReportToolbar_Menu_ITCNT11_SaveFormat_VI":
u"csv",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT11$SaveFormat$DDD$L":
u"csv",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT11$SaveFormat":
u"Csv",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT5$PageNumber$DDDState":
u"{"windowsState":"0:0:-1:0:0:0:-10000:-10000:1:0:0:0"}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT5$PageNumber$DDD$L$State":
u"{"CustomCallback":""}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu$ITCNT11$SaveFormat$DDDState":
u"{"windowsState":"0:0:-1:430:140:1:68:165:1:0:0:0"}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl":
u"{"activeTabIndex":1}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportToolbar$Menu":
u"{"selectedItemIndexPath":"","checkedState":""}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxReportViewer":
u"{"drillDown":{},"parameters":{},"cacheKey":"","currentPageIndex":0}",
u"ctl00$ctl00$ctl00$MainContent$MainContent$ReportOutputPlaceHolder$uxTabControl$uxErrorGrid":
u"{"keys":[],"callbackState":"BwQHAwIERGF0YQcnAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwAHAAcAAgtGb3JtYXRTdGF0ZQcAAgVTdGF0ZQc/BwAHAAcABwAHAAIABQAAAIAJAgtUaW1lT2ZFcnJvcgcACQIAAgADBwQCAAcAAgEHAAcAAgACAQcABwAHAAcAAg1TaG93RmlsdGVyUm93CgIB","selection":""}",
}
yield SplashFormRequest.from_response(
response=resp,
formid=form_id,
formdata=formdata,
callback=self.get_station_data,
# callback=self.parse_tags,
dont_click=True,
endpoint=u"execute",
cache_args=[u"lua_source"],
args={
u"http_method": u"POST",
u"headers": {
u"Content-Type": u"application/x-www-form-urlencoded",
},
u"lua_source": script,
})
def parse(self, response):
csrf_token = response.xpath('//input[@name="csrf_token"]/@value').get()
yield SplashFormRequest.from_response(response,
formxpath='//form',
formdata={
'csrf_token': csrf_token,
'username': '******',
'password': '******'
},
callback=self.after_login)
def start_requests(self):
frm_data = {"email": self.e_mail, "password": self.senha}
login_url = self.start_url + '/login/sign_in'
yield SplashFormRequest(login_url,
formdata=frm_data,
callback=self.sign_in_me,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={'lua_source': script_10_sec_wait},
dont_filter=True)
def parse(self, response):
# meta['splash']['args'] 包含了发往Splash的参数。
# meta['splash']['endpoint'] 指定了Splash所使用的endpoint，默认是render.html
# meta['splash']['splash_url'] 覆盖了settings.py文件中配置的Splash URL
# meta['splash']['splash_headers'] 运行你增加或修改发往Splash服务器的HTTP头部信息，注意这个不是修改发往远程web站点的HTTP头部
# meta['splash']['dont_send_headers'] 如果你不想传递headers给Splash，将它设置成True
# meta['splash']['slot_policy'] 让你自定义Splash请求的同步设置
# meta['splash']['dont_process_response'] 当你设置成True后，SplashMiddleware不会修改默认的scrapy.Response请求.
# 默认是会返回SplashResponse子类响应比如SplashTextResponse
# meta['splash']['magic_response'] 默认为True，Splash会自动设置Response的一些属性，比如response.headers, response.body等
# SplashFormRequest使用
yield SplashFormRequest(response.url, self.next_parse, formdata={'name': '111'})
def set_renavam(self, response):
error_message = response.selector.xpath(
"//ul[contains(@class,'alert-error') and not(@style)]/li/span/text()"
).get("")
if error_message:
error_msg = {
"error_type": "WRONG_CREDENTIALS",
"details": error_message
}
self.errors.append(error_msg)
self.logger.warning(error_msg)
return
# get the Captcha's options
sitekey = response.selector.xpath(
"//div[@class='g-recaptcha']/@data-sitekey").get("")
gcaptcha_txt = self.solve_captcha(sitekey, response.url)
if not gcaptcha_txt:
return
renavam_form = response.selector.xpath(
"//form[.//td[contains(text(),'Renavam')]]")
form_name = renavam_form.xpath("./@name").get("")
frm_data = {
"{}:_idcl".format(form_name):
form_name.replace("form", "btAvancar"),
'g-recaptcha-response': gcaptcha_txt
}
form_inputs = renavam_form.xpath(".//input")
for inpt in form_inputs:
inpt_name = inpt.xpath("./@name").get("")
inpt_val = inpt.xpath("./@value").get("")
if ":Renavam" in inpt_name:
inpt_val = self.renavam
frm_data.update({inpt_name: inpt_val})
url = "http://www.detran.sp.gov.br" + renavam_form.xpath(
"./@action").get("")
yield SplashFormRequest(url,
formdata=frm_data,
callback=self.get_main_page,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script_10_sec_wait,
'cookies': response.data['cookies']
},
dont_filter=True)
def parse_affiliates(self, response):
countries
for country in countries:
yield SplashFormRequest(
url='https://www.crossfit.com/affiliate-list',
formxpath=
"//div[@class='form-group']/select[@id='countryFilter']",
formdata={'option': country})
paises = response.xpath("//table[@id='affiliateTable']/tbody/tr")
for pais in paises:
yield {
'gym name': country.xpath('.//td/a/text()').get(),
'local': country.xpath('.//td/text()').get(),
'country': country
}
def parse(self, response):
if self.password == '':
self.logger.error('first add a password')
return
return SplashFormRequest.from_response(
response,
formdata=
{
'email' : self.user_name,
'pass' : self.password
},
callback = self.after_login,
formid = 'loginform',
dont_process_response = True
)
def login_me(self, response):
url = 'http://www.sefaz.ba.gov.br/scripts/ipva/dae/VeiculoCadastrado/ipva_texto_obter_desconto200.asp'
frm_data = {'txt_renavam': self.renavam, 'txt_renavam1': ''}
#yield FormRequest(url, formdata=frm_data, callback=self.get_main_page,
# errback=self.errback_func, dont_filter=True)
yield SplashFormRequest(url,
formdata=frm_data,
callback=self.get_main_page,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script,
'cookies': response.data['cookies']
},
dont_filter=True)
def get_login_page(self, response):
"""Function to get request options to login.
Used to get ReCaptcha token; image captcha value."""
# get the Captcha's options
sitekey = response.selector.xpath(
"//div[@class='g-recaptcha']/@data-sitekey").get("")
gcaptcha_txt = self.solve_captcha(sitekey, response.url)
if not gcaptcha_txt:
return
# Get options for request
EVENTTARGET = response.selector.xpath(
"//input[@id='__EVENTTARGET']/@value").get("")
EVENTARGUMENT = response.selector.xpath(
"//input[@id='__EVENTARGUMENT']/@value").get("")
VIEWSTATE = response.selector.xpath(
"//input[@id='__VIEWSTATE']/@value").get("")
VIEWSTATEGENERATOR = response.selector.xpath(
"//input[@id='__VIEWSTATEGENERATOR']/@value").get("")
EVENTVALIDATION = response.selector.xpath(
"//input[@id='__EVENTVALIDATION']/@value").get("")
frm_data = {
'__EVENTTARGET': EVENTTARGET,
'__EVENTARGUMENT': EVENTARGUMENT,
'__VIEWSTATE': VIEWSTATE,
'__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
'__EVENTVALIDATION': EVENTVALIDATION,
'ctl00$conteudoPaginaPlaceHolder$txtRenavam': self.renavam,
'ctl00$conteudoPaginaPlaceHolder$txtPlaca': self.placa,
'g-recaptcha-response': gcaptcha_txt,
'ctl00$conteudoPaginaPlaceHolder$btn_Consultar': 'Consultar'
}
yield SplashFormRequest(self.start_url,
formdata=frm_data,
callback=self.login_me,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script_10_sec_wait,
'cookies': response.data['cookies']
},
dont_filter=True)
def get_debito_calculado_ipva(self, response):
valor_do_ipva = response.selector.xpath(
"//font[contains(.,'Pagamento de cota')]/text()").get("").strip()
ano_exercicio = response.selector.xpath(
"//span[contains(.,'Ano Exercício')]/../input/@value").get(
"").strip()
data_do_vencimento = response.selector.xpath(
"//span[contains(.,'Data do Vencimento')]/../input/@value").get(
"").strip()
valor_da_cota_unica = response.selector.xpath(
"//span[contains(.,'Valor da Cota única')]/../input/@value").get(
"").strip()
print(valor_da_cota_unica)
if valor_da_cota_unica:
row_data = {
'valor_do_ipva': valor_do_ipva,
'ano_exercicio': ano_exercicio,
'data_do_vencimento': data_do_vencimento,
'valor_da_cota_unica': valor_da_cota_unica
}
if self.get_files:
url = "http://www.sefaz.ba.gov.br/scripts/ipva/dae/VeiculoCadastrado/result_dae_avulso_ipva.asp"
frm_data = {
'Lnum_cnpj_cpf_base': '',
'Lnum_cnpj_cpf_filial': '',
'Lnum_cnpj_cpf_digito': ''
}
#yield FormRequest(url, callback=self.test_file,
# errback=self.errback_func, dont_filter=True)
yield SplashFormRequest(
url,
formdata=frm_data,
callback=self.print_html_to_pdf,
#errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script,
'cookies': response.data['cookies']
},
meta={'row_data': row_data},
dont_filter=True)
else:
ipva_do_veiculo = self.result.get('ipva_do_veiculo', [])
ipva_do_veiculo.append(row_data)
self.result.update({'ipva_do_veiculo': ipva_do_veiculo})
def start_requests(self):
tickers = [
'600001',
'000777',
]
for ticker in tickers: # try all tickers one by one
if ticker[0] == '0':
data = { # prepare for the requrest
'stock': ticker,
'searchkey': '年年度报告',
'category': 'category_ndbg_szsh',
'pageNum': '1',
'pageSize': '30',
'column': 'szse_main', # for Shenzhen stock exchange
'tabName': 'fulltext',
'sortName':'',
'sortType':'',
'limit': '',
'seDate': '',
}
elif ticker[0] == '6':
data = { # prepare for the requrest
'stock': ticker,
'searchkey': '年年度报告',
'category': 'category_ndbg_szsh',
'pageNum': '1',
'pageSize': '30',
'column': 'sse', # for Shanghai stock exchange
'tabName': 'fulltext',
'sortName':'',
'sortType':'',
'limit': '',
'seDate': '',
}
else:
print("Wrong ticker")
continue
yield SplashFormRequest(
url='http://www.cninfo.com.cn/cninfo-new/announcement/query',
formdata=data,
callback=self.parse,
# args={'wait': 2}
)
def login(self, response): # 调用splash的SplashFormRequest，提交表单参数
yield SplashFormRequest.from_response(
response=response,
url=self.login_url,
endpoint='execute',
formdata={
'user_agent': '',
'cookie': ''
},
args={
'wait': 30,
'lua_source': lg,
'proxy': 'http://:1080',
'search_key': self.search_key
},
callback=self.after_login,
errback=self.error_parse,
)
def get_result_debito_ipva(self, response):
cota3 = response.url.split("cota3=")[-1]
url = "http://www.sefaz.ba.gov.br/scripts/ipva/dae/VeiculoCadastrado/debito_calculado_ipva.asp?cota3={}".format(
cota3)
today = dt.now().strftime("%d/%m/%Y")
frm_data = {'txt_dtc_pagamento': today}
#yield FormRequest(url, formdata=frm_data, callback=self.get_debito_calculado_ipva,
# errback=self.errback_func, dont_filter=True)
yield SplashFormRequest(url,
formdata=frm_data,
callback=self.get_debito_calculado_ipva,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script,
'cookies': response.data['cookies']
},
dont_filter=True)
def login_me(self, response):
login_url = "https://auth.netcombo.com.br/login"
client_id = response.selector.xpath(
"//input[@name='client_id']/@value").get("")
redirect_uri = response.selector.xpath(
"//input[@name='redirect_uri']/@value").get("")
response_type = response.selector.xpath(
"//input[@name='response_type']/@value").get("")
scope = response.selector.xpath("//input[@name='scope']/@value").get(
"")
state = response.selector.xpath("//input[@name='state']/@value").get(
"")
authMs = response.selector.xpath("//input[@name='authMs']/@value").get(
"")
frm_data = {
'Username': self.login,
'password': self.senha,
'client_id': client_id,
'redirect_uri': redirect_uri,
'response_type': response_type,
'scope': scope,
'state': state,
'authMs': authMs,
'Auth_method': 'UP'
}
print(frm_data)
yield SplashFormRequest(login_url,
formdata=frm_data,
callback=self.select_contract,
errback=self.errback_func,
endpoint='execute',
cache_args=['lua_source'],
args={
'lua_source': script_30_sec_wait,
'cookies': response.data['cookies'],
'timeout': 60,
'images': 0
},
dont_filter=True)
def get_ipva_search(self, response):
action = response.selector.xpath("//input[@id='action']/@value").get("")
csrf_token = response.selector.xpath("//input[@id='csrf_token']/@value").get("")
sitekey = response.selector.xpath(
"//script[contains(@src,'recaptcha/api')]/@src").get("").split("render=")[-1]
gcaptcha_txt = self.solve_captcha(sitekey, response.request.url,
captcha_type=5,
captcha_action='portal_consulta_renavam')
if not gcaptcha_txt:
return
frm_data = {'action': action,
'renavam': response.meta['renavam'],
'csrf_token': csrf_token,
'recaptcha_response': gcaptcha_txt}
ipva_url = "https://portalcontribuinte.sefin.ro.gov.br/Publico/__Resultado_Renavam_.jsp"
yield SplashFormRequest(ipva_url, formdata=frm_data,
endpoint='render.json', args=self.splash_args,
meta={'renavam': response.meta['renavam']},
callback=self.get_ipva_result, dont_filter=True)
def login(self,
response): # 调用splash的SplashFormRequest，提交表单参数(user、password)
yield SplashFormRequest.from_response(
response=response,
url=self.login_url,
formdata={
'email': user,
'password': password
},
endpoint='execute',
args={
'wait': 30,
'lua_source': log_lua_script,
'user_name': user,
'user_password': password,
'proxy': 'http://:1080'
},
callback=self.after_login,
errback=self.error_parse,
)
def login_me(self, response):
"""Function to get request options to login.
Used to get ReCaptcha token; image captcha value."""
# get the Captcha's options
sitekey = response.selector.xpath(
"//div[@class='g-recaptcha']/@data-sitekey").get("")
gcaptcha_txt = self.solve_captcha(sitekey, response.url)
if not gcaptcha_txt:
return
frm_data = {
'Renavam': self.renavam,
'Placa': self.placa,
'g-recaptcha-response': gcaptcha_txt}
# yield FormRequest(self.start_url, formdata=frm_data, callback=self.get_main_page, dont_filter=True)
yield SplashFormRequest(self.start_url, formdata=frm_data,
callback=self.get_main_page,
errback=self.errback_func,
endpoint='execute', cache_args=['lua_source'],
args={'lua_source': autos_detran_ro_script}, dont_filter=True)
def test_form_request_from_response():
# Copied from scrapy tests (test_from_response_submit_not_first_clickable)
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)
kwargs.setdefault('url', 'http://example.com')
kwargs.setdefault('encoding', 'utf-8')
return HtmlResponse(**kwargs)
response = _buildresponse(
"""<form action="get.php" method="GET">
<input type="submit" name="clickable1" value="clicked1">
<input type="hidden" name="one" value="1">
<input type="hidden" name="two" value="3">
<input type="submit" name="clickable2" value="clicked2">
</form>""")
req = SplashFormRequest.from_response(
response, formdata={'two': '2'}, clickdata={'name': 'clickable2'})
assert req.method == 'GET'
assert req.meta['splash']['args']['url'] == req.url
fs = cgi.parse_qs(req.url.partition('?')[2], True)
assert fs['clickable2'] == ['clicked2']
assert 'clickable1' not in fs
assert fs['one'] == ['1']
assert fs['two'] == ['2']
def test_form_request_from_response():
# Copied from scrapy tests (test_from_response_submit_not_first_clickable)
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)
kwargs.setdefault('url', 'http://example.com')
kwargs.setdefault('encoding', 'utf-8')
return HtmlResponse(**kwargs)
response = _buildresponse(
"""<form action="get.php" method="GET">
<input type="submit" name="clickable1" value="clicked1">
<input type="hidden" name="one" value="1">
<input type="hidden" name="two" value="3">
<input type="submit" name="clickable2" value="clicked2">
</form>""")
req = SplashFormRequest.from_response(
response, formdata={'two': '2'}, clickdata={'name': 'clickable2'})
assert req.method == 'GET'
assert req.meta['splash']['args']['url'] == req.url
fs = cgi.parse_qs(req.url.partition('?')[2], True)
assert fs['clickable2'] == ['clicked2']
assert 'clickable1' not in fs
assert fs['one'] == ['1']
assert fs['two'] == ['2']
def login(self, response):
logging.debug('Screenshot: %s', response.data['png'])
try:
headers = json.loads(self.rpt_mp.headers)
except json.decoder.JSONDecodeError as err:
headers = {}
formdata, url, method = fill_login_form(response.url, response.text,
response.meta['username'],
response.meta['password'])
# because following request is a new Splash request
response.meta.pop('splash')
response.meta.pop('_splash_processed')
return SplashFormRequest.from_response(response,
endpoint='render.json',
args=self.splash_args,
headers=headers,
dont_filter=True,
formname='fm',
formdata=formdata,
meta=response.meta,
callback=self.after_login)
def get_main_page(self, response):
"""Redirect to main page."""
error_message = response.selector.xpath("//div[@class='msgErro']/text()").get("")
print(error_message)
if "Nenhum registro encontrado, verifique os dados digitados" in error_message:
error_msg = {"error_type": "WRONG_CREDENTIALS",
"details": error_message}
self.errors.append(error_msg)
self.logger.warning(error_msg)
return
elif "Confirme que você não é um robô." in error_message:
self.incorrect_captcha_report(
self.captcha_service, self.g_recaptcha_id)
if self.incorrect_captcha_retries > 0:
yield Request(self.start_url, callback=self.login_me,
meta={'dont_merge_cookies': True}, dont_filter=True)
return
regex = re.compile(r'\s+')
self.result['placa'] = regex.sub(" ", response.selector.xpath("//span[text()='Placa']/../text()").get("").strip())
self.result['marca_modelo'] = regex.sub(" ", response.selector.xpath("//span[text()='Marca/Modelo']/../text()").get("").strip())
self.result['fabricacao_modelo'] = regex.sub(" ", response.selector.xpath("//span[text()='Fabricacao/Modelo']/../text()").get("").strip())
self.result['cor'] = regex.sub(" ", response.selector.xpath("//span[text()='Cor']/../text()").get("").strip())
renavam = regex.sub(" ", response.selector.xpath("//span[text()='Renavam']/../text()").get("").strip())
self.result['renavam'] = renavam
self.result['tipo'] = regex.sub(" ", response.selector.xpath("//span[text()='Tipo']/../text()").get("").strip())
self.result['carroceria'] = regex.sub(" ", response.selector.xpath("//span[text()='Carroceria']/../text()").get("").strip())
self.result['especie'] = regex.sub(" ", response.selector.xpath("//span[text()='Especie']/../text()").get("").strip())
self.result['lugares'] = regex.sub(" ", response.selector.xpath("//span[text()='Lugares']/../text()").get("").strip())
self.result['categoria'] = regex.sub(" ", response.selector.xpath("//span[text()='Categoria']/../text()").get("").strip())
self.result['potencia'] = regex.sub(" ", response.selector.xpath("//span[text()='Potência']/../text()").get("").strip())
self.result['combustivel'] = regex.sub(" ", response.selector.xpath("//span[text()='Combustível']/../text()").get("").strip())
self.result['nome_do_proprietario'] = regex.sub(" ", response.selector.xpath("//span[text()='Nome do Proprietário']/../text()").get("").strip())
self.result['situacao_lacre'] = regex.sub(" ", response.selector.xpath("//span[text()='Situação Lacre']/../text()").get("").strip())
self.result['proprietario_anterior'] = regex.sub(" ", response.selector.xpath("//span[text()='Proprietário Anterior']/../text()").get("").strip())
self.result['origem_dos_dados_do_veiculo'] = regex.sub(" ", response.selector.xpath("//span[text()='Origem dos Dados do Veículo']/../text()").get("").strip())
self.result['placa_anterior'] = regex.sub(" ", response.selector.xpath("//span[text()='Placa Anterior']/../text()").get("").strip())
self.result['municipio_de_emplacamento'] = regex.sub(" ", response.selector.xpath("//span[text()='Municipio de Emplacamento']/../text()").get("").strip())
self.result['licenciado_ate'] = regex.sub(" ", response.selector.xpath("//span[text()='Licenciado ate']/../text()").get("").strip())
self.result['adquirido_em'] = regex.sub(" ", response.selector.xpath("//span[text()='Adquirido em']/../text()").get("").strip())
self.result['situacao'] = regex.sub(" ", response.selector.xpath("//span[text()='Situação']/../text()").get("").strip())
self.result['restricao_a_venda'] = regex.sub(" ", response.selector.xpath("//span[text()='Restrição a Venda']/../text()").get("").strip())
self.result['informacoes_pendentes_originadas_das_financeiras_via_sng_sistema_nacional_de_gravame'] = regex.sub(" ", response.selector.xpath(
"//span[text()='Informações PENDENTES originadas das financeiras via SNG - Sistema Nacional de Gravame']/../text()").get("").strip())
self.result['impedimentos'] = regex.sub(" ", response.selector.xpath("//span[text()='Impedimentos']/../text()").get("").strip())
debitos_rows = response.selector.xpath("//div[@id='corpo_DebitosVeiculo']/div[@id='Integral']/table[@id='TabelaIntegral']/tbody/tr")
debitos = []
for row in debitos_rows:
descricao = regex.sub(" ", row.xpath("./td[1]/text()").get("").strip())
vencimento = regex.sub(" ", row.xpath("./td[2]/text()").get("").strip())
nominal_r = regex.sub(" ", row.xpath("./td[3]/text()").get("").strip())
corrigido_r = regex.sub(" ", row.xpath("./td[4]/text()").get("").strip())
desconto_r = regex.sub(" ", row.xpath("./td[5]/text()").get("").strip())
juros_r = regex.sub(" ", row.xpath("./td[6]/text()").get("").strip())
multa_r = regex.sub(" ", row.xpath("./td[7]/text()").get("").strip())
atual_r = regex.sub(" ", row.xpath("./td[8]/text()").get("").strip())
debitos.append({
'descricao': descricao,
'vencimento': vencimento,
'nominal_r': nominal_r,
'corrigido_r': corrigido_r,
'desconto_r': desconto_r,
'juros_r': juros_r,
'multa_r': multa_r,
'atual_r': atual_r})
self.result.update({'debitos': debitos})
infracoes_em_autuacao_rows = response.selector.xpath(
"//div[@id='corpo_AutuacoesVeiculo']//tbody/tr")
infracoes_em_autuacao = []
for row in infracoes_em_autuacao_rows:
num_auto = " ".join([s.strip() for s in row.xpath("./td[1]/text()").extract() if s.strip()])
if "veículo até o momento." in num_auto:
break
status = regex.sub(" ", " ".join(row.xpath("./td[2]/text()").extract()).strip())
descricao = regex.sub(" ", " ".join(row.xpath("./td[3]/text()").extract()).strip())
local_complemento = regex.sub(" ", " ".join(row.xpath("./td[4]/text()").extract()).strip())
valor = regex.sub(" ", " ".join(row.xpath("./td[5]/text()").extract()).strip())
infracoes_em_autuacao.append({
'num_auto': num_auto,
'status': status,
'descricao': descricao,
'local_complemento': local_complemento,
'valor': valor})
self.result.update({'infracoes_em_autuacao': infracoes_em_autuacao})
penalidades_multas_rows = response.selector.xpath(
"//div[@id='corpo_MultasVeiculo']//tbody/tr")
penalidades_multas = []
for row in penalidades_multas_rows:
num_auto = regex.sub(" ", " ".join(row.xpath("./td[1]/text()").extract()).strip())
if "veículo até o momento." in num_auto:
break
status = regex.sub(" ", " ".join(row.xpath("./td[2]/text()").extract()).strip())
descricao = regex.sub(" ", " ".join(row.xpath("./td[3]/text()").extract()).strip())
local_complemento = regex.sub(" ", " ".join(row.xpath("./td[4]/text()").extract()).strip())
valor = regex.sub(" ", " ".join(row.xpath("./td[5]/text()").extract()).strip())
penalidades_multas.append({
'num_auto': num_auto,
'status': status,
'descricao': descricao,
'local_complemento': local_complemento,
'valor': valor})
self.result.update({'penalidades_multas': penalidades_multas})
recursos_infracao_rows = response.selector.xpath(
"//div[@id='corpo_RecursosInfracao']//tbody/tr")
recursos_infracao = []
for row in recursos_infracao_rows:
processo = regex.sub(" ", " ".join(row.xpath("./td[1]/text()").extract()).strip())
if "veículo até o momento." in processo:
break
n_proc_renainf = regex.sub(" ", " ".join(row.xpath("./td[2]/text()").extract()).strip())
numero_do_auto = regex.sub(" ", " ".join(row.xpath("./td[3]/text()").extract()).strip())
detalhamento_da_infracao = regex.sub(" ", " ".join(row.xpath("./td[4]/text()").extract()).strip())
situacao_do_processo = regex.sub(" ", " ".join(row.xpath("./td[5]/text()").extract()).strip())
recursos_infracao.append({
'processo': processo,
'nº_proc_renainf': n_proc_renainf,
'numero_do_auto': numero_do_auto,
'detalhamento_da_infracao': detalhamento_da_infracao,
'situacao_do_processo': situacao_do_processo})
self.result.update({'recursos_infracao': recursos_infracao})
dare_btn = response.selector.xpath("//input[@id='BotaoIntegral']")
if self.get_files and dare_btn:
dare_url = "https://consulta.detran.ro.gov.br/CentralDeConsultasInternet/Internet/DARE.asp"
hdListaIdDebitos = response.selector.xpath("//input[@name='hdListaIdDebitos']/@value").get("")
hdPlaca = response.selector.xpath("//input[@name='hdPlaca']/@value").get("")
frm_data = {'hdListaIdDebitos': hdListaIdDebitos,
'hdPlaca': hdPlaca}
print(frm_data)
yield SplashFormRequest(dare_url, formdata=frm_data,
endpoint='render.json', args=self.splash_args,
meta={'result_key': 'debitos'},
callback=self.print_html_to_pdf, dont_filter=True)
ipva_url = "https://portalcontribuinte.sefin.ro.gov.br/Publico/ConsultaRenavam.jsp?renavam={}".format(renavam)
yield Request(ipva_url, callback=self.get_ipva_search, meta={'renavam': renavam}, dont_filter=True)
def parse(self, response):
#专利名称
title = response.xpath("//h1")
#申请公布号
openNo = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[1]')
#申请公布日
openDate = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[2]')
#申请号
applyNo = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[3]')
#申请日
applyDate = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[4]')
#申请人
applyPeople = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[5]')
#发明人
inventor = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[6]')
#地址
address = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[8]')
#分类号
classifyNo = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_linr"]/ul/li[9]')
#摘要
summery = response.xpath(
'//div[@class="cp_box"]/div/div[@class="cp_jsh"]')
#二维码
qrcodeurls = response.xpath('//div[@class="cp_box"]/a/img/@src')
#缩略图
thumb = response.xpath(
'//div[@class="cp_box"]/div[@class="cp_img"]/img/@src')
baseUrl = 'http://epub.sipo.gov.cn/'
for ti, on, od, an, ad, ap, inv, add, cf, su, qr, th in zip(
title, openNo, openDate, applyNo, applyDate, applyPeople,
inventor, address, classifyNo, summery, qrcodeurls, thumb):
item = ZlItem()
item['title'] = ti.xpath(
"string(.)").extract_first().strip().split()[1]
item['openNo'] = on.xpath("string(.)").extract_first()[6:]
item['openDate'] = od.xpath("string(.)").extract_first()[6:]
item['applyNo'] = an.xpath("string(.)").extract_first()[5:]
item['applyDate'] = ad.xpath("string(.)").extract_first()[4:]
item['applyPeople'] = ap.xpath("string(.)").extract_first()[4:]
item['inventor'] = ''.join(
inv.xpath("string(.)").extract_first()[4:].strip().split())
item['address'] = add.xpath("string(.)").extract_first()[3:]
item['classifyNo'] = cf.xpath(
"string(.)").extract_first().split()[0][4:]
item['summery'] = su.xpath("string(.)").extract_first().split()[1]
item['qrcodeurls'] = baseUrl + qr.extract()
item['thumb'] = baseUrl + th.extract()
yield item
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
yield SplashFormRequest(
url='http://epub.sipo.gov.cn/pam.action',
callback=self.downparse,
method='POST',
args={'wait': 5},
formdata={
"strWhere":
"PN='" +
"{}".format(on.xpath("string(.)").extract_first()[6:]) +
"'",
"strSources":
"pip"
},
headers=headers)
