def form_postresponse(self, response):
form_args = JavaScriptUtils.getValuesFromdoPost(
response.xpath(
"//*[@id='ctl00_PlaceHolderMain_dgvPermitList_gdvPermitList_gdvPermitListtop4btnExport']/@href"
).extract_first())
form = _get_form(response,
formname=None,
formid='aspnetForm',
formnumber=0,
formxpath=None)
formdata = _get_inputs(
form,
formdata={
'__EVENTARGUMENT':
form_args['__EVENTARGUMENT'],
'__ASYNCPOST':
'true',
'__EVENTTARGET':
form_args['__EVENTTARGET'],
'ctl00$ScriptManager1':
'ctl00$PlaceHolderMain$dgvPermitList$updatePanel|{}'.format(
form_args['__EVENTTARGET'])
},
clickdata=None,
dont_click=False,
response=response)
yield scrapy.FormRequest(url=response.url,
method='POST',
formdata=formdata,
dont_filter=True,
errback=self.handle_form_error,
callback=self.response_download)
def parse(self, response):
if self.check:
self.start = datetime.datetime.strptime(
self.start, '%Y%m%d').strftime('%m/%d/%Y')
self.end = datetime.datetime.strptime(
self.end, '%Y%m%d').strftime('%m/%d/%Y')
self.check = False
form = _get_form(response,
formname=None,
formid='aspnetForm',
formnumber=0,
formxpath=None)
formdata = _get_inputs(
form,
formdata={
'ctl00$PlaceHolderMain$generalSearchForm$txtGSStartDate':
self.start,
'ctl00$PlaceHolderMain$generalSearchForm$txtGSEndDate':
self.end,
'__EVENTTARGET': 'ctl00$PlaceHolderMain$btnNewSearch'
},
clickdata=None,
dont_click=False,
response=response)
self.modifyformdata = dict(formdata)
self.form_url = response.url
yield scrapy.FormRequest(url=response.url,
method='POST',
formdata=formdata,
dont_filter=True,
errback=self.handle_form_error,
callback=self.form_postresponse)
def fill_generic_form(self, url, body, form_descriptor):
doc = html.document_fromstring(body, base_url=url)
form = self._pick_node(doc, form_descriptor)
if form is None:
raise Exception('Generic form not found')
# Get all the possible inputs for each field
values = [self._get_field_values(form, field)
for field in form_descriptor['fields']]
for params in itertools.product(*values):
form_values = dict(_get_inputs(form, None, False, None, None))
for name, option in params:
form_values[name] = option
yield form_values.items(), form.action or form.base_url, form.method
def fill_generic_form(self, url, body, form_descriptor):
doc = html.document_fromstring(body, base_url=url)
form = self._pick_node(doc, form_descriptor)
if form is None:
raise Exception('Generic form not found')
# Get all the possible inputs for each field
values = [self._get_field_values(form, field)
for field in form_descriptor['fields']]
for params in itertools.product(*values):
form_values = OrderedDict(_get_inputs(form, None, False, None,
None))
for name, option in params:
form_values[name] = option
yield list(form_values.items()), form.action or form.base_url, form.method