def parse(self, response: scrapy.http.Response, **kwargs): entries = response.xpath( f'//div[{xpath_class(["event-search"])}]//table/tbody/tr').getall( ) for entry in entries: yield ResponseItem({'body': entry, 'meta': response.meta})
def parse(self, response: scrapy.http.Response, **kwargs): data = json.loads(response.text) if data['message']: for msg in data['message']: self.logger.warning(msg) for entry in data['list']: yield ResponseItem({'body': entry, 'meta': response.meta})
def parse(self, response: scrapy.http.Response, **kwargs): next_page_url = response.xpath( f'//li[{xpath_class(["next"])}]/a/@href').get() if next_page_url is not None: yield scrapy.Request(f'{self.base_url}{next_page_url}') entries = response.xpath( f'//article[{xpath_class(["event"])}]').getall() for entry in entries: yield ResponseItem({'body': entry, 'meta': response.meta})
def parse(self, response: scrapy.http.Response, **kwargs): yield ResponseItem({'body': response.text, 'meta': response.meta})
def parse_entry(self, response: scrapy.http.Response, **kwargs): yield ResponseItem({ 'body': response.body, 'meta': response.meta, 'url': response.url })