def start_requests(self): token = '' lines = read_cache('tokens', (self.plat_id or 'test') + '.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(token=token) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies)
def start_requests(self): token = '' lines = read_cache('tokens', (self.plat_id or 'test')+'.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(token=token) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies)
def start_requests(self): for i in self.shortlist: token = '' lines = read_cache('tokens', (self.plat_id or 'test')+'.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(token=token, page_index=i) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies, dont_filter=True)
def start_requests(self): for i in self.shortlist: token = '' lines = read_cache('tokens', (self.plat_id or 'test') + '.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(token=token, page_index=i) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies, dont_filter=True)
def start_requests(self): if not (self.plat_id and self.time_from and self.time_to): return try: loans = JiekuanItem.django_model.objects.filter(plat_id=self.plat_id, status='1', \ success_time__gte=self.time_from, \ success_time__lte=self.time_to) except Exception as e: self.logger.info('Error From Filtering Loan Objects <%s>.' % e) return for jk in loans: if not jk.bid_id: continue token = '' lines = read_cache('tokens', (self.plat_id or 'test') + '.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(id=jk.bid_id, token=token) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies, dont_filter=True)
def start_requests(self): if not (self.plat_id and self.time_from and self.time_to): return try: loans = JiekuanItem.django_model.objects.filter(plat_id=self.plat_id, status='1', \ success_time__gte=self.time_from, \ success_time__lte=self.time_to) except Exception as e: self.logger.info('Error From Filtering Loan Objects <%s>.' % e) return for jk in loans: if not jk.bid_id: continue token = '' lines = read_cache('tokens', (self.plat_id or 'test')+'.tk') if self.need_token and lines: token = lines[0] url = self.start_formated_url.format(id=jk.bid_id, token=token) from scrapy.http import Request cookies = parse_cookies(lines[1]) yield Request(url, cookies=cookies, dont_filter=True)