def sync_next_subject(self): self.logger.debug(self.subjects_content) if self.subjects_content: item = self.subjects_content.pop() subject = MaxSubjectLoader.from_dict(item) self.logger.debug('reading subject: %s', subject['name']) return http.web_open(subject['url'], meta={'subject': subject}, base_url=MAX_BASE_URL, callback=self.parse_books)
def sync_next_subject(self): self.logger.debug(self.subjects_content) if self.subjects_content: item = self.subjects_content.pop() subject = SubjectLoader.from_dict(item) self.logger.debug('reading subject: %s', subject['name']) args = self._get_book_args(subject) return http.web_open('/listaMidiatecas.processa', meta={'subject': subject}, args=args, callback=self.parse_books)
def retry_login(self, response): self.logger.info('authentication failed') original_data = response.meta.get('creds', None) if original_data: del response.meta['creds'] self._log_user(original_data, 'last login with username: %s') self.logger.info('retrying login...') new_credentials = self._read_auth(default_dict=original_data) self._log_user(new_credentials, 'login attempt with user: %s') login_handler = self.after_login(self.retry_login) yield http.web_open('/login.processa', callback=login_handler, formdata=new_credentials, meta={'creds': new_credentials}, dont_filter=True, impl=scrapy.FormRequest)
def build_download_request(self, book): return http.web_open(book['download_url'], cb_kwargs={'book': book}, callback=self.handle_download)
def start_requests(self): yield http.web_open(callback=self.synchronize)
def start_requests(self): yield http.web_open('/listaDisciplina.processa', args=SubjectSpider.subject_args, callback=self.parse_subjects)
def start_requests(self): yield http.web_open(url='/horario.htm', base_url=MAX_BASE_URL, callback=self.parse_schedule)
def start_requests(self): login_handler = self.after_login(self.retry_login) yield http.web_open(callback=login_handler)
def parse_home(self, response): logout_url = response.css('#icon-sair-perfil').xpath( './/following::a[1]/@href').get() if logout_url: return http.web_open(logout_url, callback=self.handle_exit) self.logger.debug('user already out!?')
def start_requests(self): yield http.web_open(callback=self.parse_home)