def sync_next_subject(self):
self.logger.debug(self.subjects_content)
if self.subjects_content:
item = self.subjects_content.pop()
subject = MaxSubjectLoader.from_dict(item)
self.logger.debug('reading subject: %s', subject['name'])
return http.web_open(subject['url'],
meta={'subject': subject},
base_url=MAX_BASE_URL,
callback=self.parse_books)
def sync_next_subject(self):
self.logger.debug(self.subjects_content)
if self.subjects_content:
item = self.subjects_content.pop()
subject = SubjectLoader.from_dict(item)
self.logger.debug('reading subject: %s', subject['name'])
args = self._get_book_args(subject)
return http.web_open('/listaMidiatecas.processa',
meta={'subject': subject},
args=args,
callback=self.parse_books)
def retry_login(self, response):
self.logger.info('authentication failed')
original_data = response.meta.get('creds', None)
if original_data:
del response.meta['creds']
self._log_user(original_data, 'last login with username: %s')
self.logger.info('retrying login...')
new_credentials = self._read_auth(default_dict=original_data)
self._log_user(new_credentials, 'login attempt with user: %s')
login_handler = self.after_login(self.retry_login)
yield http.web_open('/login.processa',
callback=login_handler,
formdata=new_credentials,
meta={'creds': new_credentials},
dont_filter=True,
impl=scrapy.FormRequest)
def build_download_request(self, book):
return http.web_open(book['download_url'],
cb_kwargs={'book': book},
callback=self.handle_download)
def start_requests(self):
yield http.web_open(callback=self.synchronize)
def start_requests(self):
yield http.web_open('/listaDisciplina.processa',
args=SubjectSpider.subject_args,
callback=self.parse_subjects)
def start_requests(self):
yield http.web_open(url='/horario.htm',
base_url=MAX_BASE_URL,
callback=self.parse_schedule)
def start_requests(self):
login_handler = self.after_login(self.retry_login)
yield http.web_open(callback=login_handler)
def parse_home(self, response):
logout_url = response.css('#icon-sair-perfil').xpath(
'.//following::a[1]/@href').get()
if logout_url:
return http.web_open(logout_url, callback=self.handle_exit)
self.logger.debug('user already out!?')
def start_requests(self):
yield http.web_open(callback=self.parse_home)