def start_requests(self):
for item in self.load_subjects():
subject = MaxSubjectLoader.from_dict(item)
self.logger.debug('reading subject: %s', subject['name'])
yield http.web_open(subject['url'],
meta={'subject': subject},
base_url=http.MAX_BASE_URL,
callback=self.parse_books)
def start_requests(self):
for item in self.load_subjects():
subject = SubjectLoader.from_dict(item)
self.logger.debug('reading subject: %s', subject['name'])
args = self._get_book_args(subject)
yield http.web_open('/listaMidiatecas.processa',
meta={'subject': subject},
args=args,
callback=self.parse_books)
def retry_login(self, response):
self.logger.info('authentication failed')
original_data = response.meta.get('creds', None)
if original_data:
del response.meta['creds']
self._log_user(original_data, 'last login with username: %s')
self.logger.info('retrying login...')
new_credentials = self._read_auth(default_dict=original_data)
self._log_user(new_credentials, 'login attempt with user: %s')
yield http.web_open('/login.processa',
callback=self.get_login_handler(),
formdata=new_credentials,
meta={'creds': new_credentials},
dont_filter=True,
impl=scrapy.FormRequest)
def start_requests(self):
yield http.web_open(callback=self.get_login_handler())
def parse_home(self, response):
logout_url = response.css('#icon-sair-perfil').xpath(
'.//following::a[1]/@href').get()
if logout_url:
return http.web_open(logout_url, callback=self.handle_exit)
self.logger.debug('user already out!?')
def start_requests(self):
yield http.web_open(callback=self.parse_home)
def start_requests(self):
# we must fake the authentication, sounds bad...
LoginSpider.fake_auth()
yield http.web_open(base_url=http.MAX_BASE_URL,
callback=self.synchronize)
def start_requests(self):
yield http.web_open(url='/horario.htm',
base_url=http.MAX_BASE_URL,
callback=self.parse_schedule)
def start_requests(self):
yield http.web_open('/listaDisciplina.processa',
args=SubjectSpider.subject_args,
callback=self.parse_subjects)
def start_requests(self):
yield http.web_open(callback=self.synchronize)