def advanced_search_job(self,
publication_date=0,
contract_type=None,
conty=None,
activity_domain=None,
job='',
town=''):
params = self.search_page.go(
job=quote_plus(job.encode('utf-8')),
town=quote_plus(town.encode('utf-8'))).get_post_params()
if contract_type:
self.page.url += '&employmenttype=%s' % contract_type
if conty:
self.page.url += '&countrysubdivisionfacet=%s' % conty
if activity_domain:
self.page.url += '&industryfacet=%s' % activity_domain
date_min = date.today() - timedelta(
days=publication_date) if publication_date > 0 else None
params['filterUrl'] = self.page.url
return self.call_json(params, date_min=date_min)
def search_videos(self, pattern, sortby):
pattern = pattern.replace('/', '').encode('utf-8')
if sortby is None:
url = 'en/search/%s/1' % quote_plus(pattern)
else:
url = 'en/%s/search/%s/1' % (sortby, quote_plus(pattern))
return self.index_page.go(search=url).iter_videos()
def search_job(self, pattern=None):
if pattern is None:
return []
self.search_page.go(job=quote_plus(pattern.encode('utf-8')))
assert self.search_page.is_here()
return self.page.iter_job_adverts()
def search_job(self, pattern=None):
if pattern is None:
return []
self.search_page.go(job=quote_plus(pattern.encode('utf-8')))
assert self.search_page.is_here()
return self.page.iter_job_adverts()
def login(self):
service_url = 'http://www.sfr.fr/xmscomposer/j_spring_cas_security_check'
self.location('https://www.sfr.fr/cas/login?service=%s' %
quote_plus(service_url),
no_login=True)
self.page.login(self.username, self.password)
if not self.is_logged():
raise BrowserIncorrectPassword()
def _percent_encode(s):
result = quote_plus(s).replace('+', '%20').replace('*',
'%2A').replace(
'%7E', '~')
# the implementation of the app has a bug. someone double escaped the '@' so we have to correct this
# on our end.
result = result.replace('%40', '%2540')
return result
def advanced_search_job(self, publication_date=0, contract_type=None, conty=None, activity_domain=None,
job='', town=''):
params = self.search_page.go(job=quote_plus(job.encode('utf-8')),
town=quote_plus(town.encode('utf-8'))).get_post_params()
if contract_type:
self.page.url += '&employmenttype=%s' % contract_type
if conty:
self.page.url += '&countrysubdivisionfacet=%s' % conty
if activity_domain:
self.page.url += '&industryfacet=%s' % activity_domain
date_min = date.today() - timedelta(days=publication_date) if publication_date > 0 else None
params['filterUrl'] = self.page.url
return self.call_json(params, date_min=date_min)
def search_videos(self, pattern, sortby):
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
data = {'method': 'vimeo.videos.search',
'sort': 'relevant',
'page': '1',
'full_response': '1',
'query': quote_plus(pattern.encode('utf-8'))}
self._prepare_request(self.APIURL, method='POST', headers=headers, data=data)
return self._api.go(data=data).iter_videos()
def obj_magnet(self):
hsh = Regexp(CleanText('//div[@class="trackers"]/h2'),
r'hash ([0-9a-f]+)', '\\1')(self)
name = "dn=%s" % quote_plus(
CleanText('//div[@class="downlinks"]/h2/span')(self))
trackers = [
"tr=%s" % _.text
for _ in self.xpath('//div[@class="trackers"]/dl/dt')
]
return "&".join(["magnet:?xt=urn:btih:%s" % hsh, name] + trackers)
def iter_history(self, account_id):
self.main.go()
self.history.go()
if self.message.is_here():
self.page.submit()
self.history.go()
if self.broken.is_here():
return
assert self.history.is_here()
if not self.page.is_on_right_portfolio(account_id):
self.history.go(id=quote_plus(self.page.get_compte(account_id)), period=0)
periods = self.page.get_periods()
for period in periods:
self.history.go(id=quote_plus(self.page.get_compte(account_id)), period=period)
for tr in self.page.iter_history():
yield tr
def iter_history(self, account_id):
self.history.go()
if self.message.is_here():
self.page.submit()
self.history.go()
if self.broken.is_here():
return
assert self.history.is_here()
if not self.page.is_on_right_portfolio(account_id):
self.history.go(id=quote_plus(self.page.get_compte(account_id)),
period=0)
periods = self.page.get_periods()
for period in periods:
self.history.go(id=quote_plus(self.page.get_compte(account_id)),
period=period)
for tr in self.page.iter_history():
yield tr
def search_images(self, pattern, sortby=CapImage.SEARCH_RELEVANCE, nsfw=False, **opts):
opts['q'] = quote_plus(pattern)
opts['order'] = self.SORTS[sortby]
opts['safesearch'] = nsfw
opts['key'] = self.api_key
opts['per_page'] = 20
opts['page'] = 1
while True:
self.search_api.go(params=opts)
assert self.search_api.is_here()
res = self.page.get()
for d in res['hits']:
yield self._improve_low(d)
if opts['page'] * opts['per_page'] >= res['totalHits']:
break
opts['page'] += 1
def iter_issues(self, query):
qsparts = ['repo:%s' % query.project]
if query.assignee:
qsparts.append('assignee:%s' % query.assignee)
if query.author:
qsparts.append('author:%s' % query.author)
if query.status:
qsparts.append('state:%s' % query.status)
if query.title:
qsparts.append('%s in:title' % query.title)
qs = quote_plus(' '.join(qsparts))
base_url = 'https://api.github.com/search/issues?q=%s' % qs
for json in self._paginated(base_url):
for jissue in json['items']:
issue_number = jissue['number']
yield self._make_issue(query.project, issue_number, jissue)
if not len(json['items']):
break
def iter_issues(self, query):
qsparts = ['repo:%s' % query.project]
if query.assignee:
qsparts.append('assignee:%s' % query.assignee)
if query.author:
qsparts.append('author:%s' % query.author)
if query.status:
qsparts.append('state:%s' % query.status)
if query.title:
qsparts.append('%s in:title' % query.title)
qs = quote_plus(' '.join(qsparts))
base_url = 'https://api.github.com/search/issues?q=%s' % qs
for json in self._paginated(base_url):
for jissue in json['items']:
issue_number = jissue['number']
yield self._make_issue(query.project, issue_number, jissue)
if not len(json['items']):
break
def search_images(self,
pattern,
sortby=CapImage.SEARCH_RELEVANCE,
nsfw=False,
**opts):
opts['q'] = quote_plus(pattern)
opts['order'] = self.SORTS[sortby]
opts['safesearch'] = nsfw
opts['key'] = self.api_key
opts['per_page'] = 20
opts['page'] = 1
while True:
self.search_api.go(params=opts)
assert self.search_api.is_here()
res = self.page.get()
for d in res['hits']:
yield self._improve_low(d)
if opts['page'] * opts['per_page'] >= res['totalHits']:
break
opts['page'] += 1
def search_job(self, pattern=None):
return self.adv_search.go(search='?q=%s' % quote_plus(pattern), page=1).iter_job_adverts()
def search_videos(self, pattern):
self.location(
'http://service.canal-plus.com/video/rest/search/cplus/' +
quote_plus(pattern.replace('/', '').encode('utf-8')))
return self.page.iter_results()
def get_compte(self, account_id):
values = self.doc.xpath('//option[contains(text(), $id)]/@value',
id=account_id)
assert len(values) == 1, 'could not find account %r' % account_id
return quote_plus(values[0])
def search_videos(self, pattern):
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + quote_plus(pattern.replace('/', '').encode('utf-8')))
return self.page.iter_results()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8')))
def iter_lyrics(self, criteria, pattern):
return self.browser.iter_lyrics(criteria, quote_plus(pattern.encode('utf-8')))
def login(self):
service_url = 'http://www.sfr.fr/xmscomposer/j_spring_cas_security_check'
self.location('https://www.sfr.fr/cas/login?service=%s' % quote_plus(service_url), no_login=True)
self.page.login(self.username, self.password)
if not self.is_logged():
raise BrowserIncorrectPassword()
def get_compte(self, account_id):
values = self.doc.xpath('//option[contains(text(), $id)]/@value', id=account_id)
assert len(values) == 1, 'could not find account %r' % account_id
return quote_plus(values[0])
def search_videos(self, pattern):
self.location('/search?s=%s' % (quote_plus(pattern.encode('utf-8'))))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
def search_videos(self, pattern):
return self.search_page.go(
pattern=quote_plus(pattern.encode('utf-8'))).iter_videos()
def advanced_search_job(self, job_name, place, contract, limit_date):
search = '' if not contract else contract
query = {'q': quote_plus(job_name),
'where': place,
'tm': limit_date}
return self.adv_search.go(search='%s?%s' % (search, urlencode(query)), page=1).iter_job_adverts()
def iter_lyrics(self, criteria, pattern):
return self.browser.iter_lyrics(criteria,
quote_plus(pattern.encode('utf-8')))
def _percent_encode(s):
result = quote_plus(s).replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
# the implementation of the app has a bug. someone double escaped the '@' so we have to correct this
# on our end.
result = result.replace('%40', '%2540')
return result
def iter_subtitles(self, language, pattern):
if language not in LANGUAGE_LIST:
raise LanguageNotSupported()
return self.browser.iter_subtitles(language,
quote_plus(pattern.encode('utf-8')))
def search_videos(self, pattern):
return self.search_page.go(pattern=quote_plus(pattern.encode('utf-8'))).iter_videos()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8')))
def iter_subtitles(self, language, pattern):
if language not in LANGUAGE_CONV.keys():
raise LanguageNotSupported()
return self.browser.iter_subtitles(language, quote_plus(pattern.encode('utf-8')))
def iter_documents(self):
account, = self.doc['donnees']['comptes']
statements = account['releves']
for document in statements:
d = Document()
d.date = datetime.strptime(document['dateEdition'], '%d/%m/%Y')
d.label = '%s %s' % (account['libelle'], document['dateEdition'])
d.type = DocumentTypes.STATEMENT
d.format = 'pdf'
d.id = '%s_%s' % (account['id'], document['dateEdition'].replace(
'/', ''))
d.url = '/icd/syd-front/data/syd-rce-telechargerReleve.html?b64e4000_sceau=%s' % quote_plus(
document['sceau'])
yield d
def iter_documents(self):
account, = self.doc['donnees']['comptes']
statements = account['releves']
for document in statements:
d = Document()
d.date = datetime.strptime(document['dateEdition'], '%d/%m/%Y')
d.label = '%s %s' % (account['libelle'], document['dateEdition'])
d.type = 'document'
d.format = 'pdf'
d.id = '%s_%s' % (account['id'], document['dateEdition'].replace('/', ''))
d.url = '/icd/syd-front/data/syd-rce-telechargerReleve.html?b64e4000_sceau=%s' % quote_plus(document['sceau'])
yield d