def iter_transfer_recipients(self, account): assert account.url url = urlsplit(account.url) parts = [part for part in url.path.split('/') if part] if account.type == Account.TYPE_SAVINGS: self.logger.debug('Deleting account name %s to get recipients', parts[-2]) del parts[-2] parts.append('virements') url = url._replace(path='/'.join(parts)) target = urlunsplit(url) try: self.location(target) except BrowserHTTPNotFound: return [] if self.transfer_accounts.is_here(): try: self.page.submit_account(account.id) except AccountNotFound: return [] assert self.recipients_page.is_here() return self.page.iter_recipients()
def iter_transfer_recipients(self, account): if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): return [] assert account.url # url transfer preparation url = urlsplit(account.url) parts = [part for part in url.path.split('/') if part] assert len( parts ) > 2, 'Account url missing some important part to iter recipient' account_type = parts[1] # cav, ord, epargne ... account_webid = parts[-1] try: self.transfer_main_page.go(acc_type=account_type, webid=account_webid) except BrowserHTTPNotFound: return [] # can check all account available transfer option if self.transfer_main_page.is_here(): self.transfer_accounts.go(acc_type=account_type, webid=account_webid) if self.transfer_accounts.is_here(): try: self.page.submit_account(account.id) except AccountNotFound: return [] assert self.recipients_page.is_here() return self.page.iter_recipients()
def iter_transfer_recipients(self, account): assert account.url url = urlsplit(account.url) parts = [part for part in url.path.split('/') if part] if account.type == Account.TYPE_SAVINGS: self.logger.debug('Deleting account name %s to get recipients', parts[-2]) del parts[-2] parts.append('virements') url = url._replace(path='/'.join(parts)) target = urlunsplit(url) try: self.location(target) except BrowserHTTPNotFound: return [] if self.transfer_accounts.is_here(): try: self.page.submit_account(account.id) except AccountNotFound: return [] assert self.recipients_page.is_here() return self.page.iter_recipients()
def add_cookie(self, name, value): # httplib/cookielib don't seem to like unicode cookies... if sys.version_info.major < 3: name = to_unicode(name).encode('utf-8') value = to_unicode(value).encode('utf-8') self.browser.logger.debug('adding cookie %r=%r', name, value) self.browser.session.cookies.set(name, value, domain=urlsplit(self.url).hostname)
def __init__(self, url, *args, **kwargs): self._userid = 0 v = urlsplit(url) self.PROTOCOL = v.scheme self.DOMAIN = v.netloc self.BASEPATH = v.path if self.BASEPATH.endswith('/'): self.BASEPATH = self.BASEPATH[:-1] Browser.__init__(self, *args, **kwargs) self.projects = {}
def __init__(self, url, apiurl, username, password, *args, **kwargs): url_parsed = urlsplit(url) self.PROTOCOL = url_parsed.scheme self.DOMAIN = url_parsed.netloc self.BASEPATH = url_parsed.path if self.BASEPATH.endswith('/'): self.BASEPATH = self.BASEPATH[:-1] self.apiurl = apiurl self.username = username self.password = password DomainBrowser.__init__(self, *args, **kwargs)
def location(self, url, **kwargs): try: return super(BPBrowser, self).location(url, **kwargs) except ServerError as err: if "/../" not in err.response.url: raise # this shit website includes ".." in an absolute url in the Location header # requests passes it verbatim, and the site can't handle it self.logger.debug('site has "/../" in their url, fixing url manually') parts = list(urlsplit(err.response.url)) parts[2] = os.path.abspath(parts[2]) return self.location(urlunsplit(parts))
def open(self, *args, **kwargs): try: return super(BPBrowser, self).open(*args, **kwargs) except ServerError as err: if "/../" not in err.response.url: raise # this shit website includes ".." in an absolute url in the Location header # requests passes it verbatim, and the site can't handle it self.logger.debug('site has "/../" in their url, fixing url manually') parts = list(urlsplit(err.response.url)) parts[2] = os.path.abspath(parts[2]) return self.open(urlunsplit(parts))
def __init__(self, url, apiurl, username, password, *args, **kwargs): url_parsed = urlsplit(url) self.PROTOCOL = url_parsed.scheme self.DOMAIN = url_parsed.netloc self.BASEPATH = url_parsed.path if self.BASEPATH.endswith('/'): self.BASEPATH = self.BASEPATH[:-1] self.apiurl = apiurl self.username = username self.password = password DomainBrowser.__init__(self, *args, **kwargs)
def get_token(self): vary = None if self.params.get('vary', None) is not None: vary = self.params['vary'] else: for script in self.doc.xpath('//script'): if script.text is None: continue m = re.search("'vary', '([\d-]+)'\)", script.text) if m: vary = m.group(1) break url = self.browser.absurl( '/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') headers = {'Referer': self.url} r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers) if not int(r.headers.get('Content-Length', 0)): url = self.browser.absurl( '/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx' ) headers = {'Referer': self.url} r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers) doc = r.page.doc date = None for script in doc.xpath('//script'): if script.text is None: continue m = re.search('lastConnectionDate":"([^"]*)"', script.text) if m: date = m.group(1) url = self.browser.absurl( '/cyber/ibp/ate/portal/integratedInternet.jsp') data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date headers = {'Referer': r.url} r = self.browser.open(url, data=data, headers=headers) v = urlsplit(r.url) args = dict(parse_qsl(v.query)) return args['token']
def load_default_backends(self): """ Overload a Application method. """ for backend_name, module_name, params in self.weboob.backends_config.iter_backends(): if module_name != 'redmine': continue v = urlsplit(params['url']) if v.netloc == 'symlink.me': self.load_backends(names=[backend_name]) return if not self.check_loaded_backends({'url': 'https://symlink.me'}): print("Ok, so leave now.") sys.exit(0)
def load_default_backends(self): """ Overload a Application method. """ for backend_name, module_name, params in self.weboob.backends_config.iter_backends( ): if module_name != 'redmine': continue v = urlsplit(params['url']) if v.netloc == 'symlink.me': self.load_backends(names=[backend_name]) return if not self.check_loaded_backends({'url': 'https://symlink.me'}): print("Ok, so leave now, f*g.") sys.exit(0)
def url2id(url, nopost=False): v = urlsplit(url) pagename = v.path.split('/')[-1] args = parse_qs(v.query) if pagename == 'viewforum.php': return '%d' % int(args['f'][0]) if pagename == 'viewtopic.php': if 'f' in args: s = '%d' % int(args['f'][0]) else: s = '0' s += '.%d' % int(args['t'][0]) if 'p' in args and not nopost: s += '.%d' % int(args['p'][0]) return s return None
def market_pagination(self, account_id): # Next page is handled by js. Need to build the right url by changing params in current url several_pages = self.get_pages() if several_pages: current_page, total_pages = map(int, several_pages) if current_page < total_pages: params = { 'action': 11, 'idCptSelect': self.get_dropdown_menu(account_id), 'numPage': current_page + 1, } url_to_keep = urlsplit(self.browser.url)[:3] url_to_complete = ( urlencode(params), '' ) # '' is the urlsplit().fragment needed for urlunsplit next_page_url = urlunsplit(url_to_keep + url_to_complete) return next_page_url
def get_history(self, account): if account.type in (account.TYPE_PEA, account.TYPE_MARKET): self.go_linebourse(account) return self.linebourse.iter_history(account.id) transactions = [] v = urlsplit(account.url) args = dict(parse_qsl(v.query)) args['typeRecherche'] = 10 self.location(v.path, params=args) for tr in self.page.iter_history(): transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions
def get_history(self, account): if account.type in (account.TYPE_PEA, account.TYPE_MARKET): self.go_linebourse(account) return self.linebourse.iter_history(account.id) transactions = [] v = urlsplit(account.url) args = dict(parse_qsl(v.query)) args['typeRecherche'] = 10 self.location(v.path, params=args) for tr in self.page.iter_history(): transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions
def go_recipients_list(self, account_url, account_id): # url transfer preparation url = urlsplit(account_url) parts = [part for part in url.path.split('/') if part] assert len(parts) > 2, 'Account url missing some important part to iter recipient' account_type = parts[1] # cav, ord, epargne ... account_webid = parts[-1] self.transfer_main_page.go(acc_type=account_type, webid=account_webid) # may raise a BrowserHTTPNotFound # can check all account available transfer option if self.transfer_main_page.is_here(): self.transfer_accounts.go(acc_type=account_type, webid=account_webid) if self.transfer_accounts.is_here(): self.page.submit_account(account_id) # may raise AccountNotFound
def get_token(self): vary = None if self.params.get('vary', None) is not None: vary = self.params['vary'] else: for script in self.doc.xpath('//script'): if script.text is None: continue m = re.search("'vary', '([\d-]+)'\)", script.text) if m: vary = m.group(1) break url = self.browser.absurl('/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') headers = {'Referer': self.url} # Sometime, the page is a 302 and redirect to a page where there are no information that we need, # so we try with 2 others url to further fetch token when empty page r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers) if not int(r.headers.get('Content-Length', 0)): r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers) if not int(r.headers.get('Content-Length', 0)): r = self.browser.open(url, data={'taskId': 'equipementDom'}, params={'vary': vary}, headers=headers) doc = r.page.doc date = None for script in doc.xpath('//script'): if script.text is None: continue m = re.search('lastConnectionDate":"([^"]*)"', script.text) if m: date = m.group(1) url = self.browser.absurl('/cyber/ibp/ate/portal/integratedInternet.jsp') data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date headers = {'Referer': r.url} r = self.browser.open(url, data=data, headers=headers) v = urlsplit(r.url) args = dict(parse_qsl(v.query)) return args['token']
def on_load(self): div = self.doc.xpath('//div[has-class("pagination")]')[0] strongs = div.xpath('.//strong') self.cur_page = int(strongs[0].text.strip()) self.tot_pages = int(strongs[1].text.strip()) try: url = self.doc.xpath('//h2/a')[-1].attrib['href'] except (IndexError, KeyError): url = self.url v = urlsplit(url) args = parse_qs(v.query) self.topic_id = int(args['t'][0]) self.forum_id = int(args['f'][0]) if 'f' in args else 0 self.forum_title = u'' nav = self.doc.xpath('//li[has-class("icon-home")]') if len(nav) > 0: text = nav[0].xpath('.//a')[-1].text.strip() if len(text) >= 20: text = text[:20] + u'…' self.forum_title = '[%s] ' % text
def on_load(self): div = self.doc.xpath('//div[has-class("pagination")]')[0] strongs = div.xpath('.//strong') self.cur_page = int(strongs[0].text.strip()) self.tot_pages = int(strongs[1].text.strip()) try: url = self.doc.xpath('//h2/a')[-1].attrib['href'] except (IndexError, KeyError): url = self.url v = urlsplit(url) args = parse_qs(v.query) self.topic_id = int(args['t'][0]) self.forum_id = int(args['f'][0]) if 'f' in args else 0 self.forum_title = u'' nav = self.doc.xpath('//li[has-class("icon-home")]') if len(nav) > 0: text = nav[0].xpath('.//a')[-1].text.strip() if len(text) >= 20: text = text[:20] + u'…' self.forum_title = '[%s] ' % text
def iter_history_old(self, account): if self.cache.get(account.id, None) is None: self.cache[account.id] = {} self.cache[account.id]["history"] = [] if not self.accounts.is_here() and not self.accounts2.is_here(): self.go_on_accounts_list() url = account.url if not url: return while url is not None: if self.accounts.is_here() or self.accounts2.is_here(): self.location(url) else: form = self.page.get_form(name='leftnav') form.url = url form.submit() assert self.transactions.is_here() trs = sorted_transactions( self.page.get_history(account.currency)) for tr in trs: self.cache[account.id]["history"] += [tr] yield tr if self.page.is_last(): url = None else: v = urlsplit(url) args = dict(parse_qsl(v.query)) args['BPIndex'] = int(args['BPIndex']) + 1 url = '%s?%s' % (v.path, urlencode(args)) else: for tr in self.cache[account.id]["history"]: yield tr
def get_cb_operations(self, account, month=0): """ Get CB operations. * month=0 : current operations (non debited) * month=1 : previous month operations (debited) """ if not hasattr(account, '_coming_links'): return for link in account._coming_links: v = urlsplit(self.absurl(link)) args = dict(parse_qsl(v.query)) args['MOIS'] = month self.location(v.path, params=args) for tr in self.page.get_operations(): yield tr for card_link in self.page.get_cards(): self.location(card_link) for tr in self.page.get_operations(): yield tr
def get_cb_operations(self, account, month=0): """ Get CB operations. * month=0 : current operations (non debited) * month=1 : previous month operations (debited) """ if not hasattr(account, '_coming_links'): return for link in account._coming_links: v = urlsplit(self.absurl(link)) args = dict(parse_qsl(v.query)) args['MOIS'] = month self.location(v.path, params=args) for tr in self.page.get_operations(): yield tr for card_link in self.page.get_cards(): self.location(card_link) for tr in self.page.get_operations(): yield tr
def set_variables(self): v = urlsplit(self.url) version = v.path.split('/')[1] self.base_url = 'https://banqueenligne.entreprises.labanquepostale.fr/%s' % version self.accounts_url = self.base_url + '/voscomptes/synthese/synthese.ea'
def get_token(self): url = self.doc.xpath('//frame[@name="portalHeader"]')[0].attrib['src'] v = urlsplit(url) args = dict(parse_qsl(v.query)) return args['token']
def __init__(self, *args, **kwargs): super(LCLProBrowser, self).__init__(*args, **kwargs) self.session.cookies.set("lclgen", "professionnels", domain=urlsplit(self.BASEURL).hostname) self.owner_type = AccountOwnerType.ORGANIZATION
def __init__(self, *args, **kwargs): super(LCLProBrowser, self).__init__(*args, **kwargs) self.session.cookies.set("lclgen", "professionnels", domain=urlsplit(self.BASEURL).hostname) self.owner_type = AccountOwnerType.ORGANIZATION
def __init__(self, *args, **kwargs): super(LCLProBrowser, self).__init__(*args, **kwargs) self.session.cookies.set("lclgen", "professionnels", domain=urlsplit(self.BASEURL).hostname)
def set_variables(self): v = urlsplit(self.url) version = v.path.split('/')[1] self.base_url = 'https://banqueenligne.entreprises.labanquepostale.fr/%s' % version self.accounts_url = self.base_url + '/voscomptes/synthese/synthese.ea'
def get_token(self): url = self.doc.xpath('//frame[@name="portalHeader"]')[0].attrib['src'] v = urlsplit(url) args = dict(parse_qsl(v.query)) return args['token']
def __init__(self, *args, **kwargs): super(ELCLBrowser, self).__init__(*args, **kwargs) self.session.cookies.set('lclgen', 'ecl', domain=urlsplit(self.BASEURL).hostname)
def __init__(self, *args, **kwargs): super(ELCLBrowser, self).__init__(*args, **kwargs) self.session.cookies.set('lclgen', 'ecl', domain=urlsplit(self.BASEURL).hostname)