def go_post(self, url, data=None): # most of HSBC accounts links are actually handled by js code # which convert a GET query string to POST data. # not doing so often results in logout by the site q = dict(parse_qsl(urlparse(url).query)) if data: q.update(data) url = url[:url.find('?')] self.location(url, data=q)
def on_load(self): auth_query_params = re.search(r'parent\.location = ".*#(.*)";', self.text) assert auth_query_params, 'Url query parameter with token for authentication was not found' auth_query_params = auth_query_params.group(1) params = dict(parse_qsl(auth_query_params)) self.browser.token = params.get('id_token', None) self.browser.csrf = params['access_token']
def request_access_token(self, auth_uri): self.logger.info('requesting access token') if isinstance(auth_uri, dict): values = auth_uri else: values = dict(parse_qsl(urlparse(auth_uri).query)) data = self.build_access_token_parameters(values) try: auth_response = self.do_token_request(data).json() except ClientError: raise BrowserIncorrectPassword() self.update_token(auth_response)
def get_token(self): vary = None if self.params.get('vary', None) is not None: vary = self.params['vary'] else: for script in self.doc.xpath('//script'): if script.text is None: continue m = re.search("'vary', '([\d-]+)'\)", script.text) if m: vary = m.group(1) break url = self.browser.absurl( '/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') headers = {'Referer': self.url} r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers) if not int(r.headers.get('Content-Length', 0)): url = self.browser.absurl( '/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx' ) headers = {'Referer': self.url} r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers) doc = r.page.doc date = None for script in doc.xpath('//script'): if script.text is None: continue m = re.search('lastConnectionDate":"([^"]*)"', script.text) if m: date = m.group(1) url = self.browser.absurl( '/cyber/ibp/ate/portal/integratedInternet.jsp') data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date headers = {'Referer': r.url} r = self.browser.open(url, data=data, headers=headers) v = urlsplit(r.url) args = dict(parse_qsl(v.query)) return args['token']
def prepare_url(url, fields): components = urlparse(url) query_pairs = [(f, v) for (f, v) in parse_qsl(components.query) if f not in fields] for (field, value) in fields.items(): query_pairs.append((field, value)) new_query_str = urlencode(query_pairs) new_components = (components.scheme, components.netloc, components.path, components.params, new_query_str, components.fragment) return urlunparse(new_components)
def request_access_token(self, auth_uri): self.logger.info('requesting access token') if isinstance(auth_uri, dict): values = auth_uri else: values = dict(parse_qsl(urlparse(auth_uri).query)) self.handle_callback_error(values) data = self.build_access_token_parameters(values) try: auth_response = self.do_token_request(data).json() except ClientError: raise BrowserIncorrectPassword() self.update_token(auth_response)
def get_history(self, account): if account.type in (account.TYPE_PEA, account.TYPE_MARKET): self.go_linebourse(account) return self.linebourse.iter_history(account.id) transactions = [] v = urlsplit(account.url) args = dict(parse_qsl(v.query)) args['typeRecherche'] = 10 self.location(v.path, params=args) for tr in self.page.iter_history(): transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions
def get_token(self): vary = None if self.params.get('vary', None) is not None: vary = self.params['vary'] else: for script in self.doc.xpath('//script'): if script.text is None: continue m = re.search("'vary', '([\d-]+)'\)", script.text) if m: vary = m.group(1) break url = self.browser.absurl('/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') headers = {'Referer': self.url} # Sometime, the page is a 302 and redirect to a page where there are no information that we need, # so we try with 2 others url to further fetch token when empty page r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers) if not int(r.headers.get('Content-Length', 0)): r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers) if not int(r.headers.get('Content-Length', 0)): r = self.browser.open(url, data={'taskId': 'equipementDom'}, params={'vary': vary}, headers=headers) doc = r.page.doc date = None for script in doc.xpath('//script'): if script.text is None: continue m = re.search('lastConnectionDate":"([^"]*)"', script.text) if m: date = m.group(1) url = self.browser.absurl('/cyber/ibp/ate/portal/integratedInternet.jsp') data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date headers = {'Referer': r.url} r = self.browser.open(url, data=data, headers=headers) v = urlsplit(r.url) args = dict(parse_qsl(v.query)) return args['token']
def do_login(self): self.login_page.go() self.page.login(self.username, self.password, self.lastname) # q is timestamp millisecond self.app_config.go(params={'q': int(time()*1000)}) client_id = self.page.get_client_id() params = { 'client_id': client_id, 'response_type': 'id_token token', 'redirect_uri': 'https://www.bouyguestelecom.fr/mon-compte/' } self.location('https://oauth2.bouyguestelecom.fr/authorize', params=params) fragments = dict(parse_qsl(urlparse(self.url).fragment)) self.id_personne = jwt.get_unverified_claims(fragments['id_token'])['id_personne'] authorization = 'Bearer ' + fragments['access_token'] self.headers = {'Authorization': authorization}
def prepare_url(url, fields): components = urlparse(url) query_pairs = [(f, v) for (f, v) in parse_qsl(components.query) if f not in fields] for (field, value) in fields.items(): query_pairs.append((field, value)) new_query_str = urlencode(query_pairs) new_components = ( components.scheme, components.netloc, components.path, components.params, new_query_str, components.fragment ) return urlunparse(new_components)
def iter_history_old(self, account): if self.cache.get(account.id, None) is None: self.cache[account.id] = {} self.cache[account.id]["history"] = [] if not self.accounts.is_here() and not self.accounts2.is_here(): self.go_on_accounts_list() url = account.url if not url: return while url is not None: if self.accounts.is_here() or self.accounts2.is_here(): self.location(url) else: form = self.page.get_form(name='leftnav') form.url = url form.submit() assert self.transactions.is_here() trs = sorted_transactions( self.page.get_history(account.currency)) for tr in trs: self.cache[account.id]["history"] += [tr] yield tr if self.page.is_last(): url = None else: v = urlsplit(url) args = dict(parse_qsl(v.query)) args['BPIndex'] = int(args['BPIndex']) + 1 url = '%s?%s' % (v.path, urlencode(args)) else: for tr in self.cache[account.id]["history"]: yield tr
def do_login(self): self.login_page.go() try: self.page.login(self.username, self.password, self.lastname) except ClientError as e: if e.response.status_code == 401: raise BrowserIncorrectPassword() raise if self.login_page.is_here(): msg = self.page.get_error_message() raise BrowserIncorrectPassword(msg) if self.forgotten_password_page.is_here(): # when too much attempt has been done in a short time, bouygues redirect us here, # but no message is available on this page raise BrowserIncorrectPassword() # q is timestamp millisecond self.app_config.go(params={'q': int(time() * 1000)}) client_id = self.page.get_client_id() params = { 'client_id': client_id, 'response_type': 'id_token token', 'redirect_uri': 'https://www.bouyguestelecom.fr/mon-compte/' } self.location('https://oauth2.bouyguestelecom.fr/authorize', params=params) fragments = dict(parse_qsl(urlparse(self.url).fragment)) self.id_personne = jwt.get_unverified_claims( fragments['id_token'])['id_personne'] authorization = 'Bearer ' + fragments['access_token'] self.headers = {'Authorization': authorization}
def get_cb_operations(self, account, month=0): """ Get CB operations. * month=0 : current operations (non debited) * month=1 : previous month operations (debited) """ if not hasattr(account, '_coming_links'): return for link in account._coming_links: v = urlsplit(self.absurl(link)) args = dict(parse_qsl(v.query)) args['MOIS'] = month self.location(v.path, params=args) for tr in self.page.get_operations(): yield tr for card_link in self.page.get_cards(): self.location(card_link) for tr in self.page.get_operations(): yield tr
def iter_accounts(self, next_pages): account_type = Account.TYPE_UNKNOWN params = self.get_params() actions = self.get_button_actions() for div in self.doc.xpath('//div[has-class("btit")]'): if div.text in (None, u'Synthèse'): continue account_type = self.ACCOUNT_TYPES.get(div.text.strip(), Account.TYPE_UNKNOWN) if account_type is None: # ignore services accounts self.logger.debug('Ignore account type %s', div.text.strip()) continue # Go to the full list of this kind of account, if any. btn = div.getparent().xpath('.//button[span[text()="Suite"]]') if len(btn) > 0: _params = params.copy() _params.update(actions[btn[0].attrib['id']]) next_pages.append(_params) continue currency = None for th in div.getnext().xpath('.//thead//th'): m = re.match('.*\((\w+)\)$', th.text) if m and currency is None: currency = Account.get_currency(m.group(1)) for tr in div.getnext().xpath('.//tbody/tr'): if 'id' not in tr.attrib: continue args = dict(parse_qsl(tr.attrib['id'])) tds = tr.findall('td') if len(tds) < 4 or 'identifiant' not in args: self.logger.warning('Unable to parse an account') continue account = Account() account.id = args['identifiant'].replace(' ', '') account.label = u' '.join([u''.join([txt.strip() for txt in tds[1].itertext()]), u''.join([txt.strip() for txt in tds[2].itertext()])]).strip() for pattern, _type in self.PATTERN: match = pattern.match(account.label) if match: account.type = _type break else: account.type = account_type balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') account.currency = currency or Account.get_currency(balance_text) if account.type == account.TYPE_LOAN: account.balance = - abs(account.balance) account._prev_debit = None account._next_debit = None account._params = None account._coming_params = None account._coming_count = None account._invest_params = None if balance != u'' and len(tds[3].xpath('.//a')) > 0: account._params = params.copy() account._params['dialogActionPerformed'] = 'SOLDE' account._params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] if len(tds) >= 5 and len(tds[self.COL_COMING].xpath('.//a')) > 0: _params = account._params.copy() _params['dialogActionPerformed'] = 'ENCOURS_COMPTE' # If there is an action needed before going to the cards page, save it. m = re.search('dialogActionPerformed=([\w_]+)', self.url) if m and m.group(1) != 'EQUIPEMENT_COMPLET': _params['prevAction'] = m.group(1) next_pages.append(_params) if not account._params: account._invest_params = params.copy() account._invest_params['dialogActionPerformed'] = 'CONTRAT' account._invest_params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] yield account # Needed to preserve navigation. btn = self.doc.xpath('.//button[span[text()="Retour"]]') if len(btn) > 0: _params = params.copy() _params.update(actions[btn[0].attrib['id']]) self.browser.open('/cyber/internet/ContinueTask.do', data=_params)
def get_context_token(self): parameters = dict(parse_qsl(urlparse(self.url).query)) return parameters.get('context_token', None)
def add_qs(url, **kwargs): parts = list(urlparse(url)) qs = OrderedDict(parse_qsl(parts[4])) qs.update(kwargs) parts[4] = urlencode(qs) return urlunparse(parts)
def get_token(self): url = self.doc.xpath('//frame[@name="portalHeader"]')[0].attrib['src'] v = urlsplit(url) args = dict(parse_qsl(v.query)) return args['token']
def build_authorization_uri(self): p = urlparse(self.AUTHORIZATION_URI) q = dict(parse_qsl(p.query)) q.update(self.build_authorization_parameters()) return p._replace(query=urlencode(q)).geturl()
def parse_qs(d): return dict(parse_qsl(d))
def iter_accounts(self, next_pages): account_type = Account.TYPE_UNKNOWN params = self.get_params() actions = self.get_button_actions() for div in self.doc.xpath('//div[has-class("btit")]'): if div.text in (None, u'Synthèse'): continue account_type = self.ACCOUNT_TYPES.get(div.text.strip(), Account.TYPE_UNKNOWN) if account_type is None: # ignore services accounts self.logger.debug('Ignore account type %s', div.text.strip()) continue # Go to the full list of this kind of account, if any. btn = div.getparent().xpath('.//button[span[text()="Suite"]]') if len(btn) > 0: _params = params.copy() _params.update(actions[btn[0].attrib['id']]) next_pages.append(_params) continue currency = None for th in div.getnext().xpath('.//thead//th'): m = re.match('.*\((\w+)\)$', th.text) if m and currency is None: currency = Account.get_currency(m.group(1)) for tr in div.getnext().xpath('.//tbody/tr'): if 'id' not in tr.attrib: continue args = dict(parse_qsl(tr.attrib['id'])) tds = tr.findall('td') if len(tds) < 4 or 'identifiant' not in args: self.logger.warning('Unable to parse an account') continue account = Account() account.id = args['identifiant'].replace(' ', '') account.label = u' '.join([ u''.join([txt.strip() for txt in tds[1].itertext()]), u''.join([txt.strip() for txt in tds[2].itertext()]) ]).strip() for pattern, _type in self.PATTERN: match = pattern.match(account.label) if match: account.type = _type break else: account.type = account_type balance = FrenchTransaction.clean_amount(u''.join( [txt.strip() for txt in tds[3].itertext()])) account.balance = Decimal(balance or '0.0') account.currency = currency if account.type == account.TYPE_LOAN: account.balance = -abs(account.balance) account._prev_debit = None account._next_debit = None account._params = None account._coming_params = None account._invest_params = None if balance != u'' and len(tds[3].xpath('.//a')) > 0: account._params = params.copy() account._params['dialogActionPerformed'] = 'SOLDE' account._params[ 'attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split( '_', 1)[1] if len(tds) >= 5 and len( tds[self.COL_COMING].xpath('.//a')) > 0: _params = account._params.copy() _params['dialogActionPerformed'] = 'ENCOURS_COMPTE' # If there is an action needed before going to the cards page, save it. m = re.search('dialogActionPerformed=([\w_]+)', self.url) if m and m.group(1) != 'EQUIPEMENT_COMPLET': _params['prevAction'] = m.group(1) next_pages.append(_params) if not account._params: account._invest_params = params.copy() account._invest_params['dialogActionPerformed'] = 'CONTRAT' account._invest_params[ 'attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split( '_', 1)[1] yield account # Needed to preserve navigation. btn = self.doc.xpath('.//button[span[text()="Retour"]]') if len(btn) > 0: _params = params.copy() _params.update(actions[btn[0].attrib['id']]) self.browser.open('/cyber/internet/ContinueTask.do', data=_params)