def go_post(self, url, data=None):
# most of HSBC accounts links are actually handled by js code
# which convert a GET query string to POST data.
# not doing so often results in logout by the site
q = dict(parse_qsl(urlparse(url).query))
if data:
q.update(data)
url = url[:url.find('?')]
self.location(url, data=q)
def go_post(self, url, data=None):
# most of HSBC accounts links are actually handled by js code
# which convert a GET query string to POST data.
# not doing so often results in logout by the site
q = dict(parse_qsl(urlparse(url).query))
if data:
q.update(data)
url = url[:url.find('?')]
self.location(url, data=q)
def on_load(self):
auth_query_params = re.search(r'parent\.location = ".*#(.*)";',
self.text)
assert auth_query_params, 'Url query parameter with token for authentication was not found'
auth_query_params = auth_query_params.group(1)
params = dict(parse_qsl(auth_query_params))
self.browser.token = params.get('id_token', None)
self.browser.csrf = params['access_token']
def request_access_token(self, auth_uri):
self.logger.info('requesting access token')
if isinstance(auth_uri, dict):
values = auth_uri
else:
values = dict(parse_qsl(urlparse(auth_uri).query))
data = self.build_access_token_parameters(values)
try:
auth_response = self.do_token_request(data).json()
except ClientError:
raise BrowserIncorrectPassword()
self.update_token(auth_response)
def get_token(self):
vary = None
if self.params.get('vary', None) is not None:
vary = self.params['vary']
else:
for script in self.doc.xpath('//script'):
if script.text is None:
continue
m = re.search("'vary', '([\d-]+)'\)", script.text)
if m:
vary = m.group(1)
break
url = self.browser.absurl(
'/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx')
headers = {'Referer': self.url}
r = self.browser.open(url,
data='taskId=aUniversMesComptes',
params={'vary': vary},
headers=headers)
if not int(r.headers.get('Content-Length', 0)):
url = self.browser.absurl(
'/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx'
)
headers = {'Referer': self.url}
r = self.browser.open(url,
data='taskId=aUniversMesComptes',
headers=headers)
doc = r.page.doc
date = None
for script in doc.xpath('//script'):
if script.text is None:
continue
m = re.search('lastConnectionDate":"([^"]*)"', script.text)
if m:
date = m.group(1)
url = self.browser.absurl(
'/cyber/ibp/ate/portal/integratedInternet.jsp')
data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date
headers = {'Referer': r.url}
r = self.browser.open(url, data=data, headers=headers)
v = urlsplit(r.url)
args = dict(parse_qsl(v.query))
return args['token']
def prepare_url(url, fields):
components = urlparse(url)
query_pairs = [(f, v) for (f, v) in parse_qsl(components.query)
if f not in fields]
for (field, value) in fields.items():
query_pairs.append((field, value))
new_query_str = urlencode(query_pairs)
new_components = (components.scheme, components.netloc,
components.path, components.params,
new_query_str, components.fragment)
return urlunparse(new_components)
def request_access_token(self, auth_uri):
self.logger.info('requesting access token')
if isinstance(auth_uri, dict):
values = auth_uri
else:
values = dict(parse_qsl(urlparse(auth_uri).query))
self.handle_callback_error(values)
data = self.build_access_token_parameters(values)
try:
auth_response = self.do_token_request(data).json()
except ClientError:
raise BrowserIncorrectPassword()
self.update_token(auth_response)
def get_history(self, account):
if account.type in (account.TYPE_PEA, account.TYPE_MARKET):
self.go_linebourse(account)
return self.linebourse.iter_history(account.id)
transactions = []
v = urlsplit(account.url)
args = dict(parse_qsl(v.query))
args['typeRecherche'] = 10
self.location(v.path, params=args)
for tr in self.page.iter_history():
transactions.append(tr)
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def get_history(self, account):
if account.type in (account.TYPE_PEA, account.TYPE_MARKET):
self.go_linebourse(account)
return self.linebourse.iter_history(account.id)
transactions = []
v = urlsplit(account.url)
args = dict(parse_qsl(v.query))
args['typeRecherche'] = 10
self.location(v.path, params=args)
for tr in self.page.iter_history():
transactions.append(tr)
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def get_token(self):
vary = None
if self.params.get('vary', None) is not None:
vary = self.params['vary']
else:
for script in self.doc.xpath('//script'):
if script.text is None:
continue
m = re.search("'vary', '([\d-]+)'\)", script.text)
if m:
vary = m.group(1)
break
url = self.browser.absurl('/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx')
headers = {'Referer': self.url}
# Sometime, the page is a 302 and redirect to a page where there are no information that we need,
# so we try with 2 others url to further fetch token when empty page
r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers)
if not int(r.headers.get('Content-Length', 0)):
r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers)
if not int(r.headers.get('Content-Length', 0)):
r = self.browser.open(url, data={'taskId': 'equipementDom'}, params={'vary': vary}, headers=headers)
doc = r.page.doc
date = None
for script in doc.xpath('//script'):
if script.text is None:
continue
m = re.search('lastConnectionDate":"([^"]*)"', script.text)
if m:
date = m.group(1)
url = self.browser.absurl('/cyber/ibp/ate/portal/integratedInternet.jsp')
data = 'session%%3Aate.lastConnectionDate=%s&taskId=aUniversMesComptes' % date
headers = {'Referer': r.url}
r = self.browser.open(url, data=data, headers=headers)
v = urlsplit(r.url)
args = dict(parse_qsl(v.query))
return args['token']
def do_login(self):
self.login_page.go()
self.page.login(self.username, self.password, self.lastname)
# q is timestamp millisecond
self.app_config.go(params={'q': int(time()*1000)})
client_id = self.page.get_client_id()
params = {
'client_id': client_id,
'response_type': 'id_token token',
'redirect_uri': 'https://www.bouyguestelecom.fr/mon-compte/'
}
self.location('https://oauth2.bouyguestelecom.fr/authorize', params=params)
fragments = dict(parse_qsl(urlparse(self.url).fragment))
self.id_personne = jwt.get_unverified_claims(fragments['id_token'])['id_personne']
authorization = 'Bearer ' + fragments['access_token']
self.headers = {'Authorization': authorization}
def prepare_url(url, fields):
components = urlparse(url)
query_pairs = [(f, v) for (f, v) in parse_qsl(components.query) if f not in fields]
for (field, value) in fields.items():
query_pairs.append((field, value))
new_query_str = urlencode(query_pairs)
new_components = (
components.scheme,
components.netloc,
components.path,
components.params,
new_query_str,
components.fragment
)
return urlunparse(new_components)
def iter_history_old(self, account):
if self.cache.get(account.id, None) is None:
self.cache[account.id] = {}
self.cache[account.id]["history"] = []
if not self.accounts.is_here() and not self.accounts2.is_here():
self.go_on_accounts_list()
url = account.url
if not url:
return
while url is not None:
if self.accounts.is_here() or self.accounts2.is_here():
self.location(url)
else:
form = self.page.get_form(name='leftnav')
form.url = url
form.submit()
assert self.transactions.is_here()
trs = sorted_transactions(
self.page.get_history(account.currency))
for tr in trs:
self.cache[account.id]["history"] += [tr]
yield tr
if self.page.is_last():
url = None
else:
v = urlsplit(url)
args = dict(parse_qsl(v.query))
args['BPIndex'] = int(args['BPIndex']) + 1
url = '%s?%s' % (v.path, urlencode(args))
else:
for tr in self.cache[account.id]["history"]:
yield tr
def do_login(self):
self.login_page.go()
try:
self.page.login(self.username, self.password, self.lastname)
except ClientError as e:
if e.response.status_code == 401:
raise BrowserIncorrectPassword()
raise
if self.login_page.is_here():
msg = self.page.get_error_message()
raise BrowserIncorrectPassword(msg)
if self.forgotten_password_page.is_here():
# when too much attempt has been done in a short time, bouygues redirect us here,
# but no message is available on this page
raise BrowserIncorrectPassword()
# q is timestamp millisecond
self.app_config.go(params={'q': int(time() * 1000)})
client_id = self.page.get_client_id()
params = {
'client_id': client_id,
'response_type': 'id_token token',
'redirect_uri': 'https://www.bouyguestelecom.fr/mon-compte/'
}
self.location('https://oauth2.bouyguestelecom.fr/authorize',
params=params)
fragments = dict(parse_qsl(urlparse(self.url).fragment))
self.id_personne = jwt.get_unverified_claims(
fragments['id_token'])['id_personne']
authorization = 'Bearer ' + fragments['access_token']
self.headers = {'Authorization': authorization}
def get_cb_operations(self, account, month=0):
"""
Get CB operations.
* month=0 : current operations (non debited)
* month=1 : previous month operations (debited)
"""
if not hasattr(account, '_coming_links'):
return
for link in account._coming_links:
v = urlsplit(self.absurl(link))
args = dict(parse_qsl(v.query))
args['MOIS'] = month
self.location(v.path, params=args)
for tr in self.page.get_operations():
yield tr
for card_link in self.page.get_cards():
self.location(card_link)
for tr in self.page.get_operations():
yield tr
def get_cb_operations(self, account, month=0):
"""
Get CB operations.
* month=0 : current operations (non debited)
* month=1 : previous month operations (debited)
"""
if not hasattr(account, '_coming_links'):
return
for link in account._coming_links:
v = urlsplit(self.absurl(link))
args = dict(parse_qsl(v.query))
args['MOIS'] = month
self.location(v.path, params=args)
for tr in self.page.get_operations():
yield tr
for card_link in self.page.get_cards():
self.location(card_link)
for tr in self.page.get_operations():
yield tr
def iter_accounts(self, next_pages):
account_type = Account.TYPE_UNKNOWN
params = self.get_params()
actions = self.get_button_actions()
for div in self.doc.xpath('//div[has-class("btit")]'):
if div.text in (None, u'Synthèse'):
continue
account_type = self.ACCOUNT_TYPES.get(div.text.strip(), Account.TYPE_UNKNOWN)
if account_type is None:
# ignore services accounts
self.logger.debug('Ignore account type %s', div.text.strip())
continue
# Go to the full list of this kind of account, if any.
btn = div.getparent().xpath('.//button[span[text()="Suite"]]')
if len(btn) > 0:
_params = params.copy()
_params.update(actions[btn[0].attrib['id']])
next_pages.append(_params)
continue
currency = None
for th in div.getnext().xpath('.//thead//th'):
m = re.match('.*\((\w+)\)$', th.text)
if m and currency is None:
currency = Account.get_currency(m.group(1))
for tr in div.getnext().xpath('.//tbody/tr'):
if 'id' not in tr.attrib:
continue
args = dict(parse_qsl(tr.attrib['id']))
tds = tr.findall('td')
if len(tds) < 4 or 'identifiant' not in args:
self.logger.warning('Unable to parse an account')
continue
account = Account()
account.id = args['identifiant'].replace(' ', '')
account.label = u' '.join([u''.join([txt.strip() for txt in tds[1].itertext()]),
u''.join([txt.strip() for txt in tds[2].itertext()])]).strip()
for pattern, _type in self.PATTERN:
match = pattern.match(account.label)
if match:
account.type = _type
break
else:
account.type = account_type
balance_text = u''.join([txt.strip() for txt in tds[3].itertext()])
balance = FrenchTransaction.clean_amount(balance_text)
account.balance = Decimal(balance or '0.0')
account.currency = currency or Account.get_currency(balance_text)
if account.type == account.TYPE_LOAN:
account.balance = - abs(account.balance)
account._prev_debit = None
account._next_debit = None
account._params = None
account._coming_params = None
account._coming_count = None
account._invest_params = None
if balance != u'' and len(tds[3].xpath('.//a')) > 0:
account._params = params.copy()
account._params['dialogActionPerformed'] = 'SOLDE'
account._params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1]
if len(tds) >= 5 and len(tds[self.COL_COMING].xpath('.//a')) > 0:
_params = account._params.copy()
_params['dialogActionPerformed'] = 'ENCOURS_COMPTE'
# If there is an action needed before going to the cards page, save it.
m = re.search('dialogActionPerformed=([\w_]+)', self.url)
if m and m.group(1) != 'EQUIPEMENT_COMPLET':
_params['prevAction'] = m.group(1)
next_pages.append(_params)
if not account._params:
account._invest_params = params.copy()
account._invest_params['dialogActionPerformed'] = 'CONTRAT'
account._invest_params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1]
yield account
# Needed to preserve navigation.
btn = self.doc.xpath('.//button[span[text()="Retour"]]')
if len(btn) > 0:
_params = params.copy()
_params.update(actions[btn[0].attrib['id']])
self.browser.open('/cyber/internet/ContinueTask.do', data=_params)
def get_context_token(self):
parameters = dict(parse_qsl(urlparse(self.url).query))
return parameters.get('context_token', None)
def add_qs(url, **kwargs):
parts = list(urlparse(url))
qs = OrderedDict(parse_qsl(parts[4]))
qs.update(kwargs)
parts[4] = urlencode(qs)
return urlunparse(parts)
def get_token(self):
url = self.doc.xpath('//frame[@name="portalHeader"]')[0].attrib['src']
v = urlsplit(url)
args = dict(parse_qsl(v.query))
return args['token']
def build_authorization_uri(self):
p = urlparse(self.AUTHORIZATION_URI)
q = dict(parse_qsl(p.query))
q.update(self.build_authorization_parameters())
return p._replace(query=urlencode(q)).geturl()
def parse_qs(d):
return dict(parse_qsl(d))
def get_token(self):
url = self.doc.xpath('//frame[@name="portalHeader"]')[0].attrib['src']
v = urlsplit(url)
args = dict(parse_qsl(v.query))
return args['token']
def iter_accounts(self, next_pages):
account_type = Account.TYPE_UNKNOWN
params = self.get_params()
actions = self.get_button_actions()
for div in self.doc.xpath('//div[has-class("btit")]'):
if div.text in (None, u'Synthèse'):
continue
account_type = self.ACCOUNT_TYPES.get(div.text.strip(),
Account.TYPE_UNKNOWN)
if account_type is None:
# ignore services accounts
self.logger.debug('Ignore account type %s', div.text.strip())
continue
# Go to the full list of this kind of account, if any.
btn = div.getparent().xpath('.//button[span[text()="Suite"]]')
if len(btn) > 0:
_params = params.copy()
_params.update(actions[btn[0].attrib['id']])
next_pages.append(_params)
continue
currency = None
for th in div.getnext().xpath('.//thead//th'):
m = re.match('.*\((\w+)\)$', th.text)
if m and currency is None:
currency = Account.get_currency(m.group(1))
for tr in div.getnext().xpath('.//tbody/tr'):
if 'id' not in tr.attrib:
continue
args = dict(parse_qsl(tr.attrib['id']))
tds = tr.findall('td')
if len(tds) < 4 or 'identifiant' not in args:
self.logger.warning('Unable to parse an account')
continue
account = Account()
account.id = args['identifiant'].replace(' ', '')
account.label = u' '.join([
u''.join([txt.strip() for txt in tds[1].itertext()]),
u''.join([txt.strip() for txt in tds[2].itertext()])
]).strip()
for pattern, _type in self.PATTERN:
match = pattern.match(account.label)
if match:
account.type = _type
break
else:
account.type = account_type
balance = FrenchTransaction.clean_amount(u''.join(
[txt.strip() for txt in tds[3].itertext()]))
account.balance = Decimal(balance or '0.0')
account.currency = currency
if account.type == account.TYPE_LOAN:
account.balance = -abs(account.balance)
account._prev_debit = None
account._next_debit = None
account._params = None
account._coming_params = None
account._invest_params = None
if balance != u'' and len(tds[3].xpath('.//a')) > 0:
account._params = params.copy()
account._params['dialogActionPerformed'] = 'SOLDE'
account._params[
'attribute($SEL_$%s)' %
tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split(
'_', 1)[1]
if len(tds) >= 5 and len(
tds[self.COL_COMING].xpath('.//a')) > 0:
_params = account._params.copy()
_params['dialogActionPerformed'] = 'ENCOURS_COMPTE'
# If there is an action needed before going to the cards page, save it.
m = re.search('dialogActionPerformed=([\w_]+)', self.url)
if m and m.group(1) != 'EQUIPEMENT_COMPLET':
_params['prevAction'] = m.group(1)
next_pages.append(_params)
if not account._params:
account._invest_params = params.copy()
account._invest_params['dialogActionPerformed'] = 'CONTRAT'
account._invest_params[
'attribute($SEL_$%s)' %
tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split(
'_', 1)[1]
yield account
# Needed to preserve navigation.
btn = self.doc.xpath('.//button[span[text()="Retour"]]')
if len(btn) > 0:
_params = params.copy()
_params.update(actions[btn[0].attrib['id']])
self.browser.open('/cyber/internet/ContinueTask.do', data=_params)
def build_authorization_uri(self):
p = urlparse(self.AUTHORIZATION_URI)
q = dict(parse_qsl(p.query))
q.update(self.build_authorization_parameters())
return p._replace(query=urlencode(q)).geturl()
def add_qs(url, **kwargs):
parts = list(urlparse(url))
qs = OrderedDict(parse_qsl(parts[4]))
qs.update(kwargs)
parts[4] = urlencode(qs)
return urlunparse(parts)
