def do_login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
if not self.password.isdigit() or len(self.password) != 6:
raise BrowserIncorrectPassword()
if not self.username.isdigit() or len(self.username) < 8:
raise BrowserIncorrectPassword()
self.username = self.username[:8]
self.login.stay_or_go()
try:
self.page.login(self.username, self.password)
except BrowserHTTPNotFound:
raise BrowserIncorrectPassword()
if self.login.is_here():
raise BrowserIncorrectPassword()
if self.bad_login.is_here():
error = self.page.get_error()
if error is None:
raise BrowserIncorrectPassword()
elif error.startswith('Votre session a'):
raise BrowserUnavailable('Session has expired')
elif error.startswith('Le service est momentan'):
raise BrowserUnavailable(error)
else:
raise BrowserIncorrectPassword(error)
def do_login(self):
if not self.password.isdigit() or len(self.password) != 6:
raise BrowserIncorrectPassword()
if not self.username.isdigit() or len(self.username) < 8:
raise BrowserIncorrectPassword()
self.username = self.username[:8]
self.login.stay_or_go()
try:
self.page.login(self.username, self.password)
except BrowserHTTPNotFound:
raise BrowserIncorrectPassword()
if self.login.is_here():
raise BrowserIncorrectPassword()
if self.bad_login.is_here():
error = self.page.get_error()
if error is None:
raise BrowserIncorrectPassword()
elif error.startswith('Votre session a'):
raise BrowserUnavailable('Session has expired')
elif error.startswith('Le service est momentan'):
raise BrowserUnavailable(error)
elif 'niv_auth_insuff' in error:
raise BrowserIncorrectPassword(
"Niveau d'authentification insuffisant")
else:
raise BrowserIncorrectPassword(error)
def do_login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
if not self.login.is_here():
self.location('/signin/')
response = self.open(self.page.get_script_url())
token, csrf, key, value, sessionID, cookie = self.page.get_token_and_csrf(
response.text)
self.session.cookies.update({'xppcts': cookie})
data = {}
data['ads_token_js'] = token
data['_csrf'] = csrf
data['_sessionID'] = sessionID
data[key] = value
data = urllib.urlencode(data)
self.open('/auth/verifychallenge', data=data)
res = self.page.login(self.username, self.password)
if 'LoginFailed' in res.content or 'Sorry, we can\'t log you in' in res.content or self.error.is_here(
):
raise BrowserIncorrectPassword()
if '/auth/validatecaptcha' in res.content:
raise BrowserUnavailable('captcha')
self.location('/')
if self.old_website.is_here():
self.location('https://www.paypal.com/businessexp/summary')
if self.login.is_here() or self.landing.is_here():
raise BrowserUnavailable('login failed')
self.detect_account_type()
def on_load(self):
h1 = CleanText('//h1[1]')(self.doc)
if "est indisponible" in h1:
raise BrowserUnavailable(h1)
a = Link('//a[@class="btn"][1]', default=None)(self.doc)
if not a:
raise BrowserUnavailable()
self.browser.location(a)
def _get_history_invests(self, account):
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP):
if self.page.is_account_inactive(account.id):
self.logger.warning('Account %s %s is inactive.' %
(account.label, account.id))
return []
# There is (currently ?) no history for MILLEVIE PREMIUM accounts
if "MILLEVIE" in account.label:
try:
self.page.go_life_insurance(account)
except ServerError as ex:
if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label:
self.logger.info(
"Can not reach history page for MILLEVIE PREMIUM account"
)
return []
raise
label = account.label.split()[-1]
try:
self.natixis_life_ins_his.go(id1=label[:3],
id2=label[3:5],
id3=account.id)
except BrowserHTTPError as e:
if e.response.status_code == 500:
error = json.loads(e.response.text)
raise BrowserUnavailable(error["error"])
raise
return sorted_transactions(self.page.get_history())
if account.label.startswith(
'NUANCES ') or account.label in self.insurance_accounts:
self.page.go_life_insurance(account)
if 'JSESSIONID' in self.session.cookies:
# To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session
del self.session.cookies['JSESSIONID']
try:
if not self.life_insurance.is_here(
) and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' %
self.page.get_cons_histo())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return []
return self.page.iter_history()
def on_loaded(self):
try:
h1 = self.parser.select(self.document.getroot(), 'h1', 1)
except BrokenPageError:
pass
if h1.text is not None and h1.text.startswith('Le service est moment'):
try:
raise BrowserUnavailable(self.document.xpath('//h4')[0].text)
except KeyError:
raise BrowserUnavailable(h1.text)
def __next__(self):
if self.remaining <= 0:
raise BrowserUnavailable(
'Site did not reply successfully after multiple tries')
if self.delogged:
self.browser.do_login()
self.delogged = False
if self.it is None:
self.it = self.cb()
# recreated iterator, consume previous items
try:
nb = -1
for nb, sent in enumerate(self.items):
new = next(self.it)
if hasattr(new, 'iter_fields'):
equal = dict(sent.iter_fields()) == dict(
new.iter_fields())
else:
equal = sent == new
if not equal:
# safety is not guaranteed
raise BrowserUnavailable(
'Site replied inconsistently between retries, %r vs %r',
sent, new)
except StopIteration:
raise BrowserUnavailable(
'Site replied fewer elements (%d) than last iteration (%d)',
nb + 1, len(self.items))
except self.exc_check as exc:
self.delogged = True
if self.logger:
self.logger.info('%s raised, retrying', exc)
self.it = None
self.remaining -= 1
return next(self)
# return one item
try:
obj = next(self.it)
except self.exc_check as exc:
self.delogged = True
if self.logger:
self.logger.info('%s raised, retrying', exc)
self.it = None
self.remaining -= 1
return next(self)
else:
self.items.append(obj)
return obj
def handle_polling(self):
assert self.polling_transaction, "polling_transaction is mandatory !"
data = {'n10_id_transaction': self.polling_transaction}
timeout = time.time() + self.polling_duration
while time.time() < timeout:
self.location('/sec/oob_pollingooba.json', data=data)
status = self.page.doc['donnees']['transaction_status']
if status != "in_progress":
break
time.sleep(3)
else:
status = "aborted"
self.check_polling_errors(status)
data.update({
'oob_op': "auth",
})
self.location('/sec/oob_auth.json', data=data)
if self.page.doc.get('commun', {}).get('statut').lower() == "nok":
raise BrowserUnavailable()
self.polling_transaction = None
def on_loaded(self):
try:
a = self.document.xpath('//a[@class="btn"]')[0]
except IndexError:
raise BrowserUnavailable()
else:
self.browser.location(a.attrib['href'], nologin=True)
def on_load(self):
MyHTMLPage.on_load(self)
if self.doc.xpath(u'//h2[text()="ERREUR"]'): # website sometime crash
self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea')
raise BrowserUnavailable()
def do_login(self):
if self.config['pin_code'].get():
self.validate_security_form()
if not self.page.is_logged():
raise BrowserIncorrectPassword(
"Login / Password or authentication pin_code incorrect")
return
try:
self.login.go()
except ConnectionError as e:
raise BrowserUnavailable(e)
if self.page.is_logged():
return
self.page.login(self.username, self.password)
self.page.check_user_double_auth()
if self.page.check_website_double_auth():
self.otp_form = self.page.get_security_form()
self.otp_url = self.url
raise BrowserQuestion(
Value('pin_code',
label=self.page.get_otp_message()
or 'Please type the OTP you received'))
if not self.page.is_logged():
raise BrowserIncorrectPassword(self.page.get_error_message())
def do_login(self):
if not self.username or not self.password:
raise BrowserIncorrectPassword()
# First we try to connect to the new website: if the connection
# is on the old website, we will automatically redirected.
website = self.website.replace('.fr', '')
region_domain = re.sub(r'^www\.', 'www.credit-agricole.fr/', website)
self.BASEURL = 'https://%s/' % region_domain
self.login_page.go()
if self.old_website.is_here():
self.BASEURL = 'https://%s/' % self.website
self.logger.warning('This is a regional connection, switching to old website with URL %s', self.BASEURL)
raise SiteSwitch('region')
self.do_security_check()
# accounts_url may contain '/particulier', '/professionnel', '/entreprise', '/agriculteur' or '/association'
self.accounts_url = self.page.get_accounts_url()
assert self.accounts_url, 'Could not get accounts url from security check'
try:
self.location(self.accounts_url)
except HTTPNotFound:
# Sometimes the url the json sends us back is just unavailable...
raise BrowserUnavailable()
assert self.accounts_page.is_here(), 'We failed to login after the security check: response URL is %s' % self.url
def do_security_check(self):
try:
form = self.get_security_form()
self.security_check.go(data=form)
except ServerError as exc:
# Wrongpass returns a 500 server error...
exc_json = exc.response.json()
error = exc_json.get('error')
if error:
message = error.get('message', '')
wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit")
if any(value in message for value in wrongpass_messages):
raise BrowserIncorrectPassword()
if 'obtenir un nouveau code' in message:
raise ActionNeeded(message)
code = error.get('code', '')
technical_error_messages = ('Un incident technique', 'identifiant et votre code personnel')
# Sometimes there is no error message, so we try to use the code as well
technical_error_codes = ('technical_error',)
if any(value in message for value in technical_error_messages) or \
any(value in code for value in technical_error_codes):
raise BrowserUnavailable(message)
# When a PSD2 SCA is required it also returns a 500, hopefully we can detect it
if (
exc_json.get('url') == 'dsp2/informations.html'
or exc_json.get('redirection', '').endswith('dsp2/informations.html')
):
return self.handle_sca()
raise
def _iter_accounts(self):
owner_name = self.get_profile().name.upper()
self.loans.go(account_type=self.account_type,
loans_page_label=self.loans_page_label)
for a in self.page.get_list():
yield a
self.accounts.go(account_type=self.account_type,
accounts_page_label=self.accounts_page_label)
self.multitype_av.go()
if self.multitype_av.is_here():
for a in self.page.get_av_accounts():
self.location(a._link, data=a._args)
self.location(a._link.replace("_attente",
"_detail_contrat_rep"),
data=a._args)
if self.page.get_error():
raise BrowserUnavailable(self.page.get_error())
self.page.fill_diff_currency(a)
yield a
self.accounts.go(account_type=self.account_type,
accounts_page_label=self.accounts_page_label)
if self.accounts.is_here():
for a in self.page.get_list(name=owner_name):
yield a
else:
for a in self.page.get_list():
yield a
def on_load(self):
if Dict('commun/statut')(self.doc).upper() == 'NOK':
reason = Dict('commun/raison')(self.doc)
action = Dict('commun/action')(self.doc)
if action and 'BLOCAGE' in action:
raise ActionNeeded()
if reason and 'err_tech' in reason:
# This error is temporary and usually do not happens on the next try
raise TemporaryBrowserUnavailable()
if ('le service est momentanement indisponible' in reason
and Dict('commun/origine')(self.doc) != 'cbo'):
raise BrowserUnavailable()
if reason == "niv_auth_insuff":
return
conditions = (
'pas encore géré'
in reason, # this page is not handled by SG api website
'le service est momentanement indisponible'
in reason, # can't access new website
)
assert any(conditions), 'Error %s is not handled yet' % reason
self.logger.warning('Handled Error "%s"', reason)
def on_load(self):
super(BasePage, self).on_load()
if 'Erreur' in CleanText('//div[@id="main"]/h1', default='')(self.doc):
err = CleanText('//div[@id="main"]/div[@class="content"]',
default='Site indisponible')(self.doc)
raise BrowserUnavailable(err)
def on_load(self):
if self.doc.xpath(
'//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]'
):
# Warning: it could occurs because of wrongpass, user have to change password
raise BrowserUnavailable(
CleanText('//div[@class="ngo_mu_message"]')(self.doc))
def get_list(self):
accounts = []
previous_account = None
noaccounts = self.get_from_js('_js_noMvts =', ';')
if noaccounts is not None:
assert 'avez aucun compte' in noaccounts
return []
txt = self.get_from_js('_data = new Array(', ');', is_list=True)
if txt is None:
raise BrowserUnavailable('Unable to find accounts list in scripts')
data = json.loads('[%s]' % txt.replace("'", '"'))
for line in data:
a = Account()
a.id = line[self.COL_ID].replace(' ', '')
if re.match(r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)', a.id):
a.id = str(CleanDecimal().filter(a.id))
a._acc_nb = a.id.split('_')[0] if len(a.id.split('_')) > 1 else None
a.label = MyStrip(line[self.COL_LABEL], xpath='.//div[@class="libelleCompteTDB"]')
# This account can be multiple life insurance accounts
if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *':
continue
a.balance = Decimal(FrenchTransaction.clean_amount(line[self.COL_BALANCE]))
a.currency = a.get_currency(line[self.COL_BALANCE])
a.type = self.get_account_type(a.label)
# The parent account must be created right before
if a.type == Account.TYPE_CARD:
# duplicate
if find_object(accounts, id=a.id):
self.logger.warning('Ignoring duplicate card %r', a.id)
continue
a.parent = previous_account
if line[self.COL_HISTORY] == 'true':
a._inv = False
a._link = self.get_history_link()
a._args = self.make__args_dict(line)
else:
a._inv = True
a._args = {'_ipc_eventValue': line[self.COL_ID],
'_ipc_fireEvent': line[self.COL_FIRE_EVENT],
}
a._link = self.doc.xpath('//form[@name="changePageForm"]')[0].attrib['action']
if a.type is Account.TYPE_CARD:
a.coming = a.balance
a.balance = Decimal('0.0')
accounts.append(a)
previous_account = a
return accounts
def get_token_and_csrf(self, code):
# Paypal will try to create an infinite loop to make the parse fail, based on different
# weird things like a check of 'ind\\u0435xOf' vs 'indexOf'.
cleaner_code = code.replace(r"'ind\\u0435xOf'", "'indexOf'")
# It also calls "data" which is undefined instead of a return (next call is an infinite
# recursive function). This should theorically not happen if window.domain is correctly set
# to "paypal.com" though.
cleaner_code = cleaner_code.replace("data;", "return;")
# Remove setCookie function content
cleaner_code = re.sub(r"'setCookie'.*(?=,'removeCookie')",
"'setCookie':function(){}", cleaner_code)
# Paypal will try to send a XHR, let's use a fake method to catch the values sent
cleaner_code = """
XMLHttpRequest.prototype.send = function(body)
{
window.PAYPAL_TOKENS = body;
};
function GET_JS_TOKENS()
{
return window.PAYPAL_TOKENS || "INVALID_TOKENS";
}
""" + cleaner_code
try:
raw = str(
Javascript(cleaner_code, None,
"paypal.com").call("GET_JS_TOKENS"))
raw = raw.split("&")
tokens = {}
for r in raw:
r = r.split("=")
k = r[0]
v = unquote(r[1])
if k not in ["ads_token_js", "_sessionID", "_csrf"]:
tokens["key"] = k
tokens["value"] = v
else:
tokens[k] = v
token = tokens["ads_token_js"]
sessionID = tokens["_sessionID"]
csrf = tokens["_csrf"]
key = tokens["key"]
value = tokens["value"]
except:
raise BrowserUnavailable("Could not grab tokens")
# Clean string obfuscation like: '\x70\x61\x79\x70\x61\x6c\x20\x73\x75\x63\x6b\x73'
def basic_decoder(mtc):
return repr(literal_eval(mtc.group(0)).encode('utf-8'))
cleaner_code = re.sub(r"'.*?(?<!\\)'", basic_decoder, code)
cookie = re.search(r'xppcts = (\w+);', cleaner_code).group(1)
return token, csrf, key, value, sessionID, cookie
def to_statement(self, uri):
for i in range(self.MAX_RETRIES):
self.location(uri)
if self.statement.is_here():
break
else:
raise BrowserUnavailable()
def _make_api_call(self, account, start_date, end_date, offset, max_length=50):
HEADERS = {
'Accept': "application/json",
'Content-Type': 'application/json',
}
HEADERS.update(self.get_and_update_bred_token())
call_payload = {
"account": account._number,
"poste": account._nature,
"sousPoste": account._codeSousPoste or '00',
"devise": account.currency,
"fromDate": start_date.strftime('%Y-%m-%d'),
"toDate": end_date.strftime('%Y-%m-%d'),
"from": offset,
"size": max_length, # max length of transactions
"search": "",
"categorie": "",
}
result = self.open('/transactionnel/services/applications/operations/getSearch/', data=json.dumps(call_payload), headers=HEADERS, ).json()
if int(result['erreur']['code']) != 0:
raise BrowserUnavailable("API sent back an error code")
transaction_list = result['content']['operations']
return transaction_list
def on_load(self):
for td in self.doc.getroot().cssselect('td.LibelleErreur'):
if td.text is None:
continue
msg = td.text.strip()
if 'indisponible' in msg:
raise BrowserUnavailable(msg)
def do_login(self):
if not self.password.isdigit() or len(self.password) not in (6, 7):
raise BrowserIncorrectPassword()
if not self.username.isdigit() or len(self.username) < 8:
raise BrowserIncorrectPassword()
self.username = self.username[:8]
self.main_page.go()
try:
self.page.login(self.username, self.password)
except BrowserHTTPNotFound:
raise BrowserIncorrectPassword()
assert self.login.is_here()
reason, action = self.page.get_error()
if reason == 'echec_authent':
raise BrowserIncorrectPassword()
elif reason in (
'acces_bloq',
'acces_susp',
'pas_acces_bad',
):
raise ActionNeeded()
elif reason == 'err_tech':
# there is message "Service momentanément indisponible. Veuillez réessayer."
# in SG website in that case ...
raise BrowserUnavailable()
def iter_detailed_loans(self):
self.accountspage.go()
self.where = 'start'
for loan in self.page.get_detailed_loans():
data = {
'AJAXREQUEST': '_viewRoot',
'index': 'index',
'autoScroll': '',
'javax.faces.ViewState': loan._jid,
'accountNumber': loan._id,
'index:goToConsumerLoanUI': 'index:goToConsumerLoanUI'
}
self.accountspage.go(data=data)
self.loantokenpage.go(data=data)
try:
self.loandetailpage.go()
except ServerError as exception:
json_error = json.loads(exception.response.text)
if json_error['error']['code'] == "INTERNAL_ERROR":
raise BrowserUnavailable(json_error['error']['message'])
raise
else:
self.page.getdetails(loan)
yield loan
self.return_from_loan_site()
def logged(self):
for link in self.doc.xpath('//div[@id="header_bottom"]/ul[@class="top_menu"]//ul/li/a'):
if link.text == 'logout':
return True
if link.text == 'login':
return False
raise BrowserUnavailable('Unable to determine login state')
def do_login(self):
self.app_gone = False
self.preconnection.go()
try:
self.wait_until(VisibleXPath(
'//h2[text()[contains(.,"Log on to Internet Banking")]]'),
timeout=20)
self.page.login(self.username)
self.wait_until_is_here(self.login, 10)
error = self.page.get_error()
if error:
raise BrowserIncorrectPassword(error)
self.page.get_no_secure_key()
self.wait_until_is_here(self.login, 10)
error = self.page.get_error()
if error:
raise BrowserHTTPError(error)
self.page.login_w_secure(self.password, self.secret)
if self.login.is_here():
error = self.page.get_error()
if error:
raise BrowserIncorrectPassword(error)
WebDriverWait(self.driver,
20).until(EC.title_contains("My banking"))
except TimeoutException as e:
self.logger.exception("timeout while login")
raise BrowserUnavailable(e.msg)
def load_async(self, time):
total = 0
restart = True
while restart:
restart = False
# load content of loading divs.
lst = self.doc.xpath('//input[@type="hidden" and starts-with(@id, "asynch")]')
if len(lst) > 0:
params = {}
for i, input in enumerate(lst):
params['key%s' % i] = input.attrib['name']
params['div%s' % i] = input.attrib['value']
params['time'] = time
r = self.browser.open('/AsynchAjax', params=params)
data = json.loads(r.content)
for i, d in enumerate(data['data']):
div = self.doc.xpath('//div[@id="%s"]' % d['key'])[0]
html = d['flux']
div.clear()
div.attrib['id'] = d['key'] # needed because clear removes also all attributes
div.insert(0, etree.fromstring(html, parser=etree.HTMLParser()))
if 'time' in data:
wait = float(data['time'])/1000.0
self.logger.debug('should wait %f more seconds', wait)
total += wait
if total > 120:
raise BrowserUnavailable('too long time to wait')
sleep(wait)
restart = True
def do_login(self):
try:
self.location(self.BASEURL)
except (ClientError, HTTPNotFound) as e:
if e.response.status_code in (403, 404):
# Sometimes the website makes some redirections that leads
# to a 404 or a 403 when we try to access the BASEURL
# (website is not stable).
raise BrowserUnavailable(e.message)
raise
# avoids trying to relog in while it's already on home page
if self.home_page.is_here():
return
try:
self.page.login(self.username, self.password)
except BrowserUnavailable as ex:
# HACK: some accounts with legacy password fails (legacy means not only digits).
# The website crashes, even on a web browser.
# So, if we get a specific exception AND if we have a legacy password,
# we raise WrongPass instead of BrowserUnavailable.
if 'Cette page est indisponible' in ex.message and not self.password.isdigit(
):
raise BrowserIncorrectPassword()
raise
if self.login_page.is_here():
raise BrowserIncorrectPassword()
if 'internetRescuePortal' in self.url:
# 1 more request is necessary
data = {'integrationMode': 'INTERNET_RESCUE'}
self.location('/cyber/internet/Login.do', data=data)
def on_load(self):
if CleanText(
'//script[contains(text(), "momentanément indisponible")]')(
self.doc):
raise BrowserUnavailable(
u"Le service est momentanément indisponible")
return super(ErrorPage, self).on_load()
def on_load(self):
if self.doc['commun']['statut'] == 'NOK':
reason = self.doc['commun']['raison']
if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS':
raise NoAccountsException(
"Vous n'avez pas l'autorisation de consulter : {}".format(
reason))
raise BrowserUnavailable(reason)
