def do_login(self): assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) if not self.password.isdigit() or len(self.password) != 6: raise BrowserIncorrectPassword() if not self.username.isdigit() or len(self.username) < 8: raise BrowserIncorrectPassword() self.username = self.username[:8] self.login.stay_or_go() try: self.page.login(self.username, self.password) except BrowserHTTPNotFound: raise BrowserIncorrectPassword() if self.login.is_here(): raise BrowserIncorrectPassword() if self.bad_login.is_here(): error = self.page.get_error() if error is None: raise BrowserIncorrectPassword() elif error.startswith('Votre session a'): raise BrowserUnavailable('Session has expired') elif error.startswith('Le service est momentan'): raise BrowserUnavailable(error) else: raise BrowserIncorrectPassword(error)
def do_login(self): if not self.password.isdigit() or len(self.password) != 6: raise BrowserIncorrectPassword() if not self.username.isdigit() or len(self.username) < 8: raise BrowserIncorrectPassword() self.username = self.username[:8] self.login.stay_or_go() try: self.page.login(self.username, self.password) except BrowserHTTPNotFound: raise BrowserIncorrectPassword() if self.login.is_here(): raise BrowserIncorrectPassword() if self.bad_login.is_here(): error = self.page.get_error() if error is None: raise BrowserIncorrectPassword() elif error.startswith('Votre session a'): raise BrowserUnavailable('Session has expired') elif error.startswith('Le service est momentan'): raise BrowserUnavailable(error) elif 'niv_auth_insuff' in error: raise BrowserIncorrectPassword( "Niveau d'authentification insuffisant") else: raise BrowserIncorrectPassword(error)
def do_login(self): assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) if not self.login.is_here(): self.location('/signin/') response = self.open(self.page.get_script_url()) token, csrf, key, value, sessionID, cookie = self.page.get_token_and_csrf( response.text) self.session.cookies.update({'xppcts': cookie}) data = {} data['ads_token_js'] = token data['_csrf'] = csrf data['_sessionID'] = sessionID data[key] = value data = urllib.urlencode(data) self.open('/auth/verifychallenge', data=data) res = self.page.login(self.username, self.password) if 'LoginFailed' in res.content or 'Sorry, we can\'t log you in' in res.content or self.error.is_here( ): raise BrowserIncorrectPassword() if '/auth/validatecaptcha' in res.content: raise BrowserUnavailable('captcha') self.location('/') if self.old_website.is_here(): self.location('https://www.paypal.com/businessexp/summary') if self.login.is_here() or self.landing.is_here(): raise BrowserUnavailable('login failed') self.detect_account_type()
def on_load(self): h1 = CleanText('//h1[1]')(self.doc) if "est indisponible" in h1: raise BrowserUnavailable(h1) a = Link('//a[@class="btn"][1]', default=None)(self.doc) if not a: raise BrowserUnavailable() self.browser.location(a)
def _get_history_invests(self, account): if self.home.is_here(): self.page.go_list() else: self.home.go() self.page.go_history(account._info) if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): if self.page.is_account_inactive(account.id): self.logger.warning('Account %s %s is inactive.' % (account.label, account.id)) return [] # There is (currently ?) no history for MILLEVIE PREMIUM accounts if "MILLEVIE" in account.label: try: self.page.go_life_insurance(account) except ServerError as ex: if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label: self.logger.info( "Can not reach history page for MILLEVIE PREMIUM account" ) return [] raise label = account.label.split()[-1] try: self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) except BrowserHTTPError as e: if e.response.status_code == 500: error = json.loads(e.response.text) raise BrowserUnavailable(error["error"]) raise return sorted_transactions(self.page.get_history()) if account.label.startswith( 'NUANCES ') or account.label in self.insurance_accounts: self.page.go_life_insurance(account) if 'JSESSIONID' in self.session.cookies: # To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session del self.session.cookies['JSESSIONID'] try: if not self.life_insurance.is_here( ) and not self.message.is_here(): # life insurance website is not always available raise BrowserUnavailable() self.page.submit() self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo()) except (IndexError, AttributeError) as e: self.logger.error(e) return [] return self.page.iter_history()
def on_loaded(self): try: h1 = self.parser.select(self.document.getroot(), 'h1', 1) except BrokenPageError: pass if h1.text is not None and h1.text.startswith('Le service est moment'): try: raise BrowserUnavailable(self.document.xpath('//h4')[0].text) except KeyError: raise BrowserUnavailable(h1.text)
def __next__(self): if self.remaining <= 0: raise BrowserUnavailable( 'Site did not reply successfully after multiple tries') if self.delogged: self.browser.do_login() self.delogged = False if self.it is None: self.it = self.cb() # recreated iterator, consume previous items try: nb = -1 for nb, sent in enumerate(self.items): new = next(self.it) if hasattr(new, 'iter_fields'): equal = dict(sent.iter_fields()) == dict( new.iter_fields()) else: equal = sent == new if not equal: # safety is not guaranteed raise BrowserUnavailable( 'Site replied inconsistently between retries, %r vs %r', sent, new) except StopIteration: raise BrowserUnavailable( 'Site replied fewer elements (%d) than last iteration (%d)', nb + 1, len(self.items)) except self.exc_check as exc: self.delogged = True if self.logger: self.logger.info('%s raised, retrying', exc) self.it = None self.remaining -= 1 return next(self) # return one item try: obj = next(self.it) except self.exc_check as exc: self.delogged = True if self.logger: self.logger.info('%s raised, retrying', exc) self.it = None self.remaining -= 1 return next(self) else: self.items.append(obj) return obj
def handle_polling(self): assert self.polling_transaction, "polling_transaction is mandatory !" data = {'n10_id_transaction': self.polling_transaction} timeout = time.time() + self.polling_duration while time.time() < timeout: self.location('/sec/oob_pollingooba.json', data=data) status = self.page.doc['donnees']['transaction_status'] if status != "in_progress": break time.sleep(3) else: status = "aborted" self.check_polling_errors(status) data.update({ 'oob_op': "auth", }) self.location('/sec/oob_auth.json', data=data) if self.page.doc.get('commun', {}).get('statut').lower() == "nok": raise BrowserUnavailable() self.polling_transaction = None
def on_loaded(self): try: a = self.document.xpath('//a[@class="btn"]')[0] except IndexError: raise BrowserUnavailable() else: self.browser.location(a.attrib['href'], nologin=True)
def on_load(self): MyHTMLPage.on_load(self) if self.doc.xpath(u'//h2[text()="ERREUR"]'): # website sometime crash self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea') raise BrowserUnavailable()
def do_login(self): if self.config['pin_code'].get(): self.validate_security_form() if not self.page.is_logged(): raise BrowserIncorrectPassword( "Login / Password or authentication pin_code incorrect") return try: self.login.go() except ConnectionError as e: raise BrowserUnavailable(e) if self.page.is_logged(): return self.page.login(self.username, self.password) self.page.check_user_double_auth() if self.page.check_website_double_auth(): self.otp_form = self.page.get_security_form() self.otp_url = self.url raise BrowserQuestion( Value('pin_code', label=self.page.get_otp_message() or 'Please type the OTP you received')) if not self.page.is_logged(): raise BrowserIncorrectPassword(self.page.get_error_message())
def do_login(self): if not self.username or not self.password: raise BrowserIncorrectPassword() # First we try to connect to the new website: if the connection # is on the old website, we will automatically redirected. website = self.website.replace('.fr', '') region_domain = re.sub(r'^www\.', 'www.credit-agricole.fr/', website) self.BASEURL = 'https://%s/' % region_domain self.login_page.go() if self.old_website.is_here(): self.BASEURL = 'https://%s/' % self.website self.logger.warning('This is a regional connection, switching to old website with URL %s', self.BASEURL) raise SiteSwitch('region') self.do_security_check() # accounts_url may contain '/particulier', '/professionnel', '/entreprise', '/agriculteur' or '/association' self.accounts_url = self.page.get_accounts_url() assert self.accounts_url, 'Could not get accounts url from security check' try: self.location(self.accounts_url) except HTTPNotFound: # Sometimes the url the json sends us back is just unavailable... raise BrowserUnavailable() assert self.accounts_page.is_here(), 'We failed to login after the security check: response URL is %s' % self.url
def do_security_check(self): try: form = self.get_security_form() self.security_check.go(data=form) except ServerError as exc: # Wrongpass returns a 500 server error... exc_json = exc.response.json() error = exc_json.get('error') if error: message = error.get('message', '') wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") if any(value in message for value in wrongpass_messages): raise BrowserIncorrectPassword() if 'obtenir un nouveau code' in message: raise ActionNeeded(message) code = error.get('code', '') technical_error_messages = ('Un incident technique', 'identifiant et votre code personnel') # Sometimes there is no error message, so we try to use the code as well technical_error_codes = ('technical_error',) if any(value in message for value in technical_error_messages) or \ any(value in code for value in technical_error_codes): raise BrowserUnavailable(message) # When a PSD2 SCA is required it also returns a 500, hopefully we can detect it if ( exc_json.get('url') == 'dsp2/informations.html' or exc_json.get('redirection', '').endswith('dsp2/informations.html') ): return self.handle_sca() raise
def _iter_accounts(self): owner_name = self.get_profile().name.upper() self.loans.go(account_type=self.account_type, loans_page_label=self.loans_page_label) for a in self.page.get_list(): yield a self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) self.multitype_av.go() if self.multitype_av.is_here(): for a in self.page.get_av_accounts(): self.location(a._link, data=a._args) self.location(a._link.replace("_attente", "_detail_contrat_rep"), data=a._args) if self.page.get_error(): raise BrowserUnavailable(self.page.get_error()) self.page.fill_diff_currency(a) yield a self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) if self.accounts.is_here(): for a in self.page.get_list(name=owner_name): yield a else: for a in self.page.get_list(): yield a
def on_load(self): if Dict('commun/statut')(self.doc).upper() == 'NOK': reason = Dict('commun/raison')(self.doc) action = Dict('commun/action')(self.doc) if action and 'BLOCAGE' in action: raise ActionNeeded() if reason and 'err_tech' in reason: # This error is temporary and usually do not happens on the next try raise TemporaryBrowserUnavailable() if ('le service est momentanement indisponible' in reason and Dict('commun/origine')(self.doc) != 'cbo'): raise BrowserUnavailable() if reason == "niv_auth_insuff": return conditions = ( 'pas encore géré' in reason, # this page is not handled by SG api website 'le service est momentanement indisponible' in reason, # can't access new website ) assert any(conditions), 'Error %s is not handled yet' % reason self.logger.warning('Handled Error "%s"', reason)
def on_load(self): super(BasePage, self).on_load() if 'Erreur' in CleanText('//div[@id="main"]/h1', default='')(self.doc): err = CleanText('//div[@id="main"]/div[@class="content"]', default='Site indisponible')(self.doc) raise BrowserUnavailable(err)
def on_load(self): if self.doc.xpath( '//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]' ): # Warning: it could occurs because of wrongpass, user have to change password raise BrowserUnavailable( CleanText('//div[@class="ngo_mu_message"]')(self.doc))
def get_list(self): accounts = [] previous_account = None noaccounts = self.get_from_js('_js_noMvts =', ';') if noaccounts is not None: assert 'avez aucun compte' in noaccounts return [] txt = self.get_from_js('_data = new Array(', ');', is_list=True) if txt is None: raise BrowserUnavailable('Unable to find accounts list in scripts') data = json.loads('[%s]' % txt.replace("'", '"')) for line in data: a = Account() a.id = line[self.COL_ID].replace(' ', '') if re.match(r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)', a.id): a.id = str(CleanDecimal().filter(a.id)) a._acc_nb = a.id.split('_')[0] if len(a.id.split('_')) > 1 else None a.label = MyStrip(line[self.COL_LABEL], xpath='.//div[@class="libelleCompteTDB"]') # This account can be multiple life insurance accounts if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *': continue a.balance = Decimal(FrenchTransaction.clean_amount(line[self.COL_BALANCE])) a.currency = a.get_currency(line[self.COL_BALANCE]) a.type = self.get_account_type(a.label) # The parent account must be created right before if a.type == Account.TYPE_CARD: # duplicate if find_object(accounts, id=a.id): self.logger.warning('Ignoring duplicate card %r', a.id) continue a.parent = previous_account if line[self.COL_HISTORY] == 'true': a._inv = False a._link = self.get_history_link() a._args = self.make__args_dict(line) else: a._inv = True a._args = {'_ipc_eventValue': line[self.COL_ID], '_ipc_fireEvent': line[self.COL_FIRE_EVENT], } a._link = self.doc.xpath('//form[@name="changePageForm"]')[0].attrib['action'] if a.type is Account.TYPE_CARD: a.coming = a.balance a.balance = Decimal('0.0') accounts.append(a) previous_account = a return accounts
def get_token_and_csrf(self, code): # Paypal will try to create an infinite loop to make the parse fail, based on different # weird things like a check of 'ind\\u0435xOf' vs 'indexOf'. cleaner_code = code.replace(r"'ind\\u0435xOf'", "'indexOf'") # It also calls "data" which is undefined instead of a return (next call is an infinite # recursive function). This should theorically not happen if window.domain is correctly set # to "paypal.com" though. cleaner_code = cleaner_code.replace("data;", "return;") # Remove setCookie function content cleaner_code = re.sub(r"'setCookie'.*(?=,'removeCookie')", "'setCookie':function(){}", cleaner_code) # Paypal will try to send a XHR, let's use a fake method to catch the values sent cleaner_code = """ XMLHttpRequest.prototype.send = function(body) { window.PAYPAL_TOKENS = body; }; function GET_JS_TOKENS() { return window.PAYPAL_TOKENS || "INVALID_TOKENS"; } """ + cleaner_code try: raw = str( Javascript(cleaner_code, None, "paypal.com").call("GET_JS_TOKENS")) raw = raw.split("&") tokens = {} for r in raw: r = r.split("=") k = r[0] v = unquote(r[1]) if k not in ["ads_token_js", "_sessionID", "_csrf"]: tokens["key"] = k tokens["value"] = v else: tokens[k] = v token = tokens["ads_token_js"] sessionID = tokens["_sessionID"] csrf = tokens["_csrf"] key = tokens["key"] value = tokens["value"] except: raise BrowserUnavailable("Could not grab tokens") # Clean string obfuscation like: '\x70\x61\x79\x70\x61\x6c\x20\x73\x75\x63\x6b\x73' def basic_decoder(mtc): return repr(literal_eval(mtc.group(0)).encode('utf-8')) cleaner_code = re.sub(r"'.*?(?<!\\)'", basic_decoder, code) cookie = re.search(r'xppcts = (\w+);', cleaner_code).group(1) return token, csrf, key, value, sessionID, cookie
def to_statement(self, uri): for i in range(self.MAX_RETRIES): self.location(uri) if self.statement.is_here(): break else: raise BrowserUnavailable()
def _make_api_call(self, account, start_date, end_date, offset, max_length=50): HEADERS = { 'Accept': "application/json", 'Content-Type': 'application/json', } HEADERS.update(self.get_and_update_bred_token()) call_payload = { "account": account._number, "poste": account._nature, "sousPoste": account._codeSousPoste or '00', "devise": account.currency, "fromDate": start_date.strftime('%Y-%m-%d'), "toDate": end_date.strftime('%Y-%m-%d'), "from": offset, "size": max_length, # max length of transactions "search": "", "categorie": "", } result = self.open('/transactionnel/services/applications/operations/getSearch/', data=json.dumps(call_payload), headers=HEADERS, ).json() if int(result['erreur']['code']) != 0: raise BrowserUnavailable("API sent back an error code") transaction_list = result['content']['operations'] return transaction_list
def on_load(self): for td in self.doc.getroot().cssselect('td.LibelleErreur'): if td.text is None: continue msg = td.text.strip() if 'indisponible' in msg: raise BrowserUnavailable(msg)
def do_login(self): if not self.password.isdigit() or len(self.password) not in (6, 7): raise BrowserIncorrectPassword() if not self.username.isdigit() or len(self.username) < 8: raise BrowserIncorrectPassword() self.username = self.username[:8] self.main_page.go() try: self.page.login(self.username, self.password) except BrowserHTTPNotFound: raise BrowserIncorrectPassword() assert self.login.is_here() reason, action = self.page.get_error() if reason == 'echec_authent': raise BrowserIncorrectPassword() elif reason in ( 'acces_bloq', 'acces_susp', 'pas_acces_bad', ): raise ActionNeeded() elif reason == 'err_tech': # there is message "Service momentanément indisponible. Veuillez réessayer." # in SG website in that case ... raise BrowserUnavailable()
def iter_detailed_loans(self): self.accountspage.go() self.where = 'start' for loan in self.page.get_detailed_loans(): data = { 'AJAXREQUEST': '_viewRoot', 'index': 'index', 'autoScroll': '', 'javax.faces.ViewState': loan._jid, 'accountNumber': loan._id, 'index:goToConsumerLoanUI': 'index:goToConsumerLoanUI' } self.accountspage.go(data=data) self.loantokenpage.go(data=data) try: self.loandetailpage.go() except ServerError as exception: json_error = json.loads(exception.response.text) if json_error['error']['code'] == "INTERNAL_ERROR": raise BrowserUnavailable(json_error['error']['message']) raise else: self.page.getdetails(loan) yield loan self.return_from_loan_site()
def logged(self): for link in self.doc.xpath('//div[@id="header_bottom"]/ul[@class="top_menu"]//ul/li/a'): if link.text == 'logout': return True if link.text == 'login': return False raise BrowserUnavailable('Unable to determine login state')
def do_login(self): self.app_gone = False self.preconnection.go() try: self.wait_until(VisibleXPath( '//h2[text()[contains(.,"Log on to Internet Banking")]]'), timeout=20) self.page.login(self.username) self.wait_until_is_here(self.login, 10) error = self.page.get_error() if error: raise BrowserIncorrectPassword(error) self.page.get_no_secure_key() self.wait_until_is_here(self.login, 10) error = self.page.get_error() if error: raise BrowserHTTPError(error) self.page.login_w_secure(self.password, self.secret) if self.login.is_here(): error = self.page.get_error() if error: raise BrowserIncorrectPassword(error) WebDriverWait(self.driver, 20).until(EC.title_contains("My banking")) except TimeoutException as e: self.logger.exception("timeout while login") raise BrowserUnavailable(e.msg)
def load_async(self, time): total = 0 restart = True while restart: restart = False # load content of loading divs. lst = self.doc.xpath('//input[@type="hidden" and starts-with(@id, "asynch")]') if len(lst) > 0: params = {} for i, input in enumerate(lst): params['key%s' % i] = input.attrib['name'] params['div%s' % i] = input.attrib['value'] params['time'] = time r = self.browser.open('/AsynchAjax', params=params) data = json.loads(r.content) for i, d in enumerate(data['data']): div = self.doc.xpath('//div[@id="%s"]' % d['key'])[0] html = d['flux'] div.clear() div.attrib['id'] = d['key'] # needed because clear removes also all attributes div.insert(0, etree.fromstring(html, parser=etree.HTMLParser())) if 'time' in data: wait = float(data['time'])/1000.0 self.logger.debug('should wait %f more seconds', wait) total += wait if total > 120: raise BrowserUnavailable('too long time to wait') sleep(wait) restart = True
def do_login(self): try: self.location(self.BASEURL) except (ClientError, HTTPNotFound) as e: if e.response.status_code in (403, 404): # Sometimes the website makes some redirections that leads # to a 404 or a 403 when we try to access the BASEURL # (website is not stable). raise BrowserUnavailable(e.message) raise # avoids trying to relog in while it's already on home page if self.home_page.is_here(): return try: self.page.login(self.username, self.password) except BrowserUnavailable as ex: # HACK: some accounts with legacy password fails (legacy means not only digits). # The website crashes, even on a web browser. # So, if we get a specific exception AND if we have a legacy password, # we raise WrongPass instead of BrowserUnavailable. if 'Cette page est indisponible' in ex.message and not self.password.isdigit( ): raise BrowserIncorrectPassword() raise if self.login_page.is_here(): raise BrowserIncorrectPassword() if 'internetRescuePortal' in self.url: # 1 more request is necessary data = {'integrationMode': 'INTERNET_RESCUE'} self.location('/cyber/internet/Login.do', data=data)
def on_load(self): if CleanText( '//script[contains(text(), "momentanément indisponible")]')( self.doc): raise BrowserUnavailable( u"Le service est momentanément indisponible") return super(ErrorPage, self).on_load()
def on_load(self): if self.doc['commun']['statut'] == 'NOK': reason = self.doc['commun']['raison'] if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS': raise NoAccountsException( "Vous n'avez pas l'autorisation de consulter : {}".format( reason)) raise BrowserUnavailable(reason)