class BtmonBrowser(PagesBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'http://www.btmon.com/' home = URL('$', HomePage) search = URL(r'/torrent/\?sort=relevance&f=(?P<pattern>.*)', SearchPage) torrent = URL(r'/(?P<torrent_id>.*)\.torrent\.html', TorrentPage) def get_bpc_cookie(self): if 'BPC' not in self.session.cookies: self.home.go() bpcCookie = str(self.page.content).split('BPC=')[-1].split('"')[0] self.session.cookies['BPC'] = bpcCookie def iter_torrents(self, pattern): self.get_bpc_cookie() return self.search.go(pattern=pattern).iter_torrents() def get_torrent(self, id): try: self.get_bpc_cookie() self.torrent.go(torrent_id=id) torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return
class BECMBrowser(AbstractBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.becm.fr' PARENT = 'creditmutuel' login = URL('/fr/authentification.html', LoginPage) advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage) @need_login def get_advisor(self): advisor = None if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) if self.page.get_advisor_link(): advisor = self.page.get_advisor() self.location( self.page.get_advisor_link()).page.update_advisor(advisor) else: advisor = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_advisor() link = self.page.get_agency() if link: self.location(link) self.page.update_advisor(advisor) return iter([advisor]) if advisor else iter([])
class T411Browser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.t411.ai/' home = URL('$', HomePage) search = URL( 'torrents/search/\?search=(?P<pattern>.*)&order=seeders&type=desc', SearchPage) # Order matters here: 'torrents/[^&]*' would match '/torrents/download/\?id...' and # TorrentPage would crash on the bencode data, so DownloadPage must be listed before # TorrentPage download = URL('/torrents/download/\?id=(?P<id>.*)', DownloadPage) torrent = URL('/torrents/details/\?id=(?P<id>.*)&r=1', 'torrents/[^&]*', TorrentPage) def do_login(self): self.home.go() if not self.page.logged: self.page.login(self.username, self.password) if not self.page.logged: raise BrowserIncorrectPassword() @need_login def iter_torrents(self, pattern): return self.search.go(pattern=pattern).iter_torrents() @need_login def get_torrent(self, fullid, torrent=None): try: self.torrent.go(id=fullid) torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return
class T411Browser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.t411.in/' home = URL('$', HomePage) search = URL( 'torrents/search/\?search=(?P<pattern>.*)&order=seeders&type=desc', SearchPage) torrent = URL('/torrents/details/\?id=(?P<id>.*)&r=1', 'torrents/[^&]*', TorrentPage) #def __init__(self, *args, **kwargs): # Browser.__init__(self, *args, **kwargs) def do_login(self): self.home.go() if not self.page.logged: self.page.login(self.username, self.password) if not self.page.logged: raise BrowserIncorrectPassword() @need_login def iter_torrents(self, pattern): return self.search.go(pattern=pattern).iter_torrents() @need_login def get_torrent(self, fullid, torrent=None): try: self.torrent.go(id=fullid) torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return
class CICBrowser(AbstractBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.cic.fr' PARENT = 'creditmutuel' login = URL('/sb/fr/banques/particuliers/index.html', '/(?P<subbank>.*)/fr/$', '/(?P<subbank>.*)/fr/banques/accueil.html', '/(?P<subbank>.*)/fr/banques/particuliers/index.html', LoginPage)
class BECMBrowser(AbstractBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.becm.fr' PARENT = 'creditmutuel' login = URL('/fr/authentification.html', LoginPage) advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage) def do_login(self): # Clear cookies. self.do_logout() self.login.go() if not self.page.logged: self.page.login(self.username, self.password) # Many "Credit Mutuel" customers tried to add their connection to BECM, but the BECM # website does not return any error when you try to login with correct Crédit Mutuel # credentials, therefore we must suggest them to try regular Crédit Mutuel if login fails. if self.login.is_here(): raise ActionNeeded( "La connexion au site de BECM n'a pas fonctionné avec les identifiants fournis.\ Si vous êtes client du Crédit Mutuel, veuillez réessayer en sélectionnant le module Crédit Mutuel." ) if self.verify_pass.is_here(): raise AuthMethodNotImplemented( "L'identification renforcée avec la carte n'est pas supportée." ) @need_login def get_advisor(self): advisor = None if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) if self.page.get_advisor_link(): advisor = self.page.get_advisor() self.location( self.page.get_advisor_link()).page.update_advisor(advisor) else: advisor = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_advisor() link = self.page.get_agency() if link: self.location(link) self.page.update_advisor(advisor) return iter([advisor]) if advisor else iter([])
class LimetorrentsBrowser(PagesBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.limetorrents.info/' search = URL(r'/search/all/(?P<pattern>.*)/seeds/(?P<page>[0-9]+)/', SearchPage) torrent = URL(r'/(?P<torrent_name>.*)-torrent-(?P<torrent_id>[0-9]+)\.html', TorrentPage) def iter_torrents(self, pattern): return self.search.go(pattern=pattern, page=1).iter_torrents() def get_torrent(self, id): try: self.torrent.go(torrent_id=id, torrent_name='whatever') torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return
class CICBrowser(AbstractBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.cic.fr' PARENT = 'creditmutuel' login = URL(r'/fr/authentification.html', r'/sb/fr/banques/particuliers/index.html', r'/(?P<subbank>.*)/fr/$', r'/(?P<subbank>.*)/fr/banques/accueil.html', r'/(?P<subbank>.*)/fr/banques/particuliers/index.html', LoginPage) por = URL(r'/(?P<subbank>.*)fr/banque/PORT_Synthese.aspx', PorPage) decoupled_state = URL(r'/fr/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) cancel_decoupled = URL(r'/fr/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled)
class T411Browser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.t411.si/' home = URL('$', HomePage) login = URL('/login$', LoginPage) search = URL(r'/torrents/search/\?search=(?P<pattern>.*)', SearchPage) download = URL( '/telecharger-torrent/(?P<torrent_hash>[0-9a-f]{40})/(?P<torrent_name>\w+)', DownloadPage) torrent = URL('/torrents/(?P<torrent_id>[0-9]+)/(?P<torrent_name>.*)', TorrentPage) def do_login(self): self.home.go() if not self.page.logged: self.page.login(self.username, self.password) self.home.go() if not self.page.logged: raise BrowserIncorrectPassword() @need_login def iter_torrents(self, pattern): return self.search.go(pattern=pattern).iter_torrents() @need_login def get_torrent(self, torrent): try: self.torrent.go(torrent_id=torrent.id, torrent_name=torrent.name) torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return def get_torrent_file(self, torrent): torrent = self.browser.get_torrent(torrent) if not torrent: return None resp = self.browser.open(torrent.url) return resp.content
class YggtorrentBrowser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://yggtorrent.to/' home = URL('$', HomePage) login = URL('/user/login$', LoginPage) search = URL(r'/engine/search\?name=(?P<pattern>.*)&order=desc&sort=seed&do=search', SearchPage) download = URL('/engine/download_torrent\?id=(?P<torrent_id>[0-9]+)', DownloadPage) torrent = URL('/torrent/(?P<torrent_cat>.+)/(?P<torrent_subcat>.+)/(?P<torrent_id>[0-9]+)-(?P<torrent_name>.*)', TorrentPage) def do_login(self): self.home.go() if not self.page.logged: self.page.login(self.username, self.password) self.home.go() if not self.page.logged: raise BrowserIncorrectPassword() @need_login def iter_torrents(self, pattern): return self.search.go(pattern=pattern).iter_torrents() @need_login def get_torrent(self, id): try: self.torrent.go(torrent_id=id, torrent_name='anything', torrent_cat='any', torrent_subcat='thing') torrent = self.page.get_torrent() return torrent except BrowserHTTPNotFound: return @need_login def get_torrent_file(self, id): torrent = self.browser.get_torrent(id) if not torrent: return None resp = self.browser.open(torrent.url) return resp.content
class CICBrowser(LoginBrowser): PROFILE = Wget() BASEURL = 'https://www.cic.fr' login = URL('/sb/fr/banques/particuliers/index.html', '/(?P<subbank>.*)/fr/$', '/(?P<subbank>.*)/fr/banques/accueil.html', '/(?P<subbank>.*)/fr/banques/particuliers/index.html', LoginPage) login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi', LoginErrorPage) accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi', '/(?P<subbank>.*)/fr/banque/situation_financiere.html', AccountsPage) user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx', UserSpacePage) operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*', '/(?P<subbank>.*)/fr/banque/mouvements.html.*', '/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*', OperationsPage) coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage) card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage) noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage) info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage) transfert = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_vi.html', EmptyPage) transfert_2 = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_cmweb.aspx.*', TransfertPage) change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi', ChangePasswordPage) verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*', VerifCodePage) empty = URL( '/(?P<subbank>.*)/fr/banques/index.html', '/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*', '/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*', '/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx', '/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage) currentSubBank = None __states__ = ['currentSubBank'] def do_login(self): self.login.go() if not self.page.logged: self.page.login(self.username, self.password) if not self.page.logged or self.login_error.is_here(): raise BrowserIncorrectPassword() self.getCurrentSubBank() @need_login def get_accounts_list(self): return self.accounts.stay_or_go( subbank=self.currentSubBank).iter_accounts() def get_account(self, id): assert isinstance(id, basestring) for a in self.get_accounts_list(): if a.id == id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user url = urlparse(self.url) self.currentSubBank = url.path.lstrip('/').split('/')[0] def list_operations(self, page_url): if page_url.startswith('/') or page_url.startswith('https'): self.location(page_url) else: self.location('%s/%s/fr/banque/%s' % (self.BASEURL, self.currentSubBank, page_url)) if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_history(self, account): transactions = [] last_debit = None for tr in self.list_operations(account._link_id): # to prevent redundancy with card transactions, we do not # store 'RELEVE CARTE' transaction. if tr.raw != 'RELEVE CARTE': transactions.append(tr) elif last_debit is None: last_debit = (tr.date - timedelta(days=10)).month coming_link = self.page.get_coming_link() if self.operations.is_here( ) else None if coming_link is not None: for tr in self.list_operations(coming_link): transactions.append(tr) month = 0 for card_link in account._card_links: v = urlsplit(card_link) args = dict(parse_qsl(v.query)) # useful with 12 -> 1 if int(args['mois']) < month: month = month + 1 else: month = int(args['mois']) for tr in self.list_operations(card_link): if month > last_debit: tr._is_coming = True transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions def transfer(self, account, to, amount, reason=None): # access the transfer page self.transfert.go(subbank=self.currentSubBank) # fill the form form = self.page.get_form(xpath="//form[@id='P:F']") try: form[ 'data_input_indiceCompteADebiter'] = self.page.get_from_account_index( account) form[ 'data_input_indiceCompteACrediter'] = self.page.get_to_account_index( to) except ValueError as e: raise TransferError(e.message) form['[t:dbt%3adouble;]data_input_montant_value_0_'] = '%s' % str( amount).replace('.', ',') if reason is not None: form[ '[t:dbt%3astring;x(27)]data_input_libelleCompteDebite'] = reason form[ '[t:dbt%3astring;x(31)]data_input_motifCompteCredite'] = reason del form['_FID_GoCancel'] del form['_FID_DoValidate'] form['_FID_DoValidate.x'] = str(randint(3, 125)) form['_FID_DoValidate.y'] = str(randint(3, 22)) form.submit() # look for known errors content = self.page.get_unicode_content() insufficient_amount_message = u'Le montant du virement doit être positif, veuillez le modifier' maximum_allowed_balance_message = u'Montant maximum autorisé au débit pour ce compte' if insufficient_amount_message in content: raise TransferError('The amount you tried to transfer is too low.') if maximum_allowed_balance_message in content: raise TransferError( 'The maximum allowed balance for the target account has been / would be reached.' ) # look for the known "all right" message ready_for_transfer_message = u'Confirmer un virement entre vos comptes' if ready_for_transfer_message not in content: raise TransferError('The expected message "%s" was not found.' % ready_for_transfer_message) # submit the confirmation form form = self.page.get_form(xpath="//form[@id='P:F']") del form['_FID_DoConfirm'] form['_FID_DoConfirm.x'] = str(randint(3, 125)) form['_FID_DoConfirm.y'] = str(randint(3, 22)) submit_date = datetime.now() form.submit() # look for the known "everything went well" message content = self.page.get_unicode_content() transfer_ok_message = u'Votre virement a été exécuté' if transfer_ok_message not in content: raise TransferError('The expected message "%s" was not found.' % transfer_ok_message) # We now have to return a Transfer object transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S')) transfer.amount = amount transfer.origin = account transfer.recipient = to transfer.date = submit_date return transfer
class CreditMutuelBrowser(LoginBrowser, StatesMixin): PROFILE = Wget() STATE_DURATION = 15 TIMEOUT = 30 BASEURL = 'https://www.creditmutuel.fr' login = URL('/fr/authentification.html', '/(?P<subbank>.*)fr/$', '/(?P<subbank>.*)fr/banques/accueil.html', '/(?P<subbank>.*)fr/banques/particuliers/index.html', LoginPage) login_error = URL('/(?P<subbank>.*)fr/identification/default.cgi', LoginErrorPage) accounts = URL('/(?P<subbank>.*)fr/banque/situation_financiere.cgi', '/(?P<subbank>.*)fr/banque/situation_financiere.html', AccountsPage) revolving_loan_list = URL( r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL( r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) user_space = URL( '/(?P<subbank>.*)fr/banque/espace_personnel.aspx', '/(?P<subbank>.*)fr/banque/accueil.cgi', '/(?P<subbank>.*)fr/banque/DELG_Gestion', '/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', UserSpacePage) card = URL( '/(?P<subbank>.*)fr/banque/operations_carte.cgi.*', '/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', '/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', CardPage) operations = URL( '/(?P<subbank>.*)fr/banque/mouvements.cgi.*', '/(?P<subbank>.*)fr/banque/mouvements.html.*', '/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*', r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+', OperationsPage) coming = URL('/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage) info = URL('/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage) change_pass = URL('/(?P<subbank>.*)fr/validation/change_password.cgi', '/fr/services/change_password.html', ChangePasswordPage) verify_pass = URL('/(?P<subbank>.*)fr/validation/verif_code.cgi.*', VerifCodePage) new_home = URL('/(?P<subbank>.*)fr/banque/pageaccueil.html', '/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage) empty = URL( '/(?P<subbank>.*)fr/banques/index.html', '/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*', '/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case).*', EmptyPage) por = URL('/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx', '/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx', PorPage) li = URL('/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne', '/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*', '/(?P<subbank>.*)fr/assurances/WI_ASS', '/fr/assurances/', LIAccountsPage) iban = URL('/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage) new_accounts = URL('/(?P<subbank>.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) new_operations = URL('/(?P<subbank>.*)fr/banque/mouvements.cgi', '/fr/banque/nr/nr_devbooster.aspx.*', '/(?P<subbank>.*)fr/banque/RE/aiguille.asp', '/fr/banque/mouvements.html', '/(?P<subbank>.*)fr/banque/consultation/operations', OperationsPage) advisor = URL( '/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)', '/(?P<subbank>.*)fr/infoclient/', r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*', AdvisorPage) redirect = URL( '/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) cards_activity = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) cards_list = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*', '/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) cards_ope = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) internal_transfer = URL('/(?P<subbank>.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) external_transfer = URL( '/(?P<subbank>.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) recipients_list = URL('/(?P<subbank>.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) currentSubBank = None is_new_website = False form = None logged = None __states__ = ['currentSubBank', 'form', 'logged'] accounts_list = None def do_login(self): # Clear cookies. self.do_logout() self.login.go() if not self.page.logged: self.page.login(self.username, self.password) if not self.page.logged or self.login_error.is_here(): raise BrowserIncorrectPassword() self.getCurrentSubBank() @need_login def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() self.accounts_list = [] # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: page = self.open(company).page if isinstance( company, basestring) else company self.accounts_list.extend([card for card in page.iter_cards()]) if not self.is_new_website: for a in self.accounts.stay_or_go( subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(a) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) else: for a in self.new_accounts.stay_or_go( subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(a) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) for acc in self.li.go( subbank=self.currentSubBank).iter_li_accounts(): self.accounts_list.append(acc) for acc in self.revolving_loan_list.stay_or_go( subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) return self.accounts_list def get_account(self, id): assert isinstance(id, basestring) for a in self.get_accounts_list(): if a.id == id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user paths = urlparse(self.url).path.lstrip('/').split('/') self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else "" if paths[0] in ["fr", "mabanque"]: self.is_new_website = True def list_operations(self, page): if isinstance(page, basestring): if page.startswith('/') or page.startswith( 'https') or page.startswith('?'): self.location(page) else: self.location('%s/%sfr/banque/%s' % (self.BASEURL, self.currentSubBank, page)) else: self.page = page # getting about 6 months history on new website if self.is_new_website and self.page: try: for x in range(0, 2): form = self.page.get_form( id="I1:fm", submit='//input[@name="_FID_DoActivateSearch"]') if x == 1: form.update({ [k for k in form.keys() if "DateStart" in k][0]: (datetime.now() - relativedelta(months=7)).strftime('%d/%m/%Y'), [k for k in form.keys() if "DateEnd" in k][0]: datetime.now().strftime('%d/%m/%Y') }) [ form.pop(k, None) for k in form.keys() if "_FID_Do" in k and "DoSearch" not in k ] form.submit() except (IndexError, FormNotFound): pass while self.page: try: form = self.page.get_form( '//*[@id="I1:fm"]', submit='//input[@name="_FID_DoLoadMoreTransactions"]') [ form.pop(k, None) for k in form.keys() if "_FID_Do" in k and "LoadMore" not in k ] form.submit() except (IndexError, FormNotFound): break #sometime the browser can't go further except ClientError as exc: if exc.response.status_code == 413: break raise if self.li.is_here(): return self.page.iter_history() if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_monthly_transactions(self, trs): groups = [ list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date) ] trs = [] for group in groups: tr = FrenchTransaction() tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date tr.amount = -sum([t.amount for t in group]) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs @need_login def get_history(self, account): transactions = [] if not account._link_id: raise NotImplementedError() # need to refresh the months select if account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): for tr in self.list_operations(account._link_id): transactions.append(tr) coming_link = self.page.get_coming_link() if self.operations.is_here( ) else None if coming_link is not None: for tr in self.list_operations(coming_link): transactions.append(tr) differed_date = None cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \ account._card_links if hasattr(account, '_card_links') else [] for card in cards: card_trs = [] for tr in self.list_operations(card): if hasattr(tr, '_differed_date') and ( not differed_date or tr._differed_date < differed_date): differed_date = tr._differed_date if tr.date >= datetime.now(): tr._is_coming = True elif hasattr(account, '_card_pages'): card_trs.append(tr) transactions.append(tr) if card_trs: transactions.extend(self.get_monthly_transactions(card_trs)) if differed_date is not None: # set deleted for card_summary for tr in transactions: tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \ differed_date.month <= tr.date.month and \ not hasattr(tr, '_is_manualsum') transactions = sorted_transactions(transactions) return transactions @need_login def get_investment(self, account): if account._is_inv: if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.por.go(subbank=self.currentSubBank) self.page.send_form(account) elif account.type == Account.TYPE_LIFE_INSURANCE: if not account._link_inv: return iter([]) self.location(account._link_inv) return self.page.iter_investment() return iter([]) @need_login def iter_recipients(self, origin_account): # access the transfer page self.internal_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): for recipient in self.page.iter_recipients( origin_account=origin_account): yield recipient self.external_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): origin_account._external_recipients = set() if self.page.has_transfer_categories(): for category in self.page.iter_categories(): self.page.go_on_category(category['index']) self.page.IS_PRO_PAGE = True for recipient in self.page.iter_recipients( origin_account=origin_account, category=category['name']): yield recipient else: for recipient in self.page.iter_recipients( origin_account=origin_account): yield recipient @need_login def init_transfer(self, account, to, amount, reason=None): if to.category != u'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: self.internal_transfer.go(subbank=self.currentSubBank) if self.external_transfer.is_here( ) and self.page.has_transfer_categories(): for category in self.page.iter_categories(): if category['name'] == to.category: self.page.go_on_category(category['index']) break self.page.IS_PRO_PAGE = True self.page.RECIPIENT_STRING = 'data_input_indiceBen' self.page.prepare_transfer(account, to, amount, reason) return self.page.handle_response(account, to, amount, reason) @need_login def execute_transfer(self, transfer, **params): form = self.page.get_form( id='P:F', submit='//input[@type="submit" and contains(@value, "Confirmer")]') # For the moment, don't ask the user if he confirms the duplicate. form['Bool:data_input_confirmationDoublon'] = 'true' form.submit() return self.page.create_transfer(transfer) @need_login def get_advisor(self): advisor = None if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) if self.page.get_advisor_link(): advisor = self.page.get_advisor() self.location( self.page.get_advisor_link()).page.update_advisor(advisor) else: advisor = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_advisor() link = self.page.get_agency() if link: link = link.replace(':443/', '/') self.location(link) self.page.update_advisor(advisor) return iter([advisor]) if advisor else iter([]) @need_login def get_profile(self): if not self.is_new_website: profile = self.accounts.stay_or_go( subbank=self.currentSubBank).get_profile() else: profile = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_profile() return profile def get_recipient_object(self, recipient): r = Recipient() r.iban = recipient.iban r.id = recipient.iban r.label = recipient.label r.category = recipient.category # On credit mutuel recipients are immediatly available. r.enabled_at = datetime.now().replace(microsecond=0) r.currency = u'EUR' r.bank_name = NotAvailable return r def continue_new_recipient(self, recipient, **params): if u'Clé' in params: self.page.post_code(params[u'Clé']) self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) self.page.ask_sms(self.get_recipient_object(recipient)) def send_sms(self, sms): data = {} for k, v in self.form.iteritems(): if k != 'url': data[k] = v data['otp_password'] = sms data['_FID_DoConfirm.x'] = '1' data['_FID_DoConfirm.y'] = '1' data['global_backup_hidden_key'] = '' self.location(self.form['url'], data=data) def end_new_recipient(self, recipient, **params): self.send_sms(params['code']) self.form = None self.page = None self.logged = 0 return self.get_recipient_object(recipient) def post_with_bic(self, recipient, **params): data = {} for k, v in self.form.iteritems(): if k != 'url': data[k] = v data['[t:dbt%3astring;x(11)]data_input_BIC'] = params[u'Bic'] self.location(self.form['url'], data=data) self.page.ask_sms(self.get_recipient_object(recipient)) @need_login def new_recipient(self, recipient, **params): if self.currentSubBank is None: self.getCurrentSubBank() if 'Bic' in params: return self.post_with_bic(recipient, **params) if 'code' in params: return self.end_new_recipient(recipient, **params) if u'Clé' in params: return self.continue_new_recipient(recipient, **params) self.recipients_list.go(subbank=self.currentSubBank) if self.page.has_list(): if recipient.category not in self.page.get_recipients_list(): raise AddRecipientError( 'Recipient category is not on the website available list.') self.page.go_list(recipient.category) self.page.go_to_add() if self.verify_pass.is_here(): raise AddRecipientStep( self.get_recipient_object(recipient), Value(u'Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params)
class ImdbBrowser(PagesBrowser): BASEURL = 'http://www.imdb.com' PROFILE = Wget() movie_crew = URL(r'/title/tt[0-9]*/fullcredits.*', MovieCrewPage) release = URL(r'/title/tt[0-9]*/releaseinfo.*', ReleasePage) bio = URL(r'/name/nm[0-9]*/bio.*', BiographyPage) person = URL(r'/name/nm[0-9]*/*', PersonPage) def iter_movies(self, pattern): res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&tt=on', params={'q': pattern}) jres = res.json() htmlparser = HTMLParser() for cat in ['title_popular', 'title_exact', 'title_approx']: if cat in jres: for m in jres[cat]: tdesc = unicode(m['title_description']) if '<a' in tdesc and '>' in tdesc: short_description = u'%s %s' % ( tdesc.split('<')[0].strip(', '), tdesc.split('>')[1].split('<')[0]) else: short_description = tdesc.strip(', ') movie = Movie(m['id'], htmlparser.unescape(m['title'])) movie.other_titles = NotLoaded movie.release_date = NotLoaded movie.duration = NotLoaded movie.short_description = htmlparser.unescape( short_description) movie.pitch = NotLoaded movie.country = NotLoaded movie.note = NotLoaded movie.roles = NotLoaded movie.all_release_dates = NotLoaded movie.thumbnail_url = NotLoaded yield movie def iter_persons(self, pattern): res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&nm=on', params={'q': pattern}) jres = res.json() htmlparser = HTMLParser() for cat in ['name_popular', 'name_exact', 'name_approx']: if cat in jres: for p in jres[cat]: person = Person(p['id'], htmlparser.unescape(unicode(p['name']))) person.real_name = NotLoaded person.birth_place = NotLoaded person.birth_date = NotLoaded person.death_date = NotLoaded person.gender = NotLoaded person.nationality = NotLoaded person.short_biography = NotLoaded person.short_description = htmlparser.unescape( p['description']) person.roles = NotLoaded person.thumbnail_url = NotLoaded yield person def get_movie(self, id): res = self.open( 'http://www.omdbapi.com/?apikey=b7c56eb5&i=%s&plot=full' % id) if res is not None: jres = res.json() else: return None htmlparser = HTMLParser() title = NotAvailable duration = NotAvailable release_date = NotAvailable pitch = NotAvailable country = NotAvailable note = NotAvailable short_description = NotAvailable thumbnail_url = NotAvailable other_titles = [] genres = [] roles = {} if 'Title' not in jres: return title = htmlparser.unescape(unicode(jres['Title'].strip())) if 'Poster' in jres: thumbnail_url = unicode(jres['Poster']) if 'Director' in jres: short_description = unicode(jres['Director']) if 'Genre' in jres: for g in jres['Genre'].split(', '): genres.append(g) if 'Runtime' in jres: m = re.search('(\d+?) min', jres['Runtime']) if m: duration = int(m.group(1)) if 'Released' in jres: released_string = str(jres['Released']) if released_string == 'N/A': release_date = NotAvailable else: months = { 'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12', } for st in months: released_string = released_string.replace(st, months[st]) release_date = datetime.strptime(released_string, '%d %m %Y') if 'Country' in jres: country = u'' for c in jres['Country'].split(', '): country += '%s, ' % c country = country[:-2] if 'Plot' in jres: pitch = unicode(jres['Plot']) if 'imdbRating' in jres and 'imdbVotes' in jres: note = u'%s/10 (%s votes)' % (jres['imdbRating'], jres['imdbVotes']) for r in ['Actors', 'Director', 'Writer']: if '%s' % r in jres.keys(): roles['%s' % r] = [('N/A', e) for e in jres['%s' % r].split(', ')] movie = Movie(id, title) movie.other_titles = other_titles movie.release_date = release_date movie.duration = duration movie.genres = genres movie.pitch = pitch movie.country = country movie.note = note movie.roles = roles movie.short_description = short_description movie.all_release_dates = NotLoaded movie.thumbnail_url = thumbnail_url return movie def get_person(self, id): try: self.location('http://www.imdb.com/name/%s' % id) except BrowserHTTPNotFound: return assert self.person.is_here() return self.page.get_person(id) def get_person_biography(self, id): self.location('http://www.imdb.com/name/%s/bio' % id) assert self.bio.is_here() return self.page.get_biography() def iter_movie_persons(self, movie_id, role): self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id) assert self.movie_crew.is_here() for p in self.page.iter_persons(role): yield p def iter_person_movies(self, person_id, role): self.location('http://www.imdb.com/name/%s' % person_id) assert self.person.is_here() return self.page.iter_movies(role) def iter_person_movies_ids(self, person_id): self.location('http://www.imdb.com/name/%s' % person_id) assert self.person.is_here() for movie in self.page.iter_movies_ids(): yield movie def iter_movie_persons_ids(self, movie_id): self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id) assert self.movie_crew.is_here() for person in self.page.iter_persons_ids(): yield person def get_movie_releases(self, id, country): self.location('http://www.imdb.com/title/%s/releaseinfo' % id) assert self.release.is_here() return self.page.get_movie_releases(country)
class CreditMutuelBrowser(TwoFactorBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.creditmutuel.fr' HAS_CREDENTIALS_ONLY = True STATE_DURATION = 5 TWOFA_DURATION = 60 * 24 * 90 # connexion login = URL( r'/fr/authentification.html', r'/(?P<subbank>.*)fr/$', r'/(?P<subbank>.*)fr/banques/accueil.html', r'/(?P<subbank>.*)fr/banques/particuliers/index.html', LoginPage ) login_error = URL(r'/(?P<subbank>.*)fr/identification/default.cgi', LoginErrorPage) twofa_unabled_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', TwoFAUnabledPage) mobile_confirmation = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', MobileConfirmationPage) decoupled_state = URL(r'/fr/banque/async/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) cancel_decoupled = URL(r'/fr/banque/async/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) otp_validation_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', OtpValidationPage) otp_blocked_error_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', OtpBlockedErrorPage) fiscality = URL(r'/(?P<subbank>.*)fr/banque/residencefiscale.aspx', FiscalityConfirmationPage) # accounts accounts = URL(r'/(?P<subbank>.*)fr/banque/situation_financiere.cgi', r'/(?P<subbank>.*)fr/banque/situation_financiere.html', AccountsPage) useless_page = URL(r'/(?P<subbank>.*)fr/banque/paci/defi-solidaire.html', UselessPage) revolving_loan_list = URL(r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL(r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) user_space = URL(r'/(?P<subbank>.*)fr/banque/espace_personnel.aspx', r'/(?P<subbank>.*)fr/banque/accueil.cgi', r'/(?P<subbank>.*)fr/banque/DELG_Gestion', r'/(?P<subbank>.*)fr/banque/paci_engine/engine.aspx', r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', UserSpacePage) card = URL(r'/(?P<subbank>.*)fr/banque/operations_carte.cgi.*', r'/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', r'/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', CardPage) operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi.*', r'/(?P<subbank>.*)fr/banque/mouvements.html.*', r'/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*', r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+', OperationsPage) # This loans_operations contains operation for some loans, but not all of them. loans_operations = URL(r'/(?P<subbank>.*)fr/banque/gec9.aspx.*', LoansOperationsPage) coming = URL(r'/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage) info = URL(r'/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage) change_pass = URL(r'/(?P<subbank>.*)fr/validation/change_password.cgi', '/fr/services/change_password.html', ChangePasswordPage) verify_pass = URL(r'/(?P<subbank>.*)fr/validation/verif_code.cgi.*', r'/(?P<subbank>.*)fr/validation/lst_codes.cgi.*', VerifCodePage) new_home = URL(r'/(?P<subbank>.*)fr/banque/pageaccueil.html', r'/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage) empty = URL(r'/(?P<subbank>.*)fr/banques/index.html', r'/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*', r'/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', EmptyPage) por = URL(r'/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx', r'/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx', PorPage) por_action_needed = URL(r'/(?P<subbank>.*)fr/banque/ORDR_InfosGenerales.aspx', EmptyPage) li = URL(r'/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne', r'/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*', r'/(?P<subbank>.*)fr/assurances/WI_ASS', '/fr/assurances/', LIAccountsPage) iban = URL(r'/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage) new_accounts = URL(r'/(?P<subbank>.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) new_operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi', r'/fr/banque/nr/nr_devbooster.aspx.*', r'/(?P<subbank>.*)fr/banque/RE/aiguille(liste)?.asp', '/fr/banque/mouvements.html', r'/(?P<subbank>.*)fr/banque/consultation/operations', OperationsPage) advisor = URL(r'/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)', r'/(?P<subbank>.*)fr/infoclient/', r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*', AdvisorPage) redirect = URL(r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) cards_activity = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) cards_list = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*', r'/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) cards_ope = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) cards_ope2 = URL('/(?P<subbank>.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2) cards_hist_available = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', '/(?P<subbank>.*)fr/banque/PCS1_CARDFUNCTIONS.aspx', NewCardsListPage) cards_hist_available2 = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx', NewCardsListPage) internal_transfer = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) external_transfer = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) recipients_list = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) error = URL(r'/(?P<subbank>.*)validation/infos.cgi', ErrorPage) subscription = URL(r'/(?P<subbank>.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) terms_and_conditions = URL(r'/(?P<subbank>.*)fr/banque/conditions-generales.html', r'/(?P<subbank>.*)fr/banque/coordonnees_personnelles.aspx', r'/(?P<subbank>.*)fr/banque/paci_engine/paci_wsd_pdta.aspx', r'/(?P<subbank>.*)fr/banque/reglementation-dsp2.html', ConditionsPage) currentSubBank = None is_new_website = None form = None logged = None need_clear_storage = None accounts_list = None def __init__(self, config, *args, **kwargs): self.config = config self.weboob = kwargs['weboob'] kwargs['username'] = self.config['login'].get() kwargs['password'] = self.config['password'].get() super(CreditMutuelBrowser, self).__init__(config, *args, **kwargs) self.__states__ += ( 'currentSubBank', 'form', 'logged', 'is_new_website', 'need_clear_storage', 'recipient_form', 'twofa_auth_state', 'polling_data', 'otp_data', ) self.twofa_auth_state = {} self.polling_data = {} self.otp_data = {} self.keep_session = None self.recipient_form = None self.AUTHENTICATION_METHODS = { 'resume': self.handle_polling, 'code': self.handle_sms, } def get_expire(self): if self.twofa_auth_state: expires = datetime.fromtimestamp(self.twofa_auth_state['expires']).isoformat() return expires return def load_state(self, state): # when add recipient fails, state can't be reloaded. # If state is reloaded, there is this error message: # "Navigation interdite - Merci de bien vouloir recommencer votre action." if state.get('need_clear_storage'): # only keep 'twofa_auth_state' state to avoid new 2FA state = {'twofa_auth_state': state.get('twofa_auth_state')} if state.get('polling_data') or state.get('recipient_form') or state.get('otp_data'): # can't start on an url in the middle of a validation process # or server will cancel it and launch another one if 'url' in state: state.pop('url') # if state is empty (first login), it does nothing super(CreditMutuelBrowser, self).load_state(state) def finalize_twofa(self, twofa_data): """ Go to validated 2FA url. Before following redirection, store 'auth_client_state' cookie to prove to server, for a TWOFA_DURATION, that 2FA is already done. """ self.location( twofa_data['final_url'], data=twofa_data['final_url_params'], allow_redirects=False ) for cookie in self.session.cookies: if cookie.name == 'auth_client_state': # only present if 2FA is valid self.twofa_auth_state['value'] = cookie.value # this is a token self.twofa_auth_state['expires'] = cookie.expires # this is a timestamp self.location(self.response.headers['Location']) def handle_polling(self): # 15' on website, we don't wait that much, but leave sufficient time for the user timeout = time.time() + 600.00 # 15' on webview, need not to wait that much while time.time() < timeout: data = {'transactionId': self.polling_data['polling_id']} self.decoupled_state.go(data=data) decoupled_state = self.page.get_decoupled_state() if decoupled_state == 'VALIDATED': self.logger.info('AppValidation done, going to final_url') self.finalize_twofa(self.polling_data) self.polling_data = {} return elif decoupled_state in ('CANCELLED', 'NONE'): self.polling_data = {} raise AppValidationCancelled() assert decoupled_state == 'PENDING', 'Unhandled polling state: "%s"' % decoupled_state time.sleep(5) # every second on wbesite, need to slow that down # manually cancel polling before website max duration for it self.cancel_decoupled.go(data=data) self.polling_data = {} raise AppValidationExpired() def check_otp_blocked(self): # Too much wrong OTPs, locked down after total 3 wrong inputs if self.otp_blocked_error_page.is_here(): error_msg = self.page.get_error_message() raise BrowserUnavailable(error_msg) def handle_sms(self): self.otp_data['final_url_params']['otp_password'] = self.code self.finalize_twofa(self.otp_data) ## cases where 2FA is not finalized # Too much wrong OTPs, locked down after total 3 wrong inputs self.check_otp_blocked() # OTP is expired after 15', we end up on login page if self.login.is_here(): raise BrowserIncorrectPassword("Le code de confirmation envoyé par SMS n'est plus utilisable") # Wrong OTP leads to same form with error message, re-raise BrowserQuestion elif self.otp_validation_page.is_here(): error_msg = self.page.get_error_message() if 'erroné' not in error_msg: raise BrowserUnavailable(error_msg) else: label = '%s %s' % (error_msg, self.page.get_message()) raise BrowserQuestion(Value('code', label=label)) self.otp_data = {} def check_redirections(self): self.logger.info('Checking redirections') # MobileConfirmationPage or OtpValidationPage is coming but there is no request_information location = self.response.headers.get('Location', '') if 'validation.aspx' in location and not self.is_interactive: self.check_interactive() elif location: self.location(location, allow_redirects=False) def check_auth_methods(self): if self.mobile_confirmation.is_here(): self.page.check_bypass() if self.mobile_confirmation.is_here(): self.polling_data = self.page.get_polling_data() assert self.polling_data, "Can't proceed to polling if no polling_data" raise AppValidation(self.page.get_validation_msg()) if self.otp_validation_page.is_here(): self.otp_data = self.page.get_otp_data() assert self.otp_data, "Can't proceed to SMS handling if no otp_data" raise BrowserQuestion(Value('code', label=self.page.get_message())) self.check_otp_blocked() def init_login(self): self.login.go() # 2FA already done, if valid, login() redirects to home page if self.twofa_auth_state: self.session.cookies.set('auth_client_state', self.twofa_auth_state['value']) self.page.login(self.username, self.password, redirect=True) if not self.page.logged: # 302 redirect to catch to know if polling self.page.login(self.username, self.password) self.check_redirections() # for cic, there is two redirections self.check_redirections() if self.twofa_unabled_page.is_here(): raise ActionNeeded(self.page.get_error_msg()) # when people try to log in but there are on a sub site of creditmutuel if not self.page and not self.url.startswith(self.BASEURL): raise BrowserIncorrectPassword() if self.login_error.is_here(): raise BrowserIncorrectPassword() if self.verify_pass.is_here(): raise AuthMethodNotImplemented("L'identification renforcée avec la carte n'est pas supportée.") self.check_auth_methods() self.getCurrentSubBank() def ownership_guesser(self): profile = self.get_profile() psu_names = profile.name.lower().split() for account in self.accounts_list: label = account.label.lower() # We try to find "M ou Mme" or "Mlle XXX ou M XXXX" for example (non-exhaustive exemple list) if re.search(r'.* ((m) ([\w].*|ou )?(m[ml]e)|(m[ml]e) ([\w].*|ou )(m) ).*', label): account.ownership = AccountOwnership.CO_OWNER # We check if the PSU firstname and lastname is in the account label elif all(name in label.split() for name in psu_names): account.ownership = AccountOwnership.OWNER # Card Accounts should be set with the same ownership of their parents for account in self.accounts_list: if account.type == Account.TYPE_CARD and not empty(account.parent): account.ownership = account.parent.ownership @need_login def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() self.two_cards_page = None self.accounts_list = [] self.revolving_accounts = [] self.unavailablecards = [] self.cards_histo_available = [] self.cards_list =[] self.cards_list2 =[] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) if self.cards_hist_available.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) if not self.cards_list: self.cards_hist_available2.go(subbank=self.currentSubBank) if self.cards_hist_available2.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available2 self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) self.revolving_accounts.append(acc.label.lower()) # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: # We need to return to the main page to avoid navigation error self.cards_activity.go(subbank=self.currentSubBank) page = self.open(company).page if isinstance(company, basestring) else company for card in page.iter_cards(): card2 = find_object(self.cards_list, id=card.id[:16]) if card2: # In order to keep the id of the card from the old space, we exchange the following values card._link_id = card2._link_id card._parent_id = card2._parent_id card.coming = card2.coming card._referer = card2._referer card._secondpage = card2._secondpage self.accounts_list.remove(card2) self.accounts_list.append(card) self.cards_list2.append(card) self.cards_list.extend(self.cards_list2) # Populate accounts from old website if not self.is_new_website: self.logger.info('On old creditmutuel website') self.accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank) self.page.add_por_accounts(self.accounts_list) # Populate accounts from new website else: self.new_accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank) self.page.add_por_accounts(self.accounts_list) self.li.go(subbank=self.currentSubBank) self.accounts_list.extend(self.page.iter_li_accounts()) # This type of account is like a loan, for splitting payments in smaller amounts. # Its history is irrelevant because money is debited from a checking account and # the balance is not even correct, so ignore it. excluded_label = ['etalis', 'valorisation totale'] accounts_by_id = {} for acc in self.accounts_list: if acc.label.lower() not in excluded_label: accounts_by_id[acc.id] = acc # Set the parent to loans and cards accounts for acc in self.accounts_list: if acc.type == Account.TYPE_CARD and not empty(getattr(acc, '_parent_id', None)): acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id: acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) self.accounts_list = list(accounts_by_id.values()) if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) self.ownership_guesser() return self.accounts_list def get_account(self, _id): assert isinstance(_id, basestring) for a in self.get_accounts_list(): if a.id == _id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user paths = urlparse(self.url).path.lstrip('/').split('/') self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else "" if self.currentSubBank and paths[0] == 'banqueprivee' and paths[1] == 'mabanque': self.currentSubBank = 'banqueprivee/mabanque/' if self.currentSubBank and paths[1] == "decouverte": self.currentSubBank += paths[1] + "/" if paths[0] in ["cmmabn", "fr", "mabanque", "banqueprivee"]: self.is_new_website = True def list_operations(self, page, account): if isinstance(page, basestring): if page.startswith('/') or page.startswith('https') or page.startswith('?'): self.location(page) else: try: self.location('%s/%sfr/banque/%s' % (self.BASEURL, self.currentSubBank, page)) except ServerError as e: self.logger.warning('Page cannot be visited: %s/%sfr/banque/%s: %s', self.BASEURL, self.currentSubBank, page, e) raise BrowserUnavailable() else: self.page = page # On some savings accounts, the page lands on the contract tab, and we want the situation if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label: self.page.go_on_history_tab() if self.li.is_here(): return self.page.iter_history() if self.is_new_website and self.page: try: for page in range(1, 50): # Need to reach the page with all transactions if not self.page.has_more_operations(): break form = self.page.get_form(id="I1:P:F") form['_FID_DoLoadMoreTransactions'] = '' form['_wxf2_pseq'] = page form.submit() # IndexError when form xpath returns [], StopIteration if next called on empty iterable except (StopIteration, FormNotFound): self.logger.warning('Could not get history on new website') except IndexError: # 6 months history is not available pass while self.page: try: # Submit form if their is more transactions to fetch form = self.page.get_form(id="I1:fm") if self.page.doc.xpath('boolean(//a[@class="ei_loadmorebtn"])'): form['_FID_DoLoadMoreTransactions'] = "" form.submit() else: break except (IndexError, FormNotFound): break # Sometimes the browser can't go further except ClientError as exc: if exc.response.status_code == 413: break raise if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_monthly_transactions(self, trs): date_getter = attrgetter('date') groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)] trs = [] for group in groups: if group[0].date > datetime.today().date(): continue tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date tr.amount = -sum(t.amount for t in group) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs @need_login def get_history(self, account): transactions = [] if not account._link_id: raise NotImplementedError() if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: if self.two_cards_page: # In this case, you need to return to the page where the iter account get the cards information # Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same account._referer.go(subbank=self.currentSubBank) if account._secondpage: self.location(self.page.get_second_page_link()) # Check if '000000xxxxxx0000' card have an annual history self.location(account._link_id) # The history of the card is available for 1 year with 1 month per page # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions urlstogo = self.page.get_links() self.location(account._link_id) half_history = 'firstHalf' for url in urlstogo: transactions = [] self.location(url) if 'GoMonthPrecedent' in url: # To reach the 6 last month of history you need to change this url parameter # Moreover we are on a transition page where we see the 6 next month (no scrapping here) half_history = 'secondHalf' else: history = self.page.get_history() self.tr_date = self.page.get_date() amount_summary = self.page.get_amount_summary() if self.page.has_more_operations(): for i in range(1, 100): # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) data = { '_FID_DoAddElem': '', '_wxf2_cc': 'fr-FR', '_wxf2_pmode': 'Normal', '_wxf2_pseq': i, '_wxf2_ptarget': 'C:P:updPan', 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', 'Data_ServiceListDatas_CurrentType': 'MyCards', } if 'fid=GoMonth&mois=' in self.url: m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) if m: m = m.group(1) self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data) else: self.location(self.url, data=data) if not self.page.has_more_operations_xml(): history = self.page.iter_history_xml(date=self.tr_date) # We are now with an XML page with all the transactions of the month break else: history = self.page.get_history(date=self.tr_date) for tr in history: # For regrouped transaction, we have to go through each one to get details if tr._regroup: self.location(tr._regroup) for tr2 in self.page.get_tr_merged(): tr2._is_coming = tr._is_coming tr2.date = self.tr_date transactions.append(tr2) else: transactions.append(tr) if transactions and self.tr_date < datetime.today().date(): tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date tr.amount = amount_summary tr.date = tr.rdate = tr.vdate = self.tr_date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True transactions.append(tr) for tr in sorted_transactions(transactions): yield tr else: # need to refresh the months select if account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): for tr in self.list_operations(account._link_id, account): transactions.append(tr) coming_link = self.page.get_coming_link() if self.operations.is_here() else None if coming_link is not None: for tr in self.list_operations(coming_link, account): transactions.append(tr) deferred_date = None cards = ([page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): if tr._to_delete: # Delete main transaction when subtransactions exist continue if hasattr(tr, '_deferred_date') and (not deferred_date or tr._deferred_date < deferred_date): deferred_date = tr._deferred_date if tr.date >= datetime.now(): tr._is_coming = True elif hasattr(account, '_card_pages'): card_trs.append(tr) transactions.append(tr) if card_trs: transactions.extend(self.get_monthly_transactions(card_trs)) if deferred_date is not None: # set deleted for card_summary for tr in transactions: tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and deferred_date.month <= tr.date.month and not hasattr(tr, '_is_manualsum')) for tr in sorted_transactions(transactions): yield tr @need_login def get_investment(self, account): if account._is_inv: if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.por.go(subbank=self.currentSubBank) self.page.send_form(account) elif account.type == Account.TYPE_LIFE_INSURANCE: if not account._link_inv: return iter([]) self.location(account._link_inv) return self.page.iter_investment() if account.type is Account.TYPE_PEA: liquidities = create_french_liquidity(account.balance) liquidities.label = account.label return [liquidities] return iter([]) @need_login def iter_recipients(self, origin_account): # access the transfer page self.internal_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): for recipient in self.page.iter_recipients(origin_account=origin_account): yield recipient self.external_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): origin_account._external_recipients = set() if self.page.has_transfer_categories(): for category in self.page.iter_categories(): self.page.go_on_category(category['index']) self.page.IS_PRO_PAGE = True for recipient in self.page.iter_recipients(origin_account=origin_account, category=category['name']): yield recipient else: for recipient in self.page.iter_recipients(origin_account=origin_account): yield recipient @need_login def init_transfer(self, account, to, amount, exec_date, reason=None): if to.category != 'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: self.internal_transfer.go(subbank=self.currentSubBank) if self.external_transfer.is_here() and self.page.has_transfer_categories(): for category in self.page.iter_categories(): if category['name'] == to.category: self.page.go_on_category(category['index']) break self.page.IS_PRO_PAGE = True self.page.RECIPIENT_STRING = 'data_input_indiceBen' self.page.prepare_transfer(account, to, amount, reason, exec_date) return self.page.handle_response(account, to, amount, reason, exec_date) @need_login def execute_transfer(self, transfer, **params): form = self.page.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Confirmer")]') # For the moment, don't ask the user if he confirms the duplicate. form['Bool:data_input_confirmationDoublon'] = 'true' form.submit() return self.page.create_transfer(transfer) @need_login def get_advisor(self): advisor = None if not self.is_new_website: self.logger.info('On old creditmutuel website') self.accounts.stay_or_go(subbank=self.currentSubBank) if self.page.get_advisor_link(): advisor = self.page.get_advisor() self.location(self.page.get_advisor_link()).page.update_advisor(advisor) else: advisor = self.new_accounts.stay_or_go(subbank=self.currentSubBank).get_advisor() link = self.page.get_agency() if link: link = link.replace(':443/', '/') self.location(link) self.page.update_advisor(advisor) return iter([advisor]) if advisor else iter([]) @need_login def get_profile(self): if not self.is_new_website: self.logger.info('On old creditmutuel website') profile = self.accounts.stay_or_go(subbank=self.currentSubBank).get_profile() else: profile = self.new_accounts.stay_or_go(subbank=self.currentSubBank).get_profile() return profile def get_recipient_object(self, recipient): r = Recipient() r.iban = recipient.iban r.id = recipient.iban r.label = recipient.label r.category = recipient.category # On credit mutuel recipients are immediatly available. r.enabled_at = datetime.now().replace(microsecond=0) r.currency = 'EUR' r.bank_name = NotAvailable return r def format_recipient_form(self, key): self.recipient_form['[t:xsd%3astring;]Data_KeyInput'] = key # we don't know the card id # by default all users have only one card # but to be sure, let's get it dynamically do_validate = [k for k in self.recipient_form.keys() if '_FID_DoValidate_cardId' in k] assert len(do_validate) == 1, 'There should be only one card.' self.recipient_form[do_validate[0]] = '' activate = [k for k in self.recipient_form.keys() if '_FID_GoCardAction_action' in k] for _ in activate: del self.recipient_form[_] def continue_new_recipient(self, recipient, **params): if 'Clé' in params: url = self.recipient_form.pop('url') self.format_recipient_form(params['Clé']) self.location(url, data=self.recipient_form) self.recipient_form = None if self.verify_pass.is_here(): self.page.handle_error() assert False, 'An error occured while checking the card code' self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) self.page.ask_sms(self.get_recipient_object(recipient)) def send_sms(self, sms): data = {} for k, v in self.form.items(): if k != 'url': data[k] = v data['otp_password'] = sms data['_FID_DoConfirm.x'] = '1' data['_FID_DoConfirm.y'] = '1' data['global_backup_hidden_key'] = '' self.location(self.form['url'], data=data) def end_new_recipient(self, recipient, **params): self.send_sms(params['code']) self.form = None self.page = None self.logged = 0 return self.get_recipient_object(recipient) def post_with_bic(self, recipient, **params): data = {} for k, v in self.form.items(): if k != 'url': data[k] = v data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic'] self.location(self.form['url'], data=data) self.page.ask_sms(self.get_recipient_object(recipient)) def set_new_recipient(self, recipient, **params): if self.currentSubBank is None: self.getCurrentSubBank() if 'Bic' in params: return self.post_with_bic(recipient, **params) if 'code' in params: return self.end_new_recipient(recipient, **params) if 'Clé' in params: return self.continue_new_recipient(recipient, **params) assert False, 'An error occured while adding a recipient.' @need_login def new_recipient(self, recipient, **params): if self.currentSubBank is None: self.getCurrentSubBank() self.recipients_list.go(subbank=self.currentSubBank) if self.page.has_list(): assert recipient.category in self.page.get_recipients_list(), \ 'Recipient category is not on the website available list.' self.page.go_list(recipient.category) self.page.go_to_add() if self.verify_pass.is_here(): self.recipient_form = self.page.get_recipient_form() raise AddRecipientStep(self.get_recipient_object(recipient), Value('Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params) @need_login def iter_subscriptions(self): if self.currentSubBank is None: self.getCurrentSubBank() self.subscription.go(subbank=self.currentSubBank) return self.page.iter_subscriptions() @need_login def iter_documents(self, subscription): if self.currentSubBank is None: self.getCurrentSubBank() self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'}) security_limit = 10 for i in range(security_limit): for doc in self.page.iter_documents(sub_id=subscription.id): yield doc if self.page.is_last_page(): break self.page.next_page()
class CreditMutuelBrowser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 BASEURL = 'https://www.creditmutuel.fr' login = URL('/groupe/fr/index.html', '/(?P<subbank>.*)/fr/$', '/(?P<subbank>.*)/fr/banques/accueil.html', '/(?P<subbank>.*)/fr/banques/particuliers/index.html', LoginPage) login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi', LoginErrorPage) accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi', '/(?P<subbank>.*)/fr/banque/situation_financiere.html', AccountsPage) user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx', UserSpacePage) operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*', '/(?P<subbank>.*)/fr/banque/mouvements.html.*', '/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*', OperationsPage) coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage) card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage) noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage) info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage) transfert = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_vi.html', EmptyPage) transfert_2 = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_cmweb.aspx.*', TransfertPage) change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi', ChangePasswordPage) verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*', VerifCodePage) empty = URL( '/(?P<subbank>.*)/fr/banques/index.html', '/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*', '/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*', '/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx', '/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage) por = URL('/(?P<subbank>.*)/fr/banque/POR_ValoToute.aspx', '/(?P<subbank>.*)/fr/banque/POR_SyntheseLst.aspx', PorPage) li = URL('/(?P<subbank>.*)/fr/assurances/profilass.aspx\?domaine=epargne', '/(?P<subbank>.*)/fr/assurances/consultation/WI_ASSAVI', LIAccountsPage) iban = URL('/(?P<subbank>.*)/fr/banque/rib.cgi', IbanPage) new_home = URL('/fr/banque/pageaccueil.html', '/fr/banque/welcome_pack.html', NewHomePage) new_accounts = URL('/fr/banque/comptes-et-contrats.html', AccountsPage) new_operations = URL('/fr/banque/mouvements.cgi', '/fr/banque/mouvements.html', OperationsPage) new_por = URL('/fr/banque/POR_ValoToute.aspx', '/fr/banque/POR_SyntheseLst.aspx', PorPage) new_iban = URL('/fr/banque/rib.cgi', IbanPage) redirect = URL('/fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) currentSubBank = None is_new_website = False __states__ = ['currentSubBank'] def do_login(self): # Clear cookies. self.do_logout() self.login.go() if not self.page.logged: self.page.login(self.username, self.password) if not self.page.logged or self.login_error.is_here(): raise BrowserIncorrectPassword() if not self.is_new_website: self.getCurrentSubBank() @need_login def get_accounts_list(self): if self.currentSubBank is None and not self.is_new_website: self.getCurrentSubBank() accounts = [] if not self.is_new_website: for a in self.accounts.stay_or_go( subbank=self.currentSubBank).iter_accounts(): accounts.append(a) self.iban.go(subbank=self.currentSubBank).fill_iban(accounts) self.por.go(subbank=self.currentSubBank).add_por_accounts(accounts) for acc in self.li.go( subbank=self.currentSubBank).iter_li_accounts(): accounts.append(acc) else: for a in self.new_accounts.stay_or_go().iter_accounts(): accounts.append(a) self.new_iban.go().fill_iban(accounts) self.new_por.go().add_por_accounts(accounts) return accounts def get_account(self, id): assert isinstance(id, basestring) for a in self.get_accounts_list(): if a.id == id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user url = urlparse(self.url) self.currentSubBank = url.path.lstrip('/').split('/')[0] def list_operations(self, page_url): if page_url.startswith('/') or page_url.startswith('https'): self.location(page_url) elif not self.is_new_website: self.location('%s/%s/fr/banque/%s' % (self.BASEURL, self.currentSubBank, page_url)) else: self.location('%s/fr/banque/%s' % (self.BASEURL, page_url)) if self.li.is_here(): return self.page.iter_history() if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_history(self, account): transactions = [] last_debit = None if not account._link_id: return iter([]) for tr in self.list_operations(account._link_id): # to prevent redundancy with card transactions, we do not # store 'RELEVE CARTE' transaction. if not tr.raw.startswith('RELEVE CARTE'): transactions.append(tr) elif last_debit is None: # we set the debit date to last day of month so we need to do the same form last_debit last_debit = (tr.date + relativedelta(day=31)) coming_link = self.page.get_coming_link() if self.operations.is_here( ) else None if coming_link is not None: for tr in self.list_operations(coming_link): transactions.append(tr) for card_link in account._card_links: for tr in self.list_operations(card_link): if last_debit is None or tr.date > last_debit: tr._is_coming = True transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions def get_investment(self, account): if account._is_inv: if account.type == Account.TYPE_MARKET: if not self.is_new_website: self.por.go(subbank=self.currentSubBank) else: self.new_por.go() self.page.send_form(account) elif account.type == Account.TYPE_LIFE_INSURANCE: if not account._link_inv: return iter([]) self.location(account._link_inv) return self.page.iter_investment() return iter([]) def transfer(self, account, to, amount, reason=None): if self.is_new_website: raise NotImplementedError() # access the transfer page self.transfert.go(subbank=self.currentSubBank) # fill the form form = self.page.get_form(xpath="//form[@id='P:F']") try: form[ 'data_input_indiceCompteADebiter'] = self.page.get_from_account_index( account) form[ 'data_input_indiceCompteACrediter'] = self.page.get_to_account_index( to) except ValueError as e: raise TransferError(e.message) form['[t:dbt%3adouble;]data_input_montant_value_0_'] = '%s' % str( amount).replace('.', ',') if reason is not None: form[ '[t:dbt%3astring;x(27)]data_input_libelleCompteDebite'] = reason form[ '[t:dbt%3astring;x(31)]data_input_motifCompteCredite'] = reason del form['_FID_GoCancel'] del form['_FID_DoValidate'] form['_FID_DoValidate.x'] = str(randint(3, 125)) form['_FID_DoValidate.y'] = str(randint(3, 22)) form.submit() # look for known errors content = self.page.get_unicode_content() insufficient_amount_message = u'Le montant du virement doit être positif, veuillez le modifier' maximum_allowed_balance_message = u'Montant maximum autorisé au débit pour ce compte' if insufficient_amount_message in content: raise TransferError('The amount you tried to transfer is too low.') if maximum_allowed_balance_message in content: raise TransferError( 'The maximum allowed balance for the target account has been / would be reached.' ) # look for the known "all right" message ready_for_transfer_message = u'Confirmer un virement entre vos comptes' if ready_for_transfer_message not in content: raise TransferError('The expected message "%s" was not found.' % ready_for_transfer_message) # submit the confirmation form form = self.page.get_form(xpath="//form[@id='P:F']") del form['_FID_DoConfirm'] form['_FID_DoConfirm.x'] = str(randint(3, 125)) form['_FID_DoConfirm.y'] = str(randint(3, 22)) submit_date = datetime.now() form.submit() # look for the known "everything went well" message content = self.page.get_unicode_content() transfer_ok_message = u'Votre virement a été exécuté' if transfer_ok_message not in content: raise TransferError('The expected message "%s" was not found.' % transfer_ok_message) # We now have to return a Transfer object transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S')) transfer.amount = amount transfer.origin = account transfer.recipient = to transfer.date = submit_date return transfer
class CreditMutuelBrowser(LoginBrowser): PROFILE = Wget() BASEURL = 'https://www.creditmutuel.fr' login = URL('/groupe/fr/index.html', LoginPage) login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi', LoginErrorPage) accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi', AccountsPage) user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx', UserSpacePage) operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*', '/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*', OperationsPage) coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage) card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage) noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage) info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage) transfert = URL( '/(?P<subbank>.*)/fr/banque/WI_VPLV_VirUniSaiCpt.asp\?(?P<parameters>.*)', TransfertPage) change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi', ChangePasswordPage) verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*', VerifCodePage) empty = URL( '/(?P<subbank>.*)/fr/$', '/(?P<subbank>.*)/fr/banques/index.html', '/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*', '/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*', '/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx', '/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage) currentSubBank = None def do_login(self): self.login.stay_or_go() self.page.login(self.username, self.password) if not self.page.logged or self.login_error.is_here(): raise BrowserIncorrectPassword() self.getCurrentSubBank() @need_login def get_accounts_list(self): return self.accounts.stay_or_go( subbank=self.currentSubBank).iter_accounts() def get_account(self, id): assert isinstance(id, basestring) for a in self.get_accounts_list(): if a.id == id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user url = urlparse(self.url) self.currentSubBank = url.path.lstrip('/').split('/')[0] def list_operations(self, page_url): if page_url.startswith('/'): self.location(page_url) else: self.location('%s/%s/fr/banque/%s' % (self.BASEURL, self.currentSubBank, page_url)) if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_history(self, account): transactions = [] last_debit = None for tr in self.list_operations(account._link_id): # to prevent redundancy with card transactions, we do not # store 'RELEVE CARTE' transaction. if tr.raw != 'RELEVE CARTE': transactions.append(tr) elif last_debit is None: last_debit = (tr.date - timedelta(days=10)).month coming_link = self.page.get_coming_link() if self.operations.is_here( ) else None if coming_link is not None: for tr in self.list_operations(coming_link): transactions.append(tr) month = 0 for card_link in account._card_links: v = urlsplit(card_link) args = dict(parse_qsl(v.query)) # useful with 12 -> 1 if int(args['mois']) < month: month += 1 else: month = int(args['mois']) for tr in self.list_operations(card_link): if month > last_debit: tr._is_coming = True transactions.append(tr) transactions.sort(key=lambda tr: tr.rdate, reverse=True) return transactions def transfer(self, account, to, amount, reason=None): # access the transfer page parameters = 'RAZ=ALL&Cat=6&PERM=N&CHX=A' page = self.transfert.go(subbank=self.currentSubBank, parameters=parameters) # fill the form form = self.page.get_form(name='FormVirUniSaiCpt') form['IDB'] = account[-1] form['ICR'] = to[-1] form['MTTVIR'] = '%s' % str(amount).replace('.', ',') if reason is not None: form['LIBDBT'] = reason form['LIBCRT'] = reason page = form.submit() # look for known errors content = page.response.text insufficient_amount_message = u'Montant insuffisant.' maximum_allowed_balance_message = u'Solde maximum autorisé dépassé.' if insufficient_amount_message in content: raise TransferError('The amount you tried to transfer is too low.') if maximum_allowed_balance_message in content: raise TransferError( 'The maximum allowed balance for the target account has been / would be reached.' ) # look for the known "all right" message ready_for_transfer_message = u'Confirmez un virement entre vos comptes' if ready_for_transfer_message in content: raise TransferError('The expected message "%s" was not found.' % ready_for_transfer_message) # submit the confirmation form form = page.get_form(name='FormVirUniCnf') submit_date = datetime.now() page = form.submit() # look for the known "everything went well" message content = page.response.text transfer_ok_message = u'Votre virement a été exécuté ce jour' if transfer_ok_message not in content: raise TransferError('The expected message "%s" was not found.' % transfer_ok_message) # We now have to return a Transfer object transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S')) transfer.amount = amount transfer.origin = account transfer.recipient = to transfer.date = submit_date return transfer
class CreditMutuelBrowser(LoginBrowser, StatesMixin): PROFILE = Wget() STATE_DURATION = 10 TIMEOUT = 30 BASEURL = 'https://www.creditmutuel.fr' login = URL('/fr/authentification.html', r'/(?P<subbank>.*)fr/$', r'/(?P<subbank>.*)fr/banques/accueil.html', r'/(?P<subbank>.*)fr/banques/particuliers/index.html', LoginPage) login_error = URL(r'/(?P<subbank>.*)fr/identification/default.cgi', LoginErrorPage) accounts = URL(r'/(?P<subbank>.*)fr/banque/situation_financiere.cgi', r'/(?P<subbank>.*)fr/banque/situation_financiere.html', AccountsPage) revolving_loan_list = URL( r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL( r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) user_space = URL( r'/(?P<subbank>.*)fr/banque/espace_personnel.aspx', r'/(?P<subbank>.*)fr/banque/accueil.cgi', r'/(?P<subbank>.*)fr/banque/DELG_Gestion', r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', UserSpacePage) card = URL( r'/(?P<subbank>.*)fr/banque/operations_carte.cgi.*', r'/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', r'/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', CardPage) operations = URL( r'/(?P<subbank>.*)fr/banque/mouvements.cgi.*', r'/(?P<subbank>.*)fr/banque/mouvements.html.*', r'/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*', r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+', OperationsPage) coming = URL(r'/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage) info = URL(r'/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage) change_pass = URL(r'/(?P<subbank>.*)fr/validation/change_password.cgi', '/fr/services/change_password.html', ChangePasswordPage) verify_pass = URL(r'/(?P<subbank>.*)fr/validation/verif_code.cgi.*', r'/(?P<subbank>.*)fr/validation/lst_codes.cgi.*', VerifCodePage) new_home = URL(r'/(?P<subbank>.*)fr/banque/pageaccueil.html', r'/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage) empty = URL( r'/(?P<subbank>.*)fr/banques/index.html', r'/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*', r'/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', EmptyPage) por = URL(r'/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx', r'/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx', PorPage) li = URL(r'/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne', r'/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*', r'/(?P<subbank>.*)fr/assurances/WI_ASS', '/fr/assurances/', LIAccountsPage) iban = URL(r'/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage) new_accounts = URL(r'/(?P<subbank>.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) new_operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi', r'/fr/banque/nr/nr_devbooster.aspx.*', r'/(?P<subbank>.*)fr/banque/RE/aiguille(liste)?.asp', '/fr/banque/mouvements.html', r'/(?P<subbank>.*)fr/banque/consultation/operations', OperationsPage) advisor = URL( r'/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)', r'/(?P<subbank>.*)fr/infoclient/', r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*', AdvisorPage) redirect = URL( r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) cards_activity = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) cards_list = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*', r'/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) cards_ope = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) cards_ope2 = URL('/(?P<subbank>.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2) cards_hist_available = URL( '/(?P<subbank>.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', '/(?P<subbank>.*)fr/banque/PCS1_CARDFUNCTIONS.aspx', NewCardsListPage) cards_hist_available2 = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx', NewCardsListPage) internal_transfer = URL( r'/(?P<subbank>.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) external_transfer = URL( r'/(?P<subbank>.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) recipients_list = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) error = URL(r'/(?P<subbank>.*)validation/infos.cgi', ErrorPage) subscription = URL(r'/(?P<subbank>.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) terms_and_conditions = URL( r'/(?P<subbank>.*)fr/banque/conditions-generales.html', r'/(?P<subbank>.*)fr/banque/coordonnees_personnelles.aspx', r'/(?P<subbank>.*)fr/banque/paci_engine/paci_wsd_pdta.aspx', r'/(?P<subbank>.*)fr/banque/reglementation-dsp2.html', ConditionsPage) currentSubBank = None is_new_website = None form = None logged = None need_clear_storage = None __states__ = [ 'currentSubBank', 'form', 'logged', 'is_new_website', 'need_clear_storage' ] accounts_list = None def load_state(self, state): # when add recipient fails, state can't be reloaded. If state is reloaded, there is this error message: # "Navigation interdite - Merci de bien vouloir recommencer votre action." if not state.get('need_clear_storage'): super(CreditMutuelBrowser, self).load_state(state) else: self.need_clear_storage = None def do_login(self): # Clear cookies. self.do_logout() self.login.go() if not self.page.logged: self.page.login(self.username, self.password) # when people try to log in but there are on a sub site of creditmutuel if not self.page and not self.url.startswith(self.BASEURL): raise BrowserIncorrectPassword() if not self.page.logged or self.login_error.is_here(): raise BrowserIncorrectPassword() if self.verify_pass.is_here(): raise AuthMethodNotImplemented( "L'identification renforcée avec la carte n'est pas supportée." ) self.getCurrentSubBank() def ownership_guesser(self): profile = self.get_profile() psu_names = profile.name.lower().split() for account in self.accounts_list: label = account.label.lower() # We try to find "M ou Mme" or "Mlle XXX ou M XXXX" for example (non-exhaustive exemple list) if re.search( r'.* ((m) ([\w].*|ou )?(m[ml]e)|(m[ml]e) ([\w].*|ou )(m) ).*', label): account.ownership = AccountOwnership.CO_OWNER # We check if the PSU firstname and lastname is in the account label elif all(name in label.split() for name in psu_names): account.ownership = AccountOwnership.OWNER # Card Accounts should be set with the same ownership of their parents for account in self.accounts_list: if account.type == Account.TYPE_CARD and not empty(account.parent): account.ownership = account.parent.ownership @need_login def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() self.two_cards_page = None self.accounts_list = [] self.revolving_accounts = [] self.unavailablecards = [] self.cards_histo_available = [] self.cards_list = [] self.cards_list2 = [] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) if self.cards_hist_available.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) if not self.cards_list: self.cards_hist_available2.go(subbank=self.currentSubBank) if self.cards_hist_available2.is_here(): self.unavailablecards.extend( self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available2 self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) for acc in self.revolving_loan_list.stay_or_go( subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) self.revolving_accounts.append(acc.label.lower()) # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: # We need to return to the main page to avoid navigation error self.cards_activity.go(subbank=self.currentSubBank) page = self.open(company).page if isinstance( company, basestring) else company for card in page.iter_cards(): card2 = find_object(self.cards_list, id=card.id[:16]) if card2: # In order to keep the id of the card from the old space, we exchange the following values card._link_id = card2._link_id card._parent_id = card2._parent_id card.coming = card2.coming card._referer = card2._referer card._secondpage = card2._secondpage self.accounts_list.remove(card2) self.accounts_list.append(card) self.cards_list2.append(card) self.cards_list.extend(self.cards_list2) # Populate accounts from old website if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) # Populate accounts from new website else: self.new_accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) self.li.go(subbank=self.currentSubBank) self.accounts_list.extend(self.page.iter_li_accounts()) for acc in self.cards_list: if hasattr(acc, '_parent_id'): acc.parent = find_object(self.accounts_list, id=acc._parent_id) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [ acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label) ] if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) self.ownership_guesser() return self.accounts_list def get_account(self, _id): assert isinstance(_id, basestring) for a in self.get_accounts_list(): if a.id == _id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user paths = urlparse(self.url).path.lstrip('/').split('/') self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else "" if self.currentSubBank and paths[0] == 'banqueprivee' and paths[ 1] == 'mabanque': self.currentSubBank = 'banqueprivee/mabanque/' if self.currentSubBank and paths[1] == "decouverte": self.currentSubBank += paths[1] + "/" if paths[0] in ["fr", "mabanque", "banqueprivee"]: self.is_new_website = True def list_operations(self, page, account): if isinstance(page, basestring): if page.startswith('/') or page.startswith( 'https') or page.startswith('?'): self.location(page) else: try: self.location('%s/%sfr/banque/%s' % (self.BASEURL, self.currentSubBank, page)) except ServerError as e: self.logger.warning( 'Page cannot be visited: %s/%sfr/banque/%s: %s', self.BASEURL, self.currentSubBank, page, e) raise BrowserUnavailable() else: self.page = page # On some savings accounts, the page lands on the contract tab, and we want the situation if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label: self.page.go_on_history_tab() if self.li.is_here(): return self.page.iter_history() if self.is_new_website and self.page: try: for page in range(1, 50): # Need to reach the page with all transactions if not self.page.has_more_operations(): break form = self.page.get_form(id="I1:P:F") form['_FID_DoLoadMoreTransactions'] = '' form['_wxf2_pseq'] = page form.submit() # IndexError when form xpath returns [], StopIteration if next called on empty iterable except (StopIteration, FormNotFound): self.logger.warning('Could not get history on new website') except IndexError: # 6 months history is not available pass while self.page: try: # Submit form if their is more transactions to fetch form = self.page.get_form(id="I1:fm") if self.page.doc.xpath( 'boolean(//a[@class="ei_loadmorebtn"])'): form['_FID_DoLoadMoreTransactions'] = "" form.submit() else: break except (IndexError, FormNotFound): break # Sometimes the browser can't go further except ClientError as exc: if exc.response.status_code == 413: break raise if not self.operations.is_here(): return iter([]) return self.pagination(lambda: self.page.get_history()) def get_monthly_transactions(self, trs): date_getter = attrgetter('date') groups = [ list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter) ] trs = [] for group in groups: if group[0].date > datetime.today().date(): continue tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date tr.amount = -sum(t.amount for t in group) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs @need_login def get_history(self, account): transactions = [] if not account._link_id: raise NotImplementedError() if len(account.id ) >= 16 and account.id[:16] in self.cards_histo_available: if self.two_cards_page: # In this case, you need to return to the page where the iter account get the cards information # Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same account._referer.go(subbank=self.currentSubBank) if account._secondpage: self.location(self.page.get_second_page_link()) # Check if '000000xxxxxx0000' card have an annual history self.location(account._link_id) # The history of the card is available for 1 year with 1 month per page # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions urlstogo = self.page.get_links() self.location(account._link_id) half_history = 'firstHalf' for url in urlstogo: transactions = [] self.location(url) if 'GoMonthPrecedent' in url: # To reach the 6 last month of history you need to change this url parameter # Moreover we are on a transition page where we see the 6 next month (no scrapping here) half_history = 'secondHalf' else: history = self.page.get_history() self.tr_date = self.page.get_date() amount_summary = self.page.get_amount_summary() if self.page.has_more_operations(): for i in range(1, 100): # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) data = { '_FID_DoAddElem': '', '_wxf2_cc': 'fr-FR', '_wxf2_pmode': 'Normal', '_wxf2_pseq': i, '_wxf2_ptarget': 'C:P:updPan', 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', 'Data_ServiceListDatas_CurrentType': 'MyCards', } if 'fid=GoMonth&mois=' in self.url: m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) if m: m = m.group(1) self.location( 'CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details' .format(m, half_history), data=data) else: self.location(self.url, data=data) if not self.page.has_more_operations_xml(): history = self.page.iter_history_xml( date=self.tr_date) # We are now with an XML page with all the transactions of the month break else: history = self.page.get_history(date=self.tr_date) for tr in history: # For regrouped transaction, we have to go through each one to get details if tr._regroup: self.location(tr._regroup) for tr2 in self.page.get_tr_merged(): tr2._is_coming = tr._is_coming tr2.date = self.tr_date transactions.append(tr2) else: transactions.append(tr) if transactions and self.tr_date < datetime.today().date(): tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date tr.amount = amount_summary tr.date = tr.rdate = tr.vdate = self.tr_date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True transactions.append(tr) for tr in sorted_transactions(transactions): yield tr else: # need to refresh the months select if account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): for tr in self.list_operations(account._link_id, account): transactions.append(tr) coming_link = self.page.get_coming_link( ) if self.operations.is_here() else None if coming_link is not None: for tr in self.list_operations(coming_link, account): transactions.append(tr) differed_date = None cards = ([ page.select_card(account._card_number) for page in account._card_pages ] if hasattr(account, '_card_pages') else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): if tr._to_delete: # Delete main transaction when subtransactions exist continue if hasattr(tr, '_differed_date') and ( not differed_date or tr._differed_date < differed_date): differed_date = tr._differed_date if tr.date >= datetime.now(): tr._is_coming = True elif hasattr(account, '_card_pages'): card_trs.append(tr) transactions.append(tr) if card_trs: transactions.extend( self.get_monthly_transactions(card_trs)) if differed_date is not None: # set deleted for card_summary for tr in transactions: tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and differed_date.month <= tr.date.month and not hasattr(tr, '_is_manualsum')) for tr in sorted_transactions(transactions): yield tr @need_login def get_investment(self, account): if account._is_inv: if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.por.go(subbank=self.currentSubBank) self.page.send_form(account) elif account.type == Account.TYPE_LIFE_INSURANCE: if not account._link_inv: return iter([]) self.location(account._link_inv) return self.page.iter_investment() if account.type is Account.TYPE_PEA: liquidities = create_french_liquidity(account.balance) liquidities.label = account.label return [liquidities] return iter([]) @need_login def iter_recipients(self, origin_account): # access the transfer page self.internal_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): for recipient in self.page.iter_recipients( origin_account=origin_account): yield recipient self.external_transfer.go(subbank=self.currentSubBank) if self.page.can_transfer(origin_account.id): origin_account._external_recipients = set() if self.page.has_transfer_categories(): for category in self.page.iter_categories(): self.page.go_on_category(category['index']) self.page.IS_PRO_PAGE = True for recipient in self.page.iter_recipients( origin_account=origin_account, category=category['name']): yield recipient else: for recipient in self.page.iter_recipients( origin_account=origin_account): yield recipient @need_login def init_transfer(self, account, to, amount, exec_date, reason=None): if to.category != 'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: self.internal_transfer.go(subbank=self.currentSubBank) if self.external_transfer.is_here( ) and self.page.has_transfer_categories(): for category in self.page.iter_categories(): if category['name'] == to.category: self.page.go_on_category(category['index']) break self.page.IS_PRO_PAGE = True self.page.RECIPIENT_STRING = 'data_input_indiceBen' self.page.prepare_transfer(account, to, amount, reason, exec_date) return self.page.handle_response(account, to, amount, reason, exec_date) @need_login def execute_transfer(self, transfer, **params): form = self.page.get_form( id='P:F', submit='//input[@type="submit" and contains(@value, "Confirmer")]') # For the moment, don't ask the user if he confirms the duplicate. form['Bool:data_input_confirmationDoublon'] = 'true' form.submit() return self.page.create_transfer(transfer) @need_login def get_advisor(self): advisor = None if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) if self.page.get_advisor_link(): advisor = self.page.get_advisor() self.location( self.page.get_advisor_link()).page.update_advisor(advisor) else: advisor = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_advisor() link = self.page.get_agency() if link: link = link.replace(':443/', '/') self.location(link) self.page.update_advisor(advisor) return iter([advisor]) if advisor else iter([]) @need_login def get_profile(self): if not self.is_new_website: profile = self.accounts.stay_or_go( subbank=self.currentSubBank).get_profile() else: profile = self.new_accounts.stay_or_go( subbank=self.currentSubBank).get_profile() return profile def get_recipient_object(self, recipient): r = Recipient() r.iban = recipient.iban r.id = recipient.iban r.label = recipient.label r.category = recipient.category # On credit mutuel recipients are immediatly available. r.enabled_at = datetime.now().replace(microsecond=0) r.currency = 'EUR' r.bank_name = NotAvailable return r def continue_new_recipient(self, recipient, **params): if 'Clé' in params: self.page.post_code(params['Clé']) self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) self.page.ask_sms(self.get_recipient_object(recipient)) def send_sms(self, sms): data = {} for k, v in self.form.items(): if k != 'url': data[k] = v data['otp_password'] = sms data['_FID_DoConfirm.x'] = '1' data['_FID_DoConfirm.y'] = '1' data['global_backup_hidden_key'] = '' self.location(self.form['url'], data=data) def end_new_recipient(self, recipient, **params): self.send_sms(params['code']) self.form = None self.page = None self.logged = 0 return self.get_recipient_object(recipient) def post_with_bic(self, recipient, **params): data = {} for k, v in self.form.items(): if k != 'url': data[k] = v data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic'] self.location(self.form['url'], data=data) self.page.ask_sms(self.get_recipient_object(recipient)) @need_login def new_recipient(self, recipient, **params): if self.currentSubBank is None: self.getCurrentSubBank() if 'Bic' in params: return self.post_with_bic(recipient, **params) if 'code' in params: return self.end_new_recipient(recipient, **params) if 'Clé' in params: return self.continue_new_recipient(recipient, **params) self.recipients_list.go(subbank=self.currentSubBank) if self.page.has_list(): assert recipient.category in self.page.get_recipients_list(), \ 'Recipient category is not on the website available list.' self.page.go_list(recipient.category) self.page.go_to_add() if self.verify_pass.is_here(): raise AddRecipientStep( self.get_recipient_object(recipient), Value('Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params) @need_login def iter_subscriptions(self): if self.currentSubBank is None: self.getCurrentSubBank() self.subscription.go(subbank=self.currentSubBank) return self.page.iter_subscriptions() @need_login def iter_documents(self, subscription): if self.currentSubBank is None: self.getCurrentSubBank() self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'}) security_limit = 10 for i in range(security_limit): for doc in self.page.iter_documents(sub_id=subscription.id): yield doc if self.page.is_last_page(): break self.page.next_page()