class BtmonBrowser(PagesBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'http://www.btmon.com/'
home = URL('$', HomePage)
search = URL(r'/torrent/\?sort=relevance&f=(?P<pattern>.*)', SearchPage)
torrent = URL(r'/(?P<torrent_id>.*)\.torrent\.html', TorrentPage)
def get_bpc_cookie(self):
if 'BPC' not in self.session.cookies:
self.home.go()
bpcCookie = str(self.page.content).split('BPC=')[-1].split('"')[0]
self.session.cookies['BPC'] = bpcCookie
def iter_torrents(self, pattern):
self.get_bpc_cookie()
return self.search.go(pattern=pattern).iter_torrents()
def get_torrent(self, id):
try:
self.get_bpc_cookie()
self.torrent.go(torrent_id=id)
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
class BECMBrowser(AbstractBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.becm.fr'
PARENT = 'creditmutuel'
login = URL('/fr/authentification.html', LoginPage)
advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage)
@need_login
def get_advisor(self):
advisor = None
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
if self.page.get_advisor_link():
advisor = self.page.get_advisor()
self.location(
self.page.get_advisor_link()).page.update_advisor(advisor)
else:
advisor = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_advisor()
link = self.page.get_agency()
if link:
self.location(link)
self.page.update_advisor(advisor)
return iter([advisor]) if advisor else iter([])
class T411Browser(LoginBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.t411.ai/'
home = URL('$', HomePage)
search = URL(
'torrents/search/\?search=(?P<pattern>.*)&order=seeders&type=desc',
SearchPage)
# Order matters here: 'torrents/[^&]*' would match '/torrents/download/\?id...' and
# TorrentPage would crash on the bencode data, so DownloadPage must be listed before
# TorrentPage
download = URL('/torrents/download/\?id=(?P<id>.*)', DownloadPage)
torrent = URL('/torrents/details/\?id=(?P<id>.*)&r=1', 'torrents/[^&]*',
TorrentPage)
def do_login(self):
self.home.go()
if not self.page.logged:
self.page.login(self.username, self.password)
if not self.page.logged:
raise BrowserIncorrectPassword()
@need_login
def iter_torrents(self, pattern):
return self.search.go(pattern=pattern).iter_torrents()
@need_login
def get_torrent(self, fullid, torrent=None):
try:
self.torrent.go(id=fullid)
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
class T411Browser(LoginBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.t411.in/'
home = URL('$', HomePage)
search = URL(
'torrents/search/\?search=(?P<pattern>.*)&order=seeders&type=desc',
SearchPage)
torrent = URL('/torrents/details/\?id=(?P<id>.*)&r=1', 'torrents/[^&]*',
TorrentPage)
#def __init__(self, *args, **kwargs):
# Browser.__init__(self, *args, **kwargs)
def do_login(self):
self.home.go()
if not self.page.logged:
self.page.login(self.username, self.password)
if not self.page.logged:
raise BrowserIncorrectPassword()
@need_login
def iter_torrents(self, pattern):
return self.search.go(pattern=pattern).iter_torrents()
@need_login
def get_torrent(self, fullid, torrent=None):
try:
self.torrent.go(id=fullid)
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
class CICBrowser(AbstractBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.cic.fr'
PARENT = 'creditmutuel'
login = URL('/sb/fr/banques/particuliers/index.html',
'/(?P<subbank>.*)/fr/$',
'/(?P<subbank>.*)/fr/banques/accueil.html',
'/(?P<subbank>.*)/fr/banques/particuliers/index.html',
LoginPage)
class BECMBrowser(AbstractBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.becm.fr'
PARENT = 'creditmutuel'
login = URL('/fr/authentification.html', LoginPage)
advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage)
def do_login(self):
# Clear cookies.
self.do_logout()
self.login.go()
if not self.page.logged:
self.page.login(self.username, self.password)
# Many "Credit Mutuel" customers tried to add their connection to BECM, but the BECM
# website does not return any error when you try to login with correct Crédit Mutuel
# credentials, therefore we must suggest them to try regular Crédit Mutuel if login fails.
if self.login.is_here():
raise ActionNeeded(
"La connexion au site de BECM n'a pas fonctionné avec les identifiants fournis.\
Si vous êtes client du Crédit Mutuel, veuillez réessayer en sélectionnant le module Crédit Mutuel."
)
if self.verify_pass.is_here():
raise AuthMethodNotImplemented(
"L'identification renforcée avec la carte n'est pas supportée."
)
@need_login
def get_advisor(self):
advisor = None
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
if self.page.get_advisor_link():
advisor = self.page.get_advisor()
self.location(
self.page.get_advisor_link()).page.update_advisor(advisor)
else:
advisor = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_advisor()
link = self.page.get_agency()
if link:
self.location(link)
self.page.update_advisor(advisor)
return iter([advisor]) if advisor else iter([])
class LimetorrentsBrowser(PagesBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.limetorrents.info/'
search = URL(r'/search/all/(?P<pattern>.*)/seeds/(?P<page>[0-9]+)/', SearchPage)
torrent = URL(r'/(?P<torrent_name>.*)-torrent-(?P<torrent_id>[0-9]+)\.html', TorrentPage)
def iter_torrents(self, pattern):
return self.search.go(pattern=pattern, page=1).iter_torrents()
def get_torrent(self, id):
try:
self.torrent.go(torrent_id=id, torrent_name='whatever')
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
class CICBrowser(AbstractBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.cic.fr'
PARENT = 'creditmutuel'
login = URL(r'/fr/authentification.html',
r'/sb/fr/banques/particuliers/index.html',
r'/(?P<subbank>.*)/fr/$',
r'/(?P<subbank>.*)/fr/banques/accueil.html',
r'/(?P<subbank>.*)/fr/banques/particuliers/index.html',
LoginPage)
por = URL(r'/(?P<subbank>.*)fr/banque/PORT_Synthese.aspx', PorPage)
decoupled_state = URL(r'/fr/otp/SOSD_OTP_GetTransactionState.htm',
DecoupledStatePage)
cancel_decoupled = URL(r'/fr/otp/SOSD_OTP_CancelTransaction.htm',
CancelDecoupled)
class T411Browser(LoginBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.t411.si/'
home = URL('$', HomePage)
login = URL('/login$', LoginPage)
search = URL(r'/torrents/search/\?search=(?P<pattern>.*)', SearchPage)
download = URL(
'/telecharger-torrent/(?P<torrent_hash>[0-9a-f]{40})/(?P<torrent_name>\w+)',
DownloadPage)
torrent = URL('/torrents/(?P<torrent_id>[0-9]+)/(?P<torrent_name>.*)',
TorrentPage)
def do_login(self):
self.home.go()
if not self.page.logged:
self.page.login(self.username, self.password)
self.home.go()
if not self.page.logged:
raise BrowserIncorrectPassword()
@need_login
def iter_torrents(self, pattern):
return self.search.go(pattern=pattern).iter_torrents()
@need_login
def get_torrent(self, torrent):
try:
self.torrent.go(torrent_id=torrent.id, torrent_name=torrent.name)
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
def get_torrent_file(self, torrent):
torrent = self.browser.get_torrent(torrent)
if not torrent:
return None
resp = self.browser.open(torrent.url)
return resp.content
class YggtorrentBrowser(LoginBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://yggtorrent.to/'
home = URL('$', HomePage)
login = URL('/user/login$', LoginPage)
search = URL(r'/engine/search\?name=(?P<pattern>.*)&order=desc&sort=seed&do=search', SearchPage)
download = URL('/engine/download_torrent\?id=(?P<torrent_id>[0-9]+)', DownloadPage)
torrent = URL('/torrent/(?P<torrent_cat>.+)/(?P<torrent_subcat>.+)/(?P<torrent_id>[0-9]+)-(?P<torrent_name>.*)', TorrentPage)
def do_login(self):
self.home.go()
if not self.page.logged:
self.page.login(self.username, self.password)
self.home.go()
if not self.page.logged:
raise BrowserIncorrectPassword()
@need_login
def iter_torrents(self, pattern):
return self.search.go(pattern=pattern).iter_torrents()
@need_login
def get_torrent(self, id):
try:
self.torrent.go(torrent_id=id, torrent_name='anything', torrent_cat='any', torrent_subcat='thing')
torrent = self.page.get_torrent()
return torrent
except BrowserHTTPNotFound:
return
@need_login
def get_torrent_file(self, id):
torrent = self.browser.get_torrent(id)
if not torrent:
return None
resp = self.browser.open(torrent.url)
return resp.content
class CICBrowser(LoginBrowser):
PROFILE = Wget()
BASEURL = 'https://www.cic.fr'
login = URL('/sb/fr/banques/particuliers/index.html',
'/(?P<subbank>.*)/fr/$',
'/(?P<subbank>.*)/fr/banques/accueil.html',
'/(?P<subbank>.*)/fr/banques/particuliers/index.html',
LoginPage)
login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi',
LoginErrorPage)
accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi',
'/(?P<subbank>.*)/fr/banque/situation_financiere.html',
AccountsPage)
user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx',
UserSpacePage)
operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*',
'/(?P<subbank>.*)/fr/banque/mouvements.html.*',
'/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*',
OperationsPage)
coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage)
card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage)
noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage)
info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage)
transfert = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_vi.html',
EmptyPage)
transfert_2 = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_cmweb.aspx.*',
TransfertPage)
change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi',
ChangePasswordPage)
verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*',
VerifCodePage)
empty = URL(
'/(?P<subbank>.*)/fr/banques/index.html',
'/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*',
'/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*',
'/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx',
'/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage)
currentSubBank = None
__states__ = ['currentSubBank']
def do_login(self):
self.login.go()
if not self.page.logged:
self.page.login(self.username, self.password)
if not self.page.logged or self.login_error.is_here():
raise BrowserIncorrectPassword()
self.getCurrentSubBank()
@need_login
def get_accounts_list(self):
return self.accounts.stay_or_go(
subbank=self.currentSubBank).iter_accounts()
def get_account(self, id):
assert isinstance(id, basestring)
for a in self.get_accounts_list():
if a.id == id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
url = urlparse(self.url)
self.currentSubBank = url.path.lstrip('/').split('/')[0]
def list_operations(self, page_url):
if page_url.startswith('/') or page_url.startswith('https'):
self.location(page_url)
else:
self.location('%s/%s/fr/banque/%s' %
(self.BASEURL, self.currentSubBank, page_url))
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_history(self, account):
transactions = []
last_debit = None
for tr in self.list_operations(account._link_id):
# to prevent redundancy with card transactions, we do not
# store 'RELEVE CARTE' transaction.
if tr.raw != 'RELEVE CARTE':
transactions.append(tr)
elif last_debit is None:
last_debit = (tr.date - timedelta(days=10)).month
coming_link = self.page.get_coming_link() if self.operations.is_here(
) else None
if coming_link is not None:
for tr in self.list_operations(coming_link):
transactions.append(tr)
month = 0
for card_link in account._card_links:
v = urlsplit(card_link)
args = dict(parse_qsl(v.query))
# useful with 12 -> 1
if int(args['mois']) < month:
month = month + 1
else:
month = int(args['mois'])
for tr in self.list_operations(card_link):
if month > last_debit:
tr._is_coming = True
transactions.append(tr)
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def transfer(self, account, to, amount, reason=None):
# access the transfer page
self.transfert.go(subbank=self.currentSubBank)
# fill the form
form = self.page.get_form(xpath="//form[@id='P:F']")
try:
form[
'data_input_indiceCompteADebiter'] = self.page.get_from_account_index(
account)
form[
'data_input_indiceCompteACrediter'] = self.page.get_to_account_index(
to)
except ValueError as e:
raise TransferError(e.message)
form['[t:dbt%3adouble;]data_input_montant_value_0_'] = '%s' % str(
amount).replace('.', ',')
if reason is not None:
form[
'[t:dbt%3astring;x(27)]data_input_libelleCompteDebite'] = reason
form[
'[t:dbt%3astring;x(31)]data_input_motifCompteCredite'] = reason
del form['_FID_GoCancel']
del form['_FID_DoValidate']
form['_FID_DoValidate.x'] = str(randint(3, 125))
form['_FID_DoValidate.y'] = str(randint(3, 22))
form.submit()
# look for known errors
content = self.page.get_unicode_content()
insufficient_amount_message = u'Le montant du virement doit être positif, veuillez le modifier'
maximum_allowed_balance_message = u'Montant maximum autorisé au débit pour ce compte'
if insufficient_amount_message in content:
raise TransferError('The amount you tried to transfer is too low.')
if maximum_allowed_balance_message in content:
raise TransferError(
'The maximum allowed balance for the target account has been / would be reached.'
)
# look for the known "all right" message
ready_for_transfer_message = u'Confirmer un virement entre vos comptes'
if ready_for_transfer_message not in content:
raise TransferError('The expected message "%s" was not found.' %
ready_for_transfer_message)
# submit the confirmation form
form = self.page.get_form(xpath="//form[@id='P:F']")
del form['_FID_DoConfirm']
form['_FID_DoConfirm.x'] = str(randint(3, 125))
form['_FID_DoConfirm.y'] = str(randint(3, 22))
submit_date = datetime.now()
form.submit()
# look for the known "everything went well" message
content = self.page.get_unicode_content()
transfer_ok_message = u'Votre virement a été exécuté'
if transfer_ok_message not in content:
raise TransferError('The expected message "%s" was not found.' %
transfer_ok_message)
# We now have to return a Transfer object
transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S'))
transfer.amount = amount
transfer.origin = account
transfer.recipient = to
transfer.date = submit_date
return transfer
class CreditMutuelBrowser(LoginBrowser, StatesMixin):
PROFILE = Wget()
STATE_DURATION = 15
TIMEOUT = 30
BASEURL = 'https://www.creditmutuel.fr'
login = URL('/fr/authentification.html', '/(?P<subbank>.*)fr/$',
'/(?P<subbank>.*)fr/banques/accueil.html',
'/(?P<subbank>.*)fr/banques/particuliers/index.html',
LoginPage)
login_error = URL('/(?P<subbank>.*)fr/identification/default.cgi',
LoginErrorPage)
accounts = URL('/(?P<subbank>.*)fr/banque/situation_financiere.cgi',
'/(?P<subbank>.*)fr/banque/situation_financiere.html',
AccountsPage)
revolving_loan_list = URL(
r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*',
RevolvingLoansList)
revolving_loan_details = URL(
r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*',
RevolvingLoanDetails)
user_space = URL(
'/(?P<subbank>.*)fr/banque/espace_personnel.aspx',
'/(?P<subbank>.*)fr/banque/accueil.cgi',
'/(?P<subbank>.*)fr/banque/DELG_Gestion',
'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx',
UserSpacePage)
card = URL(
'/(?P<subbank>.*)fr/banque/operations_carte.cgi.*',
'/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$',
'/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=',
CardPage)
operations = URL(
'/(?P<subbank>.*)fr/banque/mouvements.cgi.*',
'/(?P<subbank>.*)fr/banque/mouvements.html.*',
'/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*',
r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+',
OperationsPage)
coming = URL('/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage)
info = URL('/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage)
change_pass = URL('/(?P<subbank>.*)fr/validation/change_password.cgi',
'/fr/services/change_password.html', ChangePasswordPage)
verify_pass = URL('/(?P<subbank>.*)fr/validation/verif_code.cgi.*',
VerifCodePage)
new_home = URL('/(?P<subbank>.*)fr/banque/pageaccueil.html',
'/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage)
empty = URL(
'/(?P<subbank>.*)fr/banques/index.html',
'/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*',
'/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case).*',
EmptyPage)
por = URL('/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx',
'/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx', PorPage)
li = URL('/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne',
'/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*',
'/(?P<subbank>.*)fr/assurances/WI_ASS', '/fr/assurances/',
LIAccountsPage)
iban = URL('/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage)
new_accounts = URL('/(?P<subbank>.*)fr/banque/comptes-et-contrats.html',
NewAccountsPage)
new_operations = URL('/(?P<subbank>.*)fr/banque/mouvements.cgi',
'/fr/banque/nr/nr_devbooster.aspx.*',
'/(?P<subbank>.*)fr/banque/RE/aiguille.asp',
'/fr/banque/mouvements.html',
'/(?P<subbank>.*)fr/banque/consultation/operations',
OperationsPage)
advisor = URL(
'/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)',
'/(?P<subbank>.*)fr/infoclient/',
r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*',
AdvisorPage)
redirect = URL(
'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx',
RedirectPage)
cards_activity = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx',
CardsActivityPage)
cards_list = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*',
'/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr',
CardsListPage)
cards_ope = URL('/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper',
CardsOpePage)
internal_transfer = URL('/(?P<subbank>.*)fr/banque/virements/vplw_vi.html',
InternalTransferPage)
external_transfer = URL(
'/(?P<subbank>.*)fr/banque/virements/vplw_vee.html',
ExternalTransferPage)
recipients_list = URL('/(?P<subbank>.*)fr/banque/virements/vplw_bl.html',
RecipientsListPage)
currentSubBank = None
is_new_website = False
form = None
logged = None
__states__ = ['currentSubBank', 'form', 'logged']
accounts_list = None
def do_login(self):
# Clear cookies.
self.do_logout()
self.login.go()
if not self.page.logged:
self.page.login(self.username, self.password)
if not self.page.logged or self.login_error.is_here():
raise BrowserIncorrectPassword()
self.getCurrentSubBank()
@need_login
def get_accounts_list(self):
if not self.accounts_list:
if self.currentSubBank is None:
self.getCurrentSubBank()
self.accounts_list = []
# Handle cards on tiers page
self.cards_activity.go(subbank=self.currentSubBank)
companies = self.page.companies_link() if self.cards_activity.is_here() else \
[self.page] if self.is_new_website else []
for company in companies:
page = self.open(company).page if isinstance(
company, basestring) else company
self.accounts_list.extend([card for card in page.iter_cards()])
if not self.is_new_website:
for a in self.accounts.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(a)
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
else:
for a in self.new_accounts.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(a)
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
for acc in self.li.go(
subbank=self.currentSubBank).iter_li_accounts():
self.accounts_list.append(acc)
for acc in self.revolving_loan_list.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(acc)
return self.accounts_list
def get_account(self, id):
assert isinstance(id, basestring)
for a in self.get_accounts_list():
if a.id == id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
paths = urlparse(self.url).path.lstrip('/').split('/')
self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else ""
if paths[0] in ["fr", "mabanque"]:
self.is_new_website = True
def list_operations(self, page):
if isinstance(page, basestring):
if page.startswith('/') or page.startswith(
'https') or page.startswith('?'):
self.location(page)
else:
self.location('%s/%sfr/banque/%s' %
(self.BASEURL, self.currentSubBank, page))
else:
self.page = page
# getting about 6 months history on new website
if self.is_new_website and self.page:
try:
for x in range(0, 2):
form = self.page.get_form(
id="I1:fm",
submit='//input[@name="_FID_DoActivateSearch"]')
if x == 1:
form.update({
[k for k in form.keys() if "DateStart" in k][0]:
(datetime.now() -
relativedelta(months=7)).strftime('%d/%m/%Y'),
[k for k in form.keys() if "DateEnd" in k][0]:
datetime.now().strftime('%d/%m/%Y')
})
[
form.pop(k, None) for k in form.keys()
if "_FID_Do" in k and "DoSearch" not in k
]
form.submit()
except (IndexError, FormNotFound):
pass
while self.page:
try:
form = self.page.get_form(
'//*[@id="I1:fm"]',
submit='//input[@name="_FID_DoLoadMoreTransactions"]')
[
form.pop(k, None) for k in form.keys()
if "_FID_Do" in k and "LoadMore" not in k
]
form.submit()
except (IndexError, FormNotFound):
break
#sometime the browser can't go further
except ClientError as exc:
if exc.response.status_code == 413:
break
raise
if self.li.is_here():
return self.page.iter_history()
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_monthly_transactions(self, trs):
groups = [
list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date),
lambda tr: tr.date)
]
trs = []
for group in groups:
tr = FrenchTransaction()
tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date
tr.amount = -sum([t.amount for t in group])
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
@need_login
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here(
) else None
if coming_link is not None:
for tr in self.list_operations(coming_link):
transactions.append(tr)
differed_date = None
cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \
account._card_links if hasattr(account, '_card_links') else []
for card in cards:
card_trs = []
for tr in self.list_operations(card):
if hasattr(tr, '_differed_date') and (
not differed_date
or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \
differed_date.month <= tr.date.month and \
not hasattr(tr, '_is_manualsum')
transactions = sorted_transactions(transactions)
return transactions
@need_login
def get_investment(self, account):
if account._is_inv:
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.por.go(subbank=self.currentSubBank)
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
return iter([])
@need_login
def iter_recipients(self, origin_account):
# access the transfer page
self.internal_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
for recipient in self.page.iter_recipients(
origin_account=origin_account):
yield recipient
self.external_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
origin_account._external_recipients = set()
if self.page.has_transfer_categories():
for category in self.page.iter_categories():
self.page.go_on_category(category['index'])
self.page.IS_PRO_PAGE = True
for recipient in self.page.iter_recipients(
origin_account=origin_account,
category=category['name']):
yield recipient
else:
for recipient in self.page.iter_recipients(
origin_account=origin_account):
yield recipient
@need_login
def init_transfer(self, account, to, amount, reason=None):
if to.category != u'Interne':
self.external_transfer.go(subbank=self.currentSubBank)
else:
self.internal_transfer.go(subbank=self.currentSubBank)
if self.external_transfer.is_here(
) and self.page.has_transfer_categories():
for category in self.page.iter_categories():
if category['name'] == to.category:
self.page.go_on_category(category['index'])
break
self.page.IS_PRO_PAGE = True
self.page.RECIPIENT_STRING = 'data_input_indiceBen'
self.page.prepare_transfer(account, to, amount, reason)
return self.page.handle_response(account, to, amount, reason)
@need_login
def execute_transfer(self, transfer, **params):
form = self.page.get_form(
id='P:F',
submit='//input[@type="submit" and contains(@value, "Confirmer")]')
# For the moment, don't ask the user if he confirms the duplicate.
form['Bool:data_input_confirmationDoublon'] = 'true'
form.submit()
return self.page.create_transfer(transfer)
@need_login
def get_advisor(self):
advisor = None
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
if self.page.get_advisor_link():
advisor = self.page.get_advisor()
self.location(
self.page.get_advisor_link()).page.update_advisor(advisor)
else:
advisor = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_advisor()
link = self.page.get_agency()
if link:
link = link.replace(':443/', '/')
self.location(link)
self.page.update_advisor(advisor)
return iter([advisor]) if advisor else iter([])
@need_login
def get_profile(self):
if not self.is_new_website:
profile = self.accounts.stay_or_go(
subbank=self.currentSubBank).get_profile()
else:
profile = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_profile()
return profile
def get_recipient_object(self, recipient):
r = Recipient()
r.iban = recipient.iban
r.id = recipient.iban
r.label = recipient.label
r.category = recipient.category
# On credit mutuel recipients are immediatly available.
r.enabled_at = datetime.now().replace(microsecond=0)
r.currency = u'EUR'
r.bank_name = NotAvailable
return r
def continue_new_recipient(self, recipient, **params):
if u'Clé' in params:
self.page.post_code(params[u'Clé'])
self.page.add_recipient(recipient)
if self.page.bic_needed():
self.page.ask_bic(self.get_recipient_object(recipient))
self.page.ask_sms(self.get_recipient_object(recipient))
def send_sms(self, sms):
data = {}
for k, v in self.form.iteritems():
if k != 'url':
data[k] = v
data['otp_password'] = sms
data['_FID_DoConfirm.x'] = '1'
data['_FID_DoConfirm.y'] = '1'
data['global_backup_hidden_key'] = ''
self.location(self.form['url'], data=data)
def end_new_recipient(self, recipient, **params):
self.send_sms(params['code'])
self.form = None
self.page = None
self.logged = 0
return self.get_recipient_object(recipient)
def post_with_bic(self, recipient, **params):
data = {}
for k, v in self.form.iteritems():
if k != 'url':
data[k] = v
data['[t:dbt%3astring;x(11)]data_input_BIC'] = params[u'Bic']
self.location(self.form['url'], data=data)
self.page.ask_sms(self.get_recipient_object(recipient))
@need_login
def new_recipient(self, recipient, **params):
if self.currentSubBank is None:
self.getCurrentSubBank()
if 'Bic' in params:
return self.post_with_bic(recipient, **params)
if 'code' in params:
return self.end_new_recipient(recipient, **params)
if u'Clé' in params:
return self.continue_new_recipient(recipient, **params)
self.recipients_list.go(subbank=self.currentSubBank)
if self.page.has_list():
if recipient.category not in self.page.get_recipients_list():
raise AddRecipientError(
'Recipient category is not on the website available list.')
self.page.go_list(recipient.category)
self.page.go_to_add()
if self.verify_pass.is_here():
raise AddRecipientStep(
self.get_recipient_object(recipient),
Value(u'Clé', label=self.page.get_question()))
else:
return self.continue_new_recipient(recipient, **params)
class ImdbBrowser(PagesBrowser):
BASEURL = 'http://www.imdb.com'
PROFILE = Wget()
movie_crew = URL(r'/title/tt[0-9]*/fullcredits.*', MovieCrewPage)
release = URL(r'/title/tt[0-9]*/releaseinfo.*', ReleasePage)
bio = URL(r'/name/nm[0-9]*/bio.*', BiographyPage)
person = URL(r'/name/nm[0-9]*/*', PersonPage)
def iter_movies(self, pattern):
res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&tt=on',
params={'q': pattern})
jres = res.json()
htmlparser = HTMLParser()
for cat in ['title_popular', 'title_exact', 'title_approx']:
if cat in jres:
for m in jres[cat]:
tdesc = unicode(m['title_description'])
if '<a' in tdesc and '>' in tdesc:
short_description = u'%s %s' % (
tdesc.split('<')[0].strip(', '),
tdesc.split('>')[1].split('<')[0])
else:
short_description = tdesc.strip(', ')
movie = Movie(m['id'], htmlparser.unescape(m['title']))
movie.other_titles = NotLoaded
movie.release_date = NotLoaded
movie.duration = NotLoaded
movie.short_description = htmlparser.unescape(
short_description)
movie.pitch = NotLoaded
movie.country = NotLoaded
movie.note = NotLoaded
movie.roles = NotLoaded
movie.all_release_dates = NotLoaded
movie.thumbnail_url = NotLoaded
yield movie
def iter_persons(self, pattern):
res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&nm=on',
params={'q': pattern})
jres = res.json()
htmlparser = HTMLParser()
for cat in ['name_popular', 'name_exact', 'name_approx']:
if cat in jres:
for p in jres[cat]:
person = Person(p['id'],
htmlparser.unescape(unicode(p['name'])))
person.real_name = NotLoaded
person.birth_place = NotLoaded
person.birth_date = NotLoaded
person.death_date = NotLoaded
person.gender = NotLoaded
person.nationality = NotLoaded
person.short_biography = NotLoaded
person.short_description = htmlparser.unescape(
p['description'])
person.roles = NotLoaded
person.thumbnail_url = NotLoaded
yield person
def get_movie(self, id):
res = self.open(
'http://www.omdbapi.com/?apikey=b7c56eb5&i=%s&plot=full' % id)
if res is not None:
jres = res.json()
else:
return None
htmlparser = HTMLParser()
title = NotAvailable
duration = NotAvailable
release_date = NotAvailable
pitch = NotAvailable
country = NotAvailable
note = NotAvailable
short_description = NotAvailable
thumbnail_url = NotAvailable
other_titles = []
genres = []
roles = {}
if 'Title' not in jres:
return
title = htmlparser.unescape(unicode(jres['Title'].strip()))
if 'Poster' in jres:
thumbnail_url = unicode(jres['Poster'])
if 'Director' in jres:
short_description = unicode(jres['Director'])
if 'Genre' in jres:
for g in jres['Genre'].split(', '):
genres.append(g)
if 'Runtime' in jres:
m = re.search('(\d+?) min', jres['Runtime'])
if m:
duration = int(m.group(1))
if 'Released' in jres:
released_string = str(jres['Released'])
if released_string == 'N/A':
release_date = NotAvailable
else:
months = {
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}
for st in months:
released_string = released_string.replace(st, months[st])
release_date = datetime.strptime(released_string, '%d %m %Y')
if 'Country' in jres:
country = u''
for c in jres['Country'].split(', '):
country += '%s, ' % c
country = country[:-2]
if 'Plot' in jres:
pitch = unicode(jres['Plot'])
if 'imdbRating' in jres and 'imdbVotes' in jres:
note = u'%s/10 (%s votes)' % (jres['imdbRating'],
jres['imdbVotes'])
for r in ['Actors', 'Director', 'Writer']:
if '%s' % r in jres.keys():
roles['%s' % r] = [('N/A', e)
for e in jres['%s' % r].split(', ')]
movie = Movie(id, title)
movie.other_titles = other_titles
movie.release_date = release_date
movie.duration = duration
movie.genres = genres
movie.pitch = pitch
movie.country = country
movie.note = note
movie.roles = roles
movie.short_description = short_description
movie.all_release_dates = NotLoaded
movie.thumbnail_url = thumbnail_url
return movie
def get_person(self, id):
try:
self.location('http://www.imdb.com/name/%s' % id)
except BrowserHTTPNotFound:
return
assert self.person.is_here()
return self.page.get_person(id)
def get_person_biography(self, id):
self.location('http://www.imdb.com/name/%s/bio' % id)
assert self.bio.is_here()
return self.page.get_biography()
def iter_movie_persons(self, movie_id, role):
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.movie_crew.is_here()
for p in self.page.iter_persons(role):
yield p
def iter_person_movies(self, person_id, role):
self.location('http://www.imdb.com/name/%s' % person_id)
assert self.person.is_here()
return self.page.iter_movies(role)
def iter_person_movies_ids(self, person_id):
self.location('http://www.imdb.com/name/%s' % person_id)
assert self.person.is_here()
for movie in self.page.iter_movies_ids():
yield movie
def iter_movie_persons_ids(self, movie_id):
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.movie_crew.is_here()
for person in self.page.iter_persons_ids():
yield person
def get_movie_releases(self, id, country):
self.location('http://www.imdb.com/title/%s/releaseinfo' % id)
assert self.release.is_here()
return self.page.get_movie_releases(country)
class CreditMutuelBrowser(TwoFactorBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.creditmutuel.fr'
HAS_CREDENTIALS_ONLY = True
STATE_DURATION = 5
TWOFA_DURATION = 60 * 24 * 90
# connexion
login = URL(
r'/fr/authentification.html',
r'/(?P<subbank>.*)fr/$',
r'/(?P<subbank>.*)fr/banques/accueil.html',
r'/(?P<subbank>.*)fr/banques/particuliers/index.html',
LoginPage
)
login_error = URL(r'/(?P<subbank>.*)fr/identification/default.cgi', LoginErrorPage)
twofa_unabled_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', TwoFAUnabledPage)
mobile_confirmation = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', MobileConfirmationPage)
decoupled_state = URL(r'/fr/banque/async/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage)
cancel_decoupled = URL(r'/fr/banque/async/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled)
otp_validation_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', OtpValidationPage)
otp_blocked_error_page = URL(r'/(?P<subbank>.*)fr/banque/validation.aspx', OtpBlockedErrorPage)
fiscality = URL(r'/(?P<subbank>.*)fr/banque/residencefiscale.aspx', FiscalityConfirmationPage)
# accounts
accounts = URL(r'/(?P<subbank>.*)fr/banque/situation_financiere.cgi',
r'/(?P<subbank>.*)fr/banque/situation_financiere.html',
AccountsPage)
useless_page = URL(r'/(?P<subbank>.*)fr/banque/paci/defi-solidaire.html', UselessPage)
revolving_loan_list = URL(r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList)
revolving_loan_details = URL(r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails)
user_space = URL(r'/(?P<subbank>.*)fr/banque/espace_personnel.aspx',
r'/(?P<subbank>.*)fr/banque/accueil.cgi',
r'/(?P<subbank>.*)fr/banque/DELG_Gestion',
r'/(?P<subbank>.*)fr/banque/paci_engine/engine.aspx',
r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx',
UserSpacePage)
card = URL(r'/(?P<subbank>.*)fr/banque/operations_carte.cgi.*',
r'/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$',
r'/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=',
CardPage)
operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi.*',
r'/(?P<subbank>.*)fr/banque/mouvements.html.*',
r'/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*',
r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+',
OperationsPage)
# This loans_operations contains operation for some loans, but not all of them.
loans_operations = URL(r'/(?P<subbank>.*)fr/banque/gec9.aspx.*', LoansOperationsPage)
coming = URL(r'/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage)
info = URL(r'/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage)
change_pass = URL(r'/(?P<subbank>.*)fr/validation/change_password.cgi',
'/fr/services/change_password.html', ChangePasswordPage)
verify_pass = URL(r'/(?P<subbank>.*)fr/validation/verif_code.cgi.*',
r'/(?P<subbank>.*)fr/validation/lst_codes.cgi.*', VerifCodePage)
new_home = URL(r'/(?P<subbank>.*)fr/banque/pageaccueil.html',
r'/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage)
empty = URL(r'/(?P<subbank>.*)fr/banques/index.html',
r'/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*',
r'/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case|infos).*',
EmptyPage)
por = URL(r'/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx',
r'/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx',
PorPage)
por_action_needed = URL(r'/(?P<subbank>.*)fr/banque/ORDR_InfosGenerales.aspx', EmptyPage)
li = URL(r'/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne',
r'/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*',
r'/(?P<subbank>.*)fr/assurances/WI_ASS',
'/fr/assurances/', LIAccountsPage)
iban = URL(r'/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage)
new_accounts = URL(r'/(?P<subbank>.*)fr/banque/comptes-et-contrats.html', NewAccountsPage)
new_operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi',
r'/fr/banque/nr/nr_devbooster.aspx.*',
r'/(?P<subbank>.*)fr/banque/RE/aiguille(liste)?.asp',
'/fr/banque/mouvements.html',
r'/(?P<subbank>.*)fr/banque/consultation/operations', OperationsPage)
advisor = URL(r'/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)',
r'/(?P<subbank>.*)fr/infoclient/',
r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*',
AdvisorPage)
redirect = URL(r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage)
cards_activity = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage)
cards_list = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*',
r'/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr', CardsListPage)
cards_ope = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper', CardsOpePage)
cards_ope2 = URL('/(?P<subbank>.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2)
cards_hist_available = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards',
'/(?P<subbank>.*)fr/banque/PCS1_CARDFUNCTIONS.aspx', NewCardsListPage)
cards_hist_available2 = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx', NewCardsListPage)
internal_transfer = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_vi.html', InternalTransferPage)
external_transfer = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage)
recipients_list = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_bl.html', RecipientsListPage)
error = URL(r'/(?P<subbank>.*)validation/infos.cgi', ErrorPage)
subscription = URL(r'/(?P<subbank>.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage)
terms_and_conditions = URL(r'/(?P<subbank>.*)fr/banque/conditions-generales.html',
r'/(?P<subbank>.*)fr/banque/coordonnees_personnelles.aspx',
r'/(?P<subbank>.*)fr/banque/paci_engine/paci_wsd_pdta.aspx',
r'/(?P<subbank>.*)fr/banque/reglementation-dsp2.html', ConditionsPage)
currentSubBank = None
is_new_website = None
form = None
logged = None
need_clear_storage = None
accounts_list = None
def __init__(self, config, *args, **kwargs):
self.config = config
self.weboob = kwargs['weboob']
kwargs['username'] = self.config['login'].get()
kwargs['password'] = self.config['password'].get()
super(CreditMutuelBrowser, self).__init__(config, *args, **kwargs)
self.__states__ += (
'currentSubBank', 'form', 'logged', 'is_new_website',
'need_clear_storage', 'recipient_form',
'twofa_auth_state', 'polling_data', 'otp_data',
)
self.twofa_auth_state = {}
self.polling_data = {}
self.otp_data = {}
self.keep_session = None
self.recipient_form = None
self.AUTHENTICATION_METHODS = {
'resume': self.handle_polling,
'code': self.handle_sms,
}
def get_expire(self):
if self.twofa_auth_state:
expires = datetime.fromtimestamp(self.twofa_auth_state['expires']).isoformat()
return expires
return
def load_state(self, state):
# when add recipient fails, state can't be reloaded.
# If state is reloaded, there is this error message:
# "Navigation interdite - Merci de bien vouloir recommencer votre action."
if state.get('need_clear_storage'):
# only keep 'twofa_auth_state' state to avoid new 2FA
state = {'twofa_auth_state': state.get('twofa_auth_state')}
if state.get('polling_data') or state.get('recipient_form') or state.get('otp_data'):
# can't start on an url in the middle of a validation process
# or server will cancel it and launch another one
if 'url' in state:
state.pop('url')
# if state is empty (first login), it does nothing
super(CreditMutuelBrowser, self).load_state(state)
def finalize_twofa(self, twofa_data):
"""
Go to validated 2FA url. Before following redirection,
store 'auth_client_state' cookie to prove to server,
for a TWOFA_DURATION, that 2FA is already done.
"""
self.location(
twofa_data['final_url'],
data=twofa_data['final_url_params'],
allow_redirects=False
)
for cookie in self.session.cookies:
if cookie.name == 'auth_client_state':
# only present if 2FA is valid
self.twofa_auth_state['value'] = cookie.value # this is a token
self.twofa_auth_state['expires'] = cookie.expires # this is a timestamp
self.location(self.response.headers['Location'])
def handle_polling(self):
# 15' on website, we don't wait that much, but leave sufficient time for the user
timeout = time.time() + 600.00 # 15' on webview, need not to wait that much
while time.time() < timeout:
data = {'transactionId': self.polling_data['polling_id']}
self.decoupled_state.go(data=data)
decoupled_state = self.page.get_decoupled_state()
if decoupled_state == 'VALIDATED':
self.logger.info('AppValidation done, going to final_url')
self.finalize_twofa(self.polling_data)
self.polling_data = {}
return
elif decoupled_state in ('CANCELLED', 'NONE'):
self.polling_data = {}
raise AppValidationCancelled()
assert decoupled_state == 'PENDING', 'Unhandled polling state: "%s"' % decoupled_state
time.sleep(5) # every second on wbesite, need to slow that down
# manually cancel polling before website max duration for it
self.cancel_decoupled.go(data=data)
self.polling_data = {}
raise AppValidationExpired()
def check_otp_blocked(self):
# Too much wrong OTPs, locked down after total 3 wrong inputs
if self.otp_blocked_error_page.is_here():
error_msg = self.page.get_error_message()
raise BrowserUnavailable(error_msg)
def handle_sms(self):
self.otp_data['final_url_params']['otp_password'] = self.code
self.finalize_twofa(self.otp_data)
## cases where 2FA is not finalized
# Too much wrong OTPs, locked down after total 3 wrong inputs
self.check_otp_blocked()
# OTP is expired after 15', we end up on login page
if self.login.is_here():
raise BrowserIncorrectPassword("Le code de confirmation envoyé par SMS n'est plus utilisable")
# Wrong OTP leads to same form with error message, re-raise BrowserQuestion
elif self.otp_validation_page.is_here():
error_msg = self.page.get_error_message()
if 'erroné' not in error_msg:
raise BrowserUnavailable(error_msg)
else:
label = '%s %s' % (error_msg, self.page.get_message())
raise BrowserQuestion(Value('code', label=label))
self.otp_data = {}
def check_redirections(self):
self.logger.info('Checking redirections')
# MobileConfirmationPage or OtpValidationPage is coming but there is no request_information
location = self.response.headers.get('Location', '')
if 'validation.aspx' in location and not self.is_interactive:
self.check_interactive()
elif location:
self.location(location, allow_redirects=False)
def check_auth_methods(self):
if self.mobile_confirmation.is_here():
self.page.check_bypass()
if self.mobile_confirmation.is_here():
self.polling_data = self.page.get_polling_data()
assert self.polling_data, "Can't proceed to polling if no polling_data"
raise AppValidation(self.page.get_validation_msg())
if self.otp_validation_page.is_here():
self.otp_data = self.page.get_otp_data()
assert self.otp_data, "Can't proceed to SMS handling if no otp_data"
raise BrowserQuestion(Value('code', label=self.page.get_message()))
self.check_otp_blocked()
def init_login(self):
self.login.go()
# 2FA already done, if valid, login() redirects to home page
if self.twofa_auth_state:
self.session.cookies.set('auth_client_state', self.twofa_auth_state['value'])
self.page.login(self.username, self.password, redirect=True)
if not self.page.logged:
# 302 redirect to catch to know if polling
self.page.login(self.username, self.password)
self.check_redirections()
# for cic, there is two redirections
self.check_redirections()
if self.twofa_unabled_page.is_here():
raise ActionNeeded(self.page.get_error_msg())
# when people try to log in but there are on a sub site of creditmutuel
if not self.page and not self.url.startswith(self.BASEURL):
raise BrowserIncorrectPassword()
if self.login_error.is_here():
raise BrowserIncorrectPassword()
if self.verify_pass.is_here():
raise AuthMethodNotImplemented("L'identification renforcée avec la carte n'est pas supportée.")
self.check_auth_methods()
self.getCurrentSubBank()
def ownership_guesser(self):
profile = self.get_profile()
psu_names = profile.name.lower().split()
for account in self.accounts_list:
label = account.label.lower()
# We try to find "M ou Mme" or "Mlle XXX ou M XXXX" for example (non-exhaustive exemple list)
if re.search(r'.* ((m) ([\w].*|ou )?(m[ml]e)|(m[ml]e) ([\w].*|ou )(m) ).*', label):
account.ownership = AccountOwnership.CO_OWNER
# We check if the PSU firstname and lastname is in the account label
elif all(name in label.split() for name in psu_names):
account.ownership = AccountOwnership.OWNER
# Card Accounts should be set with the same ownership of their parents
for account in self.accounts_list:
if account.type == Account.TYPE_CARD and not empty(account.parent):
account.ownership = account.parent.ownership
@need_login
def get_accounts_list(self):
if not self.accounts_list:
if self.currentSubBank is None:
self.getCurrentSubBank()
self.two_cards_page = None
self.accounts_list = []
self.revolving_accounts = []
self.unavailablecards = []
self.cards_histo_available = []
self.cards_list =[]
self.cards_list2 =[]
# For some cards the validity information is only availaible on these 2 links
self.cards_hist_available.go(subbank=self.currentSubBank)
if self.cards_hist_available.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
if not self.cards_list:
self.cards_hist_available2.go(subbank=self.currentSubBank)
if self.cards_hist_available2.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available2
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(acc)
self.revolving_accounts.append(acc.label.lower())
# Handle cards on tiers page
self.cards_activity.go(subbank=self.currentSubBank)
companies = self.page.companies_link() if self.cards_activity.is_here() else \
[self.page] if self.is_new_website else []
for company in companies:
# We need to return to the main page to avoid navigation error
self.cards_activity.go(subbank=self.currentSubBank)
page = self.open(company).page if isinstance(company, basestring) else company
for card in page.iter_cards():
card2 = find_object(self.cards_list, id=card.id[:16])
if card2:
# In order to keep the id of the card from the old space, we exchange the following values
card._link_id = card2._link_id
card._parent_id = card2._parent_id
card.coming = card2.coming
card._referer = card2._referer
card._secondpage = card2._secondpage
self.accounts_list.remove(card2)
self.accounts_list.append(card)
self.cards_list2.append(card)
self.cards_list.extend(self.cards_list2)
# Populate accounts from old website
if not self.is_new_website:
self.logger.info('On old creditmutuel website')
self.accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list)
self.por.go(subbank=self.currentSubBank)
self.page.add_por_accounts(self.accounts_list)
# Populate accounts from new website
else:
self.new_accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list)
self.por.go(subbank=self.currentSubBank)
self.page.add_por_accounts(self.accounts_list)
self.li.go(subbank=self.currentSubBank)
self.accounts_list.extend(self.page.iter_li_accounts())
# This type of account is like a loan, for splitting payments in smaller amounts.
# Its history is irrelevant because money is debited from a checking account and
# the balance is not even correct, so ignore it.
excluded_label = ['etalis', 'valorisation totale']
accounts_by_id = {}
for acc in self.accounts_list:
if acc.label.lower() not in excluded_label:
accounts_by_id[acc.id] = acc
# Set the parent to loans and cards accounts
for acc in self.accounts_list:
if acc.type == Account.TYPE_CARD and not empty(getattr(acc, '_parent_id', None)):
acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable)
elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id:
acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable)
self.accounts_list = list(accounts_by_id.values())
if has_no_account and not self.accounts_list:
raise NoAccountsException(has_no_account)
self.ownership_guesser()
return self.accounts_list
def get_account(self, _id):
assert isinstance(_id, basestring)
for a in self.get_accounts_list():
if a.id == _id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
paths = urlparse(self.url).path.lstrip('/').split('/')
self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else ""
if self.currentSubBank and paths[0] == 'banqueprivee' and paths[1] == 'mabanque':
self.currentSubBank = 'banqueprivee/mabanque/'
if self.currentSubBank and paths[1] == "decouverte":
self.currentSubBank += paths[1] + "/"
if paths[0] in ["cmmabn", "fr", "mabanque", "banqueprivee"]:
self.is_new_website = True
def list_operations(self, page, account):
if isinstance(page, basestring):
if page.startswith('/') or page.startswith('https') or page.startswith('?'):
self.location(page)
else:
try:
self.location('%s/%sfr/banque/%s' % (self.BASEURL, self.currentSubBank, page))
except ServerError as e:
self.logger.warning('Page cannot be visited: %s/%sfr/banque/%s: %s', self.BASEURL, self.currentSubBank, page, e)
raise BrowserUnavailable()
else:
self.page = page
# On some savings accounts, the page lands on the contract tab, and we want the situation
if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label:
self.page.go_on_history_tab()
if self.li.is_here():
return self.page.iter_history()
if self.is_new_website and self.page:
try:
for page in range(1, 50):
# Need to reach the page with all transactions
if not self.page.has_more_operations():
break
form = self.page.get_form(id="I1:P:F")
form['_FID_DoLoadMoreTransactions'] = ''
form['_wxf2_pseq'] = page
form.submit()
# IndexError when form xpath returns [], StopIteration if next called on empty iterable
except (StopIteration, FormNotFound):
self.logger.warning('Could not get history on new website')
except IndexError:
# 6 months history is not available
pass
while self.page:
try:
# Submit form if their is more transactions to fetch
form = self.page.get_form(id="I1:fm")
if self.page.doc.xpath('boolean(//a[@class="ei_loadmorebtn"])'):
form['_FID_DoLoadMoreTransactions'] = ""
form.submit()
else:
break
except (IndexError, FormNotFound):
break
# Sometimes the browser can't go further
except ClientError as exc:
if exc.response.status_code == 413:
break
raise
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
@need_login
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem': '',
'_wxf2_cc': 'fr-FR',
'_wxf2_pmode': 'Normal',
'_wxf2_pseq': i,
'_wxf2_ptarget': 'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '',
'Data_ServiceListDatas_CurrentType': 'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)', self.url)
if m:
m = m.group(1)
self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
deferred_date = None
cards = ([page.select_card(account._card_number) for page in account._card_pages]
if hasattr(account, '_card_pages')
else account._card_links if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_deferred_date') and (not deferred_date or tr._deferred_date < deferred_date):
deferred_date = tr._deferred_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if deferred_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY
and deferred_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
@need_login
def get_investment(self, account):
if account._is_inv:
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.por.go(subbank=self.currentSubBank)
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
if account.type is Account.TYPE_PEA:
liquidities = create_french_liquidity(account.balance)
liquidities.label = account.label
return [liquidities]
return iter([])
@need_login
def iter_recipients(self, origin_account):
# access the transfer page
self.internal_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
for recipient in self.page.iter_recipients(origin_account=origin_account):
yield recipient
self.external_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
origin_account._external_recipients = set()
if self.page.has_transfer_categories():
for category in self.page.iter_categories():
self.page.go_on_category(category['index'])
self.page.IS_PRO_PAGE = True
for recipient in self.page.iter_recipients(origin_account=origin_account, category=category['name']):
yield recipient
else:
for recipient in self.page.iter_recipients(origin_account=origin_account):
yield recipient
@need_login
def init_transfer(self, account, to, amount, exec_date, reason=None):
if to.category != 'Interne':
self.external_transfer.go(subbank=self.currentSubBank)
else:
self.internal_transfer.go(subbank=self.currentSubBank)
if self.external_transfer.is_here() and self.page.has_transfer_categories():
for category in self.page.iter_categories():
if category['name'] == to.category:
self.page.go_on_category(category['index'])
break
self.page.IS_PRO_PAGE = True
self.page.RECIPIENT_STRING = 'data_input_indiceBen'
self.page.prepare_transfer(account, to, amount, reason, exec_date)
return self.page.handle_response(account, to, amount, reason, exec_date)
@need_login
def execute_transfer(self, transfer, **params):
form = self.page.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Confirmer")]')
# For the moment, don't ask the user if he confirms the duplicate.
form['Bool:data_input_confirmationDoublon'] = 'true'
form.submit()
return self.page.create_transfer(transfer)
@need_login
def get_advisor(self):
advisor = None
if not self.is_new_website:
self.logger.info('On old creditmutuel website')
self.accounts.stay_or_go(subbank=self.currentSubBank)
if self.page.get_advisor_link():
advisor = self.page.get_advisor()
self.location(self.page.get_advisor_link()).page.update_advisor(advisor)
else:
advisor = self.new_accounts.stay_or_go(subbank=self.currentSubBank).get_advisor()
link = self.page.get_agency()
if link:
link = link.replace(':443/', '/')
self.location(link)
self.page.update_advisor(advisor)
return iter([advisor]) if advisor else iter([])
@need_login
def get_profile(self):
if not self.is_new_website:
self.logger.info('On old creditmutuel website')
profile = self.accounts.stay_or_go(subbank=self.currentSubBank).get_profile()
else:
profile = self.new_accounts.stay_or_go(subbank=self.currentSubBank).get_profile()
return profile
def get_recipient_object(self, recipient):
r = Recipient()
r.iban = recipient.iban
r.id = recipient.iban
r.label = recipient.label
r.category = recipient.category
# On credit mutuel recipients are immediatly available.
r.enabled_at = datetime.now().replace(microsecond=0)
r.currency = 'EUR'
r.bank_name = NotAvailable
return r
def format_recipient_form(self, key):
self.recipient_form['[t:xsd%3astring;]Data_KeyInput'] = key
# we don't know the card id
# by default all users have only one card
# but to be sure, let's get it dynamically
do_validate = [k for k in self.recipient_form.keys() if '_FID_DoValidate_cardId' in k]
assert len(do_validate) == 1, 'There should be only one card.'
self.recipient_form[do_validate[0]] = ''
activate = [k for k in self.recipient_form.keys() if '_FID_GoCardAction_action' in k]
for _ in activate:
del self.recipient_form[_]
def continue_new_recipient(self, recipient, **params):
if 'Clé' in params:
url = self.recipient_form.pop('url')
self.format_recipient_form(params['Clé'])
self.location(url, data=self.recipient_form)
self.recipient_form = None
if self.verify_pass.is_here():
self.page.handle_error()
assert False, 'An error occured while checking the card code'
self.page.add_recipient(recipient)
if self.page.bic_needed():
self.page.ask_bic(self.get_recipient_object(recipient))
self.page.ask_sms(self.get_recipient_object(recipient))
def send_sms(self, sms):
data = {}
for k, v in self.form.items():
if k != 'url':
data[k] = v
data['otp_password'] = sms
data['_FID_DoConfirm.x'] = '1'
data['_FID_DoConfirm.y'] = '1'
data['global_backup_hidden_key'] = ''
self.location(self.form['url'], data=data)
def end_new_recipient(self, recipient, **params):
self.send_sms(params['code'])
self.form = None
self.page = None
self.logged = 0
return self.get_recipient_object(recipient)
def post_with_bic(self, recipient, **params):
data = {}
for k, v in self.form.items():
if k != 'url':
data[k] = v
data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic']
self.location(self.form['url'], data=data)
self.page.ask_sms(self.get_recipient_object(recipient))
def set_new_recipient(self, recipient, **params):
if self.currentSubBank is None:
self.getCurrentSubBank()
if 'Bic' in params:
return self.post_with_bic(recipient, **params)
if 'code' in params:
return self.end_new_recipient(recipient, **params)
if 'Clé' in params:
return self.continue_new_recipient(recipient, **params)
assert False, 'An error occured while adding a recipient.'
@need_login
def new_recipient(self, recipient, **params):
if self.currentSubBank is None:
self.getCurrentSubBank()
self.recipients_list.go(subbank=self.currentSubBank)
if self.page.has_list():
assert recipient.category in self.page.get_recipients_list(), \
'Recipient category is not on the website available list.'
self.page.go_list(recipient.category)
self.page.go_to_add()
if self.verify_pass.is_here():
self.recipient_form = self.page.get_recipient_form()
raise AddRecipientStep(self.get_recipient_object(recipient), Value('Clé', label=self.page.get_question()))
else:
return self.continue_new_recipient(recipient, **params)
@need_login
def iter_subscriptions(self):
if self.currentSubBank is None:
self.getCurrentSubBank()
self.subscription.go(subbank=self.currentSubBank)
return self.page.iter_subscriptions()
@need_login
def iter_documents(self, subscription):
if self.currentSubBank is None:
self.getCurrentSubBank()
self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'})
security_limit = 10
for i in range(security_limit):
for doc in self.page.iter_documents(sub_id=subscription.id):
yield doc
if self.page.is_last_page():
break
self.page.next_page()
class CreditMutuelBrowser(LoginBrowser):
PROFILE = Wget()
TIMEOUT = 30
BASEURL = 'https://www.creditmutuel.fr'
login = URL('/groupe/fr/index.html', '/(?P<subbank>.*)/fr/$',
'/(?P<subbank>.*)/fr/banques/accueil.html',
'/(?P<subbank>.*)/fr/banques/particuliers/index.html',
LoginPage)
login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi',
LoginErrorPage)
accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi',
'/(?P<subbank>.*)/fr/banque/situation_financiere.html',
AccountsPage)
user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx',
UserSpacePage)
operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*',
'/(?P<subbank>.*)/fr/banque/mouvements.html.*',
'/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*',
OperationsPage)
coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage)
card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage)
noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage)
info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage)
transfert = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_vi.html',
EmptyPage)
transfert_2 = URL('/(?P<subbank>.*)/fr/banque/virements/vplw_cmweb.aspx.*',
TransfertPage)
change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi',
ChangePasswordPage)
verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*',
VerifCodePage)
empty = URL(
'/(?P<subbank>.*)/fr/banques/index.html',
'/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*',
'/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*',
'/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx',
'/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage)
por = URL('/(?P<subbank>.*)/fr/banque/POR_ValoToute.aspx',
'/(?P<subbank>.*)/fr/banque/POR_SyntheseLst.aspx', PorPage)
li = URL('/(?P<subbank>.*)/fr/assurances/profilass.aspx\?domaine=epargne',
'/(?P<subbank>.*)/fr/assurances/consultation/WI_ASSAVI',
LIAccountsPage)
iban = URL('/(?P<subbank>.*)/fr/banque/rib.cgi', IbanPage)
new_home = URL('/fr/banque/pageaccueil.html',
'/fr/banque/welcome_pack.html', NewHomePage)
new_accounts = URL('/fr/banque/comptes-et-contrats.html', AccountsPage)
new_operations = URL('/fr/banque/mouvements.cgi',
'/fr/banque/mouvements.html', OperationsPage)
new_por = URL('/fr/banque/POR_ValoToute.aspx',
'/fr/banque/POR_SyntheseLst.aspx', PorPage)
new_iban = URL('/fr/banque/rib.cgi', IbanPage)
redirect = URL('/fr/banque/paci_engine/static_content_manager.aspx',
RedirectPage)
currentSubBank = None
is_new_website = False
__states__ = ['currentSubBank']
def do_login(self):
# Clear cookies.
self.do_logout()
self.login.go()
if not self.page.logged:
self.page.login(self.username, self.password)
if not self.page.logged or self.login_error.is_here():
raise BrowserIncorrectPassword()
if not self.is_new_website:
self.getCurrentSubBank()
@need_login
def get_accounts_list(self):
if self.currentSubBank is None and not self.is_new_website:
self.getCurrentSubBank()
accounts = []
if not self.is_new_website:
for a in self.accounts.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
accounts.append(a)
self.iban.go(subbank=self.currentSubBank).fill_iban(accounts)
self.por.go(subbank=self.currentSubBank).add_por_accounts(accounts)
for acc in self.li.go(
subbank=self.currentSubBank).iter_li_accounts():
accounts.append(acc)
else:
for a in self.new_accounts.stay_or_go().iter_accounts():
accounts.append(a)
self.new_iban.go().fill_iban(accounts)
self.new_por.go().add_por_accounts(accounts)
return accounts
def get_account(self, id):
assert isinstance(id, basestring)
for a in self.get_accounts_list():
if a.id == id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
url = urlparse(self.url)
self.currentSubBank = url.path.lstrip('/').split('/')[0]
def list_operations(self, page_url):
if page_url.startswith('/') or page_url.startswith('https'):
self.location(page_url)
elif not self.is_new_website:
self.location('%s/%s/fr/banque/%s' %
(self.BASEURL, self.currentSubBank, page_url))
else:
self.location('%s/fr/banque/%s' % (self.BASEURL, page_url))
if self.li.is_here():
return self.page.iter_history()
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_history(self, account):
transactions = []
last_debit = None
if not account._link_id:
return iter([])
for tr in self.list_operations(account._link_id):
# to prevent redundancy with card transactions, we do not
# store 'RELEVE CARTE' transaction.
if not tr.raw.startswith('RELEVE CARTE'):
transactions.append(tr)
elif last_debit is None:
# we set the debit date to last day of month so we need to do the same form last_debit
last_debit = (tr.date + relativedelta(day=31))
coming_link = self.page.get_coming_link() if self.operations.is_here(
) else None
if coming_link is not None:
for tr in self.list_operations(coming_link):
transactions.append(tr)
for card_link in account._card_links:
for tr in self.list_operations(card_link):
if last_debit is None or tr.date > last_debit:
tr._is_coming = True
transactions.append(tr)
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def get_investment(self, account):
if account._is_inv:
if account.type == Account.TYPE_MARKET:
if not self.is_new_website:
self.por.go(subbank=self.currentSubBank)
else:
self.new_por.go()
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
return iter([])
def transfer(self, account, to, amount, reason=None):
if self.is_new_website:
raise NotImplementedError()
# access the transfer page
self.transfert.go(subbank=self.currentSubBank)
# fill the form
form = self.page.get_form(xpath="//form[@id='P:F']")
try:
form[
'data_input_indiceCompteADebiter'] = self.page.get_from_account_index(
account)
form[
'data_input_indiceCompteACrediter'] = self.page.get_to_account_index(
to)
except ValueError as e:
raise TransferError(e.message)
form['[t:dbt%3adouble;]data_input_montant_value_0_'] = '%s' % str(
amount).replace('.', ',')
if reason is not None:
form[
'[t:dbt%3astring;x(27)]data_input_libelleCompteDebite'] = reason
form[
'[t:dbt%3astring;x(31)]data_input_motifCompteCredite'] = reason
del form['_FID_GoCancel']
del form['_FID_DoValidate']
form['_FID_DoValidate.x'] = str(randint(3, 125))
form['_FID_DoValidate.y'] = str(randint(3, 22))
form.submit()
# look for known errors
content = self.page.get_unicode_content()
insufficient_amount_message = u'Le montant du virement doit être positif, veuillez le modifier'
maximum_allowed_balance_message = u'Montant maximum autorisé au débit pour ce compte'
if insufficient_amount_message in content:
raise TransferError('The amount you tried to transfer is too low.')
if maximum_allowed_balance_message in content:
raise TransferError(
'The maximum allowed balance for the target account has been / would be reached.'
)
# look for the known "all right" message
ready_for_transfer_message = u'Confirmer un virement entre vos comptes'
if ready_for_transfer_message not in content:
raise TransferError('The expected message "%s" was not found.' %
ready_for_transfer_message)
# submit the confirmation form
form = self.page.get_form(xpath="//form[@id='P:F']")
del form['_FID_DoConfirm']
form['_FID_DoConfirm.x'] = str(randint(3, 125))
form['_FID_DoConfirm.y'] = str(randint(3, 22))
submit_date = datetime.now()
form.submit()
# look for the known "everything went well" message
content = self.page.get_unicode_content()
transfer_ok_message = u'Votre virement a été exécuté'
if transfer_ok_message not in content:
raise TransferError('The expected message "%s" was not found.' %
transfer_ok_message)
# We now have to return a Transfer object
transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S'))
transfer.amount = amount
transfer.origin = account
transfer.recipient = to
transfer.date = submit_date
return transfer
class CreditMutuelBrowser(LoginBrowser):
PROFILE = Wget()
BASEURL = 'https://www.creditmutuel.fr'
login = URL('/groupe/fr/index.html', LoginPage)
login_error = URL('/(?P<subbank>.*)/fr/identification/default.cgi',
LoginErrorPage)
accounts = URL('/(?P<subbank>.*)/fr/banque/situation_financiere.cgi',
AccountsPage)
user_space = URL('/(?P<subbank>.*)/fr/banque/espace_personnel.aspx',
UserSpacePage)
operations = URL('/(?P<subbank>.*)/fr/banque/mouvements.cgi.*',
'/(?P<subbank>.*)/fr/banque/nr/nr_devbooster.aspx.*',
OperationsPage)
coming = URL('/(?P<subbank>.*)/fr/banque/mvts_instance.cgi.*', ComingPage)
card = URL('/(?P<subbank>.*)/fr/banque/operations_carte.cgi.*', CardPage)
noop = URL('/(?P<subbank>.*)/fr/banque/CR/arrivee.asp.*', NoOperationsPage)
info = URL('/(?P<subbank>.*)/fr/banque/BAD.*', EmptyPage)
transfert = URL(
'/(?P<subbank>.*)/fr/banque/WI_VPLV_VirUniSaiCpt.asp\?(?P<parameters>.*)',
TransfertPage)
change_pass = URL('/(?P<subbank>.*)/fr/validation/change_password.cgi',
ChangePasswordPage)
verify_pass = URL('/(?P<subbank>.*)/fr/validation/verif_code.cgi.*',
VerifCodePage)
empty = URL(
'/(?P<subbank>.*)/fr/$', '/(?P<subbank>.*)/fr/banques/index.html',
'/(?P<subbank>.*)/fr/banque/paci_beware_of_phishing.*',
'/(?P<subbank>.*)/fr/validation/(?!change_password|verif_code).*',
'/(?P<subbank>.*)/fr/banque/paci_engine/static_content_manager.aspx',
'/(?P<subbank>.*)/fr/banque/DELG_Gestion.*', EmptyPage)
currentSubBank = None
def do_login(self):
self.login.stay_or_go()
self.page.login(self.username, self.password)
if not self.page.logged or self.login_error.is_here():
raise BrowserIncorrectPassword()
self.getCurrentSubBank()
@need_login
def get_accounts_list(self):
return self.accounts.stay_or_go(
subbank=self.currentSubBank).iter_accounts()
def get_account(self, id):
assert isinstance(id, basestring)
for a in self.get_accounts_list():
if a.id == id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
url = urlparse(self.url)
self.currentSubBank = url.path.lstrip('/').split('/')[0]
def list_operations(self, page_url):
if page_url.startswith('/'):
self.location(page_url)
else:
self.location('%s/%s/fr/banque/%s' %
(self.BASEURL, self.currentSubBank, page_url))
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_history(self, account):
transactions = []
last_debit = None
for tr in self.list_operations(account._link_id):
# to prevent redundancy with card transactions, we do not
# store 'RELEVE CARTE' transaction.
if tr.raw != 'RELEVE CARTE':
transactions.append(tr)
elif last_debit is None:
last_debit = (tr.date - timedelta(days=10)).month
coming_link = self.page.get_coming_link() if self.operations.is_here(
) else None
if coming_link is not None:
for tr in self.list_operations(coming_link):
transactions.append(tr)
month = 0
for card_link in account._card_links:
v = urlsplit(card_link)
args = dict(parse_qsl(v.query))
# useful with 12 -> 1
if int(args['mois']) < month:
month += 1
else:
month = int(args['mois'])
for tr in self.list_operations(card_link):
if month > last_debit:
tr._is_coming = True
transactions.append(tr)
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def transfer(self, account, to, amount, reason=None):
# access the transfer page
parameters = 'RAZ=ALL&Cat=6&PERM=N&CHX=A'
page = self.transfert.go(subbank=self.currentSubBank,
parameters=parameters)
# fill the form
form = self.page.get_form(name='FormVirUniSaiCpt')
form['IDB'] = account[-1]
form['ICR'] = to[-1]
form['MTTVIR'] = '%s' % str(amount).replace('.', ',')
if reason is not None:
form['LIBDBT'] = reason
form['LIBCRT'] = reason
page = form.submit()
# look for known errors
content = page.response.text
insufficient_amount_message = u'Montant insuffisant.'
maximum_allowed_balance_message = u'Solde maximum autorisé dépassé.'
if insufficient_amount_message in content:
raise TransferError('The amount you tried to transfer is too low.')
if maximum_allowed_balance_message in content:
raise TransferError(
'The maximum allowed balance for the target account has been / would be reached.'
)
# look for the known "all right" message
ready_for_transfer_message = u'Confirmez un virement entre vos comptes'
if ready_for_transfer_message in content:
raise TransferError('The expected message "%s" was not found.' %
ready_for_transfer_message)
# submit the confirmation form
form = page.get_form(name='FormVirUniCnf')
submit_date = datetime.now()
page = form.submit()
# look for the known "everything went well" message
content = page.response.text
transfer_ok_message = u'Votre virement a été exécuté ce jour'
if transfer_ok_message not in content:
raise TransferError('The expected message "%s" was not found.' %
transfer_ok_message)
# We now have to return a Transfer object
transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S'))
transfer.amount = amount
transfer.origin = account
transfer.recipient = to
transfer.date = submit_date
return transfer
class CreditMutuelBrowser(LoginBrowser, StatesMixin):
PROFILE = Wget()
STATE_DURATION = 10
TIMEOUT = 30
BASEURL = 'https://www.creditmutuel.fr'
login = URL('/fr/authentification.html', r'/(?P<subbank>.*)fr/$',
r'/(?P<subbank>.*)fr/banques/accueil.html',
r'/(?P<subbank>.*)fr/banques/particuliers/index.html',
LoginPage)
login_error = URL(r'/(?P<subbank>.*)fr/identification/default.cgi',
LoginErrorPage)
accounts = URL(r'/(?P<subbank>.*)fr/banque/situation_financiere.cgi',
r'/(?P<subbank>.*)fr/banque/situation_financiere.html',
AccountsPage)
revolving_loan_list = URL(
r'/(?P<subbank>.*)fr/banque/CR/arrivee.asp\?fam=CR.*',
RevolvingLoansList)
revolving_loan_details = URL(
r'/(?P<subbank>.*)fr/banque/CR/cam9_vis_lstcpt.asp.*',
RevolvingLoanDetails)
user_space = URL(
r'/(?P<subbank>.*)fr/banque/espace_personnel.aspx',
r'/(?P<subbank>.*)fr/banque/accueil.cgi',
r'/(?P<subbank>.*)fr/banque/DELG_Gestion',
r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx',
UserSpacePage)
card = URL(
r'/(?P<subbank>.*)fr/banque/operations_carte.cgi.*',
r'/(?P<subbank>.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$',
r'/(?P<subbank>.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=',
CardPage)
operations = URL(
r'/(?P<subbank>.*)fr/banque/mouvements.cgi.*',
r'/(?P<subbank>.*)fr/banque/mouvements.html.*',
r'/(?P<subbank>.*)fr/banque/nr/nr_devbooster.aspx.*',
r'(?P<subbank>.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+',
OperationsPage)
coming = URL(r'/(?P<subbank>.*)fr/banque/mvts_instance.cgi.*', ComingPage)
info = URL(r'/(?P<subbank>.*)fr/banque/BAD.*', EmptyPage)
change_pass = URL(r'/(?P<subbank>.*)fr/validation/change_password.cgi',
'/fr/services/change_password.html', ChangePasswordPage)
verify_pass = URL(r'/(?P<subbank>.*)fr/validation/verif_code.cgi.*',
r'/(?P<subbank>.*)fr/validation/lst_codes.cgi.*',
VerifCodePage)
new_home = URL(r'/(?P<subbank>.*)fr/banque/pageaccueil.html',
r'/(?P<subbank>.*)banque/welcome_pack.html', NewHomePage)
empty = URL(
r'/(?P<subbank>.*)fr/banques/index.html',
r'/(?P<subbank>.*)fr/banque/paci_beware_of_phishing.*',
r'/(?P<subbank>.*)fr/validation/(?!change_password|verif_code|image_case|infos).*',
EmptyPage)
por = URL(r'/(?P<subbank>.*)fr/banque/POR_ValoToute.aspx',
r'/(?P<subbank>.*)fr/banque/POR_SyntheseLst.aspx', PorPage)
li = URL(r'/(?P<subbank>.*)fr/assurances/profilass.aspx\?domaine=epargne',
r'/(?P<subbank>.*)fr/assurances/(consultations?/)?WI_ASS.*',
r'/(?P<subbank>.*)fr/assurances/WI_ASS', '/fr/assurances/',
LIAccountsPage)
iban = URL(r'/(?P<subbank>.*)fr/banque/rib.cgi', IbanPage)
new_accounts = URL(r'/(?P<subbank>.*)fr/banque/comptes-et-contrats.html',
NewAccountsPage)
new_operations = URL(r'/(?P<subbank>.*)fr/banque/mouvements.cgi',
r'/fr/banque/nr/nr_devbooster.aspx.*',
r'/(?P<subbank>.*)fr/banque/RE/aiguille(liste)?.asp',
'/fr/banque/mouvements.html',
r'/(?P<subbank>.*)fr/banque/consultation/operations',
OperationsPage)
advisor = URL(
r'/(?P<subbank>.*)fr/banques/contact/trouver-une-agence/(?P<page>.*)',
r'/(?P<subbank>.*)fr/infoclient/',
r'/(?P<subbank>.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*',
AdvisorPage)
redirect = URL(
r'/(?P<subbank>.*)fr/banque/paci_engine/static_content_manager.aspx',
RedirectPage)
cards_activity = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_tiers.aspx',
CardsActivityPage)
cards_list = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_ctr.*',
r'/(?P<subbank>.*)fr/banque/pro/ENC_detail_ctr',
CardsListPage)
cards_ope = URL(r'/(?P<subbank>.*)fr/banque/pro/ENC_liste_oper',
CardsOpePage)
cards_ope2 = URL('/(?P<subbank>.*)fr/banque/CRP8_SCIM_DEPCAR.aspx',
CardPage2)
cards_hist_available = URL(
'/(?P<subbank>.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards',
'/(?P<subbank>.*)fr/banque/PCS1_CARDFUNCTIONS.aspx', NewCardsListPage)
cards_hist_available2 = URL('/(?P<subbank>.*)fr/banque/SCIM_default.aspx',
NewCardsListPage)
internal_transfer = URL(
r'/(?P<subbank>.*)fr/banque/virements/vplw_vi.html',
InternalTransferPage)
external_transfer = URL(
r'/(?P<subbank>.*)fr/banque/virements/vplw_vee.html',
ExternalTransferPage)
recipients_list = URL(r'/(?P<subbank>.*)fr/banque/virements/vplw_bl.html',
RecipientsListPage)
error = URL(r'/(?P<subbank>.*)validation/infos.cgi', ErrorPage)
subscription = URL(r'/(?P<subbank>.*)fr/banque/MMU2_LstDoc.aspx',
SubscriptionPage)
terms_and_conditions = URL(
r'/(?P<subbank>.*)fr/banque/conditions-generales.html',
r'/(?P<subbank>.*)fr/banque/coordonnees_personnelles.aspx',
r'/(?P<subbank>.*)fr/banque/paci_engine/paci_wsd_pdta.aspx',
r'/(?P<subbank>.*)fr/banque/reglementation-dsp2.html', ConditionsPage)
currentSubBank = None
is_new_website = None
form = None
logged = None
need_clear_storage = None
__states__ = [
'currentSubBank', 'form', 'logged', 'is_new_website',
'need_clear_storage'
]
accounts_list = None
def load_state(self, state):
# when add recipient fails, state can't be reloaded. If state is reloaded, there is this error message:
# "Navigation interdite - Merci de bien vouloir recommencer votre action."
if not state.get('need_clear_storage'):
super(CreditMutuelBrowser, self).load_state(state)
else:
self.need_clear_storage = None
def do_login(self):
# Clear cookies.
self.do_logout()
self.login.go()
if not self.page.logged:
self.page.login(self.username, self.password)
# when people try to log in but there are on a sub site of creditmutuel
if not self.page and not self.url.startswith(self.BASEURL):
raise BrowserIncorrectPassword()
if not self.page.logged or self.login_error.is_here():
raise BrowserIncorrectPassword()
if self.verify_pass.is_here():
raise AuthMethodNotImplemented(
"L'identification renforcée avec la carte n'est pas supportée."
)
self.getCurrentSubBank()
def ownership_guesser(self):
profile = self.get_profile()
psu_names = profile.name.lower().split()
for account in self.accounts_list:
label = account.label.lower()
# We try to find "M ou Mme" or "Mlle XXX ou M XXXX" for example (non-exhaustive exemple list)
if re.search(
r'.* ((m) ([\w].*|ou )?(m[ml]e)|(m[ml]e) ([\w].*|ou )(m) ).*',
label):
account.ownership = AccountOwnership.CO_OWNER
# We check if the PSU firstname and lastname is in the account label
elif all(name in label.split() for name in psu_names):
account.ownership = AccountOwnership.OWNER
# Card Accounts should be set with the same ownership of their parents
for account in self.accounts_list:
if account.type == Account.TYPE_CARD and not empty(account.parent):
account.ownership = account.parent.ownership
@need_login
def get_accounts_list(self):
if not self.accounts_list:
if self.currentSubBank is None:
self.getCurrentSubBank()
self.two_cards_page = None
self.accounts_list = []
self.revolving_accounts = []
self.unavailablecards = []
self.cards_histo_available = []
self.cards_list = []
self.cards_list2 = []
# For some cards the validity information is only availaible on these 2 links
self.cards_hist_available.go(subbank=self.currentSubBank)
if self.cards_hist_available.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
if not self.cards_list:
self.cards_hist_available2.go(subbank=self.currentSubBank)
if self.cards_hist_available2.is_here():
self.unavailablecards.extend(
self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available2
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
for acc in self.revolving_loan_list.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(acc)
self.revolving_accounts.append(acc.label.lower())
# Handle cards on tiers page
self.cards_activity.go(subbank=self.currentSubBank)
companies = self.page.companies_link() if self.cards_activity.is_here() else \
[self.page] if self.is_new_website else []
for company in companies:
# We need to return to the main page to avoid navigation error
self.cards_activity.go(subbank=self.currentSubBank)
page = self.open(company).page if isinstance(
company, basestring) else company
for card in page.iter_cards():
card2 = find_object(self.cards_list, id=card.id[:16])
if card2:
# In order to keep the id of the card from the old space, we exchange the following values
card._link_id = card2._link_id
card._parent_id = card2._parent_id
card.coming = card2.coming
card._referer = card2._referer
card._secondpage = card2._secondpage
self.accounts_list.remove(card2)
self.accounts_list.append(card)
self.cards_list2.append(card)
self.cards_list.extend(self.cards_list2)
# Populate accounts from old website
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
# Populate accounts from new website
else:
self.new_accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
self.li.go(subbank=self.currentSubBank)
self.accounts_list.extend(self.page.iter_li_accounts())
for acc in self.cards_list:
if hasattr(acc, '_parent_id'):
acc.parent = find_object(self.accounts_list,
id=acc._parent_id)
excluded_label = ['etalis', 'valorisation totale']
self.accounts_list = [
acc for acc in self.accounts_list
if not any(w in acc.label.lower() for w in excluded_label)
]
if has_no_account and not self.accounts_list:
raise NoAccountsException(has_no_account)
self.ownership_guesser()
return self.accounts_list
def get_account(self, _id):
assert isinstance(_id, basestring)
for a in self.get_accounts_list():
if a.id == _id:
return a
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
paths = urlparse(self.url).path.lstrip('/').split('/')
self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else ""
if self.currentSubBank and paths[0] == 'banqueprivee' and paths[
1] == 'mabanque':
self.currentSubBank = 'banqueprivee/mabanque/'
if self.currentSubBank and paths[1] == "decouverte":
self.currentSubBank += paths[1] + "/"
if paths[0] in ["fr", "mabanque", "banqueprivee"]:
self.is_new_website = True
def list_operations(self, page, account):
if isinstance(page, basestring):
if page.startswith('/') or page.startswith(
'https') or page.startswith('?'):
self.location(page)
else:
try:
self.location('%s/%sfr/banque/%s' %
(self.BASEURL, self.currentSubBank, page))
except ServerError as e:
self.logger.warning(
'Page cannot be visited: %s/%sfr/banque/%s: %s',
self.BASEURL, self.currentSubBank, page, e)
raise BrowserUnavailable()
else:
self.page = page
# On some savings accounts, the page lands on the contract tab, and we want the situation
if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label:
self.page.go_on_history_tab()
if self.li.is_here():
return self.page.iter_history()
if self.is_new_website and self.page:
try:
for page in range(1, 50):
# Need to reach the page with all transactions
if not self.page.has_more_operations():
break
form = self.page.get_form(id="I1:P:F")
form['_FID_DoLoadMoreTransactions'] = ''
form['_wxf2_pseq'] = page
form.submit()
# IndexError when form xpath returns [], StopIteration if next called on empty iterable
except (StopIteration, FormNotFound):
self.logger.warning('Could not get history on new website')
except IndexError:
# 6 months history is not available
pass
while self.page:
try:
# Submit form if their is more transactions to fetch
form = self.page.get_form(id="I1:fm")
if self.page.doc.xpath(
'boolean(//a[@class="ei_loadmorebtn"])'):
form['_FID_DoLoadMoreTransactions'] = ""
form.submit()
else:
break
except (IndexError, FormNotFound):
break
# Sometimes the browser can't go further
except ClientError as exc:
if exc.response.status_code == 413:
break
raise
if not self.operations.is_here():
return iter([])
return self.pagination(lambda: self.page.get_history())
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [
list(g)
for k, g in groupby(sorted(trs, key=date_getter), date_getter)
]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
@need_login
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id
) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem':
'',
'_wxf2_cc':
'fr-FR',
'_wxf2_pmode':
'Normal',
'_wxf2_pseq':
i,
'_wxf2_ptarget':
'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber':
'',
'Data_ServiceListDatas_CurrentType':
'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)',
self.url)
if m:
m = m.group(1)
self.location(
'CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'
.format(m, half_history),
data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(
date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link(
) if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
differed_date = None
cards = ([
page.select_card(account._card_number)
for page in account._card_pages
] if hasattr(account, '_card_pages') else account._card_links
if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_differed_date') and (
not differed_date
or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(
self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type
== FrenchTransaction.TYPE_CARD_SUMMARY
and differed_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
@need_login
def get_investment(self, account):
if account._is_inv:
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.por.go(subbank=self.currentSubBank)
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
if account.type is Account.TYPE_PEA:
liquidities = create_french_liquidity(account.balance)
liquidities.label = account.label
return [liquidities]
return iter([])
@need_login
def iter_recipients(self, origin_account):
# access the transfer page
self.internal_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
for recipient in self.page.iter_recipients(
origin_account=origin_account):
yield recipient
self.external_transfer.go(subbank=self.currentSubBank)
if self.page.can_transfer(origin_account.id):
origin_account._external_recipients = set()
if self.page.has_transfer_categories():
for category in self.page.iter_categories():
self.page.go_on_category(category['index'])
self.page.IS_PRO_PAGE = True
for recipient in self.page.iter_recipients(
origin_account=origin_account,
category=category['name']):
yield recipient
else:
for recipient in self.page.iter_recipients(
origin_account=origin_account):
yield recipient
@need_login
def init_transfer(self, account, to, amount, exec_date, reason=None):
if to.category != 'Interne':
self.external_transfer.go(subbank=self.currentSubBank)
else:
self.internal_transfer.go(subbank=self.currentSubBank)
if self.external_transfer.is_here(
) and self.page.has_transfer_categories():
for category in self.page.iter_categories():
if category['name'] == to.category:
self.page.go_on_category(category['index'])
break
self.page.IS_PRO_PAGE = True
self.page.RECIPIENT_STRING = 'data_input_indiceBen'
self.page.prepare_transfer(account, to, amount, reason, exec_date)
return self.page.handle_response(account, to, amount, reason,
exec_date)
@need_login
def execute_transfer(self, transfer, **params):
form = self.page.get_form(
id='P:F',
submit='//input[@type="submit" and contains(@value, "Confirmer")]')
# For the moment, don't ask the user if he confirms the duplicate.
form['Bool:data_input_confirmationDoublon'] = 'true'
form.submit()
return self.page.create_transfer(transfer)
@need_login
def get_advisor(self):
advisor = None
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
if self.page.get_advisor_link():
advisor = self.page.get_advisor()
self.location(
self.page.get_advisor_link()).page.update_advisor(advisor)
else:
advisor = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_advisor()
link = self.page.get_agency()
if link:
link = link.replace(':443/', '/')
self.location(link)
self.page.update_advisor(advisor)
return iter([advisor]) if advisor else iter([])
@need_login
def get_profile(self):
if not self.is_new_website:
profile = self.accounts.stay_or_go(
subbank=self.currentSubBank).get_profile()
else:
profile = self.new_accounts.stay_or_go(
subbank=self.currentSubBank).get_profile()
return profile
def get_recipient_object(self, recipient):
r = Recipient()
r.iban = recipient.iban
r.id = recipient.iban
r.label = recipient.label
r.category = recipient.category
# On credit mutuel recipients are immediatly available.
r.enabled_at = datetime.now().replace(microsecond=0)
r.currency = 'EUR'
r.bank_name = NotAvailable
return r
def continue_new_recipient(self, recipient, **params):
if 'Clé' in params:
self.page.post_code(params['Clé'])
self.page.add_recipient(recipient)
if self.page.bic_needed():
self.page.ask_bic(self.get_recipient_object(recipient))
self.page.ask_sms(self.get_recipient_object(recipient))
def send_sms(self, sms):
data = {}
for k, v in self.form.items():
if k != 'url':
data[k] = v
data['otp_password'] = sms
data['_FID_DoConfirm.x'] = '1'
data['_FID_DoConfirm.y'] = '1'
data['global_backup_hidden_key'] = ''
self.location(self.form['url'], data=data)
def end_new_recipient(self, recipient, **params):
self.send_sms(params['code'])
self.form = None
self.page = None
self.logged = 0
return self.get_recipient_object(recipient)
def post_with_bic(self, recipient, **params):
data = {}
for k, v in self.form.items():
if k != 'url':
data[k] = v
data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic']
self.location(self.form['url'], data=data)
self.page.ask_sms(self.get_recipient_object(recipient))
@need_login
def new_recipient(self, recipient, **params):
if self.currentSubBank is None:
self.getCurrentSubBank()
if 'Bic' in params:
return self.post_with_bic(recipient, **params)
if 'code' in params:
return self.end_new_recipient(recipient, **params)
if 'Clé' in params:
return self.continue_new_recipient(recipient, **params)
self.recipients_list.go(subbank=self.currentSubBank)
if self.page.has_list():
assert recipient.category in self.page.get_recipients_list(), \
'Recipient category is not on the website available list.'
self.page.go_list(recipient.category)
self.page.go_to_add()
if self.verify_pass.is_here():
raise AddRecipientStep(
self.get_recipient_object(recipient),
Value('Clé', label=self.page.get_question()))
else:
return self.continue_new_recipient(recipient, **params)
@need_login
def iter_subscriptions(self):
if self.currentSubBank is None:
self.getCurrentSubBank()
self.subscription.go(subbank=self.currentSubBank)
return self.page.iter_subscriptions()
@need_login
def iter_documents(self, subscription):
if self.currentSubBank is None:
self.getCurrentSubBank()
self.subscription.go(subbank=self.currentSubBank,
params={'typ': 'doc'})
security_limit = 10
for i in range(security_limit):
for doc in self.page.iter_documents(sub_id=subscription.id):
yield doc
if self.page.is_last_page():
break
self.page.next_page()
