def set_parent_account_id(self, loan, acc):
account_parent = Coalesce(Dict('prestationCAV', default=NotAvailable),
Dict('comptePrelevement1',
default=NotAvailable),
default=NotAvailable)(acc)
loan._parent_id = None
if not empty(account_parent):
loan._parent_id = account_parent.replace(' ', '')
def parse(self, obj):
self.env['date'] = DateGuesser(CleanText('./td[1]'), Env('date_guesser'))(self)
self.env['vdate'] = NotAvailable
if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 4]')(self):
# History table with 4 columns
self.env['raw'] = CleanText('./td[2]', children=False)(self)
self.env['amount'] = CleanDecimal.French('./td[last()]')(self)
elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 5]')(self):
# History table with 5 columns
self.env['raw'] = CleanText('./td[3]', children=False)(self)
self.env['amount'] = CleanDecimal.French('./td[last()]')(self)
elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 6]')(self):
# History table with 6 columns (contains vdate)
self.env['raw'] = CleanText('./td[4]', children=False)(self)
self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self)
self.env['amount'] = CleanDecimal.French('./td[last()]')(self)
elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 7]')(self):
# History table with 7 columns
self.env['amount'] = Coalesce(
CleanDecimal.French('./td[6]', sign=lambda x: -1, default=None),
CleanDecimal.French('./td[7]', default=None)
)(self)
if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//th[a[contains(text(), "Valeur")]]')(self):
# With vdate column ('Valeur')
self.env['raw'] = CleanText('./td[4]', children=False)(self)
self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self)
else:
# Without any vdate column
self.env['raw'] = CleanText('./td[3]', children=False)(self)
else:
assert False, 'This type of history table is not handled yet!'
class item(ItemElement):
IGNORED_ACCOUNT_FAMILIES = (
'MES ASSURANCES',
'VOS ASSURANCES',
)
klass = Account
def obj_id(self):
# Loan/credit ids may be duplicated so we use the contract number for now:
if Field('type')(self) in (Account.TYPE_LOAN,
Account.TYPE_CONSUMER_CREDIT,
Account.TYPE_REVOLVING_CREDIT):
return CleanText(Dict('idElementContrat'))(self)
return CleanText(Dict('numeroCompte'))(self)
obj_number = CleanText(Dict('numeroCompte'))
obj_label = CleanText(Dict('libelleProduit'))
obj_currency = CleanCurrency(Dict('idDevise'))
obj__index = Dict('index')
obj__category = Coalesce(Dict('grandeFamilleProduitCode',
default=None),
Dict('sousFamilleProduit/niveau',
default=None),
default=None)
obj__id_element_contrat = CleanText(Dict('idElementContrat'))
obj__fam_product_code = CleanText(Dict('codeFamilleProduitBam'))
obj__fam_contract_code = CleanText(Dict('codeFamilleContratBam'))
def obj_type(self):
if CleanText(
Dict('libelleUsuelProduit'))(self) in ('HABITATION', ):
# No need to log warning for "assurance" accounts
return NotAvailable
_type = Map(CleanText(Dict('libelleUsuelProduit')),
ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self)
if _type == Account.TYPE_UNKNOWN:
self.logger.warning(
'There is an untyped account: please add "%s" to ACCOUNT_TYPES.'
% CleanText(Dict('libelleUsuelProduit'))(self))
return _type
def obj_balance(self):
balance = Dict('solde', default=None)(self)
if balance:
return Eval(float_to_decimal, balance)(self)
# We will fetch the balance with account_details
return NotAvailable
def condition(self):
# Ignore insurances (plus they all have identical IDs)
# Ignore some credits not displayed on the website
return CleanText(Dict('familleProduit/libelle', default=''))(self) not in self.IGNORED_ACCOUNT_FAMILIES \
and 'non affiche' not in CleanText(Dict('sousFamilleProduit/libelle', default=''))(self) \
and 'Inactif' not in CleanText(Dict('libelleSituationContrat', default=''))(self)
class item(ItemElement):
klass = Transaction
obj_date = Date(CleanText('.//td[@headers="date"]'), dayfirst=True)
obj_raw = Transaction.Raw('.//td[@headers="libelle"]')
obj_amount = Coalesce(
CleanDecimal.French('.//td[@headers="debit"]',
default=NotAvailable),
CleanDecimal.French('.//td[@headers="credit"]',
default=NotAvailable),
)
class item(ItemElement):
klass = Recipient
obj_id = obj_iban = Coalesce(Dict('iban'), Dict('index'))
obj_label = Dict('nom')
obj_category = 'Externe'
obj_enabled_at = dt.date.today()
obj__index = Dict('index')
def condition(self):
return Dict('actif', default=False)(self)
def obj_investments(self):
investments = []
for elem in self.xpath(
'./following-sibling::div[1]//tbody/tr'):
inv = Investment()
inv.label = CleanText('./td[1]')(elem)
inv.valuation = Coalesce(
CleanDecimal.French('./td[2]/p', default=NotAvailable),
CleanDecimal.French('./td[2]'))(elem)
investments.append(inv)
return investments
def set_loan_details(self, account):
# If there are no available details for the loan, the statut will be "NOK"
if Dict('commun/statut')(self.doc) == 'NOK':
return
else:
# There is no default value in the Coalesce because we want it to crash in case of
# unknown value to be able to add it
rate = Coalesce(
Dict('donnees/caracteristiquesReservea/tauxHorsAssurance', NotAvailable),
Dict('donnees/caracteristiquesCreditConfiance/taux', NotAvailable),
)(self.doc)
account.rate = CleanDecimal().filter(rate)
class item(ItemElement):
klass = Bill
obj_id = Format('%s_%s', Env('subid'), Dict('idFacture'))
obj_price = CleanDecimal(Dict('mntTotFacture'))
obj_url = Coalesce(
Dict('_links/facturePDF/href', default=NotAvailable),
Dict('_links/facturePDFDF/href', default=NotAvailable)
)
obj_date = MyDate(Dict('dateFacturation'))
obj_duedate = MyDate(Dict('dateLimitePaieFacture', default=NotAvailable), default=NotAvailable)
obj_label = Format('Facture %s', Dict('idFacture'))
obj_format = 'pdf'
obj_currency = 'EUR'
class item(ItemElement):
def condition(self):
return Dict('accountNumber', default=None)(self)
klass = Account
obj_id = obj_number = Dict('accountNumber')
obj_label = Coalesce(Dict('accountNatureLongLabel', default=''),
Dict('accountNatureShortLabel', default=''))
obj_iban = Dict('ibanCode')
obj_currency = Dict('currencyCode')
def obj_balance(self):
balance_value = CleanDecimal(Dict('balanceValue'))(self)
if CleanText(Dict('balanceSign'))(self) == '-':
return -balance_value
return balance_value
def iter_investment(self, account, invs=None):
if account.id not in self.investments and invs is not None:
self.investments[account.id] = []
for inv in invs:
i = Investment()
# If nothing is given to make the label, we use the ISIN instead
# We let it crash if the ISIN is not available either.
if all([inv['classification'], inv['description']]):
i.label = "%s - %s" % (inv['classification'],
inv['description'])
else:
i.label = Coalesce().filter((
inv['classification'],
inv['description'],
inv['isin'],
))
i.code = inv['isin']
if not is_isin_valid(i.code):
i.code = NotAvailable
i.code_type = NotAvailable
if u'Solde Espèces' in i.label:
i.code = 'XX-liquidity'
else:
i.code_type = Investment.CODE_TYPE_ISIN
i.quantity = CleanDecimal(default=NotAvailable).filter(
inv['nombreParts'])
i.unitprice = CleanDecimal(default=NotAvailable).filter(
inv['prixMoyenAchat'])
i.unitvalue = CleanDecimal(default=NotAvailable).filter(
inv['valeurCotation'])
i.valuation = CleanDecimal().filter(inv['montantEuro'])
# For some invests the vdate returned is None
# Consequently we set the default value at NotAvailable
i.vdate = Date(default=NotAvailable).filter(
inv['datePosition'])
i.diff = CleanDecimal(default=NotAvailable).filter(
inv['performanceEuro'])
self.investments[account.id].append(i)
return self.investments[account.id]
class item(ItemElement):
klass = Investment
def condition(self):
return Field('label')(self) not in ('Total', '')
obj_quantity = CleanDecimal.French(
'./td[contains(@data-label, "Nombre de parts")]',
default=NotAvailable)
obj_unitvalue = CleanDecimal.French(
'./td[contains(@data-label, "Valeur de la part")]',
default=NotAvailable)
def obj_valuation(self):
# Handle discrepancies between aviva & afer (Coalesce does not work here)
if CleanText('./td[contains(@data-label, "Valeur de rachat")]'
)(self):
return CleanDecimal.French(
'./td[contains(@data-label, "Valeur de rachat")]')(
self)
return CleanDecimal.French(
CleanText('./td[contains(@data-label, "Montant")]',
children=False))(self)
obj_vdate = Date(CleanText('./td[@data-label="Date de valeur"]'),
dayfirst=True,
default=NotAvailable)
obj_label = Coalesce(
CleanText('./th[@data-label="Nom du support"]/a'),
CleanText('./th[@data-label="Nom du support"]'),
CleanText('./td[@data-label="Nom du support"]'),
)
obj_code = IsinCode(Regexp(CleanText(
'./td[@data-label="Nom du support"]/a/@onclick|./th[@data-label="Nom du support"]/a/@onclick'
),
r'"(.*)"',
default=NotAvailable),
default=NotAvailable)
obj_code_type = IsinType(Field('code'))
def obj_date(self):
# The date xpath changes depending on the kind of order
return Coalesce(
Date(CleanText(
'.//div[has-class("a-span4") and not(has-class("recipient"))]/div[2]'
),
parse_func=parse_french_date,
dayfirst=True,
default=NotAvailable),
Date(CleanText(
'.//div[has-class("a-span3") and not(has-class("recipient"))]/div[2]'
),
parse_func=parse_french_date,
dayfirst=True,
default=NotAvailable),
Date(CleanText(
'.//div[has-class("a-span2") and not(has-class("recipient"))]/div[2]'
),
parse_func=parse_french_date,
dayfirst=True,
default=NotAvailable),
)(self)
class item_account_generic(ItemElement):
klass = Account
def condition(self):
# For some loans the following xpath is absent and we don't want to skip them
# Also a case of loan that is empty and has no information exists and will be ignored
return (len(
self.el.xpath('.//span[@class="number"]')
) > 0 or (Field('type')(self) == Account.TYPE_LOAN and (not bool(
self.el.
xpath(
'.//div//*[contains(text(),"pas la restitution de ces données.")]'
)
) and not bool(
self.el.xpath(
'.//div[@class="amount"]/span[contains(text(), "Contrat résilié")]'
)
) and not bool(
self.el.xpath(
'.//div[@class="amount"]/span[contains(text(), "Remboursé intégralement")]'
)
) and not bool(
self.el.xpath(
'.//div[@class="amount"]/span[contains(text(), "Prêt non débloqué")]'
)))))
obj_id = obj_number = CleanText('.//abbr/following-sibling::text()')
obj_currency = Coalesce(Currency('.//span[@class="number"]'),
Currency('.//span[@class="thick"]'))
def obj_url(self):
url = Link('./a', default=NotAvailable)(self)
if not url:
url = Regexp(Attr('.//span', 'onclick', default=''),
r'\'(https.*)\'',
default=NotAvailable)(self)
if url:
if 'CreditRenouvelable' in url:
url = Link(
u'.//a[contains(text(), "espace de gestion crédit renouvelable")]'
)(self.el)
return urljoin(self.page.url, url)
return url
def obj_label(self):
return CleanText('.//div[@class="title"]/h3')(self).upper()
def obj_ownership(self):
account_holder = CleanText('.//div[@class="title"]/span')(self)
if re.search(
r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou ?(m|mr|me|mme|mlle|mle|ml)?\b(.*)',
account_holder, re.IGNORECASE):
return AccountOwnership.CO_OWNER
elif all([n in account_holder for n in self.env['name'].split(' ')]):
return AccountOwnership.OWNER
else:
return AccountOwnership.ATTORNEY
def obj_balance(self):
if Field('type')(self) == Account.TYPE_LOAN:
balance = CleanDecimal('.//span[@class="number"]',
replace_dots=True,
default=NotAvailable)(self)
if balance:
balance = -abs(balance)
return balance
return CleanDecimal('.//span[@class="number"]',
replace_dots=True,
default=NotAvailable)(self)
def obj_coming(self):
if Field('type')(
self) == Account.TYPE_CHECKING and Field('balance')(self) != 0:
# When the balance is 0, we get a website unavailable on the history page
# and the following navigation is broken
has_coming = False
coming = 0
details_page = self.page.browser.open(Field('url')(self))
coming_op_link = Link(
'//a[contains(text(), "Opérations à venir")]',
default=NotAvailable)(details_page.page.doc)
if coming_op_link:
coming_op_link = Regexp(
Link('//a[contains(text(), "Opérations à venir")]'),
r'../(.*)')(details_page.page.doc)
coming_operations = self.page.browser.open(
self.page.browser.BASEURL + '/voscomptes/canalXHTML/CCP/' +
coming_op_link)
else:
coming_op_link = Link(
'//a[contains(text(), "Opérations en cours")]')(
details_page.page.doc)
coming_operations = self.page.browser.open(coming_op_link)
if CleanText('//span[@id="amount_total"]')(
coming_operations.page.doc):
has_coming = True
coming += CleanDecimal('//span[@id="amount_total"]',
replace_dots=True)(
coming_operations.page.doc)
if CleanText('.//dt[contains(., "Débit différé à débiter")]')(
self):
has_coming = True
coming += CleanDecimal(
'.//dt[contains(., "Débit différé à débiter")]/following-sibling::dd[1]',
replace_dots=True)(self)
return coming if has_coming else NotAvailable
return NotAvailable
def obj_iban(self):
if not Field('url')(self):
return NotAvailable
if Field('type')(self) not in (Account.TYPE_CHECKING,
Account.TYPE_SAVINGS):
return NotAvailable
details_page = self.page.browser.open(Field('url')(self)).page
rib_link = Link('//a[abbr[contains(text(), "RIB")]]',
default=NotAvailable)(details_page.doc)
if rib_link:
response = self.page.browser.open(rib_link)
return response.page.get_iban()
elif Field('type')(self) == Account.TYPE_SAVINGS:
# The rib link is available on the history page (ex: Livret A)
his_page = self.page.browser.open(Field('url')(self))
rib_link = Link('//a[abbr[contains(text(), "RIB")]]',
default=NotAvailable)(his_page.page.doc)
if rib_link:
response = self.page.browser.open(rib_link)
return response.page.get_iban()
return NotAvailable
def obj_type(self):
types = {
'comptes? bancaires?': Account.TYPE_CHECKING,
"plan d'epargne populaire": Account.TYPE_SAVINGS,
'livrets?': Account.TYPE_SAVINGS,
'epargnes? logement': Account.TYPE_SAVINGS,
"autres produits d'epargne": Account.TYPE_SAVINGS,
'compte relais': Account.TYPE_SAVINGS,
'comptes? titres? et pea': Account.TYPE_MARKET,
'compte-titres': Account.TYPE_MARKET,
'assurances? vie': Account.TYPE_LIFE_INSURANCE,
'prêt': Account.TYPE_LOAN,
'crédits?': Account.TYPE_LOAN,
'plan d\'epargne en actions': Account.TYPE_PEA,
'comptes? attente': Account.TYPE_CHECKING,
'perp': Account.TYPE_PERP,
'assurances? retraite': Account.TYPE_PERP,
}
# first trying to match with label
label = Field('label')(self)
for atypetxt, atype in types.items():
if re.findall(atypetxt,
label.lower()): # match with/without plurial in type
return atype
# then by type
type = Regexp(
CleanText(
'../../preceding-sibling::div[@class="avoirs"][1]/span[1]'),
r'(\d+) (.*)', '\\2')(self)
for atypetxt, atype in types.items():
if re.findall(atypetxt,
type.lower()): # match with/without plurial in type
return atype
return Account.TYPE_UNKNOWN
def obj__has_cards(self):
return Link('.//a[contains(@href, "consultationCarte")]',
default=None)(self)
class item(ItemElement):
def validate(self, obj):
# We skip loans with a balance of 0 because the JSON returned gives
# us no info (only `null` values on all fields), so there is nothing
# useful to display
return obj.type != Account.TYPE_LOAN or obj.balance != 0
FAMILY_TO_TYPE = {
1: Account.TYPE_CHECKING,
2: Account.TYPE_SAVINGS,
3: Account.TYPE_DEPOSIT,
4: Account.TYPE_MARKET,
5: Account.TYPE_LIFE_INSURANCE,
6: Account.TYPE_LIFE_INSURANCE,
8: Account.TYPE_LOAN,
9: Account.TYPE_LOAN,
}
LABEL_TO_TYPE = {
'PEA Espèces': Account.TYPE_PEA,
'PEA Titres': Account.TYPE_PEA,
'PEL': Account.TYPE_SAVINGS,
'Plan Epargne Retraite Particulier': Account.TYPE_PERP,
'Crédit immobilier': Account.TYPE_MORTGAGE,
'Réserve Provisio': Account.TYPE_REVOLVING_CREDIT,
'Prêt personnel': Account.TYPE_CONSUMER_CREDIT,
'Crédit Silo': Account.TYPE_REVOLVING_CREDIT,
}
klass = Account
obj_id = Dict('key')
obj_label = Coalesce(Dict('libellePersoProduit',
default=NotAvailable),
Dict('libelleProduit',
default=NotAvailable),
default=NotAvailable)
obj_currency = Currency(Dict('devise'))
obj_type = Coalesce(Map(Dict('libelleProduit'),
LABEL_TO_TYPE,
default=NotAvailable),
Map(Env('account_type'),
FAMILY_TO_TYPE,
default=NotAvailable),
default=Account.TYPE_UNKNOWN)
obj_balance = Dict('soldeDispo')
obj_coming = Dict('soldeAVenir')
obj_number = Dict('value')
obj__subscriber = Format('%s %s', Dict('titulaire/nom'),
Dict('titulaire/prenom'))
obj__iduser = Dict('titulaire/ikpi')
def obj_iban(self):
iban = Map(Dict('key'),
Env('ibans')(self),
default=NotAvailable)(self)
if not empty(iban):
if not is_iban_valid(iban):
iban = rib2iban(rebuild_rib(iban))
return iban
return None
def obj_ownership(self):
indic = Dict('titulaire/indicTitulaireCollectif',
default=None)(self)
# The boolean is in the form of a string ('true' or 'false')
if indic == 'true':
return AccountOwnership.CO_OWNER
elif indic == 'false':
if self.page.get_user_ikpi() == Dict('titulaire/ikpi')(
self):
return AccountOwnership.OWNER
return AccountOwnership.ATTORNEY
return NotAvailable
# softcap not used TODO don't pass this key when backend is ready
# deferred cb can disappear the day after the appear, so 0 as day_for_softcap
obj__bisoftcap = {
'deferred_cb': {
'softcap_day': 1000,
'day_for_softcap': 1
}
}