def obj_ownership(self): # 'groupeRoleDTO' can contains 'TITULAIRE', 'MANDATAIRE' or 'REPRESENTATION' # 'role' contains 'groupeRoleDTO' sub-categories. If the groupeRoleDTO is # 'TUTULAIRE', we have to check the role to know if it's 'TITULAIRE' or 'COTITULAIRE' ownership = Map(Dict('groupeRoleDTO'), self.ACCOUNTS_OWNERSHIP, NotAvailable)(self) if ownership == AccountOwnership.OWNER: ownership = Map(Dict('role'), self.ACCOUNTS_OWNERSHIP, NotAvailable)(self) return ownership
def obj_sensors(self): sensors = [] lastdate = DateTime(Regexp(Env('datetime'), r'(\d+)\.(\d+)\.(\d+) (\d+):(\d+)', r'\3-\2-\1 \4:\5', default=NotAvailable), default=NotAvailable)(self) forecast = Map(Env('forecast'), self.forecasts, default=NotAvailable)(self) alarm = Map(Env('alarm'), self.alarmlevel, default=u'')(self) self.add_sensor(sensors, u"Level", u"cm", self.env['levelvalue'], forecast, alarm, lastdate) self.add_sensor(sensors, u"Flow", u"m3/s", self.env['flowvalue'], forecast, alarm, lastdate) return sensors
class item(ItemElement): klass = Account TYPE = { 'Livret': Account.TYPE_SAVINGS, 'Compte': Account.TYPE_CHECKING, 'PEA': Account.TYPE_PEA, 'PEA-PME': Account.TYPE_PEA, 'Compte-titres': Account.TYPE_MARKET, 'Assurance-vie': Account.TYPE_LIFE_INSURANCE, } obj_id = CleanText( './td//div[contains(@class, "-synthese-title") or contains(@class, "-synthese-text")]' ) & Regexp(pattern=r'(\d+)') obj_label = CleanText( './td//div[contains(@class, "-synthese-title")]') obj_balance = MyDecimal( './td//div[contains(@class, "-synthese-num")]', replace_dots=True) obj_currency = FrenchTransaction.Currency( './td//div[contains(@class, "-synthese-num")]') obj_type = Map(Regexp(Field('label'), r'^([^ ]*)'), TYPE, default=Account.TYPE_UNKNOWN) def obj_url(self): return urljoin(self.page.url, CleanText('./@data-href')(self)) obj__card_balance = CleanDecimal( './td//div[@class="synthese-encours"]/div[2]', default=None) def condition(self): return not len(self.el.xpath('./td[@class="chart"]'))
class item(ItemElement): klass = GaugeSensor obj_name = Map(Dict('key'), SENSOR_NAMES) obj_gaugeid = Env('nom_court_sit') obj_id = Format('%s.%s', obj_gaugeid, Dict('key')) obj_unit = 'µg/m³' class obj_lastvalue(ItemElement): klass = GaugeMeasure obj_date = DateTime( Format( '%s %s', Env('min_donnees'), Env('date'), # "date" contains the time... ) ) obj_level = CleanDecimal(Dict('value')) class obj_geo(ItemElement): klass = GeoCoordinates obj_latitude = CleanDecimal(Env('latitude')) obj_longitude = CleanDecimal(Env('longitude')) class obj_location(ItemElement): klass = PostalAddress obj_street = Env('adresse') obj_postal_code = Env('ninsee') obj_city = Env('city') obj_region = 'Ile-de-France' obj_country = 'France'
class item(ItemElement): klass = Account obj_id = CleanText('./td[2]') obj_number = Field('id') obj_label = CleanText('./td/span[@class="gras"]') obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) # Accounts without balance will be skipped later on obj_balance = CleanDecimal.French('./td//*[@class="montant3"]', default=NotAvailable) obj_currency = Currency('./td[@class="cel-devise"]') obj_iban = None obj__form = None def obj_url(self): url = Link('./td[2]/a', default=None)(self) if url and 'BGPI' in url: # This URL is just the BGPI home page, not the account itself. # The real account URL will be set by get_account_details() in BGPISpace. return 'BGPI' return url def validate(self, obj): # Skip 'ESPE INTEG' accounts, these liquidities are already available # on the associated Market account on the Netfinca website return obj.label != 'ESPE INTEG'
class item(ItemElement): klass = Loan def condition(self): return 'Billet financier' not in CleanText('./td[1]')(self) obj_id = CleanText('./td[2]') obj_number = Field('id') obj_label = CleanText('./td[1]') obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_LOAN) obj_next_payment_amount = Env('next_payment_amount') obj_total_amount = Env('total_amount') obj_currency = Currency('./td[@class="cel-devise"]') obj_url = Link('./td[2]/a', default=None) obj_iban = None obj__form = None def obj_balance(self): balance = Env('balance')(self) return -abs(balance) def parse(self, obj): # We must handle Loan tables with 5 or 6 columns if CleanText('//tr[contains(@class, "colcelligne")][count(td) = 5]')(self): # History table with 4 columns (no loan details) self.env['next_payment_amount'] = NotAvailable self.env['total_amount'] = NotAvailable self.env['balance'] = CleanDecimal.French('./td[4]//*[@class="montant3" or @class="montant4"]', default=NotAvailable)(self) elif CleanText('//tr[contains(@class, "colcelligne")][count(td) = 6]')(self): # History table with 5 columns (contains next_payment_amount & total_amount) self.env['next_payment_amount'] = CleanDecimal.French('./td[3]//*[@class="montant3"]', default=NotAvailable)(self) self.env['total_amount'] = CleanDecimal.French('./td[4]//*[@class="montant3"]', default=NotAvailable)(self) self.env['balance'] = CleanDecimal.French('./td[5]//*[@class="montant3"]', default=NotAvailable)(self)
class item(ItemElement): klass = Account def condition(self): # Skip card coming lines return 'Encours carte' not in CleanText(TableCell('label', colspan=True))(self) obj_id = CleanText(TableCell('id', colspan=True)) obj_number = Field('id') obj_label = CleanText(TableCell('label', colspan=True)) obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) obj_currency = Currency(TableCell('currency', colspan=True)) obj_url = None # Accounts may have an 'Operations' balance or a 'Value' balance def obj_balance(self): value_balance = CleanText(TableCell('value_balance', default='', colspan=True))(self) # Skip invalid balance values in the 'Value' column (for example for Revolving credits) if value_balance not in ('', 'Montant disponible'): return CleanDecimal.French().filter(value_balance) return CleanDecimal.French(CleanText(TableCell('operation_balance', default='', colspan=True)))(self) def obj__form(self): # Account forms look like 'javascript:fwkPUAvancerForm('Releves','frm1')' # From this we extract the name (frm1) and fetch the form name on the page. script = Link('.//a', default='')(TableCell('id', colspan=True)(self)[0]) if 'javascript' in script: form_search = re.search(r'frm\d+', script) if form_search: account_form = self.page.get_form(name=form_search.group(0)) return self.page.fill_form(account_form, card=False) return None
class item(ItemElement): klass = Account obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) obj__owner = CleanText(TableCell('owner')) def obj_id(self): tablecell = TableCell('id')(self)[0] _id = tablecell.xpath('./div[position()=2]') return CleanText(_id)(self) obj_number = obj_id def obj_label(self): tablecell = TableCell('label')(self)[0] label = tablecell.xpath('./div[position()=1]') return CleanText(label)(self) def obj_balance(self): tablecell = TableCell('balance')(self)[0] balance = tablecell.xpath('./span[@class="intraday"]') return CleanDecimal.French(balance)(self) def obj_currency(self): tablecell = TableCell('balance')(self)[0] currency = tablecell.xpath('./span[@class="intraday"]') return Currency(currency)(self)
class account(ItemElement): klass = Account def condition(self): return '/outil/UWLM/ListeMouvement' in self.el.attrib[ 'onclick'] NATURE2TYPE = { '001': Account.TYPE_SAVINGS, '005': Account.TYPE_CHECKING, '006': Account.TYPE_CHECKING, '007': Account.TYPE_SAVINGS, '012': Account.TYPE_SAVINGS, '023': Account.TYPE_CHECKING, '046': Account.TYPE_SAVINGS, '049': Account.TYPE_SAVINGS, '068': Account.TYPE_MARKET, '069': Account.TYPE_SAVINGS, } obj__link_id = Format('%s&mode=55', Regexp(CleanText('./@onclick'), "'(.*)'")) obj_id = Regexp(Field('_link_id'), r'.*agence=(\w+).*compte=(\w+)', r'\1\2') obj__coming_links = [] obj_label = CleanText('.//div[@class="libelleCompte"]') obj_balance = CleanDecimal('.//td[has-class("right")]', replace_dots=True) obj_currency = FrenchTransaction.Currency( './/td[has-class("right")]') obj_type = Map(Regexp(Field('_link_id'), r'.*nature=(\w+)'), NATURE2TYPE, default=Account.TYPE_UNKNOWN) obj__market_link = None
def obj_type(self): _type = Map(CleanText(Dict('comptePrincipal/libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) if _type == Account.TYPE_UNKNOWN: self.logger.warning( 'We got an untyped account: please add "%s" to ACCOUNT_TYPES.', CleanText( Dict('comptePrincipal/libelleUsuelProduit'))(self)) return _type
class item(ItemElement): klass = Transaction obj_amount = CleanDecimal(Dict('amount')) obj_date = Date(Dict('effectiveDate')) obj_vdate = Date(Dict('operationDate')) obj_type = Map(Upper(Dict('type')), Transaction.TYPES, Transaction.TYPE_UNKNOWN) def obj_raw(self): return Transaction.Raw(Lower(Dict('label')))(self) or Format('%s %s', Field('date'), Field('amount'))(self)
def obj_iban(self): iban = Map(Dict('key'), Env('ibans')(self), default=NotAvailable)(self) if not empty(iban): if not is_iban_valid(iban): iban = rib2iban(rebuild_rib(iban)) return iban return None
def obj_type(self): if CleanText( Dict('libelleUsuelProduit'))(self) in ('HABITATION', ): # No need to log warning for "assurance" accounts return NotAvailable _type = Map(CleanText(Dict('libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) if _type == Account.TYPE_UNKNOWN: self.logger.warning( 'There is an untyped account: please add "%s" to ACCOUNT_TYPES.', CleanText(Dict('libelleUsuelProduit'))(self)) return _type
class item(ItemElement): klass = Transaction # Not sure that Dict('id') is unique and persist # wait for the full API migration obj__web_id = Eval(str, Dict('id')) obj_amount = CleanDecimal(Dict('amount')) obj_date = Date(Dict('effectiveDate')) obj_type = Map(Upper(Dict('type')), Transaction.TYPES, Transaction.TYPE_UNKNOWN) def obj_raw(self): return Transaction.Raw(Lower(Dict('detail')))(self) or Format('%s %s', Field('date'), Field('amount'))(self)
class item(ItemElement): klass = Account TYPE = { 'Livret': Account.TYPE_SAVINGS, 'Compte': Account.TYPE_CHECKING, 'PEA': Account.TYPE_PEA, 'PEA-PME': Account.TYPE_PEA, 'Compte-titres': Account.TYPE_MARKET, 'Assurance-vie': Account.TYPE_LIFE_INSURANCE, 'Crédit': Account.TYPE_LOAN, } obj_id = CleanText( './td//div[contains(@class, "-synthese-title") or contains(@class, "-synthese-text")]' ) & Regexp(pattern=r'(\d+)') obj_label = CleanText( './td//div[contains(@class, "-synthese-title")]') obj_balance = MyDecimal( './td//div[contains(@class, "-synthese-num")]', replace_dots=True) obj_currency = FrenchTransaction.Currency( './td//div[contains(@class, "-synthese-num")]') obj_type = Map(Regexp(Field('label'), r'^([^ ]*)'), TYPE, default=Account.TYPE_UNKNOWN) def obj_url(self): return urljoin(self.page.url, CleanText('./@data-href')(self)) obj__card_balance = CleanDecimal( './td//div[@class="synthese-encours"][last()]/div[2]', default=None) def condition(self): return not len(self.el.xpath('./td[@class="chart"]')) def obj_ownership(self): owner = CleanText( './td//div[contains(@class, "-synthese-text") and not(starts-with(., "N°"))]', default=None)(self) if owner: if re.search( r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', owner, re.IGNORECASE): return AccountOwnership.CO_OWNER elif all(n in owner.upper() for n in self.env['name'].split()): return AccountOwnership.OWNER return AccountOwnership.ATTORNEY
class item(ItemElement): klass = Account obj_id = CleanText('./@data-policy') obj_number = Field('id') obj_label = CleanText('.//p[has-class("a-heading")]', default=NotAvailable) obj_url = AbsoluteLink('.//a[contains(text(), "Détail")]') obj_type = Map(Regexp(CleanText('../../../div[contains(@class, "o-product-roundels-category")]'), r'Vérifier votre (.*) contrats', default=NotAvailable), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) def condition(self): # 'Prévoyance' div is for insurance contracts -- they are not bank accounts and thus are skipped ignored_accounts = ( 'Prévoyance', 'Responsabilité civile', 'Complémentaire santé', 'Protection juridique', 'Habitation', 'Automobile', ) return CleanText('../../div[has-class("o-product-tab-category")]', default=NotAvailable)(self) not in ignored_accounts
class item(ItemElement): TRANSACTION_TYPES = { 'PAIEMENT PAR CARTE': Transaction.TYPE_CARD, 'REMISE CARTE': Transaction.TYPE_CARD, 'PRELEVEMENT CARTE': Transaction.TYPE_CARD_SUMMARY, 'RETRAIT AU DISTRIBUTEUR': Transaction.TYPE_WITHDRAWAL, "RETRAIT MUR D'ARGENT": Transaction.TYPE_WITHDRAWAL, 'FRAIS': Transaction.TYPE_BANK, 'COTISATION': Transaction.TYPE_BANK, 'VIREMENT': Transaction.TYPE_TRANSFER, 'VIREMENT EN VOTRE FAVEUR': Transaction.TYPE_TRANSFER, 'VIREMENT EMIS': Transaction.TYPE_TRANSFER, 'CHEQUE EMIS': Transaction.TYPE_CHECK, 'REMISE DE CHEQUE': Transaction.TYPE_DEPOSIT, 'PRELEVEMENT': Transaction.TYPE_ORDER, 'PRELEVT': Transaction.TYPE_ORDER, 'PRELEVMNT': Transaction.TYPE_ORDER, 'REMBOURSEMENT DE PRET': Transaction.TYPE_LOAN_PAYMENT, } klass = Transaction # Transactions in foreign currencies have no 'libelleTypeOperation' # and 'libelleComplementaire' keys, hence the default values. # The CleanText() gets rid of additional spaces. obj_raw = CleanText( Format('%s %s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')), CleanText(Dict('libelleComplementaire', default='')))) obj_label = CleanText( Format('%s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')))) obj_amount = Eval(float_to_decimal, Dict('montant')) obj_type = Map(CleanText(Dict('libelleTypeOperation', default='')), TRANSACTION_TYPES, Transaction.TYPE_UNKNOWN) def obj_date(self): return dateutil.parser.parse(Dict('dateValeur')(self)) def obj_rdate(self): return dateutil.parser.parse(Dict('dateOperation')(self))
class item(ItemElement): klass = Account obj_id = Regexp(CleanText('./h3/a/@title'), r'([A-Z\d]{4}[A-Z\d\*]{3}[A-Z\d]{4})') obj_balance = CleanDecimal.French( './span/text()[1]' ) # This website has the good taste of leaving hard coded HTML comments. This is the way to pin point to the righ text item. obj_currency = Currency('./span') obj_url = AbsoluteLink('./h3/a') # account are grouped in /div based on their type, we must fetch the closest one relative to item_xpath obj_type = Map( CleanText( './ancestor::div[1]/preceding-sibling::h2[1]/button/div[@class="title-accordion"]' ), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) def obj_label(self): """ Need to get rid of the id wherever we find it in account labels like "LIV A 0123456789N MR MOMO" (livret A) as well as "0123456789N MR MOMO" (checking account) """ return CleanText('./h3/a/@title')(self).replace( '%s ' % Field('id')(self), '')
class item(ItemElement): klass = Account TYPE = { 'Livret': Account.TYPE_SAVINGS, } def obj_balance(self): balance = CleanText( self.el.xpath('./td/div/div[1]/div/span'))(self) balance = re.sub(r'[^\d\-\,]', '', balance) return Decimal( re.sub(r',(?!(\d+$))', '', balance).replace(',', '.')) obj_id = CleanText('./td/div/div[3]/span') & Regexp( pattern=r'(\d+)') obj_label = CleanText('./td/div/div[2]/span') obj_currency = FrenchTransaction.Currency( './td/div/div[1]/div/span') obj_type = Map(Regexp(Field('label'), r'^(\w+)'), TYPE, default=Account.TYPE_UNKNOWN) obj__link = CleanText('./@data-href')
class item(ItemElement): klass = Account obj_id = CleanText(Dict('codeDispositif')) obj_balance = CleanDecimal(Dict('mtBrut')) obj_currency = 'EUR' obj_type = Map(Dict('typeDispositif'), ACCOUNT_TYPES, Account.TYPE_LIFE_INSURANCE) def obj_number(self): # just the id is a kind of company id so it can be unique on a backend but not unique on multiple backends return '%s_%s' % (Field('id')(self), self.page.browser.username) def obj_label(self): try: return Dict('libelleDispositif')(self).encode( 'iso-8859-2').decode('utf8') except UnicodeError: try: return Dict('libelleDispositif')(self).encode( 'latin1').decode('utf8') except UnicodeError: return Dict('libelleDispositif')(self)
class item(ItemElement): TRANSACTION_TYPES = { 'PAIEMENT PAR CARTE': Transaction.TYPE_CARD, 'REMISE CARTE': Transaction.TYPE_CARD, 'PRELEVEMENT CARTE': Transaction.TYPE_CARD_SUMMARY, 'RETRAIT AU DISTRIBUTEUR': Transaction.TYPE_WITHDRAWAL, "RETRAIT MUR D'ARGENT": Transaction.TYPE_WITHDRAWAL, 'FRAIS': Transaction.TYPE_BANK, 'COTISATION': Transaction.TYPE_BANK, 'VIREMENT': Transaction.TYPE_TRANSFER, 'VIREMENT EN VOTRE FAVEUR': Transaction.TYPE_TRANSFER, 'VIREMENT EMIS': Transaction.TYPE_TRANSFER, 'CHEQUE EMIS': Transaction.TYPE_CHECK, 'REMISE DE CHEQUE': Transaction.TYPE_DEPOSIT, 'PRELEVEMENT': Transaction.TYPE_ORDER, 'PRELEVT': Transaction.TYPE_ORDER, 'PRELEVMNT': Transaction.TYPE_ORDER, 'REMBOURSEMENT DE PRET': Transaction.TYPE_LOAN_PAYMENT, "REMISE D'EFFETS": Transaction.TYPE_PAYBACK, 'AVOIR': Transaction.TYPE_PAYBACK, "VERSEMENT D'ESPECES": Transaction.TYPE_CASH_DEPOSIT, 'INTERETS CREDITEURS': Transaction.TYPE_BANK, } klass = Transaction obj_date = Date(Dict('dateValeur')) # Transactions in foreign currencies have no 'libelleTypeOperation' # and 'libelleComplementaire' keys, hence the default values. # The CleanText() gets rid of additional spaces. obj_raw = Transaction.Raw( CleanText( Format( '%s %s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')), CleanText(Dict('libelleComplementaire', default=''))))) # There is a key in the json called dateOperation but most of the time it is the # same as the dateValeur. If the patterns do not find the rdate in the label, # we set the value of rdate to dateOperation if dateOperation is before # dateValeur (there are cases where dateOperation is after dateValeur). def obj_rdate(self): date = Field('date')(self) # rdate is already set by `obj_raw` and the patterns. rdate = self.obj.rdate date_operation = Date(Dict('dateOperation'))(self) if rdate == date and date_operation < date: return date_operation elif rdate != date: return rdate return NotAvailable obj_label = CleanText( Format('%s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')))) obj_amount = Eval(float_to_decimal, Dict('montant')) obj_type = Map(CleanText(Dict('libelleTypeOperation', default='')), TRANSACTION_TYPES, Transaction.TYPE_UNKNOWN)
class item(ItemElement): def validate(self, obj): # We skip loans with a balance of 0 because the JSON returned gives # us no info (only `null` values on all fields), so there is nothing # useful to display return obj.type != Account.TYPE_LOAN or obj.balance != 0 FAMILY_TO_TYPE = { 1: Account.TYPE_CHECKING, 2: Account.TYPE_SAVINGS, 3: Account.TYPE_DEPOSIT, 4: Account.TYPE_MARKET, 5: Account.TYPE_LIFE_INSURANCE, 6: Account.TYPE_LIFE_INSURANCE, 8: Account.TYPE_LOAN, 9: Account.TYPE_LOAN, } LABEL_TO_TYPE = { 'PEA Espèces': Account.TYPE_PEA, 'PEA Titres': Account.TYPE_PEA, 'PEL': Account.TYPE_SAVINGS, 'Plan Epargne Retraite Particulier': Account.TYPE_PERP, 'Crédit immobilier': Account.TYPE_MORTGAGE, 'Réserve Provisio': Account.TYPE_REVOLVING_CREDIT, 'Prêt personnel': Account.TYPE_CONSUMER_CREDIT, 'Crédit Silo': Account.TYPE_REVOLVING_CREDIT, } klass = Account obj_id = Dict('key') obj_label = Coalesce(Dict('libellePersoProduit', default=NotAvailable), Dict('libelleProduit', default=NotAvailable), default=NotAvailable) obj_currency = Currency(Dict('devise')) obj_type = Coalesce(Map(Dict('libelleProduit'), LABEL_TO_TYPE, default=NotAvailable), Map(Env('account_type'), FAMILY_TO_TYPE, default=NotAvailable), default=Account.TYPE_UNKNOWN) obj_balance = Dict('soldeDispo') obj_coming = Dict('soldeAVenir') obj_number = Dict('value') obj__subscriber = Format('%s %s', Dict('titulaire/nom'), Dict('titulaire/prenom')) obj__iduser = Dict('titulaire/ikpi') def obj_iban(self): iban = Map(Dict('key'), Env('ibans')(self), default=NotAvailable)(self) if not empty(iban): if not is_iban_valid(iban): iban = rib2iban(rebuild_rib(iban)) return iban return None def obj_ownership(self): indic = Dict('titulaire/indicTitulaireCollectif', default=None)(self) # The boolean is in the form of a string ('true' or 'false') if indic == 'true': return AccountOwnership.CO_OWNER elif indic == 'false': if self.page.get_user_ikpi() == Dict('titulaire/ikpi')( self): return AccountOwnership.OWNER return AccountOwnership.ATTORNEY return NotAvailable # softcap not used TODO don't pass this key when backend is ready # deferred cb can disappear the day after the appear, so 0 as day_for_softcap obj__bisoftcap = { 'deferred_cb': { 'softcap_day': 1000, 'day_for_softcap': 1 } }