def get_card_transactions(self, latest_date, ongoing_coming): for item in self.doc.xpath('//table[@class="ca-table"][2]//tr[td]'): if CleanText('./td[2]/b')(item): # This node is a summary containing the 'date' for all following transactions. raw_date = Regexp(CleanText('./td[2]/b/text()'), r'le (.*) :')(item) if latest_date and parse_french_date( raw_date).date() > latest_date: # This summary has already been fetched continue latest_date = parse_french_date(raw_date).date() if latest_date < ongoing_coming: # This summary is anterior to the ongoing_coming so we create a transaction from it tr = FrenchTransaction() tr.date = tr.rdate = latest_date tr.raw = tr.label = CleanText('./td[2]/b/text()')(item) tr.amount = -CleanDecimal.French( './td[position()=last()]')(item) tr.type = FrenchTransaction.TYPE_CARD_SUMMARY yield tr else: # This node is a real transaction. # Its 'date' is the date of the most recently encountered summary node. tr = FrenchTransaction() tr.date = latest_date date_guesser = LinearDateGuesser(latest_date) tr.rdate = tr.bdate = DateGuesser( CleanText('./td[1]//text()'), date_guesser=date_guesser)(item) tr.label = tr.raw = CleanText('./td[2]')(item) tr.amount = CleanDecimal.French('./td[last()]')(item) tr.type = FrenchTransaction.TYPE_DEFERRED_CARD yield tr
class fill_account(ItemElement): obj_balance = CleanDecimal.French( '//ul[has-class("m-data-group")]//strong') obj_currency = Currency('//ul[has-class("m-data-group")]//strong') obj_valuation_diff = CleanDecimal.French( '//h3[contains(., "value latente")]/following-sibling::p[1]', default=NotAvailable)
class item(ItemElement): klass = Account TYPES = { u'assurance vie': Account.TYPE_LIFE_INSURANCE, u'perp': Account.TYPE_PERP, u'epargne retraite agipi pair': Account.TYPE_PERP, u'novial avenir': Account.TYPE_MADELIN, u'epargne retraite novial': Account.TYPE_LIFE_INSURANCE, } condition = lambda self: Field('balance')(self) is not NotAvailable obj_id = Regexp(CleanText('.//span[has-class("small-title")]'), r'([\d/]+)') obj_label = CleanText('.//h3[has-class("card-title")]') obj_balance = CleanDecimal.French('.//p[has-class("amount-card")]') obj_valuation_diff = CleanDecimal.French( './/p[@class="performance"]', default=NotAvailable) def obj_url(self): url = Attr('.', 'data-route')(self) # The Assurance Vie xpath recently changed so we must verify that all # the accounts now have "/savings/" instead of "/assurances-vie/". assert "/savings/" in url return url obj_currency = Currency('.//p[has-class("amount-card")]') obj__acctype = "investment" obj_type = MapIn(Lower(Field('label')), TYPES, Account.TYPE_UNKNOWN)
class item(ItemElement): klass = Account TYPES = { 'assurance vie': Account.TYPE_LIFE_INSURANCE, 'perp': Account.TYPE_PERP, 'epargne retraite agipi pair': Account.TYPE_PERP, 'epargne retraite agipi far': Account.TYPE_MADELIN, 'epargne retraite ma retraite': Account.TYPE_PER, 'novial avenir': Account.TYPE_MADELIN, 'epargne retraite novial': Account.TYPE_LIFE_INSURANCE, } obj_id = Regexp(CleanText('.//span[has-class("small-title")]'), r'([\d/]+)') obj_number = obj_id obj_label = CleanText('.//h3[has-class("card-title")]') obj_balance = CleanDecimal.French('.//p[has-class("amount-card")]') obj_valuation_diff = CleanDecimal.French( './/p[@class="performance"]', default=NotAvailable) obj_currency = Currency('.//p[has-class("amount-card")]') obj__acctype = "investment" obj_type = MapIn(Lower(Field('label')), TYPES, Account.TYPE_UNKNOWN) obj_url = Attr('.', 'data-module-open-link--link') obj_ownership = AccountOwnership.OWNER
def get_performance_history(self): # The positions of the columns depend on the age of the investment fund. # For example, if the fund is younger than 5 years, there will be not '5 ans' column. durations = [ CleanText('.')(el) for el in self.doc.xpath( '//div[contains(@class, "fpPerfglissanteclassique")]//th') ] values = [ CleanText('.')(el) for el in self.doc.xpath( '//div[contains(@class, "fpPerfglissanteclassique")]//tr[td[text()="Fonds"]]//td' ) ] matches = dict(zip(durations, values)) # We do not fill the performance dictionary if no performance is available, # otherwise it will overwrite the data obtained from the JSON with empty values. perfs = {} if matches.get('1 an'): perfs[1] = percent_to_ratio( CleanDecimal.French(default=NotAvailable).filter( matches['1 an'])) if matches.get('3 ans'): perfs[3] = percent_to_ratio( CleanDecimal.French(default=NotAvailable).filter( matches['3 ans'])) if matches.get('5 ans'): perfs[5] = percent_to_ratio( CleanDecimal.French(default=NotAvailable).filter( matches['5 ans'])) return perfs
def parse(self, obj): self.env['date'] = DateGuesser(CleanText('./td[1]'), Env('date_guesser'))(self) self.env['vdate'] = NotAvailable if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 4]')(self): # History table with 4 columns self.env['raw'] = CleanText('./td[2]', children=False)(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 5]')(self): # History table with 5 columns self.env['raw'] = CleanText('./td[3]', children=False)(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 6]')(self): # History table with 6 columns (contains vdate) self.env['raw'] = CleanText('./td[4]', children=False)(self) self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 7]')(self): # History table with 7 columns self.env['amount'] = Coalesce( CleanDecimal.French('./td[6]', sign=lambda x: -1, default=None), CleanDecimal.French('./td[7]', default=None) )(self) if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//th[a[contains(text(), "Valeur")]]')(self): # With vdate column ('Valeur') self.env['raw'] = CleanText('./td[4]', children=False)(self) self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self) else: # Without any vdate column self.env['raw'] = CleanText('./td[3]', children=False)(self) else: assert False, 'This type of history table is not handled yet!'
def obj_valuation(self): # Handle discrepancies between aviva & afer (Coalesce does not work here) if CleanText('./td[contains(@data-label, "Valeur de rachat")]' )(self): return CleanDecimal.French( './td[contains(@data-label, "Valeur de rachat")]')( self) return CleanDecimal.French( CleanText('./td[contains(@data-label, "Montant")]', children=False))(self)
class item(ItemElement): klass = Transaction obj_date = Date(CleanText('.//td[@headers="date"]'), dayfirst=True) obj_raw = Transaction.Raw('.//td[@headers="libelle"]') obj_amount = Coalesce( CleanDecimal.French('.//td[@headers="debit"]', default=NotAvailable), CleanDecimal.French('.//td[@headers="credit"]', default=NotAvailable), )
def obj_balance(self): value_balance = CleanText( TableCell('value_balance', default='', colspan=True))(self) # Skip invalid balance values in the 'Value' column (for example for Revolving credits) if value_balance not in ('', 'Montant disponible'): return CleanDecimal.French().filter(value_balance) return CleanDecimal.French( CleanText( TableCell('operation_balance', default='', colspan=True)))(self)
def obj_investments(self): investments = [] for elem in self.xpath( './following-sibling::div[1]//tbody/tr'): inv = Investment() inv.label = CleanText('./td[1]')(elem) inv.valuation = Coalesce( CleanDecimal.French('./td[2]/p', default=NotAvailable), CleanDecimal.French('./td[2]'))(elem) investments.append(inv) return investments
class item(ItemElement): klass = Account obj_id = MAIN_ID obj_label = 'Compte Bolden' obj_type = Account.TYPE_MARKET obj_currency = 'EUR' obj_balance = CleanDecimal.French( '//div[p[has-class("investor-state") and contains(text(),"Total compte Bolden :")]]/p[has-class("investor-status")]' ) obj_valuation_diff = CleanDecimal.French( '//div[has-class("rent-total")]')
def parse(self, obj): # We must handle Loan tables with 5 or 6 columns if CleanText('//tr[contains(@class, "colcelligne")][count(td) = 5]')(self): # History table with 4 columns (no loan details) self.env['next_payment_amount'] = NotAvailable self.env['total_amount'] = NotAvailable self.env['balance'] = CleanDecimal.French('./td[4]//*[@class="montant3" or @class="montant4"]', default=NotAvailable)(self) elif CleanText('//tr[contains(@class, "colcelligne")][count(td) = 6]')(self): # History table with 5 columns (contains next_payment_amount & total_amount) self.env['next_payment_amount'] = CleanDecimal.French('./td[3]//*[@class="montant3"]', default=NotAvailable)(self) self.env['total_amount'] = CleanDecimal.French('./td[4]//*[@class="montant3"]', default=NotAvailable)(self) self.env['balance'] = CleanDecimal.French('./td[5]//*[@class="montant3"]', default=NotAvailable)(self)
class item(ItemElement): klass = Investment obj_label = CleanText('.//span[@class="uppercase"]') obj_valuation = CleanDecimal.French( './/span[@class="box"][span[span[text()="Montant estimé"]]]/span[2]/span' ) obj_quantity = CleanDecimal.French( './/span[@class="box"][span[span[text()="Nombre de part"]]]/span[2]/span' ) obj_unitvalue = CleanDecimal.French( './/span[@class="box"][span[span[text()="Valeur liquidative"]]]/span[2]/span' ) obj_unitprice = CleanDecimal.French( './/span[@class="box"][span[span[text()="Prix de revient"]]]/span[2]/span', default=NotAvailable) obj_portfolio_share = Eval( lambda x: x / 100, CleanDecimal.French( './/span[@class="box"][span[span[text()="Répartition"]]]/span[2]/span' )) def obj_diff_ratio(self): # Euro funds have '-' instead of a diff_ratio value if (CleanText( './/span[@class="box"][span[span[text()="+/- value latente (%)"]]]/span[2]/span' )(self) == '-'): return NotAvailable return Eval( lambda x: x / 100, CleanDecimal.French( './/span[@class="box"][span[span[text()="+/- value latente (%)"]]]/span[2]/span', ))(self) def obj_diff(self): if Field('diff_ratio')(self) == NotAvailable: return NotAvailable return CleanDecimal.French( './/span[@class="box"][span[span[text()="+/- value latente"]]]/span[2]/span' )(self) def obj_code(self): code = CleanText('.//span[@class="cl-secondary"]')(self) if is_isin_valid(code): return code return NotAvailable def obj_code_type(self): if Field('code')(self) == NotAvailable: return NotAvailable return Investment.CODE_TYPE_ISIN
class item(ItemElement): klass = Investment obj_label = CleanText( './div[@data-label="Nom du support" or @data-label="Support cible"]/span[1]' ) obj_quantity = CleanDecimal.French( './div[contains(@data-label, "Nombre")]', default=NotAvailable) obj_unitvalue = CleanDecimal.French( './div[contains(@data-label, "Valeur")]', default=NotAvailable) obj_valuation = CleanDecimal.French( './div[contains(@data-label, "Montant")]', default=NotAvailable) obj_vdate = Env('date')
def obj_performance_history(self): # Fetching the performance history (1 year, 3 years & 5 years) perfs = {} if not self.xpath('//table[tr[td[text()="Performance"]]]'): return # Available performance history: 1 month, 3 months, 6 months, 1 year, 2 years, 3 years, 4 years & 5 years. # We need to match the durations with their respective values. durations = [CleanText('.')(el) for el in self.xpath('//table[tr[td[text()="Performance"]]]//tr//th')] values = [CleanText('.')(el) for el in self.xpath('//table[tr[td[text()="Performance"]]]//tr//td')] matches = dict(zip(durations, values)) perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1A'])) perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3A'])) perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5A'])) return perfs
def get_liquidity(self): # Not all accounts have a Liquidity element liquidity_element = CleanDecimal.French( '//td[contains(text(), "Solde espèces en euros")]//following-sibling::td[position()=1]', default=None)(self.doc) if liquidity_element: return create_french_liquidity(liquidity_element)
def handle_response(self, account, recipient, amount, reason): # handle error error_msg = CleanText('//div[@id="blocErreur"]')(self.doc) if error_msg: raise TransferBankError(message=error_msg) account_txt = CleanText('//form//h3[contains(text(), "débiter")]//following::span[1]', replace=[(' ', '')])(self.doc) recipient_txt = CleanText('//form//h3[contains(text(), "créditer")]//following::span[1]', replace=[(' ', '')])(self.doc) assert account.id in account_txt or ''.join(account.label.split()) == account_txt, 'Something went wrong' assert recipient.id in recipient_txt or ''.join(recipient.label.split()) == recipient_txt, 'Something went wrong' amount_element = self.doc.xpath('//h3[contains(text(), "Montant du virement")]//following::span[@class="price"]')[0] r_amount = CleanDecimal.French('.')(amount_element) exec_date = Date(CleanText('//h3[contains(text(), "virement")]//following::span[@class="date"]'), dayfirst=True)(self.doc) currency = FrenchTransaction.Currency('.')(amount_element) transfer = Transfer() transfer.currency = currency transfer.amount = r_amount transfer.account_iban = account.iban transfer.recipient_iban = recipient.iban transfer.account_id = account.id transfer.recipient_id = recipient.id transfer.exec_date = exec_date transfer.label = reason transfer.account_label = account.label transfer.recipient_label = recipient.label transfer.account_balance = account.balance return transfer
class get_unique_card(ItemElement): item_xpath = '//table[@class="ca-table"][@summary]' klass = Account # Transform 'n° 4999 78xx xxxx xx72' into '499978xxxxxxxx72' obj_number = CleanText('//table[@class="ca-table"][@summary]//tr[@class="ligne-impaire"]/td[@class="cel-texte"][1]', replace=[(' ', ''), ('n°', '')]) # Card ID is formatted as '499978xxxxxxxx72MrFirstnameLastname-' obj_id = Format('%s%s', Field('number'), CleanText('//table[@class="ca-table"][@summary]//caption[@class="caption"]//b', replace=[(' ', '')])) # Card label is formatted as 'Carte VISA Premier - Mr M Lastname' obj_label = Format('%s - %s', CleanText('//table[@class="ca-table"][@summary]//tr[@class="ligne-impaire ligne-bleu"]/th[@id="compte-1"]'), CleanText('//table[@class="ca-table"][@summary]//caption[@class="caption"]//b')) obj_balance = CleanDecimal(0) obj_coming = CleanDecimal.French('//table[@class="ca-table"][@summary]//tr[@class="ligne-paire"]//td[@class="cel-num"]', default=0) obj_currency = Currency(Regexp(CleanText('//th[contains(text(), "Montant en")]'), r'^Montant en (.*)')) obj_type = Account.TYPE_CARD obj__form = None
def handle_response(self, recipient): json_response = self.doc['donnees'] transfer = Transfer() transfer.id = json_response['idVirement'] transfer.label = json_response['motif'] transfer.amount = CleanDecimal.French( (CleanText(Dict('montantToDisplay'))))(json_response) transfer.currency = json_response['devise'] transfer.exec_date = Date(Dict('dateExecution'), dayfirst=True)(json_response) transfer.account_id = Format('%s%s', Dict('codeGuichet'), Dict('numeroCompte'))( json_response['compteEmetteur']) transfer.account_iban = json_response['compteEmetteur']['iban'] transfer.account_label = json_response['compteEmetteur'][ 'libelleToDisplay'] assert recipient._json_id == json_response['compteBeneficiaire']['id'] transfer.recipient_id = recipient.id transfer.recipient_iban = json_response['compteBeneficiaire']['iban'] transfer.recipient_label = json_response['compteBeneficiaire'][ 'libelleToDisplay'] return transfer
def obj_balance(self): # The page can be randomly in french or english and # the valuations can be "€12,345.67" or "12 345,67 €" try: return CleanDecimal.French('./td[6]')(self) except NumberFormatError: return CleanDecimal.US('./td[6]')(self)
def obj_balance(self): # This wonderful website randomly displays separators as '.' or ',' # For example, numbers can look like "€12,345.67" or "12 345,67 €" try: return CleanDecimal.French('./td[6]')(self) except NumberFormatError: return CleanDecimal.US('./td[6]')(self)
class item(ItemElement): klass = Transaction TRANSACTION_TYPES = { 'FACTURE CB': Transaction.TYPE_CARD, 'RETRAIT CB': Transaction.TYPE_WITHDRAWAL, "TRANSACTION INITIALE RECUE D'AVOIR": Transaction.TYPE_PAYBACK, } obj_label = CleanText(Dict('Raison sociale commerçant')) obj_amount = CleanDecimal.French(Dict('Montant EUR')) obj_original_currency = CleanText(Dict("Devise d'origine")) obj_rdate = Date(CleanText(Dict("Date d'opération")), dayfirst=True) obj_date = Date(CleanText(Dict('Date débit-crédit')), dayfirst=True) obj_type = MapIn(CleanText(Dict('Libellé opération')), TRANSACTION_TYPES) def obj_commission(self): commission = CleanDecimal.French(Dict('Commission'))(self) if commission != 0: # We don't want to return null commissions return commission return NotAvailable def obj_original_amount(self): original_amount = CleanDecimal.French( Dict("Montant d'origine"))(self) if original_amount != 0: # We don't want to return null original_amounts return original_amount return NotAvailable def obj__coming(self): return Field('date')(self) >= date.today()
def fill_diff_currency(self, account): valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) account.balance = CleanDecimal.French(Regexp(CleanText('//table[@class="v1-formbloc"]//td[@class="v1-labels"]//b[contains(text(), "Estimation du contrat")]/ancestor::td/following-sibling::td[1]'), r'^(.+) EUR'))(self.doc) # NC == Non communiqué if valuation_diff and "NC" not in valuation_diff: account.valuation_diff = MyDecimal().filter(valuation_diff) account.currency = account.get_currency(valuation_diff)
def get_account_details(self, account_id): balance = CleanDecimal.French( '//a[div[div[span[span[contains(text(), "%s")]]]]]/div[1]/div[2]/span/span' % account_id, default=NotAvailable)(self.doc) currency = Currency( '//a[div[div[span[span[contains(text(), "%s")]]]]]/div[1]/div[2]/span/span' % account_id, default=NotAvailable)(self.doc) label = CleanText( '//a[div[div[span[span[contains(text(), "%s")]]]]]/div[1]/div[1]/span/span' % account_id, default=NotAvailable)(self.doc) url = Link('//a[div[div[span[span[contains(text(), "%s")]]]]]' % account_id, default=None)(self.doc) if url: account_url = 'https://bgpi-gestionprivee.credit-agricole.fr' + url else: account_url = None return balance, currency, label, account_url
class item(ItemElement): klass = Account obj_id = CleanText('./td[2]') obj_number = Field('id') obj_label = CleanText('./td/span[@class="gras"]') obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) # Accounts without balance will be skipped later on obj_balance = CleanDecimal.French('./td//*[@class="montant3"]', default=NotAvailable) obj_currency = Currency('./td[@class="cel-devise"]') obj_iban = None obj__form = None def obj_url(self): url = Link('./td[2]/a', default=None)(self) if url and 'BGPI' in url: # This URL is just the BGPI home page, not the account itself. # The real account URL will be set by get_account_details() in BGPISpace. return 'BGPI' return url def validate(self, obj): # Skip 'ESPE INTEG' accounts, these liquidities are already available # on the associated Market account on the Netfinca website return obj.label != 'ESPE INTEG'
class item(ItemElement): klass = Account def condition(self): # Ignore cards that do not have a coming return CleanText('.//tr[1]/td[@class="cel-num"]')(self) # Transform 'n° 4999 78xx xxxx xx72' into '499978xxxxxxxx72' obj_number = CleanText('.//caption/span[@class="tdb-cartes-num"]', replace=[(' ', ''), ('n°', '')]) # The raw number is used to access multiple cards details obj__raw_number = CleanText( './/caption/span[@class="tdb-cartes-num"]') # Multiple card IDs are formatted as '499978xxxxxxxx72MrFirstnameLastname' obj_id = Format( '%s%s', Field('number'), CleanText('.//caption/span[@class="tdb-cartes-prop"]', replace=[(' ', '')])) # Card label is formatted as 'Carte VISA Premier - Mr M Lastname' obj_label = Format( '%s - %s', CleanText('.//caption/span[has-class("tdb-cartes-carte")]'), CleanText('.//caption/span[has-class("tdb-cartes-prop")]')) obj_type = Account.TYPE_CARD obj_balance = CleanDecimal(0) obj_coming = CleanDecimal.French( './/tr[1]/td[position() = last()]', default=0) obj_currency = Currency( Regexp(CleanText('//span[contains(text(), "Montants en")]'), r'^Montants en (.*)')) obj__form = None
def obj_diff_percent(self): # Euro funds have '-' instead of a diff_percent value if CleanText('.//span[@class="box"][span[span[text()="+/- value latente (%)"]]]/span[2]/span')(self) == '-': return NotAvailable return Eval( lambda x: x / 100, CleanDecimal.French('.//span[@class="box"][span[span[text()="+/- value latente (%)"]]]/span[2]/span') )(self)
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_valuation = CleanDecimal.French(Regexp( CleanText(TableCell('details')), r'^(.*?) €', # can be 100,00 € + Frais de 0,90 € ))
def get_investment_performances(self): # TODO: Handle supplementary attributes for AMFSGPage self.logger.warning('This investment leads to AMFSGPage, please handle SRRI, asset_category and recommended_period.') # Fetching the performance history (1 year, 3 years & 5 years) perfs = {} if not self.doc.xpath('//table[tr[th[contains(text(), "Performances glissantes")]]]'): return # Available performance durations are: 1 week, 1 month, 1 year, 3 years & 5 years. # We need to match the durations with their respective values. durations = [CleanText('.')(el) for el in self.doc.xpath('//table[tr[th[contains(text(), "Performances glissantes")]]]//tr[2]//th')] values = [CleanText('.')(el) for el in self.doc.xpath('//table[tr[th[contains(text(), "Performances glissantes")]]]//td')] matches = dict(zip(durations, values)) perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1 an *'])) perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3 ans *'])) perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5 ans *'])) return perfs
def parse_decimal(self, string, replace_dots): string = CleanText(None).filter(string) if string in ('-', '*'): return NotAvailable # Decimal separators can be ',' or '.' depending on the column if replace_dots: return CleanDecimal.French().filter(string) return CleanDecimal.SI().filter(string)