def obj__redacted_card(self): raw = Field('raw')(self) if not raw.startswith('FACTURE CARTE') or ' SUIVANT RELEVE DU ' in raw: return page = Async('details').loaded_page(self) return page.get_redacted_card()
def parse(self, el): try: page = Async('details').loaded_page(self) except AttributeError: page = None self.env['investments'] = list(page.get_investments( )) if page and 'numMvt' in page.url else []
def obj_investments(self): try: page = Async('invs').loaded_page(self) return list(page.iter_investments()) except AttributeError: # No investments available return list()
def obj_code(self): val = Async( 'details', CleanText( '//td[@class="libelle-normal" and contains(.,"CodeISIN")]', default=NotAvailable))(self) return val.split('CodeISIN : ')[1] if val else val
class item(ItemElement): klass = Investment # ASV.popup('/general?command=displayAVEuroEpargne') load_details = Attr('.//div[has-class("asv_fond_view")]//a', 'onclick') & Regexp(pattern="'(.*)'") & AsyncLoad obj_label = CleanText('.//span[has-class("asv_cat_lbl")]') # XXX I would like to do that... but 1. CleanText doesn't deal with default 2. default values can't be filters yet #obj_code = Async('details') & CleanText('//li[contains(text(), "Code ISIN")]/span') | (Field('label') & Format('XX%s', Slugify)) obj_code = Async('details') & CleanText('//li[contains(text(), "Code ISIN")]/span') obj_id = obj_code obj_description = Async('details') & CleanText('//h5') obj_quantity = CleanDecimal('.//dl[contains(dt/text(), "Nombre de parts")]/dd', replace_dots=True) obj_unitvalue = CleanDecimal('.//dl[contains(dt/text(), "Valeur de part")]/dd', replace_dots=True) obj_valuation = CleanDecimal('.//dl[has-class("ligne-montant")]/dd', replace_dots=True) def obj_unitprice(self): if 'eurossima' in self.el.get('class'): return self.obj.unitvalue percent = CleanDecimal('.//dl[has-class("ligne-pmvalue")]/dd', replace_dots=True)(self) return (self.obj.unitvalue / (1 + percent/Decimal('100.0'))).quantize(Decimal('1.00')) def obj_diff(self): return (self.obj.valuation - (self.obj.quantity * self.obj.unitprice)).quantize(Decimal('1.00')) def validate(self, obj): if not obj.id: obj.id = obj.code = 'XX' + Slugify(self.obj_label)(self) return True
def validate(self, obj): if obj.category == 'RELEVE CB': return raw = Async( 'details', CleanText( u'//td[contains(text(), "Libellé")]/following-sibling::*[1]|//td[contains(text(), "Nom du donneur")]/following-sibling::*[1]', default=obj.raw))(self) if raw: obj.raw = raw obj.label = raw if not obj.date: obj.date = Async( 'details', Date(CleanText( u'//td[contains(text(), "Date de l\'opération")]/following-sibling::*[1]', default=u''), default=NotAvailable))(self) obj.rdate = obj.date obj.vdate = Async( 'details', Date(CleanText( u'//td[contains(text(), "Date de valeur")]/following-sibling::*[1]', default=u''), default=NotAvailable))(self) obj.amount = Async( 'details', CleanDecimal( u'//td[contains(text(), "Montant")]/following-sibling::*[1]', replace_dots=True, default=NotAvailable))(self) return True
class item(ItemElement): klass = Account load_details = Field('url') & AsyncLoad obj_label = CleanText('.//a[@class="account--name"] | .//div[@class="account--name"]') obj_balance = CleanDecimal('.//a[has-class("account--balance")]', replace_dots=True) obj_currency = FrenchTransaction.Currency('.//a[has-class("account--balance")]') obj_valuation_diff = Async('details') & CleanDecimal('//li[h4[text()="Total des +/- values"]]/h3 |\ //li[span[text()="Total des +/- values latentes"]]/span[has-class("overview__value")]', replace_dots=True, default=NotAvailable) obj__card = Async('details') & Attr('//a[@data-modal-behavior="credit_card-modal-trigger"]', 'href', default=NotAvailable) obj__holder = None def obj_coming(self): # Don't duplicate coming (card balance with account coming) # TODO: fetch coming which is not card coming for account with cards. if self.obj__card(self): return NotAvailable return Async('details', CleanDecimal(u'//li[h4[text()="Mouvements à venir"]]/h3', replace_dots=True, default=NotAvailable))(self) def obj_id(self): id = Async('details', Regexp(CleanText('//h3[has-class("account-number")]'), r'(\d+)', default=NotAvailable))(self) if not id: raise SkipItem() return id def obj_type(self): for word in Field('label')(self).lower().split(): v = self.page.ACCOUNT_TYPES.get(word) if v: return v category = CleanText('./preceding-sibling::tr[has-class("list--accounts--master")]//h4')(self) v = self.page.ACCOUNT_TYPES.get(category) if v: return v page = Async('details').loaded_page(self) if isinstance(page, LoanPage): return Account.TYPE_LOAN return Account.TYPE_UNKNOWN def obj_url(self): link = Attr('.//a[@class="account--name"] | .//a[2]', 'href', default=NotAvailable)(self) if not self.page.browser.webid: self.page.browser.webid = re.search('\/([^\/|?|$]{32})(\/|\?|$)', link).group(1) return urljoin(self.page.url, link) def obj__webid(self): m = re.search('([a-z\d]{32})', Field('url')(self)) if m: return m.group(1) return None # We do not yield other banks accounts for the moment. def validate(self, obj): return not Async('details', CleanText(u'//h4[contains(text(), "Établissement bancaire")]'))(self) and not \ Async('details', CleanText(u'//h4/div[contains(text(), "Établissement bancaire")]'))(self)
def obj_photos(self): page_doc = Async('details').loaded_page(self).doc photos = [] for photo in page_doc.xpath('//div[@id="bxSliderContainer"]//ul//li//img'): url = Attr('.', 'src')(photo) if url[0] != '/': photos.append(HousingPhoto(url)) return photos
class item(ItemElement): klass = Investment # ASV.popup('/general?command=displayAVEuroEpargne') load_details = Attr( './/div[has-class("asv_fond_view")]//a', 'onclick') & Regexp(pattern="'(.*)'") & AsyncLoad obj_label = CleanText('.//span[has-class("asv_cat_lbl")]') obj_code = Async('details') & CleanText( '//li[contains(text(), "Code ISIN")]/span[1]') obj_id = obj_code obj_description = Async('details') & CleanText('//h5') obj_quantity = CleanDecimal( './/dl[contains(dt/text(), "Nombre de parts")]/dd', replace_dots=True) obj_unitvalue = CleanDecimal( './/dl[contains(dt/text(), "Valeur de part")]/dd', replace_dots=True) # There are two kind of lists: # - Header contains percent and valuation is in a specific row ("ligne-montant") # - Header contains valuation, there is no "ligne-montant" row, and percent is in a specific row obj_valuation = CleanDecimal( './/dl[has-class("ligne-montant")]/dd | .//dd[@data-show="header" and not(contains(text(), "%"))]', replace_dots=True) def obj_unitprice(self): if 'eurossima' in self.el.get('class') or \ 'fondsEuro' in self.el.get('class'): # in this case, the content of field is: # <span data-sort="pm_value" class="pmvalue positive">NOT_A_NUMBER</span> return self.obj.unitvalue if self.el.xpath( './/span[has-class("pmvalue")]')[0].text == u'+∞ %': percent = NotAvailable return NotAvailable else: percent = CleanDecimal('.//span[has-class("pmvalue")]', replace_dots=True)(self) return (self.obj.unitvalue / (1 + percent / Decimal('100.0'))).quantize( Decimal('1.00')) def obj_diff(self): if not self.obj.quantity: # Quantity of euro funds is null. return Decimal('0.00') if not self.obj.unitprice: return NotAvailable return (self.obj.valuation - (self.obj.quantity * self.obj.unitprice)).quantize( Decimal('1.00'))
class item(ItemElement): klass = Investment # ASV.popup('/general?command=displayAVEuroEpargne') load_details = Attr( './/div[has-class("asv_fond_view")]//a', 'onclick') & Regexp(pattern="'(.*)'") & AsyncLoad obj_label = CleanText('.//span[has-class("asv_cat_lbl")]') # XXX I would like to do that... but 1. CleanText doesn't deal with default 2. default values can't be filters yet #obj_code = Async('details') & CleanText('//li[contains(text(), "Code ISIN")]/span') | (Field('label') & Format('XX%s', Slugify)) obj_code = Async('details') & CleanText( '//li[contains(text(), "Code ISIN")]/span') obj_id = obj_code obj_description = Async('details') & CleanText('//h5') obj_quantity = CleanDecimal( './/dl[contains(dt/text(), "Nombre de parts")]/dd', replace_dots=True) obj_unitvalue = CleanDecimal( './/dl[contains(dt/text(), "Valeur de part")]/dd', replace_dots=True) # There are two kind of lists: # - Header contains percent and valuation is in a specific row ("ligne-montant") # - Header contains valuation, there is no "ligne-montant" row, and percent is in a specific row obj_valuation = CleanDecimal( './/dl[has-class("ligne-montant")]/dd | .//dd[@data-show="header" and not(contains(text(), "%"))]', replace_dots=True) def obj_unitprice(self): if 'eurossima' in self.el.get('class') or \ 'fondsEuro' in self.el.get('class'): # in this case, the content of field is: # <span data-sort="pm_value" class="pmvalue positive">NOT_A_NUMBER</span> return self.obj.unitvalue percent = CleanDecimal('.//span[has-class("pmvalue")]', replace_dots=True)(self) return (self.obj.unitvalue / (1 + percent / Decimal('100.0'))).quantize( Decimal('1.00')) def obj_diff(self): if not self.obj.quantity: # Quantity of euro funds is null. return Decimal('0.00') return (self.obj.valuation - (self.obj.quantity * self.obj.unitprice)).quantize( Decimal('1.00')) def validate(self, obj): if not obj.id: obj.id = obj.code = 'XX' + Slugify(self.obj_label)(self) return True
def obj_iban(self): rib_page = Async('iban').loaded_page(self) if 'RibPdf' in rib_page.url: return rib_page.get_iban() return Join( '', Regexp(CleanText( '//td[has-class("ColonneCode")][contains(text(), "IBAN")]' ), r'\b((?!IBAN)[A-Z0-9]+)\b', nth='*'))(rib_page.doc) or NotAvailable
def obj_balance(self): if Field('type')(self) != Account.TYPE_CARD: balance = Field('_amount')(self) if Field('type')(self) in [Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET]: page = Async('details').loaded_page(self) if isinstance(page, MarketPage): updated_balance = page.get_balance(Field('type')(self)) if updated_balance is not None: return updated_balance return balance return Decimal('0')
class item(ItemElement): klass = Account load_details = Field('_link') & AsyncLoad obj_id = Async('details') & Regexp( CleanText('//h3[has-class("account-number")]'), r'(\d+)') obj_label = CleanText( './/a[@class="account--name"] | .//div[@class="account--name"]' ) obj_balance = CleanDecimal('.//a[has-class("account--balance")]', replace_dots=True) obj_currency = FrenchTransaction.Currency( './/a[has-class("account--balance")]') obj_valuation_diff = Async('details') & CleanDecimal( '//li[h4[text()="Total des +/- values"]]/h3 |\ //li[span[text()="Total des +/- values latentes"]]/span[has-class("overview__value")]', replace_dots=True, default=NotAvailable) obj_coming = Async('details') & CleanDecimal( u'//li[h4[text()="Mouvements à venir"]]/h3', replace_dots=True, default=NotAvailable) obj__card = Async('details') & Attr( '//a[@data-modal-behavior="credit_card-modal-trigger"]', 'href', default=NotAvailable) obj__holder = None obj__webid = None def obj_type(self): return self.page.ACCOUNT_TYPES.get( CleanText( './preceding-sibling::tr[has-class("list--accounts--master")]//h4' )(self), Account.TYPE_UNKNOWN) def obj__link(self): link = Attr('.//a[@class="account--name"] | .//a[2]', 'href', default=NotAvailable)(self) if not self.page.browser.webid: self.page.browser.webid = re.search( '\/([^\/|?|$]{32})(\/|\?|$)', link).group(1) return link # We do not yield other banks accounts for the moment. def validate(self, obj): return not Async( 'details', CleanText( u'//h4[contains(text(), "Établissement bancaire")]'))( self)
def obj_details(self): page_doc = Async('details').loaded_page(self).doc return { 'GES': CleanText('//span[@id="gassymbol"]', '')(page_doc), 'DPE': CleanText('//span[@id="energysymbol"]', '')(page_doc), }
class item(ItemElement): klass = Bill load_details = Attr('./td/a', 'href') & AsyncLoad obj_id = Format('%s_%s', Env('email'), CleanDecimal(TableCell('id'))) obj__url = Async('details') & Attr('//a[contains(@href, "facture")]', 'href', default=NotAvailable) obj_date = Date(CleanText(TableCell('date'))) obj_format = u"pdf" obj_label = Async('details') & CleanText('//table/tr/td[@class="Prod"]') obj_type = u"bill" obj_price = CleanDecimal(TableCell('price'), replace_dots=True) obj_currency = u"€" def parse(self, el): self.env['email'] = self.page.browser.username
class item(ItemElement): klass = Account load_details = Link(u'//td[2]/a') & AsyncLoad obj_id = CleanText(TableCell('id')) obj_label = CleanText(TableCell('label')) obj_balance = MyDecimal(TableCell('balance')) obj_type = Env('type') obj_iban = NotAvailable obj_valuation_diff = Async('details') & MyDecimal( '//th[span[contains(text(), \ "Performance")]]/following-sibling::td[1]') obj__page = Env('page') obj__acctype = "investment" def condition(self): return CleanText(TableCell('balance'))(self) def parse(self, el): page = Async('details').loaded_page(self) type = CleanText().filter( page.doc.xpath('//th[contains(text(), \ "Cadre fiscal")]/following-sibling::td[1]')) if not type: raise SkipItem() self.env['type'] = self.page.TYPES.get(type.lower(), Account.TYPE_UNKNOWN) self.env['page'] = page
def obj_type(self): # card url is /compte/cav/xxx/carte/yyy so reverse to match "carte" before "cav" for word in Field('url')(self).lower().split('/')[::-1]: v = self.page.ACCOUNT_TYPES.get(word) if v: return v for word in Field('label')(self).replace('_', ' ').lower().split(): v = self.page.ACCOUNT_TYPES.get(word) if v: return v category = CleanText( './preceding-sibling::tr[has-class("list--accounts--master")]//h4' )(self) v = self.page.ACCOUNT_TYPES.get(category.lower()) if v: return v page = Async('details').loaded_page(self) if isinstance(page, LoanPage): return Account.TYPE_LOAN return Account.TYPE_UNKNOWN
def obj_iban(self): try: return Async( 'details', CleanText('(.//div[@class="iban"]/p)[1]', replace=[(' ', '')]))(self) except ServerError: return NotAvailable
def validate(self, obj): if obj.category == 'RELEVE CB': obj.type = Transaction.TYPE_CARD_SUMMARY obj.deleted = True raw = Async( 'details', CleanText( u'//td[contains(text(), "Libellé")]/following-sibling::*[1]|//td[contains(text(), "Nom du donneur")]/following-sibling::*[1]', default=obj.raw))(self) if raw: if obj.raw in raw or raw in obj.raw or ' ' not in obj.raw: obj.raw = raw obj.label = raw else: obj.label = '%s %s' % (obj.raw, raw) obj.raw = '%s %s' % (obj.raw, raw) if not obj.date: obj.date = Async( 'details', Date(CleanText( u'//td[contains(text(), "Date de l\'opération")]/following-sibling::*[1]', default=u''), default=NotAvailable))(self) obj.rdate = obj.date obj.vdate = Async( 'details', Date(CleanText( u'//td[contains(text(), "Date de valeur")]/following-sibling::*[1]', default=u''), default=NotAvailable))(self) obj.amount = Async( 'details', CleanDecimal( u'//td[contains(text(), "Montant")]/following-sibling::*[1]', replace_dots=True, default=NotAvailable))(self) # ugly hack to fix broken html if not obj.amount: obj.amount = Async( 'details', CleanDecimal( u'//td[contains(text(), "Montant")]/following-sibling::*[1]', replace_dots=True, default=NotAvailable))(self) return True
def parse(self, el): page = Async('details').loaded_page(self) label = CleanText(TableCell('label')(self)[0].xpath('./a[1]'))(self) # Try to get gross amount amount = None for td in page.doc.xpath('//td[em[1][contains(text(), "Total")]]/following-sibling::td'): amount = CleanDecimal('.', default=None)(td) if amount: break amount = amount or MyDecimal(TableCell('amount'))(self) if any(word in label.lower() for word in self.page.DEBIT_WORDS): amount = -amount self.env['label'] = label self.env['amount'] = amount self.env['investments'] = list(page.get_investments())
def obj_id(self): id = Async( 'details', Regexp(CleanText('//h3[has-class("account-number")]'), r'(\d+)', default=NotAvailable))(self) if not id: raise SkipItem() return id
def obj_maturity_date(self): if Field('subscription_date')(self): async_page = Async('details').loaded_page(self) date = MyDate( CleanText('//div[@class="bloc Tmargin"]/dl[2]/dd[4]', default=NotAvailable))(async_page.doc) return date return MyDate(CleanText(TableCell('maturity_date')), default=NotAvailable)(self)
class item(ItemElement): klass = Account load_details = Field('_market_link') & AsyncLoad obj_type = Account.TYPE_MARKET obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) obj_valuation_diff = Async('details') & CleanDecimal('//td[contains(text(), "value latente")]/ \ following-sibling::td[1]', replace_dots=True) obj__market_link = Regexp(Attr(TableCell('label'), 'onclick'), "'(.*?)'") obj__link_id = Async('details') & Link(u'//a[text()="Historique"]') obj__transfer_id = None def obj_id(self): return "%sbourse" % "".join(CleanText().filter((TableCell('label')(self)[0]).xpath('./div[not(b)]')).split(' - ')) def obj_label(self): return "%s Bourse" % CleanText().filter((TableCell('label')(self)[0]).xpath('./div[b]'))
def obj_type(self): type = Async('details', CleanText(u'//td[contains(text(), "Nature de l\'opération")]/following-sibling::*[1]'))(self) if not type: return Transaction.TYPE_UNKNOWN for pattern, _type in Transaction.PATTERNS: match = pattern.match(type) if match: return _type break return Transaction.TYPE_UNKNOWN
def obj_url(self): async_page = Async('details').loaded_page(self) url = Link( '//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]', default=NotAvailable)(async_page.doc) if not url: url = Link( '//a[contains(text(), "Imprimer un récapitulatif de commande")]' )(async_page.doc) return url
def obj_amount(self): am = CleanDecimal('.//td[2]', replace_dots=True, default=NotAvailable)(self) if am is not NotAvailable: return am return (Async('details') & CleanDecimal('//div//tr[2]/td[2]', replace_dots=True, default=NotAvailable))(self)
def parse(self, el): page = Async('details').loaded_page(self) type = CleanText().filter( page.doc.xpath('//th[contains(text(), \ "Cadre fiscal")]/following-sibling::td[1]')) if not type: raise SkipItem() self.env['type'] = self.page.TYPES.get(type.lower(), Account.TYPE_UNKNOWN) self.env['page'] = page
def obj_coming(self): # Don't duplicate coming (card balance with account coming) # TODO: fetch coming which is not card coming for account with cards. if self.obj__card(self): return NotAvailable return Async( 'details', CleanDecimal(u'//li[h4[text()="Mouvements à venir"]]/h3', replace_dots=True, default=NotAvailable))(self)
def obj_last_payment_date(self): xpath = '//div[@class="bloc Tmargin"]/div[@class="formline"][2]/span' if 'dont le dernier' in CleanText(xpath)(self): return MyDate( Regexp(CleanText(xpath), ' (\d{2}/\d{2}/\d{4})', default=NotAvailable))(self) async_page = Async('details').loaded_page(self) return MyDate( CleanText('//div[@class="bloc Tmargin"]/dl[1]/dd[2]'), default=NotAvailable)(async_page.doc)
class item(ItemElement): klass = Investment load_details = Regexp(Attr('./td/a', 'onclick', default=""), 'PageExterne\(\'([^\']+)', default=None) & AsyncLoad obj_label = CleanText(TableCell('label')) obj_code = Async('details') & CleanText('//td[contains(text(), "CodeISIN")]/b', default=NotAvailable) obj_quantity = CleanDecimal(TableCell('quantity'), replace_dots=True, default=NotAvailable) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True, default=NotAvailable) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True)
class item(ItemElement): klass = Investment def load_details(self): # create URL with ISIN code if exists code = Field('code')(self) if code: url = 'https://aviva.sixtelekurs.fr/opcvm.hts?isin=%s&environnement=internet' % code return self.page.browser.async_open(url=url) obj_code = Regexp(Attr('./div[@data-label="Nom du support"]/a', 'onclick', default=''), '\"([^\"]+)', default=NotAvailable) obj_quantity = MyDecimal('./div[@data-label="Nombre de parts"]', default=NotAvailable) obj_unitvalue = MyDecimal('./div[@data-label="Valeur de la part"]') obj_valuation = MyDecimal('./div[@data-label="Valeur"]', default=NotAvailable) obj_vdate = Date(CleanText('./div[@data-label="Date de valeur"]'), dayfirst=True, default=NotAvailable) obj_unitprice = Async('details') & CleanDecimal( '//td[@class="donnees"]/span[@id="VL_achat"]', default=NotAvailable) obj_diff_percent = Async('details') & CleanDecimal( '//td[@class="donnees"]/span[@id="Performance"]', default=NotAvailable) obj_description = Async('details') & CleanText( '//td[@class="donnees"]/span[@id="Nature"]', default=NotAvailable) def obj_label(self): # label xpath changes if there is a link to details label = CleanText('./div[@data-label="Nom du support"]', children=False)(self) if label: return label return CleanText('./div[@data-label="Nom du support"]/a', children=False)(self)
def validate(self, obj): if obj.category == 'RELEVE CB': return raw = Async( 'details', CleanText( u'//td[contains(text(), "Libellé")]/following-sibling::*[1]', default=obj.raw))(self) if raw: obj.raw = raw return True
def obj_iban(self): rib_page = Async('iban').loaded_page(self) if 'RibPdf' in rib_page.url: return rib_page.get_iban() return Join('', Regexp(CleanText('//td[has-class("ColonneCode")][contains(text(), "IBAN")]'), r'\b((?!IBAN)[A-Z0-9]+)\b', nth='*'))(rib_page.doc) or NotAvailable
def obj_code(self): val = Async('details', CleanText('//td[@class="libelle-normal" and contains(.,"CodeISIN")]', default=NotAvailable))(self) if val: return val.split('CodeISIN : ')[1] if val else val else: return NotAvailable