class item(ItemElement): klass = Bill def condition(self): num = Attr('.', 'data-fact_ligne', default='')(self) return self.env['subid'] == num obj_url = Attr('.//div[@class="pdf"]/a', 'href') obj__localid = Regexp(Field('url'), '&id=(.*)&date', u'\\1') obj_label = Regexp(Field('url'), '&date=(\d*)', u'\\1') obj_id = Format('%s.%s', Env('subid'), Field('_localid')) obj_date = FormatDate(Field('label')) obj_format = u"pdf" obj_type = u"bill" obj_price = CleanDecimal('div[@class="montant"]', default=Decimal(0), replace_dots=False)
def get_single_card(self, parent_id): div, = self.doc.xpath('//div[@class="infosynthese"]') ret = Account() ret.type = Account.TYPE_CARD ret.coming = CleanDecimal(Regexp( CleanText('.'), r'En cours prélevé au \d+/\d+/\d+ : ([\d\s,-]+) euros'), replace_dots=True)(div) ret.number = Regexp(CleanText('.'), 'sur votre carte n°([\d*]+)')(div) ret.id = '%s.%s' % (parent_id, ret.number) ret.currency = 'EUR' ret.label = 'CARTE %s' % ret.number ret.url = self.url return ret
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_unitprice = CleanDecimal(TableCell('unitprice', default=NotAvailable), default=NotAvailable, replace_dots=True) obj_vdate = Date(CleanText(TableCell('vdate'), replace=[('-', '')]), default=NotAvailable) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable, replace_dots=True) obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable, replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) def obj_code(self): return CleanText(TableCell('code'), replace=[('-', '')])(self) or NotAvailable
def generate_card_summary(self): tr = Transaction() text = CleanText('//div[@class="infosynthese"]') # card account: positive summary amount tr.amount = abs( CleanDecimal(Regexp(text, r'Montant imputé le \d+/\d+/\d+ : (.*) euros'), replace_dots=True)(self.doc)) tr.date = tr.rdate = Date(Regexp(text, r'Montant imputé le (\d+/\d+/\d+)'), dayfirst=True)(self.doc) tr.type = tr.TYPE_CARD_SUMMARY tr.label = 'DEBIT CARTE BANCAIRE DIFFERE' tr._coming = False return tr
def get_funding_src(self, t): if 'fundingSource' not in self.doc['data']['details']: return None funding_src_lst = [ src for src in self.doc['data']['details']['fundingSource'] ['fundingSourceList'] if src['type'] != 'BALANCE' ] assert len(funding_src_lst) <= 1 for src in funding_src_lst: tr = FrenchTransaction(t.id + '_fundingSrc') tr.amount = CleanDecimal(replace_dots=True).filter(src['amount']) tr.date = tr.rdate = t.date tr.label = tr.raw = u'Crédit depuis %s' % src['institution'] return tr
class item(ItemElement): klass = Bill obj_date = Date(Dict('date'), default=NotAvailable) obj_price = Eval(lambda x: x / 100, CleanDecimal(Dict('amount'))) obj_format = 'pdf' def obj_label(self): return 'Facture du %s' % Field('date')(self) def obj_id(self): return '%s_%s' % (Env('subid')(self), Field('date')(self).strftime('%d%m%Y')) obj_url = Format('%s%s', BrowserURL('doc_api_par'), Dict('hrefPdf')) obj__is_v2 = True
def check_data_consistency(self, account, recipient, amount, reason): try: assert CleanDecimal('//div[@class="topBox"]/div[@class="montant"]', replace_dots=True)(self.doc) == amount assert reason in CleanText('//div[@class="motif"]')(self.doc) assert account._transfer_id in CleanText( u'//div[div[@class="libelleChoix" and contains(text(), "Compte émetteur")]] \ //div[@class="infoCompte" and not(@title)]', replace=[(' ', '')])(self.doc) assert recipient._transfer_id in CleanText( u'//div[div[@class="libelleChoix" and contains(text(), "Compte destinataire")]] \ //div[@class="infoCompte" and not(@title)]', replace=[(' ', '')])(self.doc) except AssertionError: raise TransferError('data consistency failed')
class item(ItemElement): klass = Detail def condition(self): txt = self.el.xpath('td[1]')[0].text return (txt is not None) and (txt != "Date") obj_id = None obj_datetime = DateTime(CleanText('td[1]', symbols=u'à'), dayfirst=True) obj_label = Format(u'%s %s %s', CleanText('td[2]'), CleanText('td[3]'), CleanText('td[4]')) obj_price = CleanDecimal('td[5]', default=Decimal(0), replace_dots=True)
class item(ItemElement): klass = Bill obj__ref = CleanText('//input[@id="noref"]/@value') obj_id = Format('%s_%s', Env('subid'), CleanText('./@facture-id')) obj__url = Format('http://www.bouyguestelecom.fr/parcours/facture/download/index?id=%s', CleanText('./@facture-id')) obj_date = Env('date') obj_format = u"pdf" obj_label = CleanText('./text()') obj_type = u"bill" obj_price = CleanDecimal(CleanText('./span', replace=[(u' € ', '.')])) obj_currency = u"€" def parse(self, el): self.env['date'] = parse_french_date('01 %s' % CleanText('./text()')(self)).date()
class item(ItemElement): klass = Housing obj_id = Format( '%s-%s', Regexp(Env('type'), '(.*)-.*'), CleanText('./@id', replace=[('header-offer-', '')])) obj_title = CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/p[@class="offer-type"]/span/@title' ) obj_area = CleanDecimal( CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/div/div/h3/a/span[@class="offer-area-number"]', default=NotAvailable)) obj_cost = CleanDecimal(Regexp(CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/p[@class="offer-price"]/span', default=NotAvailable), '(.*) [%s%s%s]' % (u'€', u'$', u'£'), default=NotAvailable), default=Decimal(0)) obj_currency = Regexp(CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/p[@class="offer-price"]/span', default=NotAvailable), '.* ([%s%s%s])' % (u'€', u'$', u'£'), default=u'€') obj_date = Date( Regexp( CleanText( './div/div/div[has-class("offer-picture-more")]/div/p[@class="offer-update"]' ), ".*(\d{2}/\d{2}/\d{4}).*")) obj_text = CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/div/p[has-class("offer-description")]/span' ) obj_location = CleanText( './div/div/div[@class="offer-details-wrapper"]/div/div/div/div/h2' )
class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), CleanText('./td[3]')) obj_url = Attr('./td[@class="center" or @class="center pdf"]/a', 'href') obj_date = Env('date') obj_format = u"pdf" obj_type = u"bill" obj_price = CleanDecimal('./td[@class="center montant"]/span', replace_dots=True) def parse(self, el): self.env['date'] = parse_french_date(el.xpath('./td[2]')[0].text).date() def condition(self): return CleanText().filter(self.el.xpath('.//td')[-1]) != "" and len(self.el.xpath('./td[@class="center" or @class="center pdf"]/a/@href')) == 1
class item(ItemElement): klass = Bill def obj_url(self): return urljoin(self.page.url, Regexp(Dict('sOperation'), r'"(/.*\.pdf)')(self)) _num = Regexp(Field('url'), r'facture_(\d+).pdf') obj_id = Format('%s_%s', Env('subid'), _num) obj_date = Eval(datetime.fromtimestamp, Dict('sTimestamp')) obj_label = Format('Facture %s', _num) obj_price = CleanDecimal(Dict('fMontant')) obj_currency = Currency(Dict('sMontant')) obj_type = 'bill' obj_format = 'pdf'
def get_loan_account(self, account): assert account._prestation_id in Dict('donnees/tabIdAllPrestations')(self.doc), \ 'Loan with prestation id %s should be on this page ...' % account._prestation_id for acc in Dict('donnees/tabPrestations')(self.doc): if CleanText(Dict('idPrestation'))(acc) == account._prestation_id: loan = Loan() loan.id = loan.number = account.id loan.label = account.label loan.type = account.type loan.ownership = account.ownership loan.currency = Currency(Dict('capitalRestantDu/devise'))(acc) loan.balance = Eval( lambda x: x / 100, CleanDecimal(Dict('capitalRestantDu/valeur')))(acc) loan.coming = account.coming loan.total_amount = Eval( lambda x: x / 100, CleanDecimal(Dict('montantPret/valeur')))(acc) loan.next_payment_amount = Eval( lambda x: x / 100, CleanDecimal(Dict('montantEcheance/valeur')))(acc) loan.next_payment_date = self.guess_loan_monthly_repayment(acc) loan.duration = Dict('dureeNbMois')(acc) loan.maturity_date = datetime.datetime.strptime( Dict('dateFin')(acc), '%Y%m%d') self.set_parent_account_id(loan, acc) loan._internal_id = account._internal_id loan._prestation_id = account._prestation_id loan._loan_type = account._loan_type return loan
def get_operations(self, account): for tr in self.doc.xpath( '//table[@id="tabHistoriqueOperations"]/tbody/tr'): tds = tr.findall('td') if len(CleanText(None).filter(tds[-1])) == 0: continue t = Transaction() t.type = Transaction.TYPE_BANK t.parse(date=CleanText(None).filter(tds[3]), raw=CleanText(None).filter(tds[1])) t.amount = CleanDecimal(None, replace_dots=True, default=0).filter(tds[-2]) t.commission = CleanDecimal(None, replace_dots=True, default=0).filter(tds[-3]) investment = Investment() investment.label = CleanText(None).filter(tds[0]) investment.quantity = CleanDecimal(None, replace_dots=True, default=0).filter(tds[4]) investment.unitvalue = CleanDecimal(None, replace_dots=True, default=0).filter(tds[5]) t.investments = [investment] yield t
def set_loan_details(self, account): # If there are no available details for the loan, the statut will be "NOK" if Dict('commun/statut')(self.doc) == 'NOK': return else: # There is no default value in the Coalesce because we want it to crash in case of # unknown value to be able to add it rate = Coalesce( Dict('donnees/caracteristiquesReservea/tauxHorsAssurance', NotAvailable), Dict('donnees/caracteristiquesCreditConfiance/taux', NotAvailable), )(self.doc) account.rate = CleanDecimal().filter(rate)
def _get_high_tide_value(self, AM=True, jour=0): if AM: time = DateTime( CleanText('//tr[@id="MareeJours_%s"]/td[1]/b[1]' % jour))(self) value = CleanDecimal('//tr[@id="MareeJours_0"]/td[2]/b[1]', replace_dots=True)(self) else: time, value = None, None if len( XPath('//tr[@id="MareeJours_%s"]/td[1]/b' % jour)(self)) > 1: time = DateTime(CleanText( '//tr[@id="MareeJours_%s"]/td[1]/b[2]' % jour), default=None)(self) value = CleanDecimal('//tr[@id="MareeJours_0"]/td[2]/b[2]', replace_dots=True, default=None)(self) if time and value: measure = GaugeMeasure() measure.level = float(value) measure.date = time + timedelta(days=jour) return measure
class item(ItemElement): klass = Account load_details = Attr('.//a', 'href') & AsyncLoad obj__link_id = Async( 'details', Link('//li/a[contains(text(), "Mouvements")]')) obj__link_inv = Link('./td[1]/a') obj_id = CleanText('./td[2]', replace=[(' ', '')]) obj_label = CleanText('./td[1]') obj_balance = CleanDecimal('./td[3]', replace_dots=True) obj_currency = FrenchTransaction.Currency('./td[4]') obj__card_links = [] obj_type = Account.TYPE_LIFE_INSURANCE obj__is_inv = True
def iter_investment(self, label): for tr in self.doc.xpath( u'//table[@summary="Liste des échéances"]/tbody/tr'): # list containing the numerical values # element 1 is the quantity # element 2 is the valuation tds = tr.findall('td[@class="i d"]') # var containing the label td_label = tr.find('td[@class="i g"]') inv = Investment() if len(tds) <= 2: continue inv.label = CleanText(td_label)(tr) inv.quantity = CleanDecimal(tds[1], replace_dots=True)(tr) inv.valuation = CleanDecimal(tds[2], replace_dots=True)(tr) if 'PEI' in label.split()[0]: label = 'PEE' if Regexp(CleanText(td_label), '\(([\w]+).*\)$')(tr) not in label.split()[0]: continue yield inv
class item(ItemElement): klass = Housing obj_id = Format( 'colocation-%s', CleanText('./div/header/@id', replace=[('header-offer-', '')])) obj_title = CleanText( CleanHTML( './div/header/section/p[@class="property-type"]/span/@title' )) obj_area = CleanDecimal( './div/header/section/p[@class="offer-attributes"]/a/span[@class="offer-area-number"]', default=0) obj_cost = CleanDecimal('./div/header/section/p[@class="price"]', default=0) obj_currency = Regexp( CleanText('./div/header/section/p[@class="price"]', default=NotAvailable), '.* ([%s%s%s])' % (u'€', u'$', u'£'), default=u'€') obj_text = CleanText( './div/div[@class="content-offer"]/section[has-class("content-desc")]/p/span[has-class("offer-text")]/@title' ) obj_date = Date( Regexp( CleanText( './div/header/section/p[has-class("update-date")]'), ".*(\d{2}/\d{2}/\d{4}).*")) obj_location = CleanText( '(./div/div[@class="content-offer"]/section[has-class("content-desc")]/p)[1]/span/@title' )
class item(ItemElement): klass = Account def condition(self): return CleanDecimal(TableCell('balance'), replace_dots=True, default=NotAvailable)(self) is not NotAvailable TYPE = { 'COMPTE COURANT': Account.TYPE_CHECKING, 'COMPTE TRANSACTION': Account.TYPE_CHECKING, 'COMPTE ORDINAIRE': Account.TYPE_CHECKING, } TYPE_BY_LABELS = { 'CAV': Account.TYPE_CHECKING, } obj_id = CleanText(TableCell('id')) obj_label = CleanText(TableCell('label')) obj_currency = FrenchTransaction.Currency(TableCell('currency')) obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) def obj__link(self): return Link(TableCell('id')(self)[0].xpath('./a'), default=None)(self) def obj__url(self): return Link(TableCell('rib')(self)[0].xpath('./a[img[starts-with(@alt, "RIB")]]'), default=None)(self) def load_iban(self): link = Link(TableCell('rib')(self)[0].xpath('./a[img[starts-with(@alt, "RIB")]]'), default=None)(self) return self.page.browser.async_open(link) def obj_type(self): try: el_to_check = CleanText(TableCell('type'))(self) type_dict = self.TYPE except ColumnNotFound: el_to_check = Field('label')(self) type_dict = self.TYPE_BY_LABELS for k, v in type_dict.items(): if el_to_check.startswith(k): return v return Account.TYPE_UNKNOWN def obj_iban(self): rib_page = Async('iban').loaded_page(self) if 'RibPdf' in rib_page.url: return rib_page.get_iban() return Join('', Regexp(CleanText('//td[has-class("ColonneCode")][contains(text(), "IBAN")]'), r'\b((?!IBAN)[A-Z0-9]+)\b', nth='*'))(rib_page.doc) or NotAvailable
def get_list(self): no_accounts_message = self.doc.xpath( u'//span/b[contains(text(),"Votre abonnement est clôturé. Veuillez contacter votre conseiller.")]/text()' ) if no_accounts_message: raise ActionNeeded(no_accounts_message[0]) for tr in self.doc.xpath('//table[has-class("datas")]//tr'): if tr.attrib.get('class', '') == 'entete': continue cols = tr.findall('td') a = Account() a.label = unicode(cols[self.COL_ID].xpath( './/span[@class="left-underline"] | .//span[@class="left"]/a') [0].text.strip()) a.type = self.get_account_type(a.label) balance = CleanText('.')(cols[self.COL_BALANCE]) if balance == '': continue a.balance = CleanDecimal(replace_dots=True).filter(balance) a.currency = a.get_currency(balance) if cols[self.COL_ID].find('a'): a._link, a._args = self.params_from_js( cols[self.COL_ID].find('a').attrib['href']) a._acc_nb = cols[self.COL_ID].xpath( './/span[@class="right-underline"] | .//span[@class="right"]' )[0].text.replace(' ', '').strip() if hasattr(a, '_args') and a._args: a.id = '%s%s%s' % (a._acc_nb, a._args['IndiceCompte'], a._args['Indiceclassement']) else: a.id = a._acc_nb # This account can be multiple life insurance accounts if any(a.label.startswith(lab) for lab in ['ASS.VIE-BONS CAPI-SCPI-DIVERS', 'BONS CAPI-SCPI-DIVERS']) or \ u'Aucun d\\351tail correspondant pour ce compte' in tr.xpath('.//a/@href')[0]: continue if a.type is Account.TYPE_CARD: a.coming = a.balance a.balance = Decimal('0.0') a._inv = False yield a
def populate(self, accounts): cards = [] for account in accounts: for li in self.doc.xpath('//li[@class="nav-category"]'): title = CleanText().filter(li.xpath('./h3')) for a in li.xpath('./ul/li//a'): label = CleanText().filter( a.xpath('.//span[@class="nav-category__name"]')) balance_el = a.xpath( './/span[@class="nav-category__value"]') balance = CleanDecimal( replace_dots=True, default=NotAvailable).filter(balance_el) if 'CARTE' in label and balance: acc = Account() acc.balance = balance acc.label = label acc.currency = FrenchTransaction.Currency().filter( balance_el) acc._link = Link().filter(a.xpath('.')) acc._history_page = acc._link acc.id = acc._webid = Regexp( pattern='([^=]+)$').filter(Link().filter( a.xpath('.'))) acc.type = Account.TYPE_CARD if not acc in cards: cards.append(acc) elif account.label == label and account.balance == balance: if not account.type: account.type = AccountsPage.ACCOUNT_TYPES.get( title, Account.TYPE_UNKNOWN) if account.type == Account.TYPE_LOAN: account._history_page = None elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET): account._history_page = re.sub( '/$', '', Link().filter(a.xpath('.'))) elif '/compte/cav' in a.attrib[ 'href'] or not 'titulaire' in self.url: account._history_page = self.browser.other_transactions else: account._history_page = self.browser.budget_transactions account._webid = Attr( None, 'data-account-label').filter( a.xpath( './/span[@class="nav-category__name"]')) accounts.extend(cards)
class get_event(ItemElement): klass = BaseCalendarEvent obj_summary = CleanText('//div[@id="sectionHead"]/h1') obj_description = CleanHTML('//div[@id="event-item"]/div[3]/p[2]') obj_price = CleanDecimal(Regexp( CleanText('//aside[@id="detail"]/ul/li[3]'), r'Cost /[^\d]*([\d ,.]+).', default=''), default=None) obj_location = Regexp(CleanText('//aside[@id="detail"]/ul/li[2]'), r'Venue / (.+)') obj_booked_entries = Type( CleanText('//h1[@id="MembersFavouriteCount"]'), type=int) obj_status = STATUS.CONFIRMED obj_category = CATEGORIES.CONCERT _date = Date(CleanText('//aside[@id="detail"]/ul/li[1]/a[1]')) def obj_start_date(self): start_time = Time( Regexp(CleanText('//aside[@id="detail"]/ul/li[1]'), r'(\d{2}:\d{2}) -'))(self) return CombineDate(self._date, start_time)(self) def obj_end_date(self): end_time = Time( Regexp(CleanText('//aside[@id="detail"]/ul/li[1]'), r'- (\d{2}:\d{2})'))(self) end_date = CombineDate(self._date, end_time)(self) if end_date > self.obj_start_date(): end_date += timedelta(days=1) return end_date def obj_ticket(self): li_class = Attr('//li[@id="tickets"]//li[1]', 'class', default=None)(self) if li_class: if li_class == 'closed': return TICKET.CLOSED else: return TICKET.AVAILABLE return TICKET.NOTAVAILABLE
class item(ItemElement): klass = Investment obj_label = CleanText(Dict('libelle')) obj_valuation = CleanDecimal(Dict('montant')) def obj_code_type(self): if is_isin_valid(CleanText(Dict('code'))(self)): return Investment.CODE_TYPE_ISIN return NotAvailable def obj_code(self): code = CleanText(Dict('code'))(self) if is_isin_valid(code): return code return NotAvailable
def _parse_voice(self, div, string, num, inter=False): voicediv = div.xpath('div[@class="conso"]')[0] voice = Detail() voice.id = "-voice" voice.label = CleanText('div[@class="titre"]/p')(div) if inter: voice.label = voice.label + " (international)" voice.id = voice.id + "-inter" voice.price = CleanDecimal('div[@class="horsForfait"]/p/span', default=Decimal(0), replace_dots=True)(div) voice1 = CleanText('.//span[@class="actif"][1]')(voicediv) voice2 = CleanText('.//span[@class="actif"][2]')(voicediv) voice.infos = unicode(string) % (voice1, voice2) return voice
class item(ItemElement): klass = Pokemon obj_id = CleanDecimal('td[1]') obj_pokedex_id = obj_id def obj_type(self): types = [] for type in TableCell('type')(self)[0].xpath('./div'): types.append(Attr('.//img', 'alt')(type)) return types def obj_name(self): return CleanText(TableCell('name')(self)[0].xpath('./strong/a/text()'))(self)
class get_transfer(ItemElement): klass = Transfer obj_label = CleanText('//div[@id="transfer-label"]/span[@class="transfer__account-value"]') obj_amount = CleanDecimal('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]', replace_dots=True) obj_currency = CleanCurrency('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]') obj_account_label = CleanText('//span[@id="transfer-origin-account"]') obj_recipient_label = CleanText('//span[@id="transfer-destination-account"]') def obj_exec_date(self): type_ = CleanText('//div[@id="transfer-type"]/span[@class="transfer__account-value"]')(self) if type_ == 'Ponctuel': return datetime.date.today() elif type_ == 'Différé': return Date(CleanText('//div[@id="transfer-date"]/span[@class="transfer__account-value"]'), dayfirst=True)(self)
class item(ItemElement): klass = Investment condition = lambda self: not CleanText( '//div[has-class("errorConteneur")]', default=None)(self.el) obj_label = Upper(TableCell('label')) obj_code = CleanText(TableCell('code')) obj_quantity = MyDecimal(TableCell('quantity')) obj_unitprice = MyDecimal(TableCell('unitprice')) obj_unitvalue = MyDecimal(TableCell('unitvalue')) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable)
def create_transfer(self, account, recipient, amount, reason): transfer = Transfer() transfer.currency = FrenchTransaction.Currency('//div[@class="topBox"]/div[@class="montant"]')(self.doc) transfer.amount = CleanDecimal('//div[@class="topBox"]/div[@class="montant"]', replace_dots=True)(self.doc) transfer.account_iban = account.iban transfer.recipient_iban = recipient.iban transfer.account_id = account.id transfer.recipient_id = recipient.id transfer.exec_date = date.today() transfer.label = reason transfer.account_label = account.label transfer.recipient_label = recipient.label transfer._account = account transfer._recipient = recipient transfer.account_balance = account.balance return transfer
def parse(self, el): details = dict() self.env['area'] = NotAvailable for item in el.xpath('//div[@class="line"]/h2'): if 'Surface' in CleanText('./span[@class="property"]')(item): self.env['area'] = CleanDecimal(Regexp(CleanText('./span[@class="value"]'), '(.*)m.*'), replace_dots=(',', '.'))(item) else: key = u'%s' % CleanText('./span[@class="property"]')(item) if 'GES' in key or 'Classe' in key: details[key] = CleanText('./span[@class="value"]/noscript/a')(item) else: details[key] = CleanText('./span[@class="value"]')(item) self.env['details'] = details