class item(ItemElement): klass = Subscription obj__nom1 = CleanText(TableCell("nom")) obj__nom2 = CleanText( Attr('.//input[contains(@id, ".nom")]', "value")) obj__prenom1 = CleanText(TableCell("prenom")) obj__prenom2 = CleanText( Attr('.//input[contains(@id, ".prenom")]', "value")) obj__internal_id = CleanText( Attr('.//input[contains(@id, ".salaEmplPk.noIntSala")]', "value")) obj__pseudo_siret = CleanText( Attr('.//input[contains(@id, ".salaEmplPk.psdoSirt")]', "value")) obj__date_creation = Date( CleanText( Attr('.//input[contains(@id, ".dtCreation")]', "value")), dayfirst=True, ) obj_label = Format("%s %s", CleanText(Field("_prenom1")), CleanText(Field("_nom1"))) # obj_subscriber = Env("subscriber") obj__type = "employee" obj__active = Eval(lambda x: x[0].checked, (Child(TableCell("actif")))) obj_id = Field("_internal_id")
class item(ItemElement): klass = Transaction obj_id = CleanText(TableCell('id')) obj_label = CleanText(TableCell('label')) obj_type = Transaction.TYPE_BANK obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_amount = Env('amount') obj_investments = Env('investments') def parse(self, el): # We have only one history for all accounts... # And we know only on details page if it match current account. trid = CleanText(TableCell('id'))(self) if trid not in self.page.browser.cache['details']: # Thanks to stateful website : first go on details page... idt = Attr(TableCell('id')(self)[0].xpath('./a'), 'id', default=None)(self) typeop = Regexp(Attr(TableCell('id')(self)[0].xpath('./a'), 'onclick'), 'Operation.+?([A-Z_]+)')(self) form = self.page.get_history_form(idt, {'referenceOp': trid, 'typeOperation': typeop}) details_page = self.page.browser.open(form.url, data=dict(form)).page self.page.browser.cache['details'][trid] = details_page # ...then go back to history list. idt = Attr('//input[@title="Retour"]', 'id', default=None)(details_page.doc) form = self.page.get_history_form(idt) self.page.browser.open(form.url, data=dict(form)) else: details_page = self.page.browser.cache['details'][trid] # Check if page is related to the account if not len(details_page.doc.xpath('//td[contains(text(), $id)]', id=Env('accid')(self))): raise SkipItem() self.env['investments'] = list(details_page.get_investments(accid=Env('accid')(self))) self.env['amount'] = sum([i.valuation or Decimal('0') for i in self.env['investments']])
class item(ItemElement): klass = Account def condition(self): # Skip card coming lines return 'Encours carte' not in CleanText(TableCell('label', colspan=True))(self) obj_id = CleanText(TableCell('id', colspan=True)) obj_number = Field('id') obj_label = CleanText(TableCell('label', colspan=True)) obj_type = Map(Field('label'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) obj_currency = Currency(TableCell('currency', colspan=True)) obj_url = None # Accounts may have an 'Operations' balance or a 'Value' balance def obj_balance(self): value_balance = CleanText(TableCell('value_balance', default='', colspan=True))(self) # Skip invalid balance values in the 'Value' column (for example for Revolving credits) if value_balance not in ('', 'Montant disponible'): return CleanDecimal.French().filter(value_balance) return CleanDecimal.French(CleanText(TableCell('operation_balance', default='', colspan=True)))(self) def obj__form(self): # Account forms look like 'javascript:fwkPUAvancerForm('Releves','frm1')' # From this we extract the name (frm1) and fetch the form name on the page. script = Link('.//a', default='')(TableCell('id', colspan=True)(self)[0]) if 'javascript' in script: form_search = re.search(r'frm\d+', script) if form_search: account_form = self.page.get_form(name=form_search.group(0)) return self.page.fill_form(account_form, card=False) return None
class InvestItem(ItemElement): klass = Investment obj_label = CleanText(TableCell('label', support_th=True)) obj_portfolio_share = Eval(lambda x: x / 100 if x else NotAvailable, CleanDecimal(TableCell('share', support_th=True), replace_dots=True, default=NotAvailable)) obj_quantity = CleanDecimal(TableCell('quantity', support_th=True), replace_dots=True, default=NotAvailable) obj_valuation = CleanDecimal(TableCell('valuation', support_th=True), replace_dots=True, default=NotAvailable)
class item(ItemElement): klass = Transaction obj_label = CleanText(TableCell('label')) obj_type = Transaction.TYPE_BANK obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_amount = MyDecimal(TableCell('amount'))
class item(ItemElement): klass = Investment obj_code = Regexp(CleanText(TableCell('label')), r'Code ISIN : (\w+) ', default=NotAvailable) obj_quantity = MyDecimal(TableCell('quantity'), default=NotAvailable) obj_unitvalue = MyDecimal(TableCell('unitvalue'), default=NotAvailable) # Some PERP invests don't have valuation obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable) def obj_label(self): if 'FONDS EN EUROS' in CleanText(TableCell('label'))(self): return 'FONDS EN EUROS' return Regexp(CleanText(TableCell('label')), r'Libellé support : (.*) Code ISIN')(self) def obj_code_type(self): if Field('label')(self) == 'FONDS EN EUROS': return NotAvailable return Investment.CODE_TYPE_ISIN
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'))) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=Decimal('1')) obj_valuation = CleanDecimal(TableCell('support_value')) def obj_diff_ratio(self): val = self.el.xpath('.//td')[4].text_content().strip().strip('%') if val == '-': return NotAvailable return Decimal(val) / 100 def obj_code(self): if "Fonds en euros" in Field('label')(self): return NotAvailable return Regexp(Link('.//a'), r'javascript:openSupportPerformanceWindow\(\'(.*?)\', \'(.*?)\'\)', '\\2')(self) def obj_quantity(self): # default for euro funds return CleanDecimal(TableCell('quantity'), default=CleanDecimal(TableCell('support_value'))(self))(self) def condition(self): return len(self.el.xpath('.//td')) > 1
class account(ItemElement): klass = Account obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True, sign=lambda x: -1) obj_currency = FrenchTransaction.Currency(TableCell('balance')) obj_type = Account.TYPE_LOAN obj_id = Env('id') obj__transfer_id = None def obj_label(self): has_type = CleanText( './ancestor::table[.//th[contains(text(), "Nature libell")]]', default=None)(self) return CleanText('./td[3]')(self) if has_type else CleanText( './ancestor::table/preceding-sibling::div[1]')(self).split( ' - ')[0] def parse(self, el): label = Field('label')(self) trs = self.xpath( '//td[contains(text(), $label)]/ancestor::tr[1] | ./ancestor::table[1]/tbody/tr', label=label) i = [i for i in range(len(trs)) if el == trs[i]] i = i[0] if i else 0 label = label.replace(' ', '') self.env['id'] = "%s%s%s" % (Regexp( CleanText(TableCell('id')), r'(\w+)\s-\s(\w+)', r'\1\2')(self), label.replace(' ', ''), i)
class item(ItemElement): klass = Transaction obj_label = CleanText(TableCell('label')) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) obj_type = Transaction.TYPE_BANK def obj_amount(self): amount = MyDecimal( TableCell('net') if not CleanText(TableCell('brut')) (self) else TableCell('brut'))(self) return -amount if amount and any( word in Field('label')(self).lower() for word in self.page.DEBIT_WORDS) else amount def obj_date(self): return Date(CleanText(TableCell('date')), dayfirst=True, default=Field('vdate')(self))(self) def condition(self): return u"Validé" in CleanText(TableCell('status'))( self) and u"Arrêté annuel" not in Field('label')(self) def obj_investments(self): data = Attr('.', 'data-ri')(self) form = self.page.get_historyexpandall_form(data) page = self.page.browser.open(form.url, data=dict(form)).page investments = [] for table in page.doc.xpath( '//following-sibling::tr[1]//span[contains(text(), "ISIN")]/ancestor::table[1]' ): investments.extend( TableTransactionsInvestment(self.page, el=table)()) return investments
class ItemInvestment(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_quantity = MyDecimal(TableCell('quantity', default=None)) obj_unitvalue = MyDecimal(TableCell('unitvalue', default=None)) obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable) obj_code = Regexp(CleanText('.//td[contains(text(), "Isin")]'), ':[\s]+([\w]+)', default=NotAvailable) obj__invest_type = Regexp(CleanText('.//td[contains(text(), "Type")]'), ':[\s]+([\w ]+)', default=NotAvailable) def obj_valuation(self): valuation = MyDecimal(TableCell('valuation', default=None))(self) h2 = CleanText('./ancestor::div[contains(@id, "Histo")][1]/preceding-sibling::h2[1]')(self) return -valuation if valuation and any(word in h2.lower() for word in self.page.DEBIT_WORDS) else valuation def obj_portfolio_share(self): ps = MyDecimal(TableCell('portfolio_share', default=None))(self) return Eval(lambda x: x / 100, ps)(self) if not empty(ps) else NotAvailable def obj__gestion_type(self): if self.xpath('ancestor::tbody[ends-with(@id, "contratProfilTable_data")]'): # investments are nested in profiles, get profile type profile_table_el = self.xpath('ancestor::tr/ancestor::table[position() = 1]')[0] profile_table = ProfileTableInvestment(self.page, self, profile_table_el) gestion_type = profile_table.get_colnum('_gestion_type') assert gestion_type path = 'ancestor::tr/preceding-sibling::tr[@data-ri][position() = 1][1]/td[%d]' % (gestion_type + 1) return CleanText(path)(self) return NotAvailable
class item(ItemElement): klass = Transaction def condition(self): return CleanText(TableCell('date'))(self) != 'en cours' obj_label = CleanText(TableCell('label')) obj_type = Transaction.TYPE_BANK obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj__arbitration = False def obj_amount(self): credit = CleanDecimal(TableCell('credit'), default=Decimal(0))(self) # Different types of life insurances, use different columns. if TableCell('debit', default=None)(self): debit = CleanDecimal(TableCell('debit'), default=Decimal(0))(self) # In case of financial arbitration, both columns are equal if credit and debit: assert credit == debit self.obj._arbitration = True return credit else: return credit - abs(debit) else: credit2 = CleanDecimal(TableCell('credit2'), default=Decimal(0))(self) assert not (credit and credit2) return credit + credit2
class item(ItemElement): klass = Investment def condition(self): return (CleanText('./th')(self) != 'Total épargne constituée' ) and ('Détail' not in Field('label')(self)) obj_label = CleanText('./th') obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable) obj_valuation = CleanDecimal(TableCell('valuation'), default=NotAvailable) obj_portfolio_share = Eval( lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'))) def obj_code(self): code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self) return code if is_isin_valid(code) else NotAvailable def obj_code_type(self): return Investment.CODE_TYPE_ISIN if is_isin_valid( Field('code')(self)) else NotAvailable
def obj_amount(self): amount = MyDecimal( TableCell('net') if not CleanText(TableCell('brut')) (self) else TableCell('brut'))(self) return -amount if amount and any( word in Field('label')(self).lower() for word in self.page.DEBIT_WORDS) else amount
def obj_currency(self): tablecell = TableCell('balance')(self)[0] text = tablecell.xpath('./span/text()')[0] regex = '[0-9,.]* (.*)' currency = Currency().filter(re.search(regex, text).group(1)) return currency
class item(ItemElement): klass = BaseTransaction obj_label = CleanText(TableCell('label')) obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_amount = CleanDecimal(TableCell('amount'), replace_dots=True) obj__coming = False
class item(ItemElement): klass = Transaction def obj_date(self): d = Date(CleanText(TableCell('date')), dayfirst=True, default=None)(self) if d: return d return Date(CleanText(TableCell('date')), parse_func=parse_french_date)(self) obj_raw = Transaction.Raw(CleanText(TableCell('label'))) obj_amount = CleanDecimal(TableCell('amount'), replace_dots=True, default=NotAvailable) obj__is_coming = False def parse(self, el): if el.xpath('./td[2]/a'): m = re.search( '(\d+)', el.xpath('./td[2]/a')[0].get( 'data-modal-alert-behavior', '')) if m: self.env['account']._history_pages.append((Field('raw')(self),\ self.page.browser.open('%s%s%s' % (self.page.url.split('mouvements')[0], 'mouvement/', m.group(1))).page)) raise SkipItem()
class item(ItemElement): klass = Account obj_label = CleanText(TableCell('label')) obj_id = CleanText(TableCell('id')) obj_number = CleanText(TableCell('id')) obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) obj_coming = None obj_iban = None def obj_type(self): for k, v in self.page.ACCOUNT_TYPES.items(): if Field('label')(self).startswith(k): return v return Account.TYPE_UNKNOWN def obj_currency(self): currency = CleanText(TableCell('currency')(self))(self) return Account.get_currency(currency) # Required to get the investments of each "Assurances Vie" account: def obj__details(self): raw_details = CleanText( (TableCell('balance')(self)[0]).xpath('./a/@href'))(self) m = re.search(r"Window\('(.*?)',window", raw_details) if m: return m.group(1)
class item(ItemElement): klass = Investment def obj_label(self): if not CleanText(TableCell('code'))(self): return CleanText('./preceding-sibling::tr[1]/td[2]')(self) return CleanText(TableCell('label'))(self) def obj_code(self): if CleanText(TableCell('code'))(self): return CleanText(TableCell('code'))(self) return CleanText('./preceding-sibling::tr[1]/td[1]')(self) obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable) obj_unitprice = CleanDecimal(TableCell('unitprice')) obj_valuation = MyDecimal(TableCell('valuation')) obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) obj_code_type = Investment.CODE_TYPE_ISIN def condition(self): return CleanDecimal(TableCell('quantity'), default=None)(self) is not None
class item(ItemElement): def condition(self): text = CleanText('td')(self) return not text.startswith('Aucune information disponible') klass = Transaction obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_amount = MyDecimal(TableCell('amount')) obj_raw = CleanText(TableCell('label')) def obj_investments(self): inv = Investment() inv.quantity = CleanDecimal(TableCell('quantity'), replace_dots=True)(self) inv.code_type = Investment.CODE_TYPE_ISIN txt = CleanText(TableCell('name'))(self) match = re.match('(?:(.*) )?- ([^-]+)$', txt) inv.label = match.group(1) or NotAvailable inv.code = match.group(2) if inv.code in self.parent.labels: inv.label = inv.label or self.parent.labels[inv.code] elif inv.label: self.parent.labels[inv.code] = inv.label else: inv.label = inv.code return [inv]
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) obj_portfolio_share = Eval( lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) # There is no "unitvalue" information available on the "Assurances Vie" space. def obj_quantity(self): quantity = TableCell('quantity')(self) if CleanText(quantity)(self) == '-': return NotAvailable return CleanDecimal(quantity, replace_dots=True)(self) def obj_code(self): isin = CleanText(TableCell('code')(self))(self) return isin or NotAvailable def obj_code_type(self): if is_isin_valid(Field('code')(self)): return Investment.CODE_TYPE_ISIN return NotAvailable def obj_vdate(self): return self.page.get_vdate()
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) def obj_code(self): td = TableCell('label')(self)[0] return Attr('.//a', 'id', default=NotAvailable)(td) obj_code_type = Investment.CODE_TYPE_ISIN def obj_quantity(self): return MyDecimal(TableCell('quantity'))(self) or NotAvailable def obj_unitvalue(self): return MyDecimal(TableCell('unitvalue'))(self) or NotAvailable obj_valuation = MyDecimal(TableCell('valuation')) obj_portfolio_share = Eval( lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable)
class item(ItemElement): klass = Transaction obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) obj_amount = MyDecimal(TableCell('credit'), default=TableCell('debit')) obj_raw = Transaction.Raw(ReplaceEntities(Regexp(CleanText('.//script[1]'), r"toggleDetails\([^,]+,[^,]+, '(.*?)', '(.*?)', '(.*?)',", r'\1 \2 \3')))
class item(ItemElement): klass = Account obj_type = Account.TYPE_CARD obj_currency = 'EUR' obj_number = CleanText(TableCell('number')) obj_label = Format('%s %s', CleanText(TableCell('label')), obj_number) obj_id = Format('%s.%s', Env('parent_id'), obj_number) def obj_coming(self): comings = (CleanDecimal(TableCell('balance', default=None), replace_dots=True, default=None)(self), CleanDecimal(TableCell('_credit', default=None), replace_dots=True, default=None)(self), CleanDecimal(TableCell('_debit', default=None), replace_dots=True, default=None)(self)) for coming in comings: if not empty(coming): return coming else: # There should have at least 0.00 in debit column assert False def obj_url(self): td = TableCell('label')(self)[0].xpath('.//a')[0] return urljoin(self.page.url, td.attrib['href'])
class item(InvestItem): obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True, default=NotAvailable) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable)
def obj_balance(self): if CleanText(TableCell('balance'))( self) != u'Remboursé intégralement': return -abs( CleanDecimal(TableCell('balance'), replace_dots=True)(self)) return Decimal(0)
class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_quantity = MyDecimal(CleanText(TableCell('quantity'))) obj_valuation = MyDecimal(TableCell('valuation')) obj_unitvalue = MyDecimal(TableCell('unitvalue')) def obj_portfolio_share(self): if MyDecimal(TableCell('portfolio_share'), default=None)(self): return Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share')))(self) return NotAvailable def obj_code(self): for code in Field('label')(self).split(): if is_isin_valid(code): return code return NotAvailable def obj_vdate(self): if Field('unitvalue') is NotAvailable: vdate = Date(dayfirst=True, default=NotAvailable)\ .filter(Regexp(CleanText('.'), '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) or \ Date(dayfirst=True, default=NotAvailable)\ .filter(Regexp(CleanText('//tr[td[span[b[contains(text(), "Estimation du contrat")]]]]/td[2]'), '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) return vdate
def condition(self): if CleanText( TableCell('balance'))(self) != 'Prêt non débloqué': return bool(not self.xpath( '//caption[contains(text(), "Période de franchise du")]' )) return CleanText( TableCell('balance'))(self) != 'Prêt non débloqué'
def obj_date(self): d = Date(CleanText(TableCell('date')), dayfirst=True, default=None)(self) if d: return d return Date(CleanText(TableCell('date')), parse_func=parse_french_date)(self)
class item(ItemElement): klass = Transaction obj_date = Date(CleanText('//div[div[contains(text(), "Date de prélèvement")]]/following-sibling::div/div'), dayfirst=True) obj_rdate = Date(CleanText(TableCell('rdate')), dayfirst=True) obj_label = CleanText(TableCell('label')) obj_amount = CleanDecimal.SI(TableCell('amount')) obj_type = Transaction.TYPE_CARD
def parse(self, el): i = Investment() i.label = Field('label')(self) i.code = CleanText(TableCell('code'))(self) i.quantity = MyDecimal(TableCell('quantity'))(self) i.valuation = Field('amount')(self) i.vdate = Field('date')(self) self.env['investments'] = [i]
def original_unitvalue(self): tablecell = TableCell('unitvalue', colspan=True)(self)[0] text = tablecell.xpath('./text()')[0] regex = '[0-9,]* (.*)' currency = Currency().filter(re.search(regex, text).group(1)) return currency, CleanDecimal(replace_dots=True).filter(text)
class item(ItemElement): def condition(self): return 'Récapitulatif annuel' not in CleanText(TableCell('_last_document_label'))(self) klass = Subscription obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) obj_label = CleanText(TableCell('label'))
def obj_vdate(self): tablecell = TableCell('vdate', colspan=True)(self)[0] vdate_scrapped = tablecell.xpath('./preceding-sibling::td[position()=1]//span/text()')[0] # Scrapped date could be a schedule time (00:00) or a date (01/01/1970) vdate = NotAvailable if ':' in vdate_scrapped: today = datetime.date.today() h, m = [int(x) for x in vdate_scrapped.split(':')] hour = datetime.time(hour=h, minute=m) vdate = datetime.datetime.combine(today, hour) elif '/' in vdate_scrapped: vdate = datetime.datetime.strptime(vdate_scrapped, '%d/%m/%y') return vdate
def obj_code(self): # We try to get the code from <a> div. If we didn't find code in url, # we try to find it in the cell text tablecell = TableCell('label', colspan=True)(self)[0] # url find try url = tablecell.xpath('./following-sibling::td[position()=1]/div/a')[0].attrib['href'] code_match = re.search(r'sico=([A-Z0-9]*)', url) if code_match: if is_isin_valid(code_match.group(1)): return code_match.group(1) # cell text find try text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self) for code in text.split(' '): if is_isin_valid(code): return code return NotAvailable
def obj_quantity(self): tablecell = TableCell('quantity', colspan=True)(self)[0] return CleanDecimal(tablecell.xpath('./span'), replace_dots=True)(self)
def obj_label(self): tablecell = TableCell('label')(self)[0] label = tablecell.xpath('./div[position()=1]') return CleanText(label)(self)
def obj_balance(self): tablecell = TableCell('balance')(self)[0] b = tablecell.xpath('./span[@class="intraday"]') balance = CleanDecimal(replace_dots=True).filter(b) return Decimal(balance)
def obj_label(self): tablecell = TableCell('label', colspan=True)(self)[0] label = CleanText(tablecell.xpath('./following-sibling::td[@class=""]/div/a')[0])(self) return label
def obj_id(self): tablecell = TableCell('id')(self)[0] id = tablecell.xpath('./div[position()=2]') return CleanText().filter(id)
def obj_url(self): td = TableCell('id')(self)[0] return Link(td.xpath('./a'))(self)