def get_cards(self, accounts_list=None): # accounts_list is only used by get_list self.location(self.accounts_url.format(self.sag)) for idelco, parent_id in self.page.iter_idelcos(): if not self.accounts.is_here(): self.location(self.accounts_url.format(self.sag)) obj = self.page.get_idelco(idelco) if isinstance(obj, basestring): self.location(obj) else: self.page.submit_card(obj) assert self.cards.is_here() or self.cards2.is_here() if self.page.several_cards(): for account in self.page.iter_cards(): if accounts_list: account.parent = find_object(accounts_list, id=account._parent_id) yield account else: for account in self.page.iter_card(): if accounts_list: account._parent_id = parent_id account.parent = find_object(accounts_list, id=account._parent_id) yield account
def init_transfer(self, transfer, **params): self.logger.info('Going to do a new transfer') transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z/\-\?:\(\)\.,\'\+ ]+', '', transfer.label).split()).upper() if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) return self.browser.init_transfer(account, recipient, transfer)
def get_list(self): accounts = [] previous_account = None noaccounts = self.get_from_js('_js_noMvts =', ';') if noaccounts is not None: assert 'avez aucun compte' in noaccounts return [] txt = self.get_from_js('_data = new Array(', ');', is_list=True) if txt is None: raise BrowserUnavailable('Unable to find accounts list in scripts') data = json.loads('[%s]' % txt.replace("'", '"')) for line in data: a = Account() a.id = line[self.COL_ID].replace(' ', '') if re.match(r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)', a.id): a.id = str(CleanDecimal().filter(a.id)) a._acc_nb = a.id.split('_')[0] if len(a.id.split('_')) > 1 else None a.label = MyStrip(line[self.COL_LABEL], xpath='.//div[@class="libelleCompteTDB"]') # This account can be multiple life insurance accounts if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *': continue a.balance = Decimal(FrenchTransaction.clean_amount(line[self.COL_BALANCE])) a.currency = a.get_currency(line[self.COL_BALANCE]) a.type = self.get_account_type(a.label) # The parent account must be created right before if a.type == Account.TYPE_CARD: # duplicate if find_object(accounts, id=a.id): self.logger.warning('Ignoring duplicate card %r', a.id) continue a.parent = previous_account if line[self.COL_HISTORY] == 'true': a._inv = False a._link = self.get_history_link() a._args = self.make__args_dict(line) else: a._inv = True a._args = {'_ipc_eventValue': line[self.COL_ID], '_ipc_fireEvent': line[self.COL_FIRE_EVENT], } a._link = self.doc.xpath('//form[@name="changePageForm"]')[0].attrib['action'] if a.type is Account.TYPE_CARD: a.coming = a.balance a.balance = Decimal('0.0') accounts.append(a) previous_account = a return accounts
def get_life_insurance_list(self, accounts): self.life_insurances.go() for ins in self.page.iter_lifeinsurances(univers=self.current_univers): ins.parent = find_object(accounts, _number=ins._parent_number, type=Account.TYPE_CHECKING) yield ins
def get_price(self, id, price=None): product = Product(id.split('.')[0]) product.backend = self.name price = find_object(self.iter_prices([product]), id=id, error=PriceNotFound) price.shop.info = self.browser.get_shop_info(price.id.split('.', 2)[-1]) return price
def parse(self, el): if any(s in CleanText('.')(el) for s in ['Avoir disponible', 'Solde']) or self.page.is_inner(CleanText('.')(el)): self.env['category'] = u'Interne' else: self.env['category'] = u'Externe' if self.env['category'] == u'Interne': _id = Regexp(CleanText('.'), '- (.*?) -')(el) if _id == self.env['account_id']: raise SkipItem() try: account = find_object(self.page.browser.get_accounts_list(), id=_id, error=AccountNotFound) self.env['id'] = _id self.env['label'] = account.label self.env['iban'] = account.iban except AccountNotFound: self.env['id'] = Regexp(CleanText('.'), '- (.*?) -')(el).replace(' ', '') self.env['iban'] = NotAvailable label = CleanText('.')(el).split('-') holder = label[-1] if not any(string in label[-1] for string in ['Avoir disponible', 'Solde']) else label[-2] self.env['label'] = '%s %s' % (label[0].strip(), holder.strip()) self.env['bank_name'] = u'La Banque Postale' else: self.env['id'] = self.env['iban'] = Regexp(CleanText('.'), '- (.*?) -')(el).replace(' ', '') self.env['label'] = Regexp(CleanText('.'), '- (.*?) - (.*)', template='\\2')(el).strip() first_part = CleanText('.')(el).split('-')[0].strip() self.env['bank_name'] = u'La Banque Postale' if first_part in ['CCP', 'PEL'] else NotAvailable if self.env['id'] in self.parent.objects: # user add two recipients with same iban... raise SkipItem()
def iter_transfer_recipients(self, origin_account): if not self.browser.is_new_website: raise NotImplementedError() if not isinstance(origin_account, Account): origin_account = find_object(self.iter_accounts(), id=origin_account, error=AccountNotFound) return self.browser.iter_recipients(origin_account)
def iter_history(self, account): if account._history_url.startswith('javascript:') or account._history_url == '#': raise NotImplementedError() account = find_object(self.iter_accounts(), id=account.id) # this url (reached with a GET) return some transactions, but not in same format than POST method # and some transactions are duplicated and other are missing, don't take them from GET # because we don't want to manage both way in iter_history self.location(account._history_url) date_range_list = self.page.get_date_range_list() # a date_range is a couple of date like '01/03/201831/03/2018' but current month is often missing and we have to rebuild it # from first one to get very recent transaction without scrap them from 1st page (reached with GET url) if len(date_range_list): date_range_list = [self._build_next_date_range(date_range_list[0])] + date_range_list for date_range in date_range_list: date_guesser = LinearDateGuesser(datetime.datetime.strptime(date_range[10:], "%d/%m/%Y")) try: self.location(account._history_url, data={'date': date_range}) except ServerError as error: if error.response.status_code == 500: if 'RELEVE NON DISPONIBLE A CETTE PERIODE' in error.response.text: continue # just skip because it's still possible to have transactions next months # Yes, they really did that heresy... else: raise for tr in sorted_transactions(self.page.iter_history(date_guesser=date_guesser)): yield tr
def init_transfer(self, transfer, **params): if not transfer.label: raise TransferInvalidLabel() self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date)
def get_history(self, account): if account.type == Account.TYPE_LOAN: return [] headers = { 'Content-Type': 'application/json; charset=UTF-8', 'Accept': 'application/json, text/javascript, */*; q=0.01' } data = { 'contexte': '', 'dateEntree': None, 'filtreEntree': None, 'donneesEntree': json.dumps(account._formated), } items = [] self.cenet_account_history.go(data=json.dumps(data), headers=headers) # there might be some duplicate transactions regarding the card type ones # because some requests lead to the same transaction list # even with different parameters/data in the request card_tr_list = [] while True: data_out = self.page.doc['DonneesSortie'] for tr in self.page.get_history(): items.append(tr) if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY: if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate): self.logger.warning('Duplicated transaction: %s', tr) items.pop() continue card_tr_list.append(tr) tr.deleted = True tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label] donneesEntree = {} donneesEntree['Compte'] = account._formated donneesEntree['ListeOperations'] = [tr_dict[0]] deferred_data = { 'contexte': '', 'dateEntree': None, 'donneesEntree': json.dumps(donneesEntree).replace('/', '\\/'), 'filtreEntree': json.dumps(tr_dict[0]).replace('/', '\\/') } tr_detail_page = self.cenet_tr_detail.open(data=json.dumps(deferred_data), headers=headers) for tr in tr_detail_page.get_history(): items.append(tr) offset = self.page.next_offset() if not offset: break data['filtreEntree'] = json.dumps({ 'Offset': offset, }) self.cenet_account_history.go(data=json.dumps(data), headers=headers) return sorted_transactions(items)
def init_transfer(self, transfer, **params): if self.config['website'].get() != 'par': raise NotImplementedError() transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split()) self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer)
def get_history(self, account): if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): self.logger.warning('This account is not supported') raise NotImplementedError() # some accounts may exist without a link to any history page if not account._form and (not account.url or 'CATITRES' in account.url): return if account._perimeter != self.current_perimeter: self.go_perimeter(account._perimeter) if account.type not in (Account.TYPE_LOAN, Account.TYPE_CARD) and account._form: # the account needs a form submission to go to the history # but we need to get the latest form data self.location(self.accounts_url.format(self.sag)) accounts = self.page.iter_accounts() new_account = find_object(accounts, AccountNotFound, id=account.id) self.location(new_account._form.request) # card accounts need to get an updated link if account.type == Account.TYPE_CARD: account = self.get_card(account.id) if account.url and (account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(account)): self.location(account.url.format(self.sag)) if self.cards.is_here(): date_guesser = ChaoticDateGuesser(date.today()-timedelta(weeks=42)) url = self.page.url state = None notfirst = False while url: if notfirst: self.location(url) else: notfirst = True assert self.cards.is_here() for state, tr in self.page.get_history(date_guesser, state): yield tr url = self.page.get_next_url() elif self.page and not self.no_fixed_deposit_page.is_here(): date_guesser = LinearDateGuesser() self.page.order_transactions() while True: assert self.transactions.is_here() for tr in self.page.get_history(date_guesser): yield tr url = self.page.get_next_url() if url is None: break self.location(url)
def iter_accounts(self, accnum, current_univers): seen = set() accounts_list = [] for content in self.get_content(): if accnum != '00000000000' and content['numero'] != accnum: continue for poste in content['postes']: a = Account() a._number = content['numeroLong'] a._nature = poste['codeNature'] a._codeSousPoste = poste['codeSousPoste'] if 'codeSousPoste' in poste else None a._consultable = poste['consultable'] a._univers = current_univers a.id = '%s.%s' % (a._number, a._nature) if a.id in seen: # some accounts like "compte à terme fidélis" have the same _number and _nature # but in fact are kind of closed, so worthless... self.logger.warning('ignored account id %r (%r) because it is already used', a.id, poste.get('numeroDossier')) continue seen.add(a.id) a.type = self.ACCOUNT_TYPES.get(poste['codeNature'], Account.TYPE_UNKNOWN) if a.type == Account.TYPE_UNKNOWN: self.logger.warning("unknown type %s" % poste['codeNature']) if a.type == Account.TYPE_CARD: a.parent = find_object(accounts_list, _number=a._number, type=Account.TYPE_CHECKING) if 'numeroDossier' in poste and poste['numeroDossier']: a._file_number = poste['numeroDossier'] a.id += '.%s' % a._file_number if poste['postePortefeuille']: a.label = u'Portefeuille Titres' a.balance = Decimal(str(poste['montantTitres']['valeur'])) a.currency = poste['montantTitres']['monnaie']['code'].strip() if not a.balance and not a.currency and 'dateTitres' not in poste: continue accounts_list.append(a) if 'libelle' not in poste: continue a.label = ' '.join([content['intitule'].strip(), poste['libelle'].strip()]) a.balance = Decimal(str(poste['solde']['valeur'])) a.currency = poste['solde']['monnaie']['code'].strip() # Some accounts may have balance currency if 'Solde en devises' in a.label and a.currency != u'EUR': a.id += str(poste['monnaie']['codeSwift']) accounts_list.append(a) return accounts_list
def iter_account_owners(self): """ Some connections have a "Compte de Tiers" section with several people each having their own accounts. We must fetch the account for each person and store the owner of each account. """ if self.unique_accounts_list: for account in self.unique_accounts_list.values(): yield account else: self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) if self.owners_list.is_here(): self.owners = self.page.get_owners_urls() # self.accounts_list will be a dictionary of owners each # containing a dictionary of the owner's accounts. for owner in range(len(self.owners)): self.accounts_list[owner] = {} self.update_accounts_list(owner, True) # We must set an "_owner" attribute to each account. for a in self.accounts_list[owner].values(): a._owner = owner # go on cards page if there are cards accounts for a in self.accounts_list[owner].values(): if a.type == Account.TYPE_CARD: self.location(a.url) break # get all couples (card, parent) on cards page all_card_and_parent = [] if self.cbPage.is_here(): all_card_and_parent = self.page.get_all_parent_id() self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) # update cards parent and currency for a in self.accounts_list[owner].values(): if a.type == Account.TYPE_CARD: for card in all_card_and_parent: if a.id in card[0].replace(' ', ''): a.parent = find_object(self.accounts_list[owner].values(), id=card[1]) if a.parent and not a.currency: a.currency = a.parent.currency # We must get back to the owners list before moving to the next owner: self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) # Fill a dictionary will all accounts without duplicating common accounts: for owner in self.accounts_list.values(): for account in owner.values(): if account.id not in self.unique_accounts_list.keys(): self.unique_accounts_list[account.id] = account for account in self.unique_accounts_list.values(): yield account
def get_accounts_list(self): if not self.accounts_list: self.update_accounts_list() for a in self.accounts_list.values(): # Get parent of card account if a.type == Account.TYPE_CARD: card_page = self.open(a.url).page parent_id = card_page.get_parent_id() a.parent = find_object(self.accounts_list.values(), id=parent_id) yield a
def parse(self, el): if bool(CleanText('./div[@id="soldeEurosCompte"]')(self)): self.env['category'] = u'Interne' account = find_object(self.page.browser.get_accounts_list(), id=self.obj_id(self)) self.env['iban'] = account.iban if account else NotAvailable self.env['bank_name'] = u'LCL' else: self.env['category'] = u'Externe' self.env['iban'] = self.obj_id(self) self.env['bank_name'] = NotAvailable
def get_city(self, _id): cities = list(self.iter_city_search(_id)) if len(cities) == 0: raise CityNotFound() try: return find_object(cities, id=_id, error=CityNotFound) except CityNotFound: return cities[0]
def get_account(self, id): """ Get an account from its ID. :param id: ID of the account :type id: :class:`str` :rtype: :class:`Account` :raises: :class:`AccountNotFound` """ return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
def get_document(self, id): """ Get a document. :param id: ID of document :rtype: :class:`Document` :raises: :class:`DocumentNotFound` """ return find_object(self.iter_documents(id.split("#")[0]), id=id, error=DocumentNotFound)
def init_transfer(self, transfer, **params): if not transfer.label: raise TransferInvalidLabel() self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) assert account.id.isdigit() # Only 11 first character are required to do transfer account.id = account.id[:11] return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date)
def init_transfer(self, transfer, **params): if self.config['website'].get() != 'pp': raise NotImplementedError() if transfer.label is None: raise TransferInvalidLabel() self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) assert account.id.isdigit() # quantize to show 2 decimals. amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2) return self.browser.init_transfer(account, recipient, amount, transfer.label, transfer.exec_date)
def iter_sensors(self, gauge, pattern=None): if not isinstance(gauge, Gauge): gauge = find_object(self.browser.get_rivers_list(), id=gauge, error=SensorNotFound) if pattern is None: for sensor in gauge.sensors: yield sensor else: lowpattern = pattern.lower() for sensor in gauge.sensors: if lowpattern in sensor.name.lower(): yield sensor
def iter_subscription(self): subscriber = self.get_profile() self.subscription.go() for sub in self.page.iter_subscription(): sub.subscriber = subscriber.name account = find_object(self.get_accounts_list(), id=sub.id, error=AccountNotFound) sub.label = account.label yield sub
def get_accounts_on_space(self, space, fill_account=True): accounts_list = [] self.change_space(space) for acc in self.page.get_list(): acc._space = space if fill_account: try: self.fill_account(acc) except ServerError: pass assert not find_object(accounts_list, id=acc.id), 'There is a duplicate account.' accounts_list.append(acc) yield acc for loan in self.iter_detailed_loans(): loan._space = space assert not find_object(accounts_list, id=loan.id), 'There is a duplicate loan.' accounts_list.append(loan) yield loan
def init_transfer(self, transfer, **params): if self.config['website'].get() != 'par': raise NotImplementedError() self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) try: # quantize to show 2 decimals. amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2) except (AssertionError, ValueError): raise TransferError('something went wrong') return self.browser.init_transfer(account, recipient, amount, transfer)
def market_accounts_matching(self, accounts_list, market_accounts_list): for market_account in market_accounts_list: account = find_object(accounts_list, id=market_account.id) if account: account.label = market_account.label or account.label if account.type == Account.TYPE_MARKET: account.balance = market_account.balance or account.balance # Some PEA accounts are only present on the Market page but the balance includes # liquidities from the DAV PEA, so we need another request to fetch the balance: elif account.type == Account.TYPE_PEA: url = 'https://www.cabourse.credit-agricole.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.WalletVal?nump=%s:%s' self.location(url % (account.id, self.code_caisse)) account.balance = self.page.get_pea_balance()
def iter_sensors(self, gauge, pattern=None): if not isinstance(gauge, Gauge): gauge = find_object(self.iter_gauges(), id=gauge, error=SensorNotFound) gauge = self.browser.get_station_infos(gauge).next() if pattern is None: for sensor in gauge.sensors: yield sensor else: lowpattern = pattern.lower() for sensor in gauge.sensors: if lowpattern in sensor.name.lower(): yield sensor
def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: for category in self.browser.get_categories(): yield category elif collection.path_level == 1 and collection.split_path[0].startswith('vid_'): cat = find_object(self.browser.get_categories(), id=collection.split_path[0], error=None) for video in self.browser.iter_videos(cat.title): yield video else: for cat in self.browser.iter_subcategories(collection.split_path): yield cat
def iter_documents(self, subscription): """ Iter documents. :param subscription: subscription to get documents :type subscription: :class:`Subscription` :rtype: iter[:class:`Document`] """ if not isinstance(subscription, Subscription): subscription = find_object(self.iter_subscription(), id=subscription, error=SubscriptionNotFound) return self.browser.get_documents(subscription)
def init_transfer(self, transfer, **params): # There is a check on the website, transfer can't be done with too long reason. if transfer.label: transfer.label = transfer.label[:30] self.logger.info('Going to do a new transfer') if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) else: recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) try: # quantize to show 2 decimals. amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2) except (AssertionError, ValueError): raise TransferError('something went wrong') return self.browser.init_transfer(account, recipient, amount, transfer.label, transfer.exec_date)
def post_message(self, message): if not self.config['username'].get(): raise BrowserForbidden() self.browser.post( find_object(self.iter_threads(), id=message.full_id.split('.')[0]), message.content)
def get_bill(self, _id): subscription = self.get_subscription(_id.split('-')[0]) return find_object(self.browser.get_bills(subscription), id=_id, error=BillNotFound)
def get_last_measure(self, sensor_id): gauge_id = sensor_id.split('-')[0] return find_object(self.iter_sensors(gauge_id), id=sensor_id, error=SensorNotFound).lastvalue
def get_bill(self, _id): subid = _id.split('.')[0] subscription = self.get_subscription(subid) return find_object(self.iter_bills(subscription), id=_id, error=BillNotFound)
def iter_transfer_recipients(self, origin_account): if not isinstance(origin_account, Account): origin_account = find_object(self.iter_accounts(), id=origin_account, error=AccountNotFound) return self.browser.iter_recipients(origin_account)
def new_recipient(self, recipient, **params): if not re.match(u"^[-+.,:/?() éèêëïîñàâäãöôòõùûüÿ0-9a-z']+$", recipient.label, re.I): raise RecipientInvalidLabel( 'Recipient label contains invalid characters') if 'sms_code' in params and not re.match(r'^[a-z0-9]{6}$', params['sms_code'], re.I): # check before send sms code because it can crash website if code is invalid raise AddRecipientBankError("SMS code %s is invalid" % params['sms_code']) # avoid `iter_accounts` if there is only one perimeter if len(self.perimeters) > 1: accounts = list(self.iter_accounts()) assert recipient.origin_account_id, 'Origin account id is mandatory for multispace' account = find_object(accounts, id=recipient.origin_account_id, error=AccountNotFound) self.go_to_perimeter(account._perimeter) self.transfer_init_page.go() assert self.transfer_init_page.is_here() if not self.page.add_recipient_is_allowed(): if not [ rec for rec in self.page.iter_recipients() if rec.category == 'Externe' ]: raise AddRecipientBankError( 'Vous ne pouvez pas ajouter de bénéficiaires, veuillez contacter votre banque.' ) assert False, 'Xpath for a recipient add is not catched' self.location(self.page.url_list_recipients()) # there are 2 pages from where we can add a new recipient: # - RecipientListPage, but the link is sometimes missing # - TransferPage, start making a transfer with a new recipient but don't complete the transfer # but it seems dangerous since we have to set an amount, etc. # so we implement it in 2 ways with a preference for RecipientListPage if self.page.url_add_recipient(): self.logger.debug( 'good, we can add a recipient from the recipient list') else: # in this case, the link was missing self.logger.warning( 'cannot add a recipient from the recipient list page, pretending to make a transfer in order to add it' ) self.transfer_init_page.go() assert self.transfer_init_page.is_here() self.location(self.page.url_add_recipient()) if not ('sms_code' in params and self.page.can_send_code()): self.page.send_sms() # go to a GET page, so StatesMixin can reload it self.accounts.go() raise AddRecipientStep( self.build_recipient(recipient), Value('sms_code', label='Veuillez saisir le code SMS')) else: self.page.submit_code(params['sms_code']) err = hasattr(self.page, 'get_sms_error') and self.page.get_sms_error() if err: raise AddRecipientBankError(message=err) self.page.submit_recipient(recipient.label, recipient.iban) self.page.confirm_recipient() self.page.check_recipient_error() if self.transfer_page.is_here(): # in this case, we were pretending to make a transfer, just to add the recipient # go back to transfer page to abort the transfer and see the new recipient self.transfer_init_page.go() assert self.transfer_init_page.is_here() res = self.page.find_recipient(recipient.iban) assert res, 'Recipient with iban %s could not be found' % recipient.iban return res
def get_document(self, _id): return find_object(self.iter_documents(None), id=_id, error=DocumentNotFound)
def _get_sensor_by_id(self, id): gid = id.partition('.')[0] return find_object(self.iter_sensors(gid), id=id)
def _get_gauge_by_id(self, id): return find_object(self.browser.iter_gauges(), id=id)
def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() self.two_cards_page = None self.accounts_list = [] self.revolving_accounts = [] self.unavailablecards = [] self.cards_histo_available = [] self.cards_list = [] self.cards_list2 = [] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) if self.cards_hist_available.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) if not self.cards_list: self.cards_hist_available2.go(subbank=self.currentSubBank) if self.cards_hist_available2.is_here(): self.unavailablecards.extend( self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available2 self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) for acc in self.revolving_loan_list.stay_or_go( subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) self.revolving_accounts.append(acc.label.lower()) # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: # We need to return to the main page to avoid navigation error self.cards_activity.go(subbank=self.currentSubBank) page = self.open(company).page if isinstance( company, basestring) else company for card in page.iter_cards(): card2 = find_object(self.cards_list, id=card.id[:16]) if card2: # In order to keep the id of the card from the old space, we exchange the following values card._link_id = card2._link_id card._parent_id = card2._parent_id card.coming = card2.coming card._referer = card2._referer card._secondpage = card2._secondpage self.accounts_list.remove(card2) self.accounts_list.append(card) self.cards_list2.append(card) self.cards_list.extend(self.cards_list2) # Populate accounts from old website if not self.is_new_website: self.accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) # Populate accounts from new website else: self.new_accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban( self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts( self.accounts_list) self.li.go(subbank=self.currentSubBank) self.accounts_list.extend(self.page.iter_li_accounts()) for acc in self.cards_list: if hasattr(acc, '_parent_id'): acc.parent = find_object(self.accounts_list, id=acc._parent_id) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [ acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label) ] if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) self.ownership_guesser() return self.accounts_list
def get_document(self, id): return find_object(self.browser.iter_documents(), id=id, error=DocumentNotFound)
def get_document(self, bill): return find_object( self.iter_documents(bill.split("#")[0]), id=bill, error=DocumentNotFound )
def iter_history(self, account, coming=False): handled_history_types = ( Account.TYPE_CHECKING, Account.TYPE_CARD, Account.TYPE_SAVINGS, Account.TYPE_PEA, ) if account.type not in handled_history_types: self.unhandled_method(account.id) return if account.type == Account.TYPE_CARD: self.go_to_perimeter(account._perimeter) self.accounts.go() self.page.go_to_card(account._card_link) assert (self.cards_page.is_here() or self.multiple_cards_page.is_here()), \ 'Failed to reach card details for card %s.' % account.id if self.multiple_cards_page.is_here(): # We need to go to the correct card transactions with its number. card_url = self.page.get_transactions_link(account._raw_number) self.location(card_url) # When there are several future coming summaries, # we must skip the ongoing one but fetch the other ones # even if they are in the future. ongoing_coming = self.page.get_ongoing_coming() if not ongoing_coming: # This card has no available history or coming. return card_transactions = [] latest_date = None for tr in self.page.get_card_transactions(latest_date, ongoing_coming): card_transactions.append(tr) if not card_transactions: return # Pagination: we must fetch the date of the last transaction # because the summary of next transactions may not # be available on the next page latest_date = card_transactions[-1].date next_page_url = self.page.get_next_page() while next_page_url: self.location(next_page_url) for tr in self.page.get_card_transactions( latest_date, ongoing_coming): card_transactions.append(tr) next_page_url = self.page.get_next_page() for tr in sorted_transactions(card_transactions): yield tr return # Transactions of accounts without form/url or with 'CATITRES' and 'bgpi' in url cannot be handled. if not account._form and (not account.url or 'CATITRES' in account.url or 'bgpi' in account.url): self.unhandled_method(account.id) return # Access acount details: if account.url: # Refresh the session_value before going to the account URL new_session_value = 'sessionSAG=' + self.session_value updated_url = re.sub(r'sessionSAG=([^&]+)', new_session_value, account.url) self.location(updated_url) elif account._form: # We cannot use forms if we are not on the account's perimeter: # we need to go to the correct perimeter and refresh forms. # The form submission sometimes fails so we try several # times until we get to the account history page. for form in range(3): self.accounts.stay_or_go() self.go_to_perimeter(account._perimeter) # Only fetch the perimeter's regular accounts (Checking & Savings) # No need to go to Wealth, Loans or Netfinca for transactions refreshed_account = find_object( self.iter_perimeter_regular_accounts(iban=False), AccountNotFound, id=account.id) refreshed_account._form.submit() if self.failed_history.is_here(): self.logger.warning( 'Form submission failed to reach the account history, we try again.' ) continue break # 3 types of history pages were identified so far if not (self.checking_history.is_here() or self.savings_history.is_here() or self.other_savings_history.is_here()): self.unhandled_method(account.id) date_guesser = LinearDateGuesser(date_max_bump=timedelta(30)) for tr in self.page.iter_history(date_guesser=date_guesser): yield tr
def get_document(self, _id): subscription = self.get_subscription(_id.split('-')[0]) return find_object(self.browser.get_documents(subscription), id=_id, error=DocumentNotFound)
def get_account(self, _id, space=None): return find_object(self.get_accounts_list(fill_account=False, space=space), id=_id, error=AccountNotFound)
def get_history(self, account): if account.type == Account.TYPE_LOAN: return [] headers = { 'Content-Type': 'application/json; charset=UTF-8', 'Accept': 'application/json, text/javascript, */*; q=0.01' } data = { 'contexte': '', 'dateEntree': None, 'filtreEntree': None, 'donneesEntree': json.dumps(account._formated), } items = [] self.cenet_account_history.go(data=json.dumps(data), headers=headers) # there might be some duplicate transactions regarding the card type ones # because some requests lead to the same transaction list # even with different parameters/data in the request card_tr_list = [] while True: data_out = self.page.doc['DonneesSortie'] for tr in self.page.get_history(): items.append(tr) if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY: if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate): self.logger.warning('Duplicated transaction: %s', tr) items.pop() continue card_tr_list.append(tr) tr.deleted = True tr_dict = [ tr_dict2 for tr_dict2 in data_out if tr_dict2['Libelle'] == tr.label ] donneesEntree = {} donneesEntree['Compte'] = account._formated donneesEntree['ListeOperations'] = [tr_dict[0]] deferred_data = { 'contexte': '', 'dateEntree': None, 'donneesEntree': json.dumps(donneesEntree).replace('/', '\\/'), 'filtreEntree': json.dumps(tr_dict[0]).replace('/', '\\/') } tr_detail_page = self.cenet_tr_detail.open( data=json.dumps(deferred_data), headers=headers) for tr in tr_detail_page.get_history(): items.append(tr) offset = self.page.next_offset() if not offset: break data['filtreEntree'] = json.dumps({ 'Offset': offset, }) self.cenet_account_history.go(data=json.dumps(data), headers=headers) return sorted_transactions(items)
def get_document(self, _id): subid = _id.rsplit('_', 1)[0] subscription = self.get_subscription(subid) return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound)
def get_video_from_id(self, _id, category): return find_object(self.get_categories_videos(category), id=u'%s#%s' % (_id, category))
def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound)
def iter_gauge_history(self, sensor_id): gauge_id = sensor_id.split('-')[0] return find_object(self.iter_sensors(gauge_id), id=sensor_id, error=SensorNotFound).history
def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() self.two_cards_page = None self.accounts_list = [] self.revolving_accounts = [] self.unavailablecards = [] self.cards_histo_available = [] self.cards_list =[] self.cards_list2 =[] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) if self.cards_hist_available.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) if not self.cards_list: self.cards_hist_available2.go(subbank=self.currentSubBank) if self.cards_hist_available2.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) for acc in self.page.iter_accounts(): acc._referer = self.cards_hist_available2 self.accounts_list.append(acc) self.cards_list.append(acc) self.cards_histo_available.append(acc.id) for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) self.revolving_accounts.append(acc.label.lower()) # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: # We need to return to the main page to avoid navigation error self.cards_activity.go(subbank=self.currentSubBank) page = self.open(company).page if isinstance(company, basestring) else company for card in page.iter_cards(): card2 = find_object(self.cards_list, id=card.id[:16]) if card2: # In order to keep the id of the card from the old space, we exchange the following values card._link_id = card2._link_id card._parent_id = card2._parent_id card.coming = card2.coming card._referer = card2._referer card._secondpage = card2._secondpage self.accounts_list.remove(card2) self.accounts_list.append(card) self.cards_list2.append(card) self.cards_list.extend(self.cards_list2) # Populate accounts from old website if not self.is_new_website: self.logger.info('On old creditmutuel website') self.accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank) self.page.add_por_accounts(self.accounts_list) # Populate accounts from new website else: self.new_accounts.stay_or_go(subbank=self.currentSubBank) has_no_account = self.page.has_no_account() self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank) self.page.add_por_accounts(self.accounts_list) self.li.go(subbank=self.currentSubBank) self.accounts_list.extend(self.page.iter_li_accounts()) # This type of account is like a loan, for splitting payments in smaller amounts. # Its history is irrelevant because money is debited from a checking account and # the balance is not even correct, so ignore it. excluded_label = ['etalis', 'valorisation totale'] accounts_by_id = {} for acc in self.accounts_list: if acc.label.lower() not in excluded_label: accounts_by_id[acc.id] = acc # Set the parent to loans and cards accounts for acc in self.accounts_list: if acc.type == Account.TYPE_CARD and not empty(getattr(acc, '_parent_id', None)): acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id: acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) self.accounts_list = list(accounts_by_id.values()) if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) self.ownership_guesser() return self.accounts_list
def get_city(self, _id): return find_object(self.iter_city_search(_id), id=_id, error=CityNotFound)
def get_subscription(self, _id): return find_object(self.browser.get_subscriptions(), id=_id, error=SubscriptionNotFound)
def get_card(self, id): return find_object(self.get_cards(), id=id)
def get_account(self, _id): return find_object(self.iter_accounts(), id=_id, error=AccountNotFound)
def get_history(self, account): if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE): self.logger.warning('This account is not supported') raise NotImplementedError() # some accounts may exist without a link to any history page if not hasattr(account, '_form') and (not account.url or 'CATITRES' in account.url): return if account._perimeter != self.current_perimeter: self.go_perimeter(account._perimeter) if hasattr(account, '_form'): # the account needs a form submission to go to the history # but we need to get the latest form data self.location(self.accounts_url.format(self.sag)) accounts = self.page.get_list(use_links=False) new_account = find_object(accounts, AccountNotFound, id=account.id) self.location(new_account._form.request) # card accounts need to get an updated link if account.type == Account.TYPE_CARD: account = self.get_card(account.id) if account.url and (account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(account)): self.location(account.url.format(self.sag)) if self.cards.is_here(): date_guesser = ChaoticDateGuesser(date.today() - timedelta(weeks=42)) url = self.page.url state = None notfirst = False while url: if notfirst: self.location(url) else: notfirst = True assert self.cards.is_here() for state, tr in self.page.get_history(date_guesser, state): yield tr url = self.page.get_next_url() elif self.page: date_guesser = LinearDateGuesser() self.page.order_transactions() while True: assert self.transactions.is_here() for tr in self.page.get_history(date_guesser): yield tr url = self.page.get_next_url() if url is None: break self.location(url)
def get_account(self, _id): return find_object(self.get_accounts_list(), id=_id, error=AccountNotFound)
def get_account(self, id): account_list = self.get_accounts_list() return find_object(account_list, id=id)
def get_account_for_history(self, id): account_list = list(self._iter_accounts()) return find_object(account_list, id=id)
def get_account(self, _id): return find_object(self.browser.iter_account_owners(), id=_id, error=AccountNotFound)