def get_cards(self, accounts_list=None):
# accounts_list is only used by get_list
self.location(self.accounts_url.format(self.sag))
for idelco, parent_id in self.page.iter_idelcos():
if not self.accounts.is_here():
self.location(self.accounts_url.format(self.sag))
obj = self.page.get_idelco(idelco)
if isinstance(obj, basestring):
self.location(obj)
else:
self.page.submit_card(obj)
assert self.cards.is_here() or self.cards2.is_here()
if self.page.several_cards():
for account in self.page.iter_cards():
if accounts_list:
account.parent = find_object(accounts_list, id=account._parent_id)
yield account
else:
for account in self.page.iter_card():
if accounts_list:
account._parent_id = parent_id
account.parent = find_object(accounts_list, id=account._parent_id)
yield account
def init_transfer(self, transfer, **params):
self.logger.info('Going to do a new transfer')
transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z/\-\?:\(\)\.,\'\+ ]+', '', transfer.label).split()).upper()
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
return self.browser.init_transfer(account, recipient, transfer)
def get_list(self):
accounts = []
previous_account = None
noaccounts = self.get_from_js('_js_noMvts =', ';')
if noaccounts is not None:
assert 'avez aucun compte' in noaccounts
return []
txt = self.get_from_js('_data = new Array(', ');', is_list=True)
if txt is None:
raise BrowserUnavailable('Unable to find accounts list in scripts')
data = json.loads('[%s]' % txt.replace("'", '"'))
for line in data:
a = Account()
a.id = line[self.COL_ID].replace(' ', '')
if re.match(r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)', a.id):
a.id = str(CleanDecimal().filter(a.id))
a._acc_nb = a.id.split('_')[0] if len(a.id.split('_')) > 1 else None
a.label = MyStrip(line[self.COL_LABEL], xpath='.//div[@class="libelleCompteTDB"]')
# This account can be multiple life insurance accounts
if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *':
continue
a.balance = Decimal(FrenchTransaction.clean_amount(line[self.COL_BALANCE]))
a.currency = a.get_currency(line[self.COL_BALANCE])
a.type = self.get_account_type(a.label)
# The parent account must be created right before
if a.type == Account.TYPE_CARD:
# duplicate
if find_object(accounts, id=a.id):
self.logger.warning('Ignoring duplicate card %r', a.id)
continue
a.parent = previous_account
if line[self.COL_HISTORY] == 'true':
a._inv = False
a._link = self.get_history_link()
a._args = self.make__args_dict(line)
else:
a._inv = True
a._args = {'_ipc_eventValue': line[self.COL_ID],
'_ipc_fireEvent': line[self.COL_FIRE_EVENT],
}
a._link = self.doc.xpath('//form[@name="changePageForm"]')[0].attrib['action']
if a.type is Account.TYPE_CARD:
a.coming = a.balance
a.balance = Decimal('0.0')
accounts.append(a)
previous_account = a
return accounts
def get_life_insurance_list(self, accounts):
self.life_insurances.go()
for ins in self.page.iter_lifeinsurances(univers=self.current_univers):
ins.parent = find_object(accounts, _number=ins._parent_number, type=Account.TYPE_CHECKING)
yield ins
def get_price(self, id, price=None):
product = Product(id.split('.')[0])
product.backend = self.name
price = find_object(self.iter_prices([product]), id=id, error=PriceNotFound)
price.shop.info = self.browser.get_shop_info(price.id.split('.', 2)[-1])
return price
def parse(self, el):
if any(s in CleanText('.')(el) for s in ['Avoir disponible', 'Solde']) or self.page.is_inner(CleanText('.')(el)):
self.env['category'] = u'Interne'
else:
self.env['category'] = u'Externe'
if self.env['category'] == u'Interne':
_id = Regexp(CleanText('.'), '- (.*?) -')(el)
if _id == self.env['account_id']:
raise SkipItem()
try:
account = find_object(self.page.browser.get_accounts_list(), id=_id, error=AccountNotFound)
self.env['id'] = _id
self.env['label'] = account.label
self.env['iban'] = account.iban
except AccountNotFound:
self.env['id'] = Regexp(CleanText('.'), '- (.*?) -')(el).replace(' ', '')
self.env['iban'] = NotAvailable
label = CleanText('.')(el).split('-')
holder = label[-1] if not any(string in label[-1] for string in ['Avoir disponible', 'Solde']) else label[-2]
self.env['label'] = '%s %s' % (label[0].strip(), holder.strip())
self.env['bank_name'] = u'La Banque Postale'
else:
self.env['id'] = self.env['iban'] = Regexp(CleanText('.'), '- (.*?) -')(el).replace(' ', '')
self.env['label'] = Regexp(CleanText('.'), '- (.*?) - (.*)', template='\\2')(el).strip()
first_part = CleanText('.')(el).split('-')[0].strip()
self.env['bank_name'] = u'La Banque Postale' if first_part in ['CCP', 'PEL'] else NotAvailable
if self.env['id'] in self.parent.objects: # user add two recipients with same iban...
raise SkipItem()
def iter_transfer_recipients(self, origin_account):
if not self.browser.is_new_website:
raise NotImplementedError()
if not isinstance(origin_account, Account):
origin_account = find_object(self.iter_accounts(), id=origin_account, error=AccountNotFound)
return self.browser.iter_recipients(origin_account)
def iter_history(self, account):
if account._history_url.startswith('javascript:') or account._history_url == '#':
raise NotImplementedError()
account = find_object(self.iter_accounts(), id=account.id)
# this url (reached with a GET) return some transactions, but not in same format than POST method
# and some transactions are duplicated and other are missing, don't take them from GET
# because we don't want to manage both way in iter_history
self.location(account._history_url)
date_range_list = self.page.get_date_range_list()
# a date_range is a couple of date like '01/03/201831/03/2018' but current month is often missing and we have to rebuild it
# from first one to get very recent transaction without scrap them from 1st page (reached with GET url)
if len(date_range_list):
date_range_list = [self._build_next_date_range(date_range_list[0])] + date_range_list
for date_range in date_range_list:
date_guesser = LinearDateGuesser(datetime.datetime.strptime(date_range[10:], "%d/%m/%Y"))
try:
self.location(account._history_url, data={'date': date_range})
except ServerError as error:
if error.response.status_code == 500:
if 'RELEVE NON DISPONIBLE A CETTE PERIODE' in error.response.text:
continue
# just skip because it's still possible to have transactions next months
# Yes, they really did that heresy...
else:
raise
for tr in sorted_transactions(self.page.iter_history(date_guesser=date_guesser)):
yield tr
def init_transfer(self, transfer, **params):
if not transfer.label:
raise TransferInvalidLabel()
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date)
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01'
}
data = {
'contexte': '',
'dateEntree': None,
'filtreEntree': None,
'donneesEntree': json.dumps(account._formated),
}
items = []
self.cenet_account_history.go(data=json.dumps(data), headers=headers)
# there might be some duplicate transactions regarding the card type ones
# because some requests lead to the same transaction list
# even with different parameters/data in the request
card_tr_list = []
while True:
data_out = self.page.doc['DonneesSortie']
for tr in self.page.get_history():
items.append(tr)
if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY:
if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate):
self.logger.warning('Duplicated transaction: %s', tr)
items.pop()
continue
card_tr_list.append(tr)
tr.deleted = True
tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label]
donneesEntree = {}
donneesEntree['Compte'] = account._formated
donneesEntree['ListeOperations'] = [tr_dict[0]]
deferred_data = {
'contexte': '',
'dateEntree': None,
'donneesEntree': json.dumps(donneesEntree).replace('/', '\\/'),
'filtreEntree': json.dumps(tr_dict[0]).replace('/', '\\/')
}
tr_detail_page = self.cenet_tr_detail.open(data=json.dumps(deferred_data), headers=headers)
for tr in tr_detail_page.get_history():
items.append(tr)
offset = self.page.next_offset()
if not offset:
break
data['filtreEntree'] = json.dumps({
'Offset': offset,
})
self.cenet_account_history.go(data=json.dumps(data), headers=headers)
return sorted_transactions(items)
def init_transfer(self, transfer, **params):
if self.config['website'].get() != 'par':
raise NotImplementedError()
transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split())
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
transfer.amount = transfer.amount.quantize(Decimal('.01'))
return self.browser.init_transfer(account, recipient, transfer)
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP):
self.logger.warning('This account is not supported')
raise NotImplementedError()
# some accounts may exist without a link to any history page
if not account._form and (not account.url or 'CATITRES' in account.url):
return
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
if account.type not in (Account.TYPE_LOAN, Account.TYPE_CARD) and account._form:
# the account needs a form submission to go to the history
# but we need to get the latest form data
self.location(self.accounts_url.format(self.sag))
accounts = self.page.iter_accounts()
new_account = find_object(accounts, AccountNotFound, id=account.id)
self.location(new_account._form.request)
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_card(account.id)
if account.url and (account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(account)):
self.location(account.url.format(self.sag))
if self.cards.is_here():
date_guesser = ChaoticDateGuesser(date.today()-timedelta(weeks=42))
url = self.page.url
state = None
notfirst = False
while url:
if notfirst:
self.location(url)
else:
notfirst = True
assert self.cards.is_here()
for state, tr in self.page.get_history(date_guesser, state):
yield tr
url = self.page.get_next_url()
elif self.page and not self.no_fixed_deposit_page.is_here():
date_guesser = LinearDateGuesser()
self.page.order_transactions()
while True:
assert self.transactions.is_here()
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
if url is None:
break
self.location(url)
def iter_accounts(self, accnum, current_univers):
seen = set()
accounts_list = []
for content in self.get_content():
if accnum != '00000000000' and content['numero'] != accnum:
continue
for poste in content['postes']:
a = Account()
a._number = content['numeroLong']
a._nature = poste['codeNature']
a._codeSousPoste = poste['codeSousPoste'] if 'codeSousPoste' in poste else None
a._consultable = poste['consultable']
a._univers = current_univers
a.id = '%s.%s' % (a._number, a._nature)
if a.id in seen:
# some accounts like "compte à terme fidélis" have the same _number and _nature
# but in fact are kind of closed, so worthless...
self.logger.warning('ignored account id %r (%r) because it is already used', a.id, poste.get('numeroDossier'))
continue
seen.add(a.id)
a.type = self.ACCOUNT_TYPES.get(poste['codeNature'], Account.TYPE_UNKNOWN)
if a.type == Account.TYPE_UNKNOWN:
self.logger.warning("unknown type %s" % poste['codeNature'])
if a.type == Account.TYPE_CARD:
a.parent = find_object(accounts_list, _number=a._number, type=Account.TYPE_CHECKING)
if 'numeroDossier' in poste and poste['numeroDossier']:
a._file_number = poste['numeroDossier']
a.id += '.%s' % a._file_number
if poste['postePortefeuille']:
a.label = u'Portefeuille Titres'
a.balance = Decimal(str(poste['montantTitres']['valeur']))
a.currency = poste['montantTitres']['monnaie']['code'].strip()
if not a.balance and not a.currency and 'dateTitres' not in poste:
continue
accounts_list.append(a)
if 'libelle' not in poste:
continue
a.label = ' '.join([content['intitule'].strip(), poste['libelle'].strip()])
a.balance = Decimal(str(poste['solde']['valeur']))
a.currency = poste['solde']['monnaie']['code'].strip()
# Some accounts may have balance currency
if 'Solde en devises' in a.label and a.currency != u'EUR':
a.id += str(poste['monnaie']['codeSwift'])
accounts_list.append(a)
return accounts_list
def iter_account_owners(self):
"""
Some connections have a "Compte de Tiers" section with several
people each having their own accounts. We must fetch the account
for each person and store the owner of each account.
"""
if self.unique_accounts_list:
for account in self.unique_accounts_list.values():
yield account
else:
self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
if self.owners_list.is_here():
self.owners = self.page.get_owners_urls()
# self.accounts_list will be a dictionary of owners each
# containing a dictionary of the owner's accounts.
for owner in range(len(self.owners)):
self.accounts_list[owner] = {}
self.update_accounts_list(owner, True)
# We must set an "_owner" attribute to each account.
for a in self.accounts_list[owner].values():
a._owner = owner
# go on cards page if there are cards accounts
for a in self.accounts_list[owner].values():
if a.type == Account.TYPE_CARD:
self.location(a.url)
break
# get all couples (card, parent) on cards page
all_card_and_parent = []
if self.cbPage.is_here():
all_card_and_parent = self.page.get_all_parent_id()
self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'})
# update cards parent and currency
for a in self.accounts_list[owner].values():
if a.type == Account.TYPE_CARD:
for card in all_card_and_parent:
if a.id in card[0].replace(' ', ''):
a.parent = find_object(self.accounts_list[owner].values(), id=card[1])
if a.parent and not a.currency:
a.currency = a.parent.currency
# We must get back to the owners list before moving to the next owner:
self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
# Fill a dictionary will all accounts without duplicating common accounts:
for owner in self.accounts_list.values():
for account in owner.values():
if account.id not in self.unique_accounts_list.keys():
self.unique_accounts_list[account.id] = account
for account in self.unique_accounts_list.values():
yield account
def get_accounts_list(self):
if not self.accounts_list:
self.update_accounts_list()
for a in self.accounts_list.values():
# Get parent of card account
if a.type == Account.TYPE_CARD:
card_page = self.open(a.url).page
parent_id = card_page.get_parent_id()
a.parent = find_object(self.accounts_list.values(), id=parent_id)
yield a
def parse(self, el):
if bool(CleanText('./div[@id="soldeEurosCompte"]')(self)):
self.env['category'] = u'Interne'
account = find_object(self.page.browser.get_accounts_list(), id=self.obj_id(self))
self.env['iban'] = account.iban if account else NotAvailable
self.env['bank_name'] = u'LCL'
else:
self.env['category'] = u'Externe'
self.env['iban'] = self.obj_id(self)
self.env['bank_name'] = NotAvailable
def get_city(self, _id):
cities = list(self.iter_city_search(_id))
if len(cities) == 0:
raise CityNotFound()
try:
return find_object(cities, id=_id, error=CityNotFound)
except CityNotFound:
return cities[0]
def get_account(self, id):
"""
Get an account from its ID.
:param id: ID of the account
:type id: :class:`str`
:rtype: :class:`Account`
:raises: :class:`AccountNotFound`
"""
return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
def get_document(self, id):
"""
Get a document.
:param id: ID of document
:rtype: :class:`Document`
:raises: :class:`DocumentNotFound`
"""
return find_object(self.iter_documents(id.split("#")[0]),
id=id,
error=DocumentNotFound)
def init_transfer(self, transfer, **params):
if not transfer.label:
raise TransferInvalidLabel()
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
assert account.id.isdigit()
# Only 11 first character are required to do transfer
account.id = account.id[:11]
return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date)
def init_transfer(self, transfer, **params):
if self.config['website'].get() != 'pp':
raise NotImplementedError()
if transfer.label is None:
raise TransferInvalidLabel()
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
assert account.id.isdigit()
# quantize to show 2 decimals.
amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2)
return self.browser.init_transfer(account, recipient, amount, transfer.label, transfer.exec_date)
def iter_sensors(self, gauge, pattern=None):
if not isinstance(gauge, Gauge):
gauge = find_object(self.browser.get_rivers_list(), id=gauge, error=SensorNotFound)
if pattern is None:
for sensor in gauge.sensors:
yield sensor
else:
lowpattern = pattern.lower()
for sensor in gauge.sensors:
if lowpattern in sensor.name.lower():
yield sensor
def iter_subscription(self):
subscriber = self.get_profile()
self.subscription.go()
for sub in self.page.iter_subscription():
sub.subscriber = subscriber.name
account = find_object(self.get_accounts_list(), id=sub.id, error=AccountNotFound)
sub.label = account.label
yield sub
def get_accounts_on_space(self, space, fill_account=True):
accounts_list = []
self.change_space(space)
for acc in self.page.get_list():
acc._space = space
if fill_account:
try:
self.fill_account(acc)
except ServerError:
pass
assert not find_object(accounts_list, id=acc.id), 'There is a duplicate account.'
accounts_list.append(acc)
yield acc
for loan in self.iter_detailed_loans():
loan._space = space
assert not find_object(accounts_list, id=loan.id), 'There is a duplicate loan.'
accounts_list.append(loan)
yield loan
def init_transfer(self, transfer, **params):
if self.config['website'].get() != 'par':
raise NotImplementedError()
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
try:
# quantize to show 2 decimals.
amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2)
except (AssertionError, ValueError):
raise TransferError('something went wrong')
return self.browser.init_transfer(account, recipient, amount, transfer)
def market_accounts_matching(self, accounts_list, market_accounts_list):
for market_account in market_accounts_list:
account = find_object(accounts_list, id=market_account.id)
if account:
account.label = market_account.label or account.label
if account.type == Account.TYPE_MARKET:
account.balance = market_account.balance or account.balance
# Some PEA accounts are only present on the Market page but the balance includes
# liquidities from the DAV PEA, so we need another request to fetch the balance:
elif account.type == Account.TYPE_PEA:
url = 'https://www.cabourse.credit-agricole.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.WalletVal?nump=%s:%s'
self.location(url % (account.id, self.code_caisse))
account.balance = self.page.get_pea_balance()
def iter_sensors(self, gauge, pattern=None):
if not isinstance(gauge, Gauge):
gauge = find_object(self.iter_gauges(), id=gauge, error=SensorNotFound)
gauge = self.browser.get_station_infos(gauge).next()
if pattern is None:
for sensor in gauge.sensors:
yield sensor
else:
lowpattern = pattern.lower()
for sensor in gauge.sensors:
if lowpattern in sensor.name.lower():
yield sensor
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
for category in self.browser.get_categories():
yield category
elif collection.path_level == 1 and collection.split_path[0].startswith('vid_'):
cat = find_object(self.browser.get_categories(), id=collection.split_path[0], error=None)
for video in self.browser.iter_videos(cat.title):
yield video
else:
for cat in self.browser.iter_subcategories(collection.split_path):
yield cat
def iter_documents(self, subscription):
"""
Iter documents.
:param subscription: subscription to get documents
:type subscription: :class:`Subscription`
:rtype: iter[:class:`Document`]
"""
if not isinstance(subscription, Subscription):
subscription = find_object(self.iter_subscription(),
id=subscription,
error=SubscriptionNotFound)
return self.browser.get_documents(subscription)
def init_transfer(self, transfer, **params):
# There is a check on the website, transfer can't be done with too long reason.
if transfer.label:
transfer.label = transfer.label[:30]
self.logger.info('Going to do a new transfer')
if transfer.account_iban:
account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound)
else:
account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound)
if transfer.recipient_iban:
recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound)
else:
recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound)
try:
# quantize to show 2 decimals.
amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2)
except (AssertionError, ValueError):
raise TransferError('something went wrong')
return self.browser.init_transfer(account, recipient, amount, transfer.label, transfer.exec_date)
def post_message(self, message):
if not self.config['username'].get():
raise BrowserForbidden()
self.browser.post(
find_object(self.iter_threads(), id=message.full_id.split('.')[0]),
message.content)
def get_bill(self, _id):
subscription = self.get_subscription(_id.split('-')[0])
return find_object(self.browser.get_bills(subscription),
id=_id,
error=BillNotFound)
def get_last_measure(self, sensor_id):
gauge_id = sensor_id.split('-')[0]
return find_object(self.iter_sensors(gauge_id), id=sensor_id, error=SensorNotFound).lastvalue
def get_bill(self, _id):
subid = _id.split('.')[0]
subscription = self.get_subscription(subid)
return find_object(self.iter_bills(subscription), id=_id, error=BillNotFound)
def iter_transfer_recipients(self, origin_account):
if not isinstance(origin_account, Account):
origin_account = find_object(self.iter_accounts(),
id=origin_account,
error=AccountNotFound)
return self.browser.iter_recipients(origin_account)
def new_recipient(self, recipient, **params):
if not re.match(u"^[-+.,:/?() éèêëïîñàâäãöôòõùûüÿ0-9a-z']+$",
recipient.label, re.I):
raise RecipientInvalidLabel(
'Recipient label contains invalid characters')
if 'sms_code' in params and not re.match(r'^[a-z0-9]{6}$',
params['sms_code'], re.I):
# check before send sms code because it can crash website if code is invalid
raise AddRecipientBankError("SMS code %s is invalid" %
params['sms_code'])
# avoid `iter_accounts` if there is only one perimeter
if len(self.perimeters) > 1:
accounts = list(self.iter_accounts())
assert recipient.origin_account_id, 'Origin account id is mandatory for multispace'
account = find_object(accounts,
id=recipient.origin_account_id,
error=AccountNotFound)
self.go_to_perimeter(account._perimeter)
self.transfer_init_page.go()
assert self.transfer_init_page.is_here()
if not self.page.add_recipient_is_allowed():
if not [
rec for rec in self.page.iter_recipients()
if rec.category == 'Externe'
]:
raise AddRecipientBankError(
'Vous ne pouvez pas ajouter de bénéficiaires, veuillez contacter votre banque.'
)
assert False, 'Xpath for a recipient add is not catched'
self.location(self.page.url_list_recipients())
# there are 2 pages from where we can add a new recipient:
# - RecipientListPage, but the link is sometimes missing
# - TransferPage, start making a transfer with a new recipient but don't complete the transfer
# but it seems dangerous since we have to set an amount, etc.
# so we implement it in 2 ways with a preference for RecipientListPage
if self.page.url_add_recipient():
self.logger.debug(
'good, we can add a recipient from the recipient list')
else:
# in this case, the link was missing
self.logger.warning(
'cannot add a recipient from the recipient list page, pretending to make a transfer in order to add it'
)
self.transfer_init_page.go()
assert self.transfer_init_page.is_here()
self.location(self.page.url_add_recipient())
if not ('sms_code' in params and self.page.can_send_code()):
self.page.send_sms()
# go to a GET page, so StatesMixin can reload it
self.accounts.go()
raise AddRecipientStep(
self.build_recipient(recipient),
Value('sms_code', label='Veuillez saisir le code SMS'))
else:
self.page.submit_code(params['sms_code'])
err = hasattr(self.page,
'get_sms_error') and self.page.get_sms_error()
if err:
raise AddRecipientBankError(message=err)
self.page.submit_recipient(recipient.label, recipient.iban)
self.page.confirm_recipient()
self.page.check_recipient_error()
if self.transfer_page.is_here():
# in this case, we were pretending to make a transfer, just to add the recipient
# go back to transfer page to abort the transfer and see the new recipient
self.transfer_init_page.go()
assert self.transfer_init_page.is_here()
res = self.page.find_recipient(recipient.iban)
assert res, 'Recipient with iban %s could not be found' % recipient.iban
return res
def get_document(self, _id):
return find_object(self.iter_documents(None), id=_id,
error=DocumentNotFound)
def _get_sensor_by_id(self, id):
gid = id.partition('.')[0]
return find_object(self.iter_sensors(gid), id=id)
def _get_gauge_by_id(self, id):
return find_object(self.browser.iter_gauges(), id=id)
def get_accounts_list(self):
if not self.accounts_list:
if self.currentSubBank is None:
self.getCurrentSubBank()
self.two_cards_page = None
self.accounts_list = []
self.revolving_accounts = []
self.unavailablecards = []
self.cards_histo_available = []
self.cards_list = []
self.cards_list2 = []
# For some cards the validity information is only availaible on these 2 links
self.cards_hist_available.go(subbank=self.currentSubBank)
if self.cards_hist_available.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
if not self.cards_list:
self.cards_hist_available2.go(subbank=self.currentSubBank)
if self.cards_hist_available2.is_here():
self.unavailablecards.extend(
self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available2
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
for acc in self.revolving_loan_list.stay_or_go(
subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(acc)
self.revolving_accounts.append(acc.label.lower())
# Handle cards on tiers page
self.cards_activity.go(subbank=self.currentSubBank)
companies = self.page.companies_link() if self.cards_activity.is_here() else \
[self.page] if self.is_new_website else []
for company in companies:
# We need to return to the main page to avoid navigation error
self.cards_activity.go(subbank=self.currentSubBank)
page = self.open(company).page if isinstance(
company, basestring) else company
for card in page.iter_cards():
card2 = find_object(self.cards_list, id=card.id[:16])
if card2:
# In order to keep the id of the card from the old space, we exchange the following values
card._link_id = card2._link_id
card._parent_id = card2._parent_id
card.coming = card2.coming
card._referer = card2._referer
card._secondpage = card2._secondpage
self.accounts_list.remove(card2)
self.accounts_list.append(card)
self.cards_list2.append(card)
self.cards_list.extend(self.cards_list2)
# Populate accounts from old website
if not self.is_new_website:
self.accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
# Populate accounts from new website
else:
self.new_accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(
self.accounts_list)
self.por.go(subbank=self.currentSubBank).add_por_accounts(
self.accounts_list)
self.li.go(subbank=self.currentSubBank)
self.accounts_list.extend(self.page.iter_li_accounts())
for acc in self.cards_list:
if hasattr(acc, '_parent_id'):
acc.parent = find_object(self.accounts_list,
id=acc._parent_id)
excluded_label = ['etalis', 'valorisation totale']
self.accounts_list = [
acc for acc in self.accounts_list
if not any(w in acc.label.lower() for w in excluded_label)
]
if has_no_account and not self.accounts_list:
raise NoAccountsException(has_no_account)
self.ownership_guesser()
return self.accounts_list
def get_document(self, id):
return find_object(self.browser.iter_documents(), id=id, error=DocumentNotFound)
def get_document(self, bill):
return find_object(
self.iter_documents(bill.split("#")[0]),
id=bill,
error=DocumentNotFound
)
def iter_history(self, account, coming=False):
handled_history_types = (
Account.TYPE_CHECKING,
Account.TYPE_CARD,
Account.TYPE_SAVINGS,
Account.TYPE_PEA,
)
if account.type not in handled_history_types:
self.unhandled_method(account.id)
return
if account.type == Account.TYPE_CARD:
self.go_to_perimeter(account._perimeter)
self.accounts.go()
self.page.go_to_card(account._card_link)
assert (self.cards_page.is_here() or self.multiple_cards_page.is_here()), \
'Failed to reach card details for card %s.' % account.id
if self.multiple_cards_page.is_here():
# We need to go to the correct card transactions with its number.
card_url = self.page.get_transactions_link(account._raw_number)
self.location(card_url)
# When there are several future coming summaries,
# we must skip the ongoing one but fetch the other ones
# even if they are in the future.
ongoing_coming = self.page.get_ongoing_coming()
if not ongoing_coming:
# This card has no available history or coming.
return
card_transactions = []
latest_date = None
for tr in self.page.get_card_transactions(latest_date,
ongoing_coming):
card_transactions.append(tr)
if not card_transactions:
return
# Pagination: we must fetch the date of the last transaction
# because the summary of next transactions may not
# be available on the next page
latest_date = card_transactions[-1].date
next_page_url = self.page.get_next_page()
while next_page_url:
self.location(next_page_url)
for tr in self.page.get_card_transactions(
latest_date, ongoing_coming):
card_transactions.append(tr)
next_page_url = self.page.get_next_page()
for tr in sorted_transactions(card_transactions):
yield tr
return
# Transactions of accounts without form/url or with 'CATITRES' and 'bgpi' in url cannot be handled.
if not account._form and (not account.url or 'CATITRES' in account.url
or 'bgpi' in account.url):
self.unhandled_method(account.id)
return
# Access acount details:
if account.url:
# Refresh the session_value before going to the account URL
new_session_value = 'sessionSAG=' + self.session_value
updated_url = re.sub(r'sessionSAG=([^&]+)', new_session_value,
account.url)
self.location(updated_url)
elif account._form:
# We cannot use forms if we are not on the account's perimeter:
# we need to go to the correct perimeter and refresh forms.
# The form submission sometimes fails so we try several
# times until we get to the account history page.
for form in range(3):
self.accounts.stay_or_go()
self.go_to_perimeter(account._perimeter)
# Only fetch the perimeter's regular accounts (Checking & Savings)
# No need to go to Wealth, Loans or Netfinca for transactions
refreshed_account = find_object(
self.iter_perimeter_regular_accounts(iban=False),
AccountNotFound,
id=account.id)
refreshed_account._form.submit()
if self.failed_history.is_here():
self.logger.warning(
'Form submission failed to reach the account history, we try again.'
)
continue
break
# 3 types of history pages were identified so far
if not (self.checking_history.is_here()
or self.savings_history.is_here()
or self.other_savings_history.is_here()):
self.unhandled_method(account.id)
date_guesser = LinearDateGuesser(date_max_bump=timedelta(30))
for tr in self.page.iter_history(date_guesser=date_guesser):
yield tr
def get_document(self, _id):
subscription = self.get_subscription(_id.split('-')[0])
return find_object(self.browser.get_documents(subscription),
id=_id,
error=DocumentNotFound)
def get_account(self, _id, space=None):
return find_object(self.get_accounts_list(fill_account=False, space=space), id=_id, error=AccountNotFound)
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01'
}
data = {
'contexte': '',
'dateEntree': None,
'filtreEntree': None,
'donneesEntree': json.dumps(account._formated),
}
items = []
self.cenet_account_history.go(data=json.dumps(data), headers=headers)
# there might be some duplicate transactions regarding the card type ones
# because some requests lead to the same transaction list
# even with different parameters/data in the request
card_tr_list = []
while True:
data_out = self.page.doc['DonneesSortie']
for tr in self.page.get_history():
items.append(tr)
if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY:
if find_object(card_tr_list,
label=tr.label,
amount=tr.amount,
raw=tr.raw,
date=tr.date,
rdate=tr.rdate):
self.logger.warning('Duplicated transaction: %s', tr)
items.pop()
continue
card_tr_list.append(tr)
tr.deleted = True
tr_dict = [
tr_dict2 for tr_dict2 in data_out
if tr_dict2['Libelle'] == tr.label
]
donneesEntree = {}
donneesEntree['Compte'] = account._formated
donneesEntree['ListeOperations'] = [tr_dict[0]]
deferred_data = {
'contexte':
'',
'dateEntree':
None,
'donneesEntree':
json.dumps(donneesEntree).replace('/', '\\/'),
'filtreEntree':
json.dumps(tr_dict[0]).replace('/', '\\/')
}
tr_detail_page = self.cenet_tr_detail.open(
data=json.dumps(deferred_data), headers=headers)
for tr in tr_detail_page.get_history():
items.append(tr)
offset = self.page.next_offset()
if not offset:
break
data['filtreEntree'] = json.dumps({
'Offset': offset,
})
self.cenet_account_history.go(data=json.dumps(data),
headers=headers)
return sorted_transactions(items)
def get_document(self, _id):
subid = _id.rsplit('_', 1)[0]
subscription = self.get_subscription(subid)
return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound)
def get_video_from_id(self, _id, category):
return find_object(self.get_categories_videos(category),
id=u'%s#%s' % (_id, category))
def get_subscription(self, _id):
return find_object(self.iter_subscription(),
id=_id,
error=SubscriptionNotFound)
def iter_gauge_history(self, sensor_id):
gauge_id = sensor_id.split('-')[0]
return find_object(self.iter_sensors(gauge_id), id=sensor_id, error=SensorNotFound).history
def get_accounts_list(self):
if not self.accounts_list:
if self.currentSubBank is None:
self.getCurrentSubBank()
self.two_cards_page = None
self.accounts_list = []
self.revolving_accounts = []
self.unavailablecards = []
self.cards_histo_available = []
self.cards_list =[]
self.cards_list2 =[]
# For some cards the validity information is only availaible on these 2 links
self.cards_hist_available.go(subbank=self.currentSubBank)
if self.cards_hist_available.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
if not self.cards_list:
self.cards_hist_available2.go(subbank=self.currentSubBank)
if self.cards_hist_available2.is_here():
self.unavailablecards.extend(self.page.get_unavailable_cards())
for acc in self.page.iter_accounts():
acc._referer = self.cards_hist_available2
self.accounts_list.append(acc)
self.cards_list.append(acc)
self.cards_histo_available.append(acc.id)
for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts():
self.accounts_list.append(acc)
self.revolving_accounts.append(acc.label.lower())
# Handle cards on tiers page
self.cards_activity.go(subbank=self.currentSubBank)
companies = self.page.companies_link() if self.cards_activity.is_here() else \
[self.page] if self.is_new_website else []
for company in companies:
# We need to return to the main page to avoid navigation error
self.cards_activity.go(subbank=self.currentSubBank)
page = self.open(company).page if isinstance(company, basestring) else company
for card in page.iter_cards():
card2 = find_object(self.cards_list, id=card.id[:16])
if card2:
# In order to keep the id of the card from the old space, we exchange the following values
card._link_id = card2._link_id
card._parent_id = card2._parent_id
card.coming = card2.coming
card._referer = card2._referer
card._secondpage = card2._secondpage
self.accounts_list.remove(card2)
self.accounts_list.append(card)
self.cards_list2.append(card)
self.cards_list.extend(self.cards_list2)
# Populate accounts from old website
if not self.is_new_website:
self.logger.info('On old creditmutuel website')
self.accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list)
self.por.go(subbank=self.currentSubBank)
self.page.add_por_accounts(self.accounts_list)
# Populate accounts from new website
else:
self.new_accounts.stay_or_go(subbank=self.currentSubBank)
has_no_account = self.page.has_no_account()
self.accounts_list.extend(self.page.iter_accounts())
self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list)
self.por.go(subbank=self.currentSubBank)
self.page.add_por_accounts(self.accounts_list)
self.li.go(subbank=self.currentSubBank)
self.accounts_list.extend(self.page.iter_li_accounts())
# This type of account is like a loan, for splitting payments in smaller amounts.
# Its history is irrelevant because money is debited from a checking account and
# the balance is not even correct, so ignore it.
excluded_label = ['etalis', 'valorisation totale']
accounts_by_id = {}
for acc in self.accounts_list:
if acc.label.lower() not in excluded_label:
accounts_by_id[acc.id] = acc
# Set the parent to loans and cards accounts
for acc in self.accounts_list:
if acc.type == Account.TYPE_CARD and not empty(getattr(acc, '_parent_id', None)):
acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable)
elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id:
acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable)
self.accounts_list = list(accounts_by_id.values())
if has_no_account and not self.accounts_list:
raise NoAccountsException(has_no_account)
self.ownership_guesser()
return self.accounts_list
def get_city(self, _id):
return find_object(self.iter_city_search(_id), id=_id, error=CityNotFound)
def get_subscription(self, _id):
return find_object(self.browser.get_subscriptions(),
id=_id,
error=SubscriptionNotFound)
def get_card(self, id):
return find_object(self.get_cards(), id=id)
def get_account(self, _id):
return find_object(self.iter_accounts(), id=_id, error=AccountNotFound)
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA,
Account.TYPE_LIFE_INSURANCE):
self.logger.warning('This account is not supported')
raise NotImplementedError()
# some accounts may exist without a link to any history page
if not hasattr(account, '_form') and (not account.url
or 'CATITRES' in account.url):
return
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
if hasattr(account, '_form'):
# the account needs a form submission to go to the history
# but we need to get the latest form data
self.location(self.accounts_url.format(self.sag))
accounts = self.page.get_list(use_links=False)
new_account = find_object(accounts, AccountNotFound, id=account.id)
self.location(new_account._form.request)
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_card(account.id)
if account.url and (account.type != Account.TYPE_CARD
or not self.page.is_on_right_detail(account)):
self.location(account.url.format(self.sag))
if self.cards.is_here():
date_guesser = ChaoticDateGuesser(date.today() -
timedelta(weeks=42))
url = self.page.url
state = None
notfirst = False
while url:
if notfirst:
self.location(url)
else:
notfirst = True
assert self.cards.is_here()
for state, tr in self.page.get_history(date_guesser, state):
yield tr
url = self.page.get_next_url()
elif self.page:
date_guesser = LinearDateGuesser()
self.page.order_transactions()
while True:
assert self.transactions.is_here()
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
if url is None:
break
self.location(url)
def get_account(self, _id):
return find_object(self.get_accounts_list(), id=_id, error=AccountNotFound)
def get_account(self, id):
account_list = self.get_accounts_list()
return find_object(account_list, id=id)
def get_account_for_history(self, id):
account_list = list(self._iter_accounts())
return find_object(account_list, id=id)
def get_account(self, _id):
return find_object(self.browser.iter_account_owners(),
id=_id,
error=AccountNotFound)
