def get_coming(self, account):
if account.type != account.TYPE_CARD:
return []
trs = []
if not hasattr(account.parent, '_info'):
raise NotImplementedError()
# We are on the old website
if hasattr(account, '_coming_eventargument'):
if not self.cards_old.is_here():
self.home.go()
self.page.go_list()
self.page.go_cards()
self.page.go_card_coming(account._coming_eventargument)
return sorted_transactions(self.page.iter_coming())
# We are on the new website.
info = account.parent._card_links
# if info is empty, that mean there are no coming yet
if info:
for tr in self._get_history(info.copy(), account):
tr.type = tr.TYPE_DEFERRED_CARD
trs.append(tr)
return sorted_transactions(trs)
def iter_history(self, account, coming=False):
# The accounts from the "Assurances Vie" space have no available history:
if hasattr(account, '_details'):
return []
if account.type == Account.TYPE_PEA and account.label.endswith(
'Espèces'):
return []
if account.type == account.TYPE_LIFE_INSURANCE:
return self.iter_lifeinsurance_history(account, coming)
elif account.type in (account.TYPE_MARKET,
Account.TYPE_PEA) and not coming:
try:
self.market_list.go(json={}, method='POST')
except ServerError:
self.logger.warning("An Internal Server Error occurred")
return iter([])
for market_acc in self.page.get_list():
if account.number[-4:] == market_acc['securityAccountNumber'][
-4:]:
self.page = self.market_history.go(
json={
"securityAccountNumber":
market_acc['securityAccountNumber'],
})
return self.page.iter_history()
return iter([])
else:
if not self.card_to_transaction_type:
self.list_detail_card.go()
self.card_to_transaction_type = self.page.get_card_to_transaction_type(
)
data = {
"ibanCrypte":
account.id,
"pastOrPending":
1,
"triAV":
0,
"startDate":
(datetime.now() - relativedelta(years=1)).strftime('%d%m%Y'),
"endDate":
datetime.now().strftime('%d%m%Y')
}
try:
self.history.go(json=data)
except BrowserUnavailable:
# old url is still used for certain connections bu we don't know which one is,
# so the same HistoryPage is attained by the old url in another URL object
data[1]['startDate'] = (
datetime.now() - relativedelta(years=3)).strftime('%d%m%Y')
# old url authorizes up to 3 years of history
self.history_old.go(data=data)
if coming:
return sorted_transactions(self.page.iter_coming())
else:
return sorted_transactions(self.page.iter_history())
def get_ti_transactions(self, account):
self.ti_card_go()
self.page.expand(account=account)
for tr in sorted_transactions(self.page.get_history()):
yield tr
self.ti_histo_go()
self.page.expand(self.page.get_periods()[0], account=account)
for period in self.page.get_periods():
self.page.expand(period, account=account)
for tr in sorted_transactions(self.page.get_history()):
yield tr
def get_ti_transactions(self, account):
self.ti_card_go()
self.page.expand(account=account)
for tr in sorted_transactions(self.page.get_history()):
yield tr
self.ti_histo_go()
self.page.expand(self.page.get_periods()[0], account=account)
for period in self.page.get_periods():
self.page.expand(period, account=account)
for tr in sorted_transactions(self.page.get_history()):
yield tr
def iter_history(self, account, coming=False):
# The accounts from the "Assurances Vie" space have no available history:
if hasattr(account, '_details'):
return []
if account.type == Account.TYPE_PEA and account.label.endswith(
'Espèces'):
return []
if account.type == account.TYPE_LIFE_INSURANCE:
return self.iter_lifeinsurance_history(account, coming)
elif account.type in (account.TYPE_MARKET,
Account.TYPE_PEA) and not coming:
try:
self.market_list.go(data=JSON({}))
except ServerError:
self.logger.warning("An Internal Server Error occurred")
return iter([])
for market_acc in self.page.get_list():
if account.number[-4:] == market_acc['securityAccountNumber'][
-4:]:
self.page = self.market_history.go(data=JSON({
"securityAccountNumber":
market_acc['securityAccountNumber'],
}))
return self.page.iter_history()
return iter([])
else:
if not self.card_to_transaction_type:
self.list_detail_card.go()
self.card_to_transaction_type = self.page.get_card_to_transaction_type(
)
self.history.go(data=JSON({
"ibanCrypte":
account.id,
"pastOrPending":
1,
"triAV":
0,
"startDate": (datetime.now() -
relativedelta(years=2)).strftime('%d%m%Y'),
"endDate":
datetime.now().strftime('%d%m%Y')
}))
if coming:
return sorted_transactions(self.page.iter_coming())
else:
return sorted_transactions(self.page.iter_history())
def _iter_history_base(self, account):
dformat = "%Y%m%d"
# We ask for more 12 months by default, but it may not be supported for somme account types.
# To avoid duplicated transactions we exit as soon a transaction is not within the expected timeframe
for date in rrule(MONTHLY,
dtstart=(datetime.now() - relativedelta(months=11)),
until=datetime.now())[::-1]:
params = dict(identifiant=account.iban,
type_solde='C',
type_releve='Previsionnel',
type_date='O',
date_min=(date + relativedelta(days=1) -
relativedelta(months=1)).strftime(dformat),
date_max=date.strftime(dformat))
self.account_history_view.go(**params)
self.account_history.go(**params)
for transaction in sorted_transactions(self.page.iter_history()):
if transaction._coming:
self.logger.debug('skipping coming %r',
transaction.to_dict())
continue
if transaction.date > date:
self.logger.debug(
'transaction not within expected timeframe, stop iterating history: %r',
transaction.to_dict())
return
yield transaction
def get_history(self, account):
self.location(account._history_link)
if not account.type == Account.TYPE_LOAN:
if self.page.select_period():
return sorted_transactions(self.page.get_operations())
return []
def get_card_transactions(self, account, coming):
self.location(account.url)
if self.home.is_here():
# for some cards, the site redirects us to '/'...
return
for t in self.page.iter_history(is_card=True):
yield t
# For card old history, the page doesn't show the real debit date,
# so we need to parse a csv page
if not coming:
if self.deferred_card_calendar is None:
self.location(self.page.get_calendar_link())
params = {}
params['movementSearch[toDate]'] = self.get_closing_date().strftime('%d/%m/%Y')
params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y')
params['fullSearch'] = 1
self.location(account.url, params=params)
csv_link = self.page.get_csv_link()
if csv_link:
self.location(csv_link)
for t in sorted_transactions(self.page.iter_history(account_number=account.number)):
yield t
return
def get_history(self, account, coming=False):
if account.type is Account.TYPE_LOAN or not account._consultable:
raise NotImplementedError()
if account._univers != self.current_univers:
self.move_to_univers(account._univers)
today = date.today()
seen = set()
offset = 0
next_page = True
while next_page:
operation_list = self._make_api_call(
account=account,
start_date=date(day=1, month=1, year=2000), end_date=date.today(),
offset=offset, max_length=50,
)
transactions = []
for op in reversed(operation_list):
t = Transaction()
t.id = op['id']
if op['id'] in seen:
raise ParseError('There are several transactions with the same ID, probably an infinite loop')
seen.add(t.id)
d = date.fromtimestamp(op.get('dateDebit', op.get('dateOperation'))/1000)
op['details'] = [re.sub('\s+', ' ', i).replace('\x00', '') for i in op['details'] if i] # sometimes they put "null" elements...
label = re.sub('\s+', ' ', op['libelle']).replace('\x00', '')
raw = ' '.join([label] + op['details'])
vdate = date.fromtimestamp(op.get('dateValeur', op.get('dateDebit', op.get('dateOperation')))/1000)
t.parse(d, raw, vdate=vdate)
t.amount = Decimal(str(op['montant']))
t.rdate = date.fromtimestamp(op.get('dateOperation', op.get('dateDebit'))/1000)
if 'categorie' in op:
t.category = op['categorie']
t.label = label
t._coming = op['intraday']
if t._coming:
# coming transactions have a random uuid id (inconsistent between requests)
t.id = ''
t._coming |= (t.date > today)
if t.type == Transaction.TYPE_CARD and account.type == Account.TYPE_CARD:
t.type = Transaction.TYPE_DEFERRED_CARD
transactions.append(t)
# Transactions are unsorted
for t in sorted_transactions(transactions):
if coming == t._coming:
yield t
elif coming and not t._coming:
# coming transactions are at the top of history
self.logger.debug('stopping coming after %s', t)
return
next_page = bool(transactions)
offset += 50
assert offset < 30000, 'the site may be doing an infinite loop'
def get_coming(self):
transactions = list(
self.groupamaes_page.go(
page='&_pid=OperationsTraitees&_fid=GoWaitingOperations').
get_history(date_guesser=LinearDateGuesser(), coming=True))
transactions = sorted_transactions(transactions)
return transactions
def get_history(self, account, coming=False):
if account.type is Account.TYPE_LOAN or not account._consultable:
raise NotImplementedError()
if account._univers != self.current_univers:
self.move_to_univers(account._univers)
today = date.today()
seen = set()
offset = 0
next_page = True
while next_page:
operation_list = self._make_api_call(
account=account,
start_date=date(day=1, month=1, year=2000), end_date=date.today(),
offset=offset, max_length=50,
)
transactions = []
for op in reversed(operation_list):
t = Transaction()
t.id = op['id']
if op['id'] in seen:
raise ParseError('There are several transactions with the same ID, probably an infinite loop')
seen.add(t.id)
d = date.fromtimestamp(op.get('dateDebit', op.get('dateOperation'))/1000)
op['details'] = [re.sub('\s+', ' ', i).replace('\x00', '') for i in op['details'] if i] # sometimes they put "null" elements...
label = re.sub('\s+', ' ', op['libelle']).replace('\x00', '')
raw = ' '.join([label] + op['details'])
vdate = date.fromtimestamp(op.get('dateValeur', op.get('dateDebit', op.get('dateOperation')))/1000)
t.parse(d, raw, vdate=vdate)
t.amount = Decimal(str(op['montant']))
t.rdate = date.fromtimestamp(op.get('dateOperation', op.get('dateDebit'))/1000)
if 'categorie' in op:
t.category = op['categorie']
t.label = label
t._coming = op['intraday']
if t._coming:
# coming transactions have a random uuid id (inconsistent between requests)
t.id = ''
t._coming |= (t.date > today)
if t.type == Transaction.TYPE_CARD and account.type == Account.TYPE_CARD:
t.type = Transaction.TYPE_DEFERRED_CARD
transactions.append(t)
# Transactions are unsorted
for t in sorted_transactions(transactions):
if coming == t._coming:
yield t
elif coming and not t._coming:
# coming transactions are at the top of history
self.logger.debug('stopping coming after %s', t)
return
next_page = bool(transactions)
offset += 50
assert offset < 30000, 'the site may be doing an infinite loop'
def get_history(self):
transactions = list(
self.groupamaes_page.go(
page='&_pid=MenuOperations&_fid=GoOperationsTraitees').
get_history(date_guesser=LinearDateGuesser()))
transactions = sorted_transactions(transactions)
return transactions
def get_history(self, account, coming=False):
if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE) or not account._consultable:
raise NotImplementedError()
if account._univers != self.current_univers:
self.move_to_univers(account._univers)
today = date.today()
seen = set()
offset = 0
next_page = True
while next_page:
operation_list = self._make_api_call(
account=account,
start_date=date(day=1, month=1, year=2000), end_date=date.today(),
offset=offset, max_length=50,
)
transactions = self.page.iter_history(account=account, operation_list=operation_list, seen=seen, today=today, coming=coming)
# Transactions are unsorted
for t in sorted_transactions(transactions):
if coming == t._coming:
yield t
elif coming and not t._coming:
# coming transactions are at the top of history
self.logger.debug('stopping coming after %s', t)
return
next_page = len(transactions) == 50
offset += 50
# This assert supposedly prevents infinite loops,
# but some customers actually have a lot of transactions.
assert offset < 100000, 'the site may be doing an infinite loop'
def _get_history_invests(self, account):
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type == Account.TYPE_LIFE_INSURANCE:
if "MILLEVIE" in account.label:
self.page.go_life_insurance(account)
return sorted_transactions(self.natixis_life_ins_his.go(id=account.id).get_history())
if "NUANCES 3D" in account.label:
self.page.go_life_insurance(account)
try:
if not self.market.is_here() and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return iter([])
return self.page.iter_history()
def get_card_transactions(self, account, coming):
self.location(account.url)
if self.home.is_here():
# for some cards, the site redirects us to '/'...
return
for t in self.page.iter_history(is_card=True):
yield t
# For card old history, the page doesn't show the real debit date,
# so we need to parse a csv page
if not coming:
if self.deferred_card_calendar is None:
self.location(self.page.get_calendar_link())
params = {}
params['movementSearch[toDate]'] = self.get_closing_date(
).strftime('%d/%m/%Y')
params['movementSearch[fromDate]'] = (
date.today() - relativedelta(years=3)).strftime('%d/%m/%Y')
params['fullSearch'] = 1
self.location(account.url, params=params)
csv_link = self.page.get_csv_link()
if csv_link:
self.location(csv_link)
for t in sorted_transactions(
self.page.iter_history(account_number=account.number)):
yield t
return
def iter_history(self, account):
if account._history_url.startswith('javascript:') or account._history_url == '#':
raise NotImplementedError()
account = find_object(self.iter_accounts(), id=account.id)
# this url (reached with a GET) return some transactions, but not in same format than POST method
# and some transactions are duplicated and other are missing, don't take them from GET
# because we don't want to manage both way in iter_history
self.location(account._history_url)
date_range_list = self.page.get_date_range_list()
# a date_range is a couple of date like '01/03/201831/03/2018' but current month is often missing and we have to rebuild it
# from first one to get very recent transaction without scrap them from 1st page (reached with GET url)
if len(date_range_list):
date_range_list = [self._build_next_date_range(date_range_list[0])] + date_range_list
for date_range in date_range_list:
date_guesser = LinearDateGuesser(datetime.datetime.strptime(date_range[10:], "%d/%m/%Y"))
try:
self.location(account._history_url, data={'date': date_range})
except ServerError as error:
if error.response.status_code == 500:
if 'RELEVE NON DISPONIBLE A CETTE PERIODE' in error.response.text:
continue
# just skip because it's still possible to have transactions next months
# Yes, they really did that heresy...
else:
raise
for tr in sorted_transactions(self.page.iter_history(date_guesser=date_guesser)):
yield tr
def iter_history(self, account):
if hasattr(self.browser, 'get_cb_operations'):
transactions = list(self.browser.iter_history(account))
else:
transactions = [tr for tr in self.browser.iter_history(account) if not tr._coming]
transactions = sorted_transactions(transactions)
return transactions
def iter_history(self, account):
'''
Transactions are available 10 by 10 in a JSON.
To access it, we need the account 'pid' and to increment
'skip' for each transaction page until the JSON is empty.
However, transactions are not always in the chronological order.
'''
self.go_wealth_pages(account)
pid = self.page.get_pid()
skip = 0
if not pid:
self.logger.warning(
'No pid available for account %s, transactions cannot be retrieved.',
account.id)
return
transactions = []
self.go_to_transactions(pid, skip)
# Pagination:
while self.page.has_operations():
for tr in self.page.iter_history():
transactions.append(tr)
skip += 10
self.go_to_transactions(pid, skip)
for tr in sorted_transactions(transactions):
# Get investments for each transaction
params = {'oid': tr._oid, 'pid': pid}
self.history_investments.go(params=params)
if self.page.has_investments():
tr.investments = list(self.page.iter_transaction_investments())
else:
tr.investments = []
yield tr
def _get_history(self, info):
if isinstance(info['link'], list):
info['link'] = info['link'][0]
if not info['link'].startswith('HISTORIQUE'):
return
if 'measure_id' in info:
self.page.go_measure_list()
self.page.go_measure_accounts_list(info['measure_id'])
elif self.home.is_here():
self.page.go_list()
else:
self.home_tache.go(tache='CPTSYNT0')
self.page.go_history(info)
info['link'] = [info['link']]
for i in range(self.HISTORY_MAX_PAGE):
assert self.home.is_here()
# list of transactions on account page
transactions_list = []
list_form = []
for tr in self.page.get_history():
transactions_list.append(tr)
if tr.type == tr.TYPE_CARD_SUMMARY:
list_form.append(self.page.get_form_to_detail(tr))
# add detail card to list of transactions
for form in list_form:
form.submit()
if self.home.is_here() and self.page.is_access_error():
self.logger.warning('Access to card details is unavailable for this user')
continue
assert self.transaction_detail.is_here()
for tr in self.page.get_detail():
tr.type = Transaction.TYPE_DEFERRED_CARD
transactions_list.append(tr)
if self.new_website:
self.page.go_newsite_back_to_summary()
else:
self.page.go_form_to_summary()
# going back to summary goes back to first page
for j in range(i):
assert self.page.go_next()
# order by date the transactions without the summaries
transactions_list = sorted_transactions(transactions_list)
for tr in transactions_list:
yield tr
assert self.home.is_here()
if not self.page.go_next():
return
assert False, 'More than {} history pages'.format(self.HISTORY_MAX_PAGE)
def get_history(self, account):
self.location(account._history_link)
if not account.type == Account.TYPE_LOAN:
if self.page.select_period():
return sorted_transactions(self.page.get_operations())
return []
def get_card_transactions(self, account, coming):
# All card transactions can be found in the CSV (history and coming),
# however the CSV shows a maximum of 1000 transactions from all accounts.
self.location(account.url)
if self.home.is_here():
# for some cards, the site redirects us to '/'...
return
if self.deferred_card_calendar is None:
self.location(self.page.get_calendar_link())
params = {}
params['movementSearch[fromDate]'] = (
date.today() - relativedelta(years=3)).strftime('%d/%m/%Y')
params['fullSearch'] = 1
self.location(account.url, params=params)
csv_link = self.page.get_csv_link()
if csv_link:
self.location(csv_link)
# Yield past transactions as 'history' and
# transactions in the future as 'coming':
for tr in sorted_transactions(
self.page.iter_history(account_number=account.number)):
if coming and tr.date > date.today():
tr._is_coming = True
yield tr
elif not coming and tr.date <= date.today():
yield tr
def _get_history(self, info):
if isinstance(info['link'], list):
info['link'] = info['link'][0]
if not info['link'].startswith('HISTORIQUE'):
return
if 'measure_id' in info:
self.page.go_measure_list()
self.page.go_measure_accounts_list(info['measure_id'])
elif self.home.is_here():
self.page.go_list()
else:
self.home_tache.go(tache='CPTSYNT0')
self.page.go_history(info)
info['link'] = [info['link']]
for i in range(100):
assert self.home.is_here()
# list of transactions on account page
transactions_list = []
list_form = []
for tr in self.page.get_history():
transactions_list.append(tr)
if re.match('^CB [\d]{4}\*{6}[\d]{6} TOT DIF ([\w]{3,9})',
tr.label,
flags=re.IGNORECASE):
list_form.append(self.page.get_form_to_detail(tr))
# add detail card to list of transactions
for form in list_form:
form.submit()
assert self.transaction_detail.is_here()
for tr in self.page.get_detail():
tr.type = Transaction.TYPE_DEFERRED_CARD
transactions_list.append(tr)
if self.new_website:
self.page.go_newsite_back_to_summary()
else:
self.page.go_form_to_summary()
# going back to summary goes back to first page
for j in range(i):
assert self.page.go_next()
# order by date the transactions without the summaries
transactions_list = sorted_transactions(transactions_list)
for tr in transactions_list:
yield tr
assert self.home.is_here()
if not self.page.go_next():
return
assert False, 'More than 100 history pages'
def get_ti_corporate_transactions(self, account):
if account.id not in self.transactions_dict:
self.transactions_dict[account.id] = []
self.ti_histo_go()
self.page.expand(self.page.get_periods()[0], account=account)
for tr in sorted_transactions(self.page.get_history()):
self.transactions_dict[account.id].append(tr)
return self.transactions_dict[account.id]
def get_ti_corporate_transactions(self, account):
if account.id not in self.transactions_dict:
self.transactions_dict[account.id] = []
self.ti_histo_go()
self.page.expand(self.page.get_periods()[0], account=account)
for tr in sorted_transactions(self.page.get_history()):
self.transactions_dict[account.id].append(tr)
return self.transactions_dict[account.id]
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01'
}
data = {
'contexte': '',
'dateEntree': None,
'filtreEntree': None,
'donneesEntree': json.dumps(account._formated),
}
items = []
self.cenet_account_history.go(data=json.dumps(data), headers=headers)
# there might be some duplicate transactions regarding the card type ones
# because some requests lead to the same transaction list
# even with different parameters/data in the request
card_tr_list = []
while True:
data_out = self.page.doc['DonneesSortie']
for tr in self.page.get_history():
items.append(tr)
if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY:
if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate):
self.logger.warning('Duplicated transaction: %s', tr)
items.pop()
continue
card_tr_list.append(tr)
tr.deleted = True
tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label]
donneesEntree = {}
donneesEntree['Compte'] = account._formated
donneesEntree['ListeOperations'] = [tr_dict[0]]
deferred_data = {
'contexte': '',
'dateEntree': None,
'donneesEntree': json.dumps(donneesEntree).replace('/', '\\/'),
'filtreEntree': json.dumps(tr_dict[0]).replace('/', '\\/')
}
tr_detail_page = self.cenet_tr_detail.open(data=json.dumps(deferred_data), headers=headers)
for tr in tr_detail_page.get_history():
items.append(tr)
offset = self.page.next_offset()
if not offset:
break
data['filtreEntree'] = json.dumps({
'Offset': offset,
})
self.cenet_account_history.go(data=json.dumps(data), headers=headers)
return sorted_transactions(items)
def _get_history_invests(self, account):
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP):
if self.page.is_account_inactive(account.id):
self.logger.warning('Account %s %s is inactive.' %
(account.label, account.id))
return []
# There is (currently ?) no history for MILLEVIE PREMIUM accounts
if "MILLEVIE" in account.label:
try:
self.page.go_life_insurance(account)
except ServerError as ex:
if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label:
self.logger.info(
"Can not reach history page for MILLEVIE PREMIUM account"
)
return []
raise
label = account.label.split()[-1]
try:
self.natixis_life_ins_his.go(id1=label[:3],
id2=label[3:5],
id3=account.id)
except BrowserHTTPError as e:
if e.response.status_code == 500:
error = json.loads(e.response.text)
raise BrowserUnavailable(error["error"])
raise
return sorted_transactions(self.page.get_history())
if account.label.startswith(
'NUANCES ') or account.label in self.insurance_accounts:
self.page.go_life_insurance(account)
if 'JSESSIONID' in self.session.cookies:
# To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session
del self.session.cookies['JSESSIONID']
try:
if not self.life_insurance.is_here(
) and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' %
self.page.get_cons_histo())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return []
return self.page.iter_history()
def iter_history(self, account):
if account.type == Account.TYPE_LOAN:
return
elif account.type == Account.TYPE_PEA:
self.go_account_pages(account, "history")
# go on netfinca page to get pea history
acc = self.get_netfinca_account(account)
self.location(acc._market_link)
self.bourse_history.go()
if 'Liquidités' not in account.label:
self.page.go_history_filter(cash_filter="market")
else:
self.page.go_history_filter(cash_filter="liquidity")
for tr in self.page.iter_history():
yield tr
return
if account.type == Account.TYPE_LIFE_INSURANCE and account._acctype == "bank":
if not self.lifeinsurance_iframe.is_here():
self.location(account._url)
self.page.go_to_history()
# Pass account investments to try to get isin code for transaction investments
for tr in self.page.iter_history(investments=self.cache['invs'][account.id] if account.id in self.cache['invs'] else []):
yield tr
# Side investment's website
if account._acctype == "investment":
self.go_wealth_pages(account)
pagination_url = self.page.get_pagination_url()
try:
self.location(pagination_url, params={'skip': 0})
except ClientError as e:
assert e.response.status_code == 406
self.logger.info('not doing pagination for account %r, site seems broken', account)
for tr in self.page.iter_history(no_pagination=True):
yield tr
return
self.skip = 0
for tr in self.page.iter_history(pagination_url=pagination_url):
yield tr
# Main website withouth investments
elif account._acctype == "bank" and not account._hasinv and account.type != Account.TYPE_CARD:
self.go_account_pages(account, "history")
if self.page.more_history():
for tr in self.page.get_history():
yield tr
# Get deferred card history
elif account._acctype == "bank" and account.type == Account.TYPE_CARD:
for tr in sorted_transactions(self.deferred_card_transactions(account)):
if tr.date <= date.today():
yield tr
def iter_history(self, account):
account = self.get_account(account.id)
if account.type in (Account.TYPE_LOAN, Account.TYPE_PEE):
return iter([])
if account.type == Account.TYPE_LIFE_INSURANCE:
url = json.loads(
self.lifeinsurance.go(accid=account._index).content)['url']
url = self.location(url).page.get_link("opérations")
return self.location(url).page.iter_history()
elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
self._go_market_history()
if not self.page.go_account(account.label, account._owner):
return []
if not self.page.go_account_full():
return []
# Display code ISIN
self.location(self.url,
params={
'reload': 'oui',
'convertirCode': 'oui'
})
# don't rely on server-side to do the sorting, not only do you need several requests to do so
# but the site just toggles the sorting, resulting in reverse order if you browse multiple accounts
return sorted_transactions(self.page.iter_history())
# Getting a year of history
nbs = [
"UN", "DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT",
"NEUF", "DIX", "ONZE", "DOUZE"
]
trs = []
self.history.go(data=json.dumps({"index": account._index}),
page="pendingListOperations",
headers=self.json_headers)
has_deferred_cards = self.page.has_deferred_cards()
self.history.go(data=json.dumps({'index': account._index}),
page="detailcompte",
headers=self.json_headers)
self.trs = {'lastdate': None, 'list': []}
for tr in self.page.iter_history(index=account._index, nbs=nbs):
if has_deferred_cards and tr.type == Transaction.TYPE_CARD:
tr.type = Transaction.TYPE_DEFERRED_CARD
trs.append(tr)
return trs
def iter_history(self, account):
if account.type == Account.TYPE_LOAN:
return
elif account.type == Account.TYPE_PEA:
self.go_account_pages(account, "history")
# go on netfinca page to get pea history
acc = self.get_netfinca_account(account)
self.location(acc._market_link)
self.bourse_history.go()
if 'Liquidités' not in account.label:
self.page.go_history_filter(cash_filter="market")
else:
self.page.go_history_filter(cash_filter="liquidity")
for tr in self.page.iter_history():
yield tr
return
if account.type == Account.TYPE_LIFE_INSURANCE and account._acctype == "bank":
if not self.lifeinsurance_iframe.is_here():
self.location(account._url)
self.page.go_to_history()
# Pass account investments to try to get isin code for transaction investments
for tr in self.page.iter_history(investments=self.cache['invs'][account.id] if account.id in self.cache['invs'] else []):
yield tr
# Side investment's website
if account._acctype == "investment":
self.go_wealth_pages(account)
pagination_url = self.page.get_pagination_url()
try:
self.location(pagination_url, params={'skip': 0})
except ClientError as e:
assert e.response.status_code == 406
self.logger.info('not doing pagination for account %r, site seems broken', account)
for tr in self.page.iter_history(no_pagination=True):
yield tr
return
self.skip = 0
for tr in self.page.iter_history(pagination_url=pagination_url):
yield tr
# Main website withouth investments
elif account._acctype == "bank" and not account._hasinv and account.type != Account.TYPE_CARD:
self.go_account_pages(account, "history")
if self.page.more_history():
for tr in self.page.get_history():
yield tr
# Get deferred card history
elif account._acctype == "bank" and account.type == Account.TYPE_CARD:
for tr in sorted_transactions(self.deferred_card_transactions(account)):
if tr.date <= date.today():
yield tr
def get_coming(self, account):
for cb_link in account._card_links:
for _ in range(3):
self.location(cb_link)
if not self.page.is_loading():
break
time.sleep(1)
for tr in sorted_transactions(self.page.get_operations()):
yield tr
def get_coming(self, account):
for cb_link in account._card_links:
for _ in range(3):
self.location(cb_link)
if not self.page.is_loading():
break
time.sleep(1)
for tr in sorted_transactions(self.page.get_operations()):
yield tr
def get_coming(self, account):
trs = []
if not hasattr(account, '_info'):
raise NotImplementedError()
for info in account._card_links:
for tr in self._get_history(info.copy()):
tr.type = tr.TYPE_DEFERRED_CARD
tr.nopurge = True
trs.append(tr)
return sorted_transactions(trs)
def get_coming(self, account):
trs = []
if not hasattr(account, '_info'):
raise NotImplementedError()
for info in account._card_links:
for tr in self._get_history(info.copy()):
tr.type = tr.TYPE_DEFERRED_CARD
tr.nopurge = True
trs.append(tr)
return sorted_transactions(trs)
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
bourse_account = self.get_bourse_account(account)
if not bourse_account:
return iter([])
self.location(bourse_account._link_id)
assert self.bourse.is_here()
return self.page.iter_history()
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not self.goto_spirica(account):
return iter([])
return self.spirica.iter_history(account)
if account.type != Account.TYPE_CARD:
self.location(account.url.replace('tableauDeBord', 'operations'))
assert self.history.is_here() or self.loan_history.is_here()
transactions_list = []
if account.type == Account.TYPE_CHECKING:
# transaction of the day
for tr in self.page.get_today_operations():
transactions_list.append(tr)
# history
for tr in self.page.get_operations():
transactions_list.append(tr)
return sorted_transactions(transactions_list)
else:
# for summary transactions, the transactions must be on both accounts:
# negative amount on checking account, positive on card account
transactions = list(self._get_card_transactions(account))
summary = self.page.create_summary()
transactions = sorted_transactions(transactions)
if summary.amount:
transactions.insert(0, summary)
return transactions
def get_coming(self, account):
if account.type == Account.TYPE_CHECKING:
self.coming.go(account=account.id)
return self.page.get_operations()
elif account.type == Account.TYPE_CARD:
self.location(account.url.replace('operations', 'encoursCarte') + '/%s' % account._index)
transactions = list(self.page.get_operations())
if self.page.get_debit_date().month == (datetime.date.today() + relativedelta(months=1)).month:
self.location(account.url.replace('operations', 'encoursCarte') + '/%s?month=1' % account._index)
transactions += list(self.page.get_operations())
return sorted_transactions(transactions)
else:
raise NotImplementedError()
def get_coming(self, account):
if account.type == Account.TYPE_CHECKING:
self.coming.go(account=account.id)
return self.page.get_operations()
elif account.type == Account.TYPE_CARD:
self.location(account.url.replace('tableauDeBord', 'encoursCarte') + '/%s' % account._index)
transactions = list(self.page.get_operations())
if self.page.get_debit_date().month == (datetime.date.today() + relativedelta(months=1)).month:
self.location(account.url.replace('tableauDeBord', 'encoursCarte') + '/%s?month=1' % account._index)
transactions += list(self.page.get_operations())
return sorted_transactions(transactions)
else:
raise NotImplementedError()
def get_coming(self, account):
if 'gestion-sous-mandat' in account.url:
return []
transactions = list(self._get_coming_transactions(account))
for tr in self.iter_card_transactions(account):
if tr._coming:
transactions.append(tr)
transactions = sorted_transactions(transactions)
return transactions
def get_history(self, account, coming=False):
if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE
) or not account._consultable:
raise NotImplementedError()
if account._univers != self.current_univers:
self.move_to_univers(account._univers)
today = date.today()
seen = set()
offset = 0
next_page = True
end_date = date.today()
last_date = None
while next_page:
if offset == 10000:
offset = 0
end_date = last_date
operation_list = self._make_api_call(
account=account,
start_date=date(day=1, month=1, year=2000),
end_date=end_date,
offset=offset,
max_length=50,
)
transactions = self.page.iter_history(
account=account,
operation_list=operation_list,
seen=seen,
today=today,
coming=coming)
transactions = sorted_transactions(transactions)
if transactions:
last_date = transactions[-1].date
# Transactions are unsorted
for t in transactions:
if coming == t._coming:
yield t
elif coming and not t._coming:
# coming transactions are at the top of history
self.logger.debug('stopping coming after %s', t)
return
next_page = len(transactions) > 0
offset += 50
# This assert supposedly prevents infinite loops,
# but some customers actually have a lot of transactions.
assert offset < 100000, 'the site may be doing an infinite loop'
def iter_history(self, account, coming=False):
# The accounts from the "Assurances Vie" space have no available history:
if hasattr(account, '_details'):
return []
if account.type == Account.TYPE_PEA and account.label.endswith('Espèces'):
return []
if account.type == account.TYPE_LIFE_INSURANCE:
return self.iter_lifeinsurance_history(account, coming)
elif account.type in (account.TYPE_MARKET, Account.TYPE_PEA) and not coming:
try:
self.market_list.go(data=JSON({}))
except ServerError:
self.logger.warning("An Internal Server Error occurred")
return iter([])
for market_acc in self.page.get_list():
if account.number[-4:] == market_acc['securityAccountNumber'][-4:]:
self.page = self.market_history.go(data=JSON({
"securityAccountNumber": market_acc['securityAccountNumber'],
}))
return self.page.iter_history()
return iter([])
else:
if not self.card_to_transaction_type:
self.list_detail_card.go()
self.card_to_transaction_type = self.page.get_card_to_transaction_type()
self.history.go(data=JSON({
"ibanCrypte": account.id,
"pastOrPending": 1,
"triAV": 0,
"startDate": (datetime.now() - relativedelta(years=2)).strftime('%d%m%Y'),
"endDate": datetime.now().strftime('%d%m%Y')
}))
if coming:
return sorted_transactions(self.page.iter_coming())
else:
return sorted_transactions(self.page.iter_history())
def get_coming(self, account):
transactions = []
data = {'accountExternalNumber': account.id}
self.pre_comingpage.go(data=data)
assert self.comingpage.is_here()
# this page is "operations du jour" and may not have any date yet
# so don't take transactions here, but it sets the "context"
self.comingpage.go()
assert self.comingpage.is_here()
transactions += self.page.get_transactions(pattern='var jsonData =')
self.deffered_transactionpage.go(data=data)
transactions += self.page.get_transactions(pattern='var jsonData1 =', is_deffered=True)
return sorted_transactions(transactions)
def iter_transactions(self, account, coming=False):
self.select_account(account.label)
self.history.go()
self.wait_until_is_here(self.history)
if not self.date_list:
self.date_list = self.page.fetch_date_list()
for date_choice in self.date_list:
if (coming and date_choice >= self.today) or (
not coming and date_choice < self.today):
self.page.display_transactions(date_choice)
for tr in sorted_transactions(
self.page.iter_history(date=date_choice)):
yield tr
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
bourse_account = self.get_bourse_account(account)
if not bourse_account:
return iter([])
self.location(bourse_account._link_id)
assert self.bourse.is_here()
history = list(self.page.iter_history())
self.leave_espace_bourse()
return history
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not self.goto_spirica(account):
return iter([])
return self.spirica.iter_history(account)
if account.type != Account.TYPE_CARD:
self.location(account.url)
assert self.history.is_here() or self.loan_history.is_here()
transactions_list = []
if account.type == Account.TYPE_CHECKING:
# transaction of the day
for tr in self.page.get_today_operations():
transactions_list.append(tr)
# history
for tr in self.page.get_operations():
transactions_list.append(tr)
return sorted_transactions(transactions_list)
else:
# for summary transactions, the transactions must be on both accounts:
# negative amount on checking account, positive on card account
transactions = []
self.location(
account.url.replace('operations', 'encoursCarte') +
'/%s?month=1' % account._index)
if self.page.get_debit_date().month == (
datetime.date.today() - relativedelta(months=1)).month:
transactions = list(self.page.get_operations())
summary = self.page.create_summary()
if summary:
transactions.insert(0, summary)
return transactions
def get_ge_transactions(self, account):
transactions = []
self.coming.go()
self.page.expand(rib=account._rib)
link = self.page.get_link(account)
if link:
self.location(link)
transactions += self.page.get_history()
self.history.go()
for period in self.page.get_periods():
self.page.expand(period, rib=account._rib)
link = self.page.get_link(account)
if link:
self.location(link)
transactions += self.page.get_history()
self.history.go()
return sorted_transactions(transactions)
def iter_history(self, account):
''' There is now an API for the accounts history, however transactions are not
sorted by date in the JSON. The website fetches 5 years of history maximum.
For some accounts, the access to the transactions JSON is not available yet. '''
params = {
'startDate': (date.today() - relativedelta(years=2)).year,
'endDate': date.today().year,
'pid': account.id,
}
self.history.go(params=params)
error_code = self.page.get_error_code()
if error_code:
self.logger.warning('Error when trying to access the history JSON, history will be skipped for this account.')
return
for tr in sorted_transactions(self.page.iter_history()):
yield tr
def _iter_history_base(self, account):
history = []
dformat = "%Y%m%d"
for date in rrule(MONTHLY, dtstart=(datetime.now() - relativedelta(months=3)), until=datetime.now()):
self.account_history_view.go(identifiant=account.iban, type_solde='C', type_releve='Comptable', \
type_date='O', date_min=(date + relativedelta(days=1)).strftime(dformat), \
date_max=(date + relativedelta(months=1)).strftime(dformat))
self.account_history.go(identifiant=account.iban, date_min=(date + relativedelta(days=1)).strftime(dformat), \
date_max=(date + relativedelta(months=1)).strftime(dformat))
for transaction in self.page.iter_history():
if transaction._coming:
self.logger.debug('skipping coming %r', transaction.to_dict())
continue
history.append(transaction)
return sorted_transactions(history)
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here(
) else None
if coming_link is not None:
for tr in self.list_operations(coming_link):
transactions.append(tr)
differed_date = None
cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \
account._card_links if hasattr(account, '_card_links') else []
for card in cards:
card_trs = []
for tr in self.list_operations(card):
if hasattr(tr, '_differed_date') and (
not differed_date
or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \
differed_date.month <= tr.date.month and \
not hasattr(tr, '_is_manualsum')
transactions = sorted_transactions(transactions)
return transactions
def get_ge_transactions(self, account):
transactions = []
self.coming.go()
self.page.expand(rib=account._rib)
link = self.page.get_link(account)
if link:
self.location(link)
transactions += self.page.get_history()
self.history.go()
for period in self.page.get_periods():
self.page.expand(period, rib=account._rib)
link = self.page.get_link(account)
if link:
self.location(link)
transactions += self.page.get_history()
self.history.go()
return sorted_transactions(transactions)
def get_history(self, account):
if account.type == Account.TYPE_LOAN:
return []
elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
bourse_account = self.get_bourse_account(account)
if not bourse_account:
return iter([])
self.location(bourse_account._link_id)
assert self.bourse.is_here()
history = list(self.page.iter_history())
self.leave_espace_bourse()
return history
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not self.goto_spirica(account):
return iter([])
return self.spirica.iter_history(account)
if account.type != Account.TYPE_CARD:
self.location(account.url.replace('tableauDeBord', 'operations'))
assert self.history.is_here() or self.loan_history.is_here()
transactions_list = []
if account.type == Account.TYPE_CHECKING:
# transaction of the day
for tr in self.page.get_today_operations():
transactions_list.append(tr)
# history
for tr in self.page.get_operations():
transactions_list.append(tr)
return sorted_transactions(transactions_list)
else:
# for summary transactions, the transactions must be on both accounts:
# negative amount on checking account, positive on card account
transactions = []
self.location(account.url.replace('tableauDeBord', 'encoursCarte') + '/%s?month=1' % account._index)
if self.page.get_debit_date().month == (datetime.date.today() - relativedelta(months=1)).month:
transactions = list(self.page.get_operations())
summary = self.page.create_summary()
if summary:
transactions.insert(0, summary)
return transactions
def iter_history(self, account):
account = self.get_account(account.id)
if account.type is Account.TYPE_LOAN:
return iter([])
if account.type == Account.TYPE_LIFE_INSURANCE:
url = json.loads(self.lifeinsurance.go(accid=account._index).content)['url']
url = self.location(url).page.get_link(u"opérations")
return self.location(url).page.iter_history()
elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
self._go_market_history()
if not self.page.go_account(account.label, account._owner):
return []
if not self.page.go_account_full():
return []
# Display code ISIN
self.location(self.url, params={'reload': 'oui', 'convertirCode': 'oui'})
# don't rely on server-side to do the sorting, not only do you need several requests to do so
# but the site just toggles the sorting, resulting in reverse order if you browse multiple accounts
return sorted_transactions(self.page.iter_history())
# Getting a year of history
nbs = ["UN", "DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT", "NEUF", "DIX", "ONZE", "DOUZE"]
trs = []
self.history.go(data=json.dumps({"index": account._index}), page="pendingListOperations", headers=self.json_headers)
has_deferred_cards = self.page.has_deferred_cards()
self.history.go(data=json.dumps({'index': account._index}), page="detailcompte", headers=self.json_headers)
self.trs = {'lastdate': None, 'list': []}
for tr in self.page.iter_history(index=account._index, nbs=nbs):
if has_deferred_cards and tr.type == Transaction.TYPE_CARD:
tr.type = Transaction.TYPE_DEFERRED_CARD
trs.append(tr)
return trs
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
differed_date = None
cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \
account._card_links if hasattr(account, '_card_links') else []
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \
differed_date.month <= tr.date.month and \
not hasattr(tr, '_is_manualsum')
transactions = sorted_transactions(transactions)
return transactions
def _iter_history_base(self, account):
history = []
dformat = "%Y%m%d"
for date in rrule(MONTHLY,
dtstart=(datetime.now() - relativedelta(months=3)),
until=datetime.now()):
self.account_history_view.go(identifiant=account.iban, type_solde='C', type_releve='Comptable', \
type_date='O', date_min=(date + relativedelta(days=1)).strftime(dformat), \
date_max=(date + relativedelta(months=1)).strftime(dformat))
self.account_history.go(identifiant=account.iban, date_min=(date + relativedelta(days=1)).strftime(dformat), \
date_max=(date + relativedelta(months=1)).strftime(dformat))
for transaction in self.page.iter_history():
if transaction._coming:
self.logger.debug('skipping coming %r',
transaction.to_dict())
continue
history.append(transaction)
return sorted_transactions(history)
def get_coming(self, account):
trs = []
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01'
}
for card in account._cards:
data = {
'contexte': '',
'dateEntree': None,
'donneesEntree': json.dumps(card),
'filtreEntree': None
}
for tr in self.cenet_account_coming.go(data=json.dumps(data), headers=headers).get_history():
trs.append(tr)
return sorted_transactions(trs)
def _get_history_invests(self, account):
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP):
if self.page.is_account_inactive(account.id):
self.logger.warning('Account %s %s is inactive.' % (account.label, account.id))
return []
if "MILLEVIE" in account.label:
self.page.go_life_insurance(account)
label = account.label.split()[-1]
try:
self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id)
except BrowserHTTPError as e:
if e.response.status_code == 500:
error = json.loads(e.response.text)
raise BrowserUnavailable(error["error"])
raise
return sorted_transactions(self.page.get_history())
if account.label.startswith('NUANCES ') or account.label in self.insurance_accounts:
self.page.go_life_insurance(account)
if 'JSESSIONID' in self.session.cookies:
# To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session
del self.session.cookies['JSESSIONID']
try:
if not self.life_insurance.is_here() and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return []
return self.page.iter_history()
def get_coming(self, account):
if account.type == Account.TYPE_LOAN:
return []
trs = []
headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01'
}
for card in account._cards:
if card['CumulEnCours']['Montant']['Valeur'] != 0:
data = {
'contexte': '',
'dateEntree': None,
'donneesEntree': json.dumps(card),
'filtreEntree': None
}
for tr in self.cenet_account_coming.go(data=json.dumps(data), headers=headers).get_history():
trs.append(tr)
return sorted_transactions(trs)
def get_transactions(self, account):
if not self.accounts:
self.iter_accounts()
self.his_home.go()
self.page.expand()
for a in self.accounts:
if a.id == account.id:
link = self.get_link(a.number, a._owner)
if link:
self.location(link)
assert self.transactions.is_here()
# We might not be on first page.
self.page.assert_first_page_or_go_there()
if self.page.is_not_sorted():
self.page.sort()
transactions = list(self.page.get_history())
transactions = sorted_transactions(transactions)
return transactions
return iter([])
def iter_history(self, account):
transactions = sorted_transactions(self.browser.get_history(account))
return transactions
def get_history(self, account, coming=False, retry_li=True):
self._quit_li_space()
self.update_accounts_list(False)
account = self.accounts_list[account.id]
if account.url is None:
return []
if account.url.startswith('javascript') or '&Crd=' in account.url:
raise NotImplementedError()
if account.type == Account.TYPE_LIFE_INSURANCE:
if coming is True:
return []
try:
if not self._go_to_life_insurance(account):
self._quit_li_space()
return []
except (XMLSyntaxError, HTTPNotFound):
self._quit_li_space()
return []
except AccountNotFound:
self.go_post(self.js_url)
# often if we visit life insurance subsite multiple times too quickly, the site just returns an error
# so we just retry (we might relogin...)
# TODO find out how to avoid the error, or avoid relogin
if retry_li:
self.logger.warning('life insurance seems unavailable for account %s', account.id)
return self.get_history(account, coming, False)
self.logger.error('life insurance seems unavailable for account %s', account.id)
return []
self.life_insurances.go(data={'url_suivant': 'HISTORIQUECONTRATB2C', 'strMonnaie': 'EURO'})
history = [t for t in self.page.iter_history()]
self._quit_li_space()
return history
try:
self.go_post(self.accounts_list[account.id].url)
# sometime go to hsbc life insurance space do logout
except HTTPNotFound:
self.app_gone = True
self.do_logout()
self.do_login()
# If we relogin on hsbc, all link have change
if self.app_gone:
self.app_gone = False
self.update_accounts_list()
self.location(self.accounts_list[account.id].url)
if self.page is None:
return []
if self.cbPage.is_here():
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
history = list(self.page.get_history(date_guesser=guesser))
for url, params in self.page.get_params(self.url):
self.location(url, params=params)
if self.cbPage.is_here():
history.extend(self.page.get_history(date_guesser=guesser))
for tr in history:
if tr.type == tr.TYPE_UNKNOWN:
tr.type = tr.TYPE_DEFERRED_CARD
history.extend(self.get_monthly_transactions(history))
history = [tr for tr in history if (coming and tr.date > date.today()) or (not coming and tr.date <= date.today())]
history = sorted_transactions(history)
return history
elif not coming:
return self._get_history()
else:
raise NotImplementedError()
def get_history(self, account, coming=False):
if account.type == Account.TYPE_CARD:
card_transactions = []
self.go_to_account_space(account._contract)
# Deferred cards transactions have a specific JSON.
# Only three months of history available for cards.
value = 0 if coming else 1
params = {
'grandeFamilleCode': int(account._category),
'compteIdx': int(account.parent._index),
'carteIdx': int(account._index),
'rechercheEncoursDebite': value
}
self.card_history.go(params=params)
for tr in self.page.iter_card_history():
card_transactions.append(tr)
# If the card if not unique on the parent id, it is impossible
# to know which summary corresponds to which card.
if not coming and card_transactions and account._unique:
# Get card summaries from parent account
# until we reach the oldest card transaction
last_transaction = card_transactions[-1]
before_last_transaction = False
params = {
'compteIdx': int(account.parent._index),
'grandeFamilleCode': int(account.parent._category),
'idDevise': str(account.parent.currency),
'idElementContrat': str(account.parent._id_element_contrat),
}
self.history.go(params=params)
for tr in self.page.iter_history():
if tr.date < last_transaction.date:
before_last_transaction = True
break
if tr.type == Transaction.TYPE_CARD_SUMMARY:
tr.amount = -tr.amount
card_transactions.append(tr)
while self.page.has_next_page() and not before_last_transaction:
next_index = self.page.get_next_index()
params = {
'grandeFamilleCode': int(account.parent._category),
'compteIdx': int(account.parent._index),
'idDevise': str(account.parent.currency),
'startIndex': next_index,
'count': 100,
}
self.history.go(params=params)
for tr in self.page.iter_history():
if tr.date < last_transaction.date:
before_last_transaction = True
break
if tr.type == Transaction.TYPE_CARD_SUMMARY:
tr.amount = -tr.amount
card_transactions.append(tr)
for tr in sorted_transactions(card_transactions):
yield tr
return
# These three parameters are required to get the transactions for non_card accounts
if empty(account._index) or empty(account._category) or empty(account._id_element_contrat):
return
self.go_to_account_space(account._contract)
params = {
'compteIdx': int(account._index),
'grandeFamilleCode': int(account._category),
'idDevise': str(account.currency),
'idElementContrat': str(account._id_element_contrat),
}
self.history.go(params=params)
for tr in self.page.iter_history():
# For "Livret A", value dates of transactions are always
# 1st or 15th of the month so we specify a valuation date.
# Example: rdate = 21/02, date=01/02 then vdate = 01/02.
if account.type == Account.TYPE_SAVINGS:
tr.vdate = tr.date
yield tr
# Get other transactions 100 by 100:
while self.page.has_next_page():
next_index = self.page.get_next_index()
params = {
'grandeFamilleCode': int(account._category),
'compteIdx': int(account._index),
'idDevise': str(account.currency),
'startIndex': next_index,
'count': 100,
}
self.history.go(params=params)
for tr in self.page.iter_history():
yield tr
def flush(self):
# As transactions are unordered on the page, we flush only at end
# the sorted list of them.
return sorted_transactions(self.objects.values())
def iter_coming(self, account):
if hasattr(self.browser, 'get_cb_operations'):
transactions = list(self.browser.get_cb_operations(account))
return sorted_transactions(transactions)
return self.browser.iter_coming(account)
