def iter_investment(self):
for line in self.document.xpath('//table[contains(@class, "ca-data-table")]/descendant::tr[count(td)>=7]'):
for sub in line.xpath('./td[@class="info-produit"]'):
sub.drop_tree()
cells = line.findall('td')
if cells[self.COL_ID].find('div/a') is None:
continue
inv = Investment()
inv.label = unicode(cells[self.COL_ID].find('div/a').text.strip())
inv.code = cells[self.COL_ID].find('div/br').tail.strip().split(u'\xa0')[0]
inv.quantity = self.parse_decimal(cells[self.COL_QUANTITY].find('span').text)
inv.valuation = self.parse_decimal(cells[self.COL_VALUATION].text)
inv.diff = self.parse_decimal(cells[self.COL_DIFF].text_content())
if "%" in cells[self.COL_UNITPRICE].text and "%" in cells[self.COL_UNITVALUE].text:
inv.unitvalue = inv.valuation / inv.quantity
inv.unitprice = (inv.valuation - inv.diff) / inv.quantity
else:
inv.unitprice = self.parse_decimal(cells[self.COL_UNITPRICE].text)
inv.unitvalue = self.parse_decimal(cells[self.COL_UNITVALUE].text)
date = cells[self.COL_UNITVALUE].find('span').text
if ':' in date:
inv.vdate = ddate.today()
else:
day, month = map(int, date.split('/', 1))
date_guesser = LinearDateGuesser()
inv.vdate = date_guesser.guess_date(day, month)
yield inv
def get_card_transactions(self, latest_date, ongoing_coming):
for item in self.doc.xpath('//table[@class="ca-table"][2]//tr[td]'):
if CleanText('./td[2]/b')(item):
# This node is a summary containing the 'date' for all following transactions.
raw_date = Regexp(CleanText('./td[2]/b/text()'),
r'le (.*) :')(item)
if latest_date and parse_french_date(
raw_date).date() > latest_date:
# This summary has already been fetched
continue
latest_date = parse_french_date(raw_date).date()
if latest_date < ongoing_coming:
# This summary is anterior to the ongoing_coming so we create a transaction from it
tr = FrenchTransaction()
tr.date = tr.rdate = latest_date
tr.raw = tr.label = CleanText('./td[2]/b/text()')(item)
tr.amount = -CleanDecimal.French(
'./td[position()=last()]')(item)
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
yield tr
else:
# This node is a real transaction.
# Its 'date' is the date of the most recently encountered summary node.
tr = FrenchTransaction()
tr.date = latest_date
date_guesser = LinearDateGuesser(latest_date)
tr.rdate = tr.bdate = DateGuesser(
CleanText('./td[1]//text()'),
date_guesser=date_guesser)(item)
tr.label = tr.raw = CleanText('./td[2]')(item)
tr.amount = CleanDecimal.French('./td[last()]')(item)
tr.type = FrenchTransaction.TYPE_DEFERRED_CARD
yield tr
def get_history(self, account):
# some accounts may exist without a link to any history page
if account._link_id is None:
return
history_url = account._link_id
operations_count = 0
# 1st, go on the account page
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
if self.page is None:
return
# Some regions have a "Show more" (well, actually "Voir les 25
# suivants") link we have to use to get all the operations.
# However, it does not show only the 25 next results, it *adds* them
# to the current view. Therefore, we have to parse each new page using
# an offset, in order to ignore all already-fetched operations.
# This especially occurs on CA Centre.
use_expand_url = bool(self.page.expand_history_page_url())
date_guesser = LinearDateGuesser()
while True:
# we skip "operations_count" operations on each page if we are in the case described above
operations_offset = operations_count if use_expand_url else 0
for page_operation in self.page.get_history(
date_guesser, operations_count, operations_offset):
operations_count += 1
yield page_operation
history_url = self.page.expand_history_page_url(
) if use_expand_url else self.page.next_page_url()
if not history_url:
break
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
def get_history(self, account, coming=False):
if account._link_id is None:
return
if account._link_id.startswith('javascript') or '&Crd=' in account._link_id:
raise NotImplementedError()
self.location(self.accounts_list[account.id]._link_id)
#If we relogin on hsbc, all link have change
if self.app_gone:
self.app_gone = False
self.update_accounts_list()
self.location(self.accounts_list[account.id]._link_id)
if self.page is None:
return
if self.cbPage.is_here():
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
return [tr for tr in self.page.get_history(date_guesser=guesser) if (coming and tr.date > date.today()) or (not coming and tr.date <= date.today())]
elif not coming:
return self._get_history()
else:
raise NotImplementedError()
def iter_history(self, account):
self.fetch_areas()
if account._history_url.startswith('javascript:'):
raise NotImplementedError()
# Manage multiple areas
self.subscription.go()
self.location(account._area)
self.accounts.go()
# Query history for 6 last months
def format_date(d):
return datetime.date.strftime(d, '%d/%m/%Y')
today = datetime.date.today()
period = (today - relativedelta(months=6), today)
query = {'date': ''.join(map(format_date, period))}
# Let's go
self.location(account._history_url)
first_page = self.page
rest_page = self.history.go(data=query)
date_guesser = LinearDateGuesser()
return chain(
first_page.iter_history(date_guesser=date_guesser),
reversed(list(rest_page.iter_history(date_guesser=date_guesser))))
def get_coming(self):
transactions = list(
self.groupamaes_page.go(
page='&_pid=OperationsTraitees&_fid=GoWaitingOperations').
get_history(date_guesser=LinearDateGuesser(), coming=True))
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def get_history(self, account):
if not self.is_on_page(AccountsPage):
self.go_on_accounts_list()
url = account._link
coming = True
date_guesser = LinearDateGuesser()
while url is not None:
self.select_form(name='leftnav')
self.form.action = self.absurl(url)
self.submit()
assert self.is_on_page(TransactionsPage)
for tr in self.page.get_history(date_guesser):
if tr.amount > 0:
coming = False
tr._is_coming = coming
yield tr
if self.page.is_last():
url = None
else:
v = urlsplit(url)
args = dict(parse_qsl(v.query))
args['BPIndex'] = int(args['BPIndex']) + 1
url = self.buildurl(v.path, **args)
def get_coming(self):
transactions = list(
self.groupamaes_page.go(
page='&_pid=OperationsTraitees&_fid=GoWaitingOperations').
get_history(date_guesser=LinearDateGuesser(), coming=True))
transactions = sorted_transactions(transactions)
return transactions
def get_history(self):
transactions = list(
self.groupamaes_page.go(
page='&_pid=MenuOperations&_fid=GoOperationsTraitees').
get_history(date_guesser=LinearDateGuesser()))
transactions = sorted_transactions(transactions)
return transactions
def get_history(self, account):
# some accounts may exist without a link to any history page
if account._link is None:
return
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_account(account.id)
date_guesser = LinearDateGuesser()
self.location(account._link)
if self.is_on_page(CardsPage):
for tr in self.page.get_history(date_guesser):
yield tr
else:
url = self.page.get_order_by_date_url()
while url:
self.location(url)
assert self.is_on_page(TransactionsPage)
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
def obj_date(self):
_date = Regexp(CleanText('//p[has-class("line")]', replace=[(u'à', '')]),
'.*Mise en ligne le (.*)')(self)
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def obj_date(self):
_date = Regexp(
CleanText('//div[@class="upload_by"]', replace=[(u'à', '')]),
'.*- Mise en ligne le (.*).')(self)
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def obj_date(self):
_date = CleanText('./div[@class="lbc"]/div[@class="date"]',
replace=[('Aujourd\'hui', str(date.today())),
('Hier',
str((date.today() - timedelta(1))))
])(self)
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE):
self.logger.warning('This account is not supported')
raise NotImplementedError()
# some accounts may exist without a link to any history page
if account._link is None:
return
did_move = False
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
did_move = True
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
accounts = [acc for acc in self.get_cards() if acc.id == account.id]
assert len(accounts) == 1
account = accounts[0]
date_guesser = LinearDateGuesser()
if account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(account):
self.location(account._link.format(self.sag))
if self.is_on_page(CardsPage):
url = self.page.url
state = None
notfirst = False
while url:
if notfirst:
self.location(url)
else:
notfirst = True
assert self.is_on_page(CardsPage)
for state, tr in self.page.get_history(date_guesser, state):
yield tr
url = self.page.get_next_url()
elif self.page:
url = self.page.get_order_by_date_url()
while url:
self.location(url)
assert self.is_on_page(TransactionsPage)
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
# Ugly hack needed for for following card accounts history
if did_move:
self.get_accounts_list()
class item(ItemElement):
klass = Transaction
obj_date = DateGuesser(CleanText('.//span[contains(., "/")]'), LinearDateGuesser(date_max_bump=timedelta(45)))
obj_label = CleanText('.//h3/strong')
obj_amount = MyDecimal('./td[@class="al-r"]/div/span[has-class("badge")]')
def obj_type(self):
amount = Field('amount')(self)
if amount < 0:
return Transaction.TYPE_CARD
else:
return Transaction.TYPE_TRANSFER
def get_roadmap(self):
for step in self.doc.xpath(
'//table[@class="trajet_etapes"]/tr[@class="etape"]'):
roadstep = RoadStep()
roadstep.line = '%s %s' % (DepartureTypeFilter(
step.xpath('./td[@class="moyen"]'))(self),
CleanText('./td[@class="moyen"]')(step))
roadstep.start_time = DateTime(
CleanText('./th/span[@class="depart"]'),
LinearDateGuesser())(step)
roadstep.end_time = DateTime(
CleanText(
'./th/span[@class="depart"]/following-sibling::span'),
LinearDateGuesser())(step)
roadstep.departure = CleanText('./td[@class="arret"]/p/strong')(
step)
roadstep.arrival = CleanText(
'./td[@class="arret"]/p/following-sibling::p/strong')(step)
roadstep.duration = RoadMapDuration(
CleanText('./td[@class="time"]'))(step)
yield roadstep
def obj_date(self):
_date = CleanText('./section[@class="item_infos"]/aside/p[@class="item_supp"]/text()',
replace=[('Aujourd\'hui', str(date.today())),
('Hier', str((date.today() - timedelta(1))))])(self)
if not _date:
return NotAvailable
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def get_history(self, account):
if account._link_id is None:
return
self.location(account._link_id)
if self.page is None:
return
if self.cbPage.is_here():
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
return self.page.get_history(date_guesser=guesser)
else:
return self._get_history()
def get_history(self, account, coming=False):
if account._link_id is None:
return
if account._link_id.startswith('javascript') or '&Crd=' in account._link_id:
raise NotImplementedError()
if account.type == Account.TYPE_LIFE_INSURANCE:
if coming is True:
raise NotImplementedError()
try:
self._go_to_life_insurance(account.id)
except (XMLSyntaxError, HTTPNotFound, AccountNotFound):
self._quit_li_space()
return iter([])
except HTTPNotFound:
self.accounts.go()
return iter([])
self.life_insurances.go(data={'url_suivant': 'HISTORIQUECONTRATB2C', 'strMonnaie': 'EURO'})
history = [t for t in self.page.iter_history()]
self._quit_li_space()
return history
try:
self.location(self.accounts_list[account.id]._link_id)
except HTTPNotFound: # sometime go to hsbc life insurance space do logout
self.app_gone = True
self.do_logout()
self.do_login()
#If we relogin on hsbc, all link have change
if self.app_gone:
self.app_gone = False
self.update_accounts_list()
self.location(self.accounts_list[account.id]._link_id)
if self.page is None:
return
if self.cbPage.is_here():
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
return [tr for tr in self.page.get_history(date_guesser=guesser) if (coming and tr.date > date.today()) or (not coming and tr.date <= date.today())]
elif not coming:
return self._get_history()
else:
raise NotImplementedError()
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA,
Account.TYPE_LIFE_INSURANCE):
self.logger.warning('This account is not supported')
raise NotImplementedError()
# some accounts may exist without a link to any history page
if account.url is None or 'CATITRES' in account.url:
return
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_cards_or_card(account.number)
if account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(
account):
self.location(account.url.format(self.sag))
if self.cards.is_here():
date_guesser = ChaoticDateGuesser(date.today() -
timedelta(weeks=42))
url = self.page.url
state = None
notfirst = False
while url:
if notfirst:
self.location(url)
else:
notfirst = True
assert self.cards.is_here()
for state, tr in self.page.get_history(date_guesser, state):
yield tr
url = self.page.get_next_url()
elif self.page:
date_guesser = LinearDateGuesser()
self.page.order_transactions()
while True:
assert self.transactions.is_here()
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
if url is None:
break
self.location(url)
def _get_history(self, link):
num_page = 0
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
while link is not None:
self.location(link)
if self.page is None:
return
for tr in self.page.get_operations(num_page, guesser):
yield tr
link = self.page.get_next_link()
num_page += 1
def iter_investment(self):
for line in self.document.xpath(
'//table[contains(@class, "ca-data-table")]/descendant::tr[count(td)>=7]'
):
for sub in line.xpath('./td[@class="info-produit"]'):
sub.drop_tree()
cells = line.findall('td')
if cells[self.COL_ID].find('div/a') is None:
continue
inv = Investment()
inv.label = unicode(cells[self.COL_ID].find('div/a').text.strip())
inv.code = cells[self.COL_ID].find('div/br').tail.strip().split(
u'\xa0')[0]
inv.quantity = self.parse_decimal(
cells[self.COL_QUANTITY].find('span').text)
inv.valuation = self.parse_decimal(cells[self.COL_VALUATION].text)
inv.diff = self.parse_decimal(cells[self.COL_DIFF].text_content())
if "%" in cells[self.COL_UNITPRICE].text and "%" in cells[
self.COL_UNITVALUE].text:
inv.unitvalue = inv.valuation / inv.quantity
inv.unitprice = (inv.valuation - inv.diff) / inv.quantity
else:
inv.unitprice = self.parse_decimal(
cells[self.COL_UNITPRICE].text)
inv.unitvalue = self.parse_decimal(
cells[self.COL_UNITVALUE].text)
date = cells[self.COL_UNITVALUE].find('span').text
if ':' in date:
inv.vdate = ddate.today()
else:
day, month = map(int, date.split('/', 1))
date_guesser = LinearDateGuesser()
inv.vdate = date_guesser.guess_date(day, month)
yield inv
def get_history(self, account):
if hasattr(account, '_market_link') and account._market_link:
self.connexion_bourse()
self.location(account._link_id,
params={
'nump': account._market_id,
})
self.page.get_fullhistory()
for tr in self.page.iter_history():
yield tr
self.deconnexion_bourse()
elif hasattr(account, '_link_id') and account._link_id:
try:
self.location(account._link_id)
except ServerError:
return
if self.login.is_here():
# Website crashed and we are disconnected.
raise BrowserUnavailable()
date_guesser = LinearDateGuesser()
for tr in self.page.get_operations(date_guesser=date_guesser):
yield tr
elif account.type == Account.TYPE_CARD:
for tr in self.get_cb_operations(account=account, month=1):
yield tr
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._external_website:
self.logger.warning(
'This account is limited, there is no available history.')
return
if account._is_calie_account:
# TODO build parsing of history page, all-you-can-eat js in it
# follow 'account._history_url' for that
raise NotImplementedError()
else:
self.assurancevie.stay_or_go()
self.go_life_insurance_website()
assert self.av_list.is_here(
), 'Something went wrong during iter life insurance history'
# Need to be on account details page to do history request
self.av_investments.go(life_insurance_id=account.id)
self.av_history.go()
for tr in self.page.iter_history():
yield tr
self.go_back_from_life_insurance_website()
def get_history(self, account):
# some accounts may exist without a link to any history page
if account._link is None:
return
date_guesser = LinearDateGuesser()
self.location(account._link)
if self.is_on_page(CardsPage):
yield from self.page.get_history(date_guesser)
else:
url = self.page.get_order_by_date_url()
while url:
self.location(url)
assert self.is_on_page(TransactionsPage)
yield from self.page.get_history(date_guesser)
url = self.page.get_next_url()
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE):
self.logger.warning('This account is not supported')
return
# some accounts may exist without a link to any history page
if account._link is None:
return
did_move = False
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
did_move = True
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_account(account.id, no_move=True)
date_guesser = LinearDateGuesser()
self.location(account._link.format(self.sag))
if self.is_on_page(CardsPage):
for tr in self.page.get_history(date_guesser):
yield tr
else:
url = self.page.get_order_by_date_url()
while url:
self.location(url)
assert self.is_on_page(TransactionsPage)
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
# Ugly hack needed for for following card accounts history
if did_move:
self.get_accounts_list()
def iter_history(self, account):
if account._history_url.startswith('javascript:') or account._history_url == '#':
raise NotImplementedError()
account = find_object(self.iter_accounts(), id=account.id)
# this url (reached with a GET) return some transactions, but not in same format than POST method
# and some transactions are duplicated and other are missing, don't take them from GET
# because we don't want to manage both way in iter_history
# fetch csrf token
self.go_with_ssodomi(self.accounts)
# we have to update the url at this moment because history consultation has to follow immediatly accounts page consultation.
account._history_url = self.update_csrf_token(account._history_url)
self.location(account._history_url)
date_range_list = self.page.get_date_range_list()
# a date_range is a couple of date like '01/03/201831/03/2018' but current month is often missing and we have to rebuild it
# from first one to get very recent transaction without scrap them from 1st page (reached with GET url)
if len(date_range_list):
date_range_list = [self._build_next_date_range(date_range_list[0])] + date_range_list
for date_range in date_range_list:
date_guesser = LinearDateGuesser(datetime.datetime.strptime(date_range[10:], "%d/%m/%Y"))
try:
self.location(account._history_url, data={'date': date_range})
except ServerError as error:
if error.response.status_code == 500:
if 'RELEVE NON DISPONIBLE A CETTE PERIODE' in error.response.text:
continue
# just skip because it's still possible to have transactions next months
# Yes, they really did that heresy...
else:
raise
for tr in sorted_transactions(self.page.iter_history(date_guesser=date_guesser)):
yield tr
def get_history(self, account, coming=False, retry_li=True):
self._quit_li_space()
self.update_accounts_list(False)
account = self.accounts_list[account.id]
if account.url is None:
return []
if account.url.startswith('javascript') or '&Crd=' in account.url:
raise NotImplementedError()
if account.type == Account.TYPE_LIFE_INSURANCE:
if coming is True:
return []
try:
if not self._go_to_life_insurance(account):
self._quit_li_space()
return []
except (XMLSyntaxError, HTTPNotFound):
self._quit_li_space()
return []
except AccountNotFound:
self.go_post(self.js_url)
# often if we visit life insurance subsite multiple times too quickly, the site just returns an error
# so we just retry (we might relogin...)
# TODO find out how to avoid the error, or avoid relogin
if retry_li:
self.logger.warning(
'life insurance seems unavailable for account %s',
account.id)
return self.get_history(account, coming, False)
self.logger.error(
'life insurance seems unavailable for account %s',
account.id)
return []
self.life_insurances.go(data={
'url_suivant': 'HISTORIQUECONTRATB2C',
'strMonnaie': 'EURO'
})
history = [t for t in self.page.iter_history()]
self._quit_li_space()
return history
try:
self.go_post(self.accounts_list[account.id].url)
# sometime go to hsbc life insurance space do logout
except HTTPNotFound:
self.app_gone = True
self.do_logout()
self.do_login()
# If we relogin on hsbc, all link have change
if self.app_gone:
self.app_gone = False
self.update_accounts_list()
self.location(self.accounts_list[account.id].url)
if self.page is None:
return []
if self.cbPage.is_here():
guesser = LinearDateGuesser(date_max_bump=timedelta(45))
history = list(self.page.get_history(date_guesser=guesser))
for url, params in self.page.get_params(self.url):
self.location(url, params=params)
if self.cbPage.is_here():
history.extend(self.page.get_history(date_guesser=guesser))
history.extend(self.get_monthly_transactions(history))
return [
tr for tr in history
if (coming and tr.date > date.today()) or (
not coming and tr.date <= date.today())
]
elif not coming:
return self._get_history()
else:
raise NotImplementedError()
def get_history(self, account):
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA,
Account.TYPE_LIFE_INSURANCE):
self.logger.warning('This account is not supported')
raise NotImplementedError()
# some accounts may exist without a link to any history page
if not hasattr(account, '_form') and (not account.url
or 'CATITRES' in account.url):
return
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
if hasattr(account, '_form'):
# the account needs a form submission to go to the history
# but we need to get the latest form data
self.location(self.accounts_url.format(self.sag))
accounts = self.page.get_list(use_links=False)
new_account = find_object(accounts, AccountNotFound, id=account.id)
self.location(new_account._form.request)
# card accounts need to get an updated link
if account.type == Account.TYPE_CARD:
account = self.get_card(account.id)
if account.url and (account.type != Account.TYPE_CARD
or not self.page.is_on_right_detail(account)):
self.location(account.url.format(self.sag))
if self.cards.is_here():
date_guesser = ChaoticDateGuesser(date.today() -
timedelta(weeks=42))
url = self.page.url
state = None
notfirst = False
while url:
if notfirst:
self.location(url)
else:
notfirst = True
assert self.cards.is_here()
for state, tr in self.page.get_history(date_guesser, state):
yield tr
url = self.page.get_next_url()
elif self.page:
date_guesser = LinearDateGuesser()
self.page.order_transactions()
while True:
assert self.transactions.is_here()
for tr in self.page.get_history(date_guesser):
yield tr
url = self.page.get_next_url()
if url is None:
break
self.location(url)
def iter_history(self, account, coming=False):
handled_history_types = (
Account.TYPE_CHECKING,
Account.TYPE_CARD,
Account.TYPE_SAVINGS,
Account.TYPE_PEA,
)
if account.type not in handled_history_types:
self.unhandled_method(account.id)
return
if account.type == Account.TYPE_CARD:
self.go_to_perimeter(account._perimeter)
self.accounts.go()
self.page.go_to_card(account._card_link)
assert (self.cards_page.is_here() or self.multiple_cards_page.is_here()), \
'Failed to reach card details for card %s.' % account.id
if self.multiple_cards_page.is_here():
# We need to go to the correct card transactions with its number.
card_url = self.page.get_transactions_link(account._raw_number)
self.location(card_url)
# When there are several future coming summaries,
# we must skip the ongoing one but fetch the other ones
# even if they are in the future.
ongoing_coming = self.page.get_ongoing_coming()
if not ongoing_coming:
# This card has no available history or coming.
return
card_transactions = []
latest_date = None
for tr in self.page.get_card_transactions(latest_date,
ongoing_coming):
card_transactions.append(tr)
if not card_transactions:
return
# Pagination: we must fetch the date of the last transaction
# because the summary of next transactions may not
# be available on the next page
latest_date = card_transactions[-1].date
next_page_url = self.page.get_next_page()
while next_page_url:
self.location(next_page_url)
for tr in self.page.get_card_transactions(
latest_date, ongoing_coming):
card_transactions.append(tr)
next_page_url = self.page.get_next_page()
for tr in sorted_transactions(card_transactions):
yield tr
return
# Transactions of accounts without form/url or with 'CATITRES' and 'bgpi' in url cannot be handled.
if not account._form and (not account.url or 'CATITRES' in account.url
or 'bgpi' in account.url):
self.unhandled_method(account.id)
return
# Access acount details:
if account.url:
# Refresh the session_value before going to the account URL
new_session_value = 'sessionSAG=' + self.session_value
updated_url = re.sub(r'sessionSAG=([^&]+)', new_session_value,
account.url)
self.location(updated_url)
elif account._form:
# We cannot use forms if we are not on the account's perimeter:
# we need to go to the correct perimeter and refresh forms.
# The form submission sometimes fails so we try several
# times until we get to the account history page.
for form in range(3):
self.accounts.stay_or_go()
self.go_to_perimeter(account._perimeter)
# Only fetch the perimeter's regular accounts (Checking & Savings)
# No need to go to Wealth, Loans or Netfinca for transactions
refreshed_account = find_object(
self.iter_perimeter_regular_accounts(iban=False),
AccountNotFound,
id=account.id)
refreshed_account._form.submit()
if self.failed_history.is_here():
self.logger.warning(
'Form submission failed to reach the account history, we try again.'
)
continue
break
# 3 types of history pages were identified so far
if not (self.checking_history.is_here()
or self.savings_history.is_here()
or self.other_savings_history.is_here()):
self.unhandled_method(account.id)
date_guesser = LinearDateGuesser(date_max_bump=timedelta(30))
for tr in self.page.iter_history(date_guesser=date_guesser):
yield tr
def get_coming(self):
transactions = list(self.operations_futures.go().get_list(
date_guesser=LinearDateGuesser()))
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions
def get_history(self):
transactions = list(self.operations_traitees.go().get_history(
date_guesser=LinearDateGuesser()))
transactions.sort(key=lambda tr: tr.rdate, reverse=True)
return transactions