def get_monthly_transactions(self, trs):
groups = [list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date)]
trs = []
for group in groups:
tr = FrenchTransaction()
tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date
tr.amount = -sum([t.amount for t in group])
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_monthly_transactions(self, trs):
groups = [
list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date),
lambda tr: tr.date)
]
trs = []
for group in groups:
tr = FrenchTransaction()
tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date
tr.amount = -sum([t.amount for t in group])
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_history(self, account):
# checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
# adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(days=360)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
# Since the site doesn't provide the debit_date,
# we just use the date of beginning of the previous period.
# If this date + 1 month is greater than today's date,
# then the transaction is coming
end_of_period = None
previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]', default=None)(self.doc)
if previous_date:
end_of_period = (parse_french_date(' '.join(previous_date.split()[1:4])) + relativedelta(months=1)).date()
else:
previous_date = CleanText('//select[@id="viewPeriod"]/option[@selected]', default=None)(self.doc)
if previous_date:
end_of_period = parse_french_date(' '.join(previous_date.split()[:3])) + relativedelta(days=-1) + relativedelta(months=1)
end_of_period = end_of_period.date()
_id = str(int(account._idforold))
for tr in reversed(self.doc.xpath('//div[@id="txnsSection"]//tbody[@id="tableBody-txnsCard%s"]/tr[@class="tableStandardText"]' % _id)):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.parse_month(month)
date = guesser.guess_date(day, month)
vdate = None
try:
detail = cols[self.COL_TEXT].xpath('./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join([txt.strip() for txt in detail.itertext()])).strip())
if m:
vday, vmonth = m.group(1).strip().split(' ')
vday = int(vday)
vmonth = self.parse_month(vmonth)
vdate = guesser.guess_date(vday, vmonth)
detail.drop_tree()
raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
if end_of_period is not None and datetime.date.today() < end_of_period:
t._is_coming = True
else:
t._is_coming = False
t.date = t.rdate = date
t.vdate = vdate
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = parse_decimal(credit or debit) * (1 if credit else -1)
if t.raw in self.browser.SUMMARY_CARD_LABEL:
t.type = t.TYPE_CARD_SUMMARY
elif t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.date = end_of_period
t.type = t.TYPE_DEFERRED_CARD
yield t
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem': '',
'_wxf2_cc': 'fr-FR',
'_wxf2_pmode': 'Normal',
'_wxf2_pseq': i,
'_wxf2_ptarget': 'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '',
'Data_ServiceListDatas_CurrentType': 'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)', self.url)
if m:
m = m.group(1)
self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
deferred_date = None
cards = ([page.select_card(account._card_number) for page in account._card_pages]
if hasattr(account, '_card_pages')
else account._card_links if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_deferred_date') and (not deferred_date or tr._deferred_date < deferred_date):
deferred_date = tr._deferred_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if deferred_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY
and deferred_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem': '',
'_wxf2_cc': 'fr-FR',
'_wxf2_pmode': 'Normal',
'_wxf2_pseq': i,
'_wxf2_ptarget': 'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '',
'Data_ServiceListDatas_CurrentType': 'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)', self.url)
if m:
m = m.group(1)
self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
differed_date = None
cards = ([page.select_card(account._card_number) for page in account._card_pages]
if hasattr(account, '_card_pages')
else account._card_links if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY
and differed_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
def get_history(self, currency):
# checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
# adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(
days=360)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
# Since the site doesn't provide the debit_date,
# we just use the date of beginning of the previous period.
# If this date + 1 month is greater than today's date,
# then the transaction is coming
end_of_period = None
previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]',
default=None)(self.doc)
if previous_date:
end_of_period = (
parse_french_date(' '.join(previous_date.split()[1:4])) +
relativedelta(months=1)).date()
else:
previous_date = CleanText(
'//select[@id="viewPeriod"]/option[@selected]',
default=None)(self.doc)
if previous_date:
end_of_period = parse_french_date(' '.join(
previous_date.split()[:3])) + relativedelta(
days=-1) + relativedelta(months=1)
end_of_period = end_of_period.date()
for tr in reversed(
self.doc.xpath(
'//div[@id="txnsSection"]//tbody/tr[@class="tableStandardText"]'
)):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.parse_month(month)
date = guesser.guess_date(day, month)
vdate = None
try:
detail = cols[self.COL_TEXT].xpath(
'./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join(
[txt.strip() for txt in detail.itertext()])).strip())
if m:
vday, vmonth = m.group(1).strip().split(' ')
vday = int(vday)
vmonth = self.parse_month(vmonth)
vdate = guesser.guess_date(vday, vmonth)
detail.drop_tree()
raw = (' '.join([
txt.strip() for txt in cols[self.COL_TEXT].itertext()
])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
if end_of_period is not None and datetime.date.today(
) < end_of_period:
t._is_coming = True
else:
t._is_coming = False
t.date = t.rdate = date
t.vdate = vdate
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = parse_decimal(credit or debit) * (1 if credit else -1)
if t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.date = end_of_period
t.type = t.TYPE_DEFERRED_CARD
yield t
