def get_card_transactions(self, latest_date, ongoing_coming):
for item in self.doc.xpath('//table[@class="ca-table"][2]//tr[td]'):
if CleanText('./td[2]/b')(item):
# This node is a summary containing the 'date' for all following transactions.
raw_date = Regexp(CleanText('./td[2]/b/text()'),
r'le (.*) :')(item)
if latest_date and parse_french_date(
raw_date).date() > latest_date:
# This summary has already been fetched
continue
latest_date = parse_french_date(raw_date).date()
if latest_date < ongoing_coming:
# This summary is anterior to the ongoing_coming so we create a transaction from it
tr = FrenchTransaction()
tr.date = tr.rdate = latest_date
tr.raw = tr.label = CleanText('./td[2]/b/text()')(item)
tr.amount = -CleanDecimal.French(
'./td[position()=last()]')(item)
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
yield tr
else:
# This node is a real transaction.
# Its 'date' is the date of the most recently encountered summary node.
tr = FrenchTransaction()
tr.date = latest_date
date_guesser = LinearDateGuesser(latest_date)
tr.rdate = tr.bdate = DateGuesser(
CleanText('./td[1]//text()'),
date_guesser=date_guesser)(item)
tr.label = tr.raw = CleanText('./td[2]')(item)
tr.amount = CleanDecimal.French('./td[last()]')(item)
tr.type = FrenchTransaction.TYPE_DEFERRED_CARD
yield tr
def parse_transaction(self, transaction, account):
trans = []
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \
u'Created']:
return []
if transaction['transactionDescription'].startswith(u'Offre de remboursement') or transaction['transactionDescription'].startswith(u'Commande à'):
return []
t = FrenchTransaction(transaction['transactionId'])
if not transaction['transactionAmount']['currencyCode'] == account.currency:
cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if not cc:
return []
t.original_amount = Decimal('%.2f' % transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = u'' + transaction['transactionAmount']['currencyCode']
t.amount = abs(cc) if not transaction['debit'] else -abs(cc)
else:
t.amount = Decimal('%.2f' % transaction['net']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'):
payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback(transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if payback_id and payback_raw and payback_amount and payback_currency:
t_payback = FrenchTransaction(payback_id)
t_payback.amount = payback_amount
t_payback.original_currency = payback_currency
t_payback.type = FrenchTransaction.TYPE_TRANSFER
t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw)
trans.append(t_payback)
t.commission = Decimal('%.2f' % transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction['amounts'] and account.currency == transaction['amounts']['conversionFrom']['currency']:
cc = self.format_amount(transaction['amounts']['conversionFrom']['value'], transaction['isCredit'])
else:
try:
cc = self.browser.convert_amount(account, transaction, transaction['detailsLink'])
except ServerError:
self.logger.warning('Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def parse_transaction(self, transaction, account):
trans = []
if 'transactionStatus' in transaction and transaction[
'transactionStatus'] in [
u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif',
u'Payé', u'En attente', u'Rejeté', u'Expiré', u'Created',
u'Brouillon', u'Paid', u'Pending', u'Canceled'
]:
return []
for pattern in [u'Commande à', u'Offre de remboursement', u'Bill to']:
if 'description' not in transaction[
'transactionDescription'] or transaction[
'transactionDescription']['description'].startswith(
pattern):
return []
t = FrenchTransaction(transaction['transactionId'])
# Those are not really transactions.
if not 'currency' in transaction['grossAmount'] \
or transaction['transactionDescription']['description'].startswith("Conversion de devise"):
return []
original_currency = unicode(transaction['grossAmount']['currency'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = [tr['grossAmount']['amountUnformatted'] for tr in transaction['secondaryTransactions'] \
if account.currency == tr['grossAmount']['currency'] \
and (tr['grossAmount']['amountUnformatted'] < 0) == (transaction['grossAmount']['amountUnformatted'] < 0) \
and tr['transactionDescription']['description'].startswith('Conversion de devise')]
if not cc:
return []
assert len(cc) == 1
t.original_amount = Decimal(
str(transaction['grossAmount']['amountUnformatted']))
t.original_currency = original_currency
t.amount = Decimal(str(cc[0]))
else:
t.amount = Decimal(
str(transaction['netAmount']['amountUnformatted']))
date = parse_french_date(transaction['transactionTime'])
raw = "%s %s" % (transaction['transactionDescription']['description'],
transaction['transactionDescription']['name'])
if raw == "Transfert de Compte bancaire":
t.type = FrenchTransaction.TYPE_TRANSFER
t.commission = Decimal(
str(transaction['feeAmount']['amountUnformatted']))
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def parse_transaction(self, transaction, account):
page = None
if 'id' not in transaction or not transaction['date']:
return []
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
if not 'txnCurrency' in transaction['amounts']:
return []
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction[
'amounts'] and 'value' in transaction['amounts'][
'conversionFrom'] and account.currency == transaction[
'amounts']['conversionFrom']['currency']:
cc = self.format_amount(
transaction['amounts']['conversionFrom']['value'],
transaction['isCredit'])
else:
try:
page = self.return_detail_page(transaction['detailsLink'])
cc = page.get_converted_amount() if isinstance(
page, HistoryDetailsPage) else None
except ServerError:
self.logger.warning(
'Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' +
transaction['date']['year']).date()
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
if page is None and t.amount < 0:
page = self.return_detail_page(transaction['detailsLink'])
funding_src = page.get_funding_src(t) if isinstance(
page, HistoryDetailsPage) else None
return [t] if funding_src is None else ([t] + [funding_src])
def parse_transaction(self, transaction, account):
trans = []
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \
u'Created', u'Canceled']:
return []
if transaction['transactionDescription'].startswith(
u'Offre de remboursement'
) or transaction['transactionDescription'].startswith(u'Commande à'):
return []
t = FrenchTransaction(transaction['transactionId'])
original_currency = unicode(
transaction['transactionAmount']['currencyCode'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = self.browser.convert_amount(
account, transaction,
'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id='
+ transaction['transactionId'])
if not cc:
return []
t.original_amount = Decimal(
'%.2f' %
transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = original_currency
t.amount = abs(cc) if not transaction['debit'] else -abs(cc)
else:
t.amount = Decimal('%.2f' %
transaction['net']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'):
payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback(
transaction,
'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id='
+ transaction['transactionId'])
if payback_id and payback_raw and payback_amount and payback_currency:
t_payback = FrenchTransaction(payback_id)
t_payback.amount = payback_amount
t_payback.original_currency = payback_currency
t_payback.type = FrenchTransaction.TYPE_TRANSFER
t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw)
trans.append(t_payback)
t.commission = Decimal('%.2f' %
transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def get_history(self, currency):
self.MONTHS = self.FR_MONTHS if currency == 'EUR' else self.US_MONTHS
#checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
#adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(
days=360)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
for tr in reversed(
self.doc.xpath(
'//div[@id="txnsSection"]//tr[@class="tableStandardText"]')
):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.MONTHS.index(month.rstrip('.')) + 1
date = guesser.guess_date(day, month)
rdate = None
try:
detail = cols[self.COL_TEXT].xpath(
'./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join(
[txt.strip() for txt in detail.itertext()])).strip())
if m:
rday, rmonth = m.group(1).strip().split(' ')
rday = int(rday)
rmonth = self.MONTHS.index(rmonth.rstrip('.')) + 1
rdate = guesser.guess_date(rday, rmonth)
detail.drop_tree()
raw = (' '.join([
txt.strip() for txt in cols[self.COL_TEXT].itertext()
])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
t.date = date
t.rdate = rdate or date
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = CleanDecimal(replace_dots=currency == 'EUR').filter(
credit or debit) * (1 if credit else -1)
if t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.type = t.TYPE_CARD
yield t
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
cc = self.browser.convert_amount(account, transaction, transaction['detailsLink'])
if not cc:
return []
t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"])
t.original_currency = u'' + transaction['amounts']['txnCurrency']
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def get_history(self, date_guesser):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
for line in lines[1:]: # first line is balance
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
[date, label, _, amount] = [self.parser.tocleanstring(td)
for td in line.xpath('./td')]
t = Transaction(0)
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError('Unable to read card balance in history: %r' % label)
t.date = parse_french_date(m.group(1))
t.amount = -t.amount
else:
day, month = map(int, date.split('/', 1))
t.date = date_guesser.guess_date(day, month)
t.type = t.TYPE_CARD
t.rdate = t.date
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
print t
print t.label
raise
yield t
def get_history(self, date_guesser):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
for line in lines[1:]: # first line is balance
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
[date, label, _, amount
] = [self.parser.tocleanstring(td) for td in line.xpath('./td')]
t = Transaction(0)
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError(
'Unable to read card balance in history: %r' % label)
t.date = parse_french_date(m.group(1))
t.amount = -t.amount
else:
day, month = map(int, date.split('/', 1))
t.date = date_guesser.guess_date(day, month)
t.type = t.TYPE_CARD
t.rdate = t.date
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
print t
print t.label
raise
yield t
def get_funding_src(self, t):
if 'fundingSource' not in self.doc['data']['details']:
return None
funding_src_lst = [src for src in self.doc['data']['details']['fundingSource']['fundingSourceList'] if src['type'] != 'BALANCE']
assert len(funding_src_lst) <= 1
for src in funding_src_lst:
tr = FrenchTransaction(t.id+'_fundingSrc')
tr.amount = CleanDecimal(replace_dots=True).filter(src['amount'])
tr.date = tr.rdate = t.date
tr.label = tr.raw = u'Crédit depuis %s' % src['institution']
return tr
def parse_transaction(self, transaction, account):
if self.browser.is_new_api:
return self.parse_new_api_transaction(transaction, account)
t = FrenchTransaction(transaction['transactionId'])
date = parse_french_date(transaction['date'])
if not transaction['txnIsInPrimaryCurrency']:
cc = self.browser.convert_amount(account, transaction, transaction['actions']['details']['url'])
if not cc:
return
t.original_amount = self.format_amount(transaction['netAmount'], transaction["isCredit"])
t.original_currency = u'' + transaction["currencyCode"]
t.amount = self.format_amount(cc, transaction['isCredit'])
elif 'netAmount' in transaction:
t.amount = self.format_amount(transaction['netAmount'], transaction["isCredit"])
else:
return
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return t
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
cc = self.browser.convert_amount(account, transaction,
transaction['detailsLink'])
if not cc:
return []
t.original_amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction["isCredit"])
t.original_currency = u'' + transaction['amounts']['txnCurrency']
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' +
transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def get_monthly_transactions(self, trs):
groups = [list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date)]
trs = []
for group in groups:
if group[0].date > date.today():
continue
tr = FrenchTransaction()
tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date
tr.amount = -sum([t.amount for t in group])
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
trs.append(tr)
return trs
def get_history(self, currency):
self.MONTHS = self.FR_MONTHS if currency == 'EUR' else self.US_MONTHS
#checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
#adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(days=300)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
for tr in reversed(self.doc.xpath('//div[@id="txnsSection"]//tr[@class="tableStandardText"]')):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.MONTHS.index(month.rstrip('.')) + 1
date = guesser.guess_date(day, month)
rdate = None
try:
detail = cols[self.COL_TEXT].xpath('./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join([txt.strip() for txt in detail.itertext()])).strip())
if m:
rday, rmonth = m.group(1).split(' ')
rday = int(rday)
rmonth = self.MONTHS.index(rmonth.rstrip('.')) + 1
rdate = guesser.guess_date(rday, rmonth)
detail.drop_tree()
raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
t.date = date
t.rdate = rdate or date
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = CleanDecimal(replace_dots=currency == 'EUR').filter(credit or debit) * (1 if credit else -1)
if t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.type = t.TYPE_CARD
yield t
def get_history(self, date_guesser, state=None):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
debit_date = None
for i, line in enumerate(lines):
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
# It is possible to have three or four columns.
cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')]
date = cols[0]
label = cols[1]
amount = cols[-1]
t = Transaction()
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError(
'Unable to read card balance in history: %r' % label)
if state is None:
debit_date = parse_french_date(m.group(1))
else:
debit_date = state
# Skip the first line because it is balance
if i == 0:
continue
t.date = t.rdate = debit_date
# Consider the second one as a positive amount to reset balance to 0.
t.amount = -t.amount
state = t.date
else:
day, month = map(int, date.split('/', 1))
t.rdate = date_guesser.guess_date(day, month)
t.date = debit_date
t.type = t.TYPE_CARD
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
self.logger.debug(t)
self.logger.debug(t.label)
raise
yield state, t
def get_history(self, date_guesser, state=None):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
debit_date = None
for i, line in enumerate(lines):
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
# It is possible to have three or four columns.
cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')]
date = cols[0]
label = cols[1]
amount = cols[-1]
t = Transaction()
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError('Unable to read card balance in history: %r' % label)
if state is None:
debit_date = parse_french_date(m.group(1))
else:
debit_date = state
# Skip the first line because it is balance
if i == 0:
continue
t.date = t.rdate = debit_date
# Consider the second one as a positive amount to reset balance to 0.
t.amount = -t.amount
state = t.date
else:
day, month = map(int, date.split('/', 1))
t.rdate = date_guesser.guess_date(day, month)
t.date = debit_date
t.type = t.TYPE_CARD
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
self.logger.debug(t)
self.logger.debug(t.label)
raise
yield state, t
def get_funding_src(self, t):
if 'fundingSource' not in self.doc['data']['details']:
return None
funding_src_lst = [
src for src in self.doc['data']['details']['fundingSource']
['fundingSourceList'] if src['type'] != 'BALANCE'
]
assert len(funding_src_lst) <= 1
for src in funding_src_lst:
tr = FrenchTransaction(t.id + '_fundingSrc')
tr.amount = CleanDecimal(replace_dots=True).filter(src['amount'])
tr.date = tr.rdate = t.date
tr.label = tr.raw = u'Crédit depuis %s' % src['institution']
return tr
def parse_transaction(self, transaction, account):
page = None
if 'id' not in transaction or not transaction['date']:
return []
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
if not 'txnCurrency' in transaction['amounts']:
return []
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction['amounts'] and 'value' in transaction['amounts']['conversionFrom'] and account.currency == transaction['amounts']['conversionFrom']['currency']:
cc = self.format_amount(transaction['amounts']['conversionFrom']['value'], transaction['isCredit'])
else:
try:
page = self.return_detail_page(transaction['detailsLink'])
cc = page.get_converted_amount() if isinstance(page, HistoryDetailsPage) else None
except ServerError:
self.logger.warning('Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']).date()
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
if page is None and t.amount < 0:
page = self.return_detail_page(transaction['detailsLink'])
funding_src = page.get_funding_src(t) if isinstance(page, HistoryDetailsPage) else None
return [t] if funding_src is None else ([t] + [funding_src])
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_monthly_transactions(self, trs):
date_getter = attrgetter('date')
groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)]
trs = []
for group in groups:
if group[0].date > datetime.today().date():
continue
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date
tr.amount = -sum(t.amount for t in group)
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
trs.append(tr)
return trs
def get_monthly_transactions(self, trs):
groups = [
list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date),
lambda tr: tr.date)
]
trs = []
for group in groups:
if group[0].date > date.today():
continue
tr = FrenchTransaction()
tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date
tr.amount = -sum([t.amount for t in group])
tr.date = tr.rdate = tr.vdate = group[0].date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
trs.append(tr)
return trs
def parse_transaction(self, transaction, account):
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif']:
return
t = FrenchTransaction(transaction['transactionId'])
if not transaction['transactionAmount']['currencyCode'] == account.currency:
cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if not cc:
return
t.original_amount = Decimal(transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = u'' + transaction['transactionAmount']['currencyCode']
t.set_amount(cc)
else:
t.amount = Decimal(transaction['transactionAmount']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
t.commission = Decimal(transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
return t
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['activityId'])
date = parse_french_date(transaction['date'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
try:
if transaction['currencyCode'] != account.currency:
transaction = self.browser.convert_amount(account, transaction)
t.original_amount = self.format_amount(transaction['originalAmount'], transaction["isCredit"])
t.original_currency = u'' + transaction["currencyCode"]
t.amount = self.format_amount(transaction['netAmount'], transaction["isCredit"])
except KeyError:
return
t._currency = transaction['currencyCode']
return t
def get_history(self):
for tr in self.document.xpath('//div[contains(@class, "mod-listeoperations")]//table/tbody/tr'):
cols = tr.findall('td')
date = self.parser.tocleanstring(cols[0])
raw = self.parser.tocleanstring(cols[1])
label = re.sub(u' - traité le \d+/\d+', '', raw)
debit = self.parser.tocleanstring(cols[3])
if len(debit) > 0:
t = FrenchTransaction(0)
t.parse(date, raw)
t.label = label
t.set_amount(debit)
yield t
amount = self.parser.tocleanstring(cols[2])
if len(amount) > 0:
t = FrenchTransaction(0)
t.parse(date, raw)
t.label = label
t.set_amount(amount)
t.amount = - t.amount
yield t
def parse_transaction(self, transaction, account):
trans = []
# Add secondary transactions on label condition.
for t in transaction['secondaryTransactions']:
if t['transactionDescription']['description'] == u'Virement à partir de':
trans.extend(self.parse_transaction(t, account))
if 'transactionStatus' in transaction and transaction['transactionStatus'] in [u'Créé',
u'Annulé',
u'Suspendu',
u'Mis à jour',
u'Actif', u'Payé',
u'En attente',
u'Rejeté',
u'Expiré',
u'Created',
u'Brouillon',
u'Paid',
u'Pending',
u'Canceled',
u'Suspended']:
return []
for pattern in [u'Commande à', u'Offre de remboursement', u'Bill to']:
if 'description' not in transaction['transactionDescription'] or transaction['transactionDescription']['description'].startswith(pattern):
return []
t = FrenchTransaction(transaction['transactionId'])
# Those are not really transactions.
if 'grossAmount' not in transaction or not 'currency' in transaction['grossAmount'] \
or transaction['transactionDescription']['description'].startswith("Conversion de devise"):
return []
original_currency = unicode(transaction['grossAmount']['currency'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = [tr['grossAmount']['amountUnformatted'] for tr in transaction['secondaryTransactions'] \
if account.currency == tr['grossAmount']['currency'] \
and (tr['grossAmount']['amountUnformatted'] < 0) == (transaction['grossAmount']['amountUnformatted'] < 0) \
and tr['transactionDescription']['description'].startswith('Conversion de devise')]
if not cc:
return []
assert len(cc) == 1
t.original_amount = Decimal(str(transaction['netAmount']['amountUnformatted']))
t.original_currency = original_currency
t.amount = Decimal(str(cc[0]))
else:
t.amount = Decimal(str(transaction['netAmount']['amountUnformatted']))
date = parse_french_date(transaction['transactionTime'])
raw = "%s %s" % (transaction['transactionDescription']['description'], transaction['transactionDescription']['name'])
if raw == "Transfert de Compte bancaire":
t.type = FrenchTransaction.TYPE_TRANSFER
if raw == u'Annulation des frais de PayPal':
return []
# Dougs told us that commission should always be netAmount minus grossAmount
grossAmount = Decimal(str(transaction['grossAmount']['amountUnformatted']))
t.commission = Decimal(str(transaction['feeAmount']['amountUnformatted']))
if t.commission:
if original_currency == account.currency:
assert abs(t.amount - grossAmount) == abs(t.commission)
t.commission = t.amount - grossAmount
else:
t.commission = (t.commission * t.amount / t.original_amount).quantize(Decimal('.01'), rounding=ROUND_DOWN)
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem': '',
'_wxf2_cc': 'fr-FR',
'_wxf2_pmode': 'Normal',
'_wxf2_pseq': i,
'_wxf2_ptarget': 'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '',
'Data_ServiceListDatas_CurrentType': 'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)', self.url)
if m:
m = m.group(1)
self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
differed_date = None
cards = ([page.select_card(account._card_number) for page in account._card_pages]
if hasattr(account, '_card_pages')
else account._card_links if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date):
differed_date = tr._differed_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if differed_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY
and differed_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
def get_history(self, currency):
# checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
# adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(
days=360)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
# Since the site doesn't provide the debit_date,
# we just use the date of beginning of the previous period.
# If this date + 1 month is greater than today's date,
# then the transaction is coming
end_of_period = None
previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]',
default=None)(self.doc)
if previous_date:
end_of_period = (
parse_french_date(' '.join(previous_date.split()[1:4])) +
relativedelta(months=1)).date()
else:
previous_date = CleanText(
'//select[@id="viewPeriod"]/option[@selected]',
default=None)(self.doc)
if previous_date:
end_of_period = parse_french_date(' '.join(
previous_date.split()[:3])) + relativedelta(
days=-1) + relativedelta(months=1)
end_of_period = end_of_period.date()
for tr in reversed(
self.doc.xpath(
'//div[@id="txnsSection"]//tbody/tr[@class="tableStandardText"]'
)):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.parse_month(month)
date = guesser.guess_date(day, month)
vdate = None
try:
detail = cols[self.COL_TEXT].xpath(
'./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join(
[txt.strip() for txt in detail.itertext()])).strip())
if m:
vday, vmonth = m.group(1).strip().split(' ')
vday = int(vday)
vmonth = self.parse_month(vmonth)
vdate = guesser.guess_date(vday, vmonth)
detail.drop_tree()
raw = (' '.join([
txt.strip() for txt in cols[self.COL_TEXT].itertext()
])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
if end_of_period is not None and datetime.date.today(
) < end_of_period:
t._is_coming = True
else:
t._is_coming = False
t.date = t.rdate = date
t.vdate = vdate
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = parse_decimal(credit or debit) * (1 if credit else -1)
if t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.date = end_of_period
t.type = t.TYPE_DEFERRED_CARD
yield t
def get_history(self, account):
# checking if the card is still valid
if self.doc.xpath('//div[@id="errorbox"]'):
return
# adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(days=360)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
# Since the site doesn't provide the debit_date,
# we just use the date of beginning of the previous period.
# If this date + 1 month is greater than today's date,
# then the transaction is coming
end_of_period = None
previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]', default=None)(self.doc)
if previous_date:
end_of_period = (parse_french_date(' '.join(previous_date.split()[1:4])) + relativedelta(months=1)).date()
else:
previous_date = CleanText('//select[@id="viewPeriod"]/option[@selected]', default=None)(self.doc)
if previous_date:
end_of_period = parse_french_date(' '.join(previous_date.split()[:3])) + relativedelta(days=-1) + relativedelta(months=1)
end_of_period = end_of_period.date()
_id = str(int(account._idforold))
for tr in reversed(self.doc.xpath('//div[@id="txnsSection"]//tbody[@id="tableBody-txnsCard%s"]/tr[@class="tableStandardText"]' % _id)):
cols = tr.findall('td')
t = Transaction()
day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1)
day = int(day)
month = self.parse_month(month)
date = guesser.guess_date(day, month)
vdate = None
try:
detail = cols[self.COL_TEXT].xpath('./div[has-class("hiddenROC")]')[0]
except IndexError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join([txt.strip() for txt in detail.itertext()])).strip())
if m:
vday, vmonth = m.group(1).strip().split(' ')
vday = int(vday)
vmonth = self.parse_month(vmonth)
vdate = guesser.guess_date(vday, vmonth)
detail.drop_tree()
raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip()
credit = CleanText().filter(cols[self.COL_CREDIT])
debit = CleanText().filter(cols[self.COL_DEBIT])
if end_of_period is not None and datetime.date.today() < end_of_period:
t._is_coming = True
else:
t._is_coming = False
t.date = t.rdate = date
t.vdate = vdate
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = parse_decimal(credit or debit) * (1 if credit else -1)
if t.raw in self.browser.SUMMARY_CARD_LABEL:
t.type = t.TYPE_CARD_SUMMARY
elif t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.date = end_of_period
t.type = t.TYPE_DEFERRED_CARD
yield t
def parse_transaction(self, transaction, account):
trans = []
# Add secondary transactions on label condition.
for t in transaction['secondaryTransactions']:
if t['transactionDescription'][
'description'] == u'Virement à partir de':
trans.extend(self.parse_transaction(t, account))
if 'transactionStatus' in transaction and transaction[
'transactionStatus'] in [
u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif',
u'Payé', u'En attente', u'Rejeté', u'Expiré', u'Created',
u'Brouillon', u'Paid', u'Pending', u'Canceled',
u'Suspended'
]:
return []
for pattern in [u'Commande à', u'Offre de remboursement', u'Bill to']:
if 'description' not in transaction[
'transactionDescription'] or transaction[
'transactionDescription']['description'].startswith(
pattern):
return []
t = FrenchTransaction(transaction['transactionId'])
# Those are not really transactions.
if 'grossAmount' not in transaction or not 'currency' in transaction['grossAmount'] \
or transaction['transactionDescription']['description'].startswith("Conversion de devise"):
return []
original_currency = unicode(transaction['grossAmount']['currency'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = [tr['grossAmount']['amountUnformatted'] for tr in transaction['secondaryTransactions'] \
if account.currency == tr['grossAmount']['currency'] \
and (tr['grossAmount']['amountUnformatted'] < 0) == (transaction['grossAmount']['amountUnformatted'] < 0) \
and tr['transactionDescription']['description'].startswith('Conversion de devise')]
if not cc:
return []
assert len(cc) == 1
t.original_amount = Decimal(
str(transaction['netAmount']['amountUnformatted']))
t.original_currency = original_currency
t.amount = Decimal(str(cc[0]))
else:
t.amount = Decimal(
str(transaction['netAmount']['amountUnformatted']))
date = parse_french_date(transaction['transactionTime'])
raw = "%s %s" % (transaction['transactionDescription']['description'],
transaction['transactionDescription']['name'])
if raw == "Transfert de Compte bancaire":
t.type = FrenchTransaction.TYPE_TRANSFER
if raw == u'Annulation des frais de PayPal':
return []
# Dougs told us that commission should always be netAmount minus grossAmount
grossAmount = Decimal(
str(transaction['grossAmount']['amountUnformatted']))
t.commission = Decimal(
str(transaction['feeAmount']['amountUnformatted']))
if t.commission:
if original_currency == account.currency:
assert abs(t.amount - grossAmount) == abs(t.commission)
t.commission = t.amount - grossAmount
else:
t.commission = (t.commission * t.amount /
t.original_amount).quantize(
Decimal('.01'), rounding=ROUND_DOWN)
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def get_history(self, account):
transactions = []
if not account._link_id:
raise NotImplementedError()
if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available:
if self.two_cards_page:
# In this case, you need to return to the page where the iter account get the cards information
# Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same
account._referer.go(subbank=self.currentSubBank)
if account._secondpage:
self.location(self.page.get_second_page_link())
# Check if '000000xxxxxx0000' card have an annual history
self.location(account._link_id)
# The history of the card is available for 1 year with 1 month per page
# Here we catch all the url needed to be the more compatible with the catch of merged subtransactions
urlstogo = self.page.get_links()
self.location(account._link_id)
half_history = 'firstHalf'
for url in urlstogo:
transactions = []
self.location(url)
if 'GoMonthPrecedent' in url:
# To reach the 6 last month of history you need to change this url parameter
# Moreover we are on a transition page where we see the 6 next month (no scrapping here)
half_history = 'secondHalf'
else:
history = self.page.get_history()
self.tr_date = self.page.get_date()
amount_summary = self.page.get_amount_summary()
if self.page.has_more_operations():
for i in range(1, 100):
# Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break)
data = {
'_FID_DoAddElem': '',
'_wxf2_cc': 'fr-FR',
'_wxf2_pmode': 'Normal',
'_wxf2_pseq': i,
'_wxf2_ptarget': 'C:P:updPan',
'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '',
'Data_ServiceListDatas_CurrentType': 'MyCards',
}
if 'fid=GoMonth&mois=' in self.url:
m = re.search(r'fid=GoMonth&mois=(\d+)', self.url)
if m:
m = m.group(1)
self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data)
else:
self.location(self.url, data=data)
if not self.page.has_more_operations_xml():
history = self.page.iter_history_xml(date=self.tr_date)
# We are now with an XML page with all the transactions of the month
break
else:
history = self.page.get_history(date=self.tr_date)
for tr in history:
# For regrouped transaction, we have to go through each one to get details
if tr._regroup:
self.location(tr._regroup)
for tr2 in self.page.get_tr_merged():
tr2._is_coming = tr._is_coming
tr2.date = self.tr_date
transactions.append(tr2)
else:
transactions.append(tr)
if transactions and self.tr_date < datetime.today().date():
tr = FrenchTransaction()
tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date
tr.amount = amount_summary
tr.date = tr.rdate = tr.vdate = self.tr_date
tr.type = FrenchTransaction.TYPE_CARD_SUMMARY
tr._is_coming = False
tr._is_manualsum = True
transactions.append(tr)
for tr in sorted_transactions(transactions):
yield tr
else:
# need to refresh the months select
if account._link_id.startswith('ENC_liste_oper'):
self.location(account._pre_link)
if not hasattr(account, '_card_pages'):
for tr in self.list_operations(account._link_id, account):
transactions.append(tr)
coming_link = self.page.get_coming_link() if self.operations.is_here() else None
if coming_link is not None:
for tr in self.list_operations(coming_link, account):
transactions.append(tr)
deferred_date = None
cards = ([page.select_card(account._card_number) for page in account._card_pages]
if hasattr(account, '_card_pages')
else account._card_links if hasattr(account, '_card_links') else [])
for card in cards:
card_trs = []
for tr in self.list_operations(card, account):
if tr._to_delete:
# Delete main transaction when subtransactions exist
continue
if hasattr(tr, '_deferred_date') and (not deferred_date or tr._deferred_date < deferred_date):
deferred_date = tr._deferred_date
if tr.date >= datetime.now():
tr._is_coming = True
elif hasattr(account, '_card_pages'):
card_trs.append(tr)
transactions.append(tr)
if card_trs:
transactions.extend(self.get_monthly_transactions(card_trs))
if deferred_date is not None:
# set deleted for card_summary
for tr in transactions:
tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY
and deferred_date.month <= tr.date.month
and not hasattr(tr, '_is_manualsum'))
for tr in sorted_transactions(transactions):
yield tr
