def get_monthly_transactions(self, trs): groups = [list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date)] trs = [] for group in groups: tr = FrenchTransaction() tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date tr.amount = -sum([t.amount for t in group]) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs
def get_monthly_transactions(self, trs): groups = [ list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date) ] trs = [] for group in groups: tr = FrenchTransaction() tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date tr.amount = -sum([t.amount for t in group]) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs
def get_monthly_transactions(self, trs): date_getter = attrgetter('date') groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)] trs = [] for group in groups: if group[0].date > datetime.today().date(): continue tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date tr.amount = -sum(t.amount for t in group) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True trs.append(tr) return trs
def get_history(self, account): # checking if the card is still valid if self.doc.xpath('//div[@id="errorbox"]'): return # adding a time delta because amex have hard time to put the date in a good interval beginning_date = self.get_beginning_debit_date() - datetime.timedelta(days=360) end_date = self.get_end_debit_date() guesser = ChaoticDateGuesser(beginning_date, end_date) # Since the site doesn't provide the debit_date, # we just use the date of beginning of the previous period. # If this date + 1 month is greater than today's date, # then the transaction is coming end_of_period = None previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]', default=None)(self.doc) if previous_date: end_of_period = (parse_french_date(' '.join(previous_date.split()[1:4])) + relativedelta(months=1)).date() else: previous_date = CleanText('//select[@id="viewPeriod"]/option[@selected]', default=None)(self.doc) if previous_date: end_of_period = parse_french_date(' '.join(previous_date.split()[:3])) + relativedelta(days=-1) + relativedelta(months=1) end_of_period = end_of_period.date() _id = str(int(account._idforold)) for tr in reversed(self.doc.xpath('//div[@id="txnsSection"]//tbody[@id="tableBody-txnsCard%s"]/tr[@class="tableStandardText"]' % _id)): cols = tr.findall('td') t = Transaction() day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1) day = int(day) month = self.parse_month(month) date = guesser.guess_date(day, month) vdate = None try: detail = cols[self.COL_TEXT].xpath('./div[has-class("hiddenROC")]')[0] except IndexError: pass else: m = re.search(r' (\d{2} \D{3,4})', (' '.join([txt.strip() for txt in detail.itertext()])).strip()) if m: vday, vmonth = m.group(1).strip().split(' ') vday = int(vday) vmonth = self.parse_month(vmonth) vdate = guesser.guess_date(vday, vmonth) detail.drop_tree() raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip() credit = CleanText().filter(cols[self.COL_CREDIT]) debit = CleanText().filter(cols[self.COL_DEBIT]) if end_of_period is not None and datetime.date.today() < end_of_period: t._is_coming = True else: t._is_coming = False t.date = t.rdate = date t.vdate = vdate t.raw = re.sub(r'[ ]+', ' ', raw) t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip() t.amount = parse_decimal(credit or debit) * (1 if credit else -1) if t.raw in self.browser.SUMMARY_CARD_LABEL: t.type = t.TYPE_CARD_SUMMARY elif t.amount > 0: t.type = t.TYPE_ORDER else: t.date = end_of_period t.type = t.TYPE_DEFERRED_CARD yield t
def get_history(self, account): transactions = [] if not account._link_id: raise NotImplementedError() if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: if self.two_cards_page: # In this case, you need to return to the page where the iter account get the cards information # Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same account._referer.go(subbank=self.currentSubBank) if account._secondpage: self.location(self.page.get_second_page_link()) # Check if '000000xxxxxx0000' card have an annual history self.location(account._link_id) # The history of the card is available for 1 year with 1 month per page # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions urlstogo = self.page.get_links() self.location(account._link_id) half_history = 'firstHalf' for url in urlstogo: transactions = [] self.location(url) if 'GoMonthPrecedent' in url: # To reach the 6 last month of history you need to change this url parameter # Moreover we are on a transition page where we see the 6 next month (no scrapping here) half_history = 'secondHalf' else: history = self.page.get_history() self.tr_date = self.page.get_date() amount_summary = self.page.get_amount_summary() if self.page.has_more_operations(): for i in range(1, 100): # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) data = { '_FID_DoAddElem': '', '_wxf2_cc': 'fr-FR', '_wxf2_pmode': 'Normal', '_wxf2_pseq': i, '_wxf2_ptarget': 'C:P:updPan', 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', 'Data_ServiceListDatas_CurrentType': 'MyCards', } if 'fid=GoMonth&mois=' in self.url: m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) if m: m = m.group(1) self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data) else: self.location(self.url, data=data) if not self.page.has_more_operations_xml(): history = self.page.iter_history_xml(date=self.tr_date) # We are now with an XML page with all the transactions of the month break else: history = self.page.get_history(date=self.tr_date) for tr in history: # For regrouped transaction, we have to go through each one to get details if tr._regroup: self.location(tr._regroup) for tr2 in self.page.get_tr_merged(): tr2._is_coming = tr._is_coming tr2.date = self.tr_date transactions.append(tr2) else: transactions.append(tr) if transactions and self.tr_date < datetime.today().date(): tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date tr.amount = amount_summary tr.date = tr.rdate = tr.vdate = self.tr_date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True transactions.append(tr) for tr in sorted_transactions(transactions): yield tr else: # need to refresh the months select if account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): for tr in self.list_operations(account._link_id, account): transactions.append(tr) coming_link = self.page.get_coming_link() if self.operations.is_here() else None if coming_link is not None: for tr in self.list_operations(coming_link, account): transactions.append(tr) deferred_date = None cards = ([page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): if tr._to_delete: # Delete main transaction when subtransactions exist continue if hasattr(tr, '_deferred_date') and (not deferred_date or tr._deferred_date < deferred_date): deferred_date = tr._deferred_date if tr.date >= datetime.now(): tr._is_coming = True elif hasattr(account, '_card_pages'): card_trs.append(tr) transactions.append(tr) if card_trs: transactions.extend(self.get_monthly_transactions(card_trs)) if deferred_date is not None: # set deleted for card_summary for tr in transactions: tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and deferred_date.month <= tr.date.month and not hasattr(tr, '_is_manualsum')) for tr in sorted_transactions(transactions): yield tr
def get_history(self, account): transactions = [] if not account._link_id: raise NotImplementedError() if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: if self.two_cards_page: # In this case, you need to return to the page where the iter account get the cards information # Indeed, for the same position of card in the two pages the url, headers and parameters are exactly the same account._referer.go(subbank=self.currentSubBank) if account._secondpage: self.location(self.page.get_second_page_link()) # Check if '000000xxxxxx0000' card have an annual history self.location(account._link_id) # The history of the card is available for 1 year with 1 month per page # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions urlstogo = self.page.get_links() self.location(account._link_id) half_history = 'firstHalf' for url in urlstogo: transactions = [] self.location(url) if 'GoMonthPrecedent' in url: # To reach the 6 last month of history you need to change this url parameter # Moreover we are on a transition page where we see the 6 next month (no scrapping here) half_history = 'secondHalf' else: history = self.page.get_history() self.tr_date = self.page.get_date() amount_summary = self.page.get_amount_summary() if self.page.has_more_operations(): for i in range(1, 100): # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) data = { '_FID_DoAddElem': '', '_wxf2_cc': 'fr-FR', '_wxf2_pmode': 'Normal', '_wxf2_pseq': i, '_wxf2_ptarget': 'C:P:updPan', 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', 'Data_ServiceListDatas_CurrentType': 'MyCards', } if 'fid=GoMonth&mois=' in self.url: m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) if m: m = m.group(1) self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data) else: self.location(self.url, data=data) if not self.page.has_more_operations_xml(): history = self.page.iter_history_xml(date=self.tr_date) # We are now with an XML page with all the transactions of the month break else: history = self.page.get_history(date=self.tr_date) for tr in history: # For regrouped transaction, we have to go through each one to get details if tr._regroup: self.location(tr._regroup) for tr2 in self.page.get_tr_merged(): tr2._is_coming = tr._is_coming tr2.date = self.tr_date transactions.append(tr2) else: transactions.append(tr) if transactions and self.tr_date < datetime.today().date(): tr = FrenchTransaction() tr.raw = tr.label = "RELEVE CARTE %s" % self.tr_date tr.amount = amount_summary tr.date = tr.rdate = tr.vdate = self.tr_date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False tr._is_manualsum = True transactions.append(tr) for tr in sorted_transactions(transactions): yield tr else: # need to refresh the months select if account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): for tr in self.list_operations(account._link_id, account): transactions.append(tr) coming_link = self.page.get_coming_link() if self.operations.is_here() else None if coming_link is not None: for tr in self.list_operations(coming_link, account): transactions.append(tr) differed_date = None cards = ([page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): if tr._to_delete: # Delete main transaction when subtransactions exist continue if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date): differed_date = tr._differed_date if tr.date >= datetime.now(): tr._is_coming = True elif hasattr(account, '_card_pages'): card_trs.append(tr) transactions.append(tr) if card_trs: transactions.extend(self.get_monthly_transactions(card_trs)) if differed_date is not None: # set deleted for card_summary for tr in transactions: tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and differed_date.month <= tr.date.month and not hasattr(tr, '_is_manualsum')) for tr in sorted_transactions(transactions): yield tr
def get_history(self, currency): # checking if the card is still valid if self.doc.xpath('//div[@id="errorbox"]'): return # adding a time delta because amex have hard time to put the date in a good interval beginning_date = self.get_beginning_debit_date() - datetime.timedelta( days=360) end_date = self.get_end_debit_date() guesser = ChaoticDateGuesser(beginning_date, end_date) # Since the site doesn't provide the debit_date, # we just use the date of beginning of the previous period. # If this date + 1 month is greater than today's date, # then the transaction is coming end_of_period = None previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]', default=None)(self.doc) if previous_date: end_of_period = ( parse_french_date(' '.join(previous_date.split()[1:4])) + relativedelta(months=1)).date() else: previous_date = CleanText( '//select[@id="viewPeriod"]/option[@selected]', default=None)(self.doc) if previous_date: end_of_period = parse_french_date(' '.join( previous_date.split()[:3])) + relativedelta( days=-1) + relativedelta(months=1) end_of_period = end_of_period.date() for tr in reversed( self.doc.xpath( '//div[@id="txnsSection"]//tbody/tr[@class="tableStandardText"]' )): cols = tr.findall('td') t = Transaction() day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1) day = int(day) month = self.parse_month(month) date = guesser.guess_date(day, month) vdate = None try: detail = cols[self.COL_TEXT].xpath( './div[has-class("hiddenROC")]')[0] except IndexError: pass else: m = re.search(r' (\d{2} \D{3,4})', (' '.join( [txt.strip() for txt in detail.itertext()])).strip()) if m: vday, vmonth = m.group(1).strip().split(' ') vday = int(vday) vmonth = self.parse_month(vmonth) vdate = guesser.guess_date(vday, vmonth) detail.drop_tree() raw = (' '.join([ txt.strip() for txt in cols[self.COL_TEXT].itertext() ])).strip() credit = CleanText().filter(cols[self.COL_CREDIT]) debit = CleanText().filter(cols[self.COL_DEBIT]) if end_of_period is not None and datetime.date.today( ) < end_of_period: t._is_coming = True else: t._is_coming = False t.date = t.rdate = date t.vdate = vdate t.raw = re.sub(r'[ ]+', ' ', raw) t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip() t.amount = parse_decimal(credit or debit) * (1 if credit else -1) if t.amount > 0: t.type = t.TYPE_ORDER else: t.date = end_of_period t.type = t.TYPE_DEFERRED_CARD yield t