def make_xact(self, atts): xact_pathlist = self.__pathlist[:] xact_pathlist.append('Xact') atts_copy = atts.copy() currentXact = Xact() dp = DateParser() currentXact.date = dp.date_from_ymd(atts['date']) currentXact.m = currentXact.date.month currentXact.y = currentXact.date.year if (self._current_statement.maxdate == None) or ( currentXact.date > self._current_statement.maxdate): self._current_statement.maxdate = currentXact.date if (self._current_statement.mindate == None) or ( currentXact.date < self._current_statement.mindate): self._current_statement.mindate = currentXact.date self._current_statement.xactcount = self._current_statement.xactcount + 1 if atts['type'] == 'Debit': self._current_statement.debits += atts['amount'] else: self._current_statement.credits -= atts['amount'] self.__last_xact = currentXact self.__xactList.append(atts_copy)
def __make_new_statement(self): self.__xactList = [] logging.info("path - " + str(self.__pathlist)) statement_path = self.__pathlist[:] statement_path.append('Statement') if self.fixedStatement: sync_stat_key = 'scur' + str(self.__account._scur_count) logging.info('Key to find = ' + sync_stat_key) statement_path.append(sync_stat_key) satts = {} dp = DateParser() satts['date'] = dp.ymd_from_date(dp.todate()) satts['display'] = self.__filename satts['ftype'] = self.__type satts['_fixedKey'] = self.fixedStatement #true or false satts['rawdata'] = str(self.__body) self._current_statement = Statement() logging.info('Got a statement with hash - ' + self._current_statement.hash) self.facade._statkey = self._current_statement
def make_xact(self, atts): xact_pathlist = self.__pathlist[:] xact_pathlist.append('Xact') atts_copy = atts.copy() currentXact = Xact(); dp = DateParser() currentXact.date = dp.date_from_ymd(atts['date']) currentXact.m = currentXact.date.month currentXact.y = currentXact.date.year if (self._current_statement.maxdate == None) or (currentXact.date > self._current_statement.maxdate): self._current_statement.maxdate = currentXact.date if (self._current_statement.mindate == None) or ( currentXact.date < self._current_statement.mindate): self._current_statement.mindate = currentXact.date self._current_statement.xactcount = self._current_statement.xactcount + 1 if atts['type'] == 'Debit': self._current_statement.debits += atts['amount'] else: self._current_statement.credits -= atts['amount'] self.__last_xact = currentXact self.__xactList.append(atts_copy)
def set_current_balance(self, curBalance): if self.fixedStatement: dp = DateParser() self.__old_synchbal = self.__account.synchbal self.__old_synched = self.__account.synched self.__account.synchbal = curBalance self.__account.synchdate = dp.todate() self.__account.synched = True
def _processNormAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug('Norm ac path - ' + str(account_path) + ' - end' ) try: if account_path != "": # delete existing current xactions logging.debug('Processing :) norm ' ) builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[self.current_statement] # we know this is not a credit card isCCard = False # get a fixed balance somewhere?? # balance passed in for RBS # set up our statement self.statementbuilder.make_recent_dif_statement('RBS-recent', 'Scraper', None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find('table', attrs={'class' : 'ItemTable'}) if x_table == None: # could easily be no transactions logging.debug(" No xtable ======>") if x_table != None: x_body = x_table.find('tbody') inputs = x_body.findAll('tr') # build the post values up for rows in inputs: atts = {} vals = rows.findAll('td') if vals: cash = '' for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: logging.debug("date ======> " + data) dp = DateParser() try: atts['date'] = dp.ymd_from_date(dp.date_from_small(data)) except: atts['date'] == '' if i == 1: if data == 'ATM': cash = 'CASH - ' if i == 2: if data != "": extra = "" datebit = "" parts = data.split(',') if len(parts) > 1: # match RBS dates - a.la. 8062 14APR11 if re.match('\d{4}\s\d\d[A-Z]{3}\d\d', parts[0]) != None: datebit = parts[0][0:4] + ' ' + parts[0][5:7] + ' ' + parts[0][7:10] # remember pretty_display strips out any words containing a sequence of 3 or more numbers parts = parts[1:] if len(parts) > 1: extra = parts[-1] parts = parts[0:-1] data = ' '.join(parts) disp = (cash + data).strip() atts['display'] = " ".join(disp.split()) atts['extradisplay'] = " ".join( (extra + " " + datebit).split()) if i > 2: # the numbers if data != "" and data != '-': logging.debug('->' + data + '<-') amount = self.normalise_ammount(data) if i == 3: atts['amount'] = amount atts['type'] = 'Credit' if i == 4: atts['amount'] = amount atts['type'] = 'Debit' if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception('RBS parsing error - ' + str(e))
def _processCCAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug('CC ac path - ' + str(account_path) + ' - end' ) try: if account_path != "": # delete existing current xactions logging.debug('Processing :) ' ) builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[self.current_statement] # we know this is not a credit card isCCard = True # get a fixed balance somewhere?? # passed in for RBS # set up our statement self.statementbuilder.make_recent_dif_statement('RBS-recent', 'Scraper', None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find('table', attrs={'class' : 'ItemTable'}) if x_table != None: x_body = x_table.find('tbody') inputs = x_body.findAll('tr') # build the post values up for rows in inputs: atts = {} vals = rows.findAll('td') if vals: datebit = '' for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: logging.debug("date ======> " + data) dp = DateParser() try: atts['date'] = dp.ymd_from_date(dp.date_from_small(data)) except: atts['date'] == '' if i == 1: datebit = data[:-5] if i == 2: if data != 'SALE': # only keep extra xact date for Sales datebit = '' if i == 3: if data != "": atts['display'] = " ".join(data.split()) atts['extradisplay'] = datebit if i > 3: # the numbers if data != "" and data != '-': amount = self.normalise_ammount(data) if i == 4: atts['amount'] = amount atts['type'] = 'Credit' if i == 5: atts['amount'] = amount atts['type'] = 'Debit' if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception('RBS parsing error - ' + str(e))
def _processNormAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug('Norm ac path - ' + str(account_path) + ' - end') try: if account_path != "": # delete existing current xactions logging.debug('Processing :) norm ') builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[ self.current_statement] # we know this is not a credit card isCCard = False # get a fixed balance somewhere?? # balance passed in for natwest # set up our statement self.statementbuilder.make_recent_dif_statement( 'NatWest-recent', 'Scraper', None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find('table', attrs={'class': 'ItemTable'}) if x_table == None: # could easily be no transactions logging.debug(" No xtable ======>") if x_table != None: x_body = x_table.find('tbody') inputs = x_body.findAll('tr') # build the post values up for rows in inputs: atts = {} vals = rows.findAll('td') if vals: cash = '' for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: dp = DateParser() try: atts['date'] = dp.ymd_from_date( dp.date_from_small(data)) except: atts['date'] == '' if i == 1: if data == 'ATM': cash = 'CASH - ' if i == 2: if data != "": extra = "" datebit = "" parts = data.split(',') if len(parts) > 1: # match natwest dates - a.la. 8062 14APR11 if re.match( '\d{4}\s\d\d[A-Z]{3}\d\d', parts[0]) != None: datebit = parts[0][ 0:4] + ' ' + parts[0][ 5:7] + ' ' + parts[0][ 7:10] # remember pretty_display strips out any words containing a sequence of 3 or more numbers parts = parts[1:] if len(parts) > 1: extra = parts[-1] parts = parts[0:-1] data = ' '.join(parts) disp = (cash + data).strip() atts['display'] = " ".join( disp.split()) atts['extradisplay'] = " ".join( (extra + " " + datebit).split()) if i > 2: # the numbers if data != "" and data != '-': amount = self.normalise_ammount(data) if i == 3: atts['amount'] = amount atts['type'] = 'Credit' if i == 4: atts['amount'] = amount atts['type'] = 'Debit' if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception('NatWest parsing error - ' + str(e))
def _processCCAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug('CC ac path - ' + str(account_path) + ' - end') try: if account_path != "": # delete existing current xactions logging.debug('Processing :) ') builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[ self.current_statement] # we know this is not a credit card isCCard = True # get a fixed balance somewhere?? # passed in for natwest # set up our statement self.statementbuilder.make_recent_dif_statement( 'NatWest-recent', 'Scraper', None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find('table', attrs={'class': 'ItemTable'}) if x_table != None: x_body = x_table.find('tbody') inputs = x_body.findAll('tr') # build the post values up for rows in inputs: atts = {} vals = rows.findAll('td') if vals: datebit = '' for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: dp = DateParser() try: atts['date'] = dp.ymd_from_date( dp.date_from_small(data)) except: atts['date'] == '' if i == 1: datebit = data[:-5] if i == 2: if data != 'SALE': # only keep extra xact date for Sales datebit = '' if i == 3: if data != "": atts['display'] = " ".join( data.split()).encode('utf8') atts['extradisplay'] = datebit.encode( 'utf8') if i > 3: # the numbers if data != "" and data != '-': amount = self.normalise_ammount(data) if i == 4: atts['amount'] = amount atts['type'] = 'Credit' if i == 5: atts['amount'] = amount atts['type'] = 'Debit' if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception('NatWest parsing error - ' + str(e))
def processAccount(self, acCount, acName, account_path, allofit): page = self.HexToByte(allofit['body']) # save this page self.output_page("account" + str(acCount) + ".html", page) soup = BeautifulSoup(page) logging.debug('ac path - ' + str(account_path) + ' - end') if account_path != "": # delete existing current xactions logging.debug('Processing :) ') self.statementbuilder = StatementBuilder(self.facade, account_path, self.token) # need to get last statement and make a new one every time self.statementbuilder.make_recent_dif_statement( 'Fd-recent', 'Scraper', None) #TODO change this isVisa = False loginform = soup.find( 'input', attrs={'name': 'cmd_sort_referenceAscending'}) if loginform != None: isVisa = True bal_tables = soup.findAll( 'table', attrs={'class': 'fdTableBackgroundOne'}) balance_table = bal_tables[2] if balance_table <> None: vals = balance_table.findAll('td') if vals: bal = vals[1].text data = bal.replace('£', u'£') data = data.strip(u'£') if data[-1] == 'D': data = data.replace('DB', '') data = data.replace('D', '') lastbal = int(float(data) * 100) firstbal = 0 - lastbal else: data = data.replace('CR', '') data = data.replace('C', '') firstbal = int(float(data) * 100) self.statementbuilder.set_current_balance(firstbal) logging.debug( "-----------------------------*******---------------------") if isVisa: logging.debug("found visa --") acTable = soup.find('table', attrs={'class': 'fdStatTable'}) # if no table then no new data afaik if acTable != None: datarows = acTable.findAll('tr') next = False # build the post values up atts = {} isFirst = True firstbal = 0 firstdate = "" lastbal = 0 lastdate = "" doBalance = False dp = DateParser() for rows in datarows: vals = rows.findAll('td') if vals: for i, val in enumerate(vals): if val.text: data = val.text.strip() data = unescape(data) data = unicode(data) else: data = "" if data != " ": data = data.replace(' ', '') if i == 0: if data != "": try: lastdate = dp.ymd_from_date( dp.date_from_dmy(data, '/')) except: logging.warn( "Invalid FD date format - probably no transactions" ) return if firstdate == "": firstdate = lastdate atts['date'] = lastdate if (i == 1 and not isVisa) or (i == 2 and isVisa): atts['display'] = data[0:19] atts['extradisplay'] = data[19:] if (i == 2 and not isVisa) or (i == 3 and isVisa): if data != "": data = data.strip(u'£') data = data.strip(u'D') data = data.strip(u'B') if data == '': atts['amount'] = 0 else: atts['amount'] = int( float(data) * 100) atts['type'] = 'Debit' if (i == 3 and not isVisa) or (i == 4 and isVisa): if data != "": data = data.strip(u'£') data = data.strip(u'C') data = data.strip(u'R') if data == '': atts['amount'] = 0 else: atts['amount'] = int( float(data) * 100) atts['type'] = 'Credit' if not isVisa: if i == 4: data = data.strip(u'£') if data != "": lastbal = int(float(data) * 100) if isFirst: isFirst = False firstbal = lastbal doBalance = True if i == 5: if doBalance: doBalance = False if data == "D": firstbal = 0 - firstbal self.statementbuilder.set_current_balance( firstbal) self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1
def processAccount(self, acCount, acName, account_path, allofit): page = self.HexToByte( allofit['body']) # save this page self.output_page("account" + str(acCount) + ".html", page) soup = BeautifulSoup(page) logging.debug('ac path - ' + str(account_path) + ' - end' ) if account_path != "": # delete existing current xactions logging.debug('Processing :) ' ) self.statementbuilder = StatementBuilder(self.facade, account_path, self.token) # need to get last statement and make a new one every time self.statementbuilder.make_recent_dif_statement('Fd-recent', 'Scraper', None) #TODO change this isVisa = False loginform=soup.find('input', attrs={'name' : 'cmd_sort_referenceAscending'}) if loginform != None: isVisa = True bal_tables=soup.findAll('table', attrs={'class' : 'fdTableBackgroundOne'}) balance_table = bal_tables[2] if balance_table <> None: vals = balance_table.findAll('td') if vals: bal = vals[1].text data = bal.replace('£', u'£'); data = data.strip(u'£') if data[-1] == 'D': data = data.replace('DB','') data = data.replace('D','') lastbal = int( float(data) * 100 ) firstbal = 0 - lastbal else: data = data.replace('CR','') data = data.replace('C','') firstbal = int( float(data) * 100 ) self.statementbuilder.set_current_balance(firstbal) logging.debug("-----------------------------*******---------------------") if isVisa: logging.debug("found visa --") acTable=soup.find('table', attrs={'class' : 'fdStatTable'}) # if no table then no new data afaik if acTable != None: datarows=acTable.findAll('tr') next = False # build the post values up atts = {} isFirst = True firstbal = 0 firstdate = "" lastbal = 0 lastdate = "" doBalance = False dp = DateParser() for rows in datarows: vals = rows.findAll('td') if vals: for i, val in enumerate(vals): if val.text: data = val.text.strip() data = unescape(data) data = unicode(data) else: data = "" if data != " ": data = data.replace(' ','') if i == 0: if data != "": try: lastdate = dp.ymd_from_date(dp.date_from_dmy(data,'/')) except: logging.warn("Invalid FD date format - probably no transactions") return if firstdate == "": firstdate = lastdate atts['date'] = lastdate if (i == 1 and not isVisa) or (i == 2 and isVisa): atts['display'] = data[0:19] atts['extradisplay'] = data[19:] if (i == 2 and not isVisa) or (i == 3 and isVisa): if data != "": data = data.strip(u'£') data = data.strip(u'D') data = data.strip(u'B') if data == '': atts['amount'] = 0 else: atts['amount'] = int( float(data) * 100 ) atts['type'] = 'Debit' if (i == 3 and not isVisa) or (i == 4 and isVisa): if data != "": data = data.strip(u'£') data = data.strip(u'C') data = data.strip(u'R') if data == '': atts['amount'] = 0 else: atts['amount'] = int( float(data) * 100 ) atts['type'] = 'Credit' if not isVisa: if i == 4: data = data.strip(u'£') if data != "": lastbal = int( float(data) * 100 ) if isFirst: isFirst = False firstbal = lastbal doBalance = True if i == 5: if doBalance: doBalance = False if data == "D": firstbal = 0 - firstbal self.statementbuilder.set_current_balance(firstbal) self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1
def _processCCAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug("CC ac path - " + str(account_path) + " - end") try: if account_path != "": # delete existing current xactions logging.debug("Processing :) ") builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[self.current_statement] # we know this is not a credit card isCCard = True # get a fixed balance somewhere?? # passed in for natwest # set up our statement self.statementbuilder.make_recent_dif_statement("NatWest-recent", "Scraper", None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find("table", attrs={"class": "ItemTable"}) if x_table != None: x_body = x_table.find("tbody") inputs = x_body.findAll("tr") # build the post values up for rows in inputs: atts = {} vals = rows.findAll("td") if vals: datebit = "" for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: dp = DateParser() try: atts["date"] = dp.ymd_from_date(dp.date_from_small(data)) except: atts["date"] == "" if i == 1: datebit = data[:-5] if i == 2: if data != "SALE": # only keep extra xact date for Sales datebit = "" if i == 3: if data != "": atts["display"] = " ".join(data.split()).encode("utf8") atts["extradisplay"] = datebit.encode("utf8") if i > 3: # the numbers if data != "" and data != "-": amount = self.normalise_ammount(data) if i == 4: atts["amount"] = amount atts["type"] = "Credit" if i == 5: atts["amount"] = amount atts["type"] = "Debit" if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception("NatWest parsing error - " + str(e))
def _processNormAccount(self, raw, account_path, balance): soup = BeautifulSoup(raw) logging.debug("Norm ac path - " + str(account_path) + " - end") try: if account_path != "": # delete existing current xactions logging.debug("Processing :) norm ") builder = StatementBuilder(self.facade, account_path, self.token) self.statementlist.append(builder) self.statementbuilder = self.statementlist[self.current_statement] # we know this is not a credit card isCCard = False # get a fixed balance somewhere?? # balance passed in for natwest # set up our statement self.statementbuilder.make_recent_dif_statement("NatWest-recent", "Scraper", None) # now set the final balance logging.debug("Balance - - - - - - - > " + str(balance)) self.statementbuilder.set_current_balance(balance) # now find all the recent transactions x_table = soup.find("table", attrs={"class": "ItemTable"}) if x_table == None: # could easily be no transactions logging.debug(" No xtable ======>") if x_table != None: x_body = x_table.find("tbody") inputs = x_body.findAll("tr") # build the post values up for rows in inputs: atts = {} vals = rows.findAll("td") if vals: cash = "" for i, val in enumerate(vals): data = self.tidy_text(val.text) if i == 0: dp = DateParser() try: atts["date"] = dp.ymd_from_date(dp.date_from_small(data)) except: atts["date"] == "" if i == 1: if data == "ATM": cash = "CASH - " if i == 2: if data != "": extra = "" datebit = "" parts = data.split(",") if len(parts) > 1: # match natwest dates - a.la. 8062 14APR11 if re.match("\d{4}\s\d\d[A-Z]{3}\d\d", parts[0]) != None: datebit = parts[0][0:4] + " " + parts[0][5:7] + " " + parts[0][7:10] # remember pretty_display strips out any words containing a sequence of 3 or more numbers parts = parts[1:] if len(parts) > 1: extra = parts[-1] parts = parts[0:-1] data = " ".join(parts) disp = (cash + data).strip() atts["display"] = " ".join(disp.split()) atts["extradisplay"] = " ".join((extra + " " + datebit).split()) if i > 2: # the numbers if data != "" and data != "-": amount = self.normalise_ammount(data) if i == 3: atts["amount"] = amount atts["type"] = "Credit" if i == 4: atts["amount"] = amount atts["type"] = "Debit" if i == 5: self.statementbuilder.make_xact(atts) self.statementbuilder.put_statement() self.current_statement = self.current_statement + 1 except Exception, e: logging.exception("NatWest parsing error - " + str(e))
def synch_accounts(self, allofit, accountlist): user = '******' facade = Facade(user); step = allofit['step'] bankArray = allofit['credentials'] # badly worded 'credentials' - it is the list of banks with credentials - a list of one now! bank = bankArray[0] bankId = bank['bankId'] logging.info("found: - " + bankId) scc = self.getScraper(bankId, bank['credentials']) # correctly worded credentials matching_accounts = {} if scc: # find accounts that use these credentials for ac in accountlist: if ac['bankname'] == bankId: # it should do account_url = ['Person', user, 'Account', ac['keyname']] matching_accounts[ac['accountname']] = account_url # in case we are in some terible loop at least stop at 100 steps if step < 100: scrape_result = scc.getxactlist(facade, matching_accounts, 'unusedtoken', step, allofit) response = {} response['message'] = "good" response['bankid'] = bank['bankId'] response['request'] = scc.response if ("good" == scrape_result) or ("account list" == scrape_result): response['message'] = "good" elif "got account" == scrape_result: sb = scc.statementbuilder xacts = sb.getxactlist() bal = sb.getSynchBalance() id = sb.getSynchAccountID() if bal < 0: bal = 0 - bal sbalwhich = 'negative' else: sbalwhich = 'positive' strbal = str(bal) dp = DateParser() bankxact = {} bankxact["id"] = id bankxact["synchbal"] = strbal bankxact["synchbalwhich"] = sbalwhich bankxact["synchdate"] = dp.ymdhms_from_date(dp.todatetime()) bankxact["xacts"] = xacts response['bankxact'] = bankxact else: response['message'] = scrape_result else: response = {} response["message"] = "bank not supported" response["bankid"] = bank['bankId'] return response