def file_date(self, file): """Get the maximum date from the file. If a date can be extracted from the statement’s contents, return it here. This is useful for dated PDF statements… it’s often possible using regular expressions to grep out the date from a PDF converted to text. This allows the filing script to prepend a relevant date instead of using the date when the file was downloaded (the default). """ iconfig, has_header = normalize_config(self.config, file.head(-1), self.skip_lines) if Col.DATE in iconfig: reader = iter(csv.reader(open(file.name))) for _ in range(self.skip_lines): next(reader) if has_header: next(reader) max_date = None for row in reader: if not row: continue if row[0].startswith('#'): continue date_str = row[iconfig[Col.DATE]] date = parse_date_liberally(date_str, self.dateutil_kwds) if max_date is None or date > max_date: max_date = date return max_date
def file_date(self, file): max_date = None for _, row in self.get_rows(file): parsed_date = parse_date_liberally(row.order_date) if max_date is None or parsed_date > max_date: max_date = parsed_date return max_date
def _get_price_for_date(self, ticker, date=None): time.sleep(TIME_DELAY) if date == None: date_string = "" else: date_string = date.strftime("%Y%m%d") url = BASE_URL_TEMPLATE.substitute(date=date_string, ticker=ticker) try: content = requests.get(url).content soup = BeautifulSoup(content, 'html.parser') table = soup.find('table', {'class': 'table'}) tr = table.findChildren('tr')[1] data = [td.text.strip() for td in tr.findChildren('td')] parsed_date = parse_date_liberally(data[0]) date = datetime(parsed_date.year, parsed_date.month, parsed_date.day, tzinfo=utc) price = D(data[4]) return source.SourcePrice(price, date, CURRENCY) except KeyError: raise CoinmarketcapError( "Invalid response from Coinmarketcap: {}".format( repr(content))) except AttributeError: raise CoinmarketcapError( "Invalid response from Coinmarketcap: {}".format( repr(content)))
def file_date(self, file): "Get the maximum date from the file." iconfig, has_header = normalize_config( self.config, file.head(encoding=self.encoding), self.csv_dialect, self.skip_lines, ) if Col.DATE in iconfig: with open(file.name, encoding=self.encoding) as infile: reader = iter(csv.reader(infile, dialect=self.csv_dialect)) for _ in range(self.skip_lines): next(reader) if has_header: next(reader) max_date = None for row in reader: if not row: continue if row[0].startswith('#'): continue date_str = row[iconfig[Col.DATE]] date = parse_date_liberally(date_str, self.dateutil_kwds) if max_date is None or date > max_date: max_date = date return max_date
def get_transaction(self, row: Row, file_name, row_number): meta = data.new_metadata(file_name, row_number) postings = self.get_postings(row) if self.expense_account: postings.append(self.get_expense_posting(row)) t = data.Transaction( meta, parse_date_liberally(row.order_date), flags.FLAG_WARNING if len(postings) == 0 else flags.FLAG_OKAY, row.seller.strip(), row.description.strip(), self.tags, data.EMPTY_SET, # links postings) return t
def file_date(self, file): "Get the maximum date from the file." iconfig, has_header = normalize_config(self.config, file.head()) if Col.DATE in iconfig: reader = iter(csv.reader(open(file.name))) if has_header: next(reader) max_date = None for row in reader: if not row: continue date_str = row[iconfig[Col.DATE]] date = parse_date_liberally(date_str) if max_date is None or date > max_date: max_date = date return max_date
def _get_price_for_date(self, ticker, date=None): if date == None: date_string = "0" else: date_string = date.strftime("%Y%m%d") url = BASE_URL_TEMPLATE.substitute(ticker=ticker) try: content = requests.get(url).content content = content.split(b"=")[1] data = json.loads(content) price = 0 date_int = int(date_string) found_date = False if date_string != "0": for item in data: if item[0] == date_string or int(item[0]) > date_int: date = item[0] price = item[1] found_date = True break if not found_date: item = data[len(data) - 1] price = item[1] date = item[0] parsed_date = parse_date_liberally(date) date = datetime(parsed_date.year, parsed_date.month, parsed_date.day, tzinfo=utc) price = D(price) return source.SourcePrice(price, date, CURRENCY) except KeyError: raise CoinmarketcapError("Invalid response from 10jqka: {}".format( repr(content))) except AttributeError: raise CoinmarketcapError("Invalid response from 10jqka: {}".format( repr(content)))
def _get_price_for_date(self, ticker, date=None): if date == None: start_time = datetime.today().strftime('%Y-%m-%d') end_time = (datetime.today() + timedelta(days=1)).strftime('%Y-%m-%d') else: start_time = date.strftime('%Y-%m-%d') end_time = (date + timedelta(days=1)).strftime('%Y-%m-%d') data = { 'pjname': unquote(ticker.replace('_', '%')), 'erectDate': start_time, 'nothing': end_time, 'head': 'head_620.js', 'bottom': 'bottom_591.js' } try: content = requests.post(BASE_URL_TEMPLATE, data).content soup = BeautifulSoup(content, 'html.parser') table = soup.find('div', { 'class': 'BOC_main' }).findChildren('table')[0] tr = table.findChildren('tr')[1] data = [td.text.strip() for td in tr.findChildren('td')] parsed_date = parse_date_liberally(data[6]) date = datetime(parsed_date.year, parsed_date.month, parsed_date.day, tzinfo=utc) price = D(data[5]) / D(100) return source.SourcePrice(price, date, CURRENCY) except Exception as e: raise e except KeyError: raise BOCError("Invalid response from BOC: {}".format( repr(content))) except AttributeError: raise BOCError("Invalid response from BOC: {}".format( repr(content)))
def get_transaction(self, row: Row, file_name, row_number): meta = data.new_metadata( file_name, row_number, { 'receipt-url': RECEIPT_URL.format(row.order_id), 'payment-method': row.payment_instrument, }) postings = self.get_postings(row) if self.expense_account: postings.append(self.get_expense_posting(row)) t = data.Transaction( meta, parse_date_liberally(row.order_date), flags.FLAG_WARNING if len(postings) == 0 else flags.FLAG_OKAY, row.seller.strip(), row.description.strip(), self.tags, data.EMPTY_SET, # links postings) return t
def file_date(self, file): "Get the maximum date from the file." iconfig, has_header = normalize_config( self.config, file.contents(), self.skip_lines ) if Col.DATE in iconfig: reader = csv.reader(open(io.StringIO(strip_blank(file.contents())))) for _ in range(self.skip_lines): next(reader) if has_header: next(reader) max_date = None for row in reader: if not row: continue if row[0].startswith("#"): continue date_str = row[iconfig[Col.DATE]] date = parse_date_liberally(date_str, self.dateutil_kwds) if max_date is None or date > max_date: max_date = date return max_date
def get_transaction(self, row, file_name, row_number): self.assert_is_row(row) postings, categorizer_result = self.get_categorized_postings(row) if categorizer_result is None: # Make a dummy result to avoid having NoneType errors categorizer_result = CategorizerResult(self.account) if len(postings) != 2: flag = flags.FLAG_WARNING else: flag = categorizer_result.flag or flags.FLAG_OKAY return data.Transaction( # pylint: disable=not-callable data.new_metadata(file_name, row_number, self.get_extra_metadata(row)), parse_date_liberally(row.date), flag, categorizer_result.payee, categorizer_result.narration or row.description, self.tags | categorizer_result.tags, data.EMPTY_SET, # links postings)
def extract(self, file): entries = [] # Normalize the configuration to fetch by index. iconfig, has_header = normalize_config(self.config, file.head()) # Skip header, if one was detected. reader = iter(csv.reader(open(file.name))) if has_header: next(reader) def get(row, ftype): return row[iconfig[ftype]] if ftype in iconfig else None # Parse all the transactions. first_row = last_row = None for index, row in enumerate(reader, 1): if not row: continue # If debugging, print out the rows. if self.debug: print(row) if first_row is None: first_row = row last_row = row # Extract the data we need from the row, based on the configuration. date = get(row, Col.DATE) txn_date = get(row, Col.TXN_DATE) payee = get(row, Col.PAYEE) fields = filter(None, [ get(row, field) for field in (Col.NARRATION1, Col.NARRATION2, Col.NARRATION3) ]) narration = ' -- '.join(fields) tag = get(row, Col.TAG) tags = {tag} if tag is not None else data.EMPTY_SET # Create a transaction and add it to the list of new entries. meta = data.new_metadata(file.name, index) if txn_date is not None: meta['txndate'] = parse_date_liberally(txn_date) date = parse_date_liberally(date) txn = data.Transaction(meta, date, self.FLAG, payee, narration, tags, data.EMPTY_SET, []) entries.append(txn) amount_debit, amount_credit = get_amounts(iconfig, row) for amount in [amount_debit, amount_credit]: if amount is None: continue units = Amount(amount, self.currency) txn.postings.append( data.Posting(self.account, units, None, None, None, None)) # Figure out if the file is in ascending or descending order. first_date = parse_date_liberally(get(first_row, Col.DATE)) last_date = parse_date_liberally(get(last_row, Col.DATE)) is_ascending = first_date < last_date # Parse the final balance. if Col.BALANCE in iconfig: # Choose between the first or the last row based on the date. row = last_row if is_ascending else first_row date = parse_date_liberally(get( row, Col.DATE)) + datetime.timedelta(days=1) balance = D(get(row, Col.BALANCE)) meta = data.new_metadata(file.name, index) entries.append( data.Balance(meta, date, self.account, Amount(balance, self.currency), None, None)) return entries
def get_transaction( self, row: Row, file_name, row_number, existing_entries=None): flag = flags.FLAG_WARNING row = row._replace(**{ 'visit_date': parse_date_liberally(row.visit_date), 'amount_billed': data.Amount( D(row.amount_billed.replace("$", "")), self.currency), 'amount_deductible': data.Amount( D(row.amount_deductible.replace("$", "")), self.currency), 'amount_plan_paid': data.Amount( D(row.amount_plan_paid.replace("$", "")), self.currency), 'amount_plan_discount': data.Amount( D(row.amount_plan_discount.replace("$", "")), self.currency), 'amount_responsible': data.Amount( D(row.amount_responsible.replace("$", "")), self.currency), 'amount_paid_at_visit': data.Amount( D(row.amount_paid_at_visit.replace("$", "")), self.currency), 'amount_owed': data.Amount( D(row.amount_owed.replace("$", "")), self.currency), }) metadata = { 'claim-number': row.claim_number, 'claim-type': row.claim_type, 'patient': row.patient, 'provider': row.provider, 'visit-date': row.visit_date, } postings = [] if row.amount_plan_paid: postings.append(data.Posting( account=self.reimbursement_account, units=row.amount_plan_paid, cost=None, price=None, flag=None, meta={})) if row.amount_plan_discount: postings.append(data.Posting( account=self.discount_account, units=row.amount_plan_discount, cost=None, price=None, flag=None, meta={})) if not postings: return None return data.Transaction( # pylint: disable=not-callable data.new_metadata(file_name, row_number, metadata), parse_date_liberally(row.date), flag, "United Healthcare", self.narration_format.format(**row._asdict()), self.tags, set([self.link_format.format(**row._asdict())]), postings)
def test_parse_date(self): self.assertEqual(datetime.date(2014, 12, 7), date_utils.parse_date_liberally('12/7/2014')) self.assertEqual(datetime.date(2014, 12, 7), date_utils.parse_date_liberally('7-Dec-2014'))
def extract(self, file, existing_entries=None): account = self.file_account(file) entries = [] # Normalize the configuration to fetch by index. iconfig, has_header = normalize_config(self.config, file.head(), self.csv_dialect, self.skip_lines) reader = iter( csv.reader(open(file.name, encoding=self.encoding), dialect=self.csv_dialect)) # Skip garbage lines for _ in range(self.skip_lines): next(reader) # Skip header, if one was detected. if has_header: next(reader) def get(row, ftype): try: return row[iconfig[ftype]] if ftype in iconfig else None except IndexError: # FIXME: this should not happen return None # Parse all the transactions. first_row = last_row = None for index, row in enumerate(reader, 1): if not row: continue if row[0].startswith('#'): continue # If debugging, print out the rows. if self.debug: print(row) if first_row is None: first_row = row last_row = row # Extract the data we need from the row, based on the configuration. date = get(row, Col.DATE) txn_date = get(row, Col.TXN_DATE) txn_time = get(row, Col.TXN_TIME) payee = get(row, Col.PAYEE) if payee: payee = payee.strip() fields = filter(None, [ get(row, field) for field in (Col.NARRATION1, Col.NARRATION2, Col.NARRATION3) ]) narration = self.narration_sep.join( field.strip() for field in fields).replace('\n', '; ') tag = get(row, Col.TAG) tags = {tag} if tag else data.EMPTY_SET link = get(row, Col.REFERENCE_ID) links = {link} if link else data.EMPTY_SET last4 = get(row, Col.LAST4) balance = get(row, Col.BALANCE) # Create a transaction meta = data.new_metadata(file.name, index) if txn_date is not None: meta['date'] = parse_date_liberally(txn_date, self.dateutil_kwds) if txn_time is not None: meta['time'] = str(dateutil.parser.parse(txn_time).time()) if balance is not None: meta['balance'] = self.parse_amount(balance) if last4: last4_friendly = self.last4_map.get(last4.strip()) meta['card'] = last4_friendly if last4_friendly else last4 date = parse_date_liberally(date, self.dateutil_kwds) txn = data.Transaction(meta, date, self.FLAG, payee, narration, tags, links, []) # Attach one posting to the transaction amount_debit, amount_credit = self.get_amounts( iconfig, row, False, self.parse_amount) # Skip empty transactions if amount_debit is None and amount_credit is None: continue for amount in [amount_debit, amount_credit]: if amount is None: continue if self.invert_sign: amount = -amount units = Amount(amount, self.currency) txn.postings.append( data.Posting(account, units, None, None, None, None)) # Attach the other posting(s) to the transaction. txn = self.call_categorizer(txn, row) # Add the transaction to the output list entries.append(txn) # Figure out if the file is in ascending or descending order. first_date = parse_date_liberally(get(first_row, Col.DATE), self.dateutil_kwds) last_date = parse_date_liberally(get(last_row, Col.DATE), self.dateutil_kwds) is_ascending = first_date < last_date # Reverse the list if the file is in descending order if not is_ascending: entries = list(reversed(entries)) # Add a balance entry if possible if Col.BALANCE in iconfig and entries: entry = entries[-1] date = entry.date + datetime.timedelta(days=1) balance = entry.meta.get('balance', None) if balance is not None: meta = data.new_metadata(file.name, index) entries.append( data.Balance(meta, date, account, Amount(balance, self.currency), None, None)) # Remove the 'balance' metadata. for entry in entries: entry.meta.pop('balance', None) return entries
def parse_date(string, frmt=None): """Parse date from string.""" if frmt is None: return parse_date_liberally(string) return datetime.datetime.strptime(string, frmt).date()
def get_extra_metadata(self, row): self.assert_is_row(row) return { 'purchase-date': parse_date_liberally(row.transaction_date), }
def extract(self, file): """Parse and extract Beanount contents from the given file. This is called to attempt to extract some Beancount directives from the file contents. It must create the directives by instantiating the objects defined in beancount.core.data and return them. This function is used by bean-extract tool. Returns: A list of beancount.core.data object, and each of them can be converted into a command-line accounting. """ entries = [] # Normalize the configuration to fetch by index. iconfig, has_header = normalize_config(self.config, file.head(-1), self.skip_lines) reader = iter(csv.reader(open(file.name), dialect=self.csv_dialect)) # Skip garbage lines for _ in range(self.skip_lines): next(reader) # Skip header, if one was detected. if has_header: next(reader) def get(row, ftype): try: return row[iconfig[ftype]] if ftype in iconfig else None except IndexError: # FIXME: this should not happen return None # Parse all the transactions. first_row = last_row = None for index, row in enumerate(reader, 1): if not row: continue if row[0].startswith('#'): continue if row[0].startswith("-----------"): break # If debugging, print out the rows. if self.debug: print(row) if first_row is None: first_row = row last_row = row # Extract the data we need from the row, based on the configuration. status = get(row, Col.STATUS) # When the status is CLOSED, the transaction where money had not been paid should be ignored. if isinstance(status, str) and status == self.close_flag: continue # Distinguish debit or credit DRCR_status = get_debit_or_credit_status(iconfig, row, self.DRCR_dict) date = get(row, Col.DATE) txn_date = get(row, Col.TXN_DATE) txn_time = get(row, Col.TXN_TIME) account = get(row, Col.ACCOUNT) tx_type = get(row, Col.TYPE) tx_type = tx_type or "" payee = get(row, Col.PAYEE) if payee: payee = payee.strip() fields = filter(None, [ get(row, field) for field in (Col.NARRATION1, Col.NARRATION2) ]) narration = self.narration_sep.join(field.strip() for field in fields) remark = get(row, Col.REMARK) tag = get(row, Col.TAG) tags = {tag} if tag is not None else data.EMPTY_SET last4 = get(row, Col.LAST4) balance = get(row, Col.BALANCE) # Create a transaction meta = data.new_metadata(file.name, index) if txn_date is not None: meta['date'] = parse_date_liberally(txn_date, self.dateutil_kwds) if txn_time is not None: meta['time'] = str(dateutil.parser.parse(txn_time).time()) if balance is not None: meta['balance'] = D(balance) if last4: last4_friendly = self.last4_map.get(last4.strip()) meta['card'] = last4_friendly if last4_friendly else last4 date = parse_date_liberally(date, self.dateutil_kwds) # flag = flags.FLAG_WARNING if DRCR_status == Debit_or_credit.UNCERTAINTY else self.FLAG txn = data.Transaction(meta, date, self.FLAG, payee, "{}({})".format(narration, remark), tags, data.EMPTY_SET, []) # Attach one posting to the transaction amount_debit, amount_credit = get_amounts(iconfig, row, DRCR_status) # Skip empty transactions if amount_debit is None and amount_credit is None: continue for amount in [amount_debit, amount_credit]: if amount is None: continue units = Amount(amount, self.currency) # Uncertain transaction, maybe capital turnover if DRCR_status == Drcr.UNCERTAINTY: if remark and len(remark.split("-")) == 2: remarks = remark.split("-") primary_account = mapping_account( self.assets_account, remarks[1]) secondary_account = mapping_account( self.assets_account, remarks[0]) txn.postings.append( data.Posting(primary_account, -units, None, None, None, None)) txn.postings.append( data.Posting(secondary_account, None, None, None, None, None)) else: txn.postings.append( data.Posting(self.default_account, units, None, None, None, None)) # Debit or Credit transaction else: # Primary posting # Rename primary account if remark field matches one of assets account primary_account = mapping_account(self.assets_account, remark) txn.postings.append( data.Posting(primary_account, units, None, None, None, None)) # Secondary posting # Rename secondary account by credit account or debit account based on DRCR status payee_narration = payee + narration _account = self.credit_account if DRCR_status == Drcr.CREDIT else self.debit_account secondary_account = mapping_account( _account, payee_narration) # secondary_account = _account[DEFAULT] # for key in _account.keys(): # if key == DEFAULT: # continue # if re.search(key, payee_narration): # secondary_account = _account[key] # break txn.postings.append( data.Posting(secondary_account, None, None, None, None, None)) # Attach the other posting(s) to the transaction. if isinstance(self.categorizer, collections.Callable): txn = self.categorizer(txn) # Add the transaction to the output list entries.append(txn) # Figure out if the file is in ascending or descending order. first_date = parse_date_liberally(get(first_row, Col.DATE), self.dateutil_kwds) last_date = parse_date_liberally(get(last_row, Col.DATE), self.dateutil_kwds) is_ascending = first_date < last_date # Reverse the list if the file is in descending order if not is_ascending: entries = list(reversed(entries)) # Add a balance entry if possible if Col.BALANCE in iconfig and entries: entry = entries[-1] date = entry.date + datetime.timedelta(days=1) balance = entry.meta.get('balance', None) if balance: meta = data.new_metadata(file.name, index) entries.append( data.Balance(meta, date, self.default_account, Amount(balance, self.currency), None, None)) # Remove the 'balance' metadta. for entry in entries: entry.meta.pop('balance', None) return entries
def extract(self, file, existing_entries=None): entries = [] # Normalize the configuration to fetch by index. iconfig, has_header = normalize_config( self.config, file.contents(), self.skip_lines ) reader = csv.reader(io.StringIO(strip_blank(file.contents()))) # Skip garbage lines for _ in range(self.skip_lines): next(reader) # Skip header, if one was detected. if has_header: next(reader) def get(row, ftype): return row[iconfig[ftype]] if ftype in iconfig else None # Parse all the transactions. first_row = last_row = None for index, row in enumerate(reader, 1): if not row: continue if row[0].startswith("#"): continue if row[0].startswith("-----------"): break if first_row is None: first_row = row last_row = row # Extract the data we need from the row, based on the configuration. status = get(row, Col.STATUS) date = get(row, Col.DATE) txn_date = get(row, Col.TXN_DATE) txn_time = get(row, Col.TXN_TIME) account = get(row, Col.ACCOUNT) tx_type = get(row, Col.TYPE) tx_type = tx_type or "" payee = get(row, Col.PAYEE) if payee: payee = payee.strip() narration = get(row, Col.NARRATION) if narration: narration = narration.strip() # Create a transaction meta = data.new_metadata(file.name, index) if txn_date is not None: meta["date"] = parse_date_liberally(txn_date) if txn_time is not None: meta["time"] = str(dateutil.parser.parse(txn_time).time()) date = parse_date_liberally(date) txn = data.Transaction( meta, date, self.FLAG, payee, narration, data.EMPTY_SET, data.EMPTY_SET, [], ) # Attach one posting to the transaction drcr = get_DRCR_status(iconfig, row, self.drcr_dict) amount_debit, amount_credit = get_amounts(iconfig, row, drcr) # Skip empty transactions if amount_debit is None and amount_credit is None: continue for amount in [amount_debit, amount_credit]: if amount is None: continue units = Amount(amount, self.currency) if drcr == Drcr.UNCERTAINTY: if account and len(account.split("-")) == 2: accounts = account.split("-") primary_account = mapping_account( self.account_map["assets"], accounts[1] ) secondary_account = mapping_account( self.account_map["assets"], accounts[0] ) txn.postings.append( data.Posting( primary_account, -units, None, None, None, None ) ) txn.postings.append( data.Posting( secondary_account, None, None, None, None, None ) ) else: txn.postings.append( data.Posting( self.account_map["assets"]["DEFAULT"], units, None, None, None, None, ) ) else: primary_account = mapping_account( self.account_map["assets"], account ) txn.postings.append( data.Posting(primary_account, units, None, None, None, None) ) payee_narration = payee + narration account_map = self.account_map[ "credit" if drcr == Drcr.CREDIT and not ( self.refund_keyword and payee_narration.find(self.refund_keyword) != -1 ) else "debit" ] secondary_account = mapping_account( account_map, payee_narration + tx_type ) txn.postings.append( data.Posting(secondary_account, None, None, None, None, None) ) # Add the transaction to the output list logging.debug(txn) entries.append(txn) # Figure out if the file is in ascending or descending order. first_date = parse_date_liberally(get(first_row, Col.DATE)) last_date = parse_date_liberally(get(last_row, Col.DATE)) is_ascending = first_date < last_date # Reverse the list if the file is in descending order if not is_ascending: entries = list(reversed(entries)) # Add a balance entry if possible if Col.BALANCE in iconfig and entries: entry = entries[-1] date = entry.date + datetime.timedelta(days=1) balance = entry.meta.get("balance", None) if balance is not None: meta = data.new_metadata(file.name, index) entries.append( data.Balance( meta, date, account, Amount(balance, self.currency), None, None, ) ) # Remove the 'balance' metadata. for entry in entries: entry.meta.pop("balance", None) return entries