def _load(sources, log_timings, extra_validations, encoding): """Parse Beancount input, run its transformations and validate it. (This is an internal method.) This routine does all that is necessary to obtain a list of entries ready for realization and working with them. This is the principal call for of the scripts that load a ledger. It returns a list of entries transformed and ready for reporting, a list of errors, and parser's options dict. Args: sources: A list of (filename-or-string, is-filename) where the first element is a string, with either a filename or a string to be parsed directly, and the second argument is a boolean that is true if the first is a filename. You may provide a list of such arguments to be parsed. Filenames must be absolute paths. log_timings: A file object or function to write timings to, or None, if it should remain quiet. extra_validations: A list of extra validation functions to run after loading this list of entries. encoding: A string or None, the encoding to decode the input filename with. Returns: See load() or load_string(). """ assert isinstance(sources, list) and all( isinstance(el, tuple) for el in sources) if hasattr(log_timings, 'write'): log_timings = log_timings.write # Parse all the files recursively. Ensure that the entries are sorted before # running any processes on them. with misc_utils.log_time('parse', log_timings, indent=1): entries, parse_errors, options_map = _parse_recursive( sources, log_timings, encoding) entries.sort(key=data.entry_sortkey) # Run interpolation on incomplete entries. with misc_utils.log_time('booking', log_timings, indent=1): entries, balance_errors = booking.book(entries, options_map) parse_errors.extend(balance_errors) # Transform the entries. with misc_utils.log_time('run_transformations', log_timings, indent=1): entries, errors = run_transformations(entries, parse_errors, options_map, log_timings) # Validate the list of entries. with misc_utils.log_time('beancount.ops.validate', log_timings, indent=1): valid_errors = validation.validate(entries, options_map, log_timings, extra_validations) errors.extend(valid_errors) # Note: We could go hardcore here and further verify that the entries # haven't been modified by user-provided validation routines, by # comparing hashes before and after. Not needed for now. # Compute the input hash. options_map['input_hash'] = compute_input_hash(options_map['include']) return entries, errors, options_map
def read_string_or_entries(entries_or_str): """Read a string of entries or just entries. Args: entries_or_str: Either a list of directives, or a string containing directives. Returns: A list of directives. """ if isinstance(entries_or_str, str): entries, parse_errors, options_map = parser.parse_string( textwrap.dedent(entries_or_str)) # Don't accept incomplete entries either. if parser.has_auto_postings(entries): raise TestError("Entries in assertions may not use interpolation.") entries, booking_errors = booking.book(entries, options_map) errors = parse_errors + booking_errors # Don't tolerate errors. if errors: oss = io.StringIO() printer.print_errors(errors, file=oss) raise TestError("Unexpected errors in expected: {}".format(oss.getvalue())) else: assert isinstance(entries_or_str, list), "Expecting list: {}".format(entries_or_str) entries = entries_or_str return entries
def test_cost_zero(self, entries, errors, options_map): """ 2013-05-18 * "" Assets:Investments:MSFT -10 MSFT {0.00 USD} Assets:Investments:Cash 2000.00 USD """ booked_entries, booking_errors = booking.book(entries, options_map) self.assertFalse(booking_errors)
def test_zero_amount(self, entries, errors, options_map): """ 2013-05-18 * "" Assets:Investments:MSFT 0 MSFT Assets:Investments:Cash 0 USD """ booked_entries, booking_errors = booking.book(entries, options_map) self.assertEqual(0, len(booking_errors))
def test_zero_amount__with_cost(self, entries, errors, options_map): """ 2013-05-18 * "" Assets:Investments:MSFT 0 MSFT {200.00 USD} Assets:Investments:Cash 1 USD """ booked_entries, booking_errors = booking.book(entries, options_map) self.assertEqual(1, len(booking_errors)) self.assertRegex(booking_errors[0].message, 'Amount is zero')
def test_cost_negative(self, entries, errors, options_map): """ 2013-05-18 * "" Assets:Investments:MSFT -10 MSFT {-200.00 USD} Assets:Investments:Cash 2000.00 USD """ booked_entries, booking_errors = booking.book(entries, options_map) self.assertEqual(1, len(booking_errors)) self.assertRegexpMatches(booking_errors[0].message, 'Cost is negative')
def test_zero_amount(self, entries, errors, options_map): """ 2013-05-18 * "" Assets:Investments:MSFT 0 MSFT {200.00 USD} Assets:Investments:Cash 0 USD """ booked_entries, booking_errors = booking.book(entries, options_map) self.assertEqual(1, len(booking_errors)) self.assertTrue(re.search('Amount is zero', booking_errors[0].message))
def get_balances(self): if self.transaction is None: accounts = self.accounts else: entries, balance_errors = booking.book( self.entries + [self.transaction], options.OPTIONS_DEFAULTS.copy()) assert(len(balance_errors) == 0) accounts = realization.realize(entries) balances = {} for user, account in accounts["Assets"]["Receivable"].items(): last_posting = realization.find_last_active_posting(account.txn_postings) if not isinstance(last_posting, Close): positions = account.balance.reduce(convert.get_cost).get_positions() amounts = [position.units for position in positions] assert(len(amounts) < 2) if len(amounts) == 1: balances[user] = str(-amounts[0]) else: balances[user] = "0.00 EUR" return balances
def read_string_or_entries(entries_or_str, allow_incomplete=False): """Read a string of entries or just entries. Args: entries_or_str: Either a list of directives, or a string containing directives. allow_incomplete: A boolean, true if we allow incomplete inputs and perform light-weight booking. Returns: A list of directives. """ if isinstance(entries_or_str, str): entries, errors, options_map = parser.parse_string( textwrap.dedent(entries_or_str)) if allow_incomplete: # Do a simplistic local conversion in order to call the comparison. entries = [_local_booking(entry) for entry in entries] else: # Don't accept incomplete entries either. if any(parser.is_entry_incomplete(entry) for entry in entries): raise TestError( "Entries in assertions may not use interpolation.") entries, booking_errors = booking.book(entries, options_map) errors = errors + booking_errors # Don't tolerate errors. if errors: oss = io.StringIO() printer.print_errors(errors, file=oss) raise TestError("Unexpected errors in expected: {}".format( oss.getvalue())) else: assert isinstance(entries_or_str, list), "Expecting list: {}".format(entries_or_str) entries = entries_or_str return entries
def setUp(self): entries, parse_errors, options_map = parser.parse_string( self.ledger_text) self.entries, booking_errors = booking.book(entries, options_map) self.assertFalse(parse_errors) self.assertFalse(booking_errors)
def main(): argparser = argparse.ArgumentParser() ameritrade.add_args(argparser) argparser.add_argument('-i', '--ignore-errors', dest='raise_error', action='store_false', default=True, help="Raise an error on unhandled messages") argparser.add_argument( '-J', '--debug-file', '--json', action='store', help="Debug filename where to strore al the raw JSON") argparser.add_argument( '-j', '--debug-transaction', action='store', type=int, help="Process a single transaction and print debugging data about it.") argparser.add_argument('-e', '--end-date', action='store', help="Period of end date minus one year.") argparser.add_argument('-B', '--no-booking', dest='booking', action='store_false', default=True, help="Do booking to resolve lots.") argparser.add_argument('-l', '--ledger', action='store', help=("Beancount ledger to remove already imported " "transactions (optional).")) argparser.add_argument( '-g', '--group-by-underlying', action='store_true', help=("Group the transaction output by corresponding " "underlying. This is great for options.")) args = argparser.parse_args() # Open a connection and figure out the main account. api = ameritrade.open(ameritrade.config_from_args(args)) accountId = utils.GetMainAccount(api) positions = utils.GetPositions(api, accountId) # Fetch transactions. # Note that the following arguments are also honored: # endDate=datetime.date.today().isoformat()) # startDate='2014-01-01', # endDate='2015-01-01') if args.end_date: end_date = parser.parse(args.end_date).date() start_date = end_date - datetime.timedelta(days=364) start = start_date.isoformat() end = end_date.isoformat() else: start = end = None txns = api.GetTransactions(accountId=accountId, startDate=start, endDate=end) if isinstance(txns, dict): pprint(txns, sys.stderr) return txns.reverse() # Optionally write out the raw original content downloaded to a file. if args.debug_file: with open(args.debug, 'w') as ofile: ofile.write(pformat(txns)) # Process each of the transactions. entries = [] balances = collections.defaultdict(inventory.Inventory) commodities = {} for txn in txns: if args.debug_transaction and txn[ 'transactionId'] != args.debug_transaction: continue else: pprint.pprint(txn) # print('{:30} {}'.format(txn['type'], txn['description'])); continue dispatch_entries = RunDispatch(txn, balances, commodities, args.raise_error) if dispatch_entries: entries.extend(dispatch_entries) # Update a balance account of just the units. # # This is only here so that the options removal can figure out which # side is the reduction side and what sign to use on the position # change. Ideally the API would provide a side indication and we # wouldn't have to maintain any state at alll. {492fa5292636} for entry in data.filter_txns(dispatch_entries): for posting in entry.postings: balance = balances[posting.account] if posting.units is not None: balance.add_amount(posting.units) # Add a final balance entry. balance_entry = CreateBalance(api, accountId) if balance_entry: entries.append(balance_entry) if args.booking: # Book the entries. entries, balance_errors = booking.book(entries, OPTIONS_DEFAULTS.copy()) if balance_errors: printer.print_errors(balance_errors) # Remove dates on reductions when they have no prices. This is an # artifact of not being able to pass in prior balance state to the # booking code, which we will fix in v3. entries = RemoveDateReductions(entries) # Match up the trades we can in this subset of the history and pair them up # with a common random id. entries, balances = MatchTrades(entries) # Add zero prices for expired options for which we still have non-zero # positions. entries.extend(GetExpiredOptionsPrices(positions, balances)) # If a Beancount ledger has been specified, open it, read it in, and remove # all the transactions up to the latest one with the transaction id (as a # link) that's present in the ledger. if args.ledger: ledger_entries, _, __ = loader.load_file(args.ledger) # Find the date of the last transaction in the ledger with a TD # transaction id, and the full set of links to remove. links = set() last_date = None for entry in data.filter_txns(ledger_entries): for link in (entry.links or {}): if re.match(r"td-\d{9,}", link): links.add(link) last_date = entry.date # Remove all the transactions already present in the ledger. # # Also remove all the non-transactions with a date before that of the # last linked one that was found. This allows us to remove Price and # Commodity directives. (In v3, when links are available on every # directive, we can just use the links.) entries = [ entry for entry in entries if ((isinstance(entry, data.Transaction) and not entry.links & links) or (not isinstance(entry, data.Transaction) and entry.date >= last_date)) ] if args.group_by_underlying: # Group the transactions by their underlying, with org-mode separators. groups = GroupByUnderlying(entries) for (has_option, currency), group_entries in sorted(groups.items()): header = currency or "General" if has_option: header = "Options: {}".format(header) print("** {}".format(header)) print() printer.print_entries(data.sorted(group_entries), file=sys.stdout) print() print() else: # Render all the entries chronologically (the default). sentries = SortCommodityFirst(entries) printer.print_entries(sentries, file=sys.stdout)