def _load(sources, log_timings, extra_validations, encoding):
"""Parse Beancount input, run its transformations and validate it.
(This is an internal method.)
This routine does all that is necessary to obtain a list of entries ready
for realization and working with them. This is the principal call for of the
scripts that load a ledger. It returns a list of entries transformed and
ready for reporting, a list of errors, and parser's options dict.
Args:
sources: A list of (filename-or-string, is-filename) where the first
element is a string, with either a filename or a string to be parsed directly,
and the second argument is a boolean that is true if the first is a filename.
You may provide a list of such arguments to be parsed. Filenames must be absolute
paths.
log_timings: A file object or function to write timings to,
or None, if it should remain quiet.
extra_validations: A list of extra validation functions to run after loading
this list of entries.
encoding: A string or None, the encoding to decode the input filename with.
Returns:
See load() or load_string().
"""
assert isinstance(sources, list) and all(
isinstance(el, tuple) for el in sources)
if hasattr(log_timings, 'write'):
log_timings = log_timings.write
# Parse all the files recursively. Ensure that the entries are sorted before
# running any processes on them.
with misc_utils.log_time('parse', log_timings, indent=1):
entries, parse_errors, options_map = _parse_recursive(
sources, log_timings, encoding)
entries.sort(key=data.entry_sortkey)
# Run interpolation on incomplete entries.
with misc_utils.log_time('booking', log_timings, indent=1):
entries, balance_errors = booking.book(entries, options_map)
parse_errors.extend(balance_errors)
# Transform the entries.
with misc_utils.log_time('run_transformations', log_timings, indent=1):
entries, errors = run_transformations(entries, parse_errors,
options_map, log_timings)
# Validate the list of entries.
with misc_utils.log_time('beancount.ops.validate', log_timings, indent=1):
valid_errors = validation.validate(entries, options_map, log_timings,
extra_validations)
errors.extend(valid_errors)
# Note: We could go hardcore here and further verify that the entries
# haven't been modified by user-provided validation routines, by
# comparing hashes before and after. Not needed for now.
# Compute the input hash.
options_map['input_hash'] = compute_input_hash(options_map['include'])
return entries, errors, options_map
def read_string_or_entries(entries_or_str):
"""Read a string of entries or just entries.
Args:
entries_or_str: Either a list of directives, or a string containing directives.
Returns:
A list of directives.
"""
if isinstance(entries_or_str, str):
entries, parse_errors, options_map = parser.parse_string(
textwrap.dedent(entries_or_str))
# Don't accept incomplete entries either.
if parser.has_auto_postings(entries):
raise TestError("Entries in assertions may not use interpolation.")
entries, booking_errors = booking.book(entries, options_map)
errors = parse_errors + booking_errors
# Don't tolerate errors.
if errors:
oss = io.StringIO()
printer.print_errors(errors, file=oss)
raise TestError("Unexpected errors in expected: {}".format(oss.getvalue()))
else:
assert isinstance(entries_or_str, list), "Expecting list: {}".format(entries_or_str)
entries = entries_or_str
return entries
def test_cost_zero(self, entries, errors, options_map):
"""
2013-05-18 * ""
Assets:Investments:MSFT -10 MSFT {0.00 USD}
Assets:Investments:Cash 2000.00 USD
"""
booked_entries, booking_errors = booking.book(entries, options_map)
self.assertFalse(booking_errors)
def test_zero_amount(self, entries, errors, options_map):
"""
2013-05-18 * ""
Assets:Investments:MSFT 0 MSFT
Assets:Investments:Cash 0 USD
"""
booked_entries, booking_errors = booking.book(entries, options_map)
self.assertEqual(0, len(booking_errors))
def test_zero_amount__with_cost(self, entries, errors, options_map):
"""
2013-05-18 * ""
Assets:Investments:MSFT 0 MSFT {200.00 USD}
Assets:Investments:Cash 1 USD
"""
booked_entries, booking_errors = booking.book(entries, options_map)
self.assertEqual(1, len(booking_errors))
self.assertRegex(booking_errors[0].message, 'Amount is zero')
def test_cost_negative(self, entries, errors, options_map):
"""
2013-05-18 * ""
Assets:Investments:MSFT -10 MSFT {-200.00 USD}
Assets:Investments:Cash 2000.00 USD
"""
booked_entries, booking_errors = booking.book(entries, options_map)
self.assertEqual(1, len(booking_errors))
self.assertRegexpMatches(booking_errors[0].message, 'Cost is negative')
def test_zero_amount(self, entries, errors, options_map):
"""
2013-05-18 * ""
Assets:Investments:MSFT 0 MSFT {200.00 USD}
Assets:Investments:Cash 0 USD
"""
booked_entries, booking_errors = booking.book(entries, options_map)
self.assertEqual(1, len(booking_errors))
self.assertTrue(re.search('Amount is zero', booking_errors[0].message))
def get_balances(self):
if self.transaction is None:
accounts = self.accounts
else:
entries, balance_errors = booking.book(
self.entries + [self.transaction],
options.OPTIONS_DEFAULTS.copy())
assert(len(balance_errors) == 0)
accounts = realization.realize(entries)
balances = {}
for user, account in accounts["Assets"]["Receivable"].items():
last_posting = realization.find_last_active_posting(account.txn_postings)
if not isinstance(last_posting, Close):
positions = account.balance.reduce(convert.get_cost).get_positions()
amounts = [position.units for position in positions]
assert(len(amounts) < 2)
if len(amounts) == 1:
balances[user] = str(-amounts[0])
else:
balances[user] = "0.00 EUR"
return balances
def read_string_or_entries(entries_or_str, allow_incomplete=False):
"""Read a string of entries or just entries.
Args:
entries_or_str: Either a list of directives, or a string containing directives.
allow_incomplete: A boolean, true if we allow incomplete inputs and perform
light-weight booking.
Returns:
A list of directives.
"""
if isinstance(entries_or_str, str):
entries, errors, options_map = parser.parse_string(
textwrap.dedent(entries_or_str))
if allow_incomplete:
# Do a simplistic local conversion in order to call the comparison.
entries = [_local_booking(entry) for entry in entries]
else:
# Don't accept incomplete entries either.
if any(parser.is_entry_incomplete(entry) for entry in entries):
raise TestError(
"Entries in assertions may not use interpolation.")
entries, booking_errors = booking.book(entries, options_map)
errors = errors + booking_errors
# Don't tolerate errors.
if errors:
oss = io.StringIO()
printer.print_errors(errors, file=oss)
raise TestError("Unexpected errors in expected: {}".format(
oss.getvalue()))
else:
assert isinstance(entries_or_str,
list), "Expecting list: {}".format(entries_or_str)
entries = entries_or_str
return entries
def setUp(self):
entries, parse_errors, options_map = parser.parse_string(
self.ledger_text)
self.entries, booking_errors = booking.book(entries, options_map)
self.assertFalse(parse_errors)
self.assertFalse(booking_errors)
def main():
argparser = argparse.ArgumentParser()
ameritrade.add_args(argparser)
argparser.add_argument('-i',
'--ignore-errors',
dest='raise_error',
action='store_false',
default=True,
help="Raise an error on unhandled messages")
argparser.add_argument(
'-J',
'--debug-file',
'--json',
action='store',
help="Debug filename where to strore al the raw JSON")
argparser.add_argument(
'-j',
'--debug-transaction',
action='store',
type=int,
help="Process a single transaction and print debugging data about it.")
argparser.add_argument('-e',
'--end-date',
action='store',
help="Period of end date minus one year.")
argparser.add_argument('-B',
'--no-booking',
dest='booking',
action='store_false',
default=True,
help="Do booking to resolve lots.")
argparser.add_argument('-l',
'--ledger',
action='store',
help=("Beancount ledger to remove already imported "
"transactions (optional)."))
argparser.add_argument(
'-g',
'--group-by-underlying',
action='store_true',
help=("Group the transaction output by corresponding "
"underlying. This is great for options."))
args = argparser.parse_args()
# Open a connection and figure out the main account.
api = ameritrade.open(ameritrade.config_from_args(args))
accountId = utils.GetMainAccount(api)
positions = utils.GetPositions(api, accountId)
# Fetch transactions.
# Note that the following arguments are also honored:
# endDate=datetime.date.today().isoformat())
# startDate='2014-01-01',
# endDate='2015-01-01')
if args.end_date:
end_date = parser.parse(args.end_date).date()
start_date = end_date - datetime.timedelta(days=364)
start = start_date.isoformat()
end = end_date.isoformat()
else:
start = end = None
txns = api.GetTransactions(accountId=accountId,
startDate=start,
endDate=end)
if isinstance(txns, dict):
pprint(txns, sys.stderr)
return
txns.reverse()
# Optionally write out the raw original content downloaded to a file.
if args.debug_file:
with open(args.debug, 'w') as ofile:
ofile.write(pformat(txns))
# Process each of the transactions.
entries = []
balances = collections.defaultdict(inventory.Inventory)
commodities = {}
for txn in txns:
if args.debug_transaction and txn[
'transactionId'] != args.debug_transaction:
continue
else:
pprint.pprint(txn)
# print('{:30} {}'.format(txn['type'], txn['description'])); continue
dispatch_entries = RunDispatch(txn, balances, commodities,
args.raise_error)
if dispatch_entries:
entries.extend(dispatch_entries)
# Update a balance account of just the units.
#
# This is only here so that the options removal can figure out which
# side is the reduction side and what sign to use on the position
# change. Ideally the API would provide a side indication and we
# wouldn't have to maintain any state at alll. {492fa5292636}
for entry in data.filter_txns(dispatch_entries):
for posting in entry.postings:
balance = balances[posting.account]
if posting.units is not None:
balance.add_amount(posting.units)
# Add a final balance entry.
balance_entry = CreateBalance(api, accountId)
if balance_entry:
entries.append(balance_entry)
if args.booking:
# Book the entries.
entries, balance_errors = booking.book(entries,
OPTIONS_DEFAULTS.copy())
if balance_errors:
printer.print_errors(balance_errors)
# Remove dates on reductions when they have no prices. This is an
# artifact of not being able to pass in prior balance state to the
# booking code, which we will fix in v3.
entries = RemoveDateReductions(entries)
# Match up the trades we can in this subset of the history and pair them up
# with a common random id.
entries, balances = MatchTrades(entries)
# Add zero prices for expired options for which we still have non-zero
# positions.
entries.extend(GetExpiredOptionsPrices(positions, balances))
# If a Beancount ledger has been specified, open it, read it in, and remove
# all the transactions up to the latest one with the transaction id (as a
# link) that's present in the ledger.
if args.ledger:
ledger_entries, _, __ = loader.load_file(args.ledger)
# Find the date of the last transaction in the ledger with a TD
# transaction id, and the full set of links to remove.
links = set()
last_date = None
for entry in data.filter_txns(ledger_entries):
for link in (entry.links or {}):
if re.match(r"td-\d{9,}", link):
links.add(link)
last_date = entry.date
# Remove all the transactions already present in the ledger.
#
# Also remove all the non-transactions with a date before that of the
# last linked one that was found. This allows us to remove Price and
# Commodity directives. (In v3, when links are available on every
# directive, we can just use the links.)
entries = [
entry for entry in entries
if ((isinstance(entry, data.Transaction) and not entry.links
& links) or (not isinstance(entry, data.Transaction)
and entry.date >= last_date))
]
if args.group_by_underlying:
# Group the transactions by their underlying, with org-mode separators.
groups = GroupByUnderlying(entries)
for (has_option, currency), group_entries in sorted(groups.items()):
header = currency or "General"
if has_option:
header = "Options: {}".format(header)
print("** {}".format(header))
print()
printer.print_entries(data.sorted(group_entries), file=sys.stdout)
print()
print()
else:
# Render all the entries chronologically (the default).
sentries = SortCommodityFirst(entries)
printer.print_entries(sentries, file=sys.stdout)
