def process_documents(entries, options_map):
"""Check files for document directives and create documents directives automatically.
Args:
entries: A list of all directives parsed from the file.
options_map: An options dict, as is output by the parser.
We're using its 'filename' option to figure out relative path to
search for documents.
Returns:
A pair of list of all entries (including new ones), and errors
generated during the process of creating document directives.
"""
filename = options_map["filename"]
# Detect filenames that should convert into entries.
autodoc_entries = []
autodoc_errors = []
document_dirs = options_map['documents']
if document_dirs:
# Restrict to the list of valid accounts only.
accounts = getters.get_accounts(entries)
# Accumulate all the entries.
for directory in document_dirs:
new_entries, new_errors = find_documents(directory, filename,
accounts)
autodoc_entries.extend(new_entries)
autodoc_errors.extend(new_errors)
# Merge the two lists of entries and errors. Keep the entries sorted.
entries.extend(autodoc_entries)
entries.sort(key=data.entry_sortkey)
return (entries, autodoc_errors)
def test_get_accounts(self):
entries = loader.load_string(TEST_INPUT)[0]
accounts = getters.get_accounts(entries)
self.assertEqual(
{
'Assets:US:Cash', 'Assets:US:Credit-Card', 'Expenses:Grocery',
'Expenses:Coffee', 'Expenses:Restaurant'
}, accounts)
def validate_directories(entries, document_dirs):
"""Validate a directory hierarchy against a ledger's account names.
Read a ledger's list of account names and check that all the capitalized
subdirectory names under the given roots match the account names.
Args:
entries: A list of directives.
document_dirs: A list of string, the directory roots to walk and validate.
"""
# Get the list of accounts declared in the ledge.
accounts = getters.get_accounts(entries)
# For each of the roots, validate the hierarchy of directories.
for document_dir in document_dirs:
errors = validate_directory(accounts, document_dir)
for error in errors:
print("ERROR: {}".format(error))
def __init__(self, entries):
self.existing_accounts = getters.get_accounts(entries)
def main():
"""Top-level function."""
parser = argparse.ArgumentParser(description=__doc__.strip())
parser.add_argument('ledger',
help="Beancount ledger file")
parser.add_argument('config', action='store',
help='Configuration for accounts and reports.')
parser.add_argument('output',
help="Output directory to write all output files to.")
parser.add_argument('filter_reports', nargs='*',
help="Optional names of specific subset of reports to analyze.")
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose mode')
parser.add_argument('-d', '--days-price-threshold', action='store', type=int,
default=5,
help="The number of days to tolerate price latency.")
parser.add_argument('-e', '--end-date', action='store',
type=datetime.date.fromisoformat,
help="The end date to compute returns up to.")
parser.add_argument('--pdf', '--pdfs', action='store_true',
help="Render as PDFs. Default is HTML directories.")
parser.add_argument('-j', '--parallel', action='store_true',
help="Run report generation concurrently.")
parser.add_argument('-E', '--check-explicit-flows', action='store_true',
help=("Enables comparison of the general categorization method "
"with the explicit one with specialized explicit handlers "
"per signature."))
args, pipeline_args = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s: %(message)s')
logging.getLogger('matplotlib.font_manager').disabled = True
# Figure out end date.
end_date = args.end_date or datetime.date.today()
# Load the example file.
logging.info("Reading ledger: %s", args.ledger)
entries, _, options_map = loader.load_file(args.ledger)
accounts = getters.get_accounts(entries)
dcontext = options_map['dcontext']
# Load, filter and expand the configuration.
config = configlib.read_config(args.config, args.filter_reports, accounts)
os.makedirs(args.output, exist_ok=True)
with open(path.join(args.output, "config.pbtxt"), "w") as efile:
print(config, file=efile)
# Extract data from the ledger.
account_data_map = investments.extract(
entries, dcontext, config, end_date, args.check_explicit_flows,
path.join(args.output, "investments"))
# Generate output reports.
output_reports = path.join(args.output, "groups")
price_map = prices.build_price_map(entries)
poptions = PipelineOptions(pipeline_args,
runner="DirectRunner",
direct_running_mode="multi_processing",
direct_num_workers=8)
with beam.Pipeline(options=poptions) as pipeline:
report_adlist = []
for report in config.groups.group:
adlist = [account_data_map[name] for name in report.investment]
assert isinstance(adlist, list)
assert all(isinstance(ad, investments.AccountData) for ad in adlist)
report_adlist.append((report, adlist))
_ = (pipeline
| beam.Create(report_adlist)
| beam.Map(reports.generate_report_mapper, price_map, end_date)
| beam.ParDo(WriteNamedBinary, FormatFilename))
def check(entries, options_map):
"""Process the balance assertion directives.
For each Balance directive, check that their expected balance corresponds to
the actual balance computed at that time and replace failing ones by new
ones with a flag that indicates failure.
Args:
entries: A list of directives.
options_map: A dict of options, parsed from the input file.
Returns:
A pair of a list of directives and a list of balance check errors.
"""
new_entries = []
check_errors = []
# This is similar to realization, but performed in a different order, and
# where we only accumulate inventories for accounts that have balance
# assertions in them (this saves on time). Here we process the entries one
# by one along with the balance checks. We use a temporary realization in
# order to hold the incremental tree of balances, so that we can easily get
# the amounts of an account's subaccounts for making checks on parent
# accounts.
real_root = realization.RealAccount('')
# Figure out the set of accounts for which we need to compute a running
# inventory balance.
asserted_accounts = {
entry.account
for entry in entries if isinstance(entry, Balance)
}
# Add all children accounts of an asserted account to be calculated as well,
# and pre-create these accounts, and only those (we're just being tight to
# make sure).
asserted_match_list = [
account.parent_matcher(account_) for account_ in asserted_accounts
]
for account_ in getters.get_accounts(entries):
if (account_ in asserted_accounts
or any(match(account_) for match in asserted_match_list)):
realization.get_or_create(real_root, account_)
# Get the Open directives for each account.
open_close_map = getters.get_account_open_close(entries)
for entry in entries:
if isinstance(entry, Transaction):
# For each of the postings' accounts, update the balance inventory.
for posting in entry.postings:
real_account = realization.get(real_root, posting.account)
# The account will have been created only if we're meant to track it.
if real_account is not None:
# Note: Always allow negative lots for the purpose of balancing.
# This error should show up somewhere else than here.
real_account.balance.add_position(posting)
elif isinstance(entry, Balance):
# Check that the currency of the balance check is one of the allowed
# currencies for that account.
expected_amount = entry.amount
try:
open, _ = open_close_map[entry.account]
except KeyError:
check_errors.append(
BalanceError(
entry.meta,
"Account '{}' does not exist: ".format(entry.account),
entry))
continue
if (expected_amount is not None and open and open.currencies
and expected_amount.currency not in open.currencies):
check_errors.append(
BalanceError(
entry.meta,
"Invalid currency '{}' for Balance directive: ".format(
expected_amount.currency), entry))
# Check the balance against the check entry.
real_account = realization.get(real_root, entry.account)
assert real_account is not None, "Missing {}".format(entry.account)
# Sum up the current balances for this account and its
# sub-accounts. We want to support checks for parent accounts
# for the total sum of their subaccounts.
subtree_balance = inventory.Inventory()
for real_child in realization.iter_children(real_account, False):
subtree_balance += real_child.balance
# Get only the amount in the desired currency.
balance_amount = subtree_balance.get_currency_units(
expected_amount.currency)
# Check if the amount is within bounds of the expected amount.
diff_amount = amount.sub(balance_amount, expected_amount)
# Use the specified tolerance or automatically infer it.
tolerance = get_balance_tolerance(entry, options_map)
if abs(diff_amount.number) > tolerance:
check_errors.append(
BalanceError(
entry.meta,
("Balance failed for '{}': "
"expected {} != accumulated {} ({} {})").format(
entry.account, expected_amount, balance_amount,
abs(diff_amount.number),
('too much' if diff_amount.number > 0 else
'too little')), entry))
# Substitute the entry by a failing entry, with the diff_amount
# field set on it. I'm not entirely sure that this is the best
# of ideas, maybe leaving the original check intact and insert a
# new error entry might be more functional or easier to
# understand.
entry = entry._replace(meta=entry.meta.copy(),
diff_amount=diff_amount)
new_entries.append(entry)
return new_entries, check_errors
def main():
"""Top-level function."""
parser = argparse.ArgumentParser(description=__doc__.strip())
parser.add_argument('ledger', help="Beancount ledger file")
parser.add_argument('config',
action='store',
help='Configuration for accounts and reports.')
parser.add_argument('output',
help="Output directory to write all output files to.")
parser.add_argument(
'filter_reports',
nargs='*',
help="Optional names of specific subset of reports to analyze.")
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Verbose mode')
parser.add_argument('-d',
'--days-price-threshold',
action='store',
type=int,
default=5,
help="The number of days to tolerate price latency.")
parser.add_argument('-e',
'--end-date',
action='store',
type=datetime.date.fromisoformat,
help="The end date to compute returns up to.")
parser.add_argument('--pdf',
'--pdfs',
action='store_true',
help="Render as PDFs. Default is HTML directories.")
parser.add_argument('-j',
'--parallel',
action='store_true',
help="Run report generation concurrently.")
parser.add_argument(
'-E',
'--check-explicit-flows',
action='store_true',
help=("Enables comparison of the general categorization method "
"with the explicit one with specialized explicit handlers "
"per signature."))
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG,
format='%(levelname)-8s: %(message)s')
logging.getLogger('matplotlib.font_manager').disabled = True
# Figure out end date.
end_date = args.end_date or datetime.date.today()
# Load the example file.
logging.info("Reading ledger: %s", args.ledger)
entries, _, options_map = loader.load_file(args.ledger)
accounts = getters.get_accounts(entries)
dcontext = options_map['dcontext']
# Load, filter and expand the configuration.
config = configlib.read_config(args.config, args.filter_reports, accounts)
os.makedirs(args.output, exist_ok=True)
with open(path.join(args.output, "config.pbtxt"), "w") as efile:
print(config, file=efile)
# Extract data from the ledger.
account_data_map = investments.extract(
entries, dcontext, config, end_date, args.check_explicit_flows,
path.join(args.output, "investments"))
# Generate output reports.
output_reports = path.join(args.output, "reports")
pricer = reports.generate_reports(account_data_map, config,
prices.build_price_map(entries),
end_date, output_reports, args.parallel,
args.pdf)
# Generate price reports.
output_prices = path.join(args.output, "prices")
reports.generate_price_pages(account_data_map,
prices.build_price_map(entries),
output_prices)
# Output required price directives (to be filled in the source ledger by
# fetching prices).
reports.write_price_directives(
path.join(output_prices, "prices.beancount"), pricer,
args.days_price_threshold)
def __init__(self, entries):
self.existing_accounts = getters.get_accounts(entries)
def balexpr(entries, options_map):
errors = []
accounts = []
real_root = realization.RealAccount('')
balexpr_entries = [entry for entry in entries if is_balexpr_entry(entry)]
asserted_accounts = {
account_
for entry in balexpr_entries
for account_ in get_accounts_from_entry(entry)
}
asserted_match_list = [
account.parent_matcher(account_) for account_ in asserted_accounts
]
for account_ in getters.get_accounts(entries):
if (account_ in asserted_accounts
or any(match(account_) for match in asserted_match_list)):
realization.get_or_create(real_root, account_)
open_close_map = getters.get_account_open_close(entries)
current_checking_balexpr_entry = 0
for entry in entries:
if current_checking_balexpr_entry >= len(balexpr_entries):
break
while current_checking_balexpr_entry < len(
balexpr_entries) and balexpr_entries[
current_checking_balexpr_entry].date == entry.date:
checking_entry = balexpr_entries[current_checking_balexpr_entry]
current_checking_balexpr_entry += 1
accounts = get_accounts_from_entry(checking_entry)
if not accounts:
errors.append(
BalExprError(checking_entry.meta,
'No account found in the expression',
checking_entry))
continue
currency = get_expected_amount_from_entry(checking_entry).currency
error_found_in_currencies = False
for account_ in accounts:
try:
open, _ = open_close_map[account_]
except KeyError:
errors.append(
BalExprError(
checking_entry.meta,
'Invalid reference to unknown account \'{}\''.
format(account_), checking_entry))
error_found_in_currencies = True
break
if currency not in open.currencies:
errors.append(
BalExprError(checking_entry.meta,
'Currencies are inconsistent',
checking_entry))
error_found_in_currencies = True
break
if error_found_in_currencies:
continue
expression = get_expression_from_entry(checking_entry)
expected_amount = get_expected_amount_from_entry(checking_entry)
real_amount, error_msg = calcuate(expression, currency, real_root)
if error_msg:
errors.append(
BalExprError(checking_entry.meta, error_msg,
checking_entry))
continue
diff_amount = sub(real_amount, expected_amount)
if abs(diff_amount.number) > 0.005:
errors.append(
BalExprError(
checking_entry.meta,
"BalExpr failed: expected {} != accumulated {} ({} {})"
.format(expected_amount, real_amount,
abs(diff_amount.number),
('too much' if diff_amount.number > 0 else
'too little')), checking_entry))
if isinstance(entry, Transaction):
for posting in entry.postings:
real_account = realization.get(real_root, posting.account)
if real_account is not None:
real_account.balance.add_position(posting)
return entries, errors
