def read_details(filename):
"""Read the encrypted details file and create an index of its contents.
Args:
filename: A string, the name of the filename to read.
Returns:
A pair of the Details object containing the database and a mapping of
Beancount account name to its (institution, account) pair.
"""
# Read the encrypted details file.
if encryption.is_encrypted_file(filename):
contents = encryption.read_encrypted_file(filename)
else:
with open(args.details_filename, 'rb') as infile:
contents = infile.read()
# Read the ASCII protobuf database.
details = assets_pb2.Details()
text_format.Merge(contents, details)
# Create a mapping of account name to (institution, account) pairs.
mapping = {}
for institution in details.institution:
for acc in institution.account:
assert acc.beancount not in mapping, (
"Account name {} is not unique!".format(acc.beancount))
mapping[acc.beancount] = (institution, acc)
return details, mapping
def load_encrypted_file(filename,
log_timings=None,
log_errors=None,
extra_validations=None,
dedent=False,
encoding=None):
"""Load an encrypted Beancount input file.
Args:
filename: The name of an encrypted file to be parsed.
log_timings: See load_string().
log_errors: See load_string().
extra_validations: See load_string().
dedent: See load_string().
encoding: See load_string().
Returns:
A triple of (entries, errors, option_map) where "entries" is a date-sorted
list of entries from the file, "errors" a list of error objects generated
while parsing and validating the file, and "options_map", a dict of the
options parsed from the file.
"""
contents = encryption.read_encrypted_file(filename)
return load_string(contents,
log_timings=log_timings,
log_errors=log_errors,
extra_validations=extra_validations,
encoding=encoding)
def test_read_encrypted_file(self):
encrypted_file = path.join(self.tmpdir.name, 'test.beancount.asc')
self.encrypt_as_file(INPUT, encrypted_file)
with test_utils.environ('GNUPGHOME', self.ringdir):
plaintext = encryption.read_encrypted_file(encrypted_file)
self.assertEqual(INPUT, plaintext)
def test_read_encrypted_file(self):
# Import secret and public keys.
self._run_gpg('--import', stdin=TEST_PUBLIC_KEY.encode('ascii'))
self._run_gpg('--import', stdin=TEST_SECRET_KEY.encode('ascii'))
# Encrypt the Beancount plaintext file with it.
out, err = self._run_gpg('--recipient',
'beancount-test',
'--encrypt',
'--output=-',
stdin=INPUT.encode('utf8'))
encrypted_file = path.join(self.tmpdir, 'test.beancount.asc')
with open(encrypted_file, 'w') as encfile:
encfile.write(out)
with test_utils.environ('GNUPGHOME', self.ringdir):
plaintext = encryption.read_encrypted_file(encrypted_file)
self.assertEqual(INPUT, plaintext)
def _parse_recursive(sources, log_timings, encoding=None):
"""Parse Beancount input, run its transformations and validate it.
Recursively parse a list of files or strings and their include files and
return an aggregate of parsed directives, errors, and the top-level
options-map. If the same file is being parsed twice, ignore it and issue an
error.
Args:
sources: A list of (filename-or-string, is-filename) where the first
element is a string, with either a filename or a string to be parsed directly,
and the second argument is a boolean that is true if the first is a filename.
You may provide a list of such arguments to be parsed. Filenames must be absolute
paths.
log_timings: A function to write timings to, or None, if it should remain quiet.
encoding: A string or None, the encoding to decode the input filename with.
Returns:
A tuple of (entries, parse_errors, options_map).
"""
assert isinstance(sources, list) and all(
isinstance(el, tuple) for el in sources)
# Current parse state.
entries, parse_errors = [], []
options_map = None
# A stack of sources to be parsed.
source_stack = list(sources)
# A list of absolute filenames that have been parsed in the past, used to
# detect and avoid duplicates (cycles).
filenames_seen = set()
with misc_utils.log_time('beancount.parser.parser', log_timings, indent=1):
while source_stack:
source, is_file = source_stack.pop(0)
is_top_level = options_map is None
# If the file is encrypted, read it in and process it as a string.
if is_file:
cwd = path.dirname(source)
source_filename = source
if encryption.is_encrypted_file(source):
source = encryption.read_encrypted_file(source)
is_file = False
else:
# If we're parsing a string, the CWD is the current process
# working directory.
cwd = os.getcwd()
source_filename = None
if is_file:
# All filenames here must be absolute.
assert path.isabs(source)
filename = path.normpath(source)
# Check for file previously parsed... detect duplicates.
if filename in filenames_seen:
parse_errors.append(
LoadError(
data.new_metadata("<load>", 0),
'Duplicate filename parsed: "{}"'.format(filename),
None))
continue
# Check for a file that does not exist.
if not path.exists(filename):
parse_errors.append(
LoadError(data.new_metadata("<load>", 0),
'File "{}" does not exist'.format(filename),
None))
continue
# Parse a file from disk directly.
filenames_seen.add(filename)
with misc_utils.log_time('beancount.parser.parser.parse_file',
log_timings,
indent=2):
(src_entries, src_errors,
src_options_map) = parser.parse_file(filename,
encoding=encoding)
cwd = path.dirname(filename)
else:
# Encode the contents if necessary.
if encoding:
if isinstance(source, bytes):
source = source.decode(encoding)
source = source.encode('ascii', 'replace')
# Parse a string buffer from memory.
with misc_utils.log_time(
'beancount.parser.parser.parse_string',
log_timings,
indent=2):
(src_entries, src_errors,
src_options_map) = parser.parse_string(
source, source_filename)
# Merge the entries resulting from the parsed file.
entries.extend(src_entries)
parse_errors.extend(src_errors)
# We need the options from the very top file only (the very
# first file being processed). No merging of options should
# occur.
if is_top_level:
options_map = src_options_map
else:
aggregate_options_map(options_map, src_options_map)
# Add includes to the list of sources to process. chdir() for glob,
# which uses it indirectly.
include_expanded = []
with file_utils.chdir(cwd):
for include_filename in src_options_map['include']:
matched_filenames = glob.glob(include_filename,
recursive=True)
if matched_filenames:
include_expanded.extend(matched_filenames)
else:
parse_errors.append(
LoadError(
data.new_metadata("<load>", 0),
'File glob "{}" does not match any files'.
format(include_filename), None))
for include_filename in include_expanded:
if not path.isabs(include_filename):
include_filename = path.join(cwd, include_filename)
include_filename = path.normpath(include_filename)
# Add the include filenames to be processed later.
source_stack.append((include_filename, True))
# Make sure we have at least a dict of valid options.
if options_map is None:
options_map = options.OPTIONS_DEFAULTS.copy()
# Save the set of parsed filenames in options_map.
options_map['include'] = sorted(filenames_seen)
return entries, parse_errors, options_map
