def main():
# Parse command line arguments.
options = parse_args(constants.TOOL_PARSER)
log.set_verbosity(options.verbosity)
# Parse each source.
ast = []
if options.file:
for f in options.file:
s = f.read()
try:
ast += parse_to_ast(s, options.cpp, options.cpp_flag)
if options.resolve_imports:
ast, _ = resolve_imports(ast, \
os.path.dirname(f.name), options.import_path,
options.cpp, options.cpp_flag)
except CAmkESSyntaxError as e:
e.set_column(s)
log.error('%s:%s' % (f.name, str(e)))
return -1
except Exception:
log.exception('Error during lexing/parsing \'%s\''% f.name)
return -1
finally:
f.close()
else:
s = sys.stdin.read()
try:
ast += parse_to_ast(s, options.cpp, options.cpp_flag)
if options.resolve_imports:
ast, _ = resolve_imports(ast, \
os.curdir, options.import_path, options.cpp,
options.cpp_flag)
except Exception:
log.exception('Error during lexing/parsing')
return -1
ast = dedupe(ast)
if options.resolve_references:
ast = resolve_references(ast)
# Generate the output and print this.
out = show(ast)
print pretty(out)
return 0
def main():
args = parse_args(constants.TOOL_LINT)
log.set_verbosity(args.verbosity)
# Parse the input and form the AST.
ast = []
for f in args.file:
try:
items = parser.parse_to_ast(f)
except Exception as inst:
log.critical('Failed to parse input: %s' % str(inst))
return CRITICAL
if args.resolve_imports:
try:
items, _ = parser.resolve_imports(items, \
os.path.dirname(f.name), args.import_path)
except Exception as inst:
log.critical('Failed to resolve imports: %s' % str(inst))
return CRITICAL
ast += items
if args.resolve_references:
try:
ast = parser.resolve_references(ast, args.allow_forward_references)
except Exception as inst:
log.critical('Failed to resolve references: %s' % str(inst))
return CRITICAL
# Check it for inconsistencies.
ret = 0
for m in lint.check(ast):
if isinstance(m, lint.ProblemWarning):
log.warning(str(m))
if ret != ERROR:
ret = WARNING
else: # isinstance(m, lint.ProblemError)
log.error(str(m))
ret = ERROR
return ret
def main():
args = parse_args(constants.TOOL_LINT)
log.set_verbosity(args.verbosity)
# Parse the input and form the AST.
ast = []
for f in args.file:
try:
items = parser.parse_to_ast(f)
except Exception as inst:
log.critical('Failed to parse input: %s' % str(inst))
return CRITICAL
if args.resolve_imports:
try:
items, _ = parser.resolve_imports(items, \
os.path.dirname(f.name), args.import_path)
except Exception as inst:
log.critical('Failed to resolve imports: %s' % str(inst))
return CRITICAL
ast += items
if args.resolve_references:
try:
ast = parser.resolve_references(ast)
except Exception as inst:
log.critical('Failed to resolve references: %s' % str(inst))
return CRITICAL
# Check it for inconsistencies.
ret = 0
for m in lint.check(ast):
if isinstance(m, lint.ProblemWarning):
log.warning(str(m))
if ret != ERROR:
ret = WARNING
else: # isinstance(m, lint.ProblemError)
log.error(str(m))
ret = ERROR
return ret
def main(argv, out, err):
options = parse_args(argv, out, err)
log.set_verbosity(options.verbosity)
try:
filename = os.path.abspath(options.file.name)
ast, read = parse_file(filename, options)
except (ASTError, ParseError) as e:
die(options, e.args)
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' %
(options.save_ast.name, ' \\\n '.join(sorted(read))))
pickle.dump(ast, options.save_ast)
return 0
def main(argv, out, err):
options = parse_args(argv, out, err)
log.set_verbosity(options.verbosity)
try:
filename = os.path.abspath(options.file.name)
ast, read = parse_file(filename, options)
except (ASTError, ParseError) as e:
die(options, e.args)
if options.makefile_dependencies is not None:
options.makefile_dependencies.write('%s: \\\n %s\n' %
(options.save_ast.name, ' \\\n '.join(sorted(read))))
pickle.dump(ast, options.save_ast)
return 0
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# register object sizes with loader
if options.object_sizes:
register_object_sizes(
yaml.load(options.object_sizes, Loader=yaml.FullLoader))
# Ensure we were supplied equal items, outfiles and templates
if len(options.outfile) != len(options.item) != len(options.template):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in outfiles
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
ast = pickle.load(options.load_ast)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
try:
r = Renderer(options.templates)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
if options.load_object_state is not None:
render_state = pickle.load(options.load_object_state)
elif options.save_object_state is None:
render_state = None
else:
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = RenderState(obj_space=obj_space)
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
key = i.address_space
if key not in render_state.cspaces:
cnode = render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name="%s_cnode" % key,
label=key)
render_state.cspaces[key] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name="%s_group_bin_pd" % key,
label=key)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', "%s_group_bin" % key), pd)
render_state.pds[key] = pd
render_state.addr_spaces[key] = addr_space
for (item, outfile, template) in zip(options.item, options.outfile,
options.template):
key = item.split("/")
if key[0] == "component":
i = [
x for x in assembly.composition.instances if x.name == key[1]
][0]
obj_key = i.address_space
elif key[0] == "connector":
c = [
c for c in assembly.composition.connections if c.name == key[1]
][0]
if key[2] == "to":
i = c.to_ends[int(key[3])]
elif key[2] == "from":
i = c.from_ends[int(key[3])]
else:
die("Invalid connector end")
obj_key = i.instance.address_space
elif key[0] == "assembly":
i = assembly
obj_key = None
else:
die("item: \"%s\" does not have the correct formatting to render."
% item)
try:
g = r.render(i,
assembly,
template,
render_state,
obj_key,
outfile_name=outfile.name,
options=options,
my_pd=render_state.pds[obj_key] if obj_key else None)
outfile.write(g)
outfile.close()
except TemplateError as inst:
if hasattr(i, 'name'):
die(rendering_error(i.name, inst))
else:
die(rendering_error(i.parent.name, inst))
read = r.get_files_used()
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' %
(options.outfile[0].name, ' \\\n '.join(sorted(read))))
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(render_state, options.save_object_state)
sys.exit(0)
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
# Construct the compilation caches if requested.
cachea = None
cacheb = None
if options.cache:
# Construct a modified version of the command line arguments that we'll
# use in the keys to the caches. Essentially we elide --outfile and its
# parameter under the assumption that this value is never used in code
# generation. The purpose of this is to allow us to successfully cache
# ancillary outputs that we generate along the way to the current
# output. If we were to include --outfile in the key, future attempts
# to generate these ancillary outputs would unnecessarily miss the
# entries generated by this execution.
args = []
skip = False
for index, arg in enumerate(argv[1:]):
if skip:
skip = False
continue
if arg in ('--outfile', '-O'):
skip = True
continue
args.append(arg)
cachea = LevelACache(
os.path.join(options.cache_dir, version(), 'cachea'))
cacheb = LevelBCache(
os.path.join(options.cache_dir, version(), 'cacheb'))
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
if cachea is not None:
try:
cachea.flush()
except sqlite3.OperationalError as e:
# The following suppresses two spurious errors:
# 1. The database is locked. In a large, parallel build, writes
# to the level A cache are heavily contended and this error
# can occur.
# 2. The database structure is unexpected. If the CAmkES
# sources have changed *while* the runner was executing,
# the level A cache can be looking in a different place to
# where the cache was created.
# Both of these are non-critical (will just result in a
# potential future cache miss) so there's no need to alarm the
# user.
if re.search(r'database is locked', str(e)) is not None or \
re.search(r'no such table', str(e)) is not None:
log.debug('failed to flush level A cache: %s' % str(e))
else:
raise
if cacheb is not None:
try:
cacheb.flush()
except sqlite3.OperationalError as e:
# As above for the level B cache.
if re.search(r'database is locked', str(e)):
log.debug('failed to flush level B cache: %s' % str(e))
else:
raise
done_items.add((item, file))
if len(all_items - done_items) == 0:
sys.exit(ret)
# Try to find this output in the level A cache if possible. This check will
# 'hit' if the source files representing the input spec are identical to
# some previously observed execution.
if cachea is not None:
assert 'args' in locals()
assert len(options.outfile) == 1, 'level A cache only supported when requestiong ' \
'single items'
output = cachea.load(args, cwd)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level A cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag,
options.import_path, options.verbosity,
options.allow_forward_references)
ast, read = parse_file_cached(filename,
options.data_structure_cache_dir,
parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = collections.defaultdict(ShmemFactory())
kept_symbols = {}
fill_frames = {}
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
if cacheb is not None:
ast_hash = level_b_prime(ast)
assert 'args' in locals()
assert len(options.item) == 1, 'level B cache only supported when requesting ' \
'single items'
output = cacheb.load(ast_hash, args,
set(options.elf) | extra_templates)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level B cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' % (filename, ' \\\n '.join(sorted(read))))
# If we have a cache, allow outputs to be saved to it.
if options.cache:
assert cachea is not None, 'level A cache not available, though the ' \
'cache is enabled (bug in runner?)'
# The logic of this cache currently only works when a single item is requested
# on the command line
assert len(options.item) == 1, 'level A cache only supported when requesting ' \
'single items'
# Calculate the input files to the level A cache.
inputs = level_a_prime(read)
# Work out the position of the --item argument in the command line
# parameters. We will use this to cache not only outputs for this
# execution, but also outputs for ones with a different target.
item_index = None
assert 'args' in locals()
for index, arg in enumerate(args[:-1]):
if arg in ('--item', '-T'):
item_index = index + 1
break
assert item_index is not None, 'failed to find required argument ' \
'--item (bug in runner?)'
# We should already have the necessary inputs for the level B cache.
assert cacheb is not None, 'level B cache not available, though the ' \
'cache is enabled (bug in runner?)'
assert ast_hash is not None, 'AST hash not pre-computed (bug in ' \
'runner?)'
def save(item, value):
# Juggle the command line arguments to cache the predicted
# arguments for a call that would generate this item.
new_args = args[:item_index] + [item] + args[item_index + 1:]
# Save entries in both caches.
cachea.save(new_args, cwd, value, inputs)
if item != 'Makefile' and item != 'camkes-gen.cmake':
# We avoid caching the generated Makefile because it is not
# safe. The inputs to generation of the Makefile are not only
# the AST, but also the file names (`inputs`). If we cache it in
# the level B cache we risk the following scenario:
#
# 1. Generate the Makefile, caching it in the level B cache;
# 2. Modify the spec to import a file containing only white
# space and/or comments; then
# 3. Generate the Makefile, missing the level A cache, but
# hitting the level B cache.
#
# At this point, the generated Makefile is incorrect because it
# does not capture any dependencies on the imported file. We can
# now introduce something semantically relevant into this file
# (e.g. an Assembly block) and it will not be seen by the build
# system.
cacheb.save(ast_hash, new_args,
set(options.elf) | extra_templates, value)
else:
def save(item, value):
pass
def apply_capdl_filters():
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' %
name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False,
infer_asid=False,
pd=pds[group],
use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
# It's only relevant to run these filters if the final target is CapDL.
# Note, this will no longer be true if we add any other templates that
# depend on a fully formed CapDL spec. Guarding this loop with an if
# is just an optimisation and the conditional can be removed if
# desired.
filteroptions = FilterOptions(
options.architecture, options.realtime, options.largeframe,
options.largeframe_dma, options.default_priority,
options.default_max_priority, options.default_criticality,
options.default_max_criticality, options.default_affinity,
options.default_period, options.default_budget,
options.default_data, options.default_size_bits,
options.debug_fault_handlers, options.fprovide_tcb_caps)
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast,
obj_space=obj_space,
cspaces=cspaces,
elfs=elfs,
options=filteroptions,
shmem=shmem,
fill_frames=fill_frames)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(
options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps,
options.fsupport_init, options.largeframe, options.largeframe_dma,
options.architecture, options.debug_fault_handlers, options.realtime)
def instantiate_misc_template():
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly,
assembly,
template,
obj_space,
None,
shmem,
kept_symbols,
fill_frames,
imported=read,
options=renderoptions)
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
if options.item[0] in ('capdl', 'label-mapping') and options.data_structure_cache_dir is not None \
and len(options.outfile) == 1:
# It's possible that data structures required to instantiate the capdl spec
# were saved during a previous invocation of this script in the current build.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
if os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_file:
# Found a cached version of the necessary data structures
obj_space, shmem, cspaces, pds, kept_symbols, fill_frames = pickle.load(
pickle_file)
apply_capdl_filters()
instantiate_misc_template()
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError(
"No template instantiated on capdl generation fastpath")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER,
instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'],
label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
obj_space,
cspaces[i.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die([
'While rendering %s: %s' % (i.name, line)
for line in inst.args
])
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name,
c.from_ends), ('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name,
c.to_ends), ('%s/to/header' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e,
assembly,
template,
obj_space,
cspaces[e.instance.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[e.instance.address_space])
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
save(item, g)
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends), ('%s/to/' % c.name,
c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e,
assembly,
template,
obj_space,
cspaces[e.instance.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[e.instance.address_space])
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'),
('rump_config', 'rumprun')]
for special in [
bl for bl in SPECIAL_TEMPLATES if conf[i.name].get(bl[0])
]:
for t in ('%s/%s' % (i.name, special[1]), ):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
obj_space,
cspaces[i.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (i.name, line)
for line in inst.args
])
if options.data_structure_cache_dir is not None:
# At this point the capdl database is in the state required for applying capdl
# filters and generating the capdl spec. In case the capdl spec isn't the current
# target, we pickle the database here, so when the capdl spec is built, these
# data structures don't need to be regenerated.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
with open(pickle_path, 'wb') as pickle_file:
pickle.dump(
(obj_space, shmem, cspaces, pds, kept_symbols, fill_frames),
pickle_file)
for (item, outfile) in (all_items - done_items):
if item in ('capdl', 'label-mapping'):
apply_capdl_filters()
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
instantiate_misc_template()
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# register object sizes with loader
if options.object_sizes:
register_object_sizes(
yaml.load(options.object_sizes, Loader=yaml.FullLoader))
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
ast = pickle.load(options.load_ast)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs.
templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
if options.load_object_state is not None:
render_state = pickle.load(options.load_object_state)
else:
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = AllocatorState(obj_space=obj_space)
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
key = i.address_space
if key not in render_state.cspaces:
cnode = render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name="%s_cnode" % key,
label=key)
render_state.cspaces[key] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name="%s_group_bin_pd" % key,
label=key)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', "%s_group_bin" % key), pd)
render_state.pds[key] = pd
render_state.addr_spaces[key] = addr_space
for (item, outfile) in zip(options.item, options.outfile):
key = item.split("/")
if len(key) is 1:
# We are rendering something that isn't a component or connection.
i = assembly
obj_key = None
template = templates.lookup(item)
elif key[1] in [
"source", "header", "c_environment_source",
"cakeml_start_source", "cakeml_end_source", "camkesConstants",
"linker", "debug", "simple", "rump_config"
]:
# We are rendering a component template
i = [
x for x in assembly.composition.instances if x.name == key[0]
][0]
obj_key = i.address_space
template = templates.lookup(item, i)
elif key[1] in ["from", "to"]:
# We are rendering a connection template
c = [
c for c in assembly.composition.connections if c.name == key[0]
][0]
if key[1] == "to":
i = c.to_ends[int(key[-1])]
elif key[1] == "from":
i = c.from_ends[int(key[-1])]
else:
die("Invalid connector end")
obj_key = i.instance.address_space
template = templates.lookup("/".join(key[:-1]), c)
else:
die("item: \"%s\" does not have the correct formatting to lookup a template."
% item)
try:
g = r.render(i,
assembly,
template,
render_state,
obj_key,
outfile_name=outfile.name,
options=options,
my_pd=render_state.pds[obj_key] if obj_key else None)
outfile.write(g)
outfile.close()
except TemplateError as inst:
die(rendering_error(i.name, inst))
read = r.get_files_used()
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' %
(options.outfile[0].name, ' \\\n '.join(sorted(read))))
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(render_state, options.save_object_state)
sys.exit(0)
def main():
options = parse_args(constants.TOOL_RUNNER)
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options.verbosity >= 3)
log.set_verbosity(options.verbosity)
def done(s):
ret = 0
if s:
options.outfile.write(s)
options.outfile.close()
sys.exit(ret)
if not options.platform or options.platform in ('?', 'help') \
or options.platform not in PLATFORMS:
die('Valid --platform arguments are %s' % ', '.join(PLATFORMS))
if not options.file or len(options.file) > 1:
die('A single input file must be provided for this operation')
# Construct the compilation cache if requested.
cache = None
if options.cache in ('on', 'readonly', 'writeonly'):
cache = Cache(options.cache_dir)
f = options.file[0]
try:
s = f.read()
# Try to find this output in the compilation cache if possible. This is
# one of two places that we check in the cache. This check will 'hit'
# if the source files representing the input spec are identical to some
# previous execution.
if options.cache in ('on', 'readonly'):
key = [
version_hash(),
os.path.abspath(f.name), s,
cache_relevant_options(options), options.platform, options.item
]
value = cache.get(key)
assert value is None or isinstance(value, FileSet), \
'illegally cached a value for %s that is not a FileSet' % options.item
if value is not None and value.valid():
# Cache hit.
log.debug('Retrieved %(platform)s.%(item)s from cache' % \
options.__dict__)
done(value.output)
ast = parser.parse_to_ast(s, options.cpp, options.cpp_flag,
options.ply_optimise)
parser.assign_filenames(ast, f.name)
except parser.CAmkESSyntaxError as e:
e.set_column(s)
die('%s:%s' % (f.name, str(e)))
except Exception as inst:
die('While parsing \'%s\': %s' % (f.name, inst))
try:
for t in AST_TRANSFORMS[PRE_RESOLUTION]:
ast = t(ast)
except Exception as inst:
die('While transforming AST: %s' % str(inst))
try:
ast, imported = parser.resolve_imports(ast, \
os.path.dirname(os.path.abspath(f.name)), options.import_path,
options.cpp, options.cpp_flag, options.ply_optimise)
except Exception as inst:
die('While resolving imports of \'%s\': %s' % (f.name, inst))
try:
# if there are multiple assemblies, combine them now
compose_assemblies(ast)
except Exception as inst:
die('While combining assemblies: %s' % str(inst))
# If we have a readable cache check if our current target is in the cache.
# The previous check will 'miss' and this one will 'hit' when the input
# spec is identical to some previous execution modulo a semantically
# irrelevant element (e.g. an introduced comment). I.e. the previous check
# matches when the input is exactly the same and this one matches when the
# AST is unchanged.
if options.cache in ('on', 'readonly'):
key = [
version_hash(), ast,
cache_relevant_options(options), options.platform, options.item
]
value = cache.get(key)
if value is not None:
assert options.item not in NEVER_AST_CACHE, \
'%s, that is marked \'never cache\' is in your cache' % options.item
log.debug('Retrieved %(platform)s.%(item)s from cache' % \
options.__dict__)
done(value)
# If we have a writable cache, allow outputs to be saved to it.
if options.cache in ('on', 'writeonly'):
orig_ast = deepcopy(ast)
fs = FileSet(imported)
def save(item, value):
# Save an input-keyed cache entry. This one is based on the
# pre-parsed inputs to save having to derive the AST (parse the
# input) in order to locate a cache entry in following passes.
# This corresponds to the first cache check above.
key = [
version_hash(),
os.path.abspath(options.file[0].name), s,
cache_relevant_options(options), options.platform, item
]
specialised = fs.specialise(value)
if item == 'capdl':
specialised.extend(options.elf)
cache[key] = specialised
if item not in NEVER_AST_CACHE:
# Save an AST-keyed cache entry. This corresponds to the second
# cache check above.
cache[[
version_hash(), orig_ast,
cache_relevant_options(options), options.platform, item
]] = value
else:
def save(item, value):
pass
ast = parser.dedupe(ast)
try:
ast = parser.resolve_references(ast)
except Exception as inst:
die('While resolving references of \'%s\': %s' % (f.name, inst))
try:
parser.collapse_references(ast)
except Exception as inst:
die('While collapsing references of \'%s\': %s' % (f.name, inst))
try:
for t in AST_TRANSFORMS[POST_RESOLUTION]:
ast = t(ast)
except Exception as inst:
die('While transforming AST: %s' % str(inst))
try:
resolve_hierarchy(ast)
except Exception as inst:
die('While resolving hierarchy: %s' % str(inst))
# All references in the AST need to be resolved for us to continue.
unresolved = reduce(lambda a, x: a.union(x),
map(lambda x: x.unresolved(), ast), set())
if unresolved:
die('Unresolved references in input specification:\n %s' % \
'\n '.join(map(lambda x: '%(filename)s:%(lineno)s:\'%(name)s\' of type %(type)s' % {
'filename':x.filename or '<unnamed file>',
'lineno':x.lineno,
'name':x._symbol,
'type':x._type.__name__,
}, unresolved)))
# Locate the assembly
assembly = [x for x in ast if isinstance(x, AST.Assembly)]
if len(assembly) > 1:
die('Multiple assemblies found')
elif len(assembly) == 1:
assembly = assembly[0]
else:
die('No assembly found')
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = defaultdict(dict)
templates = Templates(options.platform)
map(templates.add_root, options.templates)
r = Renderer(templates.get_roots(), options)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries.
for c in (x for x in ast if isinstance(x, AST.Connector)):
if c.from_template:
templates.add(c.name, 'from.source', c.from_template)
if c.to_template:
templates.add(c.name, 'to.source', c.to_template)
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for id, i in enumerate(assembly.composition.instances):
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER,
instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'],
label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(seL4_PageDirectoryObject,
name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ('%s.source' % i.name, '%s.header' % i.name,
'%s.linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space], \
shmem, options=options, id=id, my_pd=pds[i.address_space])
save(t, g)
if options.item == t:
if not template:
log.warning('Warning: no template for %s' %
options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (i.name, inst))
# Instantiate the per-connection files.
conn_dict = {}
for id, c in enumerate(assembly.composition.connections):
tmp_name = c.name
key_from = (c.from_instance.name + '_' +
c.from_interface.name) in conn_dict
key_to = (c.to_instance.name + '_' + c.to_interface.name) in conn_dict
if not key_from and not key_to:
# We need a new connection name
conn_name = 'conn' + str(id)
c.name = conn_name
conn_dict[c.from_instance.name + '_' +
c.from_interface.name] = conn_name
conn_dict[c.to_instance.name + '_' +
c.to_interface.name] = conn_name
elif not key_to:
conn_name = conn_dict[c.from_instance.name + '_' +
c.from_interface.name]
c.name = conn_name
conn_dict[c.to_instance.name + '_' +
c.to_interface.name] = conn_name
elif not key_from:
conn_name = conn_dict[c.to_instance.name + '_' +
c.to_interface.name]
c.name = conn_name
conn_dict[c.from_instance.name + '_' +
c.from_interface.name] = conn_name
else:
continue
for t in (('%s.from.source' % tmp_name, c.from_instance.address_space),
('%s.from.header' % tmp_name, c.from_instance.address_space),
('%s.to.source' % tmp_name, c.to_instance.address_space),
('%s.to.header' % tmp_name, c.to_instance.address_space)):
try:
template = templates.lookup(t[0], c)
g = ''
if template:
g = r.render(c, assembly, template, obj_space, cspaces[t[1]], \
shmem, options=options, id=id, my_pd=pds[t[1]])
save(t[0], g)
if options.item == t[0]:
if not template:
log.warning('Warning: no template for %s' %
options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (t[0], inst))
c.name = tmp_name
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for t in (('%s.from.' % c.name, c.from_instance.address_space),
('%s.to.' % c.name, c.to_instance.address_space)):
if not options.item.startswith(t[0]):
# This is not the item we're looking for.
continue
try:
# If we've reached here then this is the exact item we're
# after.
template = templates.lookup(options.item, c)
if template is None:
raise Exception('no registered template for %s' %
options.item)
g = r.render(c, assembly, template, obj_space, cspaces[t[1]], \
shmem, options=options, id=id, my_pd=pds[t[1]])
save(options.item, g)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (options.item, inst))
# Perform any per component simple generation. This needs to happen last
# as this template needs to run after all other capabilities have been
# allocated
for id, i in enumerate(assembly.composition.instances):
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
if conf and conf.settings and [x for x in conf.settings if \
x.instance == i.name and x.attribute == 'simple' and x.value]:
for t in ('%s.simple' % i.name, ):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space], \
shmem, options=options, id=id, my_pd=pds[i.address_space])
save(t, g)
if options.item == t:
if not template:
log.warning('Warning: no template for %s' %
options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (i.name, inst))
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' % name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False,
infer_asid=False,
pd=pds[group],
use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
if options.item in ('capdl', 'label-mapping'):
# It's only relevant to run these filters if the final target is CapDL.
# Note, this will no longer be true if we add any other templates that
# depend on a fully formed CapDL spec. Guarding this loop with an if
# is just an optimisation and the conditional can be removed if
# desired.
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast,
obj_space=obj_space,
cspaces=cspaces,
elfs=elfs,
options=options,
shmem=shmem)
except Exception as inst:
die('While forming CapDL spec: %s' % str(inst))
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
try:
template = templates.lookup(options.item)
if template:
g = r.render(assembly, assembly, template, obj_space, None, \
shmem, imported=imported, options=options)
save(options.item, g)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (options.item, inst))
die('No valid element matching --item %s' % options.item)
def main():
options = parse_args(constants.TOOL_RUNNER)
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options.verbosity >= 3)
log.set_verbosity(options.verbosity)
def done(s):
ret = 0
if s:
print >>options.outfile, s
options.outfile.close()
if options.post_render_edit and \
raw_input('Edit rendered template %s [y/N]? ' % \
options.outfile.name) == 'y':
editor = os.environ.get('EDITOR', 'vim')
ret = subprocess.call([editor, options.outfile.name])
sys.exit(ret)
if not options.platform or options.platform in ['?', 'help'] \
or options.platform not in PLATFORMS:
die('Valid --platform arguments are %s' % ', '.join(PLATFORMS))
if not options.file or len(options.file) > 1:
die('A single input file must be provided for this operation')
try:
profiler = get_profiler(options.profiler, options.profile_log)
except Exception as inst:
die('Failed to create profiler: %s' % str(inst))
# Construct the compilation cache if requested.
cache = None
if options.cache in ['on', 'readonly', 'writeonly']:
cache = Cache(options.cache_dir)
f = options.file[0]
try:
with profiler('Reading input'):
s = f.read()
# Try to find this output in the compilation cache if possible. This is
# one of two places that we check in the cache. This check will 'hit'
# if the source files representing the input spec are identical to some
# previous execution.
if options.cache in ['on', 'readonly']:
with profiler('Looking for a cached version of this output'):
key = [version(), os.path.abspath(f.name), s,
cache_relevant_options(options), options.platform,
options.item]
value = cache.get(key)
if value is not None and value.valid():
# Cache hit.
assert isinstance(value, FileSet), \
'illegally cached a value for %s that is not a FileSet' % options.item
log.debug('Retrieved %(platform)s.%(item)s from cache' % \
options.__dict__)
done(value.output)
with profiler('Parsing input'):
ast = parser.parse_to_ast(s, options.cpp, options.cpp_flag, options.ply_optimise)
parser.assign_filenames(ast, f.name)
except parser.CAmkESSyntaxError as e:
e.set_column(s)
die('%s:%s' % (f.name, str(e)))
except Exception as inst:
die('While parsing \'%s\': %s' % (f.name, str(inst)))
try:
for t in AST_TRANSFORMS[PRE_RESOLUTION]:
with profiler('Running AST transform %s' % t.__name__):
ast = t(ast)
except Exception as inst:
die('While transforming AST: %s' % str(inst))
try:
with profiler('Resolving imports'):
ast, imported = parser.resolve_imports(ast, \
os.path.dirname(os.path.abspath(f.name)), options.import_path,
options.cpp, options.cpp_flag, options.ply_optimise)
except Exception as inst:
die('While resolving imports of \'%s\': %s' % (f.name, str(inst)))
try:
with profiler('Combining assemblies'):
# if there are multiple assemblies, combine them now
compose_assemblies(ast)
except Exception as inst:
die('While combining assemblies: %s' % str(inst))
with profiler('Caching original AST'):
orig_ast = deepcopy(ast)
with profiler('Deduping AST'):
ast = parser.dedupe(ast)
try:
with profiler('Resolving references'):
ast = parser.resolve_references(ast)
except Exception as inst:
die('While resolving references of \'%s\': %s' % (f.name, str(inst)))
try:
with profiler('Collapsing references'):
parser.collapse_references(ast)
except Exception as inst:
die('While collapsing references of \'%s\': %s' % (f.name, str(inst)))
try:
for t in AST_TRANSFORMS[POST_RESOLUTION]:
with profiler('Running AST transform %s' % t.__name__):
ast = t(ast)
except Exception as inst:
die('While transforming AST: %s' % str(inst))
try:
with profiler('Resolving hierarchy'):
resolve_hierarchy(ast)
except Exception as inst:
die('While resolving hierarchy: %s' % str(inst))
# If we have a readable cache check if our current target is in the cache.
# The previous check will 'miss' and this one will 'hit' when the input
# spec is identical to some previous execution modulo a semantically
# irrelevant element (e.g. an introduced comment). I.e. the previous check
# matches when the input is exactly the same and this one matches when the
# AST is unchanged.
if options.cache in ['on', 'readonly']:
with profiler('Looking for a cached version of this output'):
key = [version(), orig_ast, cache_relevant_options(options),
options.platform, options.item]
value = cache.get(key)
if value is not None:
assert options.item not in NEVER_AST_CACHE, \
'%s, that is marked \'never cache\' is in your cache' % options.item
log.debug('Retrieved %(platform)s.%(item)s from cache' % \
options.__dict__)
done(value)
# If we have a writable cache, allow outputs to be saved to it.
if options.cache in ['on', 'writeonly']:
fs = FileSet(imported)
def save(item, value):
# Save an input-keyed cache entry. This one is based on the
# pre-parsed inputs to save having to derive the AST (parse the
# input) in order to locate a cache entry in following passes.
# This corresponds to the first cache check above.
key = [version(), os.path.abspath(options.file[0].name), s,
cache_relevant_options(options), options.platform,
item]
specialised = fs.specialise(value)
if item == 'capdl':
specialised.extend(options.elf or [])
cache[key] = specialised
if item not in NEVER_AST_CACHE:
# Save an AST-keyed cache entry. This corresponds to the second
# cache check above.
cache[[version(), orig_ast, cache_relevant_options(options),
options.platform, item]] = value
else:
def save(item, value):
pass
# All references in the AST need to be resolved for us to continue.
unresolved = reduce(lambda a, x: a.union(x),
map(lambda x: x.unresolved(), ast), set())
if unresolved:
die('Unresolved references in input specification:\n %s' % \
'\n '.join(map(lambda x: '%(filename)s:%(lineno)s:\'%(name)s\' of type %(type)s' % {
'filename':x.filename or '<unnamed file>',
'lineno':x.lineno,
'name':x._symbol,
'type':x._type.__name__,
}, unresolved)))
# Locate the assembly
assembly = [x for x in ast if isinstance(x, AST.Assembly)]
if len(assembly) > 1:
die('Multiple assemblies found')
elif len(assembly) == 1:
assembly = assembly[0]
else:
die('No assembly found')
obj_space = ObjectAllocator()
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = defaultdict(dict)
# We need to create a phony instance and connection to cope with cases
# where the user has not defined any instances or connections (this would
# be an arguably useless system, but we should still support it). We append
# these to the template's view of the system below to ensure we always get
# a usable template dictionary. Note that this doesn't cause any problems
# because the phony items are named '' and thus unaddressable in ADL.
dummy_instance = AST.Instance(AST.Reference('', AST.Instance), '')
dummy_connection = AST.Connection(AST.Reference('', AST.Connector), '', \
AST.Reference('', AST.Instance), AST.Reference('', AST.Interface), \
AST.Reference('', AST.Instance), AST.Reference('', AST.Interface))
templates = Templates(options.platform,
instance=map(lambda x: x.name, assembly.composition.instances + \
[dummy_instance]), \
connection=map(lambda x: x.name, assembly.composition.connections + \
[dummy_connection]))
if options.templates:
templates.add_root(options.templates)
r = Renderer(templates.get_roots(), options)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries.
for c in [x for x in ast if isinstance(x, AST.Connector)]:
if c.from_template:
templates.add(c.name, 'from.source', c.from_template)
if c.to_template:
templates.add(c.name, 'to.source', c.to_template)
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for id, i in enumerate(assembly.composition.instances):
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER, instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'], label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(seL4_PageDirectoryObject, name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ['%s.source' % i.name, '%s.header' % i.name,
'%s.linker' % i.name]:
try:
template = templates.lookup(t, i)
g = ''
if template:
with profiler('Rendering %s' % t):
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space], \
shmem, options=options, id=id, my_pd=pds[i.address_space])
save(t, g)
if options.item == t:
if not template:
log.warning('Warning: no template for %s' % options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (i.name, str(inst)))
# Instantiate the per-connection files.
conn_dict = {}
for id, c in enumerate(assembly.composition.connections):
tmp_name = c.name
key_from = (c.from_instance.name + '_' + c.from_interface.name) in conn_dict
key_to = (c.to_instance.name + '_' + c.to_interface.name) in conn_dict
if not key_from and not key_to:
# We need a new connection name
conn_name = 'conn' + str(id)
c.name = conn_name
conn_dict[c.from_instance.name + '_' + c.from_interface.name] = conn_name
conn_dict[c.to_instance.name + '_' + c.to_interface.name] = conn_name
elif not key_to:
conn_name = conn_dict[c.from_instance.name + '_' + c.from_interface.name]
c.name = conn_name
conn_dict[c.to_instance.name + '_' + c.to_interface.name] = conn_name
elif not key_from:
conn_name = conn_dict[c.to_instance.name + '_' + c.to_interface.name]
c.name = conn_name
conn_dict[c.from_instance.name + '_' + c.from_interface.name] = conn_name
else:
continue
for t in [('%s.from.source' % tmp_name, c.from_instance.address_space),
('%s.from.header' % tmp_name, c.from_instance.address_space),
('%s.to.source' % tmp_name, c.to_instance.address_space),
('%s.to.header' % tmp_name, c.to_instance.address_space)]:
try:
template = templates.lookup(t[0], c)
g = ''
if template:
with profiler('Rendering %s' % t[0]):
g = r.render(c, assembly, template, obj_space, cspaces[t[1]], \
shmem, options=options, id=id, my_pd=pds[t[1]])
save(t[0], g)
if options.item == t[0]:
if not template:
log.warning('Warning: no template for %s' % options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (t[0], str(inst)))
c.name = tmp_name
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for t in [('%s.from.' % c.name, c.from_instance.address_space),
('%s.to.' % c.name, c.to_instance.address_space)]:
if not options.item.startswith(t[0]):
# This is not the item we're looking for.
continue
try:
# If we've reached here then this is the exact item we're
# after.
template = templates.lookup(options.item, c)
if template is None:
raise Exception('no registered template for %s' % options.item)
with profiler('Rendering %s' % options.item):
g = r.render(c, assembly, template, obj_space, cspaces[t[1]], \
shmem, options=options, id=id, my_pd=pds[t[1]])
save(options.item, g)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (options.item, str(inst)))
# Perform any per component simple generation. This needs to happen last
# as this template needs to run after all other capabilities have been
# allocated
for id, i in enumerate(assembly.composition.instances):
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
if conf and conf.settings and [x for x in conf.settings if \
x.instance == i.name and x.attribute == 'simple' and x.value]:
for t in ['%s.simple' % i.name]:
try:
template = templates.lookup(t, i)
g = ''
if template:
with profiler('Rendering %s' % t):
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space], \
shmem, options=options, id=id, my_pd=pds[i.address_space])
save(t, g)
if options.item == t:
if not template:
log.warning('Warning: no template for %s' % options.item)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (i.name, str(inst)))
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
arch = None
for e in options.elf or []:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception('duplicate ELF files of name \'%s\' encountered' % name)
elf = ELF(e, name)
if not arch:
# The spec's arch will have defaulted to ARM, but we want it to
# be the same as whatever ELF format we're parsing.
arch = elf.get_arch()
if arch == 'ARM':
obj_space.spec.arch = 'arm11'
elif arch == 'x86':
obj_space.spec.arch = 'ia32'
else:
raise NotImplementedError
else:
# All ELF files we're parsing should be the same format.
if arch != elf.get_arch():
raise Exception('ELF files are not all the same architecture')
# Pass 'False' to avoid inferring a TCB as we've already created
# our own.
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
with profiler('Deriving CapDL spec from %s' % e):
elf_spec = elf.get_spec(infer_tcb=False, infer_asid=False,
pd=pds[group], use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, str(inst)))
if options.item in ['capdl', 'label-mapping']:
# It's only relevant to run these filters if the final target is CapDL.
# Note, this will no longer be true if we add any other templates that
# depend on a fully formed CapDL spec. Guarding this loop with an if
# is just an optimisation and the conditional can be removed if
# desired.
for f in CAPDL_FILTERS:
try:
with profiler('Running CapDL filter %s' % f.__name__):
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast, obj_space=obj_space, cspaces=cspaces, elfs=elfs,
profiler=profiler, options=options, shmem=shmem)
except Exception as inst:
die('While forming CapDL spec: %s' % str(inst))
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
try:
template = templates.lookup(options.item)
g = ''
if template:
with profiler('Rendering %s' % options.item):
g = r.render(assembly, assembly, template, obj_space, None, \
shmem, imported=imported, options=options)
save(options.item, g)
done(g)
except Exception as inst:
die('While rendering %s: %s' % (options.item, str(inst)))
die('No valid element matching --item %s' % options.item)
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options, queries, filteroptions = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
done_items.add((item, file))
if len(all_items - done_items) == 0:
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(renderoptions.render_state,
options.save_object_state)
sys.exit(ret)
filename = None
if options.file is not None:
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag,
options.import_path, options.verbosity,
options.allow_forward_references,
options.save_ast, options.load_ast,
queries)
ast, read = parse_file_cached(filename, parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = RenderState(obj_space=obj_space)
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if filename and options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' % (filename, ' \\\n '.join(sorted(read))))
def apply_capdl_filters(renderoptions):
# Derive a set of usable ELF objects from the filenames we were passed.
render_state = renderoptions.render_state
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' %
name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(
infer_tcb=False,
infer_asid=False,
pd=render_state.pds[group],
use_large_frames=options.largeframe,
addr_space=render_state.addr_spaces[group])
render_state.obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast,
obj_space=render_state.obj_space,
cspaces=render_state.cspaces,
elfs=elfs,
options=filteroptions)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(
options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps,
options.fsupport_init, options.largeframe, options.largeframe_dma,
options.architecture, options.debug_fault_handlers,
options.default_stack_size, options.realtime,
options.verification_base_name, filteroptions, render_state)
def instantiate_misc_templates(renderoptions):
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly,
assembly,
template,
renderoptions.render_state,
None,
outfile_name=outfile.name,
imported=read,
options=renderoptions)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
if "camkes-gen.cmake" in options.item:
instantiate_misc_templates(renderoptions)
if options.load_object_state is not None:
# There is an assumption that if load_object_state is set, we
# skip all of the component and connector logic below.
# FIXME: refactor to clarify control flow
renderoptions.render_state = pickle.load(options.load_object_state)
apply_capdl_filters(renderoptions)
instantiate_misc_templates(renderoptions)
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError("No template instantiated on capdl generation path")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in renderoptions.render_state.cspaces:
p = Perspective(phase=RUNNER,
instance=i.name,
group=i.address_space)
cnode = renderoptions.render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name=p['cnode'],
label=i.address_space)
renderoptions.render_state.cspaces[
i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name=p['pd'],
label=i.address_space)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', p['elf_name']), pd)
renderoptions.render_state.pds[i.address_space] = pd
renderoptions.render_state.addr_spaces[
i.address_space] = addr_space
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/c_environment_source' % i.name,
'%s/cakeml_start_source' % i.name,
'%s/cakeml_end_source' % i.name, '%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(
i,
assembly,
template,
renderoptions.render_state,
i.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[i.address_space])
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name,
c.from_ends), ('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name,
c.to_ends), ('%s/to/header' % c.name, c.to_ends),
('%s/to/cakeml' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e,
assembly,
template,
renderoptions.render_state,
e.instance.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
e.instance.address_space])
except TemplateError as inst:
die(rendering_error(item, inst))
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends), ('%s/to/' % c.name,
c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e,
assembly,
template,
renderoptions.render_state,
e.instance.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
e.instance.address_space])
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in renderoptions.render_state.cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'),
('rump_config', 'rumprun')]
for special in [
bl for bl in SPECIAL_TEMPLATES
if assembly.configuration[i.name].get(bl[0])
]:
for t in ('%s/%s' % (i.name, special[1]), ):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
renderoptions.render_state,
i.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
i.address_space])
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write('Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
# Construct the compilation caches if requested.
cachea = None
cacheb = None
if options.cache:
# Construct a modified version of the command line arguments that we'll
# use in the keys to the caches. Essentially we elide --outfile and its
# parameter under the assumption that this value is never used in code
# generation. The purpose of this is to allow us to successfully cache
# ancillary outputs that we generate along the way to the current
# output. If we were to include --outfile in the key, future attempts
# to generate these ancillary outputs would unnecessarily miss the
# entries generated by this execution.
args = []
skip = False
for index, arg in enumerate(argv[1:]):
if skip:
skip = False
continue
if arg in ('--outfile', '-O'):
skip = True
continue
args.append(arg)
cachea = LevelACache(os.path.join(options.cache_dir, version(), 'cachea'))
cacheb = LevelBCache(os.path.join(options.cache_dir, version(), 'cacheb'))
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
if cachea is not None:
try:
cachea.flush()
except sqlite3.OperationalError as e:
# The following suppresses two spurious errors:
# 1. The database is locked. In a large, parallel build, writes
# to the level A cache are heavily contended and this error
# can occur.
# 2. The database structure is unexpected. If the CAmkES
# sources have changed *while* the runner was executing,
# the level A cache can be looking in a different place to
# where the cache was created.
# Both of these are non-critical (will just result in a
# potential future cache miss) so there's no need to alarm the
# user.
if re.search(r'database is locked', str(e)) is not None or \
re.search(r'no such table', str(e)) is not None:
log.debug('failed to flush level A cache: %s' % str(e))
else:
raise
if cacheb is not None:
try:
cacheb.flush()
except sqlite3.OperationalError as e:
# As above for the level B cache.
if re.search(r'database is locked', str(e)):
log.debug('failed to flush level B cache: %s' % str(e))
else:
raise
done_items.add((item, file))
if len(all_items - done_items) == 0:
sys.exit(ret)
# Try to find this output in the level A cache if possible. This check will
# 'hit' if the source files representing the input spec are identical to
# some previously observed execution.
if cachea is not None:
assert 'args' in locals()
assert len(options.outfile) == 1, 'level A cache only supported when requestiong ' \
'single items'
output = cachea.load(args, cwd)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level A cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag, options.import_path, options.verbosity, options.allow_forward_references)
ast, read = parse_file_cached(filename, options.data_structure_cache_dir, parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' % (i.name,
a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' % (i.name,
a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = collections.defaultdict(ShmemFactory())
kept_symbols = {}
fill_frames = {}
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and
(x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast if isinstance(x, Connection) and
x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
if cacheb is not None:
ast_hash = level_b_prime(ast)
assert 'args' in locals()
assert len(options.item) == 1, 'level B cache only supported when requesting ' \
'single items'
output = cacheb.load(ast_hash, args, set(options.elf) | extra_templates)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level B cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write('%s: \\\n %s\n' %
(filename, ' \\\n '.join(sorted(read))))
# If we have a cache, allow outputs to be saved to it.
if options.cache:
assert cachea is not None, 'level A cache not available, though the ' \
'cache is enabled (bug in runner?)'
# The logic of this cache currently only works when a single item is requested
# on the command line
assert len(options.item) == 1, 'level A cache only supported when requesting ' \
'single items'
# Calculate the input files to the level A cache.
inputs = level_a_prime(read)
# Work out the position of the --item argument in the command line
# parameters. We will use this to cache not only outputs for this
# execution, but also outputs for ones with a different target.
item_index = None
assert 'args' in locals()
for index, arg in enumerate(args[:-1]):
if arg in ('--item', '-T'):
item_index = index + 1
break
assert item_index is not None, 'failed to find required argument ' \
'--item (bug in runner?)'
# We should already have the necessary inputs for the level B cache.
assert cacheb is not None, 'level B cache not available, though the ' \
'cache is enabled (bug in runner?)'
assert ast_hash is not None, 'AST hash not pre-computed (bug in ' \
'runner?)'
def save(item, value):
# Juggle the command line arguments to cache the predicted
# arguments for a call that would generate this item.
new_args = args[:item_index] + [item] + args[item_index + 1:]
# Save entries in both caches.
cachea.save(new_args, cwd, value, inputs)
if item != 'Makefile' and item != 'camkes-gen.cmake':
# We avoid caching the generated Makefile because it is not
# safe. The inputs to generation of the Makefile are not only
# the AST, but also the file names (`inputs`). If we cache it in
# the level B cache we risk the following scenario:
#
# 1. Generate the Makefile, caching it in the level B cache;
# 2. Modify the spec to import a file containing only white
# space and/or comments; then
# 3. Generate the Makefile, missing the level A cache, but
# hitting the level B cache.
#
# At this point, the generated Makefile is incorrect because it
# does not capture any dependencies on the imported file. We can
# now introduce something semantically relevant into this file
# (e.g. an Assembly block) and it will not be seen by the build
# system.
cacheb.save(ast_hash, new_args,
set(options.elf) | extra_templates, value)
else:
def save(item, value):
pass
def apply_capdl_filters():
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception('duplicate ELF files of name \'%s\' encountered' % name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False, infer_asid=False,
pd=pds[group], use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
filteroptions = FilterOptions(options.architecture, options.realtime, options.largeframe,
options.largeframe_dma, options.default_priority, options.default_max_priority,
options.default_affinity, options.default_period, options.default_budget,
options.default_data, options.default_size_bits,
options.debug_fault_handlers, options.fprovide_tcb_caps)
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast, obj_space=obj_space, cspaces=cspaces, elfs=elfs,
options=filteroptions, shmem=shmem, fill_frames=fill_frames)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps, options.fsupport_init,
options.largeframe, options.largeframe_dma, options.architecture, options.debug_fault_handlers,
options.realtime)
def instantiate_misc_template():
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly, assembly, template, obj_space, None,
shmem, kept_symbols, fill_frames, imported=read, options=renderoptions)
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
if options.item[0] in ('capdl', 'label-mapping') and options.data_structure_cache_dir is not None \
and len(options.outfile) == 1:
# It's possible that data structures required to instantiate the capdl spec
# were saved during a previous invocation of this script in the current build.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
if os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_file:
# Found a cached version of the necessary data structures
obj_space, shmem, cspaces, pds, kept_symbols, fill_frames = pickle.load(pickle_file)
apply_capdl_filters()
instantiate_misc_template()
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError("No template instantiated on capdl generation fastpath")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER, instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'], label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(options.architecture).vspace().object, name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/c_environment_source' % i.name,
'%s/cakeml_start_source' % i.name, '%s/cakeml_end_source' % i.name,
'%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space],
shmem, kept_symbols, fill_frames, options=renderoptions, my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name, c.from_ends),
('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name, c.to_ends),
('%s/to/header' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e, assembly, template, obj_space,
cspaces[e.instance.address_space], shmem, kept_symbols, fill_frames,
options=renderoptions, my_pd=pds[e.instance.address_space])
except TemplateError as inst:
die(rendering_error(item, inst))
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
save(item, g)
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends),
('%s/to/' % c.name, c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e, assembly, template, obj_space,
cspaces[e.instance.address_space], shmem, kept_symbols, fill_frames,
options=renderoptions, my_pd=pds[e.instance.address_space])
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'), ('rump_config', 'rumprun')]
for special in [bl for bl in SPECIAL_TEMPLATES if conf[i.name].get(bl[0])]:
for t in ('%s/%s' % (i.name, special[1]),):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space],
shmem, kept_symbols, fill_frames, options=renderoptions, my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(i.name, inst))
if options.data_structure_cache_dir is not None:
# At this point the capdl database is in the state required for applying capdl
# filters and generating the capdl spec. In case the capdl spec isn't the current
# target, we pickle the database here, so when the capdl spec is built, these
# data structures don't need to be regenerated.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
with open(pickle_path, 'wb') as pickle_file:
pickle.dump((obj_space, shmem, cspaces, pds, kept_symbols, fill_frames), pickle_file)
for (item, outfile) in (all_items - done_items):
if item in ('capdl', 'label-mapping'):
apply_capdl_filters()
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
instantiate_misc_template()
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
