def final_spec(cspaces, obj_space, addr_spaces, elf_files, architecture):
"""
Generates a final CapDL spec file that can be given to a capdl loader application
"""
arch = lookup_architecture(architecture)
for e in [item for sublist in elf_files for item in sublist]:
name = os.path.basename(e)
elf = ELF(e, name, architecture)
cspace = cspaces[name]
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False,
infer_asid=False,
pd=addr_spaces[name].vspace_root,
addr_space=addr_spaces[name])
obj_space.merge(elf_spec)
cspace.cnode.finalise_size(arch)
# Fill in TCB object information.
# TODO: This should be generalised with what is in the Camkes filters
tcb = obj_space["tcb_%s" % name]
progsymbol = elf.get_symbol_vaddr("progname")
vsyscall = elf.get_symbol_vaddr("sel4_vsyscall")
tcb.init = [0, 0, 0, 0, 2, progsymbol, 1, 0, 0, 32, vsyscall, 0, 0]
tcb.addr = elf.get_symbol_vaddr("mainIpcBuffer")
tcb.sp = elf.get_symbol_vaddr("stack") + elf.get_symbol_size("stack")
tcb.ip = elf.get_entry_point()
return obj_space
def final_spec(args, obj_space, cspaces, addr_spaces, targets, architecture):
"""
Generates a final CapDL spec file that can be given to a capdl loader application
"""
arch = lookup_architecture(architecture)
for e, key in targets:
name = os.path.basename(e)
elf = ELF(e, name, architecture)
cspace = cspaces[key]
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False, infer_asid=False,
pd=addr_spaces[key].vspace_root, addr_space=addr_spaces[key])
obj_space.merge(elf_spec, label=key)
for slot, v in cspace.cnode.slots.items():
if v is not None and isinstance(v.referent, TCB):
tcb = v.referent
if 'cspace' in tcb and tcb['cspace'] and tcb['cspace'].referent is not cspace.cnode:
# We exclude TCBs that refer to a different CSpace
continue
funcs = {"get_vaddr": lambda x: elf.get_symbol_vaddr(x)}
tcb.ip = eval(str(tcb.ip), {"__builtins__": None}, funcs)
tcb.sp = eval(str(tcb.sp), {"__builtins__": None}, funcs)
tcb.addr = eval(str(tcb.addr), {"__builtins__": None}, funcs)
tcb.init = eval(str(tcb.init), {"__builtins__": None}, funcs)
if not args.fprovide_tcb_caps:
del cspace.cnode[slot]
cspace.cnode.finalise_size(arch=arch)
return obj_space
def start_elf(self, name):
cnode = self.objects.alloc(ObjectType.seL4_CapTableObject, "cnode_%s" % name)
arch = self.objects.spec.arch.capdl_name()
pd = self.objects.alloc(lookup_architecture(arch).vspace().object, "vspace_%s" %name)
self.current_cspace = CSpaceAllocator(cnode)
self.current_addr_space = AddressSpaceAllocator(None, pd)
self.current_cap_symbols = []
self.current_region_symbols = []
def __init__(self, name, interface, intra_index, stack_size):
self.name = name
self.interface = interface
self.intra_index = intra_index
self.stack_symbol = "_camkes_stack_%s" % name
self.stack_size = stack_size
self.ipc_symbol = "_camkes_ipc_buffer_%s" % name
self.sp = "get_vaddr(\'%s\') + %d" % (self.stack_symbol, self.stack_size + PAGE_SIZE)
self.addr = "get_vaddr(\'%s\') + %d" % (self.ipc_symbol, 2 * PAGE_SIZE - lookup_architecture(options.architecture).ipc_buffer_size())
def __init__(self, name, interface, intra_index, stack_size):
self.name = name
self.interface = interface
self.intra_index = intra_index
self.stack_symbol = "_camkes_stack_%s" % name
self.stack_size = stack_size
self.ipc_symbol = "_camkes_ipc_buffer_%s" % name
self.sp = "get_vaddr(\'%s\') + %d" % (self.stack_symbol,
self.stack_size + PAGE_SIZE)
self.addr = "get_vaddr(\'%s\') + %d" % (
self.ipc_symbol, 2 * PAGE_SIZE -
lookup_architecture(options.architecture).ipc_buffer_size())
def guard_cnode_caps(cspaces, options, **_):
'''If the templates have allocated any caps to CNodes, they will not have
the correct guards. This is due to the CNodes' sizes being automatically
calculated (during set_tcb_caps above). Correct them here.'''
arch = lookup_architecture(options.architecture)
for space in cspaces.values():
[
cap.set_guard_size(arch.word_size_bits() - cap.referent.size_bits)
for cap in space.cnode.slots.values()
if cap is not None and isinstance(cap.referent, CNode)
]
def set_tcb_caps(ast, obj_space, cspaces, elfs, options, **_):
arch = lookup_architecture(options.architecture)
assembly = ast.assembly
for group, space in cspaces.items():
cnode = space.cnode
for index, tcb in [(k, v.referent) for (k, v) in cnode.slots.items()
if v is not None and isinstance(v.referent, TCB)]:
perspective = Perspective(tcb=tcb.name, group=group)
# Finalise the CNode so that we know what its absolute size will
# be. Note that we are assuming no further caps will be added to
# the CNode after this point.
cnode.finalise_size()
# Allow the user to override CNode sizes with the 'cnode_size_bits'
# attribute.
cnode_size = assembly.configuration[group].get('cnode_size_bits')
if cnode_size is not None:
try:
if isinstance(cnode_size, six.string_types):
size = int(cnode_size, 0)
else:
size = cnode_size
except ValueError:
raise Exception('illegal value for CNode size for %s' %
group)
if size < cnode.size_bits:
raise Exception('%d-bit CNode specified for %s, but this '
'CSpace needs to be at least %d bits' %
(size, group, cnode.size_bits))
cnode.size_bits = size
cspace = Cap(cnode)
cspace.set_guard_size(arch.word_size_bits() - cnode.size_bits)
tcb['cspace'] = cspace
pd = None
pd_name = perspective['pd']
pds = [x for x in obj_space.spec.objs if x.name == pd_name]
if len(pds) > 1:
raise Exception('Multiple PDs found for %s' % group)
elif len(pds) == 1:
pd, = pds
tcb['vspace'] = Cap(pd)
# If no PD was found we were probably just not passed any ELF files
# in this pass.
if perspective['pool']:
# This TCB is part of the (cap allocator's) TCB pool.
continue
def describe_fill_frames(ast, obj_space, elfs, fill_frames, options, **_):
if not elfs:
# If we haven't been passed any ELF files this step is not relevant yet.
return
arch = lookup_architecture(options.architecture)
assembly = ast.assembly
for name in fill_frames:
# Find it in assembly.composition.instances?
instances = [
x for x in assembly.composition.instances if x.name == name
]
assert len(instances) == 1, 'Found registered fill frame with no associated ' \
'instance (template bug?)'
instance, = instances
perspective = Perspective(instance=name, group=instance.address_space)
elf_name = perspective['elf_name']
assert elf_name in elfs, 'Failed to find binary image for instance %s' % name
elf = elfs[elf_name]
# Find the vspace root
root_name = perspective['pd']
roots = [x for x in obj_space.spec.objs if x.name == root_name]
assert len(roots) == 1, 'No vspace found for instance %s' % name
root, = roots
# Go over all the fill symbols
for symbol, fill in iter(fill_frames[name]):
base = get_symbol_vaddr(elf, symbol)
assert base is not None, 'Registered fill symbol not found in elf image (template bug?)'
# Ensure this symbol is correctly aligned
assert base % PAGE_SIZE == 0, 'Fill symbol in elf image is not correctly aligned (template bug?)'
(cap, frame) = frame_for_vaddr(arch, root, base, PAGE_SIZE)
assert frame is not None, 'Failed to find frame for symbol at %x (CAmkES bug?)' % base
frame.set_fill(fill)
def apply_capdl_filters(renderoptions):
# Derive a set of usable ELF objects from the filenames we were passed.
render_state = renderoptions.render_state
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' %
name)
elf = ELF(e, name, options.architecture)
group = name.replace("_group_bin", "")
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(
infer_tcb=False,
infer_asid=False,
pd=render_state.pds[group],
use_large_frames=options.largeframe,
addr_space=render_state.addr_spaces[group])
render_state.obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
for space in render_state.cspaces.values():
for (slot, tcb) in [
(v, v.referent) for (k, v) in space.cnode.slots.items()
if v is not None and isinstance(v.referent, TCB)
]:
elf = elfs.get(tcb.elf)
funcs = {"get_vaddr": lambda x: elf[1].get_symbol_vaddr(x)}
tcb.ip = simple_eval(str(tcb.ip), functions=funcs)
tcb.sp = simple_eval(str(tcb.sp), functions=funcs)
tcb.addr = simple_eval(str(tcb.addr), functions=funcs)
if not options.fprovide_tcb_caps:
del space.cnode[slot]
space.cnode.finalise_size(
arch=lookup_architecture(options.architecture))
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
# Construct the compilation caches if requested.
cachea = None
cacheb = None
if options.cache:
# Construct a modified version of the command line arguments that we'll
# use in the keys to the caches. Essentially we elide --outfile and its
# parameter under the assumption that this value is never used in code
# generation. The purpose of this is to allow us to successfully cache
# ancillary outputs that we generate along the way to the current
# output. If we were to include --outfile in the key, future attempts
# to generate these ancillary outputs would unnecessarily miss the
# entries generated by this execution.
args = []
skip = False
for index, arg in enumerate(argv[1:]):
if skip:
skip = False
continue
if arg in ('--outfile', '-O'):
skip = True
continue
args.append(arg)
cachea = LevelACache(
os.path.join(options.cache_dir, version(), 'cachea'))
cacheb = LevelBCache(
os.path.join(options.cache_dir, version(), 'cacheb'))
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
if cachea is not None:
try:
cachea.flush()
except sqlite3.OperationalError as e:
# The following suppresses two spurious errors:
# 1. The database is locked. In a large, parallel build, writes
# to the level A cache are heavily contended and this error
# can occur.
# 2. The database structure is unexpected. If the CAmkES
# sources have changed *while* the runner was executing,
# the level A cache can be looking in a different place to
# where the cache was created.
# Both of these are non-critical (will just result in a
# potential future cache miss) so there's no need to alarm the
# user.
if re.search(r'database is locked', str(e)) is not None or \
re.search(r'no such table', str(e)) is not None:
log.debug('failed to flush level A cache: %s' % str(e))
else:
raise
if cacheb is not None:
try:
cacheb.flush()
except sqlite3.OperationalError as e:
# As above for the level B cache.
if re.search(r'database is locked', str(e)):
log.debug('failed to flush level B cache: %s' % str(e))
else:
raise
done_items.add((item, file))
if len(all_items - done_items) == 0:
sys.exit(ret)
# Try to find this output in the level A cache if possible. This check will
# 'hit' if the source files representing the input spec are identical to
# some previously observed execution.
if cachea is not None:
assert 'args' in locals()
assert len(options.outfile) == 1, 'level A cache only supported when requestiong ' \
'single items'
output = cachea.load(args, cwd)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level A cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag,
options.import_path, options.verbosity,
options.allow_forward_references)
ast, read = parse_file_cached(filename,
options.data_structure_cache_dir,
parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = collections.defaultdict(ShmemFactory())
kept_symbols = {}
fill_frames = {}
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
if cacheb is not None:
ast_hash = level_b_prime(ast)
assert 'args' in locals()
assert len(options.item) == 1, 'level B cache only supported when requesting ' \
'single items'
output = cacheb.load(ast_hash, args,
set(options.elf) | extra_templates)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level B cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' % (filename, ' \\\n '.join(sorted(read))))
# If we have a cache, allow outputs to be saved to it.
if options.cache:
assert cachea is not None, 'level A cache not available, though the ' \
'cache is enabled (bug in runner?)'
# The logic of this cache currently only works when a single item is requested
# on the command line
assert len(options.item) == 1, 'level A cache only supported when requesting ' \
'single items'
# Calculate the input files to the level A cache.
inputs = level_a_prime(read)
# Work out the position of the --item argument in the command line
# parameters. We will use this to cache not only outputs for this
# execution, but also outputs for ones with a different target.
item_index = None
assert 'args' in locals()
for index, arg in enumerate(args[:-1]):
if arg in ('--item', '-T'):
item_index = index + 1
break
assert item_index is not None, 'failed to find required argument ' \
'--item (bug in runner?)'
# We should already have the necessary inputs for the level B cache.
assert cacheb is not None, 'level B cache not available, though the ' \
'cache is enabled (bug in runner?)'
assert ast_hash is not None, 'AST hash not pre-computed (bug in ' \
'runner?)'
def save(item, value):
# Juggle the command line arguments to cache the predicted
# arguments for a call that would generate this item.
new_args = args[:item_index] + [item] + args[item_index + 1:]
# Save entries in both caches.
cachea.save(new_args, cwd, value, inputs)
if item != 'Makefile' and item != 'camkes-gen.cmake':
# We avoid caching the generated Makefile because it is not
# safe. The inputs to generation of the Makefile are not only
# the AST, but also the file names (`inputs`). If we cache it in
# the level B cache we risk the following scenario:
#
# 1. Generate the Makefile, caching it in the level B cache;
# 2. Modify the spec to import a file containing only white
# space and/or comments; then
# 3. Generate the Makefile, missing the level A cache, but
# hitting the level B cache.
#
# At this point, the generated Makefile is incorrect because it
# does not capture any dependencies on the imported file. We can
# now introduce something semantically relevant into this file
# (e.g. an Assembly block) and it will not be seen by the build
# system.
cacheb.save(ast_hash, new_args,
set(options.elf) | extra_templates, value)
else:
def save(item, value):
pass
def apply_capdl_filters():
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' %
name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False,
infer_asid=False,
pd=pds[group],
use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
# It's only relevant to run these filters if the final target is CapDL.
# Note, this will no longer be true if we add any other templates that
# depend on a fully formed CapDL spec. Guarding this loop with an if
# is just an optimisation and the conditional can be removed if
# desired.
filteroptions = FilterOptions(
options.architecture, options.realtime, options.largeframe,
options.largeframe_dma, options.default_priority,
options.default_max_priority, options.default_criticality,
options.default_max_criticality, options.default_affinity,
options.default_period, options.default_budget,
options.default_data, options.default_size_bits,
options.debug_fault_handlers, options.fprovide_tcb_caps)
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast,
obj_space=obj_space,
cspaces=cspaces,
elfs=elfs,
options=filteroptions,
shmem=shmem,
fill_frames=fill_frames)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(
options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps,
options.fsupport_init, options.largeframe, options.largeframe_dma,
options.architecture, options.debug_fault_handlers, options.realtime)
def instantiate_misc_template():
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly,
assembly,
template,
obj_space,
None,
shmem,
kept_symbols,
fill_frames,
imported=read,
options=renderoptions)
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
if options.item[0] in ('capdl', 'label-mapping') and options.data_structure_cache_dir is not None \
and len(options.outfile) == 1:
# It's possible that data structures required to instantiate the capdl spec
# were saved during a previous invocation of this script in the current build.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
if os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_file:
# Found a cached version of the necessary data structures
obj_space, shmem, cspaces, pds, kept_symbols, fill_frames = pickle.load(
pickle_file)
apply_capdl_filters()
instantiate_misc_template()
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError(
"No template instantiated on capdl generation fastpath")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER,
instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'],
label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
obj_space,
cspaces[i.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die([
'While rendering %s: %s' % (i.name, line)
for line in inst.args
])
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name,
c.from_ends), ('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name,
c.to_ends), ('%s/to/header' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e,
assembly,
template,
obj_space,
cspaces[e.instance.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[e.instance.address_space])
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
save(item, g)
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends), ('%s/to/' % c.name,
c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e,
assembly,
template,
obj_space,
cspaces[e.instance.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[e.instance.address_space])
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (item, line)
for line in inst.args
])
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'),
('rump_config', 'rumprun')]
for special in [
bl for bl in SPECIAL_TEMPLATES if conf[i.name].get(bl[0])
]:
for t in ('%s/%s' % (i.name, special[1]), ):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
obj_space,
cspaces[i.address_space],
shmem,
kept_symbols,
fill_frames,
options=renderoptions,
my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
except TemplateError as inst:
die([
'While rendering %s: %s' % (i.name, line)
for line in inst.args
])
if options.data_structure_cache_dir is not None:
# At this point the capdl database is in the state required for applying capdl
# filters and generating the capdl spec. In case the capdl spec isn't the current
# target, we pickle the database here, so when the capdl spec is built, these
# data structures don't need to be regenerated.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
with open(pickle_path, 'wb') as pickle_file:
pickle.dump(
(obj_space, shmem, cspaces, pds, kept_symbols, fill_frames),
pickle_file)
for (item, outfile) in (all_items - done_items):
if item in ('capdl', 'label-mapping'):
apply_capdl_filters()
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
instantiate_misc_template()
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
def capdl_elf_vspace(context, elf_name, cap_symbol):
pd_type = lookup_architecture("x86_64").vspace().object
return TutorialFunctions.capdl_alloc_cap(context, pd_type, "vspace_%s" % elf_name, cap_symbol)
def get_word_size(arch):
return int(lookup_architecture(arch).word_size_bits()/8)
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# register object sizes with loader
if options.object_sizes:
register_object_sizes(
yaml.load(options.object_sizes, Loader=yaml.FullLoader))
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
ast = pickle.load(options.load_ast)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs.
templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
if options.load_object_state is not None:
render_state = pickle.load(options.load_object_state)
else:
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = AllocatorState(obj_space=obj_space)
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
key = i.address_space
if key not in render_state.cspaces:
cnode = render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name="%s_cnode" % key,
label=key)
render_state.cspaces[key] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name="%s_group_bin_pd" % key,
label=key)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', "%s_group_bin" % key), pd)
render_state.pds[key] = pd
render_state.addr_spaces[key] = addr_space
for (item, outfile) in zip(options.item, options.outfile):
key = item.split("/")
if len(key) is 1:
# We are rendering something that isn't a component or connection.
i = assembly
obj_key = None
template = templates.lookup(item)
elif key[1] in [
"source", "header", "c_environment_source",
"cakeml_start_source", "cakeml_end_source", "camkesConstants",
"linker", "debug", "simple", "rump_config"
]:
# We are rendering a component template
i = [
x for x in assembly.composition.instances if x.name == key[0]
][0]
obj_key = i.address_space
template = templates.lookup(item, i)
elif key[1] in ["from", "to"]:
# We are rendering a connection template
c = [
c for c in assembly.composition.connections if c.name == key[0]
][0]
if key[1] == "to":
i = c.to_ends[int(key[-1])]
elif key[1] == "from":
i = c.from_ends[int(key[-1])]
else:
die("Invalid connector end")
obj_key = i.instance.address_space
template = templates.lookup("/".join(key[:-1]), c)
else:
die("item: \"%s\" does not have the correct formatting to lookup a template."
% item)
try:
g = r.render(i,
assembly,
template,
render_state,
obj_key,
outfile_name=outfile.name,
options=options,
my_pd=render_state.pds[obj_key] if obj_key else None)
outfile.write(g)
outfile.close()
except TemplateError as inst:
die(rendering_error(i.name, inst))
read = r.get_files_used()
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' %
(options.outfile[0].name, ' \\\n '.join(sorted(read))))
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(render_state, options.save_object_state)
sys.exit(0)
def set_tcb_info(cspaces, obj_space, elfs, options, **_):
'''Set relevant extra info for TCB objects.'''
arch = lookup_architecture(options.architecture)
for group, space in cspaces.items():
cnode = space.cnode
for index, tcb in [(k, v.referent) for (k, v) in cnode.slots.items()
if v is not None and isinstance(v.referent, TCB)]:
perspective = Perspective(group=group, tcb=tcb.name)
elf_name = perspective['elf_name']
elf = elfs.get(elf_name)
if elf is None:
# We were not passed an ELF file for this CSpace. This will be
# true in the first pass of the runner where no ELFs are passed.
continue
tcb.elf = elf_name
tcb.ip = get_symbol_vaddr(elf, perspective['entry_symbol'])
assert tcb.ip != 0, 'entry point \'%s\' of %s appears to be 0x0' \
% (perspective['entry_symbol'], tcb.name)
if perspective['pool']:
# This TCB is part of the (cap allocator's) TCB pool.
continue
stack_symbol = perspective['stack_symbol']
ipc_buffer_symbol = perspective['ipc_buffer_symbol']
# The stack should be at least three pages and the IPC buffer
# region should be exactly three pages. Note that these regions
# both include a guard page either side of the used area. It is
# assumed that the stack grows downwards.
stack_size = get_symbol_size(elf, stack_symbol)
assert stack_size is not None, 'Stack for %(name)s, ' \
'\'%(symbol)s\', not found' % {
'name':tcb.name,
'symbol':stack_symbol,
}
assert stack_size >= PAGE_SIZE * 3, 'Stack for %(name)s, ' \
'\'%(symbol)s\', is only %(size)d bytes and does not have ' \
'room for guard pages' % {
'name':tcb.name,
'symbol':stack_symbol,
'size':stack_size,
}
assert get_symbol_size(elf, ipc_buffer_symbol) == PAGE_SIZE * 3
# Move the stack pointer to the top of the stack. Extra page is
# to account for the (upper) guard page.
assert stack_size % PAGE_SIZE == 0
tcb.sp = get_symbol_vaddr(elf,
stack_symbol) + stack_size - PAGE_SIZE
tcb.addr = get_symbol_vaddr(elf, ipc_buffer_symbol) + \
2 * PAGE_SIZE - arch.ipc_buffer_size()
# Each TCB needs to be passed its 'thread_id' that is the value
# it branches on in main(). This corresponds to the slot index
# to a cap to it in the component's CNode.
tcb.init.append(index)
if options.realtime:
sc_name = perspective['sc']
if sc_name in obj_space:
# For non-passive threads, associate the sc with the tcb
sc = obj_space[sc_name]
tcb['sc_slot'] = Cap(sc)
def final_spec(cspaces, obj_space, addr_spaces, elf_files, architecture, so_files):
"""
Generates a final CapDL spec file that can be given to a capdl loader application
"""
# cspaces : dict containes all the CSpaceAllocator for this app
# obj_space : ObjectAllocator for all the objs in the spec
# addr_spaces : dict containers all the AddressSpaceAllocator for this app
print("In final_spec function")
arch = lookup_architecture(architecture)
# NOTE: handle shared lib so files here
for e in [item for sublist in so_files for item in sublist]:
name = os.path.basename(e)
name = name[3:-3]
print(name)
elf = ELF(e, name, architecture)
cspace = cspaces[name]
print('the vspace root is ' + str(addr_spaces[name].vspace_root))
print('The addr space allocator is ' + str(addr_spaces[name]))
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(
infer_tcb=False,
infer_asid=False,
pd=addr_spaces[name].vspace_root,
addr_space=addr_spaces[name]
)
obj_space.merge(elf_spec)
cspace.cnode.finalise_size(arch)
# Fill in TCB object information.
# NOTE: There will be a tcb for a shared lib file but handled in root task of capdl
# loader
# TODO: This should be generalised with what is in the Camkes filters
tcb = obj_space["tcb_%s" % name]
progsymbol = 0
vsyscall = 0
tcb.init = [0,0,0,0,2,progsymbol,1,0,0,32,vsyscall,0,0]
tcb.addr = 0
tcb.sp = 0
tcb.ip = 0
for e in [item for sublist in elf_files for item in sublist]:
name = os.path.basename(e)
print(name)
# path, name, arch
elf = ELF(e, name, architecture)
# find the cspace for current elf
cspace = cspaces[name]
print('the vspace root is ' + str(addr_spaces[name].vspace_root))
print('The addr space allocator is ' + str(addr_spaces[name]))
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(
infer_tcb=False,
infer_asid=False,
pd=addr_spaces[name].vspace_root,
addr_space=addr_spaces[name]
)
obj_space.merge(elf_spec)
cspace.cnode.finalise_size(arch)
# Fill in TCB object information.
# TODO: This should be generalised with what is in the Camkes filters
tcb = obj_space["tcb_%s" % name]
progsymbol = elf.get_symbol_vaddr("progname")
vsyscall = elf.get_symbol_vaddr("sel4_vsyscall")
tcb.init = [0,0,0,0,2,progsymbol,1,0,0,32,vsyscall,0,0]
tcb.addr = elf.get_symbol_vaddr("mainIpcBuffer");
tcb.sp = elf.get_symbol_vaddr("stack")+elf.get_symbol_size("stack");
tcb.ip = elf.get_entry_point()
return obj_space
def guard_pages(obj_space, cspaces, elfs, options, **_):
'''Introduce a guard page around each stack and IPC buffer. Note that the
templates should have ensured a three page region for each stack in order to
enable this.'''
arch = lookup_architecture(options.architecture)
for group, space in cspaces.items():
cnode = space.cnode
for index, tcb in [(k, v.referent) for (k, v) in cnode.slots.items()
if v is not None and isinstance(v.referent, TCB)]:
perspective = Perspective(group=group, tcb=tcb.name)
if perspective['pool']:
# This TCB is part of the (cap allocator's) TCB pool.
continue
elf_name = perspective['elf_name']
# Find the page directory.
pd = None
pd_name = perspective['pd']
pds = [x for x in obj_space.spec.objs if x.name == pd_name]
if len(pds) > 1:
raise Exception('Multiple PDs found for group %s' % group)
elif len(pds) == 1:
pd, = pds
tcb['vspace'] = Cap(pd)
# If no PD was found we were probably just not passed any ELF files
# in this pass.
elf = elfs.get(elf_name)
if pd and elf:
ipc_symbol = perspective['ipc_buffer_symbol']
# Find the IPC buffer's preceding guard page's virtual address.
assert get_symbol_size(elf, ipc_symbol) == PAGE_SIZE * 3
pre_guard = get_symbol_vaddr(elf, ipc_symbol)
(cap, frame) = frame_for_vaddr(arch, pd, pre_guard, PAGE_SIZE)
if frame is None:
raise Exception(
'IPC buffer region of TCB %s in '
'group %s does not appear to be backed by a frame' %
(tcb.name, group))
# Delete the page.
obj_space.remove(frame)
update_frame_in_vaddr(arch, pd, pre_guard, PAGE_SIZE, None)
# Now do the same for the following guard page. We do this
# calculation separately just in case the region crosses a PT
# boundary and the two guard pages are in separate PTs.
post_guard = pre_guard + 2 * PAGE_SIZE
(cap, frame) = frame_for_vaddr(arch, pd, post_guard, PAGE_SIZE)
if frame is None:
raise Exception(
'IPC buffer region of TCB %s in '
'group %s does not appear to be backed by a frame' %
(tcb.name, group))
obj_space.remove(frame)
update_frame_in_vaddr(arch, pd, post_guard, PAGE_SIZE, None)
# Now we do the same thing for the preceding guard page of the
# thread's stack...
stack_symbol = perspective['stack_symbol']
pre_guard = get_symbol_vaddr(elf, stack_symbol)
(cap, frame) = frame_for_vaddr(arch, pd, pre_guard, PAGE_SIZE)
if frame is None:
raise Exception(
'stack region of TCB %s in '
'group %s does not appear to be backed by a frame' %
(tcb.name, group))
obj_space.remove(frame)
update_frame_in_vaddr(arch, pd, pre_guard, PAGE_SIZE, None)
# ...and the following guard page.
stack_region_size = get_symbol_size(elf, stack_symbol)
assert stack_region_size % PAGE_SIZE == 0, \
'stack region is not page-aligned'
assert stack_region_size >= 3 * PAGE_SIZE, \
'stack region has no room for guard pages'
post_guard = pre_guard + stack_region_size - PAGE_SIZE
(cap, frame) = frame_for_vaddr(arch, pd, post_guard, PAGE_SIZE)
if frame is None:
raise Exception(
'stack region of TCB %s in '
'group %s does not appear to be backed by a frame' %
(tcb.name, group))
obj_space.remove(frame)
update_frame_in_vaddr(arch, pd, post_guard, PAGE_SIZE, None)
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options, queries, filteroptions = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
done_items.add((item, file))
if len(all_items - done_items) == 0:
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(renderoptions.render_state,
options.save_object_state)
sys.exit(ret)
filename = None
if options.file is not None:
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag,
options.import_path, options.verbosity,
options.allow_forward_references,
options.save_ast, options.load_ast,
queries)
ast, read = parse_file_cached(filename, parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = RenderState(obj_space=obj_space)
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and (
x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast
if isinstance(x, Connection) and x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if filename and options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' % (filename, ' \\\n '.join(sorted(read))))
def apply_capdl_filters(renderoptions):
# Derive a set of usable ELF objects from the filenames we were passed.
render_state = renderoptions.render_state
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception(
'duplicate ELF files of name \'%s\' encountered' %
name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(
infer_tcb=False,
infer_asid=False,
pd=render_state.pds[group],
use_large_frames=options.largeframe,
addr_space=render_state.addr_spaces[group])
render_state.obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast,
obj_space=render_state.obj_space,
cspaces=render_state.cspaces,
elfs=elfs,
options=filteroptions)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(
options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps,
options.fsupport_init, options.largeframe, options.largeframe_dma,
options.architecture, options.debug_fault_handlers,
options.default_stack_size, options.realtime,
options.verification_base_name, filteroptions, render_state)
def instantiate_misc_templates(renderoptions):
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly,
assembly,
template,
renderoptions.render_state,
None,
outfile_name=outfile.name,
imported=read,
options=renderoptions)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
if "camkes-gen.cmake" in options.item:
instantiate_misc_templates(renderoptions)
if options.load_object_state is not None:
# There is an assumption that if load_object_state is set, we
# skip all of the component and connector logic below.
# FIXME: refactor to clarify control flow
renderoptions.render_state = pickle.load(options.load_object_state)
apply_capdl_filters(renderoptions)
instantiate_misc_templates(renderoptions)
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError("No template instantiated on capdl generation path")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in renderoptions.render_state.cspaces:
p = Perspective(phase=RUNNER,
instance=i.name,
group=i.address_space)
cnode = renderoptions.render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name=p['cnode'],
label=i.address_space)
renderoptions.render_state.cspaces[
i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name=p['pd'],
label=i.address_space)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', p['elf_name']), pd)
renderoptions.render_state.pds[i.address_space] = pd
renderoptions.render_state.addr_spaces[
i.address_space] = addr_space
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/c_environment_source' % i.name,
'%s/cakeml_start_source' % i.name,
'%s/cakeml_end_source' % i.name, '%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(
i,
assembly,
template,
renderoptions.render_state,
i.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[i.address_space])
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name,
c.from_ends), ('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name,
c.to_ends), ('%s/to/header' % c.name, c.to_ends),
('%s/to/cakeml' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e,
assembly,
template,
renderoptions.render_state,
e.instance.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
e.instance.address_space])
except TemplateError as inst:
die(rendering_error(item, inst))
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends), ('%s/to/' % c.name,
c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e,
assembly,
template,
renderoptions.render_state,
e.instance.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
e.instance.address_space])
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in renderoptions.render_state.cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'),
('rump_config', 'rumprun')]
for special in [
bl for bl in SPECIAL_TEMPLATES
if assembly.configuration[i.name].get(bl[0])
]:
for t in ('%s/%s' % (i.name, special[1]), ):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i,
assembly,
template,
renderoptions.render_state,
i.address_space,
outfile_name=None,
options=renderoptions,
my_pd=renderoptions.render_state.pds[
i.address_space])
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' %
item)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# Ensure we were supplied equal items and outfiles
if len(options.outfile) != len(options.item):
err.write('Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in items or outfiles
if len(set(options.item)) != len(options.item):
err.write('Duplicate items requested through --item.\n')
return -1
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
cwd = os.getcwd()
# Build a list of item/outfile pairs that we have yet to match and process
all_items = set(zip(options.item, options.outfile))
done_items = set([])
# Construct the compilation caches if requested.
cachea = None
cacheb = None
if options.cache:
# Construct a modified version of the command line arguments that we'll
# use in the keys to the caches. Essentially we elide --outfile and its
# parameter under the assumption that this value is never used in code
# generation. The purpose of this is to allow us to successfully cache
# ancillary outputs that we generate along the way to the current
# output. If we were to include --outfile in the key, future attempts
# to generate these ancillary outputs would unnecessarily miss the
# entries generated by this execution.
args = []
skip = False
for index, arg in enumerate(argv[1:]):
if skip:
skip = False
continue
if arg in ('--outfile', '-O'):
skip = True
continue
args.append(arg)
cachea = LevelACache(os.path.join(options.cache_dir, version(), 'cachea'))
cacheb = LevelBCache(os.path.join(options.cache_dir, version(), 'cacheb'))
def done(s, file, item):
ret = 0
if s:
file.write(s)
file.close()
if cachea is not None:
try:
cachea.flush()
except sqlite3.OperationalError as e:
# The following suppresses two spurious errors:
# 1. The database is locked. In a large, parallel build, writes
# to the level A cache are heavily contended and this error
# can occur.
# 2. The database structure is unexpected. If the CAmkES
# sources have changed *while* the runner was executing,
# the level A cache can be looking in a different place to
# where the cache was created.
# Both of these are non-critical (will just result in a
# potential future cache miss) so there's no need to alarm the
# user.
if re.search(r'database is locked', str(e)) is not None or \
re.search(r'no such table', str(e)) is not None:
log.debug('failed to flush level A cache: %s' % str(e))
else:
raise
if cacheb is not None:
try:
cacheb.flush()
except sqlite3.OperationalError as e:
# As above for the level B cache.
if re.search(r'database is locked', str(e)):
log.debug('failed to flush level B cache: %s' % str(e))
else:
raise
done_items.add((item, file))
if len(all_items - done_items) == 0:
sys.exit(ret)
# Try to find this output in the level A cache if possible. This check will
# 'hit' if the source files representing the input spec are identical to
# some previously observed execution.
if cachea is not None:
assert 'args' in locals()
assert len(options.outfile) == 1, 'level A cache only supported when requestiong ' \
'single items'
output = cachea.load(args, cwd)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level A cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
filename = os.path.abspath(options.file.name)
try:
# Build the parser options
parse_options = ParserOptions(options.cpp, options.cpp_flag, options.import_path, options.verbosity, options.allow_forward_references)
ast, read = parse_file_cached(filename, options.data_structure_cache_dir, parse_options)
except (ASTError, ParseError) as e:
die(e.args)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' % (i.name,
a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' % (i.name,
a.name))
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
cspaces = {}
pds = {}
conf = assembly.configuration
shmem = collections.defaultdict(ShmemFactory())
kept_symbols = {}
fill_frames = {}
templates = Templates(options.platform)
[templates.add_root(t) for t in options.templates]
try:
r = Renderer(templates, options.cache, options.cache_dir)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
# The user may have provided their own connector definitions (with
# associated) templates, in which case they won't be in the built-in lookup
# dictionary. Let's add them now. Note, definitions here that conflict with
# existing lookup entries will overwrite the existing entries. Note that
# the extra check that the connector has some templates is just an
# optimisation; the templates module handles connectors without templates
# just fine.
extra_templates = set()
for c in (x for x in ast.items if isinstance(x, Connector) and
(x.from_template is not None or x.to_template is not None)):
try:
# Find a connection that uses this type.
connection = next(x for x in ast if isinstance(x, Connection) and
x.type == c)
# Add the custom templates and update our collection of read
# inputs. It is necessary to update the read set here to avoid
# false compilation cache hits when the source of a custom template
# has changed.
extra_templates |= templates.add(c, connection)
except TemplateError as e:
die('while adding connector %s: %s' % (c.name, e))
except StopIteration:
# No connections use this type. There's no point adding it to the
# template lookup dictionary.
pass
# Check if our current target is in the level B cache. The level A cache
# will 'miss' and this one will 'hit' when the input spec is identical to
# some previously observed execution modulo a semantically irrelevant
# element (e.g. an introduced comment).
ast_hash = None
if cacheb is not None:
ast_hash = level_b_prime(ast)
assert 'args' in locals()
assert len(options.item) == 1, 'level B cache only supported when requesting ' \
'single items'
output = cacheb.load(ast_hash, args, set(options.elf) | extra_templates)
if output is not None:
log.debug('Retrieved %(platform)s/%(item)s from level B cache' %
options.__dict__)
done(output, options.outfile[0], options.item[0])
# Add custom templates.
read |= extra_templates
# Add the CAmkES sources themselves to the accumulated list of inputs.
read |= set(path for path, _ in sources())
# Add any ELF files we were passed as inputs.
read |= set(options.elf)
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write('%s: \\\n %s\n' %
(filename, ' \\\n '.join(sorted(read))))
# If we have a cache, allow outputs to be saved to it.
if options.cache:
assert cachea is not None, 'level A cache not available, though the ' \
'cache is enabled (bug in runner?)'
# The logic of this cache currently only works when a single item is requested
# on the command line
assert len(options.item) == 1, 'level A cache only supported when requesting ' \
'single items'
# Calculate the input files to the level A cache.
inputs = level_a_prime(read)
# Work out the position of the --item argument in the command line
# parameters. We will use this to cache not only outputs for this
# execution, but also outputs for ones with a different target.
item_index = None
assert 'args' in locals()
for index, arg in enumerate(args[:-1]):
if arg in ('--item', '-T'):
item_index = index + 1
break
assert item_index is not None, 'failed to find required argument ' \
'--item (bug in runner?)'
# We should already have the necessary inputs for the level B cache.
assert cacheb is not None, 'level B cache not available, though the ' \
'cache is enabled (bug in runner?)'
assert ast_hash is not None, 'AST hash not pre-computed (bug in ' \
'runner?)'
def save(item, value):
# Juggle the command line arguments to cache the predicted
# arguments for a call that would generate this item.
new_args = args[:item_index] + [item] + args[item_index + 1:]
# Save entries in both caches.
cachea.save(new_args, cwd, value, inputs)
if item != 'Makefile' and item != 'camkes-gen.cmake':
# We avoid caching the generated Makefile because it is not
# safe. The inputs to generation of the Makefile are not only
# the AST, but also the file names (`inputs`). If we cache it in
# the level B cache we risk the following scenario:
#
# 1. Generate the Makefile, caching it in the level B cache;
# 2. Modify the spec to import a file containing only white
# space and/or comments; then
# 3. Generate the Makefile, missing the level A cache, but
# hitting the level B cache.
#
# At this point, the generated Makefile is incorrect because it
# does not capture any dependencies on the imported file. We can
# now introduce something semantically relevant into this file
# (e.g. an Assembly block) and it will not be seen by the build
# system.
cacheb.save(ast_hash, new_args,
set(options.elf) | extra_templates, value)
else:
def save(item, value):
pass
def apply_capdl_filters():
# Derive a set of usable ELF objects from the filenames we were passed.
elfs = {}
for e in options.elf:
try:
name = os.path.basename(e)
if name in elfs:
raise Exception('duplicate ELF files of name \'%s\' encountered' % name)
elf = ELF(e, name, options.architecture)
p = Perspective(phase=RUNNER, elf_name=name)
group = p['group']
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False, infer_asid=False,
pd=pds[group], use_large_frames=options.largeframe)
obj_space.merge(elf_spec, label=group)
elfs[name] = (e, elf)
except Exception as inst:
die('While opening \'%s\': %s' % (e, inst))
filteroptions = FilterOptions(options.architecture, options.realtime, options.largeframe,
options.largeframe_dma, options.default_priority, options.default_max_priority,
options.default_affinity, options.default_period, options.default_budget,
options.default_data, options.default_size_bits,
options.debug_fault_handlers, options.fprovide_tcb_caps)
for f in CAPDL_FILTERS:
try:
# Pass everything as named arguments to allow filters to
# easily ignore what they don't want.
f(ast=ast, obj_space=obj_space, cspaces=cspaces, elfs=elfs,
options=filteroptions, shmem=shmem, fill_frames=fill_frames)
except Exception as inst:
die('While forming CapDL spec: %s' % inst)
renderoptions = RenderOptions(options.file, options.verbosity, options.frpc_lock_elision,
options.fspecialise_syscall_stubs, options.fprovide_tcb_caps, options.fsupport_init,
options.largeframe, options.largeframe_dma, options.architecture, options.debug_fault_handlers,
options.realtime)
def instantiate_misc_template():
for (item, outfile) in (all_items - done_items):
try:
template = templates.lookup(item)
if template:
g = r.render(assembly, assembly, template, obj_space, None,
shmem, kept_symbols, fill_frames, imported=read, options=renderoptions)
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
if options.item[0] in ('capdl', 'label-mapping') and options.data_structure_cache_dir is not None \
and len(options.outfile) == 1:
# It's possible that data structures required to instantiate the capdl spec
# were saved during a previous invocation of this script in the current build.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
if os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_file:
# Found a cached version of the necessary data structures
obj_space, shmem, cspaces, pds, kept_symbols, fill_frames = pickle.load(pickle_file)
apply_capdl_filters()
instantiate_misc_template()
# If a template wasn't instantiated, something went wrong, and we can't recover
raise CAmkESError("No template instantiated on capdl generation fastpath")
# We're now ready to instantiate the template the user requested, but there
# are a few wrinkles in the process. Namely,
# 1. Template instantiation needs to be done in a deterministic order. The
# runner is invoked multiple times and template code needs to be
# allocated identical cap slots in each run.
# 2. Components and connections need to be instantiated before any other
# templates, regardless of whether they are the ones we are after. Some
# other templates, such as the Makefile depend on the obj_space and
# cspaces.
# 3. All actual code templates, up to the template that was requested,
# need to be instantiated. This is related to (1) in that the cap slots
# allocated are dependent on what allocations have been done prior to a
# given allocation call.
# Instantiate the per-component source and header files.
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
if i.address_space not in cspaces:
p = Perspective(phase=RUNNER, instance=i.name,
group=i.address_space)
cnode = obj_space.alloc(seL4_CapTableObject,
name=p['cnode'], label=i.address_space)
cspaces[i.address_space] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(options.architecture).vspace().object, name=p['pd'],
label=i.address_space)
pds[i.address_space] = pd
for t in ('%s/source' % i.name, '%s/header' % i.name,
'%s/c_environment_source' % i.name,
'%s/cakeml_start_source' % i.name, '%s/cakeml_end_source' % i.name,
'%s/linker' % i.name):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space],
shmem, kept_symbols, fill_frames, options=renderoptions, my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
except TemplateError as inst:
die(rendering_error(i.name, inst))
# Instantiate the per-connection files.
for c in assembly.composition.connections:
for t in (('%s/from/source' % c.name, c.from_ends),
('%s/from/header' % c.name, c.from_ends),
('%s/to/source' % c.name, c.to_ends),
('%s/to/header' % c.name, c.to_ends)):
template = templates.lookup(t[0], c)
if template is not None:
for id, e in enumerate(t[1]):
item = '%s/%d' % (t[0], id)
g = ''
try:
g = r.render(e, assembly, template, obj_space,
cspaces[e.instance.address_space], shmem, kept_symbols, fill_frames,
options=renderoptions, my_pd=pds[e.instance.address_space])
except TemplateError as inst:
die(rendering_error(item, inst))
except jinja2.exceptions.TemplateNotFound:
die('While rendering %s: missing template for %s' %
(item, c.type.name))
save(item, g)
for (target, outfile) in (all_items - done_items):
if target == item:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
break
# The following block handles instantiations of per-connection
# templates that are neither a 'source' or a 'header', as handled
# above. We assume that none of these need instantiation unless we are
# actually currently looking for them (== options.item). That is, we
# assume that following templates, like the CapDL spec, do not require
# these templates to be rendered prior to themselves.
# FIXME: This is a pretty ugly way of handling this. It would be nicer
# for the runner to have a more general notion of per-'thing' templates
# where the per-component templates, the per-connection template loop
# above, and this loop could all be done in a single unified control
# flow.
for (item, outfile) in (all_items - done_items):
for t in (('%s/from/' % c.name, c.from_ends),
('%s/to/' % c.name, c.to_ends)):
if not item.startswith(t[0]):
# This is not the item we're looking for.
continue
# If we've reached here then this is the exact item we're after.
template = templates.lookup(item, c)
if template is None:
die('no registered template for %s' % item)
for e in t[1]:
try:
g = r.render(e, assembly, template, obj_space,
cspaces[e.instance.address_space], shmem, kept_symbols, fill_frames,
options=renderoptions, my_pd=pds[e.instance.address_space])
save(item, g)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(item, inst))
# Perform any per component special generation. This needs to happen last
# as these template needs to run after all other capabilities have been
# allocated
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
assert i.address_space in cspaces
SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'), ('rump_config', 'rumprun')]
for special in [bl for bl in SPECIAL_TEMPLATES if conf[i.name].get(bl[0])]:
for t in ('%s/%s' % (i.name, special[1]),):
try:
template = templates.lookup(t, i)
g = ''
if template:
g = r.render(i, assembly, template, obj_space, cspaces[i.address_space],
shmem, kept_symbols, fill_frames, options=renderoptions, my_pd=pds[i.address_space])
save(t, g)
for (item, outfile) in (all_items - done_items):
if item == t:
if not template:
log.warning('Warning: no template for %s' % item)
done(g, outfile, item)
except TemplateError as inst:
die(rendering_error(i.name, inst))
if options.data_structure_cache_dir is not None:
# At this point the capdl database is in the state required for applying capdl
# filters and generating the capdl spec. In case the capdl spec isn't the current
# target, we pickle the database here, so when the capdl spec is built, these
# data structures don't need to be regenerated.
cache_path = os.path.realpath(options.data_structure_cache_dir)
pickle_path = os.path.join(cache_path, CAPDL_STATE_PICKLE)
with open(pickle_path, 'wb') as pickle_file:
pickle.dump((obj_space, shmem, cspaces, pds, kept_symbols, fill_frames), pickle_file)
for (item, outfile) in (all_items - done_items):
if item in ('capdl', 'label-mapping'):
apply_capdl_filters()
# Instantiate any other, miscellaneous template. If we've reached this
# point, we know the user did not request a code template.
instantiate_misc_template()
# Check if there are any remaining items
not_done = all_items - done_items
if len(not_done) > 0:
for (item, outfile) in not_done:
err.write('No valid element matching --item %s.\n' % item)
return -1
return 0
def get_word_size(arch):
return int(lookup_architecture(arch).word_size_bits() / 8)
def collapse_shared_frames(ast, obj_space, elfs, shmem, options, **_):
"""Find regions in virtual address spaces that are intended to be backed by
shared frames and adjust the capability distribution to reflect this."""
if not elfs:
# If we haven't been passed any ELF files this step is not relevant yet.
return
#for i in ast._items:
#print ("rpc start")
#print (i.__dict__)
#print ("rpc end")
arch = lookup_architecture(options.architecture)
assembly = ast.assembly
for window, mappings in shmem.items():
frames = None
exact_frames = False
# If the shared variable has an associated set of backing frames
# allocated already (ie. allocated in a template), look it up
# before collapsing the shared variable.
for mapping in mappings.values():
for _, _, _, prealloc_frames, _ in mapping:
if prealloc_frames is not None:
assert frames is None, 'Multiple sides of shared memory with' \
'preallocated frames for shared variable "%s"' % window
frames = prealloc_frames
exact_frames = True
for cnode, local_mappings in mappings.items():
for sym, permissions, paddr, _, cached_hw in local_mappings:
perspective = Perspective(cnode=cnode)
# Find this instance's ELF file.
elf_name = perspective['elf_name']
assert elf_name in elfs
elf = elfs[elf_name]
# Find this instance's page directory.
pd_name = perspective['pd']
pds = [x for x in obj_space.spec.objs if x.name == pd_name]
assert len(pds) == 1
pd = pds[0]
# Look up the ELF-local version of this symbol.
vaddr = get_symbol_vaddr(elf, sym)
assert vaddr is not None, 'shared symbol \'%s\' not found in ' \
'ELF %s (template bug?)' % (sym, elf_name)
assert vaddr != 0, 'shared symbol \'%s\' located at NULL in ELF ' \
'%s (template bug?)' % (sym, elf_name)
assert vaddr % PAGE_SIZE == 0, 'shared symbol \'%s\' not ' \
'page-aligned in ELF %s (template bug?)' % (sym, elf_name)
size = get_symbol_size(elf, sym)
assert size != 0, 'shared symbol \'%s\' has size 0 in ELF %s ' \
'(template bug?)' % (sym, elf_name)
assert size % PAGE_SIZE == 0, 'shared symbol \'%s\' in ELF %s ' \
'has a size that is not page-aligned (template bug?)' % \
(sym, elf_name)
# Infer the page table(s) and small page(s) that currently back this
# region.
map_indices = [
make_indices(arch, v, PAGE_SIZE)
for v in six.moves.range(vaddr, vaddr + size, PAGE_SIZE)
]
# Permissions that we will apply to the eventual mapping.
read = 'R' in permissions
write = 'W' in permissions
execute = 'X' in permissions
largest_frame_size, level_num = find_optimal_frame_size(
arch, vaddr, size)
if frames is None:
# First iteration of the loop; we need to discover the backing
# frames for this region.
frames = []
# We want to derive large frames if (a) this window is device
# registers and large-frame-sized (in which case the kernel
# will have created it as large frames) or (b) the user has
# requested large frame promotion.
if largest_frame_size != PAGE_SIZE and (
options.largeframe or paddr is not None):
# Grab a copy of the frame for every entry we're going to end up making
new_frames = {}
for new_vaddr in six.moves.range(
vaddr, vaddr + size, largest_frame_size):
new_frames[new_vaddr] = obj_space.alloc(
seL4_FrameObject, size=largest_frame_size)
# Iterate over every unique index in every object below this one
delete_small_frames(arch, obj_space, pd, level_num,
map_indices)
# Now insert the new frames
for new_vaddr in six.moves.range(
vaddr, vaddr + size, largest_frame_size):
frame = new_frames[new_vaddr]
cap = Cap(frame, read, write, execute)
if paddr is not None:
frame.paddr = paddr + (new_vaddr - vaddr)
cap.set_cached(cached_hw)
update_frame_in_vaddr(arch, pd, new_vaddr,
largest_frame_size, cap)
frames.append(frame)
else:
# We don't need to handle large frame promotion. Just tweak
# the permissions and optionally the physical address of
# all the current mappings.
for offset, indices in enumerate(map_indices):
(cap, frame) = lookup_vspace_indices(pd, indices)
cap.read = read
cap.write = write
cap.grant = execute
if paddr is not None:
frame.paddr = paddr + offset * PAGE_SIZE
cap.set_cached(cached_hw)
frames.append(frame)
else:
# We have already discovered frames to back this region and now
# we just need to adjust page table mappings.
assert size == sum(f.size for f in frames), 'mismatched ' \
'sizes of shared region \'%s\' (template bug?)' % window
if not exact_frames:
# We do not need to preserve the exact same frames / frame sizings, so
# we can delete the entire region ready to put in our new frames
# Delete all the underlying frames / objects for this range
delete_small_frames(arch, obj_space, pd, level_num,
map_indices)
offset = 0
for frame in frames:
cap = Cap(frame, read, write, execute)
if paddr is not None:
cap.set_cached(cached_hw)
if exact_frames:
# If we need to preserve the exact frames then we need to clear
# the range for each frame individually, up to the required level
# for that frame. This is to allow for 'weird' shared memory regions
# that have preallocated frames with different sized frames in
# the one region.
frame_map_indices = [
make_indices(arch, v, PAGE_SIZE)
for v in six.moves.range(
vaddr + offset, vaddr + offset +
frame.size, PAGE_SIZE)
]
_, frame_level_num = find_optimal_frame_size(
arch, 0, frame.size)
delete_small_frames(arch, obj_space, pd,
frame_level_num,
frame_map_indices)
# Now, with exact_frames or not, we know that the slot for this frame is
# free and we can re-insert the correct frame
update_frame_in_vaddr(arch, pd, vaddr + offset,
frame.size, cap)
offset = offset + frame.size
def main(argv, out, err):
# We need a UTF-8 locale, so bail out if we don't have one. More
# specifically, things like the version() computation traverse the file
# system and, if they hit a UTF-8 filename, they try to decode it into your
# preferred encoding and trigger an exception.
encoding = locale.getpreferredencoding().lower()
if encoding not in ('utf-8', 'utf8'):
err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred '
'encoding is %s. You can override your locale with the LANG '
'environment variable.\n' % encoding)
return -1
options = parse_args(argv, out, err)
# register object sizes with loader
if options.object_sizes:
register_object_sizes(
yaml.load(options.object_sizes, Loader=yaml.FullLoader))
# Ensure we were supplied equal items, outfiles and templates
if len(options.outfile) != len(options.item) != len(options.template):
err.write(
'Different number of items and outfiles. Required one outfile location '
'per item requested.\n')
return -1
# No duplicates in outfiles
if len(set(options.outfile)) != len(options.outfile):
err.write('Duplicate outfiles requrested through --outfile.\n')
return -1
# Save us having to pass debugging everywhere.
die = functools.partial(_die, options)
log.set_verbosity(options.verbosity)
ast = pickle.load(options.load_ast)
# Locate the assembly.
assembly = ast.assembly
if assembly is None:
die('No assembly found')
# Do some extra checks if the user asked for verbose output.
if options.verbosity >= 2:
# Try to catch type mismatches in attribute settings. Note that it is
# not possible to conclusively evaluate type correctness because the
# attributes' type system is (deliberately) too loose. That is, the
# type of an attribute can be an uninterpreted C type the user will
# provide post hoc.
for i in assembly.composition.instances:
for a in i.type.attributes:
value = assembly.configuration[i.name].get(a.name)
if value is not None:
if a.type == 'string' and not \
isinstance(value, six.string_types):
log.warning('attribute %s.%s has type string but is '
'set to a value that is not a string' %
(i.name, a.name))
elif a.type == 'int' and not \
isinstance(value, numbers.Number):
log.warning('attribute %s.%s has type int but is set '
'to a value that is not an integer' %
(i.name, a.name))
try:
r = Renderer(options.templates)
except jinja2.exceptions.TemplateSyntaxError as e:
die('template syntax error: %s' % e)
if options.load_object_state is not None:
render_state = pickle.load(options.load_object_state)
elif options.save_object_state is None:
render_state = None
else:
obj_space = ObjectAllocator()
obj_space.spec.arch = options.architecture
render_state = RenderState(obj_space=obj_space)
for i in assembly.composition.instances:
# Don't generate any code for hardware components.
if i.type.hardware:
continue
key = i.address_space
if key not in render_state.cspaces:
cnode = render_state.obj_space.alloc(
ObjectType.seL4_CapTableObject,
name="%s_cnode" % key,
label=key)
render_state.cspaces[key] = CSpaceAllocator(cnode)
pd = obj_space.alloc(lookup_architecture(
options.architecture).vspace().object,
name="%s_group_bin_pd" % key,
label=key)
addr_space = AddressSpaceAllocator(
re.sub(r'[^A-Za-z0-9]', '_', "%s_group_bin" % key), pd)
render_state.pds[key] = pd
render_state.addr_spaces[key] = addr_space
for (item, outfile, template) in zip(options.item, options.outfile,
options.template):
key = item.split("/")
if key[0] == "component":
i = [
x for x in assembly.composition.instances if x.name == key[1]
][0]
obj_key = i.address_space
elif key[0] == "connector":
c = [
c for c in assembly.composition.connections if c.name == key[1]
][0]
if key[2] == "to":
i = c.to_ends[int(key[3])]
elif key[2] == "from":
i = c.from_ends[int(key[3])]
else:
die("Invalid connector end")
obj_key = i.instance.address_space
elif key[0] == "assembly":
i = assembly
obj_key = None
else:
die("item: \"%s\" does not have the correct formatting to render."
% item)
try:
g = r.render(i,
assembly,
template,
render_state,
obj_key,
outfile_name=outfile.name,
options=options,
my_pd=render_state.pds[obj_key] if obj_key else None)
outfile.write(g)
outfile.close()
except TemplateError as inst:
if hasattr(i, 'name'):
die(rendering_error(i.name, inst))
else:
die(rendering_error(i.parent.name, inst))
read = r.get_files_used()
# Write a Makefile dependency rule if requested.
if options.makefile_dependencies is not None:
options.makefile_dependencies.write(
'%s: \\\n %s\n' %
(options.outfile[0].name, ' \\\n '.join(sorted(read))))
if options.save_object_state is not None:
# Write the render_state to the supplied outfile
pickle.dump(render_state, options.save_object_state)
sys.exit(0)
def replace_dma_frames(ast, obj_space, elfs, options, **_):
'''Locate the DMA pool (a region that needs to have frames whose mappings
can be reversed) and replace its backing frames with pre-allocated,
reversible ones.'''
if not elfs:
# If we haven't been passed any ELF files this step is not relevant yet.
return
arch = lookup_architecture(options.architecture)
assembly = ast.assembly
for i in (x for x in assembly.composition.instances
if not x.type.hardware):
perspective = Perspective(instance=i.name, group=i.address_space)
elf_name = perspective['elf_name']
assert elf_name in elfs
elf = elfs[elf_name]
# Find this instance's page directory.
pd_name = perspective['pd']
pds = [x for x in obj_space.spec.objs if x.name == pd_name]
assert len(pds) == 1
pd, = pds
sym = perspective['dma_pool_symbol']
base = get_symbol_vaddr(elf, sym)
if base is None:
# We don't have a DMA pool.
continue
assert base != 0
sz = get_symbol_size(elf, sym)
assert sz % PAGE_SIZE == 0 # DMA pool should be at least page-aligned.
# Replicate logic from the template to determine the page size used to
# back the DMA pool.
page_size = 4 * 1024
if options.largeframe_dma:
for size in reversed(page_sizes(options.architecture)):
if sz >= size:
page_size = size
break
assert sz % page_size == 0, 'DMA pool not rounded up to a multiple ' \
'of page size %d (template bug?)' % page_size
dma_frame_index = 0
def get_dma_frame(index):
'''
Find the `index`-th DMA frame. Note that these are constructed in
the component template itself.
'''
p = Perspective(instance=i.name,
group=i.address_space,
dma_frame_index=index)
name = p['dma_frame_symbol']
assert name in obj_space, "No such symbol in capdl spec %s" % name
return obj_space[name]
# Ensure paging structures are in place to map in dma frames
replace_large_frames(obj_space, arch, pd, base, sz, page_size)
for page_vaddr in six.moves.range(base, base + sz, page_size):
cap = Cap(get_dma_frame(dma_frame_index), True, True, False)
cap.set_cached(False)
update_frame_in_vaddr(arch, pd, page_vaddr, page_size, cap)
dma_frame_index = dma_frame_index + 1
