def parse_file_cached(filename, parser_options): if parser_options.load_ast is not None: return pickle.load(parser_options.load_ast) ast, read = parse_file(filename, parser_options) if parser_options.save_ast is not None: pickle.dump((ast, read), parser_options.save_ast) return ast, read
def main(argv, out, err): options = parse_args(argv, out, err) log.set_verbosity(options.verbosity) try: filename = os.path.abspath(options.file.name) ast, read = parse_file(filename, options) except (ASTError, ParseError) as e: die(options, e.args) if options.makefile_dependencies is not None: options.makefile_dependencies.write( '%s: \\\n %s\n' % (options.save_ast.name, ' \\\n '.join(sorted(read)))) pickle.dump(ast, options.save_ast) return 0
def main(argv, out, err): options = parse_args(argv, out, err) log.set_verbosity(options.verbosity) try: filename = os.path.abspath(options.file.name) ast, read = parse_file(filename, options) except (ASTError, ParseError) as e: die(options, e.args) if options.makefile_dependencies is not None: options.makefile_dependencies.write('%s: \\\n %s\n' % (options.save_ast.name, ' \\\n '.join(sorted(read)))) pickle.dump(ast, options.save_ast) return 0
def main(argv): parser = argparse.ArgumentParser(prog='python -m camkes.format', description='translate CAmkES specifications') parser.add_argument('--file', '-f', help='input file', type=argparse.FileType('r'), required=True) parser.add_argument('--cpp', action='store_true', help='Pre-process the ' 'source with CPP') parser.add_argument('--nocpp', action='store_false', dest='cpp', help='Do not pre-process the source with CPP') parser.add_argument('--cpp-flag', action='append', default=[], help='Specify a flag to pass to CPP') parser.add_argument('--import-path', '-I', help='Add this path to the list ' 'of paths to search for built-in imports. That is, add it to the list ' 'of directories that are searched to find the file "foo" when ' 'encountering an expression "import <foo>;".', action='append', default=[]) parser.add_argument('--quiet', '-q', help='No output.', dest='verbosity', default=1, action='store_const', const=0) parser.add_argument('--verbose', '-v', help='Verbose output.', dest='verbosity', action='store_const', const=2) parser.add_argument('--debug', '-D', help='Extra verbose output.', dest='verbosity', action='store_const', const=3) parser.add_argument('--outfile', '-O', help='Output to the given file.', type=argparse.FileType('w'), required=True) parser.add_argument('--allow-forward-references', action='store_true', help='allow refering to objects in your specification that are ' 'defined after the point at which they are referenced') parser.add_argument('--disallow-forward-references', action='store_false', dest='allow_forward_references', help='only permit references in ' 'specifications to objects that have been defined before that point') options = parser.parse_args(argv[1:]) try: ast, _ = parse_file(options.file.name, options) except (ASTError, ParseError) as e: sys.stderr.write('failed: %s\n' % str(e)) return -1 options.outfile.write(to_xml(ast)) return 0
def parse_file_cached(filename, data_structure_cache_dir, parser_options): if data_structure_cache_dir is None: return parse_file(filename, parser_options) return pickle_call(data_structure_cache_dir, 'ast.p', parse_file, filename, parser_options)
def parse_file_cached(filename, options): if options.data_structure_cache_dir is None: return parse_file(filename, options) return pickle_call(options, 'ast.p', parse_file, filename, options)
def main(argv): parser = argparse.ArgumentParser( prog='python -m camkes.format', description='translate CAmkES specifications') parser.add_argument('--file', '-f', help='input file', type=argparse.FileType('r'), required=True) parser.add_argument('--cpp', action='store_true', help='Pre-process the ' 'source with CPP') parser.add_argument('--nocpp', action='store_false', dest='cpp', help='Do not pre-process the source with CPP') parser.add_argument('--cpp-flag', action='append', default=[], help='Specify a flag to pass to CPP') parser.add_argument( '--import-path', '-I', help='Add this path to the list ' 'of paths to search for built-in imports. That is, add it to the list ' 'of directories that are searched to find the file "foo" when ' 'encountering an expression "import <foo>;".', action='append', default=[]) parser.add_argument('--quiet', '-q', help='No output.', dest='verbosity', default=1, action='store_const', const=0) parser.add_argument('--verbose', '-v', help='Verbose output.', dest='verbosity', action='store_const', const=2) parser.add_argument('--debug', '-D', help='Extra verbose output.', dest='verbosity', action='store_const', const=3) parser.add_argument('--outfile', '-O', help='Output to the given file.', type=argparse.FileType('w'), required=True) parser.add_argument( '--allow-forward-references', action='store_true', help='allow refering to objects in your specification that are ' 'defined after the point at which they are referenced') parser.add_argument( '--disallow-forward-references', action='store_false', dest='allow_forward_references', help='only permit references in ' 'specifications to objects that have been defined before that point') options = parser.parse_args(argv[1:]) try: ast, _ = parse_file(options.file.name, options) except (ASTError, ParseError) as e: sys.stderr.write('failed: %s\n' % str(e)) return -1 options.outfile.write(to_xml(ast)) return 0
def main(argv, out, err): # We need a UTF-8 locale, so bail out if we don't have one. More # specifically, things like the version() computation traverse the file # system and, if they hit a UTF-8 filename, they try to decode it into your # preferred encoding and trigger an exception. encoding = locale.getpreferredencoding().lower() if encoding not in ('utf-8', 'utf8'): err.write('CAmkES uses UTF-8 encoding, but your locale\'s preferred ' 'encoding is %s. You can override your locale with the LANG ' 'environment variable.\n' % encoding) return -1 options, queries = parse_args(argv, out, err) # register object sizes with loader if options.object_sizes: register_object_sizes(yaml.load(options.object_sizes)) # Ensure we were supplied equal items and outfiles if len(options.outfile) != len(options.item): err.write( 'Different number of items and outfiles. Required one outfile location ' 'per item requested.\n') return -1 # No duplicates in items or outfiles if len(set(options.item)) != len(options.item): err.write('Duplicate items requested through --item.\n') return -1 if len(set(options.outfile)) != len(options.outfile): err.write('Duplicate outfiles requrested through --outfile.\n') return -1 # Save us having to pass debugging everywhere. die = functools.partial(_die, options) log.set_verbosity(options.verbosity) # Build a list of item/outfile pairs that we have yet to match and process all_items = set(zip(options.item, options.outfile)) done_items = set([]) def done(s, file, item, r, read): ret = 0 if s: file.write(s) file.close() done_items.add((item, file)) if len(all_items - done_items) == 0: read |= r.get_files_used() # Write a Makefile dependency rule if requested. if options.makefile_dependencies is not None: options.makefile_dependencies.write( '%s: \\\n %s\n' % (options.outfile[0].name, ' \\\n '.join(sorted(read)))) if options.save_object_state is not None: # Write the render_state to the supplied outfile pickle.dump(renderoptions.render_state, options.save_object_state) sys.exit(ret) try: if options.load_ast is not None: ast = pickle.load(options.load_ast) read = set() else: if options.file is not None: filename = os.path.abspath(options.file.name) else: die("Need file to load ast from") # Build the parser options parse_options = ParserOptions(options.cpp, options.cpp_flag, options.import_path, options.verbosity, options.allow_forward_references, queries) ast, read = parse_file(filename, parse_options) if options.save_ast is not None: pickle.dump(ast, options.save_ast) except (ASTError, ParseError) as e: die(e.args) # Locate the assembly. assembly = ast.assembly if assembly is None: die('No assembly found') # Do some extra checks if the user asked for verbose output. if options.verbosity >= 2: # Try to catch type mismatches in attribute settings. Note that it is # not possible to conclusively evaluate type correctness because the # attributes' type system is (deliberately) too loose. That is, the # type of an attribute can be an uninterpreted C type the user will # provide post hoc. for i in assembly.composition.instances: for a in i.type.attributes: value = assembly.configuration[i.name].get(a.name) if value is not None: if a.type == 'string' and not \ isinstance(value, six.string_types): log.warning('attribute %s.%s has type string but is ' 'set to a value that is not a string' % (i.name, a.name)) elif a.type == 'int' and not \ isinstance(value, numbers.Number): log.warning('attribute %s.%s has type int but is set ' 'to a value that is not an integer' % (i.name, a.name)) obj_space = ObjectAllocator() obj_space.spec.arch = options.architecture render_state = RenderState(obj_space=obj_space) templates = Templates(options.platform) [templates.add_root(t) for t in options.templates] try: r = Renderer(templates) except jinja2.exceptions.TemplateSyntaxError as e: die('template syntax error: %s' % e) # The user may have provided their own connector definitions (with # associated) templates, in which case they won't be in the built-in lookup # dictionary. Let's add them now. Note, definitions here that conflict with # existing lookup entries will overwrite the existing entries. Note that # the extra check that the connector has some templates is just an # optimisation; the templates module handles connectors without templates # just fine. for c in (x for x in ast.items if isinstance(x, Connector) and ( x.from_template is not None or x.to_template is not None)): try: # Find a connection that uses this type. connection = next(x for x in ast if isinstance(x, Connection) and x.type == c) # Add the custom templates and update our collection of read # inputs. templates.add(c, connection) except TemplateError as e: die('while adding connector %s: %s' % (c.name, e)) except StopIteration: # No connections use this type. There's no point adding it to the # template lookup dictionary. pass def apply_capdl_filters(renderoptions): # Derive a set of usable ELF objects from the filenames we were passed. render_state = renderoptions.render_state elfs = {} for e in options.elf: try: name = os.path.basename(e) if name in elfs: raise Exception( 'duplicate ELF files of name \'%s\' encountered' % name) elf = ELF(e, name, options.architecture) group = name.replace("_group_bin", "") # Avoid inferring a TCB as we've already created our own. elf_spec = elf.get_spec( infer_tcb=False, infer_asid=False, pd=render_state.pds[group], use_large_frames=options.largeframe, addr_space=render_state.addr_spaces[group]) render_state.obj_space.merge(elf_spec, label=group) elfs[name] = (e, elf) except Exception as inst: die('While opening \'%s\': %s' % (e, inst)) for space in render_state.cspaces.values(): for (slot, tcb) in [ (v, v.referent) for (k, v) in space.cnode.slots.items() if v is not None and isinstance(v.referent, TCB) ]: elf = elfs.get(tcb.elf) funcs = {"get_vaddr": lambda x: elf[1].get_symbol_vaddr(x)} tcb.ip = simple_eval(str(tcb.ip), functions=funcs) tcb.sp = simple_eval(str(tcb.sp), functions=funcs) tcb.addr = simple_eval(str(tcb.addr), functions=funcs) if not options.fprovide_tcb_caps: del space.cnode[slot] space.cnode.finalise_size( arch=lookup_architecture(options.architecture)) renderoptions = RenderOptions( options.file, options.verbosity, options.frpc_lock_elision, options.fspecialise_syscall_stubs, options.fprovide_tcb_caps, options.largeframe, options.largeframe_dma, options.architecture, options.debug_fault_handlers, options.default_priority, options.default_max_priority, options.default_affinity, options.default_period, options.default_budget, options.default_data, options.default_size_bits, options.default_stack_size, options.realtime, options.verification_base_name, render_state) def instantiate_misc_templates(renderoptions): for (item, outfile) in (all_items - done_items): try: template = templates.lookup(item) if template: g = r.render(assembly, assembly, template, renderoptions.render_state, None, outfile_name=outfile.name, options=renderoptions) done(g, outfile, item, r, read) except TemplateError as inst: die(rendering_error(item, inst)) if "camkes-gen.cmake" in options.item: instantiate_misc_templates(renderoptions) if options.load_object_state is not None: # There is an assumption that if load_object_state is set, we # skip all of the component and connector logic below. # FIXME: refactor to clarify control flow renderoptions.render_state = pickle.load(options.load_object_state) apply_capdl_filters(renderoptions) instantiate_misc_templates(renderoptions) # If a template wasn't instantiated, something went wrong, and we can't recover raise CAmkESError("No template instantiated on capdl generation path") # We're now ready to instantiate the template the user requested, but there # are a few wrinkles in the process. Namely, # 1. Template instantiation needs to be done in a deterministic order. The # runner is invoked multiple times and template code needs to be # allocated identical cap slots in each run. # 2. Components and connections need to be instantiated before any other # templates, regardless of whether they are the ones we are after. Some # other templates, such as the Makefile depend on the obj_space and # cspaces. # 3. All actual code templates, up to the template that was requested, # need to be instantiated. This is related to (1) in that the cap slots # allocated are dependent on what allocations have been done prior to a # given allocation call. # Instantiate the per-component source and header files. for i in assembly.composition.instances: # Don't generate any code for hardware components. if i.type.hardware: continue if i.address_space not in renderoptions.render_state.cspaces: cnode = renderoptions.render_state.obj_space.alloc( ObjectType.seL4_CapTableObject, name="%s_cnode" % i.address_space, label=i.address_space) renderoptions.render_state.cspaces[ i.address_space] = CSpaceAllocator(cnode) pd = obj_space.alloc(lookup_architecture( options.architecture).vspace().object, name="%s_group_bin_pd" % i.address_space, label=i.address_space) addr_space = AddressSpaceAllocator( re.sub(r'[^A-Za-z0-9]', '_', "%s_group_bin" % i.address_space), pd) renderoptions.render_state.pds[i.address_space] = pd renderoptions.render_state.addr_spaces[ i.address_space] = addr_space for t in ('%s/source' % i.name, '%s/header' % i.name, '%s/c_environment_source' % i.name, '%s/cakeml_start_source' % i.name, '%s/cakeml_end_source' % i.name, '%s/linker' % i.name): try: template = templates.lookup(t, i) g = '' if template: g = r.render( i, assembly, template, renderoptions.render_state, i.address_space, outfile_name=None, options=renderoptions, my_pd=renderoptions.render_state.pds[i.address_space]) for (item, outfile) in (all_items - done_items): if item == t: if not template: log.warning('Warning: no template for %s' % item) done(g, outfile, item, r, read) break except TemplateError as inst: die(rendering_error(i.name, inst)) # Instantiate the per-connection files. for c in assembly.composition.connections: for t in (('%s/from/source' % c.name, c.from_ends), ('%s/from/header' % c.name, c.from_ends), ('%s/to/source' % c.name, c.to_ends), ('%s/to/header' % c.name, c.to_ends), ('%s/to/cakeml' % c.name, c.to_ends)): template = templates.lookup(t[0], c) if template is not None: for id, e in enumerate(t[1]): item = '%s/%d' % (t[0], id) g = '' try: g = r.render(e, assembly, template, renderoptions.render_state, e.instance.address_space, outfile_name=None, options=renderoptions, my_pd=renderoptions.render_state.pds[ e.instance.address_space]) except TemplateError as inst: die(rendering_error(item, inst)) except jinja2.exceptions.TemplateNotFound: die('While rendering %s: missing template for %s' % (item, c.type.name)) for (target, outfile) in (all_items - done_items): if target == item: if not template: log.warning('Warning: no template for %s' % item) done(g, outfile, item, r, read) break # The following block handles instantiations of per-connection # templates that are neither a 'source' or a 'header', as handled # above. We assume that none of these need instantiation unless we are # actually currently looking for them (== options.item). That is, we # assume that following templates, like the CapDL spec, do not require # these templates to be rendered prior to themselves. # FIXME: This is a pretty ugly way of handling this. It would be nicer # for the runner to have a more general notion of per-'thing' templates # where the per-component templates, the per-connection template loop # above, and this loop could all be done in a single unified control # flow. for (item, outfile) in (all_items - done_items): for t in (('%s/from/' % c.name, c.from_ends), ('%s/to/' % c.name, c.to_ends)): if not item.startswith(t[0]): # This is not the item we're looking for. continue # If we've reached here then this is the exact item we're after. template = templates.lookup(item, c) if template is None: die('no registered template for %s' % item) for e in t[1]: try: g = r.render(e, assembly, template, renderoptions.render_state, e.instance.address_space, outfile_name=None, options=renderoptions, my_pd=renderoptions.render_state.pds[ e.instance.address_space]) done(g, outfile, item, r, read) except TemplateError as inst: die(rendering_error(item, inst)) # Perform any per component special generation. This needs to happen last # as these template needs to run after all other capabilities have been # allocated for i in assembly.composition.instances: # Don't generate any code for hardware components. if i.type.hardware: continue assert i.address_space in renderoptions.render_state.cspaces SPECIAL_TEMPLATES = [('debug', 'debug'), ('simple', 'simple'), ('rump_config', 'rumprun')] for special in [ bl for bl in SPECIAL_TEMPLATES if assembly.configuration[i.name].get(bl[0]) ]: for t in ('%s/%s' % (i.name, special[1]), ): try: template = templates.lookup(t, i) g = '' if template: g = r.render(i, assembly, template, renderoptions.render_state, i.address_space, outfile_name=None, options=renderoptions, my_pd=renderoptions.render_state.pds[ i.address_space]) for (item, outfile) in (all_items - done_items): if item == t: if not template: log.warning('Warning: no template for %s' % item) done(g, outfile, item, r, read) except TemplateError as inst: die(rendering_error(i.name, inst)) # Check if there are any remaining items not_done = all_items - done_items if len(not_done) > 0: for (item, outfile) in not_done: err.write('No valid element matching --item %s.\n' % item) return -1 return 0