def all_sdk(self, sdk_path_list = None) : sdks = {} sdk_search_paths = self.context["sdk_search_path"] if isinstance(sdk_path_list, list) : sdk_search_paths += sdk_path_list elif isinstance(sdk_path_list, str) and os.path.exists(sdk_path_list) : sdk_search_paths.append(sdk_path_list) sdk_search_paths = list(set(sdk_search_paths)) for sdk_search_path in sdk_search_paths : for path, dirs, files in os.walk(sdk_search_path) : for sdk_dir in dirs : sdk_path = os.path.join(path, sdk_dir) try : sdk = SDKConfig(sdk_path) sdk_id = sdk.get_config("id") if not sdks.has_key(sdk_id) : sdks[sdk_id] = sdk except : continue break return sdk_search_paths, sdks
def get_fragment_grammar(): # Match header [mapping] header = Suppress("[") + Suppress("mapping") + Suppress("]") # There are three possible patterns for mapping entries: # obj:symbol (scheme) # obj (scheme) # * (scheme) obj = Fragment.ENTITY.setResultsName("object") symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol") scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")") pattern1 = Group(obj + symbol + scheme) pattern2 = Group(obj + scheme) pattern3 = Group(Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme) mapping_entry = pattern1 | pattern2 | pattern3 # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default # A normal grouping is one with a non-default condition. The default grouping is one which contains the # default condition mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings") normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar()) default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION)) normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries) default_group = Group(default_condition + mapping_entries).setResultsName("default_group") normal_groups = Group(ZeroOrMore(normal_group)).setResultsName("normal_groups") # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. archive = Suppress("archive") + Suppress(":") + Fragment.ENTITY.setResultsName("archive") entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries") mapping = Group(header + archive + entries) mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries)) mapping.ignore("#" + restOfLine) return mapping
def get_fragment_grammar(sdkconfig, fragment_file): # Match header [mapping] header = Suppress("[") + Suppress("mapping") + Suppress("]") # There are three possible patterns for mapping entries: # obj:symbol (scheme) # obj (scheme) # * (scheme) obj = Fragment.ENTITY.setResultsName("object") symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol") scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName( "scheme") + Suppress(")") pattern1 = Group(obj + symbol + scheme) pattern2 = Group(obj + scheme) pattern3 = Group( Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme) mapping_entry = pattern1 | pattern2 | pattern3 # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default # A normal grouping is one with a non-default condition. The default grouping is one which contains the # default condition mapping_entries = Group( ZeroOrMore(mapping_entry)).setResultsName("mappings") normal_condition = Suppress(":") + originalTextFor( SDKConfig.get_expression_grammar()) default_condition = Optional( Suppress(":") + Literal(DeprecatedMapping.DEFAULT_CONDITION)) normal_group = Group( normal_condition.setResultsName("condition") + mapping_entries) default_group = Group(default_condition + mapping_entries).setResultsName("default_group") normal_groups = Group( ZeroOrMore(normal_group)).setResultsName("normal_groups") # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. archive = Suppress("archive") + Suppress( ":") + Fragment.ENTITY.setResultsName("archive") entries = Suppress("entries") + Suppress(":") + ( normal_groups + default_group).setResultsName("entries") mapping = Group(header + archive + entries) mapping.ignore("#" + restOfLine) def parsed_deprecated_mapping(pstr, loc, toks): fragment = Mapping() fragment.archive = toks[0].archive fragment.name = re.sub(r"[^0-9a-zA-Z]+", "_", fragment.archive) fragment.deprecated = True fragment.entries = set() condition_true = False for entries in toks[0].entries[0]: condition = next(iter(entries.condition.asList())).strip() condition_val = sdkconfig.evaluate_expression(condition) if condition_val: for entry in entries[1]: fragment.entries.add( (entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) condition_true = True break if not fragment.entries and not condition_true: try: entries = toks[0].entries[1][1] except IndexError: entries = toks[0].entries[1][0] for entry in entries: fragment.entries.add( (entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) if not fragment.entries: fragment.entries.add(("*", None, "default")) dep_warning = str( ParseFatalException( pstr, loc, "Warning: Deprecated old-style mapping fragment parsed in file %s." % fragment_file)) print(dep_warning) return fragment mapping.setParseAction(parsed_deprecated_mapping) return mapping
def main(): argparser = argparse.ArgumentParser(description='ESP-IDF linker script generator') argparser.add_argument( '--input', '-i', help='Linker template file', type=argparse.FileType('r')) argparser.add_argument( '--fragments', '-f', type=argparse.FileType('r'), help='Input fragment files', nargs='+') argparser.add_argument( '--libraries-file', type=argparse.FileType('r'), help='File that contains the list of libraries in the build') argparser.add_argument( '--output', '-o', help='Output linker script', type=str) argparser.add_argument( '--config', '-c', help='Project configuration') argparser.add_argument( '--kconfig', '-k', help='IDF Kconfig file') argparser.add_argument( '--check-mapping', help='Perform a check if a mapping (archive, obj, symbol) exists', action='store_true' ) argparser.add_argument( '--check-mapping-exceptions', help='Mappings exempted from check', type=argparse.FileType('r') ) argparser.add_argument( '--env', '-e', action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') argparser.add_argument('--env-file', type=argparse.FileType('r'), help='Optional file to load environment variables from. Contents ' 'should be a JSON object where each key/value pair is a variable.') argparser.add_argument( '--objdump', help='Path to toolchain objdump') args = argparser.parse_args() input_file = args.input fragment_files = [] if not args.fragments else args.fragments libraries_file = args.libraries_file config_file = args.config output_path = args.output kconfig_file = args.kconfig objdump = args.objdump check_mapping = args.check_mapping if args.check_mapping_exceptions: check_mapping_exceptions = [line.strip() for line in args.check_mapping_exceptions] else: check_mapping_exceptions = None try: sections_infos = SectionsInfo() for library in libraries_file: library = library.strip() if library: dump = StringIO(subprocess.check_output([objdump, '-h', library]).decode()) dump.name = library sections_infos.add_sections_info(dump) generation_model = GenerationModel(check_mapping, check_mapping_exceptions) _update_environment(args) # assign args.env and args.env_file to os.environ sdkconfig = SDKConfig(kconfig_file, config_file) for fragment_file in fragment_files: try: fragment_file = FragmentFile(fragment_file, sdkconfig) except (ParseException, ParseFatalException) as e: # ParseException is raised on incorrect grammar # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate # keys, key unsupported by fragment, unexpected number of values, etc.) raise LdGenFailure('failed to parse %s\n%s' % (fragment_file.name, str(e))) generation_model.add_fragments_from_file(fragment_file) mapping_rules = generation_model.generate_rules(sections_infos) script_model = TemplateModel(input_file) script_model.fill(mapping_rules) with tempfile.TemporaryFile('w+') as output: script_model.write(output) output.seek(0) if not os.path.exists(os.path.dirname(output_path)): try: os.makedirs(os.path.dirname(output_path)) except OSError as exc: if exc.errno != errno.EEXIST: raise with open(output_path, 'w') as f: # only create output file after generation has suceeded f.write(output.read()) except LdGenFailure as e: print('linker script generation failed for %s\nERROR: %s' % (input_file.name, e)) sys.exit(1)
def __init__(self, fragment_file, sdkconfig): try: fragment_file = open(fragment_file, "r") except TypeError: pass path = os.path.realpath(fragment_file.name) indent_stack = [1] class parse_ctx: fragment = None # current fragment key = "" # current key keys = list() # list of keys parsed key_grammar = None # current key grammar @staticmethod def reset(): parse_ctx.fragment_instance = None parse_ctx.key = "" parse_ctx.keys = list() parse_ctx.key_grammar = None def fragment_type_parse_action(toks): parse_ctx.reset() parse_ctx.fragment = FRAGMENT_TYPES[ toks[0]]() # create instance of the fragment return None def expand_conditionals(toks, stmts): try: stmt = toks["value"] stmts.append(stmt) except KeyError: try: conditions = toks["conditional"] for condition in conditions: try: _toks = condition[1] _cond = condition[0] if sdkconfig.evaluate_expression(_cond): expand_conditionals(_toks, stmts) break except IndexError: expand_conditionals(condition[0], stmts) except KeyError: for tok in toks: expand_conditionals(tok, stmts) def key_body_parsed(pstr, loc, toks): stmts = list() expand_conditionals(toks, stmts) if parse_ctx.key_grammar.min and len( stmts) < parse_ctx.key_grammar.min: raise ParseFatalException( pstr, loc, "fragment requires at least %d values for key '%s'" % (parse_ctx.key_grammar.min, parse_ctx.key)) if parse_ctx.key_grammar.max and len( stmts) > parse_ctx.key_grammar.max: raise ParseFatalException( pstr, loc, "fragment requires at most %d values for key '%s'" % (parse_ctx.key_grammar.max, parse_ctx.key)) try: parse_ctx.fragment.set_key_value(parse_ctx.key, stmts) except Exception as e: raise ParseFatalException( pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, e.message)) return None key = Word(alphanums + "_") + Suppress(":") key_stmt = Forward() condition_block = indentedBlock(key_stmt, indent_stack) key_stmts = OneOrMore(condition_block) key_body = Suppress(key) + key_stmts key_body.setParseAction(key_body_parsed) condition = originalTextFor( SDKConfig.get_expression_grammar()).setResultsName("condition") if_condition = Group( Suppress("if") + condition + Suppress(":") + condition_block) elif_condition = Group( Suppress("elif") + condition + Suppress(":") + condition_block) else_condition = Group( Suppress("else") + Suppress(":") + condition_block) conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName("conditional") def key_parse_action(pstr, loc, toks): key = toks[0] if key in parse_ctx.keys: raise ParseFatalException( pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key) parse_ctx.key = key parse_ctx.keys.append(key) try: parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars( )[key] key_grammar = parse_ctx.key_grammar.grammar except KeyError: raise ParseFatalException( pstr, loc, "key '%s' is not supported by fragment" % key) except Exception as e: raise ParseFatalException( pstr, loc, "unable to parse key '%s'; %s" % (key, e.message)) key_stmt << (conditional | Group(key_grammar).setResultsName("value")) return None def name_parse_action(pstr, loc, toks): parse_ctx.fragment.name = toks[0] key.setParseAction(key_parse_action) ftype = Word(alphas).setParseAction(fragment_type_parse_action) fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name") fid.setParseAction(name_parse_action) header = Suppress("[") + ftype + fid + Suppress("]") def fragment_parse_action(pstr, loc, toks): key_grammars = parse_ctx.fragment.get_key_grammars() required_keys = set( [k for (k, v) in key_grammars.items() if v.required]) present_keys = required_keys.intersection(set(parse_ctx.keys)) if present_keys != required_keys: raise ParseFatalException( pstr, loc, "required keys %s for fragment not found" % list(required_keys - present_keys)) return parse_ctx.fragment fragment_stmt = Forward() fragment_block = indentedBlock(fragment_stmt, indent_stack) fragment_if_condition = Group( Suppress("if") + condition + Suppress(":") + fragment_block) fragment_elif_condition = Group( Suppress("elif") + condition + Suppress(":") + fragment_block) fragment_else_condition = Group( Suppress("else") + Suppress(":") + fragment_block) fragment_conditional = ( fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) + Optional(fragment_else_condition)).setResultsName("conditional") fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName("value") fragment.setParseAction(fragment_parse_action) fragment.ignore("#" + restOfLine) deprecated_mapping = DeprecatedMapping.get_fragment_grammar( sdkconfig, fragment_file.name).setResultsName("value") fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional)) def fragment_stmt_parsed(pstr, loc, toks): stmts = list() expand_conditionals(toks, stmts) return stmts parser = ZeroOrMore(fragment_stmt) parser.setParseAction(fragment_stmt_parsed) self.fragments = parser.parseFile(fragment_file, parseAll=True) for fragment in self.fragments: fragment.path = path
def main(): argparser = argparse.ArgumentParser(description="ESP-IDF linker script generator") argparser.add_argument( "--input", "-i", help="Linker template file", type=argparse.FileType("r")) argparser.add_argument( "--fragments", "-f", type=argparse.FileType("r"), help="Input fragment files", nargs="+") argparser.add_argument( "--libraries-file", type=argparse.FileType("r"), help="File that contains the list of libraries in the build") argparser.add_argument( "--output", "-o", help="Output linker script", type=str) argparser.add_argument( "--config", "-c", help="Project configuration") argparser.add_argument( "--kconfig", "-k", help="IDF Kconfig file") argparser.add_argument( "--env", "-e", action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') argparser.add_argument('--env-file', type=argparse.FileType('r'), help='Optional file to load environment variables from. Contents ' 'should be a JSON object where each key/value pair is a variable.') argparser.add_argument( "--objdump", help="Path to toolchain objdump") args = argparser.parse_args() input_file = args.input fragment_files = [] if not args.fragments else args.fragments libraries_file = args.libraries_file config_file = args.config output_path = args.output kconfig_file = args.kconfig objdump = args.objdump try: sections_infos = SectionsInfo() for library in libraries_file: library = library.strip() if library: dump = StringIO(subprocess.check_output([objdump, "-h", library]).decode()) dump.name = library sections_infos.add_sections_info(dump) generation_model = GenerationModel() _update_environment(args) # assign args.env and args.env_file to os.environ sdkconfig = SDKConfig(kconfig_file, config_file) for fragment_file in fragment_files: try: fragment_file = FragmentFile(fragment_file, sdkconfig) except (ParseException, ParseFatalException) as e: # ParseException is raised on incorrect grammar # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate # keys, key unsupported by fragment, unexpected number of values, etc.) raise LdGenFailure("failed to parse %s\n%s" % (fragment_file.name, str(e))) generation_model.add_fragments_from_file(fragment_file) mapping_rules = generation_model.generate_rules(sections_infos) script_model = TemplateModel(input_file) script_model.fill(mapping_rules) with tempfile.TemporaryFile("w+") as output: script_model.write(output) output.seek(0) if not os.path.exists(os.path.dirname(output_path)): try: os.makedirs(os.path.dirname(output_path)) except OSError as exc: if exc.errno != errno.EEXIST: raise with open(output_path, "w") as f: # only create output file after generation has suceeded f.write(output.read()) except LdGenFailure as e: print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e)) sys.exit(1)
def main(): argparser = argparse.ArgumentParser( description="ESP-IDF linker script generator") argparser.add_argument("--input", "-i", help="Linker template file", type=argparse.FileType("r")) argparser.add_argument("--fragments", "-f", type=argparse.FileType("r"), help="Input fragment files", nargs="+") argparser.add_argument("--sections", "-s", type=argparse.FileType("r"), help="Library sections info") argparser.add_argument("--output", "-o", help="Output linker script", type=str) argparser.add_argument("--config", "-c", help="Project configuration", type=argparse.FileType("r")) argparser.add_argument("--kconfig", "-k", help="IDF Kconfig file", type=argparse.FileType("r")) argparser.add_argument( "--env", "-e", action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') args = argparser.parse_args() input_file = args.input fragment_files = [] if not args.fragments else args.fragments config_file = args.config output_path = args.output kconfig_file = args.kconfig sections = args.sections try: sections_infos = SectionsInfo() if sections: section_info_contents = [ s.strip() for s in sections.read().split("\n") ] section_info_contents = [s for s in section_info_contents if s] else: section_info_contents = [] for sections_info_file in section_info_contents: with open(sections_info_file) as sections_info_file_obj: sections_infos.add_sections_info(sections_info_file_obj) generation_model = GenerationModel() for fragment_file in fragment_files: fragment_file = FragmentFileModel(fragment_file) generation_model.add_fragments_from_file(fragment_file) sdkconfig = SDKConfig(kconfig_file, config_file, args.env) mapping_rules = generation_model.generate_rules( sdkconfig, sections_infos) script_model = TemplateModel(input_file) script_model.fill(mapping_rules, sdkconfig) with tempfile.TemporaryFile("w+") as output: script_model.write(output) output.seek(0) with open( output_path, "w" ) as f: # only create output file after generation has suceeded f.write(output.read()) except LdGenFailure as e: print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e)) sys.exit(1)
def parse_expression(self, expression): parser = SDKConfig.get_expression_grammar() return parser.parseString(expression, parseAll=True)
def main(): argparser = argparse.ArgumentParser( description="ESP-IDF linker script generator") argparser.add_argument("--input", "-i", help="Linker template file", type=argparse.FileType("r")) argparser.add_argument("--fragments", "-f", type=argparse.FileType("r"), help="Input fragment files", nargs="+") argparser.add_argument("--sections", "-s", type=argparse.FileType("r"), help="Library sections info", nargs="+") argparser.add_argument("--output", "-o", help="Output linker script", type=argparse.FileType("w")) argparser.add_argument("--config", "-c", help="Project configuration", type=argparse.FileType("r")) argparser.add_argument("--kconfig", "-k", help="IDF Kconfig file", type=argparse.FileType("r")) argparser.add_argument( "--env", "-e", action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') args = argparser.parse_args() input_file = args.input fragment_files = [] if not args.fragments else args.fragments config_file = args.config output_file = args.output sections_info_files = [] if not args.sections else args.sections kconfig_file = args.kconfig try: sections_infos = SectionsInfo() for sections_info_file in sections_info_files: sections_infos.add_sections_info(sections_info_file) generation_model = GenerationModel() for fragment_file in fragment_files: fragment_file = FragmentFileModel(fragment_file) generation_model.add_fragments_from_file(fragment_file) sdkconfig = SDKConfig(kconfig_file, config_file, args.env) mapping_rules = generation_model.generate_rules( sdkconfig, sections_infos) script_model = TemplateModel(input_file) script_model.fill(mapping_rules, sdkconfig) script_model.write(output_file) except Exception as e: print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e.message)) # Delete the file so the entire build will fail; and not use an outdated script. os.remove(output_file.name) # Print traceback and exit traceback.print_exc() sys.exit(1)
def setUp(self): self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
def parse_expression(self, expression): parser = SDKConfig.get_expression_grammar() return parser.parseString(expression, parseAll=True)