def all_sdk(self, sdk_path_list = None) :
sdks = {}
sdk_search_paths = self.context["sdk_search_path"]
if isinstance(sdk_path_list, list) :
sdk_search_paths += sdk_path_list
elif isinstance(sdk_path_list, str) and os.path.exists(sdk_path_list) :
sdk_search_paths.append(sdk_path_list)
sdk_search_paths = list(set(sdk_search_paths))
for sdk_search_path in sdk_search_paths :
for path, dirs, files in os.walk(sdk_search_path) :
for sdk_dir in dirs :
sdk_path = os.path.join(path, sdk_dir)
try :
sdk = SDKConfig(sdk_path)
sdk_id = sdk.get_config("id")
if not sdks.has_key(sdk_id) :
sdks[sdk_id] = sdk
except :
continue
break
return sdk_search_paths, sdks
def get_fragment_grammar():
# Match header [mapping]
header = Suppress("[") + Suppress("mapping") + Suppress("]")
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName("object")
symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")")
pattern1 = Group(obj + symbol + scheme)
pattern2 = Group(obj + scheme)
pattern3 = Group(Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme)
mapping_entry = pattern1 | pattern2 | pattern3
# To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
# A normal grouping is one with a non-default condition. The default grouping is one which contains the
# default condition
mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings")
normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar())
default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries)
default_group = Group(default_condition + mapping_entries).setResultsName("default_group")
normal_groups = Group(ZeroOrMore(normal_group)).setResultsName("normal_groups")
# Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
archive = Suppress("archive") + Suppress(":") + Fragment.ENTITY.setResultsName("archive")
entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries")
mapping = Group(header + archive + entries)
mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
mapping.ignore("#" + restOfLine)
return mapping
def get_fragment_grammar(sdkconfig, fragment_file):
# Match header [mapping]
header = Suppress("[") + Suppress("mapping") + Suppress("]")
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName("object")
symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName(
"scheme") + Suppress(")")
pattern1 = Group(obj + symbol + scheme)
pattern2 = Group(obj + scheme)
pattern3 = Group(
Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") +
scheme)
mapping_entry = pattern1 | pattern2 | pattern3
# To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
# A normal grouping is one with a non-default condition. The default grouping is one which contains the
# default condition
mapping_entries = Group(
ZeroOrMore(mapping_entry)).setResultsName("mappings")
normal_condition = Suppress(":") + originalTextFor(
SDKConfig.get_expression_grammar())
default_condition = Optional(
Suppress(":") + Literal(DeprecatedMapping.DEFAULT_CONDITION))
normal_group = Group(
normal_condition.setResultsName("condition") + mapping_entries)
default_group = Group(default_condition +
mapping_entries).setResultsName("default_group")
normal_groups = Group(
ZeroOrMore(normal_group)).setResultsName("normal_groups")
# Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
archive = Suppress("archive") + Suppress(
":") + Fragment.ENTITY.setResultsName("archive")
entries = Suppress("entries") + Suppress(":") + (
normal_groups + default_group).setResultsName("entries")
mapping = Group(header + archive + entries)
mapping.ignore("#" + restOfLine)
def parsed_deprecated_mapping(pstr, loc, toks):
fragment = Mapping()
fragment.archive = toks[0].archive
fragment.name = re.sub(r"[^0-9a-zA-Z]+", "_", fragment.archive)
fragment.deprecated = True
fragment.entries = set()
condition_true = False
for entries in toks[0].entries[0]:
condition = next(iter(entries.condition.asList())).strip()
condition_val = sdkconfig.evaluate_expression(condition)
if condition_val:
for entry in entries[1]:
fragment.entries.add(
(entry.object,
None if entry.symbol == '' else entry.symbol,
entry.scheme))
condition_true = True
break
if not fragment.entries and not condition_true:
try:
entries = toks[0].entries[1][1]
except IndexError:
entries = toks[0].entries[1][0]
for entry in entries:
fragment.entries.add(
(entry.object,
None if entry.symbol == '' else entry.symbol,
entry.scheme))
if not fragment.entries:
fragment.entries.add(("*", None, "default"))
dep_warning = str(
ParseFatalException(
pstr, loc,
"Warning: Deprecated old-style mapping fragment parsed in file %s."
% fragment_file))
print(dep_warning)
return fragment
mapping.setParseAction(parsed_deprecated_mapping)
return mapping
def main():
argparser = argparse.ArgumentParser(description='ESP-IDF linker script generator')
argparser.add_argument(
'--input', '-i',
help='Linker template file',
type=argparse.FileType('r'))
argparser.add_argument(
'--fragments', '-f',
type=argparse.FileType('r'),
help='Input fragment files',
nargs='+')
argparser.add_argument(
'--libraries-file',
type=argparse.FileType('r'),
help='File that contains the list of libraries in the build')
argparser.add_argument(
'--output', '-o',
help='Output linker script',
type=str)
argparser.add_argument(
'--config', '-c',
help='Project configuration')
argparser.add_argument(
'--kconfig', '-k',
help='IDF Kconfig file')
argparser.add_argument(
'--check-mapping',
help='Perform a check if a mapping (archive, obj, symbol) exists',
action='store_true'
)
argparser.add_argument(
'--check-mapping-exceptions',
help='Mappings exempted from check',
type=argparse.FileType('r')
)
argparser.add_argument(
'--env', '-e',
action='append', default=[],
help='Environment to set when evaluating the config file', metavar='NAME=VAL')
argparser.add_argument('--env-file', type=argparse.FileType('r'),
help='Optional file to load environment variables from. Contents '
'should be a JSON object where each key/value pair is a variable.')
argparser.add_argument(
'--objdump',
help='Path to toolchain objdump')
args = argparser.parse_args()
input_file = args.input
fragment_files = [] if not args.fragments else args.fragments
libraries_file = args.libraries_file
config_file = args.config
output_path = args.output
kconfig_file = args.kconfig
objdump = args.objdump
check_mapping = args.check_mapping
if args.check_mapping_exceptions:
check_mapping_exceptions = [line.strip() for line in args.check_mapping_exceptions]
else:
check_mapping_exceptions = None
try:
sections_infos = SectionsInfo()
for library in libraries_file:
library = library.strip()
if library:
dump = StringIO(subprocess.check_output([objdump, '-h', library]).decode())
dump.name = library
sections_infos.add_sections_info(dump)
generation_model = GenerationModel(check_mapping, check_mapping_exceptions)
_update_environment(args) # assign args.env and args.env_file to os.environ
sdkconfig = SDKConfig(kconfig_file, config_file)
for fragment_file in fragment_files:
try:
fragment_file = FragmentFile(fragment_file, sdkconfig)
except (ParseException, ParseFatalException) as e:
# ParseException is raised on incorrect grammar
# ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
# keys, key unsupported by fragment, unexpected number of values, etc.)
raise LdGenFailure('failed to parse %s\n%s' % (fragment_file.name, str(e)))
generation_model.add_fragments_from_file(fragment_file)
mapping_rules = generation_model.generate_rules(sections_infos)
script_model = TemplateModel(input_file)
script_model.fill(mapping_rules)
with tempfile.TemporaryFile('w+') as output:
script_model.write(output)
output.seek(0)
if not os.path.exists(os.path.dirname(output_path)):
try:
os.makedirs(os.path.dirname(output_path))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
with open(output_path, 'w') as f: # only create output file after generation has suceeded
f.write(output.read())
except LdGenFailure as e:
print('linker script generation failed for %s\nERROR: %s' % (input_file.name, e))
sys.exit(1)
def __init__(self, fragment_file, sdkconfig):
try:
fragment_file = open(fragment_file, "r")
except TypeError:
pass
path = os.path.realpath(fragment_file.name)
indent_stack = [1]
class parse_ctx:
fragment = None # current fragment
key = "" # current key
keys = list() # list of keys parsed
key_grammar = None # current key grammar
@staticmethod
def reset():
parse_ctx.fragment_instance = None
parse_ctx.key = ""
parse_ctx.keys = list()
parse_ctx.key_grammar = None
def fragment_type_parse_action(toks):
parse_ctx.reset()
parse_ctx.fragment = FRAGMENT_TYPES[
toks[0]]() # create instance of the fragment
return None
def expand_conditionals(toks, stmts):
try:
stmt = toks["value"]
stmts.append(stmt)
except KeyError:
try:
conditions = toks["conditional"]
for condition in conditions:
try:
_toks = condition[1]
_cond = condition[0]
if sdkconfig.evaluate_expression(_cond):
expand_conditionals(_toks, stmts)
break
except IndexError:
expand_conditionals(condition[0], stmts)
except KeyError:
for tok in toks:
expand_conditionals(tok, stmts)
def key_body_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
if parse_ctx.key_grammar.min and len(
stmts) < parse_ctx.key_grammar.min:
raise ParseFatalException(
pstr, loc,
"fragment requires at least %d values for key '%s'" %
(parse_ctx.key_grammar.min, parse_ctx.key))
if parse_ctx.key_grammar.max and len(
stmts) > parse_ctx.key_grammar.max:
raise ParseFatalException(
pstr, loc,
"fragment requires at most %d values for key '%s'" %
(parse_ctx.key_grammar.max, parse_ctx.key))
try:
parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to add key '%s'; %s" % (parse_ctx.key, e.message))
return None
key = Word(alphanums + "_") + Suppress(":")
key_stmt = Forward()
condition_block = indentedBlock(key_stmt, indent_stack)
key_stmts = OneOrMore(condition_block)
key_body = Suppress(key) + key_stmts
key_body.setParseAction(key_body_parsed)
condition = originalTextFor(
SDKConfig.get_expression_grammar()).setResultsName("condition")
if_condition = Group(
Suppress("if") + condition + Suppress(":") + condition_block)
elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + condition_block)
else_condition = Group(
Suppress("else") + Suppress(":") + condition_block)
conditional = (if_condition + Optional(OneOrMore(elif_condition)) +
Optional(else_condition)).setResultsName("conditional")
def key_parse_action(pstr, loc, toks):
key = toks[0]
if key in parse_ctx.keys:
raise ParseFatalException(
pstr, loc,
"duplicate key '%s' value definition" % parse_ctx.key)
parse_ctx.key = key
parse_ctx.keys.append(key)
try:
parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars(
)[key]
key_grammar = parse_ctx.key_grammar.grammar
except KeyError:
raise ParseFatalException(
pstr, loc, "key '%s' is not supported by fragment" % key)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to parse key '%s'; %s" % (key, e.message))
key_stmt << (conditional
| Group(key_grammar).setResultsName("value"))
return None
def name_parse_action(pstr, loc, toks):
parse_ctx.fragment.name = toks[0]
key.setParseAction(key_parse_action)
ftype = Word(alphas).setParseAction(fragment_type_parse_action)
fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
fid.setParseAction(name_parse_action)
header = Suppress("[") + ftype + fid + Suppress("]")
def fragment_parse_action(pstr, loc, toks):
key_grammars = parse_ctx.fragment.get_key_grammars()
required_keys = set(
[k for (k, v) in key_grammars.items() if v.required])
present_keys = required_keys.intersection(set(parse_ctx.keys))
if present_keys != required_keys:
raise ParseFatalException(
pstr, loc, "required keys %s for fragment not found" %
list(required_keys - present_keys))
return parse_ctx.fragment
fragment_stmt = Forward()
fragment_block = indentedBlock(fragment_stmt, indent_stack)
fragment_if_condition = Group(
Suppress("if") + condition + Suppress(":") + fragment_block)
fragment_elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + fragment_block)
fragment_else_condition = Group(
Suppress("else") + Suppress(":") + fragment_block)
fragment_conditional = (
fragment_if_condition +
Optional(OneOrMore(fragment_elif_condition)) +
Optional(fragment_else_condition)).setResultsName("conditional")
fragment = (header +
OneOrMore(indentedBlock(key_body, indent_stack,
False))).setResultsName("value")
fragment.setParseAction(fragment_parse_action)
fragment.ignore("#" + restOfLine)
deprecated_mapping = DeprecatedMapping.get_fragment_grammar(
sdkconfig, fragment_file.name).setResultsName("value")
fragment_stmt << (Group(deprecated_mapping) | Group(fragment)
| Group(fragment_conditional))
def fragment_stmt_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
return stmts
parser = ZeroOrMore(fragment_stmt)
parser.setParseAction(fragment_stmt_parsed)
self.fragments = parser.parseFile(fragment_file, parseAll=True)
for fragment in self.fragments:
fragment.path = path
def main():
argparser = argparse.ArgumentParser(description="ESP-IDF linker script generator")
argparser.add_argument(
"--input", "-i",
help="Linker template file",
type=argparse.FileType("r"))
argparser.add_argument(
"--fragments", "-f",
type=argparse.FileType("r"),
help="Input fragment files",
nargs="+")
argparser.add_argument(
"--libraries-file",
type=argparse.FileType("r"),
help="File that contains the list of libraries in the build")
argparser.add_argument(
"--output", "-o",
help="Output linker script",
type=str)
argparser.add_argument(
"--config", "-c",
help="Project configuration")
argparser.add_argument(
"--kconfig", "-k",
help="IDF Kconfig file")
argparser.add_argument(
"--env", "-e",
action='append', default=[],
help='Environment to set when evaluating the config file', metavar='NAME=VAL')
argparser.add_argument('--env-file', type=argparse.FileType('r'),
help='Optional file to load environment variables from. Contents '
'should be a JSON object where each key/value pair is a variable.')
argparser.add_argument(
"--objdump",
help="Path to toolchain objdump")
args = argparser.parse_args()
input_file = args.input
fragment_files = [] if not args.fragments else args.fragments
libraries_file = args.libraries_file
config_file = args.config
output_path = args.output
kconfig_file = args.kconfig
objdump = args.objdump
try:
sections_infos = SectionsInfo()
for library in libraries_file:
library = library.strip()
if library:
dump = StringIO(subprocess.check_output([objdump, "-h", library]).decode())
dump.name = library
sections_infos.add_sections_info(dump)
generation_model = GenerationModel()
_update_environment(args) # assign args.env and args.env_file to os.environ
sdkconfig = SDKConfig(kconfig_file, config_file)
for fragment_file in fragment_files:
try:
fragment_file = FragmentFile(fragment_file, sdkconfig)
except (ParseException, ParseFatalException) as e:
# ParseException is raised on incorrect grammar
# ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
# keys, key unsupported by fragment, unexpected number of values, etc.)
raise LdGenFailure("failed to parse %s\n%s" % (fragment_file.name, str(e)))
generation_model.add_fragments_from_file(fragment_file)
mapping_rules = generation_model.generate_rules(sections_infos)
script_model = TemplateModel(input_file)
script_model.fill(mapping_rules)
with tempfile.TemporaryFile("w+") as output:
script_model.write(output)
output.seek(0)
if not os.path.exists(os.path.dirname(output_path)):
try:
os.makedirs(os.path.dirname(output_path))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
with open(output_path, "w") as f: # only create output file after generation has suceeded
f.write(output.read())
except LdGenFailure as e:
print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e))
sys.exit(1)
def main():
argparser = argparse.ArgumentParser(
description="ESP-IDF linker script generator")
argparser.add_argument("--input",
"-i",
help="Linker template file",
type=argparse.FileType("r"))
argparser.add_argument("--fragments",
"-f",
type=argparse.FileType("r"),
help="Input fragment files",
nargs="+")
argparser.add_argument("--sections",
"-s",
type=argparse.FileType("r"),
help="Library sections info")
argparser.add_argument("--output",
"-o",
help="Output linker script",
type=str)
argparser.add_argument("--config",
"-c",
help="Project configuration",
type=argparse.FileType("r"))
argparser.add_argument("--kconfig",
"-k",
help="IDF Kconfig file",
type=argparse.FileType("r"))
argparser.add_argument(
"--env",
"-e",
action='append',
default=[],
help='Environment to set when evaluating the config file',
metavar='NAME=VAL')
args = argparser.parse_args()
input_file = args.input
fragment_files = [] if not args.fragments else args.fragments
config_file = args.config
output_path = args.output
kconfig_file = args.kconfig
sections = args.sections
try:
sections_infos = SectionsInfo()
if sections:
section_info_contents = [
s.strip() for s in sections.read().split("\n")
]
section_info_contents = [s for s in section_info_contents if s]
else:
section_info_contents = []
for sections_info_file in section_info_contents:
with open(sections_info_file) as sections_info_file_obj:
sections_infos.add_sections_info(sections_info_file_obj)
generation_model = GenerationModel()
for fragment_file in fragment_files:
fragment_file = FragmentFileModel(fragment_file)
generation_model.add_fragments_from_file(fragment_file)
sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
mapping_rules = generation_model.generate_rules(
sdkconfig, sections_infos)
script_model = TemplateModel(input_file)
script_model.fill(mapping_rules, sdkconfig)
with tempfile.TemporaryFile("w+") as output:
script_model.write(output)
output.seek(0)
with open(
output_path, "w"
) as f: # only create output file after generation has suceeded
f.write(output.read())
except LdGenFailure as e:
print("linker script generation failed for %s\nERROR: %s" %
(input_file.name, e))
sys.exit(1)
def parse_expression(self, expression):
parser = SDKConfig.get_expression_grammar()
return parser.parseString(expression, parseAll=True)
def main():
argparser = argparse.ArgumentParser(
description="ESP-IDF linker script generator")
argparser.add_argument("--input",
"-i",
help="Linker template file",
type=argparse.FileType("r"))
argparser.add_argument("--fragments",
"-f",
type=argparse.FileType("r"),
help="Input fragment files",
nargs="+")
argparser.add_argument("--sections",
"-s",
type=argparse.FileType("r"),
help="Library sections info",
nargs="+")
argparser.add_argument("--output",
"-o",
help="Output linker script",
type=argparse.FileType("w"))
argparser.add_argument("--config",
"-c",
help="Project configuration",
type=argparse.FileType("r"))
argparser.add_argument("--kconfig",
"-k",
help="IDF Kconfig file",
type=argparse.FileType("r"))
argparser.add_argument(
"--env",
"-e",
action='append',
default=[],
help='Environment to set when evaluating the config file',
metavar='NAME=VAL')
args = argparser.parse_args()
input_file = args.input
fragment_files = [] if not args.fragments else args.fragments
config_file = args.config
output_file = args.output
sections_info_files = [] if not args.sections else args.sections
kconfig_file = args.kconfig
try:
sections_infos = SectionsInfo()
for sections_info_file in sections_info_files:
sections_infos.add_sections_info(sections_info_file)
generation_model = GenerationModel()
for fragment_file in fragment_files:
fragment_file = FragmentFileModel(fragment_file)
generation_model.add_fragments_from_file(fragment_file)
sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
mapping_rules = generation_model.generate_rules(
sdkconfig, sections_infos)
script_model = TemplateModel(input_file)
script_model.fill(mapping_rules, sdkconfig)
script_model.write(output_file)
except Exception as e:
print("linker script generation failed for %s\nERROR: %s" %
(input_file.name, e.message))
# Delete the file so the entire build will fail; and not use an outdated script.
os.remove(output_file.name)
# Print traceback and exit
traceback.print_exc()
sys.exit(1)
def setUp(self):
self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
def parse_expression(self, expression):
parser = SDKConfig.get_expression_grammar()
return parser.parseString(expression, parseAll=True)