def _lookup_ident_value(self, ident, settings):
"""Look up an identifier's value"""
if settings.has_value(ident):
lookup_val = settings.get_value(ident)
if lookup_val is None:
Log.E("{} has no value".format(ident))
# Discover the data type from context
m = Matcher.new(lookup_val)
if lookup_val in RESERVED_IDENT_MAP:
mapped_ident = RESERVED_IDENT_MAP[lookup_val]
lookup_val = mapped_ident[RESERVED_IDENT.VALUE]
self.ident_class = mapped_ident[RESERVED_IDENT.CLASS]
elif m.is_fullmatch(self.STRING_CONFIG_RE):
lookup_val = m[1]
self.ident_class = StringToken
elif lookup_val.isnumeric():
lookup_val = int(lookup_val)
self.ident_class = IntToken
elif m.is_fullmatch(settings.ident_re):
# Value is an identifier, look up its value
self._lookup_ident_value(lookup_val, settings)
return
else:
Log.E("Malformed config value {}".format(lookup_val))
self.value = lookup_val
else:
Log.E("{} not defined".format(ident))
def generate(self, definitions, out_dir):
"""Generate a settings file"""
java_file = self.filename.lower()
package_dir = os.path.dirname(java_file)
package_name = os.path.relpath(package_dir, start=out_dir)
package_name = self.PACKAGE_RE.sub(".", package_name)
class_name = self.CLASS_RE.sub("", os.path.basename(self.filename))
os.makedirs(Util.get_abs_path(package_dir, out_dir), exist_ok=True)
with LineReader.OpenFileCM(java_file, "wt") as hdr_cm:
if not hdr_cm:
Log.E("Could not open {}: {}".format(self.filename,
hdr_cm.get_err_msg()))
hdr_file = hdr_cm.get_file()
# File header
hdr_file.write("\n".join([
"/* GENERATED BY GLOBIFEST -- DO NOT EDIT */", "",
"package {}".format(package_name), "",
"public final class {}".format(class_name), "{\n"
]))
# Add values
template = " public final static {} {} = {};\n"
for d in definitions:
ptype = d.param.get_type()
pid = d.param.get_identifier()
value = d.value
# Write implicit values first
implicit_id = None
for implicit_value in d.param.get_implicit_values():
hdr_file.write(
template.format("int", implicit_value[0],
implicit_value[1]))
if value == implicit_value[1]:
implicit_id = implicit_value[0]
# Write the parameter
if ptype == DefTree.PARAM_TYPE.INT:
hdr_file.write(template.format("int", pid, value))
elif ptype == DefTree.PARAM_TYPE.FLOAT:
# Default type is double precision
hdr_file.write(template.format("double", pid, value))
elif ptype == DefTree.PARAM_TYPE.STRING:
hdr_file.write(template.format("String", pid, value))
elif ptype == DefTree.PARAM_TYPE.BOOL:
hdr_file.write(
template.format("boolean", pid, value.lower()))
elif ptype == DefTree.PARAM_TYPE.ENUM:
if implicit_id:
value = implicit_id
hdr_file.write(template.format("int", pid, value))
else:
# TODO: Handle more complex literal types
Log.E("Unhandled value of type {}".format(
str(d.param.ptype)))
# File footer
hdr_file.write("}\n")
def log_error(self, err_text):
"""
Log an error
@note This does not return
"""
Log.E("{}: {}".format(self.line_info, err_text), stackframe=3)
def on_param(self, param):
"""Handle a parameter"""
try:
value = self.settings.get_value(param.get_identifier())
except KeyError:
Log.E("Undefined value {}".format(param))
self.out.append(Util.Container(param=param, value=value))
def log_error(self, msg):
"""
Log an error
@note The error may not be fatal, so it should be handled as well.
"""
Log.E(msg, err_type=self.err_ctx, is_fatal=self.err_fatal)
def _eval_check_types(self):
"""
Override type checking for logical binary operators
Both tokens must be BoolType
"""
for t in self.tokens:
if not t.matches_class(BoolToken):
Log.E("{} is not a boolean value".format(t.get_value()))
def _eval_check_types(self):
"""
Implement type checking for a binary operator
The operator's tokens must match types
"""
tok1 = self.tokens[0]
tok2 = self.tokens[1]
if not tok1.matches_type(tok2):
Log.E("Type mismatch: {}({}) {} {}({})".format(
tok1.get_name(), tok1.TOKEN_TYPE, self.OP_TEXT,
tok2.get_name(), tok2.TOKEN_TYPE))
# Strings (not Ident with string values) have restrictions on which operators can be used.
is_string_tok = (tok1.matches_class(StringToken)) or (
tok2.matches_class(StringToken))
if is_string_tok and not self.WORKS_WITH_STRING:
Log.E("Type '{}' cannot be used with operator '{}'".format(
tok1.TOKEN_TYPE, self.OP_TEXT))
def _eval_check_type(self, base_type):
"""
Implement type checking for a unary operator
The operator value must match the base token type
"""
tok = self.tokens[0]
if not tok.matches_class(base_type):
Log.E("{} must be '{}'".format(tok.get_value(),
base_type.TOKEN_TYPE))
def generate(self, definitions, out_dir):
"""Generate a settings file"""
args = dict(DEFINITIONS=definitions,
OUT_DIR=out_dir,
OUT_FILE=self.filename,
PARAM_TYPE=DefTree.PARAM_TYPE,
g_print=lambda msg: Log.I(" {}".format(str(msg))),
g_debug=lambda msg: Log.D(" {}".format(str(msg))),
g_err=lambda msg: Log.E(msg, stackframe=3))
runpy.run_path(self.formatter,
init_globals=args,
run_name="<globifest>")
def error(self, text):
"""
Logs a non-fatal error and sets the parse status
"""
err_text = text
if self.is_flag_set(FLAGS.DEBUG):
frame = Util.get_stackframe(2)
if frame:
err_text = err_text + "\n @{}:{}".format(
frame.f_code.co_filename, frame.f_lineno)
Log.E(err_text, is_fatal=False, stackframe=3)
self.status = PARSE_STATUS.ERROR
def __init__(self,
pid,
ptitle,
ptype,
pdesc="",
pdefault=None,
metadata=None):
self.pid = pid
self.ptitle = ptitle
self.ptype = ptype
self.pdesc = pdesc
self.pdefault = pdefault
self.metadata = metadata
if (self.ptype == PARAM_TYPE.ENUM) and (not self.metadata):
Log.E("ENUM must have metadata")
def extract_zipfile(zip_filename, out_dir, prefix=None, do_test=True):
"""
Extract an archive using zipfile
If prefix is provided then only files with the prefix are used, and
the prefix is removed from the output path
"""
with zipfile.ZipFile(zip_filename, mode="r") as zip_fh:
if do_test:
bad_file = zip_fh.testzip()
if bad_file:
Log.E("Corrupt archive at: {}".format(bad_file))
else:
Log.D(" Archive tested OK")
# Check if doing path replacement, if so, normalize paths
if prefix:
Log.D(" Using prefix '{}'".format(prefix))
Log.D(" Extracting to: {}".format(out_dir))
for file in zip_fh.infolist():
if prefix:
if not file.filename.startswith(prefix):
continue
orig_fn = file.filename
start_idx = len(prefix)
try:
while file.filename[start_idx] in ["/", "\\"]:
# This is a directory; fix up for the user so it does not
# extract to the root of the host's filesystem.
start_idx += 1
except IndexError:
# Just the prefix, possibly with trailing slashes; just skip.
continue
file.filename = file.filename[start_idx:]
Log.D(" {} => {}".format(orig_fn, file.filename))
else:
Log.D(" {}".format(file.filename))
zip_fh.extract(file, path=out_dir)
def undefine(self, name):
"""Undefine a value"""
if name in self.implicit_configs:
Log.E("Cannot undefine an implicit value")
self.configs.pop(name, None)
def error(self, action, sys_msg):
"""Log an error"""
Log.E("Could not {} {}: {}".format(action, self.err_file_name,
sys_msg))
def log_error(self, text):
"""Log an error performing an action"""
Log.E(" {}: {}".format(self.ACTION_TYPE, text), stackframe=3)
def generate(self, _definitions, _out_dir):
"""Generate a settings file"""
Log.E("Error generating {}: algorithm undefined", self.filename)
def build_project(in_fname, out_dir, settings, callbacks=Util.Container()):
"""
Build a project with the given settings
"""
project, prj_dir, out_dir = read_project(in_fname, out_dir)
Log.I("Project: {}".format(project.get_name()))
Log.I("PrjDir: {}".format(prj_dir))
Log.I("OutDir: {}".format(out_dir))
os.makedirs(out_dir, exist_ok=True)
# Check external project dependencies
for dep_name, dependency in project.get_dependencies():
Log.I("Checking dependency {}...".format(dep_name))
dep_out_dir = os.path.join(out_dir, dep_name)
os.makedirs(dep_out_dir, exist_ok=True)
dependency.setup(dep_out_dir)
# Set up build configuration
Log.I("Build configuration:")
setting_re = re.compile("([^=]+)=(.+)")
cfg_container = Util.Container() # Unordered, for tracking purposes
for cfg_entry in settings:
m = Matcher.new(cfg_entry)
if not m.is_fullmatch(setting_re):
Log.E("Malformed setting: {}".format(cfg_entry))
if cfg_container.get(m[1]):
Log.E("Conflicting/Duplicate setting: {}".format(cfg_entry))
Log.I(" {}: {}".format(m[1], m[2]))
variant = project.get_target(m[1], m[2])
# Update the filename with the absolute path
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[m[1]] = variant
# Validate that all layers are specified
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
if variant is None:
variant_names = project.get_variant_names(layer)
if len(variant_names) == 1:
# None specified, but there is only one
variant = project.get_target(layer, variant_names[0])
Log.D(" **Default selected for layer {}**".format(layer))
Log.I(" {}: {}".format(layer, variant.name))
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[layer] = variant
else:
Log.E("Must specify variant for layer {}".format(layer))
Log.I("Generating settings in layer order:")
effective_settings = Settings.new()
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
Log.I(" {}: {}".format(layer, variant.filename))
layer_config = build_config(variant.filename)
effective_settings.extend(layer_config.get_settings())
# Generate a metadata object to communicate information back to the caller
metadata = Util.Container(prj_dir=prj_dir,
out_dir=out_dir,
settings=effective_settings)
#### PREBUILD CALLBACK ####
if callbacks.get("prebuild"):
callbacks.prebuild(callbacks.get("arg"), metadata)
# Prepare storage for package processing
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] = []
all_manifests = []
Log.I("Processing packages...")
for pkg in project.get_packages():
pkg_file = get_pkg_file(project, pkg, prj_dir, out_dir)
if pkg_file is None:
Log.I("Unknown file root {}".format(str(pkg.file_root)))
pkg_root = get_pkg_root(project, pkg, pkg_file, out_dir)
if pkg_root is None:
Log.I("Unknown package root {}".format(str(pkg.file_root)))
Log.I(" {}".format(pkg_file))
manifest = build_manifest(pkg_file, effective_settings, pkg_root)
all_manifests.append(manifest)
pkg_dir = os.path.dirname(pkg_file)
manifest_out = manifest.get_output()
# Replace all file paths with absolute paths
for k in ManifestParser.FILE_LABELS:
manifest_out[k] = [
Util.get_abs_path(x, pkg_dir) for x in manifest_out[k]
]
# Aggregate all public labels
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] += manifest_out[pub_key]
# Dump all the files on extreme mode
if Log.Logger.has_level(Log.LEVEL.EXTREME):
for k, v in manifest_out:
Log.X(" {}:".format(k))
for f in v:
Log.X(" {}".format(f))
for cfg in manifest.get_configs():
Log.I(" Post-processing {}".format(cfg.definition_abs))
defs = cfg.def_tree.get_relevant_params(effective_settings)
for gen in cfg.generators:
gen_file = Util.get_abs_path(gen.get_filename(), pkg_dir)
gen_file = os.path.relpath(gen_file, start=pkg_dir)
gen_file = os.path.normpath(gen_file)
gen_file = os.path.join(out_dir, gen_file)
# Update the filename in the generator
gen.filename = gen_file
Log.I(" Generating {}".format(gen_file))
if gen.get_formatter():
formatter_filename = Util.get_abs_path(
gen.get_formatter(), pkg_dir)
Log.I(" Executing {}".format(formatter_filename))
#### GENERATOR CALLBACK ####
if callbacks.get("generator"):
# Let the build script intercept the generator without any filesystem changes
callbacks.generator(callbacks.get("arg", None), metadata,
defs, gen)
else:
os.makedirs(os.path.dirname(gen_file), exist_ok=True)
gen.generate(defs, out_dir)
#### POSTPROCESS CALLBACK ####
if callbacks.get("postprocess"):
callbacks.postprocess(callbacks.get("arg"), metadata)
# Add public data into each manifest and build
if callbacks.get("target"):
for manifest in all_manifests:
tables = Util.Container()
manifest_out = manifest.get_output()
for k, v in manifest_out:
if not k in ManifestParser.PUBLIC_LABELS:
tables[k] = v
manifest_name = os.path.relpath(manifest.get_filename(),
start=prj_dir)
manifest_name = os.path.normpath(manifest_name)
manifest_name = os.path.splitext(manifest_name)[0]
#### BUILD TARGET CALLBACK ####
callbacks.target(callbacks.get("arg", None), metadata,
manifest_name, tables)
#### POSTBUILD CALLBACK ####
if callbacks.get("postbuild"):
callbacks.postbuild(callbacks.get("arg", None), metadata)
def __init__(self, text):
try:
self.value = int(text)
except ValueError:
Log.E("{} is not a number".format(text))
def get_value(self, name):
"""Stub; should never be called"""
Log.E("Internal error: cannot return value",
err_type=Log.ERROR.RUNTIME)
def generate(self, definitions, out_dir):
"""Generate a settings file"""
include_guard = os.path.relpath(self.filename, start=out_dir)
include_guard = "_{}".format(
CGenerator.INCLUDE_GUARD_REPLACE.sub("_", include_guard))
include_guard = include_guard.upper()
with LineReader.OpenFileCM(self.filename, "wt") as hdr_cm:
if not hdr_cm:
Log.E("Could not open {}: {}".format(self.filename,
hdr_cm.get_err_msg()))
hdr_file = hdr_cm.get_file()
# File header
hdr_file.write("\n".join([
"/* GENERATED BY GLOBIFEST -- DO NOT EDIT */", "",
"#ifndef {}".format(include_guard),
"#define {}".format(include_guard), "\n"
]))
# Add values; preprocessor directives are used for maximum type flexibility
for d in definitions:
ptype = d.param.get_type()
pid = d.param.get_identifier()
value = d.value
# Write implicit values first
implicit_id = None
for implicit_value in d.param.get_implicit_values():
hdr_file.write("#define {} ({})\n".format(
implicit_value[0], implicit_value[1]))
if value == implicit_value[1]:
implicit_id = implicit_value[0]
# Write the parameter
if ptype in [DefTree.PARAM_TYPE.INT, DefTree.PARAM_TYPE.FLOAT]:
# Parentheses prevent conflicts with surrounding code
# Default type of INT is int (i.e., signed literal)
# Default type of FLOAT is double precision
hdr_file.write("#define {} ({})\n".format(pid, value))
elif ptype == DefTree.PARAM_TYPE.STRING:
# Strings are not surrounded to allow compile-time concatenation
hdr_file.write("#define {} {}\n".format(pid, value))
elif ptype == DefTree.PARAM_TYPE.BOOL:
# Define as 1/0, since C89 did not define TRUE and FALSE values
if value == "FALSE":
value = 0
else:
value = 1
hdr_file.write("#define {} ({})\n".format(pid, value))
elif ptype == DefTree.PARAM_TYPE.ENUM:
if implicit_id:
hdr_file.write("#define {} {}\n".format(
pid, implicit_id))
else:
hdr_file.write("#define {} ({})\n".format(pid, value))
else:
# TODO: Handle more complex literal types:
# - Integral types U/L/UL/LL/ULL
# - Float suffixes F/L for floats
Log.E("Unhandled value of type {}".format(
str(d.param.ptype)))
# File footer
hdr_file.write("\n#endif /* {} */\n".format(include_guard))
def set_value(self, name, value):
"""Set/overwrite a value"""
if name in self.implicit_configs:
Log.E("Cannot set an implicit value")
self.configs[name] = value
def evaluate(self, expr):
"""Evaluate the logical expression"""
self.expr = expr
BOUNDED_PARSER_FLAGS = 0
if self.get_debug_mode():
BOUNDED_PARSER_FLAGS |= StatefulParser.FLAGS.DEBUG
op = None
tok = None
while True:
# Fill up the operation
if op:
if tok:
op.add_token(tok)
tok = None
err_text = op.get_error()
if err_text:
Log.E(err_text)
if op.is_full():
# Evaluate and use the result as the next token
tok = BoolToken(op.evaluate())
op = None
continue
if self.expr == "":
if op:
Log.E("Operator '{}' missing argument".format(op.OP_TEXT))
break
# Identify the next token
m = Matcher.new(self.expr)
if m.is_fullmatch(self.whitespace_re):
self.expr = m[1]
continue
if m.is_fullmatch(self.int_re):
self.debug("INT: " + m[1])
if tok:
Log.E("Unexpected integer '{}'".format(m[1]))
tok = IntToken(m[1])
self.expr = m[2]
continue
if m.is_fullmatch(self.ident_re):
if tok:
Log.E("Unexpected identifier '{}'".format(m[1]))
ident = m[1]
if ident in RESERVED_IDENT_MAP:
mapped_ident = RESERVED_IDENT_MAP[ident]
mapped_class = mapped_ident[RESERVED_IDENT.CLASS]
mapped_value = mapped_ident[RESERVED_IDENT.VALUE]
self.debug("RESERVED: {} class={} value={}".format(
ident, mapped_class.TOKEN_TYPE, mapped_value))
tok = mapped_class(mapped_value)
else:
self.debug("IDENT: " + ident)
tok = IdentToken(m[1], self)
self.expr = m[2]
continue
if m.is_fullmatch(self.op_re):
self.debug("OP: " + m[1])
if op is not None:
Log.E("Spurious operator '{}' after operator '{}'".format(
m[1], op.OP_TEXT))
if m[1] == "!":
if tok:
Log.E("Unexpected operator '{}'".format(m[1]))
op = OpInverse(self)
elif not tok:
Log.E("Operator '{}' missing value".format(m[1]))
elif (m[1] == "==") or (m[1] == "="):
op = OpCompareEq(self)
elif m[1] == "!=":
op = OpCompareNe(self)
elif m[1] == "<":
op = OpCompareLt(self)
elif m[1] == "<=":
op = OpCompareLe(self)
elif m[1] == ">":
op = OpCompareGt(self)
elif m[1] == ">=":
op = OpCompareGe(self)
elif m[1] == "&&":
op = OpLogicalAnd(self)
elif m[1] == "||":
op = OpLogicalOr(self)
else:
Log.E("Unknown operator {}".format(m[1]))
self.expr = m[2]
continue
if self.expr[0] == "(":
string_parser = BoundedStatefulParser.new(
self.expr, "(", ")",
BOUNDED_PARSER_FLAGS | StatefulParser.FLAGS.MULTI_LEVEL)
self.debug("PAREN: " + string_parser.get_parsed_text())
if StatefulParser.PARSE_STATUS.FINISHED != string_parser.get_status(
):
self.debug(string_parser.get_debug_log())
Log.E("Malformed parenthetical in expression: " +
self.expr)
tok = BoolToken(self.evaluate(string_parser.get_parsed_text()))
self.expr = string_parser.get_remaining_text()
continue
if self.expr[0] == "\"":
string_parser = BoundedStatefulParser.new(
self.expr, "\"", flags=BOUNDED_PARSER_FLAGS)
if StatefulParser.PARSE_STATUS.FINISHED != string_parser.get_status(
):
Log.E("Malformed string in expression: " + self.expr)
self.debug("STRING DQ: " + string_parser.get_parsed_text())
tok = StringToken(string_parser.get_parsed_text())
self.expr = string_parser.get_remaining_text()
continue
if self.expr[0] == "'":
string_parser = BoundedStatefulParser.new(
self.expr, "'", flags=BOUNDED_PARSER_FLAGS)
if StatefulParser.PARSE_STATUS.FINISHED != string_parser.get_status(
):
Log.E("Malformed string in expression: " + self.expr)
self.debug("STRING SQ: " + string_parser.get_parsed_text())
tok = StringToken(string_parser.get_parsed_text())
self.expr = string_parser.get_remaining_text()
continue
Log.E("Bad expression: " + self.expr)
if tok is None:
Log.E("Cannot evaluate expression")
# Convert lone identifier tokens to their value type for evaluation
if isinstance(tok, IdentToken):
self.debug("convert->{}".format(tok.ident_class.TOKEN_TYPE))
tok = tok.ident_class(tok.value)
if isinstance(tok, BoolToken):
return tok.value
elif isinstance(tok, IntToken):
return tok.value != 0
else:
Log.E("Expression of type '{}' is not convertible to bool".format(
tok.TOKEN_TYPE))
def add_token(self, token):
"""Add a token to be evaluated by this operation"""
if not isinstance(token, TokenBase):
Log.E("{} is not a token".format(str(token)))
self.tokens.append(token)
def _eval_get_expr(self):
Log.E("Internal error: No default expression for unary operator")