def setup(self, out_dir):
"""Run setup actions for the external dependency"""
self.out_dir = out_dir
# Build the contents of the cache file. This must include enough information to verify
# the reproducibility of the actions, so it contains all input parameters to the
# dependency.
cache_file_contents = [
"***Delete this file to re-run dependency setup***",
"name={}".format(self.name), "out_dir={}".format(self.out_dir)
]
for a in self.actions:
cache_file_contents.append("action {} {}".format(
a.ACTION_TYPE, a.arg))
if _SetupBeginAction(cache_file_contents).run(self):
Log.I("{} up to date".format(self.name))
return
else:
Log.I("Setting up {}".format(self.name))
inputs = []
for a in self.actions:
inputs = a.run(self, inputs)
Log.D(" => {}".format(inputs))
_SetupCompleteAction(cache_file_contents).run(self)
def generate(self, definitions, out_dir):
"""Generate a settings file"""
args = dict(DEFINITIONS=definitions,
OUT_DIR=out_dir,
OUT_FILE=self.filename,
PARAM_TYPE=DefTree.PARAM_TYPE,
g_print=lambda msg: Log.I(" {}".format(str(msg))),
g_debug=lambda msg: Log.D(" {}".format(str(msg))),
g_err=lambda msg: Log.E(msg, stackframe=3))
runpy.run_path(self.formatter,
init_globals=args,
run_name="<globifest>")
def extract_zipfile(zip_filename, out_dir, prefix=None, do_test=True):
"""
Extract an archive using zipfile
If prefix is provided then only files with the prefix are used, and
the prefix is removed from the output path
"""
with zipfile.ZipFile(zip_filename, mode="r") as zip_fh:
if do_test:
bad_file = zip_fh.testzip()
if bad_file:
Log.E("Corrupt archive at: {}".format(bad_file))
else:
Log.D(" Archive tested OK")
# Check if doing path replacement, if so, normalize paths
if prefix:
Log.D(" Using prefix '{}'".format(prefix))
Log.D(" Extracting to: {}".format(out_dir))
for file in zip_fh.infolist():
if prefix:
if not file.filename.startswith(prefix):
continue
orig_fn = file.filename
start_idx = len(prefix)
try:
while file.filename[start_idx] in ["/", "\\"]:
# This is a directory; fix up for the user so it does not
# extract to the root of the host's filesystem.
start_idx += 1
except IndexError:
# Just the prefix, possibly with trailing slashes; just skip.
continue
file.filename = file.filename[start_idx:]
Log.D(" {} => {}".format(orig_fn, file.filename))
else:
Log.D(" {}".format(file.filename))
zip_fh.extract(file, path=out_dir)
def log_debug(self, text):
"""Log a debug message pertinent to the action"""
Log.D(" {}: {}".format(self.ACTION_TYPE, text))
def build_project(in_fname, out_dir, settings, callbacks=Util.Container()):
"""
Build a project with the given settings
"""
project, prj_dir, out_dir = read_project(in_fname, out_dir)
Log.I("Project: {}".format(project.get_name()))
Log.I("PrjDir: {}".format(prj_dir))
Log.I("OutDir: {}".format(out_dir))
os.makedirs(out_dir, exist_ok=True)
# Check external project dependencies
for dep_name, dependency in project.get_dependencies():
Log.I("Checking dependency {}...".format(dep_name))
dep_out_dir = os.path.join(out_dir, dep_name)
os.makedirs(dep_out_dir, exist_ok=True)
dependency.setup(dep_out_dir)
# Set up build configuration
Log.I("Build configuration:")
setting_re = re.compile("([^=]+)=(.+)")
cfg_container = Util.Container() # Unordered, for tracking purposes
for cfg_entry in settings:
m = Matcher.new(cfg_entry)
if not m.is_fullmatch(setting_re):
Log.E("Malformed setting: {}".format(cfg_entry))
if cfg_container.get(m[1]):
Log.E("Conflicting/Duplicate setting: {}".format(cfg_entry))
Log.I(" {}: {}".format(m[1], m[2]))
variant = project.get_target(m[1], m[2])
# Update the filename with the absolute path
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[m[1]] = variant
# Validate that all layers are specified
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
if variant is None:
variant_names = project.get_variant_names(layer)
if len(variant_names) == 1:
# None specified, but there is only one
variant = project.get_target(layer, variant_names[0])
Log.D(" **Default selected for layer {}**".format(layer))
Log.I(" {}: {}".format(layer, variant.name))
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[layer] = variant
else:
Log.E("Must specify variant for layer {}".format(layer))
Log.I("Generating settings in layer order:")
effective_settings = Settings.new()
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
Log.I(" {}: {}".format(layer, variant.filename))
layer_config = build_config(variant.filename)
effective_settings.extend(layer_config.get_settings())
# Generate a metadata object to communicate information back to the caller
metadata = Util.Container(prj_dir=prj_dir,
out_dir=out_dir,
settings=effective_settings)
#### PREBUILD CALLBACK ####
if callbacks.get("prebuild"):
callbacks.prebuild(callbacks.get("arg"), metadata)
# Prepare storage for package processing
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] = []
all_manifests = []
Log.I("Processing packages...")
for pkg in project.get_packages():
pkg_file = get_pkg_file(project, pkg, prj_dir, out_dir)
if pkg_file is None:
Log.I("Unknown file root {}".format(str(pkg.file_root)))
pkg_root = get_pkg_root(project, pkg, pkg_file, out_dir)
if pkg_root is None:
Log.I("Unknown package root {}".format(str(pkg.file_root)))
Log.I(" {}".format(pkg_file))
manifest = build_manifest(pkg_file, effective_settings, pkg_root)
all_manifests.append(manifest)
pkg_dir = os.path.dirname(pkg_file)
manifest_out = manifest.get_output()
# Replace all file paths with absolute paths
for k in ManifestParser.FILE_LABELS:
manifest_out[k] = [
Util.get_abs_path(x, pkg_dir) for x in manifest_out[k]
]
# Aggregate all public labels
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] += manifest_out[pub_key]
# Dump all the files on extreme mode
if Log.Logger.has_level(Log.LEVEL.EXTREME):
for k, v in manifest_out:
Log.X(" {}:".format(k))
for f in v:
Log.X(" {}".format(f))
for cfg in manifest.get_configs():
Log.I(" Post-processing {}".format(cfg.definition_abs))
defs = cfg.def_tree.get_relevant_params(effective_settings)
for gen in cfg.generators:
gen_file = Util.get_abs_path(gen.get_filename(), pkg_dir)
gen_file = os.path.relpath(gen_file, start=pkg_dir)
gen_file = os.path.normpath(gen_file)
gen_file = os.path.join(out_dir, gen_file)
# Update the filename in the generator
gen.filename = gen_file
Log.I(" Generating {}".format(gen_file))
if gen.get_formatter():
formatter_filename = Util.get_abs_path(
gen.get_formatter(), pkg_dir)
Log.I(" Executing {}".format(formatter_filename))
#### GENERATOR CALLBACK ####
if callbacks.get("generator"):
# Let the build script intercept the generator without any filesystem changes
callbacks.generator(callbacks.get("arg", None), metadata,
defs, gen)
else:
os.makedirs(os.path.dirname(gen_file), exist_ok=True)
gen.generate(defs, out_dir)
#### POSTPROCESS CALLBACK ####
if callbacks.get("postprocess"):
callbacks.postprocess(callbacks.get("arg"), metadata)
# Add public data into each manifest and build
if callbacks.get("target"):
for manifest in all_manifests:
tables = Util.Container()
manifest_out = manifest.get_output()
for k, v in manifest_out:
if not k in ManifestParser.PUBLIC_LABELS:
tables[k] = v
manifest_name = os.path.relpath(manifest.get_filename(),
start=prj_dir)
manifest_name = os.path.normpath(manifest_name)
manifest_name = os.path.splitext(manifest_name)[0]
#### BUILD TARGET CALLBACK ####
callbacks.target(callbacks.get("arg", None), metadata,
manifest_name, tables)
#### POSTBUILD CALLBACK ####
if callbacks.get("postbuild"):
callbacks.postbuild(callbacks.get("arg", None), metadata)