def generate(self, definitions, out_dir):
"""Generate a settings file"""
java_file = self.filename.lower()
package_dir = os.path.dirname(java_file)
package_name = os.path.relpath(package_dir, start=out_dir)
package_name = self.PACKAGE_RE.sub(".", package_name)
class_name = self.CLASS_RE.sub("", os.path.basename(self.filename))
os.makedirs(Util.get_abs_path(package_dir, out_dir), exist_ok=True)
with LineReader.OpenFileCM(java_file, "wt") as hdr_cm:
if not hdr_cm:
Log.E("Could not open {}: {}".format(self.filename,
hdr_cm.get_err_msg()))
hdr_file = hdr_cm.get_file()
# File header
hdr_file.write("\n".join([
"/* GENERATED BY GLOBIFEST -- DO NOT EDIT */", "",
"package {}".format(package_name), "",
"public final class {}".format(class_name), "{\n"
]))
# Add values
template = " public final static {} {} = {};\n"
for d in definitions:
ptype = d.param.get_type()
pid = d.param.get_identifier()
value = d.value
# Write implicit values first
implicit_id = None
for implicit_value in d.param.get_implicit_values():
hdr_file.write(
template.format("int", implicit_value[0],
implicit_value[1]))
if value == implicit_value[1]:
implicit_id = implicit_value[0]
# Write the parameter
if ptype == DefTree.PARAM_TYPE.INT:
hdr_file.write(template.format("int", pid, value))
elif ptype == DefTree.PARAM_TYPE.FLOAT:
# Default type is double precision
hdr_file.write(template.format("double", pid, value))
elif ptype == DefTree.PARAM_TYPE.STRING:
hdr_file.write(template.format("String", pid, value))
elif ptype == DefTree.PARAM_TYPE.BOOL:
hdr_file.write(
template.format("boolean", pid, value.lower()))
elif ptype == DefTree.PARAM_TYPE.ENUM:
if implicit_id:
value = implicit_id
hdr_file.write(template.format("int", pid, value))
else:
# TODO: Handle more complex literal types
Log.E("Unhandled value of type {}".format(
str(d.param.ptype)))
# File footer
hdr_file.write("}\n")
def read_project(in_fname, out_dir):
"""
Read project
@return Tuple containing (Project object with parsed result,
Project directory, output directory)
"""
project = Project.new(in_fname, err_fatal=True)
parser = ProjectParser.new(project)
reader = LineReader.new(parser)
reader.read_file_by_name(in_fname)
cwd = os.getcwd()
prj_dir = Util.get_abs_path(os.path.dirname(project.get_filename()), cwd)
out_dir = Util.get_abs_path(out_dir, cwd)
return (project, prj_dir, out_dir)
def get_pkg_file(project, pkg, prj_dir, out_dir):
"""
Get package file
@param project A Project object
@param pkg An element from project.get_packages()
@param prj_dir The top-level project directory
@param out_dir The top-level output directory
@return the absolute path to the package file, or None if indeterminate
"""
if pkg.file_root == project.ROOT.SOURCE:
# File is relative to the project directory
pkg_file = Util.get_abs_path(pkg.filename, prj_dir)
elif pkg.file_root == project.ROOT.DEPENDENCY:
# File is relative to the dependency's output directory
# (external manifest)
pkg_file = Util.get_abs_path(pkg.filename,
os.path.join(out_dir, pkg.module_id))
else:
pkg_file = None
return pkg_file
def _include_file(self, filename):
"""
Include the contents of another file as if it was directly placed in this file
"""
abs_filename = Util.get_abs_path(filename, self.prj_root)
# Save the project root so that files paths can be relative to the included file
old_prj_root = self.prj_root
self.prj_root = os.path.dirname(abs_filename)
# Read the file using this object as the parser
reader = LineReader.new(self, do_end=False)
reader.read_file_by_name(abs_filename)
# Restore the original project root
self.prj_root = old_prj_root
def _config_end(self, context):
"""End a config block"""
abs_def_file = Util.get_abs_path(context.ctx.definition, self.pkg_root)
if self.validate_files or self.def_parser:
if not os.path.isfile(abs_def_file):
self.log_error("'{}' is not a file".format(abs_def_file))
cfg = Util.Container(definition=context.ctx.definition,
generators=context.ctx.generators)
if self.def_parser:
cfg["definition_abs"] = abs_def_file
cfg["def_tree"] = self.def_parser(abs_def_file)
self.settings.add_implicit_configs(
cfg.def_tree.get_implicit_values())
self.manifest.add_config(cfg)
def run(self, ext_dep, inputs):
"""Run the action"""
if inputs == []:
dest = self.get_action_file(ext_dep, "dep")
else:
self.check_input(inputs, 1)
dest = Util.get_abs_path(inputs[0], ext_dep.out_dir)
if os.path.isfile(dest):
# Downloaded file is probably corrupt, delete it
self.log_info("Deleting old {}".format(dest))
os.unlink(dest)
self.log_info("Downloading file from {} as {}".format(self.arg, dest))
with urllib.request.urlopen(self.arg) as response, open(
dest, 'wb') as out_file:
shutil.copyfileobj(response, out_file)
return [dest]
def process_param_config(self, name, value):
"""Process a config parameter"""
if not value:
self.manifest_parser.log_error("Bad parameter: {}".format(value))
if name == "generate":
token = value.split(" ", 1)
if len(token) != 2:
self.manifest_parser.log_error(
"Incorrect values for parameter: generate")
gen_format = token[0].lower()
if gen_format in PROHIBITED_GENERATORS:
self.manifest_parser.log_error(
"Format {} is prohibited".format(token[0]))
generator = Generators.factory(gen_format=gen_format,
filename=token[1])
if generator is None:
self.manifest_parser.log_error("Invalid format '{}'".format(
token[0]))
self.ctx.generators.append(generator)
elif name == "generate_s":
token = value.split(" ", 1)
if len(token) != 2:
self.manifest_parser.log_error(
"Incorrect values for parameter: generate_s")
generator = Generators.factory(gen_format="_custom",
filename=token[1],
formatter=Util.get_abs_path(
token[0],
self.manifest_parser.pkg_root))
if generator is None:
self.manifest_parser.log_error("Invalid format '{}'".format(
token[0]))
self.ctx.generators.append(generator)
elif self.is_unique_element(CONFIG_ELEMENTS, name):
# No additional validation required for these elements
self.ctx[CONFIG_ELEMENTS[name]] = value
else:
self.manifest_parser.debug("not found: {}".format(name))
def _include_file(self, filename):
"""
Include the contents of another file as if it was directly placed in this file
"""
abs_filename = Util.get_abs_path(filename, self.pkg_root)
if not self.validate_files:
# For testing, just add the include file as a source
self.debug("ADD_AUX: {}".format(abs_filename))
self.manifest.add_entry("aux_files", abs_filename)
return
# Save the package root so that files paths can be relative to the included file
old_pkg_root = self.pkg_root
self.pkg_root = os.path.dirname(abs_filename)
# Read the file using this object as the parser
reader = LineReader.new(self, do_end=False)
reader.read_file_by_name(abs_filename)
# Restore the original package root
self.pkg_root = old_pkg_root
def _parse_entry(self, entry):
"""
Parse entry
"""
cur_context = self.context_stack[-1]
if not cur_context.label:
self.log_error("Missing label for entry {}".format(entry))
if self.validate_files:
# When validating files (i.e., a real build), change to absolute path
entry = Util.get_abs_path(entry, self.pkg_root)
if cur_context.label in FILE_LABELS:
path = None
for f in glob.iglob(entry):
path = pathlib.Path(f)
if not path.is_file():
self.log_error("'{}' is not a file".format(f))
if path is None:
self.log_error(
"'{}' does not match any files".format(entry))
elif cur_context.label in PATH_LABELS:
path = pathlib.Path(entry)
if not path.is_dir():
self.log_error("'{}' is not a directory".format(entry))
# If this is parsed in a condition context, skip over unmatching entries
if not cur_context.is_condition_met():
self.debug("SKIP_ENTRY: {}".format(entry))
return
if (cur_context.label in FILE_LABELS) and (self.validate_files):
for f in glob.iglob(entry):
self.debug("ADD_FILE: {}".format(f))
self.manifest.add_entry(cur_context.label, f)
else:
self.debug("ADD_ENTRY: {}".format(entry))
self.manifest.add_entry(cur_context.label, entry)
def get_action_file(self, ext_dep, extension=None):
"""Create and return a single output filename for this action"""
action_dir = Util.get_abs_path(
"{}.{}".format(self.ACTION_TYPE, extension), ext_dep.out_dir)
return action_dir
def get_action_folder(self, ext_dep):
"""Return an output directory for this action"""
action_dir = Util.get_abs_path(self.ACTION_TYPE, ext_dep.out_dir)
return action_dir
def _open_project(self):
try:
project, prj_dir, out_dir = Builder.read_project(
self.project_file, self.out_dir)
except Log.GlobifestException as e:
tkinter.messagebox.showerror(self.APP_TITLE, str(e))
return
def_trees = list()
self._opendir = os.path.dirname(self.project_file)
for pkg in project.get_packages():
pkg_file = Builder.get_pkg_file(project, pkg, prj_dir, out_dir)
if not pkg_file:
tkinter.messagebox.showerror(
self.APP_TITLE,
"Unknown file root {}".format(str(pkg.file_root)))
return
pkg_root = Builder.get_pkg_root(project, pkg, pkg_file, out_dir)
if pkg_root is None:
tkinter.messagebox.showerror(
self.APP_TITLE,
"Unknown package root {}".format(str(pkg.file_root)))
return
try:
manifest = Builder.build_manifest(pkg_file,
ManifestParser.ConfigsOnly(),
pkg_root,
validate_files=False)
except Log.GlobifestException as e:
tkinter.messagebox.showerror(self.APP_TITLE, str(e))
return
pkg_dir = os.path.dirname(pkg_file)
for cfg in manifest.get_configs():
cfg.definition = Util.get_abs_path(cfg.definition, pkg_dir)
def_tree = Builder.build_definition(cfg.definition)
def_trees.append(def_tree)
# Aggregate DefTrees into a single list
forest = DefTree.DefForest()
for tree in def_trees:
tree.walk(forest)
self.project = project
# Clear GUI elements
self._clear_gui()
# Set the window title
self.app_root.title("{} - {}".format(self.APP_TITLE,
self.project.get_name()))
# Walk the forest to populate the config tree
forest.walk(CfgTreeObserver(self.cfg_tree, self.param_tbl),
child_sorter=gui_child_sorter,
param_sorter=gui_param_sorter)
# Set up layer/variant boxes
self.settings_view_tbl.clear()
self.settings_cache.clear()
layer_names = project.get_layer_names()
if layer_names:
for layer in layer_names:
variant_names = project.get_variant_names(layer)
self.settings_view_tbl[layer] = (variant_names[0]
if variant_names else "")
# Build the settings cache
layer_cache = Util.Container()
self.settings_cache[layer] = layer_cache
for variant in variant_names:
variant_cache = Util.Container()
layer_cache[variant] = variant_cache
variant_target = self.project.get_target(layer, variant)
variant_target.filename = Util.get_abs_path(
variant_target.filename, prj_dir)
variant_cache["target"] = variant_target
variant_cache["config"] = Builder.build_config(
variant_target.filename)
value_list = []
for layer in layer_names:
value_list.append("{}={}".format(
layer, self.settings_view_tbl[layer]))
self.settings_layer_cmb.configure(values=value_list)
self.cur_layer.set(layer_names[0])
# Variant will propagate via layer change
self.file_menu.entryconfig("Close", state="normal")
if not self.settings_cache:
tkinter.messagebox.showinfo(
self.APP_TITLE,
"{} defines no settings!".format(self.project_file))
self._close_project()
def build_project(in_fname, out_dir, settings, callbacks=Util.Container()):
"""
Build a project with the given settings
"""
project, prj_dir, out_dir = read_project(in_fname, out_dir)
Log.I("Project: {}".format(project.get_name()))
Log.I("PrjDir: {}".format(prj_dir))
Log.I("OutDir: {}".format(out_dir))
os.makedirs(out_dir, exist_ok=True)
# Check external project dependencies
for dep_name, dependency in project.get_dependencies():
Log.I("Checking dependency {}...".format(dep_name))
dep_out_dir = os.path.join(out_dir, dep_name)
os.makedirs(dep_out_dir, exist_ok=True)
dependency.setup(dep_out_dir)
# Set up build configuration
Log.I("Build configuration:")
setting_re = re.compile("([^=]+)=(.+)")
cfg_container = Util.Container() # Unordered, for tracking purposes
for cfg_entry in settings:
m = Matcher.new(cfg_entry)
if not m.is_fullmatch(setting_re):
Log.E("Malformed setting: {}".format(cfg_entry))
if cfg_container.get(m[1]):
Log.E("Conflicting/Duplicate setting: {}".format(cfg_entry))
Log.I(" {}: {}".format(m[1], m[2]))
variant = project.get_target(m[1], m[2])
# Update the filename with the absolute path
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[m[1]] = variant
# Validate that all layers are specified
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
if variant is None:
variant_names = project.get_variant_names(layer)
if len(variant_names) == 1:
# None specified, but there is only one
variant = project.get_target(layer, variant_names[0])
Log.D(" **Default selected for layer {}**".format(layer))
Log.I(" {}: {}".format(layer, variant.name))
variant.filename = Util.get_abs_path(variant.filename, prj_dir)
cfg_container[layer] = variant
else:
Log.E("Must specify variant for layer {}".format(layer))
Log.I("Generating settings in layer order:")
effective_settings = Settings.new()
for layer in project.get_layer_names():
variant = cfg_container.get(layer)
Log.I(" {}: {}".format(layer, variant.filename))
layer_config = build_config(variant.filename)
effective_settings.extend(layer_config.get_settings())
# Generate a metadata object to communicate information back to the caller
metadata = Util.Container(prj_dir=prj_dir,
out_dir=out_dir,
settings=effective_settings)
#### PREBUILD CALLBACK ####
if callbacks.get("prebuild"):
callbacks.prebuild(callbacks.get("arg"), metadata)
# Prepare storage for package processing
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] = []
all_manifests = []
Log.I("Processing packages...")
for pkg in project.get_packages():
pkg_file = get_pkg_file(project, pkg, prj_dir, out_dir)
if pkg_file is None:
Log.I("Unknown file root {}".format(str(pkg.file_root)))
pkg_root = get_pkg_root(project, pkg, pkg_file, out_dir)
if pkg_root is None:
Log.I("Unknown package root {}".format(str(pkg.file_root)))
Log.I(" {}".format(pkg_file))
manifest = build_manifest(pkg_file, effective_settings, pkg_root)
all_manifests.append(manifest)
pkg_dir = os.path.dirname(pkg_file)
manifest_out = manifest.get_output()
# Replace all file paths with absolute paths
for k in ManifestParser.FILE_LABELS:
manifest_out[k] = [
Util.get_abs_path(x, pkg_dir) for x in manifest_out[k]
]
# Aggregate all public labels
for pub_key in ManifestParser.PUBLIC_LABELS:
metadata[pub_key] += manifest_out[pub_key]
# Dump all the files on extreme mode
if Log.Logger.has_level(Log.LEVEL.EXTREME):
for k, v in manifest_out:
Log.X(" {}:".format(k))
for f in v:
Log.X(" {}".format(f))
for cfg in manifest.get_configs():
Log.I(" Post-processing {}".format(cfg.definition_abs))
defs = cfg.def_tree.get_relevant_params(effective_settings)
for gen in cfg.generators:
gen_file = Util.get_abs_path(gen.get_filename(), pkg_dir)
gen_file = os.path.relpath(gen_file, start=pkg_dir)
gen_file = os.path.normpath(gen_file)
gen_file = os.path.join(out_dir, gen_file)
# Update the filename in the generator
gen.filename = gen_file
Log.I(" Generating {}".format(gen_file))
if gen.get_formatter():
formatter_filename = Util.get_abs_path(
gen.get_formatter(), pkg_dir)
Log.I(" Executing {}".format(formatter_filename))
#### GENERATOR CALLBACK ####
if callbacks.get("generator"):
# Let the build script intercept the generator without any filesystem changes
callbacks.generator(callbacks.get("arg", None), metadata,
defs, gen)
else:
os.makedirs(os.path.dirname(gen_file), exist_ok=True)
gen.generate(defs, out_dir)
#### POSTPROCESS CALLBACK ####
if callbacks.get("postprocess"):
callbacks.postprocess(callbacks.get("arg"), metadata)
# Add public data into each manifest and build
if callbacks.get("target"):
for manifest in all_manifests:
tables = Util.Container()
manifest_out = manifest.get_output()
for k, v in manifest_out:
if not k in ManifestParser.PUBLIC_LABELS:
tables[k] = v
manifest_name = os.path.relpath(manifest.get_filename(),
start=prj_dir)
manifest_name = os.path.normpath(manifest_name)
manifest_name = os.path.splitext(manifest_name)[0]
#### BUILD TARGET CALLBACK ####
callbacks.target(callbacks.get("arg", None), metadata,
manifest_name, tables)
#### POSTBUILD CALLBACK ####
if callbacks.get("postbuild"):
callbacks.postbuild(callbacks.get("arg", None), metadata)
