def generate(self, definitions, out_dir): """Generate a settings file""" java_file = self.filename.lower() package_dir = os.path.dirname(java_file) package_name = os.path.relpath(package_dir, start=out_dir) package_name = self.PACKAGE_RE.sub(".", package_name) class_name = self.CLASS_RE.sub("", os.path.basename(self.filename)) os.makedirs(Util.get_abs_path(package_dir, out_dir), exist_ok=True) with LineReader.OpenFileCM(java_file, "wt") as hdr_cm: if not hdr_cm: Log.E("Could not open {}: {}".format(self.filename, hdr_cm.get_err_msg())) hdr_file = hdr_cm.get_file() # File header hdr_file.write("\n".join([ "/* GENERATED BY GLOBIFEST -- DO NOT EDIT */", "", "package {}".format(package_name), "", "public final class {}".format(class_name), "{\n" ])) # Add values template = " public final static {} {} = {};\n" for d in definitions: ptype = d.param.get_type() pid = d.param.get_identifier() value = d.value # Write implicit values first implicit_id = None for implicit_value in d.param.get_implicit_values(): hdr_file.write( template.format("int", implicit_value[0], implicit_value[1])) if value == implicit_value[1]: implicit_id = implicit_value[0] # Write the parameter if ptype == DefTree.PARAM_TYPE.INT: hdr_file.write(template.format("int", pid, value)) elif ptype == DefTree.PARAM_TYPE.FLOAT: # Default type is double precision hdr_file.write(template.format("double", pid, value)) elif ptype == DefTree.PARAM_TYPE.STRING: hdr_file.write(template.format("String", pid, value)) elif ptype == DefTree.PARAM_TYPE.BOOL: hdr_file.write( template.format("boolean", pid, value.lower())) elif ptype == DefTree.PARAM_TYPE.ENUM: if implicit_id: value = implicit_id hdr_file.write(template.format("int", pid, value)) else: # TODO: Handle more complex literal types Log.E("Unhandled value of type {}".format( str(d.param.ptype))) # File footer hdr_file.write("}\n")
def read_project(in_fname, out_dir): """ Read project @return Tuple containing (Project object with parsed result, Project directory, output directory) """ project = Project.new(in_fname, err_fatal=True) parser = ProjectParser.new(project) reader = LineReader.new(parser) reader.read_file_by_name(in_fname) cwd = os.getcwd() prj_dir = Util.get_abs_path(os.path.dirname(project.get_filename()), cwd) out_dir = Util.get_abs_path(out_dir, cwd) return (project, prj_dir, out_dir)
def get_pkg_file(project, pkg, prj_dir, out_dir): """ Get package file @param project A Project object @param pkg An element from project.get_packages() @param prj_dir The top-level project directory @param out_dir The top-level output directory @return the absolute path to the package file, or None if indeterminate """ if pkg.file_root == project.ROOT.SOURCE: # File is relative to the project directory pkg_file = Util.get_abs_path(pkg.filename, prj_dir) elif pkg.file_root == project.ROOT.DEPENDENCY: # File is relative to the dependency's output directory # (external manifest) pkg_file = Util.get_abs_path(pkg.filename, os.path.join(out_dir, pkg.module_id)) else: pkg_file = None return pkg_file
def _include_file(self, filename): """ Include the contents of another file as if it was directly placed in this file """ abs_filename = Util.get_abs_path(filename, self.prj_root) # Save the project root so that files paths can be relative to the included file old_prj_root = self.prj_root self.prj_root = os.path.dirname(abs_filename) # Read the file using this object as the parser reader = LineReader.new(self, do_end=False) reader.read_file_by_name(abs_filename) # Restore the original project root self.prj_root = old_prj_root
def _config_end(self, context): """End a config block""" abs_def_file = Util.get_abs_path(context.ctx.definition, self.pkg_root) if self.validate_files or self.def_parser: if not os.path.isfile(abs_def_file): self.log_error("'{}' is not a file".format(abs_def_file)) cfg = Util.Container(definition=context.ctx.definition, generators=context.ctx.generators) if self.def_parser: cfg["definition_abs"] = abs_def_file cfg["def_tree"] = self.def_parser(abs_def_file) self.settings.add_implicit_configs( cfg.def_tree.get_implicit_values()) self.manifest.add_config(cfg)
def run(self, ext_dep, inputs): """Run the action""" if inputs == []: dest = self.get_action_file(ext_dep, "dep") else: self.check_input(inputs, 1) dest = Util.get_abs_path(inputs[0], ext_dep.out_dir) if os.path.isfile(dest): # Downloaded file is probably corrupt, delete it self.log_info("Deleting old {}".format(dest)) os.unlink(dest) self.log_info("Downloading file from {} as {}".format(self.arg, dest)) with urllib.request.urlopen(self.arg) as response, open( dest, 'wb') as out_file: shutil.copyfileobj(response, out_file) return [dest]
def process_param_config(self, name, value): """Process a config parameter""" if not value: self.manifest_parser.log_error("Bad parameter: {}".format(value)) if name == "generate": token = value.split(" ", 1) if len(token) != 2: self.manifest_parser.log_error( "Incorrect values for parameter: generate") gen_format = token[0].lower() if gen_format in PROHIBITED_GENERATORS: self.manifest_parser.log_error( "Format {} is prohibited".format(token[0])) generator = Generators.factory(gen_format=gen_format, filename=token[1]) if generator is None: self.manifest_parser.log_error("Invalid format '{}'".format( token[0])) self.ctx.generators.append(generator) elif name == "generate_s": token = value.split(" ", 1) if len(token) != 2: self.manifest_parser.log_error( "Incorrect values for parameter: generate_s") generator = Generators.factory(gen_format="_custom", filename=token[1], formatter=Util.get_abs_path( token[0], self.manifest_parser.pkg_root)) if generator is None: self.manifest_parser.log_error("Invalid format '{}'".format( token[0])) self.ctx.generators.append(generator) elif self.is_unique_element(CONFIG_ELEMENTS, name): # No additional validation required for these elements self.ctx[CONFIG_ELEMENTS[name]] = value else: self.manifest_parser.debug("not found: {}".format(name))
def _include_file(self, filename): """ Include the contents of another file as if it was directly placed in this file """ abs_filename = Util.get_abs_path(filename, self.pkg_root) if not self.validate_files: # For testing, just add the include file as a source self.debug("ADD_AUX: {}".format(abs_filename)) self.manifest.add_entry("aux_files", abs_filename) return # Save the package root so that files paths can be relative to the included file old_pkg_root = self.pkg_root self.pkg_root = os.path.dirname(abs_filename) # Read the file using this object as the parser reader = LineReader.new(self, do_end=False) reader.read_file_by_name(abs_filename) # Restore the original package root self.pkg_root = old_pkg_root
def _parse_entry(self, entry): """ Parse entry """ cur_context = self.context_stack[-1] if not cur_context.label: self.log_error("Missing label for entry {}".format(entry)) if self.validate_files: # When validating files (i.e., a real build), change to absolute path entry = Util.get_abs_path(entry, self.pkg_root) if cur_context.label in FILE_LABELS: path = None for f in glob.iglob(entry): path = pathlib.Path(f) if not path.is_file(): self.log_error("'{}' is not a file".format(f)) if path is None: self.log_error( "'{}' does not match any files".format(entry)) elif cur_context.label in PATH_LABELS: path = pathlib.Path(entry) if not path.is_dir(): self.log_error("'{}' is not a directory".format(entry)) # If this is parsed in a condition context, skip over unmatching entries if not cur_context.is_condition_met(): self.debug("SKIP_ENTRY: {}".format(entry)) return if (cur_context.label in FILE_LABELS) and (self.validate_files): for f in glob.iglob(entry): self.debug("ADD_FILE: {}".format(f)) self.manifest.add_entry(cur_context.label, f) else: self.debug("ADD_ENTRY: {}".format(entry)) self.manifest.add_entry(cur_context.label, entry)
def get_action_file(self, ext_dep, extension=None): """Create and return a single output filename for this action""" action_dir = Util.get_abs_path( "{}.{}".format(self.ACTION_TYPE, extension), ext_dep.out_dir) return action_dir
def get_action_folder(self, ext_dep): """Return an output directory for this action""" action_dir = Util.get_abs_path(self.ACTION_TYPE, ext_dep.out_dir) return action_dir
def _open_project(self): try: project, prj_dir, out_dir = Builder.read_project( self.project_file, self.out_dir) except Log.GlobifestException as e: tkinter.messagebox.showerror(self.APP_TITLE, str(e)) return def_trees = list() self._opendir = os.path.dirname(self.project_file) for pkg in project.get_packages(): pkg_file = Builder.get_pkg_file(project, pkg, prj_dir, out_dir) if not pkg_file: tkinter.messagebox.showerror( self.APP_TITLE, "Unknown file root {}".format(str(pkg.file_root))) return pkg_root = Builder.get_pkg_root(project, pkg, pkg_file, out_dir) if pkg_root is None: tkinter.messagebox.showerror( self.APP_TITLE, "Unknown package root {}".format(str(pkg.file_root))) return try: manifest = Builder.build_manifest(pkg_file, ManifestParser.ConfigsOnly(), pkg_root, validate_files=False) except Log.GlobifestException as e: tkinter.messagebox.showerror(self.APP_TITLE, str(e)) return pkg_dir = os.path.dirname(pkg_file) for cfg in manifest.get_configs(): cfg.definition = Util.get_abs_path(cfg.definition, pkg_dir) def_tree = Builder.build_definition(cfg.definition) def_trees.append(def_tree) # Aggregate DefTrees into a single list forest = DefTree.DefForest() for tree in def_trees: tree.walk(forest) self.project = project # Clear GUI elements self._clear_gui() # Set the window title self.app_root.title("{} - {}".format(self.APP_TITLE, self.project.get_name())) # Walk the forest to populate the config tree forest.walk(CfgTreeObserver(self.cfg_tree, self.param_tbl), child_sorter=gui_child_sorter, param_sorter=gui_param_sorter) # Set up layer/variant boxes self.settings_view_tbl.clear() self.settings_cache.clear() layer_names = project.get_layer_names() if layer_names: for layer in layer_names: variant_names = project.get_variant_names(layer) self.settings_view_tbl[layer] = (variant_names[0] if variant_names else "") # Build the settings cache layer_cache = Util.Container() self.settings_cache[layer] = layer_cache for variant in variant_names: variant_cache = Util.Container() layer_cache[variant] = variant_cache variant_target = self.project.get_target(layer, variant) variant_target.filename = Util.get_abs_path( variant_target.filename, prj_dir) variant_cache["target"] = variant_target variant_cache["config"] = Builder.build_config( variant_target.filename) value_list = [] for layer in layer_names: value_list.append("{}={}".format( layer, self.settings_view_tbl[layer])) self.settings_layer_cmb.configure(values=value_list) self.cur_layer.set(layer_names[0]) # Variant will propagate via layer change self.file_menu.entryconfig("Close", state="normal") if not self.settings_cache: tkinter.messagebox.showinfo( self.APP_TITLE, "{} defines no settings!".format(self.project_file)) self._close_project()
def build_project(in_fname, out_dir, settings, callbacks=Util.Container()): """ Build a project with the given settings """ project, prj_dir, out_dir = read_project(in_fname, out_dir) Log.I("Project: {}".format(project.get_name())) Log.I("PrjDir: {}".format(prj_dir)) Log.I("OutDir: {}".format(out_dir)) os.makedirs(out_dir, exist_ok=True) # Check external project dependencies for dep_name, dependency in project.get_dependencies(): Log.I("Checking dependency {}...".format(dep_name)) dep_out_dir = os.path.join(out_dir, dep_name) os.makedirs(dep_out_dir, exist_ok=True) dependency.setup(dep_out_dir) # Set up build configuration Log.I("Build configuration:") setting_re = re.compile("([^=]+)=(.+)") cfg_container = Util.Container() # Unordered, for tracking purposes for cfg_entry in settings: m = Matcher.new(cfg_entry) if not m.is_fullmatch(setting_re): Log.E("Malformed setting: {}".format(cfg_entry)) if cfg_container.get(m[1]): Log.E("Conflicting/Duplicate setting: {}".format(cfg_entry)) Log.I(" {}: {}".format(m[1], m[2])) variant = project.get_target(m[1], m[2]) # Update the filename with the absolute path variant.filename = Util.get_abs_path(variant.filename, prj_dir) cfg_container[m[1]] = variant # Validate that all layers are specified for layer in project.get_layer_names(): variant = cfg_container.get(layer) if variant is None: variant_names = project.get_variant_names(layer) if len(variant_names) == 1: # None specified, but there is only one variant = project.get_target(layer, variant_names[0]) Log.D(" **Default selected for layer {}**".format(layer)) Log.I(" {}: {}".format(layer, variant.name)) variant.filename = Util.get_abs_path(variant.filename, prj_dir) cfg_container[layer] = variant else: Log.E("Must specify variant for layer {}".format(layer)) Log.I("Generating settings in layer order:") effective_settings = Settings.new() for layer in project.get_layer_names(): variant = cfg_container.get(layer) Log.I(" {}: {}".format(layer, variant.filename)) layer_config = build_config(variant.filename) effective_settings.extend(layer_config.get_settings()) # Generate a metadata object to communicate information back to the caller metadata = Util.Container(prj_dir=prj_dir, out_dir=out_dir, settings=effective_settings) #### PREBUILD CALLBACK #### if callbacks.get("prebuild"): callbacks.prebuild(callbacks.get("arg"), metadata) # Prepare storage for package processing for pub_key in ManifestParser.PUBLIC_LABELS: metadata[pub_key] = [] all_manifests = [] Log.I("Processing packages...") for pkg in project.get_packages(): pkg_file = get_pkg_file(project, pkg, prj_dir, out_dir) if pkg_file is None: Log.I("Unknown file root {}".format(str(pkg.file_root))) pkg_root = get_pkg_root(project, pkg, pkg_file, out_dir) if pkg_root is None: Log.I("Unknown package root {}".format(str(pkg.file_root))) Log.I(" {}".format(pkg_file)) manifest = build_manifest(pkg_file, effective_settings, pkg_root) all_manifests.append(manifest) pkg_dir = os.path.dirname(pkg_file) manifest_out = manifest.get_output() # Replace all file paths with absolute paths for k in ManifestParser.FILE_LABELS: manifest_out[k] = [ Util.get_abs_path(x, pkg_dir) for x in manifest_out[k] ] # Aggregate all public labels for pub_key in ManifestParser.PUBLIC_LABELS: metadata[pub_key] += manifest_out[pub_key] # Dump all the files on extreme mode if Log.Logger.has_level(Log.LEVEL.EXTREME): for k, v in manifest_out: Log.X(" {}:".format(k)) for f in v: Log.X(" {}".format(f)) for cfg in manifest.get_configs(): Log.I(" Post-processing {}".format(cfg.definition_abs)) defs = cfg.def_tree.get_relevant_params(effective_settings) for gen in cfg.generators: gen_file = Util.get_abs_path(gen.get_filename(), pkg_dir) gen_file = os.path.relpath(gen_file, start=pkg_dir) gen_file = os.path.normpath(gen_file) gen_file = os.path.join(out_dir, gen_file) # Update the filename in the generator gen.filename = gen_file Log.I(" Generating {}".format(gen_file)) if gen.get_formatter(): formatter_filename = Util.get_abs_path( gen.get_formatter(), pkg_dir) Log.I(" Executing {}".format(formatter_filename)) #### GENERATOR CALLBACK #### if callbacks.get("generator"): # Let the build script intercept the generator without any filesystem changes callbacks.generator(callbacks.get("arg", None), metadata, defs, gen) else: os.makedirs(os.path.dirname(gen_file), exist_ok=True) gen.generate(defs, out_dir) #### POSTPROCESS CALLBACK #### if callbacks.get("postprocess"): callbacks.postprocess(callbacks.get("arg"), metadata) # Add public data into each manifest and build if callbacks.get("target"): for manifest in all_manifests: tables = Util.Container() manifest_out = manifest.get_output() for k, v in manifest_out: if not k in ManifestParser.PUBLIC_LABELS: tables[k] = v manifest_name = os.path.relpath(manifest.get_filename(), start=prj_dir) manifest_name = os.path.normpath(manifest_name) manifest_name = os.path.splitext(manifest_name)[0] #### BUILD TARGET CALLBACK #### callbacks.target(callbacks.get("arg", None), metadata, manifest_name, tables) #### POSTBUILD CALLBACK #### if callbacks.get("postbuild"): callbacks.postbuild(callbacks.get("arg", None), metadata)