def logfile(spec): """Determine logfile given a parameters specification.""" if logdir is None: return None return os.path.join( logdir, "{}_{}.log".format(options["graph_attr"], Parameters.format_spec(spec)), )
def dump_output(out, path=None, params_spec=None): from taskgraph.parameters import Parameters params_name = Parameters.format_spec(params_spec) fh = None if path: # Substitute params name into file path if necessary if params_spec and "{params}" not in path: name, ext = os.path.splitext(path) name += "_{params}" path = name + ext path = path.format(params=params_name) fh = open(path, "w") else: print( "Dumping result with parameters from {}:".format(params_name), file=sys.stderr, ) print(out + "\n", file=fh)
def show_taskgraph(options): from taskgraph.parameters import Parameters from taskgraph.util.vcs import get_repository if options.pop("verbose", False): logging.root.setLevel(logging.DEBUG) repo = None cur_ref = None diffdir = None output_file = options["output_file"] if options["diff"]: repo = get_repository(os.getcwd()) if not repo.working_directory_clean(): print( "abort: can't diff taskgraph with dirty working directory", file=sys.stderr, ) return 1 # We want to return the working directory to the current state # as best we can after we're done. In all known cases, using # branch or bookmark (which are both available on the VCS object) # as `branch` is preferable to a specific revision. cur_ref = repo.branch or repo.head_ref[:12] diffdir = tempfile.mkdtemp() atexit.register(shutil.rmtree, diffdir) # make sure the directory gets cleaned up options["output_file"] = os.path.join( diffdir, f"{options['graph_attr']}_{cur_ref}") print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr) parameters: List[Any[str, Parameters]] = options.pop("parameters") if not parameters: kwargs = { "target-kind": options.get("target_kind"), } parameters = [Parameters(strict=False, **kwargs)] # will use default values for param in parameters[:]: if isinstance(param, str) and os.path.isdir(param): parameters.remove(param) parameters.extend([ p.as_posix() for p in Path(param).iterdir() if p.suffix in (".yml", ".json") ]) logdir = None if len(parameters) > 1: # Log to separate files for each process instead of stderr to # avoid interleaving. basename = os.path.basename(os.getcwd()) logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename) if not os.path.isdir(logdir): os.makedirs(logdir) else: # Only setup logging if we have a single parameter spec. Otherwise # logging will go to files. This is also used as a hook for Gecko # to setup its `mach` based logging. setup_logging() generate_taskgraph(options, parameters, logdir) if options["diff"]: assert diffdir is not None assert repo is not None # Some transforms use global state for checks, so will fail # when running taskgraph a second time in the same session. # Reload all taskgraph modules to avoid this. for mod in sys.modules.copy(): if mod != __name__ and mod.startswith("taskgraph"): del sys.modules[mod] if options["diff"] == "default": base_ref = repo.base_ref else: base_ref = options["diff"] try: repo.update(base_ref) base_ref = repo.head_ref[:12] options["output_file"] = os.path.join( diffdir, f"{options['graph_attr']}_{base_ref}") print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr) generate_taskgraph(options, parameters, logdir) finally: repo.update(cur_ref) # Generate diff(s) diffcmd = [ "diff", "-U20", "--report-identical-files", f"--label={options['graph_attr']}@{base_ref}", f"--label={options['graph_attr']}@{cur_ref}", ] for spec in parameters: base_path = os.path.join(diffdir, f"{options['graph_attr']}_{base_ref}") cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_ref}") params_name = None if len(parameters) > 1: params_name = Parameters.format_spec(spec) base_path += f"_{params_name}" cur_path += f"_{params_name}" try: proc = subprocess.run( diffcmd + [base_path, cur_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True, ) diff_output = proc.stdout returncode = 0 except subprocess.CalledProcessError as e: # returncode 1 simply means diffs were found if e.returncode != 1: print(e.stderr, file=sys.stderr) raise diff_output = e.output returncode = e.returncode dump_output( diff_output, # Don't bother saving file if no diffs were found. Log to # console in this case instead. path=None if returncode == 0 else output_file, params_spec=spec if len(parameters) > 1 else None, ) if options["format"] != "json": print( "If you were expecting differences in task bodies " 'you should pass "-J"\n', file=sys.stderr, ) if len(parameters) > 1: print("See '{}' for logs".format(logdir), file=sys.stderr)