def logfile(spec):
"""Determine logfile given a parameters specification."""
if logdir is None:
return None
return os.path.join(
logdir,
"{}_{}.log".format(options["graph_attr"],
Parameters.format_spec(spec)),
)
def dump_output(out, path=None, params_spec=None):
from taskgraph.parameters import Parameters
params_name = Parameters.format_spec(params_spec)
fh = None
if path:
# Substitute params name into file path if necessary
if params_spec and "{params}" not in path:
name, ext = os.path.splitext(path)
name += "_{params}"
path = name + ext
path = path.format(params=params_name)
fh = open(path, "w")
else:
print(
"Dumping result with parameters from {}:".format(params_name),
file=sys.stderr,
)
print(out + "\n", file=fh)
def show_taskgraph(options):
from taskgraph.parameters import Parameters
from taskgraph.util.vcs import get_repository
if options.pop("verbose", False):
logging.root.setLevel(logging.DEBUG)
repo = None
cur_ref = None
diffdir = None
output_file = options["output_file"]
if options["diff"]:
repo = get_repository(os.getcwd())
if not repo.working_directory_clean():
print(
"abort: can't diff taskgraph with dirty working directory",
file=sys.stderr,
)
return 1
# We want to return the working directory to the current state
# as best we can after we're done. In all known cases, using
# branch or bookmark (which are both available on the VCS object)
# as `branch` is preferable to a specific revision.
cur_ref = repo.branch or repo.head_ref[:12]
diffdir = tempfile.mkdtemp()
atexit.register(shutil.rmtree,
diffdir) # make sure the directory gets cleaned up
options["output_file"] = os.path.join(
diffdir, f"{options['graph_attr']}_{cur_ref}")
print(f"Generating {options['graph_attr']} @ {cur_ref}",
file=sys.stderr)
parameters: List[Any[str, Parameters]] = options.pop("parameters")
if not parameters:
kwargs = {
"target-kind": options.get("target_kind"),
}
parameters = [Parameters(strict=False,
**kwargs)] # will use default values
for param in parameters[:]:
if isinstance(param, str) and os.path.isdir(param):
parameters.remove(param)
parameters.extend([
p.as_posix() for p in Path(param).iterdir()
if p.suffix in (".yml", ".json")
])
logdir = None
if len(parameters) > 1:
# Log to separate files for each process instead of stderr to
# avoid interleaving.
basename = os.path.basename(os.getcwd())
logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename)
if not os.path.isdir(logdir):
os.makedirs(logdir)
else:
# Only setup logging if we have a single parameter spec. Otherwise
# logging will go to files. This is also used as a hook for Gecko
# to setup its `mach` based logging.
setup_logging()
generate_taskgraph(options, parameters, logdir)
if options["diff"]:
assert diffdir is not None
assert repo is not None
# Some transforms use global state for checks, so will fail
# when running taskgraph a second time in the same session.
# Reload all taskgraph modules to avoid this.
for mod in sys.modules.copy():
if mod != __name__ and mod.startswith("taskgraph"):
del sys.modules[mod]
if options["diff"] == "default":
base_ref = repo.base_ref
else:
base_ref = options["diff"]
try:
repo.update(base_ref)
base_ref = repo.head_ref[:12]
options["output_file"] = os.path.join(
diffdir, f"{options['graph_attr']}_{base_ref}")
print(f"Generating {options['graph_attr']} @ {base_ref}",
file=sys.stderr)
generate_taskgraph(options, parameters, logdir)
finally:
repo.update(cur_ref)
# Generate diff(s)
diffcmd = [
"diff",
"-U20",
"--report-identical-files",
f"--label={options['graph_attr']}@{base_ref}",
f"--label={options['graph_attr']}@{cur_ref}",
]
for spec in parameters:
base_path = os.path.join(diffdir,
f"{options['graph_attr']}_{base_ref}")
cur_path = os.path.join(diffdir,
f"{options['graph_attr']}_{cur_ref}")
params_name = None
if len(parameters) > 1:
params_name = Parameters.format_spec(spec)
base_path += f"_{params_name}"
cur_path += f"_{params_name}"
try:
proc = subprocess.run(
diffcmd + [base_path, cur_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
)
diff_output = proc.stdout
returncode = 0
except subprocess.CalledProcessError as e:
# returncode 1 simply means diffs were found
if e.returncode != 1:
print(e.stderr, file=sys.stderr)
raise
diff_output = e.output
returncode = e.returncode
dump_output(
diff_output,
# Don't bother saving file if no diffs were found. Log to
# console in this case instead.
path=None if returncode == 0 else output_file,
params_spec=spec if len(parameters) > 1 else None,
)
if options["format"] != "json":
print(
"If you were expecting differences in task bodies "
'you should pass "-J"\n',
file=sys.stderr,
)
if len(parameters) > 1:
print("See '{}' for logs".format(logdir), file=sys.stderr)
