def run_tasks(tasks, exe, in_path, csv_names):
"""Run validator command-line on all tasks."""
csv_count = len(csv_names)
logs = [''] * csv_count
# Group tasks by unique CSV index and section.
groups = []
csv_index = -1
section = None
for task in tasks:
if task.csv_index != csv_index or task.section != section:
csv_index = task.csv_index
section = task.section
groups.append((csv_index, section, []))
groups[-1][2].append(task)
for (csv_index, section, group_tasks) in groups:
# Run validation command on all tasks in the group, splitting up the command
# to limit the number of parameters for each run.
status(TASK_SECTION_FORMAT % (csv_names[csv_index], section),
util.LOG_COLOR_CYAN)
cmd_args = [exe]
for task in group_tasks:
if len(cmd_args) >= MAX_FILES_PER_COMMAND:
logs[csv_index] += run_command(cmd_args)
cmd_args = [exe]
cmd_args.append(util.join_path(in_path, task.src))
if len(cmd_args) > 1:
logs[csv_index] += run_command(cmd_args)
return logs
def run_command(cmd_args):
"""Run validator command-line."""
# Spawn the command.
command = get_process_command(cmd_args)
status(command, util.LOG_COLOR_DARK_CYAN)
# TODO: This doesn't interleave stdout and stderr in the correct
# order, and there appears to be no reasonable way to do that in Python. As a
# work-around, we're forcing the exe to write all messages to stdout.
code = 0
try:
output = subprocess.check_output(
cmd_args, stderr=subprocess.STDOUT, universal_newlines=True)
except subprocess.CalledProcessError as e:
output = e.output
code = e.returncode
output = output.replace('\r', '')
if code != 0:
# Failure without error lines likely indicates the process crashed.
if output:
output += '\n'
output += 'ERROR: Exit code %s (0x%s).' % (code, util.uhex32(code))
if (code & 0xffffffff) == 0xc0000005: # Windows-specific error code.
output += ' [Access Violation]'
print(util.colorize_process_output(output))
return output
def main():
util.enable_ansi_colors()
(args, unknown_args) = parse_args()
out_path = util.norm_abspath(args.out_dir)
if args.clean:
clean_output_directory(out_path)
return 0
# Parse CSVs for tasks.
(tasks, csv_names, csv_tasks) = get_csv_tasks(args.csv)
if not tasks:
return 1
usd_type = args.type
# Choose the number of concurrent conversion processes.
process_count = get_process_count(args.processes, args.process_max,
len(tasks))
type_text = usd_type if usd_type else 'usda+usdz'
status('Converting %s models to %s with %s processes.' %
(len(tasks), type_text, process_count))
failed_tasks = []
in_path = util.norm_abspath(args.in_dir)
# Output .usda, .usdz, or .usd- for both.
ext = ('.' + usd_type) if usd_type else '.usd-'
exe_args = unknown_args + util.split_args(args.args)
tag_tracker = util.TagTracker()
(_, failed_tasks, _) = run_conversion(csv_tasks, tasks, args.exe, exe_args,
ext, in_path, out_path, csv_names,
process_count, '', '', tag_tracker)
if failed_tasks:
return 1
else:
return 0
def generate_csv_results(in_path, out_path, golden_path, test_path, csv_name,
log_text, task_count, tag_tracker, nodiff,
diff_command):
"""Write per-CSV result text and diff."""
tag_stats = tag_tracker.get_per_tag_stats(False)
out_text = ('%s%s\n%s%s\nValidated %s models.%s\n') % (
log_text, STATS_HEADER, tag_stats, SUMMARY_HEADER, task_count,
tag_tracker.get_summary_suffix(False))
# Write the result file to the test output directory so it's included in
# diffs.
result_name = csv_name + '_result.txt'
result_path = util.join_path(test_path, result_name)
with open(result_path, 'w') as result_file:
result = out_text.replace(in_path, '${IN}')
result_file.write(result)
# Write the log file to the base output directory so it's not diff'ed (we
# don't want it diff'ed because it contains full paths and can vary based
# on input/output directories).
log_name = csv_name + '_log.txt'
log_path = os.path.abspath(util.join_path(out_path, log_name))
line_header = colorize('%s.csv: ' % csv_name, util.LOG_COLOR_CYAN)
status('%sValidated %s models.%s' %
(line_header, task_count, tag_tracker.get_summary_suffix(True)))
status(' Writing log to: %s' % (log_path))
with open(log_path, 'w') as log_file:
log_file.write(out_text)
# Diff vs golden files.
mismatch_count = 0
missing_count = 0
extra_count = 0
if not nodiff:
status('%sDiffing vs golden.' % (line_header))
diffs_name = csv_name + '_diffs.txt'
(_, mismatch_count, missing_count,
extra_count) = diff.diff_with_golden(golden_path, test_path, out_path, [],
result_name, diff_command, diffs_name)
return (mismatch_count, missing_count, extra_count)
def main():
util.enable_ansi_colors()
(args, _) = parse_args()
out_path = util.norm_abspath(args.out_dir)
# Parse CSVs for tasks.
(tasks, csv_names, csv_tasks) = get_csv_tasks(args.csv)
if not tasks:
return 1
status('Validating %s models.' % len(tasks))
in_path = util.norm_abspath(args.in_dir)
golden_path = util.norm_abspath(args.golden_dir)
test_path = util.join_path(out_path, 'test')
# Convert usda and/or usdz files.
start_time = time.time()
exe = args.exe
logs = run_tasks(tasks, exe, in_path, csv_names)
task_end_time = time.time()
csv_count = len(csv_names)
tag_tracker = util.TagTracker()
csv_tag_trackers = [None] * csv_count
for csv_index, log_text in enumerate(logs):
csv_tag_tracker = util.TagTracker.from_output(log_text)
tag_tracker.add_from_tracker(csv_tag_tracker)
csv_tag_trackers[csv_index] = csv_tag_tracker
status(STATS_HEADER, util.LOG_COLOR_CYAN)
tag_stats = tag_tracker.get_per_tag_stats(True)
status(tag_stats)
status(SUMMARY_HEADER, util.LOG_COLOR_CYAN)
task_delta_time = task_end_time - start_time
tag_summary = tag_tracker.get_summary_suffix(True)
status('Validated %s models in %.2fs.%s' %
(len(tasks), task_delta_time, tag_summary))
util.make_directories(out_path)
util.make_directories(test_path)
mismatch_total = 0
missing_total = 0
extra_total = 0
# Write per-CSV output.
for csv_index, csv_name in enumerate(csv_names):
status('')
log_text = logs[csv_index]
csv_task_count = len(csv_tasks[csv_index])
(mismatch_count, missing_count, extra_count) = generate_csv_results(
in_path, out_path, golden_path, test_path, csv_name, log_text,
csv_task_count, csv_tag_trackers[csv_index], args.nodiff,
args.diff_command)
mismatch_total += mismatch_count
missing_total += missing_count
extra_total += extra_count
error_total = tag_tracker.totals[util.SEVERITY_ERROR]
if args.summary_out:
with open(args.summary_out, 'w') as summary_file:
summary_file.write(
get_summary(error_total, mismatch_total, missing_total, extra_total))
# Return exit code for use by unit tests.
err = 0
if error_total:
err |= 1
if mismatch_total or missing_total or extra_total:
err |= 2
return err
def main():
util.enable_ansi_colors()
(args, unknown_args) = parse_args()
out_path = util.norm_abspath(args.out_dir)
if args.clean:
clean_output_directory(out_path)
return 0
# Parse CSVs for tasks.
(tasks, csv_names, csv_tasks) = get_csv_tasks(args.csv)
if not tasks:
return 1
usd_type = args.type
# Choose the number of concurrent conversion processes.
process_count = get_process_count(args.processes, args.process_max,
len(tasks))
type_text = usd_type if usd_type else 'usda+usdz'
status('Converting %s models to %s with %s processes.' %
(len(tasks), type_text, process_count))
complete_tasks = []
failed_tasks = []
in_path = util.norm_abspath(args.in_dir)
golden_path = util.norm_abspath(args.golden_dir)
test_path = join_path(out_path, 'test')
# Output .usda, .usdz, or .usd- for both.
ext = ('.' + usd_type) if usd_type else '.usd-'
match_total = 0
mismatch_total = 0
missing_total = 0
extra_total = 0
if args.redeploy:
# We only deploy USDZ files.
usd_type = 'usdz'
complete_tasks = tasks
else:
exe_args = unknown_args + util.split_args(args.args)
# Write the log file to the base output directory so it's not diff'ed (we
# don't want it diff'ed because it contains full paths and can vary based
# on input/output directories).
log_path = out_path
# Write the result file to the test output directory so it's included in
# diffs.
result_path = test_path
tag_tracker = util.TagTracker()
(complete_tasks, failed_tasks, task_delta_time) = run_conversion(
csv_tasks, tasks, args.exe, exe_args, ext, in_path, test_path,
csv_names, process_count, log_path, result_path, tag_tracker)
# Do per CSV diffs.
if not args.nodiff:
diff_start_time = time.time()
for csv_index, csv_name in enumerate(csv_names):
# Diff vs golden files.
line_header = util.colorize('%s.csv: ' % csv_name,
util.LOG_COLOR_CYAN)
status('%sDiffing vs golden.' % (line_header))
diffs_name = csv_name + '_diffs.txt'
result_name = csv_name + '_result.txt'
(match_count, mismatch_count,
missing_count, extra_count) = diff.diff_with_golden(
golden_path, test_path, out_path, csv_tasks[csv_index],
result_name, args.diff_command, diffs_name)
match_total += match_count
mismatch_total += mismatch_count
missing_total += missing_count
extra_total += extra_count
diff_end_time = time.time()
status(DIFF_SUMMARY_HEADER, util.LOG_COLOR_CYAN)
diff_delta_time = diff_end_time - diff_start_time
tag_summary = tag_tracker.get_summary_suffix(True)
status('Converted %s models in %.2fs. %s failed.%s' %
(len(complete_tasks), task_delta_time, len(failed_tasks),
tag_summary))
diff_summary = diff.get_summary_suffix(match_total, mismatch_total,
missing_total, extra_total)
status('Diffed in %.2fs.%s' % (diff_delta_time, diff_summary))
# Deploy USDZ files and index.html to local directory for upload.
if not args.nodeploy and (not usd_type or usd_type == 'usdz'):
changed_tasks = []
if not args.nodiff:
changed_tasks = diff.get_changed_tasks(complete_tasks, golden_path,
test_path)
if args.deploy_all:
deploy_tasks = tasks
complete_tasks = tasks
else:
deploy_tasks = changed_tasks
deploy_path = util.norm_abspath(args.deploy_dir)
deploy.copy_to_deploy_dir(deploy_tasks, in_path, test_path,
deploy_path, args.deploy_gltf)
html = deploy.generate_html(tasks, changed_tasks, args.deploy_gltf)
html_path = join_path(deploy_path, 'index.html')
with open(html_path, 'w') as html_file:
html_file.write(html)
deploy.copy_html_resources(deploy_path)
if args.summary_out:
with open(args.summary_out, 'w') as summary_file:
summary_file.write(
get_summary(failed_tasks, mismatch_total, missing_total,
extra_total))
# Return exit code for use by unit tests.
err = 0
if failed_tasks:
err |= 1
if mismatch_total or missing_total or extra_total:
err |= 2
return err