def main():
args = create_parser().parse_args()
test_cases = []
try:
test_cases = configure_output_dir(args.output, args.test_cases)
except Exception as e:
LOG.error(repr(e))
return 1
# Defines the characteristics of machines which used to execute this script
try:
gpu = system_info.get_gpu()
except:
LOG.error("Can't get gpu name")
gpu = 'Unknown'
Renderer.PLATFORM = {
'GPU': gpu,
'OS': platform.system(),
'PLUGIN': extract_plugin_versions()
}
Renderer.TOOL = args.tool
Renderer.LOG = LOG
Renderer.ASSETS_PATH = args.res_path
Renderer.BASELINE_PATH = os.path.join(args.res_path, "..", "rpr_houdini_autotests_baselines")
Renderer.PACKAGE = args.package_name
[case.render() for case in
[Renderer(case, args.output, args.update_refs, args.resolution_x, args.resolution_y, args.retries, args.rpr_traces_path) for case in
test_cases]
]
def main():
args = create_args_parser()
work_dir = os.path.abspath(args.output_dir)
if not os.path.exists(os.path.join(args.output_dir, "Color")):
os.makedirs(os.path.join(args.output_dir, "Color"))
if not os.path.exists(os.path.join(args.output_dir, "render_tool_logs")):
os.makedirs(os.path.join(args.output_dir, "render_tool_logs"))
try:
test_cases_path = os.path.realpath(
os.path.join(work_dir, 'test_cases.json'))
shutil.copyfile(args.tests_list, test_cases_path)
except Exception as e:
main_logger.error("Can't copy test_cases.json")
main_logger.error(str(e))
exit(-1)
try:
with open(test_cases_path, 'r') as file:
tests_list = json.load(file)
except OSError as e:
main_logger.error("Failed to read test cases json. ")
main_logger.error(str(e))
exit(-1)
render_device = get_gpu()
system_pl = platform.system()
current_conf = set(platform.system()) if not render_device else {
platform.system(), render_device
}
main_logger.info("Detected GPUs: {}".format(render_device))
main_logger.info("PC conf: {}".format(current_conf))
main_logger.info("Creating predefined errors json...")
# save pre-defined reports with error status
prepare_cases(args, tests_list, render_device, current_conf)
with open(test_cases_path, 'w') as file:
json.dump(tests_list, file, indent=4)
# run cases
execute_cases(args, tests_list, test_cases_path, current_conf, work_dir)
return 0
def prepare_cases(args, cases, platform_config, engine, test_cases_path):
for case in cases:
# there is list with lists of gpu/os/engine/gpu&os/gpu
# for example: [['Darwin', 'Hybrid'], ['Windows', 'Radeon RX Vega'], ['GeForce GTX 1080 Ti', 'Northstar64']]
# with that skip_for case will be skipped on OSX, GeForce GTX 1080 Ti and Windows with Vega
main_logger.info("info: {}".format(case.get('status', '')))
case_status = TEST_IGNORE_STATUS if is_case_skipped(
case, platform_config, args.engine) else TEST_CRASH_STATUS
if case_status == TEST_IGNORE_STATUS:
with open(test_cases_path, "w") as f:
case['status'] = case_status
if 'aovs' in case:
set_aovs_group_status(case, case_status)
json.dump(cases, f, indent=4)
report = RENDER_REPORT_BASE.copy()
report.update(RENDER_REPORT_EC_PACK.copy())
report.update({
'test_case':
case['case'],
'test_status':
case_status,
'test_group':
args.package_name,
'scene_name':
case['scene'],
'render_device':
get_gpu(),
'width':
args.resolution_x,
'height':
args.resolution_y,
'iterations':
args.pass_limit,
'tool':
"Core",
'date_time':
datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S"),
'render_color_path':
os.path.join('Color', case['case'] + ".png"),
'file_name':
case['case'] + ".png"
})
def configure_context(args):
ps = platform.system()
engine = None
for e in ["Hybrid", "HybridPro", "Northstar64"]: # get rpr engine
if e in args.engine:
engine = e
break
args.tool = os.path.abspath(args.tool)
tool_path = os.path.dirname(args.tool)
args.output = os.path.abspath(args.output)
# set unix systems executive file permissions
if ps != "Windows":
os.system('chmod +x {}'.format(os.path.abspath(args.tool)))
gpu_name = get_gpu()
os_name = get_machine_info().get('os', 'Unknown')
if not gpu_name:
main_logger.error("Can't get gpu name")
if os_name == 'Unknown':
main_logger.error("Can't get os name")
render_platform = {os_name, gpu_name}
return ps, engine, tool_path, render_platform
def generate_json_for_report(case_name, dir_with_json, engine):
cfg_json = os.path.join(dir_with_json,
"{}_original.json".format(case_name))
if os.path.exists(cfg_json):
with open(cfg_json) as f:
test_json = json.loads(f.read().replace("\\", "\\\\"))
report_name = cfg_json.replace("original", "RPR")
report = core_config.RENDER_REPORT_BASE.copy()
report.update(core_config.RENDER_REPORT_EC_PACK.copy())
report["core_version"] = core_ver_str(int(test_json["version"], 16))
report["minor_version"] = test_json["version.minor"]
report["gpu_memory_total"] = test_json["gpumem.total.mb"]
report["gpu_memory_max"] = test_json["gpumem.max.alloc.mb"]
report["gpu_memory_usage"] = test_json["gpumem.usage.mb"]
report["system_memory_usage"] = test_json["sysmem.usage.mb"]
report["render_mode"] = "GPU"
report["render_device"] = get_gpu()
report["test_group"] = dir_with_json.split(os.path.sep)[-1]
report["scene_name"] = test_json["input"].split(os.path.sep)[-1]
report["test_case"] = case_name
if type(test_json["input"]) is str:
report["file_name"] = test_json["input"].split(
os.path.sep)[-1] + ".png"
elif type(test_json["input"]) is list:
report["file_name"] = test_json["input"][0].split(
os.path.sep)[-1] + ".png"
report["render_color_path"] = os.path.join("Color", case_name + ".png")
report["tool"] = "Core"
report["render_time"] = test_json["render.time.ms"] / 1000
report['date_time'] = datetime.datetime.now().strftime(
"%m/%d/%Y %H:%M:%S")
report['difference_color'] = 0
report['test_status'] = "passed"
report['width'] = test_json['width']
report['height'] = test_json['height']
report['iterations'] = test_json['iteration']
report['group_timeout_exceeded'] = False
with open(os.path.join(dir_with_json, report_name), 'r') as f:
rpr_report = json.load(f)[0]
report['tahoe_log'] = rpr_report['tahoe_log']
report['core_scene_configuration'] = rpr_report[
'core_scene_configuration']
with open(os.path.join(dir_with_json, report_name), 'w') as f:
json.dump([report], f, indent=' ')
if 'aovs' in test_json.keys():
for key, value in test_json['aovs'].items():
report["render_time"] = 0.0
if type(value) is str:
report['file_name'] = value.split(os.path.sep)[-1]
report['render_color_path'] = value
elif type(value) is list:
report['file_name'] = value[0].split(os.path.sep)[-1]
report['render_color_path'] = value[0]
report['test_case'] = case_name + key
with open(
os.path.join(dir_with_json,
report_name.replace('_RPR',
key + '_RPR')),
'r') as file:
report['tahoe_log'] = json.load(file)[0]['tahoe_log']
with open(
os.path.join(dir_with_json,
report_name.replace('_RPR',
key + '_RPR')),
'w') as file:
json.dump([report], file, indent=4)
continue
if 'aovs' in config_json.keys():
for key, value in config_json['aovs'].items():
value_with_engine = value.replace('.png', engine + '.png')
report = RENDER_REPORT_BASE.copy()
report.update(RENDER_REPORT_EC_PACK.copy())
report.update({
'test_case':
case['case'] + key,
'test_status':
case_status,
'test_group':
args.package_name,
'render_device':
get_gpu(),
'render_color_path':
os.path.join('Color', value_with_engine),
'file_name':
value_with_engine
})
if case['status'] == TEST_IGNORE_STATUS:
report.update({'group_timeout_exceeded': False})
with open(
os.path.join(args.output,
report['test_case'] + CASE_REPORT_SUFFIX),
'w') as file:
json.dump([report], file, indent=4)
copyfile(
def main():
args = create_args_parser()
with open(args.tests_list, 'r') as file:
try:
tests_list = json.loads(file.read())
except json.decoder.JSONDecodeError as err:
main_logger.error(str(err))
exit(1)
if not os.path.exists(os.path.join(args.output_dir, "Color")):
os.makedirs(os.path.join(args.output_dir, "Color"))
# TODO: try-catch on file reading
if not os.path.exists(
os.path.join(args.render_path, 'config.original.json')):
shutil.copyfile(os.path.join(args.render_path, 'config.json'),
os.path.join(args.render_path, 'config.original.json'))
with open(os.path.join(args.render_path, 'config.original.json'),
'r') as file:
config_tmp = json.loads(file.read())
render_device = get_gpu()
main_logger.info("Creating predefined errors json...")
for test in tests_list:
# TODO: save scene name instead of scene sub path
report = RENDER_REPORT_BASE.copy()
report.update({
'test_status':
TEST_CRASH_STATUS
if test['status'] == 'active' else TEST_IGNORE_STATUS,
'render_device':
render_device,
'test_case':
test['case'],
'scene_name':
test['scene_sub_path'],
'tool':
args.render_engine,
'file_name':
test['case'] + test['file_ext'],
'date_time':
datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S"),
'script_info':
test['script_info'],
'test_group':
args.test_group,
'render_color_path':
'Color/' + test['case'] + test['file_ext']
})
# TODO: refactor img paths
try:
shutil.copyfile(
os.path.join(ROOT_DIR_PATH, 'jobs_launcher', 'common', 'img',
report['test_status'] + test['file_ext']),
os.path.join(args.output_dir, 'Color',
test['case'] + test['file_ext']))
except (OSError, FileNotFoundError) as err:
main_logger.error("Can't create img stub: {}".format(str(err)))
with open(
os.path.join(args.output_dir,
test["case"] + CASE_REPORT_SUFFIX), "w") as file:
json.dump([report], file, indent=4)
for test in [x for x in tests_list if x['status'] == 'active']:
main_logger.info("Processing test case: {}".format(test['case']))
frame_ae = str(
update_viewer_config(
test=test,
engine=args.render_engine,
scene_path=args.scene_path,
render_quality=args.render_quality,
draw_engine=args.draw_engine,
render_path=args.render_path,
tmp=config_tmp,
))
# remove old images
main_logger.info(os.listdir(args.render_path))
old_images = [
x for x in os.listdir(args.render_path) if os.path.isfile(x) and (
x.startswith('img0') or x.endswith('.txt'))
]
main_logger.info("Detected old renderers: {}".format(str(old_images)))
for img in old_images:
try:
os.remove(os.path.join(args.render_path, img))
except OSError as err:
main_logger.error(str(err))
os.chdir(args.render_path)
if platform.system() == 'Windows':
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer.exe"))
elif platform.system() == 'Linux':
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer"))
else:
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer"))
p = psutil.Popen(viewer_run_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stderr, stdout = b"", b""
start_time = time.time()
test_case_status = TEST_CRASH_STATUS
try:
stdout, stderr = p.communicate(timeout=test['render_time'])
except (TimeoutError, psutil.TimeoutExpired,
subprocess.TimeoutExpired) as err:
main_logger.error("Aborted by timeout. {}".format(str(err)))
for child in reversed(p.children(recursive=True)):
child.terminate()
p.terminate()
else:
test_case_status = TEST_SUCCESS_STATUS
finally:
render_time = time.time() - start_time
main_logger.info(render_time)
try:
shutil.copyfile(
os.path.join(
args.render_path,
'img{0}{1}'.format(frame_ae.zfill(4),
test['file_ext'])),
os.path.join(args.output_dir, 'Color',
test['case'] + test['file_ext']))
except FileNotFoundError as err:
main_logger.error("Image {} not found".format(
'img{0}{1}'.format(frame_ae.zfill(4), test['file_ext'])))
main_logger.error(str(err))
test_case_status = TEST_CRASH_STATUS
with open(os.path.join(args.output_dir, test['case'] + '_app.log'),
'w') as file:
file.write("-----[STDOUT]------\n\n")
file.write(stdout.decode("UTF-8"))
with open(os.path.join(args.output_dir, test['case'] + '_app.log'),
'a') as file:
file.write("\n-----[STDERR]-----\n\n")
file.write(stderr.decode("UTF-8"))
render_time_bench = -0.0
try:
for bench_txt in os.listdir(args.render_path):
if os.path.isfile(bench_txt) and bench_txt.startswith(
'scene.gltf_'):
with open(os.path.join(args.render_path, bench_txt),
"r") as file:
main_logger.info("render_time has been parsed")
render_time_bench = float(
file.readlines()[-1].split(";")[-1])
main_logger.info(render_time)
except Exception as err:
main_logger.error("Error during bench_txt parsing: {}".format(
str(err)))
# Up to date test case status
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'r') as file:
test_case_report = json.loads(file.read())[0]
test_case_report["test_status"] = test_case_status
test_case_report["render_time"] = render_time
test_case_report["render_time_bench"] = render_time_bench
test_case_report[
"render_color_path"] = "Color/" + test_case_report[
"file_name"]
test_case_report["render_log"] = test['case'] + '_app.log'
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'w') as file:
json.dump([test_case_report], file, indent=4)
return 0
def main(args):
if args.testType in ['Support_2019', 'Support_2018']:
args.tool = re.sub('[0-9]{4}', args.testType[-4:], args.tool)
if which(args.tool) is None:
core_config.main_logger.error('Can\'t find tool ' + args.tool)
exit(-1)
core_config.main_logger.info('Make "base_functions.py"')
try:
cases = json.load(
open(
os.path.realpath(
os.path.join(
os.path.abspath(args.output).replace('\\', '/'),
'test_cases.json'))))
except Exception as e:
core_config.logging.error("Can't load test_cases.json")
core_config.main_logger.error(str(e))
exit(-1)
try:
with open(os.path.join(os.path.dirname(__file__),
'base_functions.py')) as f:
script = f.read()
except OSError as e:
core_config.main_logger.error(str(e))
return 1
if os.path.exists(
os.path.join(os.path.dirname(__file__), 'extensions',
args.testType + '.py')):
with open(
os.path.join(os.path.dirname(__file__), 'extensions',
args.testType + '.py')) as f:
extension_script = f.read()
script = script.split('# place for extension functions')
script = script[0] + extension_script + script[1]
work_dir = os.path.abspath(args.output).replace('\\', '/')
res_path = os.path.abspath(args.res_path).replace('\\', '/')
maya_scenes = {x.get('scene', '') for x in cases if x.get('scene', '')}
check_licenses(args.res_path, maya_scenes, args.testType)
script = script.format(work_dir=work_dir,
testType=args.testType,
render_device=args.render_device,
res_path=res_path,
pass_limit=args.pass_limit,
resolution_x=args.resolution_x,
resolution_y=args.resolution_y,
SPU=args.SPU,
threshold=args.threshold)
with open(os.path.join(args.output, 'base_functions.py'), 'w') as file:
file.write(script)
if (os.path.exists(os.path.join(os.path.dirname(__file__),
args.testCases))):
with open(os.path.join(os.path.dirname(__file__),
args.testCases)) as f:
tc = f.read()
test_cases = json.loads(tc)[args.testType]
necessary_cases = [
item for item in cases if item['case'] in test_cases
]
core_config.main_logger.info('Create empty report files')
if not os.path.exists(os.path.join(work_dir, 'Color')):
os.makedirs(os.path.join(work_dir, 'Color'))
copyfile(
os.path.abspath(
os.path.join(work_dir, '..', '..', '..', '..', 'jobs_launcher',
'common', 'img', 'error.jpg')),
os.path.join(work_dir, 'Color', 'failed.jpg'))
gpu = get_gpu()
if not gpu:
core_config.main_logger.error("Can't get gpu name")
render_platform = {platform.system(), gpu}
for case in cases:
if sum([
render_platform & set(skip_conf) == set(skip_conf)
for skip_conf in case.get('skip_on', '')
]):
for i in case['skip_on']:
skip_on = set(i)
if render_platform.intersection(skip_on) == skip_on:
case['status'] = 'skipped'
if case['status'] != 'done':
if case['status'] == 'inprogress':
case['status'] = 'fail'
template = core_config.RENDER_REPORT_BASE
template['test_case'] = case['case']
template['render_device'] = get_gpu()
template['test_status'] = 'error'
template['script_info'] = case['script_info']
template['scene_name'] = case.get('scene', '')
template['file_name'] = 'failed.jpg'
template['render_color_path'] = os.path.join('Color', 'failed.jpg')
template['test_group'] = args.testType
template['date_time'] = datetime.now().strftime(
'%m/%d/%Y %H:%M:%S')
with open(
os.path.join(work_dir, case['case'] +
core_config.CASE_REPORT_SUFFIX), 'w') as f:
f.write(json.dumps([template], indent=4))
def prepare_empty_reports(args, current_conf):
main_logger.info('Create empty report files')
if args.is_inventor:
baseline_path_tr = os.path.join(
'c:/TestResources/usd_inventor_autotests_baselines', args.test_group)
else:
baseline_path_tr = os.path.join(
'c:/TestResources/usd_rprviewer_autotests_baselines', args.test_group)
baseline_path = os.path.join(
args.output, os.path.pardir, os.path.pardir, os.path.pardir, 'Baseline', args.test_group)
if not os.path.exists(baseline_path):
os.makedirs(baseline_path)
os.makedirs(os.path.join(baseline_path, 'Color'))
copyfile(os.path.abspath(os.path.join(args.output, '..', '..', '..', '..', 'jobs_launcher',
'common', 'img', 'error.jpg')), os.path.join(args.output, 'Color', 'failed.jpg'))
with open(os.path.join(os.path.abspath(args.output), "test_cases.json"), "r") as json_file:
cases = json.load(json_file)
for case in cases:
if utils.is_case_skipped(case, current_conf, args.is_inventor):
case['status'] = 'skipped'
if case['status'] != 'done' and case['status'] != 'error':
if case["status"] == 'inprogress':
case['status'] = 'active'
test_case_report = RENDER_REPORT_BASE.copy()
test_case_report["render_time"] = 0.0
test_case_report['test_case'] = case['case']
test_case_report['case_functions'] = case['functions']
test_case_report['render_device'] = get_gpu()
test_case_report['script_info'] = case['script_info']
test_case_report['scene_name'] = case.get('scene', '')
test_case_report['test_group'] = args.test_group
test_case_report['execution_time'] = 0.0
test_case_report['date_time'] = datetime.now().strftime(
'%m/%d/%Y %H:%M:%S')
test_case_report[SCREENS_PATH_KEY] = os.path.join(args.output, "Color", case["case"] + '_SCREENS')
test_case_report['tool'] = args.tool_name
test_case_report['render_version'] = os.getenv('TOOL_VERSION', default='')
if 'jira_issue' in case:
test_case_report['jira_issue'] = case['jira_issue']
if case['status'] == 'skipped':
test_case_report['test_status'] = 'skipped'
test_case_report['file_name'] = case['case'] + case.get('extension', '.jpg')
test_case_report['render_color_path'] = os.path.join('Color', test_case_report['file_name'])
test_case_report['group_timeout_exceeded'] = False
try:
skipped_case_image_path = os.path.join(args.output, 'Color', test_case_report['file_name'])
if not os.path.exists(skipped_case_image_path):
copyfile(os.path.join(args.output, '..', '..', '..', '..', 'jobs_launcher',
'common', 'img', "skipped.jpg"), skipped_case_image_path)
except OSError or FileNotFoundError as err:
main_logger.error("Can't create img stub: {}".format(str(err)))
else:
test_case_report['test_status'] = 'error'
test_case_report['file_name'] = 'failed.jpg'
test_case_report['render_color_path'] = os.path.join('Color', 'failed.jpg')
case_path = os.path.join(args.output, case['case'] + CASE_REPORT_SUFFIX)
if os.path.exists(case_path):
with open(case_path) as f:
case_json = json.load(f)[0]
test_case_report["number_of_tries"] = case_json["number_of_tries"]
with open(case_path, "w") as f:
f.write(json.dumps([test_case_report], indent=4))
copy_baselines(args, case, baseline_path, baseline_path_tr)
if __name__ == "__main__":
main_logger.info("simpleRender start working...")
args = createArgsParser().parse_args()
try:
os.makedirs(args.output)
if not os.path.exists(os.path.join(args.output, "Color")):
os.makedirs(os.path.join(args.output, "Color"))
if not os.path.exists(os.path.join(args.output, "render_tool_logs")):
os.makedirs(os.path.join(args.output, "render_tool_logs"))
if not os.path.exists(os.path.join(args.output, "execution_logs")):
os.makedirs(os.path.join(args.output, "execution_logs"))
render_device = get_gpu()
system_pl = platform.system()
current_conf = set(platform.system()) if not render_device else {platform.system(), render_device}
main_logger.info("Detected GPUs: {}".format(render_device))
main_logger.info("PC conf: {}".format(current_conf))
main_logger.info("Creating predefined errors json...")
if 'Inventor' in args.tool_name:
args.is_inventor = True
else:
args.is_inventor = False
copy_test_cases(args)
prepare_empty_reports(args, current_conf)
#TODO do not open inventor each time
exit(execute_tests(args, current_conf))
core_config.main_logger.info('Make "base_functions.py"')
try:
cases = json.load(
open(
os.path.realpath(
os.path.join(os.path.abspath(args.output),
'test_cases.json'))))
except Exception as e:
core_config.logging.error('Can\'t load test_cases.json')
core_config.main_logger.error(str(e))
group_failed(args)
exit(-1)
gpu = get_gpu()
if not gpu:
core_config.main_logger.error("Can't get gpu name")
render_platform = {platform.system(), gpu}
for case in cases:
if is_case_skipped(case, render_platform):
case['status'] = 'skipped'
else:
case['status'] = 'active'
template = {}
template['render_device'] = get_gpu()
template['render_time'] = 0
template['number_of_tries'] = 0
template['onnx'] = case['onnx']
def prepare_empty_reports(args, current_conf):
main_logger.info('Create empty report files')
copyfile(
os.path.abspath(
os.path.join(args.output, '..', '..', '..', '..', 'jobs_launcher',
'common', 'img', 'error.jpg')),
os.path.join(args.output, 'Color', 'failed.jpg'))
with open(os.path.join(os.path.abspath(args.output), "test_cases.json"),
"r") as json_file:
cases = json.load(json_file)
for case in cases:
if is_case_skipped(case, current_conf):
case['status'] = 'skipped'
if case['status'] != 'done' and case['status'] != 'error':
if case["status"] == 'inprogress':
case['status'] = 'active'
test_case_report = RENDER_REPORT_BASE.copy()
test_case_report['test_case'] = case['case']
test_case_report['render_device'] = get_gpu()
test_case_report['render_duration'] = -0.0
test_case_report['script_info'] = case['script_info']
test_case_report['geometry'] = case['geometry']
test_case_report['material_file'] = case['material_file']
test_case_report['material_path'] = case['material_path']
test_case_report['plugin'] = args.plugin
test_case_report['render_engine'] = args.plugin
test_case_report['iterations'] = int(case.get('iterations', 50))
test_case_report['test_group'] = args.test_group
test_case_report['tool'] = 'HybridVsNs'
test_case_report['date_time'] = datetime.now().strftime(
'%m/%d/%Y %H:%M:%S')
if case['status'] == 'skipped':
test_case_report['test_status'] = 'skipped'
test_case_report['file_name'] = case['case'] + case.get(
'extension', '.jpg')
test_case_report['render_color_path'] = os.path.join(
'Color', test_case_report['file_name'])
test_case_report['group_timeout_exceeded'] = False
try:
skipped_case_image_path = os.path.join(
args.output, 'Color', test_case_report['file_name'])
if not os.path.exists(skipped_case_image_path):
copyfile(
os.path.join(args.output, '..', '..', '..', '..',
'jobs_launcher', 'common', 'img',
"skipped.jpg"),
skipped_case_image_path)
except OSError or FileNotFoundError as err:
main_logger.error("Can't create img stub: {}".format(
str(err)))
else:
test_case_report['test_status'] = 'error'
test_case_report['file_name'] = 'failed.jpg'
test_case_report['render_color_path'] = os.path.join(
'Color', 'failed.jpg')
case_path = os.path.join(args.output,
case['case'] + CASE_REPORT_SUFFIX)
if os.path.exists(case_path):
with open(case_path) as f:
case_json = json.load(f)[0]
test_case_report["number_of_tries"] = case_json[
"number_of_tries"]
with open(case_path, "w") as f:
f.write(json.dumps([test_case_report], indent=4))
def main(args):
perf_count.event_record(args.output, 'Prepare tests', True)
if "Cycles" not in args.engine:
if args.testType in ['Athena']:
ATHENA_DIR = athena_disable(False, args.tool)
else:
ATHENA_DIR = athena_disable(True, args.tool)
core_config.main_logger.info('Make "base_functions.py"')
try:
cases = json.load(
open(
os.path.realpath(
os.path.join(os.path.abspath(args.output),
'test_cases.json'))))
except Exception as e:
core_config.main_logger.error("Can't load test_cases.json")
core_config.main_logger.error(str(e))
group_failed(args)
exit(-1)
try:
with open(os.path.join(os.path.dirname(__file__),
'base_functions.py')) as f:
script = f.read()
except OSError as e:
core_config.main_logger.error(str(e))
return 1
if os.path.exists(
os.path.join(os.path.dirname(__file__), 'extensions',
args.testType + '.py')):
with open(
os.path.join(os.path.dirname(__file__), 'extensions',
args.testType + '.py')) as f:
extension_script = f.read()
script = script.split('# place for extension functions')
script = script[0] + "ATHENA_DIR=\"{}\"\n".format(
ATHENA_DIR.replace('\\', '\\\\')) + extension_script + script[1]
work_dir = os.path.abspath(args.output)
script = script.format(work_dir=work_dir,
testType=args.testType,
render_device=args.render_device,
res_path=args.res_path,
pass_limit=args.pass_limit,
resolution_x=args.resolution_x,
resolution_y=args.resolution_y,
SPU=args.SPU,
threshold=args.threshold,
engine=args.engine,
retries=args.retries)
with open(os.path.join(args.output, 'base_functions.py'), 'w') as file:
file.write(script)
if os.path.exists(args.testCases) and args.testCases:
with open(args.testCases) as f:
test_cases = json.load(f)['groups'][args.testType]
if test_cases:
necessary_cases = [
item for item in cases if item['case'] in test_cases
]
cases = necessary_cases
core_config.main_logger.info('Create empty report files')
if not os.path.exists(os.path.join(work_dir, 'Color')):
os.makedirs(os.path.join(work_dir, 'Color'))
copyfile(
os.path.abspath(
os.path.join(work_dir, '..', '..', '..', '..', 'jobs_launcher',
'common', 'img', 'error.jpg')),
os.path.join(work_dir, 'Color', 'failed.jpg'))
gpu = get_gpu()
if not gpu:
core_config.main_logger.error("Can't get gpu name")
render_platform = {platform.system(), gpu}
system_pl = platform.system()
if "Cycles" not in args.engine:
baseline_dir = 'rpr_blender_autotests_baselines'
if args.engine == 'FULL2':
baseline_dir = baseline_dir + '-NorthStar'
elif args.engine == 'LOW':
baseline_dir = baseline_dir + '-HybridLow'
elif args.engine == 'MEDIUM':
baseline_dir = baseline_dir + '-HybridMedium'
elif args.engine == 'HIGH':
baseline_dir = baseline_dir + '-HybridHigh'
elif args.engine == 'HYBRIDPRO':
baseline_dir = baseline_dir + '-HybridPro'
if system_pl == "Windows":
baseline_path_tr = os.path.join('c:/TestResources', baseline_dir,
args.testType)
else:
baseline_path_tr = os.path.expandvars(
os.path.join('$CIS_TOOLS/../TestResources', baseline_dir,
args.testType))
baseline_path = os.path.join(work_dir, os.path.pardir, os.path.pardir,
os.path.pardir, 'Baseline', args.testType)
if not os.path.exists(baseline_path):
os.makedirs(baseline_path)
os.makedirs(os.path.join(baseline_path, 'Color'))
for case in cases:
if is_case_skipped(case, render_platform, args.engine):
case['status'] = 'skipped'
if case['status'] != 'done' and case['status'] != 'error':
if case["status"] == 'inprogress':
case['status'] = 'active'
case['number_of_tries'] = case.get('number_of_tries', 0) + 1
template = core_config.RENDER_REPORT_BASE.copy()
template['test_case'] = case['case']
template['case_functions'] = case['functions']
template['render_device'] = get_gpu()
template['script_info'] = case['script_info']
template['scene_name'] = case.get('scene', '')
template['test_group'] = args.testType
template['date_time'] = datetime.now().strftime(
'%m/%d/%Y %H:%M:%S')
if 'jira_issue' in case:
template['jira_issue'] = case['jira_issue']
if case['status'] == 'skipped':
template['test_status'] = 'skipped'
template['file_name'] = case['case'] + case.get(
'extension', '.jpg')
template['render_color_path'] = os.path.join(
'Color', template['file_name'])
template['group_timeout_exceeded'] = False
try:
skipped_case_image_path = os.path.join(
args.output, 'Color', template['file_name'])
if not os.path.exists(skipped_case_image_path):
copyfile(
os.path.join(work_dir, '..', '..', '..', '..',
'jobs_launcher', 'common', 'img',
"skipped.jpg"),
skipped_case_image_path)
except OSError or FileNotFoundError as err:
core_config.main_logger.error(
"Can't create img stub: {}".format(str(err)))
else:
template['test_status'] = 'error'
template['file_name'] = 'failed.jpg'
template['render_color_path'] = os.path.join(
'Color', 'failed.jpg')
case_path = os.path.join(
work_dir, case['case'] + core_config.CASE_REPORT_SUFFIX)
if os.path.exists(case_path):
with open(case_path) as f:
case_json = json.load(f)[0]
template["number_of_tries"] = case_json["number_of_tries"]
with open(case_path, 'w') as f:
f.write(json.dumps([template], indent=4))
if 'Update' not in args.update_refs:
try:
copyfile(
os.path.join(baseline_path_tr, case['case'] +
core_config.CASE_REPORT_SUFFIX),
os.path.join(baseline_path, case['case'] +
core_config.CASE_REPORT_SUFFIX))
with open(
os.path.join(
baseline_path, case['case'] +
core_config.CASE_REPORT_SUFFIX)) as baseline:
baseline_json = json.load(baseline)
for thumb in [''] + core_config.THUMBNAIL_PREFIXES:
if thumb + 'render_color_path' and os.path.exists(
os.path.join(
baseline_path_tr,
baseline_json[thumb + 'render_color_path'])):
copyfile(
os.path.join(
baseline_path_tr,
baseline_json[thumb + 'render_color_path']),
os.path.join(
baseline_path,
baseline_json[thumb + 'render_color_path']))
except:
sleep(2)
utils.open_usdviewer_tab(args, case, current_try, "materials",
screens_path)
select_part_tower(args, case, current_try, screens_path)
if "selection_delay" in case:
utils.select_material(args, case, current_try, material_name,
screens_path, material_in_row, material_row,
case["selection_delay"])
else:
utils.select_material(args, case, current_try, material_name,
screens_path, material_in_row, material_row)
# on Nvidia materials require to be loaded
if utils.viewer_reopened and "GeForce" in get_gpu():
sleep(75)
if utils.viewer_reopened:
utils.open_usdviewer_tab(args, case, current_try, "lighting",
screens_path)
utils.set_lighting(args, case, current_try, "Kloppenheim",
screens_path)
# on Nvidia materials require to be loaded
if utils.viewer_reopened and "GeForce" in get_gpu():
sleep(75)
utils.open_usdviewer_tab(args, case, current_try, "render", screens_path)
utils.render(args, case, current_try, screens_path)
utils.save_image(args, case, current_try, image_path, screens_path)
def main():
args = create_args_parser()
# TODO: remove code duplicate
# remove old images
old_images = [
x for x in os.listdir(args.render_path)
if os.path.isfile(x) and x.startswith('img0')
]
main_logger.info(os.listdir(args.render_path))
if old_images:
main_logger.info("Detected old renderer: {}".format(str(old_images)))
for img in old_images:
try:
os.remove(os.path.join(args.render_path, img))
except OSError as err:
main_logger.error(str(err))
if not os.path.exists(os.path.join(args.output_dir, "Color")):
os.makedirs(os.path.join(args.output_dir, "Color"))
try:
test_cases_path = os.path.realpath(
os.path.join(os.path.abspath(args.output_dir), 'test_cases.json'))
shutil.copyfile(args.tests_list, test_cases_path)
except:
main_logger.error("Can't copy test_cases.json")
main_logger.error(str(e))
exit(-1)
try:
with open(test_cases_path, 'r') as file:
tests_list = json.load(file)
except OSError as e:
main_logger.error("Failed to read test cases json. ")
main_logger.error(str(e))
exit(-1)
# TODO: try-catch on file reading
if not os.path.exists(
os.path.join(args.render_path, 'config.original.json')):
main_logger.info("First execution - create copy of config.json")
shutil.copyfile(os.path.join(args.render_path, 'config.json'),
os.path.join(args.render_path, 'config.original.json'))
render_device = get_gpu()
system_pl = platform.system()
current_conf = set(platform.system()) if not render_device else {
platform.system(), render_device
}
main_logger.info("Detected GPUs: {}".format(render_device))
main_logger.info("PC conf: {}".format(current_conf))
main_logger.info("Creating predefined errors json...")
if system_pl == "Windows":
baseline_path_tr = os.path.join(
'c:/TestResources/rpr_viewer_autotests_baselines', args.test_group)
else:
baseline_path_tr = os.path.expandvars(
os.path.join(
'$CIS_TOOLS/../TestResources/rpr_viewer_autotests_baselines',
args.test_group))
baseline_path = os.path.join(args.output_dir, os.path.pardir,
os.path.pardir, os.path.pardir, 'Baseline',
args.test_group)
if not os.path.exists(baseline_path):
os.makedirs(baseline_path)
os.makedirs(os.path.join(baseline_path, 'Color'))
# save pre-defined reports with error status
for test in tests_list:
# for each case create config from default
with open(os.path.join(args.render_path, 'config.original.json'),
'r') as file:
config_tmp = json.loads(file.read())
# TODO: save scene name instead of scene sub path
report = copy.deepcopy(RENDER_REPORT_BASE)
# if 'engine' exist in case.json - set it; else - engine from xml
engine = test['config_parameters'].get('engine', args.render_engine)
is_skipped = is_case_skipped(test, current_conf)
test_status = TEST_IGNORE_STATUS if is_skipped else TEST_CRASH_STATUS
main_logger.info(
"Case: {}; Engine: {}; Skip here: {}; Predefined status: {};".
format(test['case'], engine, bool(is_skipped), test_status))
report.update({
'test_status':
test_status,
'render_device':
render_device,
'test_case':
test['case'],
'scene_name':
test['scene_sub_path'],
'tool':
engine,
'file_name':
test['case'] + test['file_ext'],
'date_time':
datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S"),
'script_info':
test['script_info'],
'test_group':
args.test_group,
'render_color_path':
'Color/' + test['case'] + test['file_ext'],
'testcase_timeout':
test['render_time']
})
if 'Update' not in args.update_refs:
try:
shutil.copyfile(
os.path.join(baseline_path_tr,
test['case'] + CASE_REPORT_SUFFIX),
os.path.join(baseline_path,
test['case'] + CASE_REPORT_SUFFIX))
with open(
os.path.join(baseline_path, test['case'] +
CASE_REPORT_SUFFIX)) as baseline:
baseline_json = json.load(baseline)
for thumb in [''] + THUMBNAIL_PREFIXES:
if os.path.exists(
os.path.join(
baseline_path_tr,
baseline_json[thumb + 'render_color_path'])):
shutil.copyfile(
os.path.join(
baseline_path_tr,
baseline_json[thumb + 'render_color_path']),
os.path.join(
baseline_path,
baseline_json[thumb + 'render_color_path']))
except:
main_logger.error('Failed to copy baseline ' + os.path.join(
baseline_path_tr, test['case'] + CASE_REPORT_SUFFIX))
if test_status == TEST_IGNORE_STATUS:
report.update({'group_timeout_exceeded': False})
test['status'] = TEST_IGNORE_STATUS
try:
shutil.copyfile(
os.path.join(ROOT_DIR_PATH, 'jobs_launcher', 'common', 'img',
report['test_status'] + test['file_ext']),
os.path.join(args.output_dir, 'Color',
test['case'] + test['file_ext']))
except (OSError, FileNotFoundError) as err:
main_logger.error("Can't create img stub: {}".format(str(err)))
with open(
os.path.join(args.output_dir,
test["case"] + CASE_REPORT_SUFFIX), "w") as file:
json.dump([report], file, indent=4)
with open(test_cases_path, 'w') as file:
json.dump(tests_list, file, indent=4)
# run cases
was_success = False
case_number = 0
skip_group = False
for test in [
x for x in tests_list if x['status'] == 'active'
and not is_case_skipped(x, current_conf)
]:
case_number += 1
skip_group = skip_group or (not was_success and case_number
== args.error_until_group_failed)
if not skip_group:
main_logger.info("\nProcessing test case: {}".format(test['case']))
engine = test['config_parameters'].get('engine',
args.render_engine)
frame_ae = str(
update_viewer_config(test=test,
engine=engine,
render_path=args.render_path,
scene_path=args.scene_path,
tmp=copy.deepcopy(config_tmp)))
if frame_ae == '0':
main_logger.info(
"Case with infinity loop. Abort by timeout is expected. Will save 5th frame"
)
# remove old images
old_images = [
x for x in os.listdir(args.render_path)
if os.path.isfile(os.path.join(args.render_path, x))
and x.startswith('img0')
]
main_logger.info(os.listdir(args.render_path))
if old_images:
main_logger.info("Detected old renderer: {}".format(
str(old_images)))
for img in old_images:
try:
os.remove(os.path.join(args.render_path, img))
except OSError as err:
main_logger.error(str(err))
os.chdir(args.render_path)
if platform.system() == 'Windows':
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer.exe"))
elif platform.system() == 'Linux':
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer"))
os.system('chmod +x {}'.format(viewer_run_path))
else:
viewer_run_path = os.path.normpath(
os.path.join(args.render_path, "RadeonProViewer"))
os.system('chmod +x {}'.format(viewer_run_path))
i = 0
test_case_status = TEST_CRASH_STATUS
while i < args.retries and test_case_status == TEST_CRASH_STATUS:
main_logger.info("Try #" + str(i))
i += 1
p = psutil.Popen(viewer_run_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stderr, stdout = b"", b""
start_time = time.time()
test_case_status = TEST_CRASH_STATUS
aborted_by_timeout = False
try:
stdout, stderr = p.communicate(timeout=test['render_time'])
except (TimeoutError, psutil.TimeoutExpired,
subprocess.TimeoutExpired) as err:
main_logger.error("Aborted by timeout. {}".format(
str(err)))
# RS_CONF_IT_022 - RS_CONF_IT_028
if frame_ae == '0':
test_case_status = TEST_SUCCESS_STATUS
frame_ae = '50'
for child in reversed(p.children(recursive=True)):
child.terminate()
p.terminate()
stdout, stderr = p.communicate()
aborted_by_timeout = True
else:
test_case_status = TEST_SUCCESS_STATUS
render_time = time.time() - start_time
error_messages = []
try:
shutil.copyfile(
os.path.join(
args.render_path,
'img{0}{1}'.format(frame_ae.zfill(4),
test['file_ext'])),
os.path.join(args.output_dir, 'Color',
test['case'] + test['file_ext']))
test_case_status = TEST_SUCCESS_STATUS
except FileNotFoundError as err:
image_not_found_str = "Image {} not found".format(
'img{0}{1}'.format(frame_ae.zfill(4),
test['file_ext']))
error_messages.append(image_not_found_str)
main_logger.error(image_not_found_str)
main_logger.error(str(err))
test_case_status = TEST_CRASH_STATUS
with open(os.path.join(args.output_dir, test['case'] + '_app.log'),
'w') as file:
file.write("-----[STDOUT]------\n\n")
file.write(stdout.decode("UTF-8"))
with open(os.path.join(args.output_dir, test['case'] + '_app.log'),
'a') as file:
file.write("\n-----[STDERR]-----\n\n")
file.write(stderr.decode("UTF-8"))
# Up to date test case status
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'r') as file:
test_case_report = json.loads(file.read())[0]
if error_messages:
test_case_report["message"] = test_case_report[
"message"] + error_messages
test_case_report["test_status"] = test_case_status
test_case_report["render_time"] = render_time
test_case_report[
"render_color_path"] = "Color/" + test_case_report[
"file_name"]
test_case_report["render_log"] = test['case'] + '_app.log'
test_case_report["group_timeout_exceeded"] = False
test_case_report[
"testcase_timeout_exceeded"] = aborted_by_timeout
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'w') as file:
json.dump([test_case_report], file, indent=4)
if test_case_status == TEST_SUCCESS_STATUS:
was_success = True
else:
# Up to date test case status
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'r') as file:
test_case_report = json.loads(file.read())[0]
test_case_report["message"] = [
'Test execution was interrupted due to the first ' +
str(args.error_until_group_failed) +
' cases resulting in an error'
]
test_case_report["group_timeout_exceeded"] = False
with open(
os.path.join(args.output_dir,
test['case'] + CASE_REPORT_SUFFIX),
'w') as file:
json.dump([test_case_report], file, indent=4)
test["status"] = test_case_status
with open(test_cases_path, 'w') as file:
json.dump(tests_list, file, indent=4)
return 0
