def doImageJob(imageJob):
name = imageJob.name.replace('.frag', '')
fragFile = name + '.frag'
jsonFile = name + '.json'
png = 'image.png'
log = 'log.txt'
res = tt.ImageJobResult()
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
writeToFile(imageJob.fragmentSource, fragFile)
writeToFile(imageJob.uniformsInfo, jsonFile)
remove(png)
remove(log)
getimageResult = getImageVulkanAndroid(fragFile)
# Try to get our own log file in any case
if os.path.exists('log.txt'):
res.log += '\n#### LOG START\n'
with open(log, 'r') as f:
res.log += f.read()
res.log += '\n#### LOG END\n'
# Always add ADB logcat
adb('logcat -b crash -b system -d > logcat.txt')
res.log += '\n#### ADB LOGCAT START\n'
with open('logcat.txt', 'r') as f:
res.log += f.read()
res.log += '\n#### ADB LOGCAT END\n'
if getimageResult == 'crash':
res.status = tt.JobStatus.CRASH
elif getimageResult == 'timeout':
res.status = tt.JobStatus.TIMEOUT
else:
assert (getimageResult == 'success')
res.status = tt.JobStatus.SUCCESS
with open(png, 'rb') as f:
res.PNG = f.read()
return res
def do_image_job(args, image_job):
name = image_job.name
if name.endswith('.frag'):
name = remove_end(name, '.frag')
frag_file = name + '.frag'
json_file = name + '.json'
png = 'image_0.png'
log = 'vklog.txt'
frag_spv_file = name + '.frag.spv'
vert_spv_file = name + '.vert.spv'
res = tt.ImageJobResult()
skip_render = image_job.skipRender
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
write_to_file(image_job.fragmentSource, frag_file)
write_to_file(image_job.uniformsInfo, json_file)
prepare_shaders(frag_file, frag_spv_file, vert_spv_file)
# Optimize
if args.spirvopt:
frag_spv_file_opt = frag_spv_file + '.opt'
cmd = os.path.dirname(HERE) + '/../../bin/' + get_bin_type() + '/spirv-opt ' + \
args.spirvopt + ' ' + frag_spv_file + ' -o ' + frag_spv_file_opt
try:
res.log += 'spirv-opt flags: ' + args.spirvopt + '\n'
print('Calling spirv-opt with flags: ' + args.spirvopt)
subprocess.run(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
timeout=TIMEOUT_SPIRV_OPT)
except subprocess.CalledProcessError as err:
# spirv-opt failed, early return
res.log += 'Error triggered by spirv-opt\n'
res.log += 'COMMAND:\n' + err.cmd + '\n'
res.log += 'RETURNCODE: ' + str(err.returncode) + '\n'
if err.stdout:
res.log += 'STDOUT:\n' + err.stdout + '\n'
if err.stderr:
res.log += 'STDERR:\n' + err.stderr + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
except subprocess.TimeoutExpired as err:
# spirv-opt timed out, early return
res.log += 'Timeout from spirv-opt\n'
res.log += 'COMMAND:\n' + err.cmd + '\n'
res.log += 'TIMEOUT: ' + str(err.timeout) + ' sec\n'
if err.stdout:
res.log += 'STDOUT:\n' + err.stdout + '\n'
if err.stderr:
res.log += 'STDERR:\n' + err.stderr + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
frag_spv_file = frag_spv_file_opt
remove(png)
remove(log)
if args.linux:
vkrun.run_linux(vert_spv_file, frag_spv_file, json_file, skip_render)
else:
wait_for_screen = not args.force
vkrun.run_android(vert_spv_file, frag_spv_file, json_file, skip_render,
wait_for_screen)
if os.path.isfile(log):
with open(log, 'r', encoding='utf-8', errors='ignore') as f:
res.log += f.read()
if os.path.isfile(png):
with open(png, 'rb') as f:
res.PNG = f.read()
if os.path.isfile('STATUS'):
with open('STATUS', 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
elif status == 'SANITY_ERROR':
res.status = tt.JobStatus.SANITY_ERROR
elif status == 'UNEXPECTED_ERROR':
res.status = tt.JobStatus.UNEXPECTED_ERROR
elif status == 'NONDET':
res.status = tt.JobStatus.NONDET
with open('nondet0.png', 'rb') as f:
res.PNG = f.read()
with open('nondet1.png', 'rb') as f:
res.PNG2 = f.read()
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
def main():
description = (
'Uses the piglit GLES3 shader runner to render shader jobs.')
parser = argparse.ArgumentParser(description=description)
# Required
parser.add_argument('worker_name',
help='The name that will refer to this worker.')
# Optional
parser.add_argument(
'--server',
default='http://localhost:8080',
help='Server URL to connect to (default: http://localhost:8080 )')
args = parser.parse_args()
gfuzz_common.tool_on_path('shader_runner_gles3')
gfuzz_common.log('Worker: ' + args.worker_name)
server = args.server + '/request'
gfuzz_common.log('server: ' + server)
# Get worker info
worker_info_json_string = '{}'
gfuzz_common.log('Dumping glxinfo to file for worker info string...')
try:
dump_glxinfo(WORKER_INFO_FILE)
with gfuzz_common.open_helper(WORKER_INFO_FILE, 'r') as info_file:
worker_info_json_string = info_file.read()
except Exception as ex:
gfuzz_common.log(str(ex))
gfuzz_common.log('Could not get worker info, continuing without it.')
service = None
worker = None
while True:
if not service:
gfuzz_common.log('Connecting to server...')
service, worker = thrift_connect(server, args.worker_name,
worker_info_json_string)
if not service:
gfuzz_common.log('Failed to connect, retrying...')
time.sleep(1)
continue
assert worker
os.makedirs(worker, exist_ok=True)
try:
job = service.getJob(worker)
if job.noJob is not None:
gfuzz_common.log("No job")
elif job.skipJob is not None:
gfuzz_common.log("Skip job")
service.jobDone(worker, job)
else:
assert job.imageJob
if job.imageJob.computeSource:
gfuzz_common.log("Got a compute job, but this worker "
"doesn't support compute shaders.")
job.imageJob.result = tt.ImageJobResult()
job.imageJob.result.status = tt.JobStatus.UNEXPECTED_ERROR
else:
gfuzz_common.log("#### Image job: " + job.imageJob.name)
job.imageJob.result = do_image_job(job.imageJob,
work_dir=worker)
gfuzz_common.log("Sending back, results status: {}".format(
job.imageJob.result.status))
service.jobDone(worker, job)
gfuzz_common.remove(worker)
continue
except (TApplicationException, ConnectionError):
gfuzz_common.log(
"Connection to server lost. Re-initialising client.")
service = None
time.sleep(1)
def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult:
"""
Does an image job. Sets up directories and some files, then delegates to run_image_job to
convert the job to a shader_test and run it. Sets a global logfile to log to for the lifetime
of the function. Gets the status of the shader job from a file that is written to by
run_image_job.
:param image_job: the image job containing the shader/uniforms.
:param work_dir: the directory to work in.
:return: the result of the image job, including the log, PNG and status.
"""
# Output directory is based on the name of job.
output_dir = os.path.join(work_dir, image_job.name)
# Delete and create output directory.
gfuzz_common.remove(output_dir)
os.makedirs(output_dir, exist_ok=True)
name = image_job.name
if name.endswith('.frag'):
name = gfuzz_common.remove_end(name, '.frag')
frag_file = os.path.join(output_dir, name + FRAG_SUFFIX)
json_file = os.path.join(output_dir, name + JSON_SUFFIX)
log_file = os.path.join(output_dir, LOGFILE_NAME)
status_file = os.path.join(output_dir, STATUS_FILENAME)
png_file = os.path.join(output_dir, name + PNG_SUFFIX)
nondet_0 = os.path.join(output_dir, NONDET0_PNG)
nondet_1 = os.path.join(output_dir, NONDET1_PNG)
gfuzz_common.write_to_file(image_job.fragmentSource, frag_file)
gfuzz_common.write_to_file(image_job.uniformsInfo, json_file)
res = tt.ImageJobResult()
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
with gfuzz_common.open_helper(log_file, 'w') as f:
try:
gfuzz_common.set_logfile(f)
run_image_job(json_file, status_file, png_file, output_dir,
image_job.skipRender)
except Exception as ex:
gfuzz_common.log(str(ex))
gfuzz_common.log('Removing status file and continuing...')
gfuzz_common.remove(status_file)
finally:
gfuzz_common.unset_logfile()
if os.path.isfile(log_file):
with gfuzz_common.open_helper(log_file, 'r') as f:
res.log += f.read()
if os.path.isfile(png_file):
with gfuzz_common.open_bin_helper(png_file, 'rb') as f:
res.PNG = f.read()
if os.path.isfile(status_file):
with gfuzz_common.open_helper(status_file, 'r') as f:
status = f.read().rstrip()
if status == STATUS_SUCCESS:
res.status = tt.JobStatus.SUCCESS
elif status == STATUS_CRASH:
res.status = tt.JobStatus.CRASH
elif status == STATUS_TIMEOUT:
res.status = tt.JobStatus.TIMEOUT
elif status == STATUS_UNEXPECTED:
res.status = tt.JobStatus.UNEXPECTED_ERROR
elif status == STATUS_NONDET:
res.status = tt.JobStatus.NONDET
with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f:
res.PNG = f.read()
with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f:
res.PNG2 = f.read()
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
def do_compute_job(
args,
comp_job: tt.ImageJob,
spirv_opt_args: Optional[List[str]],
work_dir: str
) -> tt.ImageJobResult:
# Output directory is based on the name of job.
output_dir = os.path.join(work_dir, comp_job.name)
# Delete and create output directory.
gfuzz_common.remove(output_dir)
os.makedirs(output_dir, exist_ok=True)
tmpcomp = os.path.join(output_dir, 'tmp.comp')
tmpjson = os.path.join(output_dir, 'tmp.json')
log_file = os.path.join(output_dir, runspv.LOGFILE_NAME)
ssbo_json_file = os.path.join(output_dir, 'ssbo.json')
# Output files from running the app.
status_file = os.path.join(output_dir, 'STATUS')
gfuzz_common.write_to_file(comp_job.computeSource, tmpcomp)
gfuzz_common.write_to_file(comp_job.computeInfo, tmpjson)
res = tt.ImageJobResult()
res.log = '#### Start compute shader\n\n'
assert not args.legacy_worker
# Set runspv logger. Use try-finally to clean up.
with gfuzz_common.open_helper(log_file, 'w') as f:
try:
runspv.log_to_file = f
runspv.run_compute_amber(
comp_original=tmpcomp,
json_file=tmpjson,
output_dir=output_dir,
force=args.force,
is_android=(args.target == 'android'),
skip_render=comp_job.skipRender,
spirv_opt_args=spirv_opt_args
)
except Exception as ex:
runspv.log('Exception: ' + str(ex))
runspv.log('Removing STATUS file.')
gfuzz_common.remove(status_file)
runspv.log('Continuing.')
finally:
runspv.log_to_file = None
if os.path.isfile(log_file):
with gfuzz_common.open_helper(log_file, 'r') as f:
res.log += f.read()
if os.path.isfile(status_file):
with gfuzz_common.open_helper(status_file, 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
assert (os.path.isfile(ssbo_json_file))
with gfuzz_common.open_helper(ssbo_json_file, 'r') as f:
res.computeOutputs = f.read()
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
def do_image_job(
args,
image_job,
spirv_opt_args: Optional[List[str]],
work_dir: str
) -> tt.ImageJobResult:
# Output directory is based on the name of job.
output_dir = os.path.join(work_dir, image_job.name)
# Delete and create output directory.
gfuzz_common.remove(output_dir)
os.makedirs(output_dir, exist_ok=True)
name = image_job.name
if name.endswith('.frag'):
name = gfuzz_common.remove_end(name, '.frag')
# TODO(324): the worker currently assumes that no vertex shader is present in the image job.
vert_file = prepare_vert_file(output_dir) if args.legacy_worker else None
frag_file = os.path.join(output_dir, name + '.frag')
json_file = os.path.join(output_dir, name + '.json')
png_file = os.path.join(output_dir, 'image_0.png')
log_file = os.path.join(output_dir, runspv.LOGFILE_NAME)
status_file = os.path.join(output_dir, 'STATUS')
nondet_0 = os.path.join(output_dir, 'nondet0.png')
nondet_1 = os.path.join(output_dir, 'nondet1.png')
res = tt.ImageJobResult()
skip_render = image_job.skipRender
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
gfuzz_common.write_to_file(image_job.fragmentSource, frag_file)
gfuzz_common.write_to_file(image_job.uniformsInfo, json_file)
# Set runspv logger. Use try-finally to clean up.
with gfuzz_common.open_helper(log_file, 'w') as f:
try:
runspv.log_to_file = f
resolved_spirvopt_args = resolve_spirvopt_args(spirv_opt_args)
if args.legacy_worker:
if args.target == 'host':
runspv.run_image_host_legacy(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
skip_render=skip_render,
spirv_opt_args=resolved_spirvopt_args,
)
else:
assert args.target == 'android'
runspv.run_image_android_legacy(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
force=args.force,
skip_render=skip_render,
spirv_opt_args=resolved_spirvopt_args,
)
else:
runspv.run_image_amber(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
force=args.force,
is_android=(args.target == 'android'),
skip_render=skip_render,
spirv_opt_args=resolved_spirvopt_args
)
except Exception as ex:
runspv.log('Exception: ' + str(ex))
runspv.log('Removing STATUS file.')
gfuzz_common.remove(status_file)
runspv.log('Continuing.')
finally:
runspv.log_to_file = None
if os.path.isfile(log_file):
with gfuzz_common.open_helper(log_file, 'r') as f:
res.log += f.read()
if os.path.isfile(png_file):
with gfuzz_common.open_bin_helper(png_file, 'rb') as f:
res.PNG = f.read()
if os.path.isfile(status_file):
with gfuzz_common.open_helper(status_file, 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
elif status == 'SANITY_ERROR':
res.status = tt.JobStatus.SANITY_ERROR
elif status == 'UNEXPECTED_ERROR':
res.status = tt.JobStatus.UNEXPECTED_ERROR
elif status == 'NONDET':
res.status = tt.JobStatus.NONDET
with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f:
res.PNG = f.read()
with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f:
res.PNG2 = f.read()
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
def doImageJob(imageJob):
name = imageJob.name.replace('.frag', '')
fragFile = name + '.frag'
jsonFile = name + '.json'
png = name + '.png'
log = name + '.log'
qpa = name + '.qpa'
vk = name + '.vk'
writeToFile(imageJob.fragmentSource, fragFile)
writeToFile(imageJob.uniformsInfo, jsonFile)
if os.path.isfile(png):
os.remove(png)
# FIXME: translate runVk.sh as a python function in this file
# cmd = '../runVk.sh ' + fragFile
# runVkCompleted = False
for i in range(2):
getimage = getImageVulkanAndroid(fragFile)
if (getimage):
break
else:
print('get image failed, restart dEQP')
# Hugues: do not use the "-W" (wait) adb flag here, it hangs
adb('shell am force-stop com.drawelements.deqp')
time.sleep(0.5)
adb('shell am start -n com.drawelements.deqp/.execserver.ServiceStarter'
)
print('Sleep and retry...')
time.sleep(0.5)
res = tt.ImageJobResult()
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
if not (os.path.isfile(qpa)):
# if no QPA, something went really wrong
res.status = tt.JobStatus.UNEXPECTED_ERROR
res.log = 'Not even a QPA file found?'
else:
# Store QPA as log
with open(qpa, 'r') as f:
res.log = f.read()
# Vulkan calls trace
if os.path.isfile(vk):
res.log += '\n#### VK CALLS TRACE START\n'
with open(vk) as f:
res.log += f.read()
res.log += '#### VK CALLS TRACE END\n'
# Set status
if os.path.isfile(png):
res.status = tt.JobStatus.SUCCESS
with open(png, 'rb') as f:
res.PNG = f.read()
elif fileContains(qpa, '#endTestCaseResult'):
res.status = tt.JobStatus.COMPILE_ERROR
else:
res.status = tt.JobStatus.CRASH
# adb log
adb('logcat -b crash -b system -d > logcat.txt')
res.log += '\n#### ADB LOGCAT START\n'
with open('logcat.txt', 'r') as f:
res.log += f.read()
res.log += '\n#### ADB LOGCAT END\n'
return res
def doImageJob(args, imageJob):
name = imageJob.name.replace('.frag', '')
fragFile = name + '.frag'
jsonFile = name + '.json'
png = 'image.png'
log = 'vklog.txt'
res = tt.ImageJobResult()
skipRender = imageJob.skipRender
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
writeToFile(imageJob.fragmentSource, fragFile)
writeToFile(imageJob.uniformsInfo, jsonFile)
prepareShaders(fragFile)
shutil.copy(jsonFile, 'test.json')
# Optimize
if args.spirvopt:
cmd = os.path.dirname(HERE) + '/../../bin/' + getBinType(
) + '/spirv-opt ' + args.spirvopt + ' test.frag.spv -o test.frag.spv.opt'
try:
res.log += 'spirv-opt flags: ' + args.spirvopt + '\n'
print('Calling spirv-opt with flags: ' + args.spirvopt)
subprocess.run(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
timeout=TIMEOUT_SPIRVOPT)
except subprocess.CalledProcessError as err:
# spirv-opt failed, early return
res.log += 'Error triggered by spirv-opt\n'
res.log += 'COMMAND:\n' + err.cmd + '\n'
res.log += 'RETURNCODE: ' + str(err.returncode) + '\n'
if err.stdout:
res.log += 'STDOUT:\n' + err.stdout + '\n'
if err.stderr:
res.log += 'STDERR:\n' + err.stderr + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
except subprocess.TimeoutExpired as err:
# spirv-opt timed out, early return
res.log += 'Timeout from spirv-opt\n'
res.log += 'COMMAND:\n' + err.cmd + '\n'
res.log += 'TIMEOUT: ' + str(err.timeout) + ' sec\n'
if err.stdout:
res.log += 'STDOUT:\n' + err.stdout + '\n'
if err.stderr:
res.log += 'STDERR:\n' + err.stderr + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
shutil.move('test.frag.spv.opt', 'test.frag.spv')
remove(png)
remove(log)
if args.linux:
vkrun.run_linux('test.vert.spv', 'test.frag.spv', 'test.json',
skipRender)
else:
vkrun.run_android('test.vert.spv', 'test.frag.spv', 'test.json',
skipRender)
if os.path.exists(log):
with open(log, 'r') as f:
res.log += f.read()
if os.path.exists(png):
with open(png, 'rb') as f:
res.PNG = f.read()
if os.path.exists('STATUS'):
with open('STATUS', 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res