def open_helper(file: str, mode: str):
"""
Helper function to open a file with UTF-8 format, ignoring errors.
Wraps around runspv.open_helper(), which wraps open() in turn.
:param file: the file to open - same as the file argument of open().
:param mode: the mode to open the file in - same as the mode argument of open().
:return: a File object opened in UTF-8 format and ignoring errors.
"""
return runspv.open_helper(file, mode)
def main_helper(args) -> None:
"""
Turn a GraphicsFuzz .json file into a spirv-fuzz .facts file, with one fact per word of uniform
data.
:param args: command-line arguments
:return: None
"""
parser = argparse.ArgumentParser()
# Required arguments
parser.add_argument('shader_job', help='Shader job (.json) file.')
parser.add_argument('output_file', help='Output file for facts.')
args = parser.parse_args(args)
# Generate uniform facts from .json file
fact_list = [] # type: List[dict]
# Turn the information about uniform values into facts for spirv-fuzz.
with runspv.open_helper(args.shader_job, 'r') as f:
j = json.load(f) # type: Dict[str, Any]
for name, entry in j.items():
# Skip compute shader data
if name == "$compute":
continue
scalar_uniform_functions = ['glUniform1i', 'glUniform1f']
vector_uniform_functions = [
'glUniform2i', 'glUniform3i', 'glUniform4i', 'glUniform2f',
'glUniform3f', 'glUniform4f'
]
if not (entry["func"] in scalar_uniform_functions
or entry["func"] in vector_uniform_functions):
print("Ignoring unsupported uniform function " + entry["func"])
continue
# Make a separate fact for every component value of each uniform.
for i in range(0, len(entry["args"])):
# Every uniform is in its own struct, so we index into the first element of that struct
# using index 0. If the uniform is a vector, we need to further index into the vector.
if entry["func"] in scalar_uniform_functions:
# The uniform is a scalar.
assert i == 0
index_list = [0]
else:
assert entry["func"] in vector_uniform_functions
# The uniform is a vector, so we have two levels of indexing.
index_list = [0, i]
# We need to pass the component value as an integer. If it is a float, we need to
# reinterpret its bits as an integer.
int_representation = entry["args"][i]
if isinstance(int_representation, float):
int_representation = struct.unpack(
'<I', struct.pack('<f', entry["args"][i]))[0]
uniform_buffer_element_descriptor = dict(descriptorSet=0,
binding=entry["binding"],
index=index_list)
fact_constant_uniform = dict(uniformBufferElementDescriptor=
uniform_buffer_element_descriptor,
constantWord=[int_representation])
fact_list.append(dict(constantUniformFact=fact_constant_uniform))
with runspv.open_helper(args.output_file, 'w') as f:
f.write(json.dumps(dict(fact=fact_list), indent=1, sort_keys=True))
def write_to_file(content, filename):
with runspv.open_helper(filename, 'w') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser()
# Required arguments
parser.add_argument(
'worker',
help='Worker name to identify to the server')
parser.add_argument(
'target',
help=runspv.TARGET_HELP)
# Optional arguments
parser.add_argument(
'--force',
action='store_true',
help=runspv.FORCE_OPTION_HELP)
parser.add_argument(
'--legacy-worker',
action='store_true',
help=runspv.LEGACY_OPTION_HELP)
parser.add_argument(
'--serial',
help=runspv.SERIAL_OPTION_HELP)
parser.add_argument(
'--server',
default='http://localhost:8080',
help='Server URL (default: http://localhost:8080 )')
parser.add_argument(
'--spirvopt',
help=runspv.SPIRV_OPT_OPTION_HELP)
parser.add_argument(
'--local-shader-job',
help='Execute a single, locally stored shader job (for debugging), instead of using the '
'server.')
args = parser.parse_args()
spirv_args = None # type: Optional[List[str]]
if args.spirvopt:
spirv_args = args.spirvopt.split()
# Check the target is known.
if not (args.target == 'android' or args.target == 'host'):
raise ValueError('Target must be "android" or "host"')
# Record whether or not we are targeting Android.
is_android = (args.target == 'android')
# Check the optional arguments are consistent with the target.
if not is_android and args.force:
raise ValueError('"force" option is only compatible with "android" target')
if not is_android and args.serial:
raise ValueError('"serial" option is only compatible with "android" target')
print('Worker: ' + args.worker)
server = args.server + '/request'
print('server: ' + server)
if args.serial:
os.environ['ANDROID_SERIAL'] = args.serial
service = None
worker = None
# Get worker info
worker_info_file = 'worker_info.json'
remove(worker_info_file)
worker_info_json_string = '{}'
try:
if is_android:
runspv.dump_info_android_legacy(wait_for_screen=not args.force)
else:
runspv.dump_info_host_legacy()
if not os.path.isfile(worker_info_file):
raise Exception(
'Failed to retrieve worker information. If targeting Android, make sure '
'the app permission to write to external storage is enabled.'
)
with runspv.open_helper(worker_info_file, 'r') as f:
worker_info_json_string = f.read()
except Exception as ex:
if args.legacy_worker:
raise ex
else:
print(ex)
print('Continuing without worker information.')
# Main loop
while True:
if is_android \
and 'ANDROID_SERIAL' in os.environ and \
not is_device_available(os.environ['ANDROID_SERIAL']):
raise Exception(
'#### ABORT: device {} is not available (either offline or not connected?)'
.format(os.environ['ANDROID_SERIAL'])
)
# Special case: local shader job for debugging.
if args.local_shader_job:
assert args.local_shader_job.endswith('.json'), \
'Expected local shader job "{}" to end with .json'
shader_job_prefix = remove_end(args.local_shader_job, '.json')
fake_job = tt.ImageJob()
fake_job.name = os.path.basename(shader_job_prefix)
assert os.path.isfile(args.local_shader_job), \
'Shader job {} does not exist'.format(args.local_shader_job)
with runspv.open_helper(args.local_shader_job) as f:
fake_job.uniformsInfo = f.read()
if os.path.isfile(shader_job_prefix + '.frag'):
with runspv.open_helper(shader_job_prefix + '.frag', 'r') as f:
fake_job.fragmentSource = f.read()
if os.path.isfile(shader_job_prefix + '.vert'):
with runspv.open_helper(shader_job_prefix + '.vert', 'r') as f:
fake_job.vertexSource = f.read()
if os.path.isfile(shader_job_prefix + '.comp'):
with runspv.open_helper(shader_job_prefix + '.comp', 'r') as f:
fake_job.computeSource = f.read()
fake_job.computeInfo = fake_job.uniformsInfo
do_image_job(args, fake_job, spirv_args, work_dir='out')
return
if not service:
service, worker = get_service(server, args, worker_info_json_string)
if not service:
print("Cannot connect to server, retry in a second...")
time.sleep(1)
continue
assert worker is not None
os.makedirs(worker, exist_ok=True)
try:
job = service.getJob(worker)
if job.noJob is not None:
print("No job")
elif job.skipJob is not None:
print("Skip job")
service.jobDone(worker, job)
else:
assert job.imageJob is not None
if job.imageJob.computeSource:
print("#### Compute job: " + job.imageJob.name)
job.imageJob.result = do_compute_job(
args,
job.imageJob,
spirv_args,
work_dir=worker
)
else:
print("#### Image job: " + job.imageJob.name)
job.imageJob.result = do_image_job(
args,
job.imageJob,
spirv_args,
work_dir=worker
)
print("Send back, results status: {}".format(job.imageJob.result.status))
service.jobDone(worker, job)
continue
except (TApplicationException, ConnectionError):
print("Connection to server lost. Re-initialising client.")
service = None
time.sleep(1)
def do_compute_job(
args,
comp_job: tt.ImageJob,
spirv_opt_args: Optional[List[str]],
work_dir: str
) -> tt.ImageJobResult:
# Output directory is based on the name of job.
output_dir = os.path.join(work_dir, comp_job.name)
# Delete and create output directory.
remove(output_dir)
os.makedirs(output_dir, exist_ok=True)
tmpcomp = os.path.join(output_dir, 'tmp.comp')
tmpjson = os.path.join(output_dir, 'tmp.json')
log_file = os.path.join(output_dir, runspv.LOGFILE_NAME)
ssbo_json_file = os.path.join(output_dir, 'ssbo.json')
# Output files from running the app.
status_file = os.path.join(output_dir, 'STATUS')
write_to_file(comp_job.computeSource, tmpcomp)
write_to_file(comp_job.computeInfo, tmpjson)
res = tt.ImageJobResult()
res.log = '#### Start compute shader\n\n'
assert not args.legacy_worker
# Set runspv logger. Use try-finally to clean up.
with runspv.open_helper(log_file, 'w') as f:
try:
runspv.log_to_file = f
runspv.run_compute_amber(
comp_original=tmpcomp,
json_file=tmpjson,
output_dir=output_dir,
force=args.force,
is_android=(args.target == 'android'),
skip_render=comp_job.skipRender,
spirv_opt_args=spirv_opt_args,
)
except Exception as ex:
runspv.log('Exception: ' + str(ex))
runspv.log('Removing STATUS file.')
remove(status_file)
runspv.log('Continuing.')
finally:
runspv.log_to_file = None
if os.path.isfile(log_file):
with runspv.open_helper(log_file, 'r') as f:
res.log += f.read()
if os.path.isfile(status_file):
with runspv.open_helper(status_file, 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
assert (os.path.isfile(ssbo_json_file))
with runspv.open_helper(ssbo_json_file, 'r') as f:
res.computeOutputs = f.read()
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res
def do_image_job(
args,
image_job,
spirv_opt_args: Optional[List[str]],
work_dir: str
) -> tt.ImageJobResult:
# Output directory is based on the name of job.
output_dir = os.path.join(work_dir, image_job.name)
# Delete and create output directory.
remove(output_dir)
os.makedirs(output_dir, exist_ok=True)
name = image_job.name
if name.endswith('.frag'):
name = remove_end(name, '.frag')
# TODO(324): the worker currently assumes that no vertex shader is present in the image job.
vert_file = prepare_vert_file(output_dir) if args.legacy_worker else None
frag_file = os.path.join(output_dir, name + '.frag')
json_file = os.path.join(output_dir, name + '.json')
png_file = os.path.join(output_dir, 'image_0.png')
log_file = os.path.join(output_dir, runspv.LOGFILE_NAME)
status_file = os.path.join(output_dir, 'STATUS')
nondet_0 = os.path.join(output_dir, 'nondet0.png')
nondet_1 = os.path.join(output_dir, 'nondet1.png')
res = tt.ImageJobResult()
skip_render = image_job.skipRender
# Set nice defaults to fields we will not update anyway
res.passSanityCheck = True
res.log = 'Start: ' + name + '\n'
write_to_file(image_job.fragmentSource, frag_file)
write_to_file(image_job.uniformsInfo, json_file)
# Set runspv logger. Use try-finally to clean up.
with runspv.open_helper(log_file, 'w') as f:
try:
runspv.log_to_file = f
if args.legacy_worker:
if args.target == 'host':
runspv.run_image_host_legacy(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
skip_render=skip_render,
spirv_opt_args=spirv_opt_args,
)
else:
assert args.target == 'android'
runspv.run_image_android_legacy(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
force=args.force,
skip_render=skip_render,
spirv_opt_args=spirv_opt_args,
)
else:
runspv.run_image_amber(
vert_original=vert_file,
frag_original=frag_file,
json_file=json_file,
output_dir=output_dir,
force=args.force,
is_android=(args.target == 'android'),
skip_render=skip_render,
spirv_opt_args=spirv_opt_args,
)
except Exception as ex:
runspv.log('Exception: ' + str(ex))
runspv.log('Removing STATUS file.')
remove(status_file)
runspv.log('Continuing.')
finally:
runspv.log_to_file = None
if os.path.isfile(log_file):
with runspv.open_helper(log_file, 'r') as f:
res.log += f.read()
if os.path.isfile(png_file):
with runspv.open_bin_helper(png_file, 'rb') as f:
res.PNG = f.read()
if os.path.isfile(status_file):
with runspv.open_helper(status_file, 'r') as f:
status = f.read().rstrip()
if status == 'SUCCESS':
res.status = tt.JobStatus.SUCCESS
elif status == 'CRASH':
res.status = tt.JobStatus.CRASH
elif status == 'TIMEOUT':
res.status = tt.JobStatus.TIMEOUT
elif status == 'SANITY_ERROR':
res.status = tt.JobStatus.SANITY_ERROR
elif status == 'UNEXPECTED_ERROR':
res.status = tt.JobStatus.UNEXPECTED_ERROR
elif status == 'NONDET':
res.status = tt.JobStatus.NONDET
with runspv.open_bin_helper(nondet_0, 'rb') as f:
res.PNG = f.read()
with runspv.open_bin_helper(nondet_1, 'rb') as f:
res.PNG2 = f.read()
else:
res.log += '\nUnknown status value: ' + status + '\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
else:
# Not even a status file?
res.log += '\nNo STATUS file\n'
res.status = tt.JobStatus.UNEXPECTED_ERROR
return res