def timeout(process, callback):
for (timeout, complaint) in config.timeouts():
try:
return_code = process.wait(timeout)
if return_code != 0:
return "ERROR: compressor returned status {0}".format(return_code)
else:
return callback()
except subprocess.TimeoutExpired:
print(complaint)
# timed out
p.kill()
return float('inf')
def my_compressor(fname, paranoia, base, algorithms):
try:
def error_str(suffix):
return 'multi_compressor: {0} with {1} (paranoia={2}) on {3}: {4}'\
.format(base, algorithms, paranoia, fname, suffix)
if paranoia:
# Slow but makes sure the results are valid.
# Uses the standard Compressor interface to compress the file then decompress it,
# and verifies the decompressed file is the same as the original
compressor = build_my_compressor(base, algorithms)
return compressed_filesize(compressor, fname, paranoia)
else:
# Uses the MultiCompressor interface with the measure command.
# Runs commands in the same JVM, one after the other, and doesn't write any files.
# Faster, but doesn't perform any verification checks.
global multi_compressor
if not multi_compressor:
multi_compressor = run_multicompressor()
cmd = 'measure {0} '.format(corpus_path(fname))
cmd += ' '.join(my_compressor_end_args(base, algorithms)) + '\n'
try:
multi_compressor.stdin.write(cmd)
except BrokenPipeError:
print("WARNING: MultiCompressor has quit, restarting.")
multi_compressor.kill() # make sure it's dead
multi_compressor = run_multicompressor()
# if this still fails, propagate the exception (only retry once)
multi_compressor.stdin.write(cmd)
for (timeout, complaint) in config.timeouts():
ready_read, _, _ = select.select([multi_compressor.stdout, multi_compressor.stderr], [], [],
timeout)
if not ready_read:
print(error_str(complaint))
if multi_compressor.stderr in ready_read:
for line in multi_compressor.stderr:
print(error_str("received error from MultiCompressor: " + line.strip()))
if multi_compressor.stdout in ready_read:
out = multi_compressor.stdout.readline().strip()
prefix = 'BITS WRITTEN: '
if out.find(prefix) != 0:
raise RuntimeError(error_str("unexpected output: '" + out + "'"))
bits = int(out[len(prefix):])
return bits / 8 # compressed filesize in bytes
multi_compressor.kill()
multi_compressor = None
return float('inf')
except Exception:
print(error_str("error executing task: " + traceback.format_exc()))
return float('inf')
