def check_result(suite_data, results_queue):
previous_unfinished_count = None
while True:
suite_data_unfinished = get_unfinished_runs(suite_data)
if len(suite_data_unfinished) == 0:
break
try:
item = results_queue.get(False)
except queue.Empty:
unfinished_count = len(suite_data_unfinished)
if previous_unfinished_count != unfinished_count:
logging.info(f"Waiting for {unfinished_count} unfinished runs:")
for run_id, test_data in suite_data_unfinished.items():
logging.info(f" test case - {test_data['index']}, dag_id - {test_data['dag_id']}, run_id - {run_id}")
previous_unfinished_count = unfinished_count
sleep(10) # sleep for 10 second before trying to fetch new results from the empty queue
continue
run_id = item["run_id"]
test_data = suite_data_unfinished[run_id] # if this fails, look for a bug
logging.info(f"Check results from the test case {test_data['index']} that runs DAG {test_data['dag_id']} as {run_id}")
try:
compare(test_data["output"], item["results"])
except (CompareFail, KeyError) as ex: # catch KeyError in case output field is missing for tool that should fail
if not test_data.get("should_fail", None): # do not report error if tool should fail
test_data["error"] = str(ex)
finally:
test_data["finished"] = True
rmtree(test_data["job"]["outputs_folder"])
def test_compare_contents_success(self):
expected = {
"location": "cores.txt",
"size": 2,
"class": "File",
"contents": "2\n"
}
actual = {
"basename": "cores.txt",
"checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
"class": "File",
"location":
"file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
"path": get_data("tests/test-data/cores.txt"),
"size": 2
}
compare(expected, actual)
def check_result(suite_data, results_queue):
while True:
suite_data_unfinished = get_unfinished_runs(suite_data)
if len(suite_data_unfinished) == 0:
break
try:
item = results_queue.get()
except queue.Empty:
sleep(3) # sleep for 3 second before trying to fetch new results from the empty queue
continue
run_id = item["run_id"]
test_data = suite_data_unfinished[run_id] # if this fails, look for a bug
logging.info(f"Check results from the test case {test_data['index']} that runs DAG {test_data['dag_id']} as {run_id}")
try:
compare(test_data["output"], item["results"])
except CompareFail as ex:
test_data["error"] = str(ex)
finally:
test_data["finished"] = True
rmtree(test_data["job"]["outputs_folder"])
def evaluate_result(data):
processed = 0
while processed < len(data):
try:
item = RESULTS_QUEUE.get()
except queue.Empty:
continue
processed = processed + 1
try:
compare(data[item["run_id"]]["output"], item["results"])
logging.error(f"""\nSuccess {item["dag_id"]}: {item["run_id"]}""")
except CompareFail as ex:
data[item["run_id"]]["error"] = str(ex)
logging.error(f"""\nFail {item["dag_id"]}: {item["run_id"]}""")
logging.debug(f"""{data[item["run_id"]]["error"]}""")
finally:
try:
output_folder = data[item["run_id"]]["output_folder"]
shutil.rmtree(output_folder)
logging.debug(f"""Delete output directory {output_folder}""")
except Exception as ex:
logging.error(f"""Failed to delete temporary output directory \n{ex}""")
def run_test(
args, # type: argparse.Namespace
test, # type: Dict[str, str]
test_number, # type: int
total_tests, # type: int
timeout # type: int
): # type: (...) -> TestResult
global templock
out = {} # type: Dict[str,Any]
outdir = outstr = outerr = None
test_command = [] # type: List[str]
duration = 0.0
prefix = ""
suffix = ""
if sys.stderr.isatty():
prefix = "\r"
else:
suffix = "\n"
try:
process = None # type: subprocess.Popen
test_command = prepare_test_command(args.tool, args.args,
args.testargs, test)
if test.get("short_name"):
sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" %
(prefix, test_number, total_tests,
test.get("short_name"), test.get("doc"), suffix))
else:
sys.stderr.write(
"%sTest [%i/%i] %s%s\n" %
(prefix, test_number, total_tests, test.get("doc"), suffix))
sys.stderr.flush()
start_time = time.time()
stderr = subprocess.PIPE if not args.verbose else None
process = subprocess.Popen(test_command,
stdout=subprocess.PIPE,
stderr=stderr)
outstr, outerr = [
var.decode('utf-8') for var in process.communicate(timeout=timeout)
]
return_code = process.poll()
duration = time.time() - start_time
if return_code:
raise subprocess.CalledProcessError(return_code,
" ".join(test_command))
out = json.loads(outstr)
except ValueError as err:
_logger.error(str(err))
_logger.error(outstr)
_logger.error(outerr)
except subprocess.CalledProcessError as err:
if err.returncode == UNSUPPORTED_FEATURE:
return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration,
args.classname)
if test.get("should_fail", False):
return TestResult(0, outstr, outerr, duration, args.classname)
_logger.error(u"""Test failed: %s""",
" ".join([quote(tc) for tc in test_command]))
_logger.error(test.get("doc"))
_logger.error(u"Returned non-zero")
_logger.error(outerr)
return TestResult(1, outstr, outerr, duration, args.classname,
str(err))
except (yamlscanner.ScannerError, TypeError) as err:
_logger.error(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.error(outstr)
_logger.error(u"Parse error %s", str(err))
_logger.error(outerr)
except KeyboardInterrupt:
_logger.error(u"""Test interrupted: %s""",
u" ".join([quote(tc) for tc in test_command]))
raise
except subprocess.TimeoutExpired:
_logger.error(u"""Test timed out: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.error(test.get("doc"))
return TestResult(2, outstr, outerr, timeout, args.classname,
"Test timed out")
finally:
if process is not None and process.returncode is None:
_logger.error(u"""Terminating lingering process""")
process.terminate()
for _ in range(0, 3):
time.sleep(1)
if process.poll() is not None:
break
if process.returncode is None:
process.kill()
fail_message = ''
if test.get("should_fail", False):
_logger.warning(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.warning(test.get("doc"))
_logger.warning(u"Returned zero but it should be non-zero")
return TestResult(1, outstr, outerr, duration, args.classname)
try:
compare(test.get("output"), out)
except CompareFail as ex:
_logger.warning(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.warning(test.get("doc"))
_logger.warning(u"Compare failure %s", ex)
fail_message = str(ex)
if outdir:
shutil.rmtree(outdir, True)
return TestResult((1 if fail_message else 0), outstr, outerr, duration,
args.classname, fail_message)
def run_test(
args, i, tests
): # type: (argparse.Namespace, int, List[Dict[str, str]]) -> TestResult
global templock
out = {} # type: Dict[str,Any]
outdir = outstr = outerr = test_command = None
duration = 0.0
t = tests[i]
prefix = ""
suffix = ""
if sys.stderr.isatty():
prefix = "\r"
else:
suffix = "\n"
try:
test_command = [args.tool]
test_command.extend(args.args)
# Add additional arguments given in test case
if args.testargs is not None:
for testarg in args.testargs:
(test_case_name, prefix) = testarg.split('==')
if test_case_name in t:
test_command.extend([prefix, t[test_case_name]])
# Add prefixes if running on MacOSX so that boot2docker writes to /Users
with templock:
if 'darwin' in sys.platform and args.tool == 'cwltool':
outdir = tempfile.mkdtemp(
prefix=os.path.abspath(os.path.curdir))
test_command.extend([
"--tmp-outdir-prefix={}".format(outdir),
"--tmpdir-prefix={}".format(outdir)
])
else:
outdir = tempfile.mkdtemp()
test_command.extend(
["--outdir={}".format(outdir), "--quiet", t["tool"]])
if t.get("job"):
test_command.append(t["job"])
sys.stderr.write("%sTest [%i/%i] %s\n" %
(prefix, i + 1, len(tests), suffix))
sys.stderr.flush()
start_time = time.time()
stderr = subprocess.PIPE if not args.verbose else None
process = subprocess.Popen(test_command,
stdout=subprocess.PIPE,
stderr=stderr)
outstr, outerr = [var.decode('utf-8') for var in process.communicate()]
return_code = process.poll()
duration = time.time() - start_time
if return_code:
raise subprocess.CalledProcessError(return_code,
" ".join(test_command))
out = json.loads(outstr)
except ValueError as v:
_logger.error(str(v))
_logger.error(outstr)
_logger.error(outerr)
except subprocess.CalledProcessError as err:
if err.returncode == UNSUPPORTED_FEATURE:
return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration,
args.classname)
elif t.get("should_fail", False):
return TestResult(0, outstr, outerr, duration, args.classname)
else:
_logger.error(u"""Test failed: %s""",
" ".join([pipes.quote(tc) for tc in test_command]))
_logger.error(t.get("doc"))
_logger.error("Returned non-zero")
_logger.error(outerr)
return TestResult(1, outstr, outerr, duration, args.classname,
str(err))
except (yamlscanner.ScannerError, TypeError) as e:
_logger.error(u"""Test failed: %s""",
" ".join([pipes.quote(tc) for tc in test_command]))
_logger.error(outstr)
_logger.error(u"Parse error %s", str(e))
_logger.error(outerr)
except KeyboardInterrupt:
_logger.error(u"""Test interrupted: %s""",
" ".join([pipes.quote(tc) for tc in test_command]))
raise
fail_message = ''
if t.get("should_fail", False):
_logger.warning(u"""Test failed: %s""",
" ".join([pipes.quote(tc) for tc in test_command]))
_logger.warning(t.get("doc"))
_logger.warning(u"Returned zero but it should be non-zero")
return TestResult(1, outstr, outerr, duration, args.classname)
try:
compare(t.get("output"), out)
except CompareFail as ex:
_logger.warning(u"""Test failed: %s""",
" ".join([pipes.quote(tc) for tc in test_command]))
_logger.warning(t.get("doc"))
_logger.warning(u"Compare failure %s", ex)
fail_message = str(ex)
if outdir:
shutil.rmtree(outdir, True)
return TestResult((1 if fail_message else 0), outstr, outerr, duration,
args.classname, fail_message)
