def runtime_analysis(config, overall_report):
"""Run test suites on executable and return a list containing the result of each test suite.
Each list item is a dictionary describing the result of running that test suite.
"""
test_case_report_list = []
for test_suite in config.get_test_suite():
report = dict()
report['stdout_stream'] = ''
report['stderr_stream'] = ''
report['outfile'] = ''
input_for_stdin = config.get_test_suite_input_for_stdin(test_suite)
# using Popen instead of run because I need access to the pid
# See comment under "except subprocess.TimeoutExpired:"
infile = "xinfile_" + uuid.uuid4().hex[0:16] + ".txt"
outfile = "xoutfile_" + uuid.uuid4().hex[0:16] + ".txt"
p = subprocess.Popen(['./run_jail.sh',
config.output_filename,
str(len(test_suite)), infile, outfile], # command
stdout=subprocess.PIPE, # capture stdout
stderr=subprocess.PIPE, # capture stderr
stdin=subprocess.PIPE, # capture stdin
universal_newlines=True, # use text mode for std* file objects
start_new_session=True, # otherwise killing the process group will also kill the Python interpreter
)
try:
# send test suite input
with open(infile, "w") as f:
f.write(input_for_stdin)
(stdout_stream, stderr_stream) = p.communicate(timeout=config.timeout)
report['return_code'] = p.returncode
report['stderr_stream'] += stderr_stream
report['stdout_stream'] += stdout_stream
with open(outfile, "r") as f:
current_outfile = f.read()
report['outfile'] += current_outfile
# check if test cases passed
ret_output_match = config.check_for_output_match(current_outfile, test_suite)
report['test_suite'] = test_suite
report['output_match'] = ret_output_match
except subprocess.TimeoutExpired:
# kill the process group so that all child processes spawned by the process are also killed
# The child need to be killed because, in addition to wasting CPU cycles,
# it can hold stdout and then Python will wait indefinitely even if the timeout is expired
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
report['timeout'] = True
finally:
test_case_report_list.append(report)
overall_report['runtime_analysis_done'] = True
return overall_report, test_case_report_list
sys.stderr.write(
"[ERROR] Cannot find file specified as "
"``install requirements`` (%s)\n" % requirements_file
)
sys.exit(1)
exclude_packages = config.get_exclude_packages()
packages = [x for x in find_packages() if x not in exclude_packages]
setup(
name=config.get_project_name(), # name of your app
version=version,
license=config.get_project_license(), # license
# test configuration
test_suite=config.get_test_suite(),
tests_require=tests_require,
# install configuration
install_requires=install_requires,
# your app description
description=config.get_project_desc(),
long_description=open('README.rst').read(),
# author info
author=config.get_author_name(),
author_email=config.get_author_email(),
# app location
url=config.get_project_url(),