def run(systest, tag, branch):
""" Runs (create a container from an image) the specified systest. """
test_tag = docker.get_namespace() + systest + "-" + tag + "-" + branch
try:
ccall("docker run -it -d --name " + test_tag + " " + test_tag)
shutil.rmtree("Output", ignore_errors=True)
shutil.rmtree("Logs", ignore_errors=True)
ccall("docker cp " + test_tag + ":Output . ")
except CalledProcessError as e:
print("TEST FAILED WITH: {}".format(e))
raise STRunException()
def add_output_files(output_dir, output_log_dir, success):
if success:
# Everything passes, no need to commit anything, remove previous output
ccall("git rm -r --ignore-unmatch {}".format(output_log_dir))
elif os.path.isdir(output_dir):
if os.path.isdir(output_log_dir):
# overwrite previous output to get rid of artifacts
ccall("git rm -rf {}".format(output_log_dir))
ccall("mv {} {}".format(output_dir, output_log_dir))
ccall("git add .")
def add_readme(job_path,
type='test',
output_enabled=False,
output_missing=False,
logs_missing=False,
message=None):
"""
Create a README.md at the location specified by readme_path.
"""
job_link = os.environ["TRAVIS_JOB_WEB_URL"]
job_name = os.environ["TRAVIS_JOB_NAME"]
job_success = True if (os.environ["TRAVIS_TEST_RESULT"] == '0') else False
branch = os.environ["TRAVIS_BRANCH"]
pr_branch = os.environ["TRAVIS_PULL_REQUEST_BRANCH"]
is_pr = False if (pr_branch == "") else True
if (output_missing or logs_missing or message):
additional_info = True
else:
additional_info = False
with open(os.path.join('templates', 'readme_template', 'README.md')) as f:
tmp = Template(f.read())
readme_rendered = tmp.render(type=type,
job_name=job_name,
job_success=job_success,
branch=branch,
pr_branch=pr_branch,
is_pr=is_pr,
job_link=job_link,
output_enabled=output_enabled,
output_missing=output_missing,
additional_info=additional_info,
logs_missing=logs_missing,
message=message)
with chdir(job_path):
with open("README.md", "w") as f:
f.write(readme_rendered)
ccall("git add README.md")
def run(systest, tag, branch):
""" Runs (create a container from an image) the specified systest. """
test_tag = docker.get_namespace() + systest + "-" + tag + "-" + branch
# remove any currently running containers with same name
ccall("docker container stop {}; docker container rm {}".format(
test_tag, test_tag))
ccall("docker run -it -d --name " + test_tag + " " + test_tag)
shutil.rmtree("Output", ignore_errors=True)
ccall("docker cp " + test_tag + ":Output . ")
def run_compose(systest, branch, local, tag, force_rebuild, rm_all):
""" Runs necessary systemtest with docker compose """
test_dirname = "TestCompose_{systest}".format(systest=systest)
test_basename = systest.split('.')[0]
adapter_base_name = "-".join([tag, branch])
# set up environment variables, to detect precice base image, that we
# should run with and docker images location
commands_main = ["""export PRECICE_BASE=-{base}; {extra_cmd} docker-compose config &&
bash ../../silent_compose.sh""".format(base =
adapter_base_name, extra_cmd =\
"export SYSTEST_REMOTE={remote};".format(
remote = docker.get_namespace()) if local else "" ),
"docker cp tutorial-data:/Output ."]
# rebuild tutorials image if needed
if force_rebuild:
commands_main.insert(0, "docker-compose build --no-cache")
commands_cleanup = ["docker-compose down -v"]
test_path = os.path.join(os.getcwd(), 'tests', test_dirname)
with common.chdir(test_path):
# cleanup previous results
shutil.rmtree("Output", ignore_errors=True)
try:
for command in commands_main:
ccall(command)
#compare results
path_to_ref = os.path.join(os.getcwd(), "referenceOutput")
path_to_otp = os.path.join(os.getcwd(), "Output")
comparison(path_to_ref, path_to_otp)
if rm_all:
for command in commands_cleanup:
ccall(command)
except (CalledProcessError, IncorrectOutput) as e:
# cleanup in either case
if rm_all:
for command in commands_cleanup:
ccall(command)
# generate a report of failurs for local tests
if local:
raise e
print("TESTS FAILED WITH: {}".format(e))
sys.exit(1)
def run_compose(systest,
branch,
local,
tag,
force_rebuild,
rm_all=False,
verbose=False):
""" Runs necessary systemtest with docker compose """
test_dirname = "TestCompose_{systest}".format(systest=systest)
test_basename = systest.split('.')[0]
adapter_base_name = "-".join([tag, branch])
# set up environment variables, to detect precice base image, that we
# should run with and docker images location
export_cmd = "export PRECICE_BASE=-{}; ".format(adapter_base_name)
extra_cmd = "export SYSTEST_REMOTE={}; ".format(
docker.get_namespace()) if local else ""
compose_config_cmd = "mkdir Logs; docker-compose config && "
compose_exec_cmd = "bash ../../silent_compose.sh {}".format(
'debug' if verbose else "")
copy_cmd = "docker cp tutorial-data:/Output ."
log_cmd = "docker-compose logs > Logs/container.log"
commands_main = [
export_cmd + extra_cmd + compose_config_cmd + compose_exec_cmd,
copy_cmd, log_cmd
]
# rebuild tutorials image if needed
if force_rebuild:
commands_main.insert(0, "docker-compose build --no-cache")
commands_cleanup = ["docker-compose down -v"]
test_path = os.path.join(os.getcwd(), 'tests', test_dirname)
with common.chdir(test_path):
# cleanup previous results
shutil.rmtree("Output", ignore_errors=True)
shutil.rmtree("Logs", ignore_errors=True)
try:
for command in commands_main:
ccall(command)
#compare results
path_to_ref = os.path.join(os.getcwd(), "referenceOutput")
path_to_otp = os.path.join(os.getcwd(), "Output")
comparison(path_to_ref, path_to_otp)
if rm_all:
for command in commands_cleanup:
ccall(command)
except CalledProcessError as e:
print("TEST FAILED WITH: {}".format(e))
raise STRunException()
except IncorrectOutput as e:
print("TEST FAILED WITH: {}".format(e))
raise STValidateException()
finally:
# cleanup in either case
if rm_all:
for command in commands_cleanup:
ccall(command)
def run(systest, tag, branch):
""" Runs (create a container from an image) the specified systest. """
test_tag = docker.get_namespace() + systest + "-" + tag + "-" + branch
ccall("docker run -it -d --name " + test_tag + " " + test_tag)
shutil.rmtree("Output", ignore_errors=True)
ccall("docker cp " + test_tag + ":Output . ")
choices=["Ubuntu1804", "Ubuntu1604"],
default="Ubuntu1604")
args = parser.parse_args()
if __name__ == "__main__":
systest = args.test
# Creating new logfile. if it exists, truncate content.
log = open("log_" + systest, "w")
foam_version = 4.1 if args.os == "1604" else 5
# Saving versions of used software in system test systest.
if systest == "of-ccx":
log.write("OpenFOAM version: {}\n".format(foam_version))
ccall([
"echo OpenFOAM-adapter Version: $(git ls-remote https://github.com/precice/openfoam-adapter.git | tail -1)"
],
stdout=log)
log.write("CalculiX version: 2.12\n")
ccall([
"echo CalculiX-adapter Version: $(git ls-remote https://github.com/precice/calculix-adapter.git | tail -1)"
],
stdout=log)
ccall([
"echo tutorials Version: $(git ls-remote https://github.com/precice/tutorials.git | tail -1)"
],
stdout=log)
elif systest == "of-of":
log.write("OpenFOAM version: {}\n".format(foam_version))
ccall([
"echo OpenFOAM-adapter Version: $(git ls-remote https://github.com/precice/openfoam-adapter.git | tail -1)"
],
type = 'adapter'
elif args.precice:
type = 'precice'
job_id = os.environ["TRAVIS_JOB_ID"]
job_result = os.environ["TRAVIS_TEST_RESULT"]
job_success = True if (job_result == '0') else False
job_name = os.environ["TRAVIS_JOB_NAME"]
build_folder = os.environ["TRAVIS_BUILD_NUMBER"] # example: "1832"
job_folder_unpadded = os.environ["TRAVIS_JOB_NUMBER"] # example: "1832.8"
job_folder = "{}.{:02d}".format(build_folder,
int(job_folder_unpadded.split('.')[1]))
ccall("git clone -b {st_branch} https://github.com/precice/precice_st_output".\
format(st_branch=args.st_branch))
# Path to repository folder
repo_path = os.path.join(os.getcwd(), repo_folder)
# Path to job folder
job_path = os.path.join(os.getcwd(), repo_folder, build_folder, job_folder)
output_missing = False
# Path to Logs folder inside a job folder
log_path = os.path.join(job_path, "Logs")
ccall("mkdir -p {}".format(log_path))
# Path to Output folder inside a job folder
output_path = os.path.join(job_path, "Output")
if args.adapter:
def add_job_log(systest, failed, log_dir):
with chdir(log_dir):
log_name = "log_{test}.md".format(test = systest)
with open(log_name, "w") as log:
create_job_log(systest, log, failed)
ccall("git add {log_name}".format(log_name = log_name))
if not os.path.isdir(output_dir):
commit_msg_lines = ["Failed to produce results"]
else:
commit_msg_lines = ["Output != Reference build number: {}".format(travis_build_number)]
return commit_msg_lines + ["Build url: {}".format(travis_job_web_url)]
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Push information about the test to the output repository')
parser.add_argument('-t', '--test', help="Choose systemtest, results of which to push")
parser.add_argument('-s', '--success', action='store_true' ,help="Whether test was successfull")
parser.add_argument('-b', '--base', type=str, help="Base image of the test", default="Ubuntu1604.home")
args = parser.parse_args()
ccall("git clone https://github.com/precice/precice_st_output")
test_type = "Test" if args.test == "bindings" else "TestCompose"
test_name = "{Type}_{test}.{base}".format(Type = test_type, test =
args.test, base = args.base)
if not os.path.isdir(test_name):
test_name = test_name.split(".")[0]
log_dir = os.path.join(os.getcwd(), "precice_st_output", args.base)
output_log_dir = os.path.join(log_dir, "Output_{}_{}".format(test_type, args.test))
output_dir = os.path.join(os.getcwd(), "tests", test_name, "Output")
ccall("mkdir -p {}".format(log_dir))
os.chdir(log_dir)
add_job_log(args.test, not args.success, log_dir)
test_names = args.systemtest
tests = []
for test_name in test_names:
tests += get_test_variants(test_name)
tests = filter_tests(tests, args.dockerfile)
# Checking for older docker containers
lst2 = docker.get_containers()
print(lst2)
if lst2:
print("Deleting following docker containers:", lst2)
answer = input("\nOk? (yes/no)\n")
if answer in ["yes", "y"]:
for x in lst2:
ccall("docker container rm -f " + x)
else:
print(
"BE CAREFUL!: Not deleting previous containers can later lead to problems."
)
# Building preCICE
print("\n\nBuilding preCICE docker image with choosen branch\n\n")
with common.chdir(os.path.join(os.getcwd(), 'precice')):
docker.build_image("precice-" + args.dockerfile.lower() + "-" +
args.branch,
args.dockerfile,
build_args={"branch": args.branch},
force_rebuild="precice" in args.force_rebuild)
# Starting system tests
failed = []
if __name__ == "__main__":
test_names = args.systemtest
tests = []
for test_name in test_names:
tests += get_test_variants(test_name)
tests = filter_tests(tests, args.dockerfile)
# Checking for older docker containers
lst2 = docker.get_containers()
if lst2:
print("Deleting following docker containers:", lst2)
answer = input("\nOk? (yes/no)\n")
if answer in ["yes", "y"]:
for x in lst2:
ccall("docker container rm -f " + x)
else:
print("BE CAREFUL!: Not deleting previous containers can later lead to problems.")
# Building preCICE
print("\n\nBuilding preCICE docker image with choosen branch\n\n")
docker.build_image("precice-" + args.dockerfile.lower() + "-" + args.branch, args.dockerfile,
build_args = {"branch" : args.branch},
force_rebuild = "precice" in args.force_rebuild)
# Starting system tests
failed = []
success = []
for test in tests:
test_basename = determine_test_name(test)
print("\n\nStarting system test %s\n\n" % test)
try:
