def run_build(name, build):
try:
with open("builds/%s.txt" % name, "w") as output:
try:
prepend = ""
append = ""
if "target_os" in build:
prepend = "./rethinkdb/scripts/VirtuaBuild/vm_access.py --vm-name %s --command \"" % build["target_os"]
append = "\""
remotely.run(
command_line = """
tar --extract --gzip --touch --file=rethinkdb.tar.gz -- rethinkdb
(%(command_line)s) 2>&1
""" % { "command_line": prepend + build["command_line"] + append},
stdout = output,
inputs = ["rethinkdb.tar.gz"],
outputs = build["products"],
on_begin_transfer = lambda: result_log.write("builds", name, status = "uploading", start_time = time.time()),
on_begin_script = lambda: result_log.write("builds", name, status = "running", start_time = time.time()),
on_end_script = lambda: result_log.write("builds", name, status = "ok", end_time = time.time()),
constraint = "build-ready",
timeout = 30, # minutes
exclude = ["magneto", "newton"] # magneto is ubuntu 12.04, see below.
)
# magneto has ubuntu 12.04, so we can't run executables built by it on our
# 11.10 systems. We can run ubuntu 11.10-compiled executables on magneto,
# we think, because we manually copied a few libs over to /usr/local/lib on
# magneto.
except remotely.ScriptFailedError:
result_log.write("builds", name, status = "fail", end_time = time.time())
except Exception, e:
result_log.write("builds", name, status = "bug", end_time = time.time(), traceback = traceback.format_exc())
def run_build(name, build):
try:
with open("builds/%s.txt" % name, "w") as output:
try:
prepend = ""
append = ""
if "target_os" in build:
prepend = "./rethinkdb/scripts/VirtuaBuild/vm_access.py --vm-name %s --command \"" % build["target_os"]
append = "\""
remotely.run(
command_line = """
tar --extract --gzip --touch --file=rethinkdb.tar.gz -- rethinkdb
(%(command_line)s) 2>&1
""" % { "command_line": prepend + build["command_line"] + append},
stdout = output,
inputs = ["rethinkdb.tar.gz"],
outputs = build["products"],
on_begin_transfer = lambda: result_log.write("builds", name, status = "uploading", start_time = time.time()),
on_begin_script = lambda: result_log.write("builds", name, status = "running", start_time = time.time()),
on_end_script = lambda: result_log.write("builds", name, status = "ok", end_time = time.time()),
constraint = "build-ready",
timeout = 30, # minutes
exclude = ["magneto", "newton"] # magneto is ubuntu 12.04, see below.
)
# magneto has ubuntu 12.04, so we can't run executables built by it on our
# 11.10 systems. We can run ubuntu 11.10-compiled executables on magneto,
# we think, because we manually copied a few libs over to /usr/local/lib on
# magneto.
except remotely.ScriptFailedError:
result_log.write("builds", name, status = "fail", end_time = time.time())
except Exception, e:
result_log.write("builds", name, status = "bug", end_time = time.time(), traceback = traceback.format_exc())
def run_rethinkdb_test_remotely(dependencies, command_line, stdout_file, zipfile_path, on_begin_transfer = lambda: None, on_begin_script = lambda: None, on_end_script = lambda status: None, constraint = None):
# We must have an exit code of zero even if the test-script fails so that
# `remotely` will copy `output_from_test` even if the test script fails. So
# we have to trap a non-zero exit status and communicate it some other way.
# We do this by printing it to `stdout` with `exitcode:` before it. But then
# we need a way to distinguish that from normal lines of `stdout`. So then
# we use `sed` to put `stdout:` before every line of real output. This is
# the same trick that `remotely` uses internally; every line of script
# output is actually transmitted over the network as
# `STDOUT:stdout:<actual line>`.
class ExitCodeDemuxer(object):
def __init__(self):
self.buffer = ""
def write(self, text):
self.buffer += text
lines = self.buffer.split("\n")
for line in lines[:-1]:
if line.startswith("stdout:"):
stdout_file.write(line[line.find(":")+1:]+"\n")
elif line.startswith("exitcode:"):
self.exit_code = int(line[line.find(":")+1:].strip())
else:
raise ValueError("Bad line: %r" % line)
self.buffer = lines[-1]
if len(lines) > 1:
stdout_file.flush()
demuxer = ExitCodeDemuxer()
remotely.run("""
set +e
export RETHINKDB=$(pwd)/rethinkdb/
mkdir output_from_test
echo 'stdout:About to start test on ' `hostname` '...'
(cd output_from_test; echo "stdout:$PWD"; PYTHONUNBUFFERED=1 %(command_line)s 2>&1 | sed -u s/^/stdout:/)
echo "exitcode:$?"
zip -r %(zipfile_name)s output_from_test >/dev/null
""" % { "command_line": command_line, "zipfile_name": os.path.basename(zipfile_path) },
stdout = demuxer,
inputs = dependencies,
outputs = [os.path.basename(zipfile_path)],
output_root = os.path.dirname(zipfile_path),
on_begin_transfer = on_begin_transfer,
on_begin_script = on_begin_script,
on_end_script = lambda: on_end_script("pass" if demuxer.exit_code == 0 else "fail"),
constraint = constraint
)
def run_rethinkdb_test_remotely(dependencies, command_line, stdout_file, zipfile_path, on_begin_transfer = lambda: None, on_begin_script = lambda: None, on_end_script = lambda status: None, constraint = None):
# We must have an exit code of zero even if the test-script fails so that
# `remotely` will copy `output_from_test` even if the test script fails. So
# we have to trap a non-zero exit status and communicate it some other way.
# We do this by printing it to `stdout` with `exitcode:` before it. But then
# we need a way to distinguish that from normal lines of `stdout`. So then
# we use `sed` to put `stdout:` before every line of real output. This is
# the same trick that `remotely` uses internally; every line of script
# output is actually transmitted over the network as
# `STDOUT:stdout:<actual line>`.
class ExitCodeDemuxer(object):
def __init__(self):
self.buffer = ""
def write(self, text):
self.buffer += text
lines = self.buffer.split("\n")
for line in lines[:-1]:
if line.startswith("stdout:"):
stdout_file.write(line[line.find(":")+1:]+"\n")
elif line.startswith("exitcode:"):
self.exit_code = int(line[line.find(":")+1:].strip())
else:
raise ValueError("Bad line: %r" % line)
self.buffer = lines[-1]
if len(lines) > 1:
stdout_file.flush()
demuxer = ExitCodeDemuxer()
remotely.run("""
set +e
export RETHINKDB=$(pwd)/rethinkdb/
mkdir output_from_test
echo 'stdout:About to start test on ' `hostname` '...'
(cd output_from_test; echo "stdout:$PWD"; PYTHONUNBUFFERED=1 %(command_line)s 2>&1 | sed -u s/^/stdout:/)
echo "exitcode:$?"
zip -r %(zipfile_name)s output_from_test >/dev/null
""" % { "command_line": command_line, "zipfile_name": os.path.basename(zipfile_path) },
stdout = demuxer,
inputs = dependencies,
outputs = [os.path.basename(zipfile_path)],
output_root = os.path.dirname(zipfile_path),
on_begin_transfer = on_begin_transfer,
on_begin_script = on_begin_script,
on_end_script = lambda: on_end_script("pass" if demuxer.exit_code == 0 else "fail"),
constraint = constraint
)