Example #1
0
        def run_build(name, build):
            try:
                with open("builds/%s.txt" % name, "w") as output:
                    try:
                        prepend = ""
                        append = ""
                        if "target_os" in build:
                            prepend = "./rethinkdb/scripts/VirtuaBuild/vm_access.py --vm-name %s --command \"" % build["target_os"]
                            append = "\""
                        remotely.run(
                            command_line = """
tar --extract --gzip --touch --file=rethinkdb.tar.gz -- rethinkdb
(%(command_line)s) 2>&1
""" % { "command_line": prepend + build["command_line"] + append},
                            stdout = output,
                            inputs = ["rethinkdb.tar.gz"],
                            outputs = build["products"],
                            on_begin_transfer = lambda: result_log.write("builds", name, status = "uploading", start_time = time.time()),
                            on_begin_script = lambda: result_log.write("builds", name, status = "running", start_time = time.time()),
                            on_end_script = lambda: result_log.write("builds", name, status = "ok", end_time = time.time()),
                            constraint = "build-ready",
                            timeout = 30,   # minutes
                            exclude = ["magneto", "newton"]  # magneto is ubuntu 12.04, see below.
                            )
                        # magneto has ubuntu 12.04, so we can't run executables built by it on our
                        # 11.10 systems.  We can run ubuntu 11.10-compiled executables on magneto,
                        # we think, because we manually copied a few libs over to /usr/local/lib on
                        # magneto.
                    except remotely.ScriptFailedError:
                        result_log.write("builds", name, status = "fail", end_time = time.time())
            except Exception, e:
                result_log.write("builds", name, status = "bug", end_time = time.time(), traceback = traceback.format_exc())
        def run_build(name, build):
            try:
                with open("builds/%s.txt" % name, "w") as output:
                    try:
                        prepend = ""
                        append = ""
                        if "target_os" in build:
                            prepend = "./rethinkdb/scripts/VirtuaBuild/vm_access.py --vm-name %s --command \"" % build["target_os"]
                            append = "\""
                        remotely.run(
                            command_line = """
tar --extract --gzip --touch --file=rethinkdb.tar.gz -- rethinkdb
(%(command_line)s) 2>&1
""" % { "command_line": prepend + build["command_line"] + append},
                            stdout = output,
                            inputs = ["rethinkdb.tar.gz"],
                            outputs = build["products"],
                            on_begin_transfer = lambda: result_log.write("builds", name, status = "uploading", start_time = time.time()),
                            on_begin_script = lambda: result_log.write("builds", name, status = "running", start_time = time.time()),
                            on_end_script = lambda: result_log.write("builds", name, status = "ok", end_time = time.time()),
                            constraint = "build-ready",
                            timeout = 30,   # minutes
                            exclude = ["magneto", "newton"]  # magneto is ubuntu 12.04, see below.
                            )
                        # magneto has ubuntu 12.04, so we can't run executables built by it on our
                        # 11.10 systems.  We can run ubuntu 11.10-compiled executables on magneto,
                        # we think, because we manually copied a few libs over to /usr/local/lib on
                        # magneto.
                    except remotely.ScriptFailedError:
                        result_log.write("builds", name, status = "fail", end_time = time.time())
            except Exception, e:
                result_log.write("builds", name, status = "bug", end_time = time.time(), traceback = traceback.format_exc())
Example #3
0
def run_rethinkdb_test_remotely(dependencies, command_line, stdout_file, zipfile_path, on_begin_transfer = lambda: None, on_begin_script = lambda: None, on_end_script = lambda status: None, constraint = None):
    # We must have an exit code of zero even if the test-script fails so that
    # `remotely` will copy `output_from_test` even if the test script fails. So
    # we have to trap a non-zero exit status and communicate it some other way.
    # We do this by printing it to `stdout` with `exitcode:` before it. But then
    # we need a way to distinguish that from normal lines of `stdout`. So then
    # we use `sed` to put `stdout:` before every line of real output. This is
    # the same trick that `remotely` uses internally; every line of script
    # output is actually transmitted over the network as
    # `STDOUT:stdout:<actual line>`.
    class ExitCodeDemuxer(object):
        def __init__(self):
            self.buffer = ""
        def write(self, text):
            self.buffer += text
            lines = self.buffer.split("\n")
            for line in lines[:-1]:
                if line.startswith("stdout:"):
                    stdout_file.write(line[line.find(":")+1:]+"\n")
                elif line.startswith("exitcode:"):
                    self.exit_code = int(line[line.find(":")+1:].strip())
                else:
                    raise ValueError("Bad line: %r" % line)
            self.buffer = lines[-1]
            if len(lines) > 1:
                stdout_file.flush()
    demuxer = ExitCodeDemuxer()
    remotely.run("""
set +e
export RETHINKDB=$(pwd)/rethinkdb/
mkdir output_from_test
echo 'stdout:About to start test on ' `hostname` '...'
(cd output_from_test; echo "stdout:$PWD"; PYTHONUNBUFFERED=1 %(command_line)s 2>&1 | sed -u s/^/stdout:/)
echo "exitcode:$?"
zip -r %(zipfile_name)s output_from_test >/dev/null
""" % { "command_line": command_line, "zipfile_name": os.path.basename(zipfile_path) },
        stdout = demuxer,
        inputs = dependencies,
        outputs = [os.path.basename(zipfile_path)],
        output_root = os.path.dirname(zipfile_path),
        on_begin_transfer = on_begin_transfer,
        on_begin_script = on_begin_script,
        on_end_script = lambda: on_end_script("pass" if demuxer.exit_code == 0 else "fail"),
        constraint = constraint
        )
def run_rethinkdb_test_remotely(dependencies, command_line, stdout_file, zipfile_path, on_begin_transfer = lambda: None, on_begin_script = lambda: None, on_end_script = lambda status: None, constraint = None):
    # We must have an exit code of zero even if the test-script fails so that
    # `remotely` will copy `output_from_test` even if the test script fails. So
    # we have to trap a non-zero exit status and communicate it some other way.
    # We do this by printing it to `stdout` with `exitcode:` before it. But then
    # we need a way to distinguish that from normal lines of `stdout`. So then
    # we use `sed` to put `stdout:` before every line of real output. This is
    # the same trick that `remotely` uses internally; every line of script
    # output is actually transmitted over the network as
    # `STDOUT:stdout:<actual line>`.
    class ExitCodeDemuxer(object):
        def __init__(self):
            self.buffer = ""
        def write(self, text):
            self.buffer += text
            lines = self.buffer.split("\n")
            for line in lines[:-1]:
                if line.startswith("stdout:"):
                    stdout_file.write(line[line.find(":")+1:]+"\n")
                elif line.startswith("exitcode:"):
                    self.exit_code = int(line[line.find(":")+1:].strip())
                else:
                    raise ValueError("Bad line: %r" % line)
            self.buffer = lines[-1]
            if len(lines) > 1:
                stdout_file.flush()
    demuxer = ExitCodeDemuxer()
    remotely.run("""
set +e
export RETHINKDB=$(pwd)/rethinkdb/
mkdir output_from_test
echo 'stdout:About to start test on ' `hostname` '...'
(cd output_from_test; echo "stdout:$PWD"; PYTHONUNBUFFERED=1 %(command_line)s 2>&1 | sed -u s/^/stdout:/)
echo "exitcode:$?"
zip -r %(zipfile_name)s output_from_test >/dev/null
""" % { "command_line": command_line, "zipfile_name": os.path.basename(zipfile_path) },
        stdout = demuxer,
        inputs = dependencies,
        outputs = [os.path.basename(zipfile_path)],
        output_root = os.path.dirname(zipfile_path),
        on_begin_transfer = on_begin_transfer,
        on_begin_script = on_begin_script,
        on_end_script = lambda: on_end_script("pass" if demuxer.exit_code == 0 else "fail"),
        constraint = constraint
        )