コード例 #1
0
def runPerformanceTest():
    def remoteCode():
        os.chdir('{}/maxscale-performance-test/'.format(os.environ['HOME']))
        if 'COMP_WORDBREAKS' in os.environ:
            del os.environ['COMP_WORDBREAKS']

        logFile = open('{}/results_{}'.format(builddir, buildnumber), 'w')
        process = subprocess.Popen([
            './bin/performance_test', '-v', '--server-config', '{}/{}'.format(
                os.environ['HOME'], networkConfigPath), '--remote-test-app',
            '{}/.config/performance_test/run_sysbench.sh'.format(
                os.environ['HOME']), '--db-server-2-config',
            'slave-config.sql.erb', '--db-server-3-config',
            'slave-config.sql.erb', '--db-server-4-config',
            'slave-config.sql.erb', '--mariadb-version', version,
            '--maxscale-config', perf_cnf_template, '--maxscale-version',
            target, '--keep-servers', 'true'
        ],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT)
        for byteLine in process.stdout:
            line = byteLine.decode("utf-8", "replace")
            sys.stdout.write(line)
            logFile.write(line)
        process.wait()
        logFile.close()
        sys.exit(process.returncode)

    return support.executePythonScript('Run performance tests', remoteCode)
コード例 #2
0
def removeSnapshotLock():
    """
    Compares $build_full_name and $HOME/mdbci/${name}_snapshot_lock content
    and calls remove lock if they are equal
    """
    def remoteCode():
        lockFile = "{}/{}_snapshot_lock".format(MDBCI_VM_PATH, name)
        if not os.path.exists(lockFile):
            print(
                "Lock file {} does not exist, doing nothing".format(lockFile))
            sys.exit(0)

        buildFullName = "{}-{}".format(buildername, buildnumber)
        lockerSource = open(lockFile).read().strip()
        if lockerSource != buildFullName:
            print(
                "Lock file was created not by the current task, {} != {}, doing nothing"
                .format(buildFullName, lockerSource))
            sys.exit(0)
        os.remove(lockFile)
        sys.exit(0)

    return support.executePythonScript("Remove leftover snapshot locks",
                                       remoteCode,
                                       haltOnFailure=False,
                                       alwaysRun=True)
コード例 #3
0
def testConnecton():
    """
    Tests if nodes specified in performance-test_network_config are available
    :return:
    """
    def remoteCode():
        import re
        with open('{}/{}'.format(os.environ['HOME'], networkConfigPath),
                  encoding="UTF-8") as configFile:
            networkConfig = {}
            configRegexp = re.compile(
                r'^(\S+)_(network|whoami|keyfile)=(\S+)$')
            for line in configFile:
                if configRegexp.match(line):
                    match = configRegexp.search(line)
                    if match.group(1) not in networkConfig:
                        networkConfig[match.group(1)] = {}
                    networkConfig[match.group(1)].update(
                        {match.group(2): match.group(3)})

            for node, config in networkConfig.items():
                host = '{}@{}'.format(config['whoami'], config['network'])
                result = subprocess.call(
                    'ssh -i {} -o ConnectTimeout=20 {} exit'.format(
                        config['keyfile'], host),
                    shell=True)
                if result:
                    sys.exit(
                        subprocess.call('sudo $HOME/restart_vpn.sh',
                                        shell=True))

        sys.exit(0)

    return support.executePythonScript(
        'Testing connection to the remote machine', remoteCode)
コード例 #4
0
def remoteRunScriptAndLog():
    """
    Runs shell script which name is given in a property 'script_name' of a builder on a worker
    and save results to the log file
    """
    def remote():
        if not os.path.exists("maxscale-system-test/mdbci"):
            os.mkdir("default-maxscale-branch")
            subprocess.run([
                "git", "clone", repository, "default-maxscale-branch/MaxScale"
            ])
            shutil.copytree(
                "default-maxscale-branch/MaxScale/maxscale-system-test/mdbci",
                "maxscale-system-test")

        logFile = open(buildLogFile, "w")
        process = subprocess.Popen(
            ["maxscale-system-test/mdbci/{}".format(script_name)],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        for byteLine in process.stdout:
            line = byteLine.decode("utf-8", "replace")
            sys.stdout.write(line)
            sys.stdout.flush()
            logFile.write(line)
        process.wait()
        logFile.close()

        testLogFile = open(resultFile, "w")
        testLogFile.write(str(process.returncode))
        testLogFile.close()
        sys.exit(process.returncode)

    return support.executePythonScript("Run MaxScale tests using MDBCI",
                                       remote)
コード例 #5
0
def remoteParseCtestLogAndStoreIt():
    """Parse ctest results and store them in the LOGS directory"""
    def remote():
        buildId = "{}-{}".format(buildername, buildnumber)
        outputDirectory = os.path.join(builddir, buildId, "ctest_sublogs")
        subprocess.run([
            "{}/scripts/parse_ctest_log.py".format(builddir), buildLogFile,
            "-o",
            os.path.join(builddir, "results_{}".format(buildnumber)), "-r",
            "-f", "-j", jsonResultsFile, "-s", outputDirectory
        ])

        storeDirectory = os.path.join(HOME, "LOGS", buildId, "LOGS")
        for logDirectory in os.listdir(outputDirectory):
            targetDirectory = os.path.join(storeDirectory, logDirectory)
            os.umask(0o002)
            os.makedirs(targetDirectory, exist_ok=True)
            shutil.copy(
                os.path.join(outputDirectory, logDirectory, "ctest_sublog"),
                targetDirectory)

    return support.executePythonScript(
        "Parse ctest results log and save it to logs directory",
        remote,
        alwaysRun=True)
コード例 #6
0
def removeLock():
    """Remove vagrant lock if it was left by the build script"""
    def remoteCode():
        lockFile = "{}/vagrant_lock".format(HOME)
        if os.path.exists(lockFile):
            buildFullName = "{}-{}".format(buildername, buildnumber)
            lockerSource = open(lockFile).read().strip()
            if lockerSource == buildFullName:
                os.remove(lockFile)
            else:
                print(
                    "Lock file was created not by the current task, {} != {}, doing nothing"
                    .format(buildFullName, lockerSource))
        else:
            print(
                "Lock file {} does not exist, doing nothing".format(lockFile))

        if try_already_running == "yes":
            snapshotLockFile = "{}/{}_snapshot_lock".format(MDBCI_VM_PATH, box)
            if os.path.exists(snapshotLockFile):
                print("Releasing lock for already running VM")
                os.remove(snapshotLockFile)
        sys.exit(0)

    return support.executePythonScript("Remove leftover vagrant locks",
                                       remoteCode,
                                       haltOnFailure=False,
                                       alwaysRun=True)
コード例 #7
0
def createBuildSteps():
    buildSteps = []
    buildSteps.append(steps.SetProperties(properties=configureBuildProperties))
    buildSteps.extend(common.cloneRepository())
    buildSteps.extend(
        support.executePythonScript("Build MDBCI", remoteBuildMdbci))
    buildSteps.extend(publishMdbci())
    buildSteps.extend(common.cleanBuildDir())
    return buildSteps
コード例 #8
0
def createBuildSteps():
    buildSteps = []
    buildSteps.extend(common.configureMdbciVmPathProperty())
    buildSteps.append(steps.SetProperties(properties=configureBuildProperties))
    buildSteps.extend(common.cloneRepository())
    buildSteps.extend(
        support.executePythonScript("Create full repo", remoteBuildMaxscale))
    buildSteps.extend(common.cleanBuildDir())
    buildSteps.extend(common.destroyVirtualMachine())
    return buildSteps
コード例 #9
0
def destroyVirtualMachine():
    """Destroy virtual machine if it was not destroied after the build"""
    def remoteCode():
        if not os.path.exists(mdbciConfig):
            print("MDBCI configuration does not exist")
            sys.exit(0)

        os.system("$HOME/mdbci/mdbci destroy {}".format(mdbciConfig))

    def shouldRun(step):
        if step.getProperty(
                "try_already_running") == "yes" or step.getProperty(
                    "do_not_destroy_vm") == "yes":
            return False
        return True

    return support.executePythonScript("Destroy leftover virtual machines",
                                       remoteCode,
                                       haltOnFailure=False,
                                       alwaysRun=True,
                                       doStepIf=shouldRun)
コード例 #10
0
def remoteStoreCoredumps():
    """Find the coredumps and store them in the LOGS directory"""
    def remote():
        result = subprocess.check_output([
            "{}/scripts/coredump_finder.py".format(builddir),
            "{}-{}".format(buildername, buildnumber), "url"
        ])
        coredumpLogFile = open(
            os.path.join(builddir, "coredumps_{}".format(buildnumber)), "w")
        coredumpLogFile.write("COREDUMPS \\\n")
        if result == "":
            coredumpLogFile.write(
                "Coredumps were not found for build {}".format(buildnumber))
        else:
            for dump in result.decode("utf_8").split("\n"):
                coredumpLogFile.write("{} \\\n".format(dump))
        buildId = "{}-{}".format(buildername, buildnumber)
        shutil.copy(buildLogFile, os.path.join(HOME, "LOGS", buildId))

    return support.executePythonScript("Find and store coredumps",
                                       remote,
                                       alwaysRun=True)