def runPerformanceTest(): def remoteCode(): os.chdir('{}/maxscale-performance-test/'.format(os.environ['HOME'])) if 'COMP_WORDBREAKS' in os.environ: del os.environ['COMP_WORDBREAKS'] logFile = open('{}/results_{}'.format(builddir, buildnumber), 'w') process = subprocess.Popen([ './bin/performance_test', '-v', '--server-config', '{}/{}'.format( os.environ['HOME'], networkConfigPath), '--remote-test-app', '{}/.config/performance_test/run_sysbench.sh'.format( os.environ['HOME']), '--db-server-2-config', 'slave-config.sql.erb', '--db-server-3-config', 'slave-config.sql.erb', '--db-server-4-config', 'slave-config.sql.erb', '--mariadb-version', version, '--maxscale-config', perf_cnf_template, '--maxscale-version', target, '--keep-servers', 'true' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for byteLine in process.stdout: line = byteLine.decode("utf-8", "replace") sys.stdout.write(line) logFile.write(line) process.wait() logFile.close() sys.exit(process.returncode) return support.executePythonScript('Run performance tests', remoteCode)
def removeSnapshotLock(): """ Compares $build_full_name and $HOME/mdbci/${name}_snapshot_lock content and calls remove lock if they are equal """ def remoteCode(): lockFile = "{}/{}_snapshot_lock".format(MDBCI_VM_PATH, name) if not os.path.exists(lockFile): print( "Lock file {} does not exist, doing nothing".format(lockFile)) sys.exit(0) buildFullName = "{}-{}".format(buildername, buildnumber) lockerSource = open(lockFile).read().strip() if lockerSource != buildFullName: print( "Lock file was created not by the current task, {} != {}, doing nothing" .format(buildFullName, lockerSource)) sys.exit(0) os.remove(lockFile) sys.exit(0) return support.executePythonScript("Remove leftover snapshot locks", remoteCode, haltOnFailure=False, alwaysRun=True)
def testConnecton(): """ Tests if nodes specified in performance-test_network_config are available :return: """ def remoteCode(): import re with open('{}/{}'.format(os.environ['HOME'], networkConfigPath), encoding="UTF-8") as configFile: networkConfig = {} configRegexp = re.compile( r'^(\S+)_(network|whoami|keyfile)=(\S+)$') for line in configFile: if configRegexp.match(line): match = configRegexp.search(line) if match.group(1) not in networkConfig: networkConfig[match.group(1)] = {} networkConfig[match.group(1)].update( {match.group(2): match.group(3)}) for node, config in networkConfig.items(): host = '{}@{}'.format(config['whoami'], config['network']) result = subprocess.call( 'ssh -i {} -o ConnectTimeout=20 {} exit'.format( config['keyfile'], host), shell=True) if result: sys.exit( subprocess.call('sudo $HOME/restart_vpn.sh', shell=True)) sys.exit(0) return support.executePythonScript( 'Testing connection to the remote machine', remoteCode)
def remoteRunScriptAndLog(): """ Runs shell script which name is given in a property 'script_name' of a builder on a worker and save results to the log file """ def remote(): if not os.path.exists("maxscale-system-test/mdbci"): os.mkdir("default-maxscale-branch") subprocess.run([ "git", "clone", repository, "default-maxscale-branch/MaxScale" ]) shutil.copytree( "default-maxscale-branch/MaxScale/maxscale-system-test/mdbci", "maxscale-system-test") logFile = open(buildLogFile, "w") process = subprocess.Popen( ["maxscale-system-test/mdbci/{}".format(script_name)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for byteLine in process.stdout: line = byteLine.decode("utf-8", "replace") sys.stdout.write(line) sys.stdout.flush() logFile.write(line) process.wait() logFile.close() testLogFile = open(resultFile, "w") testLogFile.write(str(process.returncode)) testLogFile.close() sys.exit(process.returncode) return support.executePythonScript("Run MaxScale tests using MDBCI", remote)
def remoteParseCtestLogAndStoreIt(): """Parse ctest results and store them in the LOGS directory""" def remote(): buildId = "{}-{}".format(buildername, buildnumber) outputDirectory = os.path.join(builddir, buildId, "ctest_sublogs") subprocess.run([ "{}/scripts/parse_ctest_log.py".format(builddir), buildLogFile, "-o", os.path.join(builddir, "results_{}".format(buildnumber)), "-r", "-f", "-j", jsonResultsFile, "-s", outputDirectory ]) storeDirectory = os.path.join(HOME, "LOGS", buildId, "LOGS") for logDirectory in os.listdir(outputDirectory): targetDirectory = os.path.join(storeDirectory, logDirectory) os.umask(0o002) os.makedirs(targetDirectory, exist_ok=True) shutil.copy( os.path.join(outputDirectory, logDirectory, "ctest_sublog"), targetDirectory) return support.executePythonScript( "Parse ctest results log and save it to logs directory", remote, alwaysRun=True)
def removeLock(): """Remove vagrant lock if it was left by the build script""" def remoteCode(): lockFile = "{}/vagrant_lock".format(HOME) if os.path.exists(lockFile): buildFullName = "{}-{}".format(buildername, buildnumber) lockerSource = open(lockFile).read().strip() if lockerSource == buildFullName: os.remove(lockFile) else: print( "Lock file was created not by the current task, {} != {}, doing nothing" .format(buildFullName, lockerSource)) else: print( "Lock file {} does not exist, doing nothing".format(lockFile)) if try_already_running == "yes": snapshotLockFile = "{}/{}_snapshot_lock".format(MDBCI_VM_PATH, box) if os.path.exists(snapshotLockFile): print("Releasing lock for already running VM") os.remove(snapshotLockFile) sys.exit(0) return support.executePythonScript("Remove leftover vagrant locks", remoteCode, haltOnFailure=False, alwaysRun=True)
def createBuildSteps(): buildSteps = [] buildSteps.append(steps.SetProperties(properties=configureBuildProperties)) buildSteps.extend(common.cloneRepository()) buildSteps.extend( support.executePythonScript("Build MDBCI", remoteBuildMdbci)) buildSteps.extend(publishMdbci()) buildSteps.extend(common.cleanBuildDir()) return buildSteps
def createBuildSteps(): buildSteps = [] buildSteps.extend(common.configureMdbciVmPathProperty()) buildSteps.append(steps.SetProperties(properties=configureBuildProperties)) buildSteps.extend(common.cloneRepository()) buildSteps.extend( support.executePythonScript("Create full repo", remoteBuildMaxscale)) buildSteps.extend(common.cleanBuildDir()) buildSteps.extend(common.destroyVirtualMachine()) return buildSteps
def destroyVirtualMachine(): """Destroy virtual machine if it was not destroied after the build""" def remoteCode(): if not os.path.exists(mdbciConfig): print("MDBCI configuration does not exist") sys.exit(0) os.system("$HOME/mdbci/mdbci destroy {}".format(mdbciConfig)) def shouldRun(step): if step.getProperty( "try_already_running") == "yes" or step.getProperty( "do_not_destroy_vm") == "yes": return False return True return support.executePythonScript("Destroy leftover virtual machines", remoteCode, haltOnFailure=False, alwaysRun=True, doStepIf=shouldRun)
def remoteStoreCoredumps(): """Find the coredumps and store them in the LOGS directory""" def remote(): result = subprocess.check_output([ "{}/scripts/coredump_finder.py".format(builddir), "{}-{}".format(buildername, buildnumber), "url" ]) coredumpLogFile = open( os.path.join(builddir, "coredumps_{}".format(buildnumber)), "w") coredumpLogFile.write("COREDUMPS \\\n") if result == "": coredumpLogFile.write( "Coredumps were not found for build {}".format(buildnumber)) else: for dump in result.decode("utf_8").split("\n"): coredumpLogFile.write("{} \\\n".format(dump)) buildId = "{}-{}".format(buildername, buildnumber) shutil.copy(buildLogFile, os.path.join(HOME, "LOGS", buildId)) return support.executePythonScript("Find and store coredumps", remote, alwaysRun=True)