def buildTestSourceReleaseQA(releaseType, releaseTarball, srcDir, binaryTarball=None, short=False, testlist=None, nprocs=1):
srcTestDir = utils.createTemporaryDirectory("release_testing")
if releaseType == "binary":
tarball = releaseTarball
if testlist is None:
testlist = "binary"
elif (releaseType in sourceReleaseTypes):
tarball = binaryTarball
if testlist is None:
testlist = releaseType
else:
raise Exception("buildTestSourceReleaseQA: unknown release type %s" % releaseType)
if tarball == None:
raise Exception("buildTestSourceReleaseQA: no binary tarball specified for release type %s" % releaseType)
testdir = utils.extractArchive(tarball, srcTestDir)
log.info("Testing customer release %s in directory %s" % (releaseType, testdir))
if releaseType != "binary":
try:
print releaseType
print srcDir
print releaseTarball
print binaryTarball
print testdir
buildCustomerRelease(releaseType, srcDir, releaseTarball, binaryTarball, testdir)
except Exception, e:
log.error("Source release failed: %s", e)
# note: leaves temp directory intact
return(list(), ["build source release"], list())
def testCustomerRelease(releaseType, releaseTarball, srcDir, binaryTarball=None, short=True, testlist=None, buildOnly=False, nprocs=1,parentTmpDir=None):
"""
Test a customer release by
binary release and vitamind release:
extract it
run tests
source release
extract binary release
extract source release
build source release and install into binary releaes
run tests
@param releaseType One of binaryReleaseTypes or sourceReleaseTypes
@param releaseTarball Absolute pathname to the tarball being tested
@param srcDir Absolute path to source tree (where tests may reside)
@param binaryTarball Absolute pathname to the binary tarball (used/required only
for the tools source release if buildOnly is False)
@param short If true, run short versions of hte tests
@param testlist Name of testlist. If None, then defaults to release type
@param buildOnly For source releases, just build -- don't run the tests.
@retval tuple of (pass, fail, disabled) tests
"""
# Require binary tarball if we're building a source release and
# either 1) we will test the release or 2) the release requires spnupic libraries to build
if releaseType in sourceReleaseTypes and binaryTarball is None:
if buildOnly == False or releaseType in releasesRequiringSPNupic:
raise Exception("testCustomerRelease: no binary tarball specified for release type %s" % releaseType)
if testlist is None:
testlist = releaseType
tmpDir = utils.createTemporaryDirectory("release_testing", parentDir=parentTmpDir)
if releaseType in binaryReleaseTypes:
tarballToExtract = releaseTarball
elif releaseType in sourceReleaseTypes:
tarballToExtract = binaryTarball
else:
raise Exception("testCustomerRelease: unknown release type '%s'" % releaseType)
if tarballToExtract is not None:
testdir = utils.extractArchive(tarballToExtract, tmpDir)
else:
testdir = os.path.join(tmpDir, "binary")
utils.createDir(testdir)
passed = list()
failed = list()
disabled = list()
log.info("Testing customer release %s in directory %s" % (releaseType, testdir))
if releaseType in sourceReleaseTypes:
try:
buildCustomerRelease(releaseType, srcDir, releaseTarball, binaryTarball, testdir, nprocs=nprocs)
except Exception, e:
log.error("Source release failed: %s", e)
failed = ["build source release"]
def buildPlugins(binaryTarball, pluginTarball, srcDir, integrateScript=None, pluginList=None):
"""
Test a plugin source release by
1. extract binary tarball
2. extract algorithm source tarball
3. integrate test code into algorithm source tarball
if integrate script is not None, run: integrateScript algorithmSrcDir srcDir
4. build from algorithm source and install into binary release
4a. move all plugins from lib to testlib directory in binary release
@param binaryTarball Absolute pathname to the binary tarball
@param pluginTarball Absolute path to plugin source release tarball
@param srcDir Absolute path to source tree (where tests may reside)
@param integrateScript Script for integrating test code into algorithm source directory
@param testlist Name of testlist. If None, no tests are performed.
@param short If True, short version of tests is run
@param pluginList File containing a list of directories that will be integrated into the source
Must be set if integrateScript is set
@retval tuple of (binaryDir, tmpDir)
binaryDir is in tmpDir; tmpDir must be deleted by called
"""
# XXX PORT FOR WINDOWS
# METHOD IS CURRENTLY UNUSED?
if integrateScript is not None:
assert os.path.exists(integrateScript), "integrateScript '%s' does not exist" % integrateScript
assert pluginList is not None
assert os.path.exists(pluginList), "pluginList '%s' does not exist" % pluginList
tmpDir = utils.backquote("mktemp -d -q /tmp/plugin_test.XXXXXX")
binaryDir = utils.extractArchive(binaryTarball, tmpDir)
pluginSrcDir = utils.extractArchive(pluginTarball, tmpDir)
try:
# Move the original plugins out of the way. Save them in case we want to use t hem.
libDir = os.path.join(binaryDir, "lib")
origPluginDir = os.path.join(binaryDir, "origLib")
os.mkdir(origPluginDir)
for file in os.listdir(libDir):
(base, ext) = os.path.splitext(file)
if ext == ".so" or ext == ".dylib":
os.rename(os.path.join(libDir, file),
os.path.join(origPluginDir, file))
if integrateScript is not None:
command = ["python", integrateScript, pluginSrcDir, srcDir, pluginList]
utils.runCommand(command)
buildDir = os.path.join(tmpDir, "build")
os.mkdir(buildDir)
if integrateScript is not None:
utils.changeDir(pluginSrcDir)
command = "/bin/sh autogen.sh"
utils.runCommand(command)
# configure
utils.changeDir(buildDir)
command = "%s/configure --prefix=%s" % (pluginSrcDir, binaryDir)
utils.runCommand(command)
# build
command = "make -k install"
utils.runCommand(command)
# With an integrate script, put all the plugins in a "testLib" directory
# and let the tests copy in the plugins they need
if integrateScript is not None:
testPluginDir = os.path.join(binaryDir, "testLib")
os.mkdir(testPluginDir)
for file in os.listdir(libDir):
(base, ext) = os.path.splitext(file)
if ext == ".so" or ext == ".dylib":
os.rename(os.path.join(libDir, file),
os.path.join(testPluginDir, file))
except Exception, e:
# import traceback
# traceback.print_exc()
raise Exception("Caught exception in integrate/build: %s" % e)
def buildCustomerRelease(releaseType, releaseSpecDir, releaseTarball, binaryTarball, prefix=None, nprocs=1,parentTmpDir=None):
"""From the given source release tarball, compile and verify that the
build product is correct.
@param releaseType -- one of the source release types
@param releaseSpecDir -- trunk/release directory containing manifest files for this release
@param releaseTarball -- tarball containing the release to be built.
@param binaryTarball -- tarball containing the binary release. This must be provided
when building the tools source release, which requires prebuilt libraries that
only exist in the binary release. These libraries are copied in to the source
release tree prior to building the source release.
Throws an exception if the build fails or the build product does not
exactly match the manifest file.
If prefix is not None, install the build product into the prefix directory
No return value"""
log.info("Building customer %s release", releaseType)
origDir = os.getcwd()
if not releaseType in sourceReleaseTypes:
raise Exception("buildCustomerRelease: unknown release type %s" % releaseType)
tarballBasename = os.path.basename(releaseTarball)
(dirName, ext) = os.path.splitext(tarballBasename)
manifest = os.path.abspath(os.path.join(releaseSpecDir,
"release",
releaseType+"_release",
"manifests",
"build_output.manifest"))
tempdir = utils.createTemporaryDirectory("build_"+releaseType, parentDir=parentTmpDir)
installdir = os.path.join(tempdir, "install")
utils.createDir(installdir)
builddir = os.path.join(tempdir, "build")
utils.createDir(builddir)
srcdir = utils.extractArchive(releaseTarball, tempdir)
# Use the precompiled runtime library for the full source build.
if releaseType in releasesRequiringSPNupic:
if binaryTarball is None:
raise Exception("No binary tarball provided when building tools release, which requires prebuilt libraries")
binarydir = utils.extractArchive(binaryTarball, tempdir)
log.info("Copying libruntime.a from binary release into source release")
if getArch() != "win32":
smartCopy(os.path.join(binarydir, "lib", "libruntime.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libruntime.a"))
smartCopy(os.path.join(binarydir, "lib", "libnode.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libnode.a"))
smartCopy(os.path.join(binarydir, "lib", "libipcserial.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libipcserial.a"))
else:
smartCopy(os.path.join(binarydir, "lib", "release", "runtime.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "runtime.lib"))
smartCopy(os.path.join(binarydir, "lib", "release", "node.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "node.lib"))
smartCopy(os.path.join(binarydir, "lib", "release", "ipcserial.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "ipcserial.lib"))
build.build(srcdir, builddir, installdir, assertions=False, customerRelease=True, nprocs=nprocs)
# To verify the build product, we install from manifest and then
# confirm that the installed tree has the same number of files as the original tree.
# An exception in the installFromManifest phase usually means there is a file in
# the manifest that was not built. Then if the number of files is the same, it usually
# means that a file was built that was not in the manifest.
tempInstallDir = os.path.join(tempdir, "temp_install")
installFromManifest(manifest, installdir, tempInstallDir,
level=0, overwrite=False, destdirExists=False)
installedFiles = []
for root,dirs,files in os.walk(tempInstallDir):
installedFiles += [os.path.join(root, f) for f in files]
# normalize the list of files
prefixLen = len(tempInstallDir)
installedFiles = [f[prefixLen:] for f in installedFiles]
installedFiles = sorted(installedFiles)
builtFiles = []
for root,dirs,files in os.walk(installdir):
builtFiles += [os.path.join(root, f) for f in files]
# normalize the list of files
prefixLen = len(installdir)
builtFiles = [f[prefixLen:] for f in builtFiles]
builtFiles = sorted(builtFiles)
if len(installedFiles) != len(builtFiles):
log.error("Manifest error when building %s release", releaseType)
log.error("Built files: %d Files in manifest: %d", len(builtFiles), len(installedFiles))
# we know that every file in installedFiles is also in builtFiles
# because installed dir was created from built dir
for f in installedFiles:
builtFiles.remove(f)
for f in builtFiles:
log.error("File '%s' installed but not in build_output.manifest" % f)
raise Exception("Error building %s release -- file(s) missing from build_output.manifest" % releaseType)
else:
log.info("Release %s built successfully. Build produces %d files", releaseType, len(builtFiles))
# copy all built files from directory where built into the the prefix dir, if specified
if prefix is not None:
log.info("Installing build products from '%s' release into binary release at '%s'",
releaseType, prefix)
installFromManifest(manifest, installdir, prefix,
level=0, overwrite=True, destdirExists=True)
# clean up
os.chdir(origDir)
utils.remove(tempdir)
