def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
git.clone(nupicCoreRemote, logger=logger)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
git.resetHard(sha=nupicCoreSha, logger=logger)
except CommandFailedError:
logger.exception("nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
raise
def buildNuPICCore(env, nupicCoreSHA, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSHA: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSHA)
git.resetHard(nupicCoreSHA)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
runWithOutput("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release", env,
logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError("nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote, logger=logger)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch, logger=logger)
git.resetHard(sha=nupicSha, logger=logger)
except CommandFailedError:
logger.exception("NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def buildNuPICCore(env, nupicCoreSHA, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSHA: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSHA)
git.resetHard(nupicCoreSHA)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
runWithOutput(
"cmake ../../src -DCMAKE_INSTALL_PREFIX=../release", env,
logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError(
"nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSHA,
logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSHA: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.")
with changeToWorkingDir(buildWorkspace):
git.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSHA:
try:
git.resetHard(nupicCoreSHA)
except CommandFailedError:
logger.exception(
"nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSHA)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch)
git.resetHard(nupicSha)
except CommandFailedError:
logger.exception(
"NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def prepFakerootFromGit(fakeroot, installDirectory, repoDirectory, gitURL, sha=None):
"""Clone a git repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new git clone
:param repoDirectory: what to name the cloned directory
:param gitURL: git URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s", gitURL, fakeroot, installDirectory, repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
git.resetHard()
else:
g_logger.info("No sha specified, using head of master")
gitVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot, installDirectory, repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot, installDirectory, "%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return gitVersionData
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
capnpTmp = buildCapnp(env, logger)
# install pre-reqs into the build workspace for isolation
runWithOutput(command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DCMAKE_PREFIX_PATH={} "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(
capnpTmp, libdir),
env=env, logger=logger)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
nupicBindingsEnv = env.copy()
nupicBindingsEnv["CPPFLAGS"] = "-I{}".format(
os.path.join(capnpTmp, "include"))
nupicBindingsEnv["LDFLAGS"] = "-L{}".format(
os.path.join(capnpTmp, "lib"))
command = (
"python setup.py install --prefix={} --nupic-core-dir={}".format(
buildWorkspace, os.path.join(os.getcwd(), "build", "release")))
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=nupicBindingsEnv, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
def setRemoteURL(remote, url, path, logger):
"""
Sets a git remote's url.
:param str remote: Which git remote to alter
:param str url: What to set the url to
:param str path: git directory to reset
:param logger: An initialized logger object
:raises CommandFailedError: if git set-url fails
"""
assert logger
assert isinstance(path,
basestring), ("path must be a string, but is %r" % path)
assert isinstance(
remote, basestring), ("remote must be a string, but is %r" % (remote))
assert isinstance(url,
basestring), "url must be a string, but is %r" % (url)
logger.debug("* Setting url for %s to %s in %s", remote, url, path)
with changeToWorkingDir(path):
return executeCommand(command="git set-url %s %s" % (remote, url),
logger=logger)
def clean(path, arguments, logger):
"""
Changes to path, then runs git clean.
:param str path: git directory to clean
:param str arguments: str containing optional extra command line arguments
for git clean, as you would type them on the command line. If you wanted
to do `git clean -fd`, you'd set arguments to "-fd".
:param logger: An initialized logger object
:raises CommandFailedError: if git clean fails
"""
assert logger
assert isinstance(arguments,
basestring), ("arguments must be a string, but is %r" %
arguments)
assert isinstance(path,
basestring), "path must be a string, but is %r" % path
command = ["git", "clean"]
if arguments:
command.append(arguments)
logger.debug("* Running %s in %s", command, path)
with changeToWorkingDir(path):
return executeCommand(command=command, logger=logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env, logger, nupicCoreRemote, nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
elif s3.checkIfNuPICCoreInS3(g_config, nupicCoreSha):
fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSha, logger)
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
# Cached nupic.core builds don't work on OS X, so clean it up and rebuild
if "darwin" in sys.platform:
shutil.rmtree(os.path.join(nupicCoreDir, "build"))
boolBuildNupicCore = True
logger.debug("Retrieved nupic.core from S3; saved to: %s",
nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally or in S3.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC wheel, but only upload to S3 from a linux box
cacheNuPIC(env, nupicSha, "darwin" not in sys.platform, logger)
# Cache nupic.core, but only upload to S3 from a linux box
cacheNuPICCore(env, buildWorkspace, nupicCoreSha, "darwin"
not in sys.platform, logger)
def runUnitTests(env, buildWorkspace):
"""
Calls `htm-it/run_tests.sh` to run the unit tests
:param dict env: Current environ set for HTM_IT_HOME, etc
:param str buildWorkspace: /path/to/buildWorkspace
:returns: return True if tests are successful
:rtype: bool
"""
rawResultsFile = os.path.join(buildWorkspace, "numenta-apps", "htm.it", "tests",
"results", "py2", "xunit", "jenkins",
"results.xml")
finalResultsFile = os.path.join(prepareResultsDir(),
"unit_tests_%s_results.xml" %
getBuildNumber(logger=g_logger))
with changeToWorkingDir(os.path.join(buildWorkspace, "numenta-apps", "htm.it")):
try:
runWithOutput(command=("./run_tests.sh --unit --language py --results "
"jenkins"),
env=env,
logger=g_logger)
except CommandFailedError:
g_logger.exception("Failed to run unit tests")
raise
finally:
shutil.move(rawResultsFile, finalResultsFile)
return analyzeResults(resultsPath=finalResultsFile)
def buildGrok(env, pipelineConfig, logger):
"""
Builds Grok with given Grok SHA.
:param env: The environment which will be set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:param logger: Logger object.
:raises
infrastructure.utilities.exceptions.BuildFailureException:
This exception is raised if build fails.
"""
try:
sitePackagesDir = os.path.join(env["PRODUCTS"],
"grok/lib/python2.7/site-packages")
if not os.path.exists(sitePackagesDir):
os.makedirs(sitePackagesDir)
# Setup the baseline configuration
with changeToWorkingDir(env["GROK_HOME"]):
runWithOutput("python setup.py configure_grok",
env=env,
logger=logger)
except:
logger.exception("Unknown failure")
raise BuildFailureException("Grok building failed. Exiting.")
def constructPreBuiltGrokFakeroot(self):
"""
Construct fakeroot from prebuilt grok
:returns: SHA of the products repo in the fakeroot
:rtype: tuple
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.debug("Creating %s", productsDirectory)
mkpath(productsDirectory)
copy_tree(config.productsDir, productsDirectory)
iteration = git.getCommitCount(productsDirectory)
with changeToWorkingDir(productsDirectory):
actualSHA = git.getCurrentSha()
# Set extra python path
self.setPythonPath()
# Clean Grok Scripts
self.cleanScripts()
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger):
"""
Downloads archieved nupic.core from S3
:param buildWorkspace: The workspace where nupic.core will be built
:param nupicCoreSHA: The SHA of the nupic.core build that needs to be
fetched
"""
logger.info("Downloading nupic.core from S3.")
cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA
with changeToWorkingDir(buildWorkspace):
nupicCoreFilePath = s3.downloadFileFromS3("builds.numenta.com",
"builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA,
logger)
logger.info("Untarring %s", nupicCoreFilePath)
command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir)
try:
os.makedirs(cachedDir)
runWithOutput(command, logger=logger)
except OSError:
logger.exception("Cached nupic.core already exists at %s", cachedDir)
raise
except CommandFailedError:
logger.exception("Failed while untarring cached nupic.core: %s", command)
raise
else:
logger.info("nupic.core downloaded from S3 & stored at %s", cachedDir)
def cleanseFakeroot(fakeroot, installDirectory, repoDirectory):
"""Clean up a fakeroot by running prepare_repo_for_packaging if present in
the repoDirectory. Will be called by prepFakerootFromGit and
prepFakerootFromDirectory for each of their subdirectories.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: To construct the path relative to fakeroot for
cleanup.
:param repoDirectory: It also used to construct the path relative to fakeroot
for cleanup.
"""
# Don't count on prepare_repo_for_packaging coping well if it is not run
# from the root of the repo checkout, so store the pwd and cd into the repo
# checkout before running cleaner or removing .YOMP
g_logger.debug("Cleaning fakeroot: %s", fakeroot)
workpath = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if os.path.isdir(workpath):
with changeToWorkingDir(workpath):
cleanerScript = "%s/%s/%s/prepare_repo_for_packaging" % (fakeroot,
installDirectory,
repoDirectory)
if os.path.isfile(cleanerScript):
if os.path.isfile("/tmp/noclean.rpms"):
g_logger.info("Found /tmp/noclean.rpms, skipping cleanup script")
else:
g_logger.info("Found %s, executing", cleanerScript)
runWithOutput("%s --destroy-all-my-work" % cleanerScript)
else:
g_logger.debug("Optional cleanup script %s not found, skipping",
cleanerScript)
def clean(path, arguments, logger):
"""
Changes to path, then runs git clean.
:param str path: git directory to clean
:param str arguments: str containing optional extra command line arguments
for git clean, as you would type them on the command line. If you wanted
to do `git clean -fd`, you'd set arguments to "-fd".
:param logger: An initialized logger object
:raises CommandFailedError: if git clean fails
"""
assert logger
assert isinstance(arguments, basestring), (
"arguments must be a string, but is %r" % arguments)
assert isinstance(path, basestring), "path must be a string, but is %r" % path
command = ["git", "clean"]
if arguments:
command.append(arguments)
logger.debug("* Running %s in %s", command, path)
with changeToWorkingDir(path):
return executeCommand(command=command, logger=logger)
def runUnitTests(env, buildWorkspace):
"""
Calls `grok/run_tests.sh` to run the unit tests
:param dict env: Current environ set for GROK_HOME, etc
:param str buildWorkspace: /path/to/buildWorkspace
:returns: return True if tests are successful
:rtype: bool
"""
rawResultsFile = os.path.join(buildWorkspace, "numenta-apps", "grok", "tests",
"results", "py2", "xunit", "jenkins",
"results.xml")
finalResultsFile = os.path.join(prepareResultsDir(),
"unit_tests_%s_results.xml" %
getBuildNumber(logger=g_logger))
with changeToWorkingDir(os.path.join(buildWorkspace, "numenta-apps", "grok")):
try:
runWithOutput(command=("./run_tests.sh --unit --language py --results "
"jenkins"),
env=env,
logger=g_logger)
except CommandFailedError:
g_logger.exception("Failed to run unit tests")
raise
finally:
shutil.move(rawResultsFile, finalResultsFile)
return analyzeResults(resultsPath=finalResultsFile)
def prepFakerootFromDirectory(fakeroot,
sourceDirectory,
installDirectory,
baseDirectory):
"""
Prepare a fakeroot from a directory by cloning a source directory to its
top level.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param sourceDirectory: Directory to copy from
:param baseDirectory: Where to copy the files to create the fakeroot
:param installDirectory: Where to create the baseDirectory
:raises: infrastructure.utilities.exceptions.MissingDirectoryError
if the given sourceDirectory is not found.
"""
g_logger.info("Prepping fakeroot in %s from %s", fakeroot, sourceDirectory)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if not os.path.isdir(sourceDirectory):
g_logger.error("%s is not a directory!", sourceDirectory)
raise MissingDirectoryError("Directory not found!")
targetDirectory = "%s/%s/%s" % (fakeroot, installDirectory, baseDirectory)
os.makedirs(targetDirectory)
# Find the top level files/dirs in the source directory and copy them to
# the fakeroot
sourceFiles = os.listdir(sourceDirectory)
for eachFile in sourceFiles:
g_logger.info("Copying %s to %s...", eachFile, targetDirectory)
runWithOutput("rsync --exclude '.*.un~' -av %s/%s %s" % (sourceDirectory,
eachFile,
targetDirectory))
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (baseDirectory, eachFile))
def uploadToSauceLab(apkPath, apkName, uploadName, logger):
"""
Uploads the APK to saucelab
:param apkPath: Path to the APK
:param apkName: Name of the APK in the artifacts directory
:param uploadName: Name of the apk to upload to saucelab
:param logger: An initialized logger.
:raises: CommandFailedError when the `curl` command fails.
"""
user = os.environ["SAUCE_USER_NAME"]
key = os.environ["SAUCE_KEY"]
sauceUrl = "https://saucelabs.com/rest/v1/storage"
command = ("curl -u %s:%s -X POST"
" %s/%s/%s?overwrite=true"
" -H Content-Type:application/octet-stream --data-binary @%s" %
(user, key, sauceUrl, user, uploadName, apkName))
with changeToWorkingDir(apkPath):
try:
logger.info("---------------- Uploading to saucelabs ----------------")
runWithOutput(command, logger=logger)
except CommandFailedError:
logger.exception("Failed to upload APK to saucelab.")
raise
def cleanseFakeroot(fakeroot, installDirectory, repoDirectory):
"""Clean up a fakeroot by running prepare_repo_for_packaging if present in
the repoDirectory. Will be called by prepFakerootFromGit and
prepFakerootFromDirectory for each of their subdirectories.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: To construct the path relative to fakeroot for
cleanup.
:param repoDirectory: It also used to construct the path relative to fakeroot
for cleanup.
"""
# Don't count on prepare_repo_for_packaging coping well if it is not run
# from the root of the repo checkout, so store the pwd and cd into the repo
# checkout before running cleaner or removing .git
g_logger.debug("Cleaning fakeroot: %s", fakeroot)
workpath = os.path.join(fakeroot, installDirectory, repoDirectory)
if os.path.isdir(workpath):
with changeToWorkingDir(workpath):
cleanerScript = os.path.join(fakeroot,
installDirectory,
repoDirectory,
"prepare_repo_for_packaging")
if os.path.isfile(cleanerScript):
if os.path.isfile("/tmp/noclean.rpms"):
g_logger.info("Found /tmp/noclean.rpms, skipping cleanup script")
else:
g_logger.info("Found %s, executing", cleanerScript)
runWithOutput(command=(cleanerScript, "--destroy-all-my-work"),
logger=g_logger)
else:
g_logger.debug("Optional cleanup script %s not found, skipping",
cleanerScript)
def buildHtmIt(env, pipelineConfig, logger):
"""
Builds HTM-IT with given HTM-IT SHA.
:param env: The environment which will be set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htmitRemote": "[email protected]:Numenta/numenta-apps.git",
"htmitBranch": "master",
"htmItSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:param logger: Logger object.
:raises
infrastructure.utilities.exceptions.BuildFailureException:
This exception is raised if build fails.
"""
try:
sitePackagesDir = os.path.join(env["PRODUCTS"], "htm-it/lib/python2.7/site-packages")
if not os.path.exists(sitePackagesDir):
os.makedirs(sitePackagesDir)
# Setup the baseline configuration
with changeToWorkingDir(env["HTM_IT_HOME"]):
runWithOutput("python setup.py configure_htm_it", env=env, logger=logger)
except:
logger.exception("Unknown failure")
raise BuildFailureException("HTM-IT building failed. Exiting.")
def setRemoteURL(remote, url, path, logger):
"""
Sets a git remote's url.
:param str remote: Which git remote to alter
:param str url: What to set the url to
:param str path: git directory to reset
:param logger: An initialized logger object
:raises CommandFailedError: if git set-url fails
"""
assert logger
assert isinstance(path, basestring), (
"path must be a string, but is %r" % path)
assert isinstance(remote, basestring), (
"remote must be a string, but is %r" % (remote))
assert isinstance(url, basestring), "url must be a string, but is %r" % (url)
logger.debug("* Setting url for %s to %s in %s", remote, url, path)
with changeToWorkingDir(path):
return executeCommand(command="git set-url %s %s" % (remote, url),
logger=logger)
def constructPreBuiltHTMITFakeroot(self):
"""
Construct fakeroot from prebuilt htm-it
:returns: SHA of the products repo in the fakeroot
:rtype: tuple
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.debug("Creating %s", productsDirectory)
mkpath(productsDirectory)
copy_tree(config.productsDir, productsDirectory)
iteration = git.getCommitCount(productsDirectory, logger=logger)
with changeToWorkingDir(productsDirectory):
actualSHA = git.getCurrentSha(logger=logger)
# Set extra python path
self.setPythonPath()
# Clean HTM-IT Scripts
self.cleanScripts()
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger):
"""
Downloads archieved nupic.core from S3
:param buildWorkspace: The workspace where nupic.core will be built
:param nupicCoreSHA: The SHA of the nupic.core build that needs to be
fetched
"""
logger.info("Downloading nupic.core from S3.")
cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA
with changeToWorkingDir(buildWorkspace):
nupicCoreFilePath = s3.downloadFileFromS3(
"builds.numenta.com",
"builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA, logger)
logger.info("Untarring %s", nupicCoreFilePath)
command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir)
try:
os.makedirs(cachedDir)
runWithOutput(command, logger=logger)
except OSError:
logger.exception("Cached nupic.core already exists at %s",
cachedDir)
raise
except CommandFailedError:
logger.exception("Failed while untarring cached nupic.core: %s",
command)
raise
else:
logger.info("nupic.core downloaded from S3 & stored at %s",
cachedDir)
def runTests(env, logger):
"""
Runs NuPIC tests.
:param env: The environment which will be set for runnung tests.
:raises:
infrastructure.utilities.exceptions.NupicBuildFailed
if the given SHA is not found.
"""
logger.debug("Running NuPIC Tests.")
with changeToWorkingDir(env["NUPIC"]):
try:
log.printEnv(env, logger)
runWithOutput("bin/py_region_test", env, logger)
testCommand = "scripts/run_nupic_tests -u --coverage --results xml"
runWithOutput(testCommand, env, logger)
except:
logger.exception("NuPIC Tests have failed.")
raise
else:
resultFile = glob.glob("%s/tests/results/xunit/*/*.xml" %
env["NUPIC"])[0]
logger.debug("Copying results to results folder.")
shutil.move(resultFile, createOrReplaceResultsDir())
logger.info("NuPIC tests have passed")
def runTests(env, logger):
"""
Runs NuPIC tests.
:param env: The environment which will be set for runnung tests.
:raises:
infrastructure.utilities.exceptions.NupicBuildFailed
if the given SHA is not found.
"""
logger.debug("Running NuPIC Tests.")
with changeToWorkingDir(env["NUPIC"]):
try:
log.printEnv(env, logger)
runWithOutput("bin/py_region_test", env, logger)
testCommand = "scripts/run_nupic_tests -u --coverage --results xml"
runWithOutput(testCommand, env, logger)
except:
logger.exception("NuPIC Tests have failed.")
raise
else:
resultFile = glob.glob("%s/tests/results/xunit/*/*.xml" % env["NUPIC"])[0]
logger.debug("Copying results to results folder.")
shutil.move(resultFile, createOrReplaceResultsDir())
logger.info("NuPIC tests have passed")
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def executeBuildProcess(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
:param dict env: dictionary of environment variables
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicRemote: location of nupic remote, e.g.,
https://github.com/numenta/nupic
:param str nupicBranch: which branch to build, e.g., master
:param str nupicSha: which nupic commit SHA to build
:param str nupicCoreRemote: location of nupic.core remote, e.g.,
https://github.com/numenta/nupic.core
:param str nupicCoreSha: which nupic.core commit SHA to build
:param logger:
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(env=env,
nupicCoreRemote=nupicCoreRemote,
nupicCoreSha=nupicCoreSha)
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
addNupicCoreToEnv(env, nupicCoreDir)
buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace)
buildNuPIC(env, logger, buildWorkspace)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace,
nupicCoreSha=nupicCoreSha,
logger=logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(env,
logger, nupicCoreRemote, nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
elif s3.checkIfNuPICCoreInS3(g_config, nupicCoreSha):
fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSha, logger)
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
# Cached nupic.core builds don't work on OS X, so clean it up and rebuild
if "darwin" in sys.platform:
shutil.rmtree(os.path.join(nupicCoreDir, "build"))
boolBuildNupicCore = True
logger.debug("Retrieved nupic.core from S3; saved to: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally or in S3.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC wheel, but only upload to S3 from a linux box
cacheNuPIC(env, nupicSha, "darwin" not in sys.platform, logger)
# Cache nupic.core, but only upload to S3 from a linux box
cacheNuPICCore(env, buildWorkspace, nupicCoreSha,
"darwin" not in sys.platform, logger)
def downloadOrCreateNuPICWheel(env, pipelineConfig):
"""
Downloads the NuPIC wheel from S3 for a given SHA. If a wheel is not found
for a particular SHA then NuPIC wheel is built for this SHA and uploaded to
S3. The building and uploading of NuPIC wheel is delegated to nupic.
While downloading a wheel from S3, if no NuPIC SHA is provided then the
correct wheel version is read from
"stable_nupic_version/nupic-package-version.txt" and if SHA is provided then
correct wheel version is read from "stable_nupic_version/<SHA>".
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"nupicRemote": "[email protected]:numenta/nupic.git",
"nupicBranch": "master",
"nupicSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The absolute path of the NuPIC wheel.
:rtype: string
"""
g_config = yaml.load(resource_stream(__name__, "../conf/config.yaml"))
nupicSha = pipelineConfig["nupicSha"]
if nupicSha:
path = "stable_nupic_version/%s" % nupicSha
else:
path = "stable_nupic_version/nupic-package-version.txt"
bucketName = g_config["S3_MAPPING_BUCKET"]
try:
with open(s3.downloadFileFromS3(bucketName, path, g_logger), "r") as fHandle:
contents = fHandle.readline().strip()
pipelineConfig["nupicSha"] = contents.split(":")[0].strip()
wheelFile = contents.split(":")[1].strip()
except AttributeError:
g_logger.debug("NuPIC wheel for %s not found in S3", nupicSha)
g_logger.debug("Building NuPIC wheel for %s" % nupicSha)
builder.buildNuPIC(env, pipelineConfig, g_logger)
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
pipelineConfig["nupicBuilt"] = True
else:
g_logger.debug("Downloading NuPIC wheel from S3 : %s" % wheelFile)
with changeToWorkingDir(env["BUILD_WORKSPACE"]):
wheelFilePath = s3.downloadFileFromS3(bucketName,
"builds_nupic_wheel/%s" % wheelFile,
g_logger)
pipelineConfig["nupicBuilt"] = False
return wheelFilePath
def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
diagnostics.printEnv(env=env, logger=logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
runWithOutput(
("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={}/libpython2.7.so").format(
libdir
),
env=env,
logger=logger,
)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --force"
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError("nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def buildCapnp(env, logger):
"""Builds capnp
:param dict env: The environment which will be set before building.
:param logger: An initialized logger
:returns: Prefix path for capnp.
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
mkdirp("capnp_tmp")
with changeToWorkingDir("capnp_tmp"):
runWithOutput([
"curl", "-O",
"https://capnproto.org/capnproto-c++-0.5.2.tar.gz"
],
env=env,
logger=logger)
runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"],
env=env,
logger=logger)
capnpTmp = os.getcwd()
with changeToWorkingDir("capnproto-c++-0.5.2"):
capnpEnv = env.copy()
capnpEnv["CXXFLAGS"] = (
"-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type "
"-Wunused -Wno-unused-parameter")
runWithOutput([
"./configure", "--disable-shared",
"--prefix={}".format(capnpTmp)
],
env=capnpEnv,
logger=logger)
runWithOutput("make -j4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
return capnpTmp
except Exception:
logger.exception("capnp building failed due to unknown reason.")
raise
else:
logger.info("capnp building was successful.")
def preBuildSetup(env, pipelineConfig):
"""
Clone the Grok repo if needed and get it set to the right remote, branch,
and SHA. Once set, if the NuPIC parameters need to be revised, take care
of that now, too.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"nupicRemote": "[email protected]:numenta/nupic.git",
"nupicBranch": "master",
"nupicSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file",
"wheelFilePath": "/path/to/wheel/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone Grok if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["GROK_HOME"]):
git.clone(pipelineConfig["grokRemote"], directory="products")
with changeToWorkingDir(env["GROK_HOME"]):
if pipelineConfig["grokSha"]:
g_logger.debug("Resetting to %s" % pipelineConfig["grokSha"])
git.resetHard(pipelineConfig["grokSha"])
else:
grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
pipelineConfig["grokBranch"])
pipelineConfig["grokSha"] = grokSha
g_logger.debug("Resetting to %s" % grokSha)
git.resetHard(grokSha)
wheelFilePath = downloadOrCreateNuPICWheel(env, pipelineConfig)
pipelineConfig["wheelFilePath"] = wheelFilePath
def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
diagnostics.printEnv(env=env, logger=logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(
("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir),
env=env,
logger=logger)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --force"
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError(
"nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def prepFakerootFromGit(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None):
"""Clone a YOMP repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new YOMP clone
:param repoDirectory: what to name the cloned directory
:param YOMPURL: YOMP URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s",
YOMPURL,
fakeroot,
installDirectory,
repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
YOMP.resetHard()
else:
g_logger.info("No sha specified, using head of master")
YOMPVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot,
installDirectory,
repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return YOMPVersionData
def buildGrok(env, pipelineConfig, logger):
"""
Builds Grok with given Grok SHA.
:param env: The environment which will be set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"nupicRemote": "[email protected]:numenta/nupic.git",
"nupicBranch": "master",
"nupicSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file",
"wheelFilePath": "/path/to/wheel/file"
}
:param logger: Logger object.
:raises
infrastructure.utilities.exceptions.BuildFailureException:
This exception is raised if build fails.
"""
try :
sitePackagesDir = os.path.join(env["PRODUCTS"],
"grok/lib/python2.7/site-packages")
if not os.path.exists(sitePackagesDir):
os.makedirs(sitePackagesDir)
with changeToWorkingDir(env["PRODUCTS"]):
installNuPICWheel(env, env["GROK_HOME"],
pipelineConfig["wheelFilePath"], logger)
# Setup the baseline configuration
with changeToWorkingDir(env["GROK_HOME"]):
runWithOutput("python setup.py configure_grok", env=env, logger=logger)
except:
logger.exception("Unknown failure")
raise BuildFailureException("Grok building failed. Exiting.")
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env=env, nupicCoreRemote=nupicCoreRemote, nupicCoreSha=nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace,
nupicCoreSha=nupicCoreSha,
logger=logger)
def getNuPICCoreDetails(env, logger):
"""
Reads .nupic_modules to find nupic.core SHA and remote.
:param env: The environment dict.
:returns: A tuple consisting of `string` representing nupic.core remote
and `string` representing nupicCoreSHA
"""
with changeToWorkingDir(env["NUPIC"]):
locals = {}
execfile(".nupic_modules", {}, locals)
return locals["NUPIC_CORE_REMOTE"], locals["NUPIC_CORE_COMMITISH"]
def getNuPICCoreDetails(env, logger):
"""
Reads .nupic_modules to find nupic.core SHA and remote.
:param env: The environment dict.
:returns: A tuple consisting of `string` representing nupic.core remote
and `string` representing nupicCoreSHA
"""
with changeToWorkingDir(env["NUPIC"]):
locals = {}
execfile(".nupic_modules", {}, locals)
return locals["NUPIC_CORE_REMOTE"], locals["NUPIC_CORE_COMMITISH"]
def buildCapnp(env, logger):
"""Builds capnp
:param dict env: The environment which will be set before building.
:param logger: An initialized logger
:returns: Prefix path for capnp.
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
mkdirp("capnp_tmp")
with changeToWorkingDir("capnp_tmp"):
runWithOutput(
["curl", "-O", "https://capnproto.org/capnproto-c++-0.5.2.tar.gz"],
env=env, logger=logger)
runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"],
env=env, logger=logger)
capnpTmp = os.getcwd()
with changeToWorkingDir("capnproto-c++-0.5.2"):
capnpEnv = env.copy()
capnpEnv["CXXFLAGS"] = (
"-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type "
"-Wunused -Wno-unused-parameter")
runWithOutput(
["./configure", "--disable-shared",
"--prefix={}".format(capnpTmp)],
env=capnpEnv, logger=logger)
runWithOutput("make -j4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
return capnpTmp
except Exception:
logger.exception("capnp building failed due to unknown reason.")
raise
else:
logger.info("capnp building was successful.")
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htm-itRemote": "[email protected]:Numenta/numenta-apps.git",
"htm-itBranch": "master",
"htm-itSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM-IT_HOME"]):
git.clone(gitURL=pipelineConfig["htm-itRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM-IT_HOME"]):
if pipelineConfig["htm-itSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htm-itSha"])
git.resetHard(sha=pipelineConfig["htm-itSha"], logger=g_logger)
else:
htm-itSha = git.getShaFromRemoteBranch(pipelineConfig["htm-itRemote"],
pipelineConfig["htm-itBranch"],
logger=g_logger)
pipelineConfig["htm-itSha"] = htm-itSha
g_logger.debug("Resetting to %s", htm-itSha)
git.resetHard(sha=htm-itSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htmitRemote": "[email protected]:Numenta/numenta-apps.git",
"htmitBranch": "master",
"htmItSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM_IT_HOME"]):
git.clone(gitURL=pipelineConfig["htmitRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM_IT_HOME"]):
if pipelineConfig["htmItSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htmItSha"])
git.resetHard(sha=pipelineConfig["htmItSha"], logger=g_logger)
else:
htmItSha = git.getShaFromRemoteBranch(pipelineConfig["htmitRemote"],
pipelineConfig["htmitBranch"],
logger=g_logger)
pipelineConfig["htmItSha"] = htmItSha
g_logger.debug("Resetting to %s", htmItSha)
git.resetHard(sha=htmItSha, logger=g_logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env=env, nupicCoreRemote=nupicCoreRemote, nupicCoreSha=nupicCoreSha
)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace, nupicCoreSha=nupicCoreSha, logger=logger)
def getNuPICCoreDetails(env, logger, nupicCoreRemote, nupicCoreSha):
"""
Reads .nupic_modules to find nupic.core SHA and remote.
:param env: The environment dict.
:returns: A tuple consisting of `string` representing nupic.core remote
and `string` representing nupicCoreSha and a `string` representing the branch
"""
with changeToWorkingDir(env["NUPIC"]):
core = {}
execfile(".nupic_modules", {}, core)
remote = core["NUPIC_CORE_REMOTE"] if not nupicCoreRemote else nupicCoreRemote
committish = core["NUPIC_CORE_COMMITISH"] if nupicCoreSha == "None" else nupicCoreSha
return remote, committish
def getCommitCount(path):
"""
Get the commit count from a YOMP directory tree
@param: path to YOMP directory
@raises: infrastructure.utilities.exceptions.CommandFailedError:
if path isn't in a YOMP checkout
@returns: total commit count for the YOMP directory
@rtype: string
"""
with changeToWorkingDir(path):
return executeCommand("YOMP rev-list HEAD --count")
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)
def getCommitCount(path):
"""
Get the commit count from a YOMP directory tree
@param: path to YOMP directory
@raises: infrastructure.utilities.exceptions.CommandFailedError:
if path isn't in a YOMP checkout
@returns: total commit count for the YOMP directory
@rtype: string
"""
with changeToWorkingDir(path):
return executeCommand("YOMP rev-list HEAD --count")
def getNuPICCoreDetails(env, nupicCoreRemote, nupicCoreSha):
"""
Reads .nupic_modules to find nupic.core SHA and remote.
:param env: The environment dict.
:returns: A tuple consisting of `string` representing nupic.core remote
and `string` representing nupicCoreSha
"""
with changeToWorkingDir(env["NUPIC"]):
core = {}
execfile(".nupic_modules", {}, core)
remote = core["NUPIC_CORE_REMOTE"] if not nupicCoreRemote else nupicCoreRemote
committish = (
core["NUPIC_CORE_COMMITISH"] if nupicCoreSha == "None" else nupicCoreSha)
return remote, committish
def loadGitDescribeFromDirectory(gitDirectory):
"""
Load & parse git describe data from gitDirectory
:param gitDirectory: path to a git clone.
"""
versionData = {}
with changeToWorkingDir(gitDirectory):
try:
rawVersion = runWithOutput("git describe --long --tags --abbrev=40").strip().split("-")
versionData["version"] = rawVersion[0]
versionData["commitsSinceTag"] = rawVersion[1]
versionData["sha"] = rawVersion[2]
except RuntimeError:
versionData = None
return versionData
def buildNuPIC(env, logger, buildWorkspace):
"""
Builds NuPIC
:param env: The environment which will be set before building
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
# Build
with changeToWorkingDir(env["NUPIC"]):
try:
try:
shutil.rmtree("build")
except OSError:
# didn't exist, so just pass
pass
# install requirements
command = ("pip", "install",
"--install-option=--prefix=%s" % env["NTA"],
"--requirement", "external/common/requirements.txt")
runWithOutput(command=command, env=env, logger=logger)
# also install pycapnp
command = ("pip", "install",
"--install-option=--prefix=%s" % env["NTA"],
"pycapnp==0.5.5")
runWithOutput(command=command, env=env, logger=logger)
# need to remove this folder for wheel build to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --prefix=%s" % buildWorkspace
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except:
logger.exception("Failed while building nupic")
raise
else:
open("nupic.stamp", "a").close()
logger.debug("NuPIC building was successful.")
def buildNuPIC(env, logger):
"""
Builds NuPIC
:param env: The environment which will be set before building
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building NuPIC------------"
log.printEnv(env, logger)
# Build
with changeToWorkingDir(env["NUPIC"]):
try:
try:
shutil.rmtree("build")
except OSError:
# didn't exist, so just pass
pass
# install requirements
runWithOutput(
"pip install --install-option=--prefix=%s --requirement "
"external/common/requirements.txt" % env["NTA"],
env=env,
logger=logger)
# need to remove this folder for wheel build to work
shutil.rmtree("external/linux32arm")
# build the wheel
command = (
"python setup.py bdist_wheel bdist_egg --nupic-core-dir=%s" %
os.path.join(env["NUPIC_CORE_DIR"], "build", "release"))
# Building on jenkins, not local
if "JENKINS_HOME" in env:
command += " upload -r numenta-pypi"
runWithOutput(command, env=env, logger=logger)
except:
logger.exception("Failed while building nupic")
raise NupicBuildFailed("NuPIC building failed.")
else:
open("nupic.stamp", "a").close()
logger.debug("NuPIC building was successful.")
def loadGitDescribeFromDirectory(YOMPDirectory):
"""
Load & parse YOMP describe data from YOMPDirectory
:param YOMPDirectory: path to a YOMP clone.
"""
versionData = {}
with changeToWorkingDir(YOMPDirectory):
try:
rawVersion = runWithOutput("YOMP describe --long --tags --abbrev=40")\
.strip().split("-")
versionData["version"] = rawVersion[0]
versionData["commitsSinceTag"] = rawVersion[1]
versionData["sha"] = rawVersion[2]
except RuntimeError:
versionData = None
return versionData
def getCommitCount(path, logger):
"""
Get the commit count from a git directory tree
:param str path: path to git directory
:param logger: logger for additional debug info
:raises infrastructure.utilities.exceptions.CommandFailedError:
if path isn't in a git checkout
:returns: total commit count for the git directory
:rtype: str
"""
with changeToWorkingDir(path):
command = ("git", "rev-list", "HEAD", "--count")
return executeCommand(command=command, logger=logger)
def cacheNuPICCore(buildWorkspace, nupicCoreSha, logger):
"""
Caches nupic.core to /var/build/NuPIC.core/<SHA>
:param buildWorkspace: The buildWorkspace were nupic.core is built
:param nupicSha: A `string` representing SHA
:param logger: initialized logger
:raises CommandFailedError: if the tar process fails before upload.
"""
cachedPath = "/var/build/nupic.core/%s" % nupicCoreSha
if not os.path.isdir(cachedPath):
logger.info("Caching nupic.core to %s", cachedPath)
with changeToWorkingDir(buildWorkspace):
shutil.copytree("nupic.core", ("/var/build/nupic.core/%s/nupic.core" %
nupicCoreSha))
else:
logger.debug("Cached nupic.core already exists.")
def cacheNuPICCore(env, buildWorkspace, nupicCoreSHA, uploadToS3, logger):
"""
Caches nupic.core to /var/build/NuPIC.core/<SHA> and uploads to S3
:param env: The environment dict
:param buildWorkspace: The buildWorkspace were nupic.core is built
:param nupicSha: A `string` representing SHA
:param uploadToS3: `boolean` defining whether to upload to S3 or not
:raises: CommandFailedError if the tar process fails before upload.
"""
cachedPath = "/var/build/nupic.core/%s" % nupicCoreSHA
if not os.path.isdir(cachedPath):
logger.info("Caching nupic.core to %s", cachedPath)
with changeToWorkingDir(buildWorkspace):
shutil.copytree(
"nupic.core",
("/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA))
if uploadToS3:
nupicCoreZip = "nupic.core-%s.zip" % nupicCoreSHA
logger.info("Archiving nupic.core to %s", nupicCoreZip)
command = "tar czf %s nupic.core" % nupicCoreZip
nupicCoreZipPath = "%s/%s" % (buildWorkspace, nupicCoreZip)
try:
runWithOutput(command, env, logger=logger)
logger.debug("Uploading %s to S3.", nupicCoreZip)
s3.uploadToS3(g_config, nupicCoreZipPath,
"builds_nupic_core", logger)
except:
logger.exception("Archiving nupic.core failed.")
raise CommandFailedError("Archiving nupic.core failed.")
else:
logger.info("nupic.core cached locally and to S3.")
else:
logger.debug("Cached nupic.core already exists.")
def runUnitTests(env, pipeline, grokSha, logger):
"""
Runs tests listed in files present at {GROK_HOME}/tests/ci/
:param env: Current environ set for GROK_HOME, etc
:param pipeline: name of repository which has triggered this build
:param grokSha: grok SHA used current run
:returns: return True if tests are successful
:rtype: bool
"""
# Print environment for debug purposes
printEnv(env, logger)
buildWorkspace = os.environ["BUILD_WORKSPACE"]
task = "_".join([pipeline, grokSha, str(uuid.uuid4())])
xunitSuccess = True
with open(
os.path.join(env["GROK_HOME"], "tests/ci/test_commands_xunit.txt"),
"r") as tests:
xunitTests = [test.strip() % dict(globals().items() + \
locals().items()) for test in tests]
with changeToWorkingDir(os.path.join(buildWorkspace, "products")):
g_logger.debug(os.getcwd())
for xunitTest in xunitTests:
logger.info("-------Running %s -------" % xunitTest)
xunitSuccess = runTestCommand(xunitTest, env, logger)
logger.info("\n\n###### COMPLETED %s tests ######\n\n" % xunitTest)
if "WORKSPACE" in os.environ:
# `WORKSPACE` should only be set by Jenkins and we only want to record
# the test results if we're on Jenkins
logger.info("\n\n###### Recording Results %s######\n\n" %
xunitTest)
recordXunitTestsResults(task)
if not xunitSuccess:
logger.error("-------Failed %s -------" % xunitTest)
break
return xunitSuccess
