def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
git.clone(nupicCoreRemote, logger=logger)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
git.resetHard(sha=nupicCoreSha, logger=logger)
except CommandFailedError:
logger.exception("nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
raise
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch)
git.resetHard(nupicSha)
except CommandFailedError:
logger.exception(
"NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSHA,
logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSHA: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.")
with changeToWorkingDir(buildWorkspace):
git.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSHA:
try:
git.resetHard(nupicCoreSHA)
except CommandFailedError:
logger.exception(
"nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSHA)
def buildNuPICCore(env, nupicCoreSHA, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSHA: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSHA)
git.resetHard(nupicCoreSHA)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
runWithOutput(
"cmake ../../src -DCMAKE_INSTALL_PREFIX=../release", env,
logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError(
"nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def buildNuPICCore(env, nupicCoreSHA, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSHA: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSHA)
git.resetHard(nupicCoreSHA)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
runWithOutput("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release", env,
logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError("nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def prepFakerootFromGit(fakeroot, installDirectory, repoDirectory, gitURL, sha=None):
"""Clone a git repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new git clone
:param repoDirectory: what to name the cloned directory
:param gitURL: git URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s", gitURL, fakeroot, installDirectory, repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
git.resetHard()
else:
g_logger.info("No sha specified, using head of master")
gitVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot, installDirectory, repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot, installDirectory, "%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return gitVersionData
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote, logger=logger)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch, logger=logger)
git.resetHard(sha=nupicSha, logger=logger)
except CommandFailedError:
logger.exception("NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
capnpTmp = buildCapnp(env, logger)
# install pre-reqs into the build workspace for isolation
runWithOutput(command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DCMAKE_PREFIX_PATH={} "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(
capnpTmp, libdir),
env=env, logger=logger)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
nupicBindingsEnv = env.copy()
nupicBindingsEnv["CPPFLAGS"] = "-I{}".format(
os.path.join(capnpTmp, "include"))
nupicBindingsEnv["LDFLAGS"] = "-L{}".format(
os.path.join(capnpTmp, "lib"))
command = (
"python setup.py install --prefix={} --nupic-core-dir={}".format(
buildWorkspace, os.path.join(os.getcwd(), "build", "release")))
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=nupicBindingsEnv, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
diagnostics.printEnv(env=env, logger=logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
runWithOutput(
("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={}/libpython2.7.so").format(
libdir
),
env=env,
logger=logger,
)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --force"
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError("nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def preBuildSetup(env, pipelineConfig):
"""
Clone the Grok repo if needed and get it set to the right remote, branch,
and SHA. Once set, if the NuPIC parameters need to be revised, take care
of that now, too.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"nupicRemote": "[email protected]:numenta/nupic.git",
"nupicBranch": "master",
"nupicSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file",
"wheelFilePath": "/path/to/wheel/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone Grok if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["GROK_HOME"]):
git.clone(pipelineConfig["grokRemote"], directory="products")
with changeToWorkingDir(env["GROK_HOME"]):
if pipelineConfig["grokSha"]:
g_logger.debug("Resetting to %s" % pipelineConfig["grokSha"])
git.resetHard(pipelineConfig["grokSha"])
else:
grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
pipelineConfig["grokBranch"])
pipelineConfig["grokSha"] = grokSha
g_logger.debug("Resetting to %s" % grokSha)
git.resetHard(grokSha)
wheelFilePath = downloadOrCreateNuPICWheel(env, pipelineConfig)
pipelineConfig["wheelFilePath"] = wheelFilePath
def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
diagnostics.printEnv(env=env, logger=logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(
("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir),
env=env,
logger=logger)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --force"
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError(
"nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def prepFakerootFromGit(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None):
"""Clone a git repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new git clone
:param repoDirectory: what to name the cloned directory
:param gitURL: git URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL=gitURL, logger=g_logger, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(pathspec=sha, logger=g_logger)
git.resetHard(sha=sha, logger=g_logger)
else:
g_logger.info("No sha specified, using head of master")
gitVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot,
installDirectory,
repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return gitVersionData
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htm-itRemote": "[email protected]:Numenta/numenta-apps.git",
"htm-itBranch": "master",
"htm-itSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM-IT_HOME"]):
git.clone(gitURL=pipelineConfig["htm-itRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM-IT_HOME"]):
if pipelineConfig["htm-itSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htm-itSha"])
git.resetHard(sha=pipelineConfig["htm-itSha"], logger=g_logger)
else:
htm-itSha = git.getShaFromRemoteBranch(pipelineConfig["htm-itRemote"],
pipelineConfig["htm-itBranch"],
logger=g_logger)
pipelineConfig["htm-itSha"] = htm-itSha
g_logger.debug("Resetting to %s", htm-itSha)
git.resetHard(sha=htm-itSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htmitRemote": "[email protected]:Numenta/numenta-apps.git",
"htmitBranch": "master",
"htmItSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM_IT_HOME"]):
git.clone(gitURL=pipelineConfig["htmitRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM_IT_HOME"]):
if pipelineConfig["htmItSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htmItSha"])
git.resetHard(sha=pipelineConfig["htmItSha"], logger=g_logger)
else:
htmItSha = git.getShaFromRemoteBranch(pipelineConfig["htmitRemote"],
pipelineConfig["htmitBranch"],
logger=g_logger)
pipelineConfig["htmItSha"] = htmItSha
g_logger.debug("Resetting to %s", htmItSha)
git.resetHard(sha=htmItSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
"""
Clone the Grok repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone Grok if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["GROK_HOME"]):
git.clone(pipelineConfig["grokRemote"], directory="products")
with changeToWorkingDir(env["GROK_HOME"]):
if pipelineConfig["grokSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["grokSha"])
git.resetHard(pipelineConfig["grokSha"])
else:
grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
pipelineConfig["grokBranch"])
pipelineConfig["grokSha"] = grokSha
g_logger.debug("Resetting to %s", grokSha)
git.resetHard(grokSha)
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4)
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
if isVersionGreaterOrEqual(nupicVersion, "0.3.5"):
srcDir = "../.."
else:
srcDir = "../../src"
pyExtensionsDir = "../../bindings/py/nupic/bindings"
# install pre-reqs into the build workspace for isolation
runWithOutput(command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env, logger=logger)
# also install pycapnp
command = ("pip", "install",
"--install-option=--prefix=%s" % buildWorkspace,
"pycapnp==0.5.5")
runWithOutput(command=command, env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
includeDir = sysconfig.get_config_var("INCLUDEPY")
runWithOutput(("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so "
"-DPYTHON_INCLUDE_DIR={pythonIncludeDir} "
"-DPY_EXTENSIONS_DIR={pyExtensionsDir}").format(
srcDir=srcDir,
pythonLibDir=libdir,
pythonIncludeDir=includeDir,
pyExtensionsDir=pyExtensionsDir),
env=env, logger=logger)
newEnv = dict(env)
newEnv["VERBOSE"] = "1"
runWithOutput("make -j 1", env=newEnv, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --prefix={}".format(buildWorkspace)
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4)
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
capnpTmp = buildCapnp(env, logger)
if isVersionGreaterOrEqual(nupicVersion, "0.3.5"):
srcDir = "../.."
else:
srcDir = "../../src"
# install pre-reqs into the build workspace for isolation
runWithOutput(
command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env,
logger=logger)
# also install pycapnp
command = ("pip", "install",
"--install-option=--prefix=%s" % buildWorkspace,
"pycapnp==0.5.5")
runWithOutput(command=command, env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
includeDir = sysconfig.get_config_var("INCLUDEPY")
runWithOutput(
("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release "
"-DCMAKE_PREFIX_PATH={capnpPrefixPath} "
"-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so "
"-DPYTHON_INCLUDE_DIR={pythonIncludeDir}").format(
srcDir=srcDir,
capnpPrefixPath=capnpTmp,
pythonLibDir=libdir,
pythonIncludeDir=includeDir),
env=env,
logger=logger)
runWithOutput("VERBOSE=1 make -j 1", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
nupicBindingsEnv = env.copy()
nupicBindingsEnv["CPPFLAGS"] = "-I{}".format(
os.path.join(capnpTmp, "include"))
nupicBindingsEnv["LDFLAGS"] = "-L{}".format(
os.path.join(capnpTmp, "lib"))
command = (
"python setup.py install --prefix={} --nupic-core-dir={}".
format(buildWorkspace,
os.path.join(os.getcwd(), "build", "release")))
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=nupicBindingsEnv, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
