def main(args):
"""
Main function for the pipeline. Executes all sub-tasks
:param dict args: Parsed command line arguments
:returns: /path/to/pipelineJson
:rtype: str
"""
logger = initPipelineLogger("manifest", logLevel=args.logLevel)
buildWorkspace = os.environ.get(
"BUILD_WORKSPACE", jenkins.defineBuildWorkspace(logger=logger))
mkdirp(buildWorkspace)
manifest = vars(args)
# Update buildWorkspace in manifest section for pipelineJson
manifest.update({"buildWorkspace": buildWorkspace})
manifestEnv = {"manifest": manifest}
with open("%s/%s_pipeline.json" % (buildWorkspace, args.pipeline),
'w') as fp:
fp.write(json.dumps(manifestEnv, ensure_ascii=False))
logger.debug(json.dumps(manifestEnv))
pipelineJsonPath = "%s/%s_pipeline.json" % (buildWorkspace, args.pipeline)
logger.info("Pipeline JSON path: %s", pipelineJsonPath)
return pipelineJsonPath
def main(args):
"""
Main function for the pipeline. Executes all sub-tasks
:param dict args: Parsed command line arguments
:returns: /path/to/pipelineJson
:rtype: str
"""
logger = initPipelineLogger("manifest", logLevel=args.logLevel)
buildWorkspace = os.environ.get("BUILD_WORKSPACE",
jenkins.defineBuildWorkspace(logger=logger))
mkdirp(buildWorkspace)
manifest = vars(args)
# Update buildWorkspace in manifest section for pipelineJson
manifest.update({"buildWorkspace": buildWorkspace})
manifestEnv = {"manifest": manifest}
with open("%s/%s_pipeline.json" % (buildWorkspace, args.pipeline), 'w') as fp:
fp.write(json.dumps(manifestEnv, ensure_ascii=False))
logger.debug(json.dumps(manifestEnv))
pipelineJsonPath = "%s/%s_pipeline.json" % (buildWorkspace, args.pipeline)
logger.info("Pipeline JSON path: %s", pipelineJsonPath)
return pipelineJsonPath
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
capnpTmp = buildCapnp(env, logger)
# install pre-reqs into the build workspace for isolation
runWithOutput(command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DCMAKE_PREFIX_PATH={} "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(
capnpTmp, libdir),
env=env, logger=logger)
runWithOutput("make -j 4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
nupicBindingsEnv = env.copy()
nupicBindingsEnv["CPPFLAGS"] = "-I{}".format(
os.path.join(capnpTmp, "include"))
nupicBindingsEnv["LDFLAGS"] = "-L{}".format(
os.path.join(capnpTmp, "lib"))
command = (
"python setup.py install --prefix={} --nupic-core-dir={}".format(
buildWorkspace, os.path.join(os.getcwd(), "build", "release")))
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=nupicBindingsEnv, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
def createOrReplaceDir(dirname, logger):
"""
Creates a dirname dir in workspace. As a initial cleanup also
deletes dirname if already present
:param str dirname: Directory name that should be created inside workspace
:returns: path to created dirname
:rtype: str
"""
workspace = getWorkspace(logger=logger)
if os.path.exists(os.path.join(workspace, dirname)):
shutil.rmtree("%s/%s" % (workspace, dirname))
mkdirp("%s/%s" % (workspace, dirname))
return os.path.join(workspace, dirname)
def createOrReplaceDir(dirname, logger):
"""
Creates a dirname dir in workspace. As a initial cleanup also
deletes dirname if already present
:param str dirname: Directory name that should be created inside workspace
:returns: path to created dirname
:rtype: str
"""
workspace = getWorkspace(logger=logger)
if os.path.exists(os.path.join(workspace, dirname)):
shutil.rmtree("%s/%s" % (workspace, dirname))
mkdirp("%s/%s" % (workspace, dirname))
return os.path.join(workspace, dirname)
def buildCapnp(env, logger):
"""Builds capnp
:param dict env: The environment which will be set before building.
:param logger: An initialized logger
:returns: Prefix path for capnp.
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
mkdirp("capnp_tmp")
with changeToWorkingDir("capnp_tmp"):
runWithOutput([
"curl", "-O",
"https://capnproto.org/capnproto-c++-0.5.2.tar.gz"
],
env=env,
logger=logger)
runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"],
env=env,
logger=logger)
capnpTmp = os.getcwd()
with changeToWorkingDir("capnproto-c++-0.5.2"):
capnpEnv = env.copy()
capnpEnv["CXXFLAGS"] = (
"-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type "
"-Wunused -Wno-unused-parameter")
runWithOutput([
"./configure", "--disable-shared",
"--prefix={}".format(capnpTmp)
],
env=capnpEnv,
logger=logger)
runWithOutput("make -j4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
return capnpTmp
except Exception:
logger.exception("capnp building failed due to unknown reason.")
raise
else:
logger.info("capnp building was successful.")
def buildCapnp(env, logger):
"""Builds capnp
:param dict env: The environment which will be set before building.
:param logger: An initialized logger
:returns: Prefix path for capnp.
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
mkdirp("capnp_tmp")
with changeToWorkingDir("capnp_tmp"):
runWithOutput(
["curl", "-O", "https://capnproto.org/capnproto-c++-0.5.2.tar.gz"],
env=env, logger=logger)
runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"],
env=env, logger=logger)
capnpTmp = os.getcwd()
with changeToWorkingDir("capnproto-c++-0.5.2"):
capnpEnv = env.copy()
capnpEnv["CXXFLAGS"] = (
"-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type "
"-Wunused -Wno-unused-parameter")
runWithOutput(
["./configure", "--disable-shared",
"--prefix={}".format(capnpTmp)],
env=capnpEnv, logger=logger)
runWithOutput("make -j4", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
return capnpTmp
except Exception:
logger.exception("capnp building failed due to unknown reason.")
raise
else:
logger.info("capnp building was successful.")
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4)
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
capnpTmp = buildCapnp(env, logger)
if isVersionGreaterOrEqual(nupicVersion, "0.3.5"):
srcDir = "../.."
else:
srcDir = "../../src"
# install pre-reqs into the build workspace for isolation
runWithOutput(
command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env,
logger=logger)
# also install pycapnp
command = ("pip", "install",
"--install-option=--prefix=%s" % buildWorkspace,
"pycapnp==0.5.5")
runWithOutput(command=command, env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
includeDir = sysconfig.get_config_var("INCLUDEPY")
runWithOutput(
("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release "
"-DCMAKE_PREFIX_PATH={capnpPrefixPath} "
"-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so "
"-DPYTHON_INCLUDE_DIR={pythonIncludeDir}").format(
srcDir=srcDir,
capnpPrefixPath=capnpTmp,
pythonLibDir=libdir,
pythonIncludeDir=includeDir),
env=env,
logger=logger)
runWithOutput("VERBOSE=1 make -j 1", env=env, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
nupicBindingsEnv = env.copy()
nupicBindingsEnv["CPPFLAGS"] = "-I{}".format(
os.path.join(capnpTmp, "include"))
nupicBindingsEnv["LDFLAGS"] = "-L{}".format(
os.path.join(capnpTmp, "lib"))
command = (
"python setup.py install --prefix={} --nupic-core-dir={}".
format(buildWorkspace,
os.path.join(os.getcwd(), "build", "release")))
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=nupicBindingsEnv, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion):
"""
Builds nupic.core
:param dict env: The environment which will be set before building.
:param str nupicCoreSha: The SHA which will be built.
:param logger: An initialized logger
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4)
:raises infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
git.resetHard(sha=nupicCoreSha, logger=logger)
if isVersionGreaterOrEqual(nupicVersion, "0.3.5"):
srcDir = "../.."
else:
srcDir = "../../src"
pyExtensionsDir = "../../bindings/py/nupic/bindings"
# install pre-reqs into the build workspace for isolation
runWithOutput(command=("pip install -r bindings/py/requirements.txt "
"--install-option=--prefix=%s "
"--ignore-installed" % buildWorkspace),
env=env, logger=logger)
# also install pycapnp
command = ("pip", "install",
"--install-option=--prefix=%s" % buildWorkspace,
"pycapnp==0.5.5")
runWithOutput(command=command, env=env, logger=logger)
shutil.rmtree("build", ignore_errors=True)
mkdirp("build/scripts")
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var("LIBDIR")
includeDir = sysconfig.get_config_var("INCLUDEPY")
runWithOutput(("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so "
"-DPYTHON_INCLUDE_DIR={pythonIncludeDir} "
"-DPY_EXTENSIONS_DIR={pyExtensionsDir}").format(
srcDir=srcDir,
pythonLibDir=libdir,
pythonIncludeDir=includeDir,
pyExtensionsDir=pyExtensionsDir),
env=env, logger=logger)
newEnv = dict(env)
newEnv["VERBOSE"] = "1"
runWithOutput("make -j 1", env=newEnv, logger=logger)
runWithOutput("make install", env=env, logger=logger)
# need to remove this folder to allow the caching process to work
shutil.rmtree("external/linux32arm")
# build the distributions
command = "python setup.py install --prefix={}".format(buildWorkspace)
# Building on jenkins, not local
if "JENKINS_HOME" in os.environ:
command += " bdist_wheel bdist_egg upload -r numenta-pypi"
runWithOutput(command=command, env=env, logger=logger)
except:
logger.exception("Failed to build nupic.core")
raise
else:
logger.info("nupic.core building was successful.")
