def main(args): """ Main function for the pipeline. Executes all sub-tasks :param dict args: Parsed command line arguments :returns: /path/to/pipelineJson :rtype: str """ logger = initPipelineLogger("manifest", logLevel=args.logLevel) buildWorkspace = os.environ.get( "BUILD_WORKSPACE", jenkins.defineBuildWorkspace(logger=logger)) mkdirp(buildWorkspace) manifest = vars(args) # Update buildWorkspace in manifest section for pipelineJson manifest.update({"buildWorkspace": buildWorkspace}) manifestEnv = {"manifest": manifest} with open("%s/%s_pipeline.json" % (buildWorkspace, args.pipeline), 'w') as fp: fp.write(json.dumps(manifestEnv, ensure_ascii=False)) logger.debug(json.dumps(manifestEnv)) pipelineJsonPath = "%s/%s_pipeline.json" % (buildWorkspace, args.pipeline) logger.info("Pipeline JSON path: %s", pipelineJsonPath) return pipelineJsonPath
def main(args): """ Main function for the pipeline. Executes all sub-tasks :param dict args: Parsed command line arguments :returns: /path/to/pipelineJson :rtype: str """ logger = initPipelineLogger("manifest", logLevel=args.logLevel) buildWorkspace = os.environ.get("BUILD_WORKSPACE", jenkins.defineBuildWorkspace(logger=logger)) mkdirp(buildWorkspace) manifest = vars(args) # Update buildWorkspace in manifest section for pipelineJson manifest.update({"buildWorkspace": buildWorkspace}) manifestEnv = {"manifest": manifest} with open("%s/%s_pipeline.json" % (buildWorkspace, args.pipeline), 'w') as fp: fp.write(json.dumps(manifestEnv, ensure_ascii=False)) logger.debug(json.dumps(manifestEnv)) pipelineJsonPath = "%s/%s_pipeline.json" % (buildWorkspace, args.pipeline) logger.info("Pipeline JSON path: %s", pipelineJsonPath) return pipelineJsonPath
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace): """ Builds nupic.core :param dict env: The environment which will be set before building. :param str nupicCoreSha: The SHA which will be built. :param logger: An initialized logger :param str buildWorkspace: /path/to/buildWorkspace :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: logger.debug("Building nupic.core SHA : %s ", nupicCoreSha) git.resetHard(sha=nupicCoreSha, logger=logger) capnpTmp = buildCapnp(env, logger) # install pre-reqs into the build workspace for isolation runWithOutput(command=("pip install -r bindings/py/requirements.txt " "--install-option=--prefix=%s " "--ignore-installed" % buildWorkspace), env=env, logger=logger) shutil.rmtree("build", ignore_errors=True) mkdirp("build/scripts") with changeToWorkingDir("build/scripts"): libdir = sysconfig.get_config_var('LIBDIR') runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DCMAKE_PREFIX_PATH={} " "-DPYTHON_LIBRARY={}/libpython2.7.so").format( capnpTmp, libdir), env=env, logger=logger) runWithOutput("make -j 4", env=env, logger=logger) runWithOutput("make install", env=env, logger=logger) # need to remove this folder to allow the caching process to work shutil.rmtree("external/linux32arm") # build the distributions nupicBindingsEnv = env.copy() nupicBindingsEnv["CPPFLAGS"] = "-I{}".format( os.path.join(capnpTmp, "include")) nupicBindingsEnv["LDFLAGS"] = "-L{}".format( os.path.join(capnpTmp, "lib")) command = ( "python setup.py install --prefix={} --nupic-core-dir={}".format( buildWorkspace, os.path.join(os.getcwd(), "build", "release"))) # Building on jenkins, not local if "JENKINS_HOME" in os.environ: command += " bdist_wheel bdist_egg upload -r numenta-pypi" runWithOutput(command=command, env=nupicBindingsEnv, logger=logger) except: logger.exception("Failed to build nupic.core") raise else: logger.info("nupic.core building was successful.")
def createOrReplaceDir(dirname, logger): """ Creates a dirname dir in workspace. As a initial cleanup also deletes dirname if already present :param str dirname: Directory name that should be created inside workspace :returns: path to created dirname :rtype: str """ workspace = getWorkspace(logger=logger) if os.path.exists(os.path.join(workspace, dirname)): shutil.rmtree("%s/%s" % (workspace, dirname)) mkdirp("%s/%s" % (workspace, dirname)) return os.path.join(workspace, dirname)
def buildCapnp(env, logger): """Builds capnp :param dict env: The environment which will be set before building. :param logger: An initialized logger :returns: Prefix path for capnp. :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: mkdirp("capnp_tmp") with changeToWorkingDir("capnp_tmp"): runWithOutput([ "curl", "-O", "https://capnproto.org/capnproto-c++-0.5.2.tar.gz" ], env=env, logger=logger) runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"], env=env, logger=logger) capnpTmp = os.getcwd() with changeToWorkingDir("capnproto-c++-0.5.2"): capnpEnv = env.copy() capnpEnv["CXXFLAGS"] = ( "-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type " "-Wunused -Wno-unused-parameter") runWithOutput([ "./configure", "--disable-shared", "--prefix={}".format(capnpTmp) ], env=capnpEnv, logger=logger) runWithOutput("make -j4", env=env, logger=logger) runWithOutput("make install", env=env, logger=logger) return capnpTmp except Exception: logger.exception("capnp building failed due to unknown reason.") raise else: logger.info("capnp building was successful.")
def buildCapnp(env, logger): """Builds capnp :param dict env: The environment which will be set before building. :param logger: An initialized logger :returns: Prefix path for capnp. :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: mkdirp("capnp_tmp") with changeToWorkingDir("capnp_tmp"): runWithOutput( ["curl", "-O", "https://capnproto.org/capnproto-c++-0.5.2.tar.gz"], env=env, logger=logger) runWithOutput(["tar", "zxf", "capnproto-c++-0.5.2.tar.gz"], env=env, logger=logger) capnpTmp = os.getcwd() with changeToWorkingDir("capnproto-c++-0.5.2"): capnpEnv = env.copy() capnpEnv["CXXFLAGS"] = ( "-fPIC -std=c++11 -m64 -fvisibility=hidden -Wall -Wreturn-type " "-Wunused -Wno-unused-parameter") runWithOutput( ["./configure", "--disable-shared", "--prefix={}".format(capnpTmp)], env=capnpEnv, logger=logger) runWithOutput("make -j4", env=env, logger=logger) runWithOutput("make install", env=env, logger=logger) return capnpTmp except Exception: logger.exception("capnp building failed due to unknown reason.") raise else: logger.info("capnp building was successful.")
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion): """ Builds nupic.core :param dict env: The environment which will be set before building. :param str nupicCoreSha: The SHA which will be built. :param logger: An initialized logger :param str buildWorkspace: /path/to/buildWorkspace :param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4) :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: logger.debug("Building nupic.core SHA : %s ", nupicCoreSha) git.resetHard(sha=nupicCoreSha, logger=logger) capnpTmp = buildCapnp(env, logger) if isVersionGreaterOrEqual(nupicVersion, "0.3.5"): srcDir = "../.." else: srcDir = "../../src" # install pre-reqs into the build workspace for isolation runWithOutput( command=("pip install -r bindings/py/requirements.txt " "--install-option=--prefix=%s " "--ignore-installed" % buildWorkspace), env=env, logger=logger) # also install pycapnp command = ("pip", "install", "--install-option=--prefix=%s" % buildWorkspace, "pycapnp==0.5.5") runWithOutput(command=command, env=env, logger=logger) shutil.rmtree("build", ignore_errors=True) mkdirp("build/scripts") with changeToWorkingDir("build/scripts"): libdir = sysconfig.get_config_var("LIBDIR") includeDir = sysconfig.get_config_var("INCLUDEPY") runWithOutput( ("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release " "-DCMAKE_PREFIX_PATH={capnpPrefixPath} " "-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so " "-DPYTHON_INCLUDE_DIR={pythonIncludeDir}").format( srcDir=srcDir, capnpPrefixPath=capnpTmp, pythonLibDir=libdir, pythonIncludeDir=includeDir), env=env, logger=logger) runWithOutput("VERBOSE=1 make -j 1", env=env, logger=logger) runWithOutput("make install", env=env, logger=logger) # need to remove this folder to allow the caching process to work shutil.rmtree("external/linux32arm") # build the distributions nupicBindingsEnv = env.copy() nupicBindingsEnv["CPPFLAGS"] = "-I{}".format( os.path.join(capnpTmp, "include")) nupicBindingsEnv["LDFLAGS"] = "-L{}".format( os.path.join(capnpTmp, "lib")) command = ( "python setup.py install --prefix={} --nupic-core-dir={}". format(buildWorkspace, os.path.join(os.getcwd(), "build", "release"))) # Building on jenkins, not local if "JENKINS_HOME" in os.environ: command += " bdist_wheel bdist_egg upload -r numenta-pypi" runWithOutput(command=command, env=nupicBindingsEnv, logger=logger) except: logger.exception("Failed to build nupic.core") raise else: logger.info("nupic.core building was successful.")
def buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion): """ Builds nupic.core :param dict env: The environment which will be set before building. :param str nupicCoreSha: The SHA which will be built. :param logger: An initialized logger :param str buildWorkspace: /path/to/buildWorkspace :param str nupicVersion: which version of NuPIC we're building (e.g. 0.3.4) :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: logger.debug("Building nupic.core SHA : %s ", nupicCoreSha) git.resetHard(sha=nupicCoreSha, logger=logger) if isVersionGreaterOrEqual(nupicVersion, "0.3.5"): srcDir = "../.." else: srcDir = "../../src" pyExtensionsDir = "../../bindings/py/nupic/bindings" # install pre-reqs into the build workspace for isolation runWithOutput(command=("pip install -r bindings/py/requirements.txt " "--install-option=--prefix=%s " "--ignore-installed" % buildWorkspace), env=env, logger=logger) # also install pycapnp command = ("pip", "install", "--install-option=--prefix=%s" % buildWorkspace, "pycapnp==0.5.5") runWithOutput(command=command, env=env, logger=logger) shutil.rmtree("build", ignore_errors=True) mkdirp("build/scripts") with changeToWorkingDir("build/scripts"): libdir = sysconfig.get_config_var("LIBDIR") includeDir = sysconfig.get_config_var("INCLUDEPY") runWithOutput(("cmake {srcDir} -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={pythonLibDir}/libpython2.7.so " "-DPYTHON_INCLUDE_DIR={pythonIncludeDir} " "-DPY_EXTENSIONS_DIR={pyExtensionsDir}").format( srcDir=srcDir, pythonLibDir=libdir, pythonIncludeDir=includeDir, pyExtensionsDir=pyExtensionsDir), env=env, logger=logger) newEnv = dict(env) newEnv["VERBOSE"] = "1" runWithOutput("make -j 1", env=newEnv, logger=logger) runWithOutput("make install", env=env, logger=logger) # need to remove this folder to allow the caching process to work shutil.rmtree("external/linux32arm") # build the distributions command = "python setup.py install --prefix={}".format(buildWorkspace) # Building on jenkins, not local if "JENKINS_HOME" in os.environ: command += " bdist_wheel bdist_egg upload -r numenta-pypi" runWithOutput(command=command, env=env, logger=logger) except: logger.exception("Failed to build nupic.core") raise else: logger.info("nupic.core building was successful.")