def buildNuPICCore(env, nupicCoreSha, logger): """ Builds nupic.core :param env: The environment which will be set before building. :param nupicCoreSha: The SHA which will be built. :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ print "\n----------Building nupic.core------------" log.printEnv(env, logger) with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: logger.debug("Building nupic.core SHA : %s ", nupicCoreSha) YOMP.resetHard(nupicCoreSha) runWithOutput("mkdir -p build/scripts", env, logger) with changeToWorkingDir("build/scripts"): libdir = sysconfig.get_config_var('LIBDIR') runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir), env, logger) runWithOutput("make -j 4", env, logger) runWithOutput("make install", env, logger) except CommandFailedError: raise NupicBuildFailed("nupic.core building failed.Exiting") except: raise PipelineError("nupic.core building failed due to unknown reason.") else: logger.info("nupic.core building was successful.")
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger): """ This method clones NuPIC repo if it is not present and checks out to required nupicBranch :param env: The environment which will be used before building. :param buildWorkspace: The workspace where NuPIC should be built :param nupicRemote: URL for NuPIC remote repository :param nupicBranch: The NuPIC branch which will be used to build :param nupicSha: NuPIC SHA used for current run. :raises: infrastructure.utilities.exceptions.MissingSHAError if the given SHA is not found. """ try: with changeToWorkingDir(buildWorkspace): if not os.path.isdir(env["NUPIC"]): YOMP.clone(nupicRemote) with changeToWorkingDir(env["NUPIC"]): YOMP.fetch(nupicRemote, nupicBranch) YOMP.resetHard(nupicSha) except CommandFailedError: logger.exception("NuPIC checkout failed with %s," " this sha might not exist.", nupicSha)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger): """ Fetch nupic.core from YOMPhub :param buildWorkspace: The workspace where nupic.core should be built :param nupicCoreRemote: URL for nupic.core remote repository :param nupicCoreSha: The SHA of the nupic.core build that needs to be fetched :raises: infrastructure.utilities.exceptions.MissingSHAError if the given SHA is not found. """ logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote)) with changeToWorkingDir(buildWorkspace): if not os.path.isdir("nupic.core"): YOMP.clone(nupicCoreRemote) nupicCoreDir = buildWorkspace + "/nupic.core" with changeToWorkingDir(nupicCoreDir): if nupicCoreSha: try: YOMP.resetHard(nupicCoreSha) except CommandFailedError: logger.exception("nupic.core checkout failed with %s," " this sha might not exist.", nupicCoreSha)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger): """ This method clones NuPIC repo if it is not present and checks out to required nupicBranch :param env: The environment which will be used before building. :param buildWorkspace: The workspace where NuPIC should be built :param nupicRemote: URL for NuPIC remote repository :param nupicBranch: The NuPIC branch which will be used to build :param nupicSha: NuPIC SHA used for current run. :raises: infrastructure.utilities.exceptions.MissingSHAError if the given SHA is not found. """ try: with changeToWorkingDir(buildWorkspace): if not os.path.isdir(env["NUPIC"]): YOMP.clone(nupicRemote) with changeToWorkingDir(env["NUPIC"]): YOMP.fetch(nupicRemote, nupicBranch) YOMP.resetHard(nupicSha) except CommandFailedError: logger.exception( "NuPIC checkout failed with %s," " this sha might not exist.", nupicSha)
def buildNuPICCore(env, nupicCoreSha, logger): """ Builds nupic.core :param env: The environment which will be set before building. :param nupicCoreSha: The SHA which will be built. :raises infrastructure.utilities.exceptions.NupicBuildFailed: This exception is raised if build fails. """ print "\n----------Building nupic.core------------" log.printEnv(env, logger) with changeToWorkingDir(env["NUPIC_CORE_DIR"]): try: logger.debug("Building nupic.core SHA : %s ", nupicCoreSha) YOMP.resetHard(nupicCoreSha) runWithOutput("mkdir -p build/scripts", env, logger) with changeToWorkingDir("build/scripts"): libdir = sysconfig.get_config_var('LIBDIR') runWithOutput( ("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir), env, logger) runWithOutput("make -j 4", env, logger) runWithOutput("make install", env, logger) except CommandFailedError: raise NupicBuildFailed("nupic.core building failed.Exiting") except: raise PipelineError( "nupic.core building failed due to unknown reason.") else: logger.info("nupic.core building was successful.")
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger): """ Fetch nupic.core from YOMPhub :param buildWorkspace: The workspace where nupic.core should be built :param nupicCoreRemote: URL for nupic.core remote repository :param nupicCoreSha: The SHA of the nupic.core build that needs to be fetched :raises: infrastructure.utilities.exceptions.MissingSHAError if the given SHA is not found. """ logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote)) with changeToWorkingDir(buildWorkspace): if not os.path.isdir("nupic.core"): YOMP.clone(nupicCoreRemote) nupicCoreDir = buildWorkspace + "/nupic.core" with changeToWorkingDir(nupicCoreDir): if nupicCoreSha: try: YOMP.resetHard(nupicCoreSha) except CommandFailedError: logger.exception( "nupic.core checkout failed with %s," " this sha might not exist.", nupicCoreSha)
def YOMPCloneIntoFakeroot(fakeroot, installDirectory, repoDirectory, YOMPURL, sha=None, logger=None): """ Clone a YOMP repository into a specific path in a fakeroot @param fakeroot: path to the directory to use as the root of the RPM's install tree @param installDirectory: Where to put the new YOMP clone @param repoDirectory: what to name the cloned directory @param YOMPURL: YOMP URL used to clone @param sha (optional): SHA to checkout once we've cloned the repository @param logger - Optional logger object, will be used to output more debugging information. @returns the SHA of the resulting YOMP clone. We may not have been invoked with a specific SHA (we normally build tip of master, for example), but we always want to include the exact SHA packaged in our RPM descriptions. """ if logger: logger.debug("Prepping fakeroot in %s", fakeroot) installPath = "%s/%s" % (fakeroot, installDirectory) with changeToWorkingDir(installPath): if logger: logger.debug("Cloning %s into %s/%s/%s", YOMPURL, fakeroot, installDirectory, repoDirectory) YOMP.clone(YOMPURL, directory=repoDirectory) workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory) with changeToWorkingDir(workDirectory): if sha: YOMP.resetHard() logger.debug("Checking out SHA %s in %s", sha, workDirectory) YOMP.checkout(sha) else: logger.debug("No SHA specified, using head of master") return YOMP.getCurrentSha()
def prepFakerootFromGit(fakeroot, installDirectory, repoDirectory, YOMPURL, sha=None): """Clone a YOMP repository and make a fakeroot out of it. :param fakeroot: path to the directory to use as the root of the RPM's install tree :param installDirectory: Where to put the new YOMP clone :param repoDirectory: what to name the cloned directory :param YOMPURL: YOMP URL used to clone :param sha (optional): SHA to checkout once we've cloned the repository """ g_logger.debug("Prepping fakeroot in %s", fakeroot) installPath = commonFakerootPrep(fakeroot, installDirectory) with changeToWorkingDir(installPath): g_logger.info("Cloning %s into %s/%s/%s", YOMPURL, fakeroot, installDirectory, repoDirectory) YOMP.clone(YOMPURL, directory=repoDirectory) workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory) if sha: with changeToWorkingDir(workDirectory): g_logger.info("Checking out SHA %s in %s", sha, workDirectory) YOMP.checkout(sha) YOMP.resetHard() else: g_logger.info("No sha specified, using head of master") YOMPVersionData = loadGitDescribeFromDirectory(workDirectory) sourceFiles = os.listdir("%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)) for directoryEntry in sourceFiles: cleanseFakeroot(fakeroot, installDirectory, "%s/%s" % (repoDirectory, directoryEntry)) cleanseFakeroot(fakeroot, installDirectory, repoDirectory) return YOMPVersionData
def preBuildSetup(env, pipelineConfig): """ Clone the YOMP repo if needed and get it set to the right remote, branch, and SHA. :param env: The environment variable which is set before building :param pipelineConfig: dict of the pipeline config values, e.g.: { "buildWorkspace": "/path/to/build/in", "YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP", "YOMPBranch": "master", "YOMPSha": "HEAD", "pipelineParams": "{dict of parameters}", "pipelineJson": "/path/to/json/file" } :returns: The updated pipelineConfig dict :rtype: dict """ log.printEnv(env, g_logger) # Clone YOMP if needed, otherwise, setup remote with changeToWorkingDir(pipelineConfig["buildWorkspace"]): if not os.path.isdir(env["YOMP_HOME"]): YOMP.clone(pipelineConfig["YOMPRemote"], directory="products") with changeToWorkingDir(env["YOMP_HOME"]): if pipelineConfig["YOMPSha"]: g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"]) YOMP.resetHard(pipelineConfig["YOMPSha"]) else: YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"], pipelineConfig["YOMPBranch"]) pipelineConfig["YOMPSha"] = YOMPSha g_logger.debug("Resetting to %s", YOMPSha) YOMP.resetHard(YOMPSha)