def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
YOMP.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
YOMP.fetch(nupicRemote, nupicBranch)
YOMP.resetHard(nupicSha)
except CommandFailedError:
logger.exception("NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger):
"""
Fetch nupic.core from YOMPhub
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
YOMP.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
YOMP.resetHard(nupicCoreSha)
except CommandFailedError:
logger.exception("nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
YOMP.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
YOMP.fetch(nupicRemote, nupicBranch)
YOMP.resetHard(nupicSha)
except CommandFailedError:
logger.exception(
"NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger):
"""
Fetch nupic.core from YOMPhub
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
YOMP.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
YOMP.resetHard(nupicCoreSha)
except CommandFailedError:
logger.exception(
"nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
def YOMPCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None,
logger=None):
"""
Clone a YOMP repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new YOMP clone
@param repoDirectory: what to name the cloned directory
@param YOMPURL: YOMP URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting YOMP clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
YOMPURL,
fakeroot,
installDirectory,
repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
YOMP.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return YOMP.getCurrentSha()
def YOMPCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None,
logger=None):
"""
Clone a YOMP repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new YOMP clone
@param repoDirectory: what to name the cloned directory
@param YOMPURL: YOMP URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting YOMP clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s", YOMPURL, fakeroot,
installDirectory, repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory,
repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
YOMP.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return YOMP.getCurrentSha()
def prepFakerootFromGit(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None):
"""Clone a YOMP repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new YOMP clone
:param repoDirectory: what to name the cloned directory
:param YOMPURL: YOMP URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s",
YOMPURL,
fakeroot,
installDirectory,
repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
YOMP.resetHard()
else:
g_logger.info("No sha specified, using head of master")
YOMPVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot,
installDirectory,
repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return YOMPVersionData
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)