def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch)
git.resetHard(nupicSha)
except CommandFailedError:
logger.exception(
"NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
git.clone(nupicCoreRemote, logger=logger)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
git.resetHard(sha=nupicCoreSha, logger=logger)
except CommandFailedError:
logger.exception("nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
raise
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSHA,
logger):
"""
Fetch nupic.core from github
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSHA: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.")
with changeToWorkingDir(buildWorkspace):
git.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSHA:
try:
git.resetHard(nupicCoreSHA)
except CommandFailedError:
logger.exception(
"nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSHA)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises infrastructure.utilities.exceptions.MissingSHAError:
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
git.clone(nupicRemote, logger=logger)
with changeToWorkingDir(env["NUPIC"]):
git.fetch(nupicRemote, nupicBranch, logger=logger)
git.resetHard(sha=nupicSha, logger=logger)
except CommandFailedError:
logger.exception("NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def prepFakerootFromGit(fakeroot, installDirectory, repoDirectory, gitURL, sha=None):
"""Clone a git repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new git clone
:param repoDirectory: what to name the cloned directory
:param gitURL: git URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s", gitURL, fakeroot, installDirectory, repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
git.resetHard()
else:
g_logger.info("No sha specified, using head of master")
gitVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot, installDirectory, repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot, installDirectory, "%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return gitVersionData
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def gitCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None,
logger=None):
"""
Clone a git repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new git clone
@param repoDirectory: what to name the cloned directory
@param gitURL: git URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting git clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
git.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return git.getCurrentSha()
def preBuildSetup(env, pipelineConfig):
"""
Clone the Grok repo if needed and get it set to the right remote, branch,
and SHA. Once set, if the NuPIC parameters need to be revised, take care
of that now, too.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"nupicRemote": "[email protected]:numenta/nupic.git",
"nupicBranch": "master",
"nupicSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file",
"wheelFilePath": "/path/to/wheel/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone Grok if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["GROK_HOME"]):
git.clone(pipelineConfig["grokRemote"], directory="products")
with changeToWorkingDir(env["GROK_HOME"]):
if pipelineConfig["grokSha"]:
g_logger.debug("Resetting to %s" % pipelineConfig["grokSha"])
git.resetHard(pipelineConfig["grokSha"])
else:
grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
pipelineConfig["grokBranch"])
pipelineConfig["grokSha"] = grokSha
g_logger.debug("Resetting to %s" % grokSha)
git.resetHard(grokSha)
wheelFilePath = downloadOrCreateNuPICWheel(env, pipelineConfig)
pipelineConfig["wheelFilePath"] = wheelFilePath
def prepFakerootFromGit(fakeroot,
installDirectory,
repoDirectory,
gitURL,
sha=None):
"""Clone a git repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new git clone
:param repoDirectory: what to name the cloned directory
:param gitURL: git URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s",
gitURL,
fakeroot,
installDirectory,
repoDirectory)
git.clone(gitURL=gitURL, logger=g_logger, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
git.checkout(pathspec=sha, logger=g_logger)
git.resetHard(sha=sha, logger=g_logger)
else:
g_logger.info("No sha specified, using head of master")
gitVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot,
installDirectory,
repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return gitVersionData
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htm-itRemote": "[email protected]:Numenta/numenta-apps.git",
"htm-itBranch": "master",
"htm-itSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM-IT_HOME"]):
git.clone(gitURL=pipelineConfig["htm-itRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM-IT_HOME"]):
if pipelineConfig["htm-itSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htm-itSha"])
git.resetHard(sha=pipelineConfig["htm-itSha"], logger=g_logger)
else:
htm-itSha = git.getShaFromRemoteBranch(pipelineConfig["htm-itRemote"],
pipelineConfig["htm-itBranch"],
logger=g_logger)
pipelineConfig["htm-itSha"] = htm-itSha
g_logger.debug("Resetting to %s", htm-itSha)
git.resetHard(sha=htm-itSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
"""
Clone the HTM-IT repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"htmitRemote": "[email protected]:Numenta/numenta-apps.git",
"htmitBranch": "master",
"htmItSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
diagnostics.printEnv(env=env, logger=g_logger)
# Clone HTM-IT if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["HTM_IT_HOME"]):
git.clone(gitURL=pipelineConfig["htmitRemote"],
logger=g_logger)
with changeToWorkingDir(env["HTM_IT_HOME"]):
if pipelineConfig["htmItSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["htmItSha"])
git.resetHard(sha=pipelineConfig["htmItSha"], logger=g_logger)
else:
htmItSha = git.getShaFromRemoteBranch(pipelineConfig["htmitRemote"],
pipelineConfig["htmitBranch"],
logger=g_logger)
pipelineConfig["htmItSha"] = htmItSha
g_logger.debug("Resetting to %s", htmItSha)
git.resetHard(sha=htmItSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
"""
Clone the Grok repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"grokRemote": "[email protected]:Numenta/numenta-apps.git",
"grokBranch": "master",
"grokSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone Grok if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["GROK_HOME"]):
git.clone(pipelineConfig["grokRemote"], directory="products")
with changeToWorkingDir(env["GROK_HOME"]):
if pipelineConfig["grokSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["grokSha"])
git.resetHard(pipelineConfig["grokSha"])
else:
grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
pipelineConfig["grokBranch"])
pipelineConfig["grokSha"] = grokSha
g_logger.debug("Resetting to %s", grokSha)
git.resetHard(grokSha)
