def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
YOMP.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir),
env, logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError("nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger):
"""
Fetch nupic.core from YOMPhub
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
YOMP.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
YOMP.resetHard(nupicCoreSha)
except CommandFailedError:
logger.exception(
"nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
def constructPreBuiltYOMPFakeroot(self):
"""
Construct fakeroot from prebuilt YOMP
:returns: SHA of the products repo in the fakeroot
:rtype: tuple
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.debug("Creating %s", productsDirectory)
mkpath(productsDirectory)
copy_tree(config.productsDir, productsDirectory)
iteration = YOMP.getCommitCount(productsDirectory)
with changeToWorkingDir(productsDirectory):
actualSHA = YOMP.getCurrentSha()
# Set extra python path
self.setPythonPath()
# Clean YOMP Scripts
self.cleanScripts()
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger):
"""
Fetch nupic.core from YOMPhub
:param buildWorkspace: The workspace where nupic.core should be built
:param nupicCoreRemote: URL for nupic.core remote repository
:param nupicCoreSha: The SHA of the nupic.core build that needs to be
fetched
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
logger.info("Cloning nupic.core from GitHub.: {}".format(nupicCoreRemote))
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir("nupic.core"):
YOMP.clone(nupicCoreRemote)
nupicCoreDir = buildWorkspace + "/nupic.core"
with changeToWorkingDir(nupicCoreDir):
if nupicCoreSha:
try:
YOMP.resetHard(nupicCoreSha)
except CommandFailedError:
logger.exception("nupic.core checkout failed with %s,"
" this sha might not exist.", nupicCoreSha)
def buildNuPICCore(env, nupicCoreSha, logger):
"""
Builds nupic.core
:param env: The environment which will be set before building.
:param nupicCoreSha: The SHA which will be built.
:raises
infrastructure.utilities.exceptions.NupicBuildFailed:
This exception is raised if build fails.
"""
print "\n----------Building nupic.core------------"
log.printEnv(env, logger)
with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
try:
logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
YOMP.resetHard(nupicCoreSha)
runWithOutput("mkdir -p build/scripts", env, logger)
with changeToWorkingDir("build/scripts"):
libdir = sysconfig.get_config_var('LIBDIR')
runWithOutput(
("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
"-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir),
env, logger)
runWithOutput("make -j 4", env, logger)
runWithOutput("make install", env, logger)
except CommandFailedError:
raise NupicBuildFailed("nupic.core building failed.Exiting")
except:
raise PipelineError(
"nupic.core building failed due to unknown reason.")
else:
logger.info("nupic.core building was successful.")
def constructPreBuiltYOMPFakeroot(self):
"""
Construct fakeroot from prebuilt YOMP
:returns: SHA of the products repo in the fakeroot
:rtype: tuple
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.debug("Creating %s", productsDirectory)
mkpath(productsDirectory)
copy_tree(config.productsDir, productsDirectory)
iteration = YOMP.getCommitCount(productsDirectory)
with changeToWorkingDir(productsDirectory):
actualSHA = YOMP.getCurrentSha()
# Set extra python path
self.setPythonPath()
# Clean YOMP Scripts
self.cleanScripts()
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def constructInfrastructureFakeroot(self):
"""
Construct our fakeroot directory tree
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
productsDirectory = self.productsDirectory
srvPath = os.path.join(fakeroot, "opt", "numenta")
logger.debug("Creating %s", srvPath)
mkpath(srvPath)
logger.debug("Cloning %s into %s...", fakeroot, config.YOMPURL)
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
installDirectory = os.path.join("opt", "numenta")
fakerootSHA = rpm.YOMPCloneIntoFakeroot(fakeroot=fakeroot,
installDirectory=installDirectory,
repoDirectory="products",
YOMPURL=config.YOMPURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = YOMP.getCommitCount(productsDirectory)
logger.debug("Commit count in %s is %s", productsDirectory, iteration)
logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA)
# Clean everything not whitelisted out of products so we don't conflict
# with YOMP or taurus rpms
purgeDirectory(path=productsDirectory,
whitelist=["__init__.py",
"infrastructure" ],
logger=logger)
# Clean out infrastructure, too - we only want the utilities
infraPath = os.path.join(productsDirectory, "infrastructure")
purgeDirectory(path=infraPath,
whitelist=["__init__.py",
"DEPENDENCIES.md",
"infrastructure",
"LICENSE",
"README.md",
"requirements.txt",
"setup.py"],
logger=logger)
return (iteration, fakerootSHA)
def constructInfrastructureFakeroot(self):
"""
Construct our fakeroot directory tree
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
productsDirectory = self.productsDirectory
srvPath = os.path.join(fakeroot, "opt", "numenta")
logger.debug("Creating %s", srvPath)
mkpath(srvPath)
logger.debug("Cloning %s into %s...", fakeroot, config.YOMPURL)
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
installDirectory = os.path.join("opt", "numenta")
fakerootSHA = rpm.YOMPCloneIntoFakeroot(
fakeroot=fakeroot,
installDirectory=installDirectory,
repoDirectory="products",
YOMPURL=config.YOMPURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = YOMP.getCommitCount(productsDirectory)
logger.debug("Commit count in %s is %s", productsDirectory, iteration)
logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA)
# Clean everything not whitelisted out of products so we don't conflict
# with YOMP or taurus rpms
purgeDirectory(path=productsDirectory,
whitelist=["__init__.py", "infrastructure"],
logger=logger)
# Clean out infrastructure, too - we only want the utilities
infraPath = os.path.join(productsDirectory, "infrastructure")
purgeDirectory(path=infraPath,
whitelist=[
"__init__.py", "DEPENDENCIES.md", "infrastructure",
"LICENSE", "README.md", "requirements.txt",
"setup.py"
],
logger=logger)
return (iteration, fakerootSHA)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
YOMP.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
YOMP.fetch(nupicRemote, nupicBranch)
YOMP.resetHard(nupicSha)
except CommandFailedError:
logger.exception("NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
:param env: The environment which will be used before building.
:param buildWorkspace: The workspace where NuPIC should be built
:param nupicRemote: URL for NuPIC remote repository
:param nupicBranch: The NuPIC branch which will be used to build
:param nupicSha: NuPIC SHA used for current run.
:raises: infrastructure.utilities.exceptions.MissingSHAError
if the given SHA is not found.
"""
try:
with changeToWorkingDir(buildWorkspace):
if not os.path.isdir(env["NUPIC"]):
YOMP.clone(nupicRemote)
with changeToWorkingDir(env["NUPIC"]):
YOMP.fetch(nupicRemote, nupicBranch)
YOMP.resetHard(nupicSha)
except CommandFailedError:
logger.exception(
"NuPIC checkout failed with %s,"
" this sha might not exist.", nupicSha)
def YOMPCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None,
logger=None):
"""
Clone a YOMP repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new YOMP clone
@param repoDirectory: what to name the cloned directory
@param YOMPURL: YOMP URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting YOMP clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s",
YOMPURL,
fakeroot,
installDirectory,
repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
YOMP.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return YOMP.getCurrentSha()
def YOMPCloneIntoFakeroot(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None,
logger=None):
"""
Clone a YOMP repository into a specific path in a fakeroot
@param fakeroot: path to the directory to use as the root of the RPM's
install tree
@param installDirectory: Where to put the new YOMP clone
@param repoDirectory: what to name the cloned directory
@param YOMPURL: YOMP URL used to clone
@param sha (optional): SHA to checkout once we've cloned the repository
@param logger - Optional logger object, will be used to output more
debugging information.
@returns the SHA of the resulting YOMP clone. We may not have been invoked
with a specific SHA (we normally build tip of master, for example), but we
always want to include the exact SHA packaged in our RPM descriptions.
"""
if logger:
logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = "%s/%s" % (fakeroot, installDirectory)
with changeToWorkingDir(installPath):
if logger:
logger.debug("Cloning %s into %s/%s/%s", YOMPURL, fakeroot,
installDirectory, repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory,
repoDirectory)
with changeToWorkingDir(workDirectory):
if sha:
YOMP.resetHard()
logger.debug("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
else:
logger.debug("No SHA specified, using head of master")
return YOMP.getCurrentSha()
def prepFakerootFromGit(fakeroot,
installDirectory,
repoDirectory,
YOMPURL,
sha=None):
"""Clone a YOMP repository and make a fakeroot out of it.
:param fakeroot: path to the directory to use as the root of the RPM's
install tree
:param installDirectory: Where to put the new YOMP clone
:param repoDirectory: what to name the cloned directory
:param YOMPURL: YOMP URL used to clone
:param sha (optional): SHA to checkout once we've cloned the repository
"""
g_logger.debug("Prepping fakeroot in %s", fakeroot)
installPath = commonFakerootPrep(fakeroot, installDirectory)
with changeToWorkingDir(installPath):
g_logger.info("Cloning %s into %s/%s/%s",
YOMPURL,
fakeroot,
installDirectory,
repoDirectory)
YOMP.clone(YOMPURL, directory=repoDirectory)
workDirectory = "%s/%s/%s" % (fakeroot, installDirectory, repoDirectory)
if sha:
with changeToWorkingDir(workDirectory):
g_logger.info("Checking out SHA %s in %s", sha, workDirectory)
YOMP.checkout(sha)
YOMP.resetHard()
else:
g_logger.info("No sha specified, using head of master")
YOMPVersionData = loadGitDescribeFromDirectory(workDirectory)
sourceFiles = os.listdir("%s/%s/%s" % (fakeroot,
installDirectory,
repoDirectory))
for directoryEntry in sourceFiles:
cleanseFakeroot(fakeroot,
installDirectory,
"%s/%s" % (repoDirectory, directoryEntry))
cleanseFakeroot(fakeroot, installDirectory, repoDirectory)
return YOMPVersionData
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)
def preBuildSetup(env, pipelineConfig):
"""
Clone the YOMP repo if needed and get it set to the right remote, branch,
and SHA.
:param env: The environment variable which is set before building
:param pipelineConfig: dict of the pipeline config values, e.g.:
{
"buildWorkspace": "/path/to/build/in",
"YOMPRemote": "[email protected]:Numenta/numenta-apps.YOMP",
"YOMPBranch": "master",
"YOMPSha": "HEAD",
"pipelineParams": "{dict of parameters}",
"pipelineJson": "/path/to/json/file"
}
:returns: The updated pipelineConfig dict
:rtype: dict
"""
log.printEnv(env, g_logger)
# Clone YOMP if needed, otherwise, setup remote
with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
if not os.path.isdir(env["YOMP_HOME"]):
YOMP.clone(pipelineConfig["YOMPRemote"], directory="products")
with changeToWorkingDir(env["YOMP_HOME"]):
if pipelineConfig["YOMPSha"]:
g_logger.debug("Resetting to %s", pipelineConfig["YOMPSha"])
YOMP.resetHard(pipelineConfig["YOMPSha"])
else:
YOMPSha = YOMP.getShaFromRemoteBranch(pipelineConfig["YOMPRemote"],
pipelineConfig["YOMPBranch"])
pipelineConfig["YOMPSha"] = YOMPSha
g_logger.debug("Resetting to %s", YOMPSha)
YOMP.resetHard(YOMPSha)
def constructSaltcellarFakeroot(self):
"""
Make a saltcellar fakeroot
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
srvPath = os.path.join(fakeroot, "srv")
logger.debug("Creating saltcellar fakeroot in %s", srvPath)
productsPath = os.path.join(fakeroot, "products")
mkpath(srvPath)
logger.debug("Cloning...")
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
fakerootSHA = rpm.YOMPCloneIntoFakeroot(fakeroot=fakeroot,
installDirectory="/",
repoDirectory="products",
YOMPURL=config.YOMPURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = YOMP.getCommitCount(productsPath)
logger.debug("Commit count in %s is %s", productsPath, iteration)
# Move the saltcellar to /srv/salt
logger.debug("Moving saltcellar to %s/salt", srvPath)
logger.debug("srvPath: %s", srvPath)
logger.debug("productsPath: %s", productsPath)
logger.debug("%s/infrastructure/saltcellar", productsPath)
logger.debug("Checking for %s/infrastructure/saltcellar", productsPath)
logger.debug(
os.path.exists("%s/infrastructure/saltcellar" % productsPath))
os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"),
os.path.join(srvPath, "salt"))
# Now that we have the salt formulas, nuke the rest of products out of
# the fakeroot
logger.debug("Deleting products from fakeroot")
rmrf(productsPath)
# Finally, scrub the private data out of /srv/salt
if not config.numenta_internal_only:
logger.debug("Sanitizing /srv/salt")
self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
else:
logger.critical(
"Baking numenta-internal rpm, not sanitizing /srv/salt")
return (iteration, fakerootSHA)
def constructSaltcellarFakeroot(self):
"""
Make a saltcellar fakeroot
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
srvPath = os.path.join(fakeroot, "srv")
logger.debug("Creating saltcellar fakeroot in %s", srvPath)
productsPath = os.path.join(fakeroot, "products")
mkpath(srvPath)
logger.debug("Cloning...")
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
fakerootSHA = rpm.YOMPCloneIntoFakeroot(fakeroot=fakeroot,
installDirectory="/",
repoDirectory="products",
YOMPURL=config.YOMPURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = YOMP.getCommitCount(productsPath)
logger.debug("Commit count in %s is %s", productsPath, iteration)
# Move the saltcellar to /srv/salt
logger.debug("Moving saltcellar to %s/salt", srvPath)
logger.debug("srvPath: %s", srvPath)
logger.debug("productsPath: %s", productsPath)
logger.debug("%s/infrastructure/saltcellar", productsPath)
logger.debug("Checking for %s/infrastructure/saltcellar",
productsPath)
logger.debug(os.path.exists("%s/infrastructure/saltcellar" %
productsPath))
os.rename(os.path.join(productsPath, "infrastructure",
"saltcellar"),
os.path.join(srvPath, "salt"))
# Now that we have the salt formulas, nuke the rest of products out of
# the fakeroot
logger.debug("Deleting products from fakeroot")
rmrf(productsPath)
# Finally, scrub the private data out of /srv/salt
if not config.numenta_internal_only:
logger.debug("Sanitizing /srv/salt")
self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
else:
logger.critical("Baking numenta-internal rpm, not sanitizing /srv/salt")
return (iteration, fakerootSHA)
createOrReplaceArtifactsDir
)
from infrastructure.utilities import logger as log
from infrastructure.utilities import s3
from infrastructure.utilities.env import addNupicCoreToEnv
from infrastructure.utilities.exceptions import (
CommandFailedError,
NupicBuildFailed,
PipelineError
)
from infrastructure.utilities.path import changeToWorkingDir
from infrastructure.utilities.cli import runWithOutput
SCRIPTS_DIR = os.path.join(YOMP.getGitRootFolder(), "nupic-pipeline", "scripts")
ARTIFACTS_DIR = createOrReplaceArtifactsDir()
DOXYFILE = "docs/Doxyfile"
INIT_FILE = "nupic/__init__.py"
VERSION_FILE = "VERSION"
g_config = yaml.load(
resource_stream(__name__, "../../../conf/nupic/config.yaml"))
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
This method clones NuPIC repo if it is not present
and checks out to required nupicBranch
def constructYOMPFakeroot(self):
"""
Construct a YOMP fakeroot directory tree.
1. Add any directories specified with --extend-pythonpath to the PYTHONPATH
we will be using for setup.py, build scripts and the cleanup scripts.
2. Install any wheels that have been specied by --use-wheel
3. Run setup.py in any directories that have been specified with
--setup-py-dir. Uses the arguments specfied by --setup-py-arguments.
4. Run any build scripts specified by --build-script
5. Run any cleanup scripts specified by --cleanup-script
6. Purge any files or directories at the top level of the checkout that were
not whitelisted with --whitelist.
:returns: (iteration, actualSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
logger.info("Preparing YOMP fakeroot in %s\n", fakeroot)
actualSHA = self.installProductsIntoYOMPFakeroot()
productsDirectory = self.productsDirectory
YOMPPath = os.path.join(productsDirectory, "YOMP")
iteration = YOMP.getCommitCount(productsDirectory)
# Extend PYTHONPATH for setup.py, build & cleanup scripts
# pythonpathExtensions
logger.debug("**************************************************")
logger.info("Phase 1: Preparing PYTHONPATH and installing wheels")
# Set extra python path
self.setPythonPath()
environment = self.environment
sitePackagesDirectory = "%s/YOMP/lib/python2.7/site-packages" % \
productsDirectory
# Install wheels if any have been specified
with changeToWorkingDir(YOMPPath):
for wheel in config.wheels:
logger.info("Installing %s", os.path.basename(wheel))
if not os.path.exists(wheel):
raise InvalidParametersError("%s does not exist!" % wheel)
pipCommand = "pip install %s --no-deps --target=%s" % \
(wheel, sitePackagesDirectory)
logger.debug("pip command: %s", pipCommand)
runWithOutput(pipCommand)
logger.debug("wheel install complete")
# Run setup.py if specified
logger.info("Phase 2: Running setup.py commands")
for pyDir in config.setupPyDirs:
pyDirPath = "%s/%s" % (productsDirectory, pyDir)
logger.debug("Changing to %s", pyDirPath)
with changeToWorkingDir(pyDirPath):
setupCommand = "python setup.py develop --prefix=%s/YOMP" % \
productsDirectory
logger.debug("Running %s", setupCommand)
runWithOutput(setupCommand, env=environment)
# Run any build scripts. We assume that they should be run in the
# directory they're in.
logger.info("Phase 3: Running build scripts...")
for builder in config.buildScripts:
builderPath = "%s/%s" % (fakeroot, builder)
logger.debug("Attempting to run %s", builderPath)
if not os.path.exists(builderPath):
raise InvalidParametersError("%s does not exist!" % builderPath)
workDirectory = os.path.dirname(builderPath)
logger.debug("Changing to %s", workDirectory)
with changeToWorkingDir(workDirectory):
runWithOutput(builderPath, env=environment)
# Run any cleanup scripts. We assume that they should be run in the
# directory they're in.
logger.info("Phase 4: Running cleanup scripts...")
# Clean Scripts
self.cleanScripts()
logger.info("Phase 5: Purge anything not whitelisted.")
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def constructYOMPFakeroot(self):
"""
Construct a YOMP fakeroot directory tree.
1. Add any directories specified with --extend-pythonpath to the PYTHONPATH
we will be using for setup.py, build scripts and the cleanup scripts.
2. Install any wheels that have been specied by --use-wheel
3. Run setup.py in any directories that have been specified with
--setup-py-dir. Uses the arguments specfied by --setup-py-arguments.
4. Run any build scripts specified by --build-script
5. Run any cleanup scripts specified by --cleanup-script
6. Purge any files or directories at the top level of the checkout that were
not whitelisted with --whitelist.
:returns: (iteration, actualSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
logger.info("Preparing YOMP fakeroot in %s\n", fakeroot)
actualSHA = self.installProductsIntoYOMPFakeroot()
productsDirectory = self.productsDirectory
YOMPPath = os.path.join(productsDirectory, "YOMP")
iteration = YOMP.getCommitCount(productsDirectory)
# Extend PYTHONPATH for setup.py, build & cleanup scripts
# pythonpathExtensions
logger.debug("**************************************************")
logger.info("Phase 1: Preparing PYTHONPATH and installing wheels")
# Set extra python path
self.setPythonPath()
environment = self.environment
sitePackagesDirectory = "%s/YOMP/lib/python2.7/site-packages" % \
productsDirectory
# Install wheels if any have been specified
with changeToWorkingDir(YOMPPath):
for wheel in config.wheels:
logger.info("Installing %s", os.path.basename(wheel))
if not os.path.exists(wheel):
raise InvalidParametersError("%s does not exist!" % wheel)
pipCommand = "pip install %s --no-deps --target=%s" % \
(wheel, sitePackagesDirectory)
logger.debug("pip command: %s", pipCommand)
runWithOutput(pipCommand)
logger.debug("wheel install complete")
# Run setup.py if specified
logger.info("Phase 2: Running setup.py commands")
for pyDir in config.setupPyDirs:
pyDirPath = "%s/%s" % (productsDirectory, pyDir)
logger.debug("Changing to %s", pyDirPath)
with changeToWorkingDir(pyDirPath):
setupCommand = "python setup.py develop --prefix=%s/YOMP" % \
productsDirectory
logger.debug("Running %s", setupCommand)
runWithOutput(setupCommand, env=environment)
# Run any build scripts. We assume that they should be run in the
# directory they're in.
logger.info("Phase 3: Running build scripts...")
for builder in config.buildScripts:
builderPath = "%s/%s" % (fakeroot, builder)
logger.debug("Attempting to run %s", builderPath)
if not os.path.exists(builderPath):
raise InvalidParametersError("%s does not exist!" %
builderPath)
workDirectory = os.path.dirname(builderPath)
logger.debug("Changing to %s", workDirectory)
with changeToWorkingDir(workDirectory):
runWithOutput(builderPath, env=environment)
# Run any cleanup scripts. We assume that they should be run in the
# directory they're in.
logger.info("Phase 4: Running cleanup scripts...")
# Clean Scripts
self.cleanScripts()
logger.info("Phase 5: Purge anything not whitelisted.")
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
import yaml
from infrastructure.utilities import YOMP
from infrastructure.utilities.jenkins import (createOrReplaceResultsDir,
createOrReplaceArtifactsDir)
from infrastructure.utilities import logger as log
from infrastructure.utilities import s3
from infrastructure.utilities.env import addNupicCoreToEnv
from infrastructure.utilities.exceptions import (CommandFailedError,
NupicBuildFailed,
PipelineError)
from infrastructure.utilities.path import changeToWorkingDir
from infrastructure.utilities.cli import runWithOutput
SCRIPTS_DIR = os.path.join(YOMP.getGitRootFolder(), "nupic-pipeline",
"scripts")
ARTIFACTS_DIR = createOrReplaceArtifactsDir()
DOXYFILE = "docs/Doxyfile"
INIT_FILE = "nupic/__init__.py"
VERSION_FILE = "VERSION"
g_config = yaml.load(
resource_stream(__name__, "../../../conf/nupic/config.yaml"))
def fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
logger):
"""
This method clones NuPIC repo if it is not present
