def installNuPICWheel(env, installDir, wheelFilePath, logger):
"""
Install a NuPIC Wheel to a specified location.
:param env: The environment dict
:param installDir: The root folder to install to. NOTE: pip will automatically
create lib/pythonX.Y/site-packages and bin folders to install libraries
and executables to. Make sure both of those sub-folder locations are already
on your PYTHONPATH and PATH respectively.
:param wheelFilePath: location of the NuPIC wheel that will be installed
:param logger: initialized logger object
"""
try:
if installDir is None:
raise PipelineError("Please provide NuPIC install directory.")
logger.debug("Installing %s to %s", wheelFilePath, installDir)
pipCommand = ("pip install %s --install-option=--prefix=%s" %
(wheelFilePath, installDir))
runWithOutput(pipCommand, env=env, logger=logger)
except:
logger.exception("Failed to install NuPIC wheel")
raise CommandFailedError("Installing NuPIC wheel failed.")
else:
logger.debug("NuPIC wheel installed successfully.")
def getShaFromRemoteBranch(YOMPRemoteRepo, YOMPRemoteBranch):
"""
Get the actual SHA of the current HEAD of a remote repo / branch.
@param YOMPRemoteRepo: The URL of the remote repo,
e.g., [email protected]:numenta/nupic.YOMP
@param YOMPRemoteBranch: The name of the remote branch, e.g., master
@raises:
@return: A `String` representing the SHA
@rtype: String
"""
shaList = executeCommand("YOMP ls-remote %s" % YOMPRemoteRepo)
if YOMPRemoteBranch == "master":
return shaList.split("\t")[0]
else:
formattedBranchName = "refs/heads/%s" % YOMPRemoteBranch
for refShaPair in shaList.split("\n"):
pair = refShaPair.split()
if pair[1] == formattedBranchName:
return pair[0]
raise CommandFailedError(
"Could not find the specified branch %s on %s" % YOMPRemoteBranch,
YOMPRemoteRepo)
def runWithOutput(command, env=None, logger=None):
"""
Run a command, printing as the command executes.
@param command: Command to run. Can be str or list
@param env: environment variables to use while running command
@param printEnv: Whether or not to print the environment passed to command
@param logger: optional logger for additional debug info if desired
"""
try:
if env is None:
env = os.environ
if logger is not None:
diagnostics.printEnv(env, logger)
logger.debug("**********> %s", command)
if isinstance(command, basestring):
command = command.strip().split(" ")
check_call(command, env=env)
except CalledProcessError:
errMessage = "Failed to execute: %s" % (command, )
raise CommandFailedError(errMessage)
# Catch other exceptions, add info about what command triggered them
except Exception as e:
errMessage = "Failed to execute: %s; original=%r" % (
command,
e,
)
if logger is not None:
logger.exception(errMessage)
raise
def runWithRetries(command, retries=1, delay=1, logger=None):
"""
Run a command up to retries times until it succeeds.
@param command: Command line to run
@param retries: maximum number of retries
@param delay: delay in seconds between retries
@param printEnv: whether or not to print the environment passed to command
@param logger: logger for additional debug info if desired
@raises infrastructure.utilities.exceptions.CommandFailedError
if the command doesn't succeed after trying retries times
"""
attempts = 0
while attempts < retries:
attempts = attempts + 1
try:
runWithOutput(command, printEnv=printEnv, logger=logger)
return
except CommandFailedError:
if logger is not None:
logger.debug(
"Attempt %s to '%s' failed, retrying in %s seconds...",
attempts, command, delay)
time.sleep(delay)
errMessage = "%s failed after %s attempts" % (command, retries)
raise CommandFailedError(errMessage)
def getShaFromRemoteBranch(gitRemoteRepo, gitRemoteBranch, logger):
"""
Get the actual SHA of the current HEAD of a remote repo / branch.
:param str gitRemoteRepo: The URL of the remote repo,
e.g., [email protected]:numenta/nupic.git
:param str gitRemoteBranch: The name of the remote branch, e.g., master
:param logger: logger for additional debug info
:raises infrastructure.utilities.exceptions.CommandFailedError: if the
command fails
:returns: A `String` representing the SHA
:rtype: str
"""
command = ("git", "ls-remote", gitRemoteRepo)
shaList = executeCommand(command=command, logger=logger)
if gitRemoteBranch == "master":
return shaList.split("\t")[0]
else:
formattedBranchName = "refs/heads/%s" % gitRemoteBranch
for refShaPair in shaList.split("\n"):
pair = refShaPair.split()
if pair[1] == formattedBranchName:
return pair[0]
raise CommandFailedError(
"Could not find the specified branch %s on %s" % gitRemoteBranch,
gitRemoteRepo)
def runWithRetries(command, retries=1, delay=1, logger=None):
"""
Run a command up to retries times until it succeeds.
@param command: Command line to run
@param retries: maximum number of retries
@param delay: delay in seconds between retries
@raises infrastructure.utilities.exceptions.CommandFailedError
if the command doesn't succeed after trying retries times
"""
attempts = 0
while attempts < retries:
attempts = attempts + 1
try:
runWithOutput(command)
return
except CommandFailedError:
if logger:
logger.debug(
"Attempt %s to '%s' failed, retrying in %s seconds...",
attempts, command, delay)
time.sleep(delay)
raise CommandFailedError("%s failed after %s attempts" %
(command, retries))
def executeCommand(command, env=os.environ, logger=None):
"""
Execute a command and return the raw output
@param command: String or list containing the exact command to execute.
@param env: The environment required to execute the command which is passed.
By default use os.environ
@param logger: logger for additional debug info if desired
@raises
infrastructure.utilities.exceptions.CommandFailedError if
the command fails
@returns: A str representing the raw output of the command
@rtype: string
"""
try:
if logger:
log.printEnv(env, logger)
logger.debug(command)
if isinstance(command, basestring):
command = command.strip().split(" ")
return check_output(command, env=env).strip()
except CalledProcessError:
raise CommandFailedError("Failed to execute command: %s", command)
def executeCommand(command, env=None, logger=None):
"""
Execute a command and return the raw output
@param command: String or list containing the exact command to execute.
@param env: The environment required to execute the command which is passed.
By default use os.environ
@param printEnv: whether or not to print the environment passed to command
@param logger: logger for additional debug info if desired
@raises
infrastructure.utilities.exceptions.CommandFailedError if
the command fails
@returns: A str representing the raw output of the command
@rtype: string
"""
try:
if env is None:
env = os.environ
if logger is not None:
diagnostics.printEnv(env, logger)
logger.debug("**********> %s", command)
if isinstance(command, basestring):
command = command.strip().split(" ")
return check_output(command, env=env).strip()
except CalledProcessError as e:
errMessage = "Failed to execute: %s; original=%r" % (command, e,)
raise CommandFailedError(errMessage)
def runWithOutput(command, env=os.environ, logger=None):
"""
Run a command, printing as the command executes.
@param command: Command to run. Can be str or list
@param env: environment variables to use while running command
@param logger: optional logger for additional debug info if desired
"""
try:
if logger:
log.printEnv(env, logger)
logger.debug(command)
if isinstance(command, basestring):
command = command.strip().split(" ")
check_call(command, env=env)
except CalledProcessError:
raise CommandFailedError("Failed to execute command: %s" % command)
def cacheNuPICCore(env, buildWorkspace, nupicCoreSHA, uploadToS3, logger):
"""
Caches nupic.core to /var/build/NuPIC.core/<SHA> and uploads to S3
:param env: The environment dict
:param buildWorkspace: The buildWorkspace were nupic.core is built
:param nupicSha: A `string` representing SHA
:param uploadToS3: `boolean` defining whether to upload to S3 or not
:raises: CommandFailedError if the tar process fails before upload.
"""
cachedPath = "/var/build/nupic.core/%s" % nupicCoreSHA
if not os.path.isdir(cachedPath):
logger.info("Caching nupic.core to %s", cachedPath)
with changeToWorkingDir(buildWorkspace):
shutil.copytree(
"nupic.core",
("/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA))
if uploadToS3:
nupicCoreZip = "nupic.core-%s.zip" % nupicCoreSHA
logger.info("Archiving nupic.core to %s", nupicCoreZip)
command = "tar czf %s nupic.core" % nupicCoreZip
nupicCoreZipPath = "%s/%s" % (buildWorkspace, nupicCoreZip)
try:
runWithOutput(command, env, logger=logger)
logger.debug("Uploading %s to S3.", nupicCoreZip)
s3.uploadToS3(g_config, nupicCoreZipPath,
"builds_nupic_core", logger)
except:
logger.exception("Archiving nupic.core failed.")
raise CommandFailedError("Archiving nupic.core failed.")
else:
logger.info("nupic.core cached locally and to S3.")
else:
logger.debug("Cached nupic.core already exists.")