def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env, logger, nupicCoreRemote, nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
elif s3.checkIfNuPICCoreInS3(g_config, nupicCoreSha):
fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSha, logger)
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
# Cached nupic.core builds don't work on OS X, so clean it up and rebuild
if "darwin" in sys.platform:
shutil.rmtree(os.path.join(nupicCoreDir, "build"))
boolBuildNupicCore = True
logger.debug("Retrieved nupic.core from S3; saved to: %s",
nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally or in S3.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC wheel, but only upload to S3 from a linux box
cacheNuPIC(env, nupicSha, "darwin" not in sys.platform, logger)
# Cache nupic.core, but only upload to S3 from a linux box
cacheNuPICCore(env, buildWorkspace, nupicCoreSha, "darwin"
not in sys.platform, logger)
def executeBuildProcess(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
:param dict env: dictionary of environment variables
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicRemote: location of nupic remote, e.g.,
https://github.com/numenta/nupic
:param str nupicBranch: which branch to build, e.g., master
:param str nupicSha: which nupic commit SHA to build
:param str nupicCoreRemote: location of nupic.core remote, e.g.,
https://github.com/numenta/nupic.core
:param str nupicCoreSha: which nupic.core commit SHA to build
:param logger:
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(env=env,
nupicCoreRemote=nupicCoreRemote,
nupicCoreSha=nupicCoreSha)
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
addNupicCoreToEnv(env, nupicCoreDir)
buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace)
buildNuPIC(env, logger, buildWorkspace)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace,
nupicCoreSha=nupicCoreSha,
logger=logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(env,
logger, nupicCoreRemote, nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
elif s3.checkIfNuPICCoreInS3(g_config, nupicCoreSha):
fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSha, logger)
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
# Cached nupic.core builds don't work on OS X, so clean it up and rebuild
if "darwin" in sys.platform:
shutil.rmtree(os.path.join(nupicCoreDir, "build"))
boolBuildNupicCore = True
logger.debug("Retrieved nupic.core from S3; saved to: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally or in S3.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC wheel, but only upload to S3 from a linux box
cacheNuPIC(env, nupicSha, "darwin" not in sys.platform, logger)
# Cache nupic.core, but only upload to S3 from a linux box
cacheNuPICCore(env, buildWorkspace, nupicCoreSha,
"darwin" not in sys.platform, logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha,
nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env=env, nupicCoreRemote=nupicCoreRemote, nupicCoreSha=nupicCoreSha)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace,
nupicCoreSha=nupicCoreSha,
logger=logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
if isReleaseVersion(nupicBranch, nupicSha):
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
for targetFile in [VERSION_FILE, DOXYFILE, INIT_FILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env=env, nupicCoreRemote=nupicCoreRemote, nupicCoreSha=nupicCoreSha
)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha, logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSha, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace, nupicCoreSha=nupicCoreSha, logger=logger)
def fullBuild(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
nupicCoreRemote, nupicCoreSHA = getNuPICCoreDetails(env, logger)
boolBuildNupicCore = False
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSHA, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
elif s3.checkIfNuPICCoreInS3(g_config, nupicCoreSHA):
fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger)
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA
# Cached nupic.core builds don't work on OS X, so clean it up and rebuild
if "darwin" in sys.platform:
shutil.rmtree(os.path.join(nupicCoreDir, "build"))
boolBuildNupicCore = True
logger.debug("Retrieved nupic.core from S3; saved to: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally or in S3.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSHA,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
boolBuildNupicCore = True
addNupicCoreToEnv(env, nupicCoreDir)
if boolBuildNupicCore:
buildNuPICCore(env, nupicCoreSHA, logger)
buildNuPIC(env, logger)
installDir = os.path.join(env["NUPIC"], "build/release")
wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
installNuPICWheel(env, installDir, wheelFilePath, logger)
runTests(env, logger)
# Cache NuPIC wheel, but only upload to S3 from a linux box
cacheNuPIC(env, nupicSha, "darwin" not in sys.platform, logger)
# Cache nupic.core, but only upload to S3 from a linux box
cacheNuPICCore(env, buildWorkspace, nupicCoreSHA,
"darwin" not in sys.platform, logger)
def executeBuildProcess(env, buildWorkspace, nupicRemote, nupicBranch,
nupicSha, nupicCoreRemote, nupicCoreSha, logger):
"""
Run a full build of the NuPIC pipeline, including validating and, if
necessary, installing nupic.core
:param dict env: dictionary of environment variables
:param str buildWorkspace: /path/to/buildWorkspace
:param str nupicRemote: location of nupic remote, e.g.,
https://github.com/numenta/nupic
:param str nupicBranch: which branch to build, e.g., master
:param str nupicSha: which nupic commit SHA to build
:param str nupicCoreRemote: location of nupic.core remote, e.g.,
https://github.com/numenta/nupic.core
:param str nupicCoreSha: which nupic.core commit SHA to build
:param logger:
"""
fetchNuPIC(env, buildWorkspace, nupicRemote, nupicBranch, nupicSha, logger)
# If this is a release version, then update __init__.py with the right
# version number. This will ensure the proper version number is tagged in
# the wheel file
with changeToWorkingDir(os.path.join(buildWorkspace, "nupic")):
with open(VERSION_FILE, "r") as f:
devVersion = f.read().strip()
if isReleaseVersion(nupicBranch, nupicSha):
nupicVersion = nupicSha
for targetFile in [VERSION_FILE, DOXYFILE]:
logger.debug("\tUpdating %s...", targetFile)
replaceInFile(devVersion, nupicSha, targetFile)
else:
nupicVersion = devVersion[:5]
nupicCoreRemote, nupicCoreSha = getNuPICCoreDetails(
env=env, nupicCoreRemote=nupicCoreRemote, nupicCoreSha=nupicCoreSha)
nupicCoreDir = ""
if checkIfProjectExistsLocallyForSHA("nupic.core", nupicCoreSha, logger):
nupicCoreDir = "/var/build/nupic.core/%s/nupic.core" % nupicCoreSha
logger.debug("Found local nupic.core at: %s", nupicCoreDir)
else:
logger.debug("Did not find nupic.core locally.")
fetchNuPICCoreFromGH(buildWorkspace, nupicCoreRemote, nupicCoreSha,
logger)
nupicCoreDir = "%s/nupic.core" % buildWorkspace
logger.debug("Building nupic.core at: %s", nupicCoreDir)
addNupicCoreToEnv(env, nupicCoreDir)
buildNuPICCore(env, nupicCoreSha, logger, buildWorkspace, nupicVersion)
buildNuPIC(env, logger, buildWorkspace)
runTests(env, logger)
# Cache NuPIC
cacheNuPIC(env=env, nupicSha=nupicSha, logger=logger)
# Cache nupic.core
cacheNuPICCore(buildWorkspace=buildWorkspace,
nupicCoreSha=nupicCoreSha,
logger=logger)
