def downloadOrCreateNuPICWheel(env, pipelineConfig): """ Downloads the NuPIC wheel from S3 for a given SHA. If a wheel is not found for a particular SHA then NuPIC wheel is built for this SHA and uploaded to S3. The building and uploading of NuPIC wheel is delegated to nupic. While downloading a wheel from S3, if no NuPIC SHA is provided then the correct wheel version is read from "stable_nupic_version/nupic-package-version.txt" and if SHA is provided then correct wheel version is read from "stable_nupic_version/<SHA>". :param env: The environment variable which is set before building :param pipelineConfig: dict of the pipeline config values, e.g.: { "buildWorkspace": "/path/to/build/in", "grokRemote": "[email protected]:Numenta/numenta-apps.git", "grokBranch": "master", "grokSha": "HEAD", "nupicRemote": "[email protected]:numenta/nupic.git", "nupicBranch": "master", "nupicSha": "HEAD", "pipelineParams": "{dict of parameters}", "pipelineJson": "/path/to/json/file" } :returns: The absolute path of the NuPIC wheel. :rtype: string """ g_config = yaml.load(resource_stream(__name__, "../conf/config.yaml")) nupicSha = pipelineConfig["nupicSha"] if nupicSha: path = "stable_nupic_version/%s" % nupicSha else: path = "stable_nupic_version/nupic-package-version.txt" bucketName = g_config["S3_MAPPING_BUCKET"] try: with open(s3.downloadFileFromS3(bucketName, path, g_logger), "r") as fHandle: contents = fHandle.readline().strip() pipelineConfig["nupicSha"] = contents.split(":")[0].strip() wheelFile = contents.split(":")[1].strip() except AttributeError: g_logger.debug("NuPIC wheel for %s not found in S3", nupicSha) g_logger.debug("Building NuPIC wheel for %s" % nupicSha) builder.buildNuPIC(env, pipelineConfig, g_logger) wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0] pipelineConfig["nupicBuilt"] = True else: g_logger.debug("Downloading NuPIC wheel from S3 : %s" % wheelFile) with changeToWorkingDir(env["BUILD_WORKSPACE"]): wheelFilePath = s3.downloadFileFromS3(bucketName, "builds_nupic_wheel/%s" % wheelFile, g_logger) pipelineConfig["nupicBuilt"] = False return wheelFilePath
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger): """ Downloads archieved nupic.core from S3 :param buildWorkspace: The workspace where nupic.core will be built :param nupicCoreSHA: The SHA of the nupic.core build that needs to be fetched """ logger.info("Downloading nupic.core from S3.") cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA with changeToWorkingDir(buildWorkspace): nupicCoreFilePath = s3.downloadFileFromS3( "builds.numenta.com", "builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA, logger) logger.info("Untarring %s", nupicCoreFilePath) command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir) try: os.makedirs(cachedDir) runWithOutput(command, logger=logger) except OSError: logger.exception("Cached nupic.core already exists at %s", cachedDir) raise except CommandFailedError: logger.exception("Failed while untarring cached nupic.core: %s", command) raise else: logger.info("nupic.core downloaded from S3 & stored at %s", cachedDir)
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger): """ Downloads archieved nupic.core from S3 :param buildWorkspace: The workspace where nupic.core will be built :param nupicCoreSHA: The SHA of the nupic.core build that needs to be fetched """ logger.info("Downloading nupic.core from S3.") cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA with changeToWorkingDir(buildWorkspace): nupicCoreFilePath = s3.downloadFileFromS3("builds.numenta.com", "builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA, logger) logger.info("Untarring %s", nupicCoreFilePath) command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir) try: os.makedirs(cachedDir) runWithOutput(command, logger=logger) except OSError: logger.exception("Cached nupic.core already exists at %s", cachedDir) raise except CommandFailedError: logger.exception("Failed while untarring cached nupic.core: %s", command) raise else: logger.info("nupic.core downloaded from S3 & stored at %s", cachedDir)
def downloadGrokRPM(grokSHA, workDir): """ Download the grok rpm for a given SHA :param grokSHA - SHA we need a Grok RPM for :param workDir - Where to store the downloaded RPM :returns full path to the Grok RPM :rtype string """ grokRpmName = getMappingsFromShaToRpm( repo="grok", sha=grokSHA, s3MappingBucket=g_config["S3_MAPPING_BUCKET"], logger=g_logger) g_logger.debug("Found grok RPM name %s, downloading", grokRpmName) with changeToWorkingDir(workDir): downloadFileFromS3(bucketName=S3_YUM_BUCKET, path="s3/x86_64/%s" % grokRpmName, logger=g_logger) return "%s/%s" % (workDir, grokRpmName)
def downloadGrokRPM(grokSHA, workDir): """ Download the grok rpm for a given SHA :param grokSHA - SHA we need a Grok RPM for :param workDir - Where to store the downloaded RPM :returns full path to the Grok RPM :rtype string """ grokRpmName = getMappingsFromShaToRpm( repo="grok", sha=grokSHA, s3MappingBucket=g_config["S3_MAPPING_BUCKET"], logger=g_logger) g_logger.debug("Found grok RPM name %s, downloading", grokRpmName) with changeToWorkingDir(workDir): downloadFileFromS3(bucketName=S3_YUM_BUCKET, path="yum/x86_64/%s" % grokRpmName, logger=g_logger) return "%s/%s" % (workDir, grokRpmName)