コード例 #1
0
ファイル: build.py プロジェクト: chetan51/numenta-apps
def downloadOrCreateNuPICWheel(env, pipelineConfig):
  """
    Downloads the NuPIC wheel from S3 for a given SHA. If a wheel is not found
    for a particular SHA then NuPIC wheel is built for this SHA and uploaded to
    S3. The building and uploading of NuPIC wheel is delegated to nupic.

    While downloading a wheel from S3, if no NuPIC SHA is provided then the
    correct wheel version is read from
    "stable_nupic_version/nupic-package-version.txt" and if SHA is provided then
    correct wheel version is read from "stable_nupic_version/<SHA>".

    :param env: The environment variable which is set before building
    :param pipelineConfig: dict of the pipeline config values, e.g.:
      {
        "buildWorkspace": "/path/to/build/in",
        "grokRemote": "[email protected]:Numenta/numenta-apps.git",
        "grokBranch": "master",
        "grokSha": "HEAD",
        "nupicRemote": "[email protected]:numenta/nupic.git",
        "nupicBranch": "master",
        "nupicSha": "HEAD",
        "pipelineParams": "{dict of parameters}",
        "pipelineJson": "/path/to/json/file"
      }
    :returns: The absolute path of the NuPIC wheel.
    :rtype: string
  """
  g_config = yaml.load(resource_stream(__name__, "../conf/config.yaml"))

  nupicSha = pipelineConfig["nupicSha"]

  if nupicSha:
    path = "stable_nupic_version/%s" % nupicSha
  else:
    path = "stable_nupic_version/nupic-package-version.txt"

  bucketName = g_config["S3_MAPPING_BUCKET"]

  try:
    with open(s3.downloadFileFromS3(bucketName, path, g_logger), "r") as fHandle:
      contents = fHandle.readline().strip()
      pipelineConfig["nupicSha"] = contents.split(":")[0].strip()
      wheelFile = contents.split(":")[1].strip()
  except AttributeError:
    g_logger.debug("NuPIC wheel for %s not found in S3", nupicSha)
    g_logger.debug("Building NuPIC wheel for %s" % nupicSha)
    builder.buildNuPIC(env, pipelineConfig, g_logger)
    wheelFilePath = glob.glob("%s/dist/*.whl" % env["NUPIC"])[0]
    pipelineConfig["nupicBuilt"] = True
  else:
    g_logger.debug("Downloading NuPIC wheel from S3 : %s" % wheelFile)
    with changeToWorkingDir(env["BUILD_WORKSPACE"]):
      wheelFilePath = s3.downloadFileFromS3(bucketName,
                                            "builds_nupic_wheel/%s" % wheelFile,
                                            g_logger)
    pipelineConfig["nupicBuilt"] = False

  return wheelFilePath
コード例 #2
0
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger):
    """
    Downloads archieved nupic.core from S3

    :param buildWorkspace: The workspace where nupic.core will be built
    :param nupicCoreSHA: The SHA of the nupic.core build that needs to be
      fetched
  """
    logger.info("Downloading nupic.core from S3.")
    cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA
    with changeToWorkingDir(buildWorkspace):
        nupicCoreFilePath = s3.downloadFileFromS3(
            "builds.numenta.com",
            "builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA, logger)

        logger.info("Untarring %s", nupicCoreFilePath)
        command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir)
        try:
            os.makedirs(cachedDir)
            runWithOutput(command, logger=logger)
        except OSError:
            logger.exception("Cached nupic.core already exists at %s",
                             cachedDir)
            raise
        except CommandFailedError:
            logger.exception("Failed while untarring cached nupic.core: %s",
                             command)
            raise
        else:
            logger.info("nupic.core downloaded from S3 & stored at %s",
                        cachedDir)
コード例 #3
0
def fetchNuPICCoreFromS3(buildWorkspace, nupicCoreSHA, logger):
  """
    Downloads archieved nupic.core from S3

    :param buildWorkspace: The workspace where nupic.core will be built
    :param nupicCoreSHA: The SHA of the nupic.core build that needs to be
      fetched
  """
  logger.info("Downloading nupic.core from S3.")
  cachedDir = "/var/build/nupic.core/%s" % nupicCoreSHA
  with changeToWorkingDir(buildWorkspace):
    nupicCoreFilePath = s3.downloadFileFromS3("builds.numenta.com",
                          "builds_nupic_core/nupic.core-%s.zip" % nupicCoreSHA,
                          logger)

    logger.info("Untarring %s", nupicCoreFilePath)
    command = "tar xzvf %s -C %s" % (nupicCoreFilePath, cachedDir)
    try:
      os.makedirs(cachedDir)
      runWithOutput(command, logger=logger)
    except OSError:
      logger.exception("Cached nupic.core already exists at %s", cachedDir)
      raise
    except CommandFailedError:
      logger.exception("Failed while untarring cached nupic.core: %s", command)
      raise
    else:
      logger.info("nupic.core downloaded from S3 & stored at %s", cachedDir)
コード例 #4
0
ファイル: bake_ami.py プロジェクト: chetan51/numenta-apps
def downloadGrokRPM(grokSHA, workDir):
  """
  Download the grok rpm for a given SHA

  :param grokSHA - SHA we need a Grok RPM for
  :param workDir - Where to store the downloaded RPM

  :returns full path to the Grok RPM

  :rtype string
  """
  grokRpmName = getMappingsFromShaToRpm(
                  repo="grok",
                  sha=grokSHA,
                  s3MappingBucket=g_config["S3_MAPPING_BUCKET"],
                  logger=g_logger)
  g_logger.debug("Found grok RPM name %s, downloading", grokRpmName)
  with changeToWorkingDir(workDir):
    downloadFileFromS3(bucketName=S3_YUM_BUCKET,
                       path="s3/x86_64/%s" % grokRpmName,
                       logger=g_logger)
  return "%s/%s" % (workDir, grokRpmName)
コード例 #5
0
ファイル: bake_ami.py プロジェクト: sergius/numenta-apps
def downloadGrokRPM(grokSHA, workDir):
    """
  Download the grok rpm for a given SHA

  :param grokSHA - SHA we need a Grok RPM for
  :param workDir - Where to store the downloaded RPM

  :returns full path to the Grok RPM

  :rtype string
  """
    grokRpmName = getMappingsFromShaToRpm(
        repo="grok",
        sha=grokSHA,
        s3MappingBucket=g_config["S3_MAPPING_BUCKET"],
        logger=g_logger)
    g_logger.debug("Found grok RPM name %s, downloading", grokRpmName)
    with changeToWorkingDir(workDir):
        downloadFileFromS3(bucketName=S3_YUM_BUCKET,
                           path="yum/x86_64/%s" % grokRpmName,
                           logger=g_logger)
    return "%s/%s" % (workDir, grokRpmName)