Exemplo n.º 1
0
def executeCommand(command, env=None, logger=None):
  """
  Execute a command and return the raw output

  @param command: String or list containing the exact command to execute.

  @param env: The environment required to execute the command which is passed.
              By default use os.environ

  @param printEnv: whether or not to print the environment passed to command

  @param logger: logger for additional debug info if desired

  @raises
    infrastructure.utilities.exceptions.CommandFailedError if
    the command fails

  @returns: A str representing the raw output of the command

  @rtype: string
  """
  try:
    if env is None:
      env = os.environ
    if logger is not None:
      diagnostics.printEnv(env, logger)
      logger.debug("**********> %s", command)
    if isinstance(command, basestring):
      command = command.strip().split(" ")
    return check_output(command, env=env).strip()
  except CalledProcessError as e:
    errMessage = "Failed to execute: %s; original=%r" % (command, e,)
    raise CommandFailedError(errMessage)
Exemplo n.º 2
0
def runWithOutput(command, env=None, logger=None):
  """
  Run a command, printing as the command executes.

  @param command: Command to run. Can be str or list

  @param env: environment variables to use while running command

  @param printEnv: Whether or not to print the environment passed to command

  @param logger: optional logger for additional debug info if desired
  """
  try:
    if env is None:
      env = os.environ
    if logger is not None:
      diagnostics.printEnv(env, logger)
      logger.debug("**********> %s", command)
    if isinstance(command, basestring):
      command = command.strip().split(" ")
    check_call(command, env=env)
  except CalledProcessError as e:
    errMessage = "Failed to execute: %s; original=%r" % (command, e,)
    raise CommandFailedError(errMessage)
  # Catch other exceptions, add info about what command triggered them
  except Exception as e:
    errMessage = "Failed to execute: %s; original=%r" % (command, e,)
    if logger is not None:
      logger.exception(errMessage)
    raise
def buildNuPICCore(env, nupicCoreSha, logger):
    """
    Builds nupic.core

    :param env: The environment which will be set before building.
    :param nupicCoreSha: The SHA which will be built.

    :raises
      infrastructure.utilities.exceptions.NupicBuildFailed:
      This exception is raised if build fails.
  """
    print "\n----------Building nupic.core------------"
    diagnostics.printEnv(env=env, logger=logger)
    with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
        try:
            logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
            git.resetHard(nupicCoreSha)
            runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
            with changeToWorkingDir("build/scripts"):
                libdir = sysconfig.get_config_var("LIBDIR")
                runWithOutput(
                    ("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release " "-DPYTHON_LIBRARY={}/libpython2.7.so").format(
                        libdir
                    ),
                    env=env,
                    logger=logger,
                )
                runWithOutput("make -j 4", env=env, logger=logger)
                runWithOutput("make install", env=env, logger=logger)

            # need to remove this folder to allow the caching process to work
            shutil.rmtree("external/linux32arm")

            # build the distributions
            command = "python setup.py install --force"
            # Building on jenkins, not local
            if "JENKINS_HOME" in os.environ:
                command += " bdist_wheel bdist_egg upload -r numenta-pypi"
            runWithOutput(command=command, env=env, logger=logger)
        except CommandFailedError:
            raise NupicBuildFailed("nupic.core building failed.Exiting")
        except:
            raise PipelineError("nupic.core building failed due to unknown reason.")
        else:
            logger.info("nupic.core building was successful.")
Exemplo n.º 4
0
def buildNuPICCore(env, nupicCoreSha, logger):
    """
    Builds nupic.core

    :param env: The environment which will be set before building.
    :param nupicCoreSha: The SHA which will be built.

    :raises
      infrastructure.utilities.exceptions.NupicBuildFailed:
      This exception is raised if build fails.
  """
    print "\n----------Building nupic.core------------"
    diagnostics.printEnv(env=env, logger=logger)
    with changeToWorkingDir(env["NUPIC_CORE_DIR"]):
        try:
            logger.debug("Building nupic.core SHA : %s ", nupicCoreSha)
            git.resetHard(nupicCoreSha)
            runWithOutput("mkdir -p build/scripts", env=env, logger=logger)
            with changeToWorkingDir("build/scripts"):
                libdir = sysconfig.get_config_var('LIBDIR')
                runWithOutput(
                    ("cmake ../../src -DCMAKE_INSTALL_PREFIX=../release "
                     "-DPYTHON_LIBRARY={}/libpython2.7.so").format(libdir),
                    env=env,
                    logger=logger)
                runWithOutput("make -j 4", env=env, logger=logger)
                runWithOutput("make install", env=env, logger=logger)

            # need to remove this folder to allow the caching process to work
            shutil.rmtree("external/linux32arm")

            # build the distributions
            command = "python setup.py install --force"
            # Building on jenkins, not local
            if "JENKINS_HOME" in os.environ:
                command += " bdist_wheel bdist_egg upload -r numenta-pypi"
            runWithOutput(command=command, env=env, logger=logger)
        except CommandFailedError:
            raise NupicBuildFailed("nupic.core building failed.Exiting")
        except:
            raise PipelineError(
                "nupic.core building failed due to unknown reason.")
        else:
            logger.info("nupic.core building was successful.")
Exemplo n.º 5
0
def buildNuPIC(env, logger):
    """
    Builds NuPIC

    :param env: The environment which will be set before building

    :raises
      infrastructure.utilities.exceptions.NupicBuildFailed:
      This exception is raised if build fails.
  """
    print "\n----------Building NuPIC------------"
    diagnostics.printEnv(env=env, logger=logger)

    # Build
    with changeToWorkingDir(env["NUPIC"]):
        try:
            try:
                shutil.rmtree("build")
            except OSError:
                # didn't exist, so just pass
                pass
            # need to remove this folder to allow the caching process to work
            shutil.rmtree("external/linux32arm")

            # build the distributions
            command = ("python setup.py install bdist_wheel bdist_egg "
                       "--nupic-core-dir=%s" %
                       os.path.join(env["NUPIC_CORE_DIR"], "build", "release"))
            # Building on jenkins, not local
            if "JENKINS_HOME" in os.environ:
                command += " upload -r numenta-pypi"

            runWithOutput(command=command, env=env, logger=logger)
        except:
            logger.exception("Failed while building nupic")
            raise NupicBuildFailed("NuPIC building failed.")
        else:
            open("nupic.stamp", "a").close()
            logger.debug("NuPIC building was successful.")
Exemplo n.º 6
0
def runUnitTests(env):
    """
    Runs tests listed in files present at {GROK_HOME}/tests/ci/

    :param env: Current environ set for GROK_HOME, etc
    :param pipeline: name of repository which has triggered this build
    :param grokSha: grok SHA used current run
    :returns: return True if tests are successful
    :rtype: bool

  """
    # Print environment for debug purposes
    printEnv(env, g_logger)
    buildWorkspace = os.environ["BUILD_WORKSPACE"]

    xunitSuccess = True
    with open(
            os.path.join(env["GROK_HOME"], "tests/ci/test_commands_xunit.txt"),
            "r") as tests:
        xunitTests = [test.strip() % dict(globals().items() + \
                      locals().items()) for test in tests]

    with changeToWorkingDir(os.path.join(buildWorkspace, "products")):
        g_logger.debug(os.getcwd())
        for xunitTest in xunitTests:
            g_logger.info("-------Running %s -------", xunitTest)
            xunitSuccess = runTestCommand(xunitTest, env)
            g_logger.info("\n\n###### COMPLETED %s tests ######\n\n",
                          xunitTest)
            if not xunitSuccess:
                g_logger.error("-------Failed %s -------", xunitTest)
                break

    if "WORKSPACE" in os.environ:
        # `WORKSPACE` should only be set by Jenkins and we only want to record
        # the test results if we're on Jenkins
        recordXunitTestsResults()

    return xunitSuccess
def buildNuPIC(env, logger):
    """
    Builds NuPIC

    :param env: The environment which will be set before building

    :raises
      infrastructure.utilities.exceptions.NupicBuildFailed:
      This exception is raised if build fails.
  """
    print "\n----------Building NuPIC------------"
    diagnostics.printEnv(env=env, logger=logger)

    # Build
    with changeToWorkingDir(env["NUPIC"]):
        try:
            try:
                shutil.rmtree("build")
            except OSError:
                # didn't exist, so just pass
                pass
            # need to remove this folder to allow the caching process to work
            shutil.rmtree("external/linux32arm")

            # build the distributions
            command = "python setup.py install bdist_wheel bdist_egg " "--nupic-core-dir=%s" % os.path.join(
                env["NUPIC_CORE_DIR"], "build", "release"
            )
            # Building on jenkins, not local
            if "JENKINS_HOME" in os.environ:
                command += " upload -r numenta-pypi"

            runWithOutput(command=command, env=env, logger=logger)
        except:
            logger.exception("Failed while building nupic")
            raise NupicBuildFailed("NuPIC building failed.")
        else:
            open("nupic.stamp", "a").close()
            logger.debug("NuPIC building was successful.")
Exemplo n.º 8
0
def preBuildSetup(env, pipelineConfig):
  """
    Clone the HTM-IT repo if needed and get it set to the right remote, branch,
    and SHA.

    :param env: The environment variable which is set before building
    :param pipelineConfig: dict of the pipeline config values, e.g.:
      {
        "buildWorkspace": "/path/to/build/in",
        "htm-itRemote": "[email protected]:Numenta/numenta-apps.git",
        "htm-itBranch": "master",
        "htm-itSha": "HEAD",
        "pipelineParams": "{dict of parameters}",
        "pipelineJson": "/path/to/json/file"
      }

    :returns: The updated pipelineConfig dict
    :rtype: dict
  """
  diagnostics.printEnv(env=env, logger=g_logger)

  # Clone HTM-IT if needed, otherwise, setup remote
  with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
    if not os.path.isdir(env["HTM-IT_HOME"]):
      git.clone(gitURL=pipelineConfig["htm-itRemote"],
                logger=g_logger)

  with changeToWorkingDir(env["HTM-IT_HOME"]):
    if pipelineConfig["htm-itSha"]:
      g_logger.debug("Resetting to %s", pipelineConfig["htm-itSha"])
      git.resetHard(sha=pipelineConfig["htm-itSha"], logger=g_logger)
    else:
      htm-itSha = git.getShaFromRemoteBranch(pipelineConfig["htm-itRemote"],
                                           pipelineConfig["htm-itBranch"],
                                           logger=g_logger)
      pipelineConfig["htm-itSha"] = htm-itSha
      g_logger.debug("Resetting to %s", htm-itSha)
      git.resetHard(sha=htm-itSha, logger=g_logger)
def preBuildSetup(env, pipelineConfig):
  """
    Clone the HTM-IT repo if needed and get it set to the right remote, branch,
    and SHA.

    :param env: The environment variable which is set before building
    :param pipelineConfig: dict of the pipeline config values, e.g.:
      {
        "buildWorkspace": "/path/to/build/in",
        "htmitRemote": "[email protected]:Numenta/numenta-apps.git",
        "htmitBranch": "master",
        "htmItSha": "HEAD",
        "pipelineParams": "{dict of parameters}",
        "pipelineJson": "/path/to/json/file"
      }

    :returns: The updated pipelineConfig dict
    :rtype: dict
  """
  diagnostics.printEnv(env=env, logger=g_logger)

  # Clone HTM-IT if needed, otherwise, setup remote
  with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
    if not os.path.isdir(env["HTM_IT_HOME"]):
      git.clone(gitURL=pipelineConfig["htmitRemote"],
                logger=g_logger)

  with changeToWorkingDir(env["HTM_IT_HOME"]):
    if pipelineConfig["htmItSha"]:
      g_logger.debug("Resetting to %s", pipelineConfig["htmItSha"])
      git.resetHard(sha=pipelineConfig["htmItSha"], logger=g_logger)
    else:
      htmItSha = git.getShaFromRemoteBranch(pipelineConfig["htmitRemote"],
                                           pipelineConfig["htmitBranch"],
                                           logger=g_logger)
      pipelineConfig["htmItSha"] = htmItSha
      g_logger.debug("Resetting to %s", htmItSha)
      git.resetHard(sha=htmItSha, logger=g_logger)
Exemplo n.º 10
0
def preBuildSetup(env, pipelineConfig):
  """
    Clone the Grok repo if needed and get it set to the right remote, branch,
    and SHA.

    :param env: The environment variable which is set before building
    :param pipelineConfig: dict of the pipeline config values, e.g.:
      {
        "buildWorkspace": "/path/to/build/in",
        "grokRemote": "[email protected]:Numenta/numenta-apps.git",
        "grokBranch": "master",
        "grokSha": "HEAD",
        "pipelineParams": "{dict of parameters}",
        "pipelineJson": "/path/to/json/file"
      }

    :returns: The updated pipelineConfig dict
    :rtype: dict
  """
  log.printEnv(env, g_logger)

  # Clone Grok if needed, otherwise, setup remote
  with changeToWorkingDir(pipelineConfig["buildWorkspace"]):
    if not os.path.isdir(env["GROK_HOME"]):
      git.clone(pipelineConfig["grokRemote"], directory="products")

  with changeToWorkingDir(env["GROK_HOME"]):
    if pipelineConfig["grokSha"]:
      g_logger.debug("Resetting to %s", pipelineConfig["grokSha"])
      git.resetHard(pipelineConfig["grokSha"])
    else:
      grokSha = git.getShaFromRemoteBranch(pipelineConfig["grokRemote"],
                                           pipelineConfig["grokBranch"])
      pipelineConfig["grokSha"] = grokSha
      g_logger.debug("Resetting to %s", grokSha)
      git.resetHard(grokSha)
Exemplo n.º 11
0
  rpmName = "htm-it"
  try:
    syncRpm = False
    sha = htm-itSha
    rpmExists = checkRpmExists(rpmName, sha, rpmNameDetails, config, logger)
    with shell_env(**env):
      if not rpmExists:
        logger.info("Creating %s rpm.", rpmName)

        # Clean stale rpms
        with changeToWorkingDir(OPERATIONS_SCRIPTS):
          try:
            # Delete any previously created rpm
            for name in glob.glob("nta-products-htm-it-*.rpm"):
              os.remove(name)
            log.printEnv(env, logger)
            infrastuctureCommonPath = os.path.join(PRODUCTS_PATH,
                                                   "infrastructure",
                                                   "infrastructure")

            command = ("%s/create-numenta-rpm" % infrastuctureCommonPath +
                       " --rpm-flavor htm-it" +
                       " --debug" +
                       " --cleanup-script htm-it/htm-it/pipeline/scripts/rpm-creator" +
                       "/clean-htm-it-tree-for-packaging" +
                       " --whitelist htm-it" +
                       " --whitelist nta.utils" +
                       " --whitelist htmengine" +
                       " --whitelist infrastructure" +
                       " --whitelist install-htm-it.sh" +
                       " --base-version " + releaseVersion +
Exemplo n.º 12
0
def buildRpms(env, grokSha, releaseVersion, artifactsDir, logger, config,
              grokRemote):
    """
  Builds an rpm for grok

  Takes the sha according to grok and checks that the sha.json file
  is present (also checks if the rpm is present on rpmbuild and in S3), if
  not it creates the rpm.

  :param env: The environment variables which is set.
  :param grokSha: The grok sha.
  :param releaseVersion: The product version which will be used
                         in the name of RPM
  :param artifactsDir: In this directory the artifacts will be stored.
  :param config: This is a dict of configuration data here we are using
                   AWS secret and access.
  :returns: syncRpmStatus(It is list which will help recongnize if RPM's rpm
            should be synced) and rpmNameDetails(It is a dict which contains the
            RPM name of Grok)
  :raises: infrastructure.utilities.exceptions.MissingRPMError,
           when RPM is not found.
           infrastructure.utilities.exceptions.FailedToMoveRPM,
           if there is some error while moving RPM's to
           rpmbuild.groksolutions.com
  """

    rpmNameDetails = {}
    rpmName = "grok"
    try:
        syncRpm = False
        sha = grokSha
        rpmExists = checkRpmExists(rpmName, sha, rpmNameDetails, config,
                                   logger)
        with shell_env(**env):
            if not rpmExists:
                logger.info("Creating %s rpm.", rpmName)

                # Clean stale rpms
                with changeToWorkingDir(OPERATIONS_SCRIPTS):
                    try:
                        # Delete any previously created rpm
                        for name in glob.glob("nta-products-grok-*.rpm"):
                            os.remove(name)
                        log.printEnv(env, logger)
                        infrastuctureCommonPath = os.path.join(
                            PRODUCTS_PATH, "infrastructure", "infrastructure")

                        command = (
                            "%s/create-numenta-rpm" % infrastuctureCommonPath +
                            " --rpm-flavor grok" + " --debug" +
                            " --cleanup-script grok/grok/pipeline/scripts/rpm-creator"
                            + "/clean-grok-tree-for-packaging" +
                            " --whitelist grok" + " --whitelist nta.utils" +
                            " --whitelist htmengine" +
                            " --whitelist infrastructure" +
                            " --whitelist install-grok.sh" +
                            " --base-version " + releaseVersion +
                            " --description Grok-installed-from-products-repo"
                            + " --rpm-name nta-products-grok" +
                            " --tempdir /tmp/grokbuild" +
                            " --setup-py-arguments develop" +
                            " --log-level debug" +
                            " --setup-py-dir nta.utils" +
                            " --setup-py-dir htmengine" +
                            " --setup-py-dir infrastructure" +
                            " --extend-pythonpath grok/lib/python2.7/site-packages"
                            + " --sha " + grokSha + " --artifact opt" +
                            " --git-url " + grokRemote)
                        # Due to some environment issue's I have used local here,
                        # we can change this later.
                        # fixme https://jira.numenta.com/browse/TAUR-797
                        from fabric.api import local
                        local(command)
                        # getting name of the RPM created
                        nameOfRpmCreated = glob.glob(
                            "nta-products-grok-*.rpm").pop()
                        if not nameOfRpmCreated:
                            raise exceptions.MissingRPMError(
                                "%s rpm name not found exiting" % rpmName)
                        # Creating artifact
                        with open("%s.txt" % rpmName, "w") as fp:
                            fp.write(nameOfRpmCreated)

                        logger.info(
                            "\n\n######### %s RPM created #########\n\n" %
                            rpmName)
                    except:
                        raise exceptions.RPMBuildingError(
                            "Failed while creating %s RPM." % rpmName)
                    else:
                        syncRpm = True

                filename = os.path.join(OPERATIONS_SCRIPTS, "%s.txt" % rpmName)
                # updating rpm details
                rpmNameDetails.update({rpmName: nameOfRpmCreated})
                # moving the rpms name to artifacts directory
                move(filename, artifactsDir)
                shaFileName = createShaFile(nameOfRpmCreated, sha)
                # move rpmname to rpmbuild
                status = moveRpmsToRpmbuild(nameOfRpmCreated, config, logger)
                if status:
                    uploadShaFiletoBucket(rpmName, shaFileName, logger)
                    # deleting the rpm after copying to rpmbuild
                    os.remove("%s/%s" % (OPERATIONS_SCRIPTS, nameOfRpmCreated))
                else:
                    raise exceptions.FailedToMoveRPM("Failed to move rpms to "
                                                     "rpmbuilder machine")
            else:
                logger.info(
                    "RPM for %s with %s sha already exists,"
                    "skipping creation of rpm!!", rpmName, sha)
        return syncRpm, rpmNameDetails
    except Exception:
        logger.exception("RPM building failed.")
        raise
Exemplo n.º 13
0
def buildRpms(env, grokSha, releaseVersion,
              artifactsDir, logger, config, grokRemote):
  """
  Builds an rpm for grok

  Takes the sha according to grok and checks that the sha.json file
  is present (also checks if the rpm is present on rpmbuild and in S3), if
  not it creates the rpm.

  :param env: The environment variables which is set.
  :param grokSha: The grok sha.
  :param releaseVersion: The product version which will be used
                         in the name of RPM
  :param artifactsDir: In this directory the artifacts will be stored.
  :param config: This is a dict of configuration data here we are using
                   AWS secret and access.
  :returns: syncRpmStatus(It is list which will help recongnize if RPM's rpm
            should be synced) and rpmNameDetails(It is a dict which contains the
            RPM name of Grok)
  :raises: infrastructure.utilities.exceptions.MissingRPMError,
           when RPM is not found.
           infrastructure.utilities.exceptions.FailedToMoveRPM,
           if there is some error while moving RPM's to
           rpmbuild.groksolutions.com
  """

  rpmNameDetails = {}
  rpmName = "grok"
  try:
    syncRpm = False
    sha = grokSha
    rpmExists = checkRpmExists(rpmName, sha, rpmNameDetails, config, logger)
    with shell_env(**env):
      if not rpmExists:
        logger.info("Creating %s rpm.", rpmName)

        # Clean stale rpms
        with changeToWorkingDir(OPERATIONS_SCRIPTS):
          try:
            # Delete any previously created rpm
            for name in glob.glob("nta-products-grok-*.rpm"):
              os.remove(name)
            log.printEnv(env, logger)
            infrastuctureCommonPath = os.path.join(PRODUCTS_PATH,
                                                   "infrastructure",
                                                   "infrastructure")

            command = ("%s/create-numenta-rpm" % infrastuctureCommonPath +
                       " --rpm-flavor grok" +
                       " --debug" +
                       " --cleanup-script grok/grok/pipeline/scripts/rpm-creator" +
                       "/clean-grok-tree-for-packaging" +
                       " --whitelist grok" +
                       " --whitelist nta.utils" +
                       " --whitelist htmengine" +
                       " --whitelist infrastructure" +
                       " --whitelist install-grok.sh" +
                       " --base-version " + releaseVersion +
                       " --description Grok-installed-from-products-repo" +
                       " --rpm-name nta-products-grok" +
                       " --tempdir /tmp/grokbuild" +
                       " --setup-py-arguments develop" +
                       " --log-level debug" +
                       " --setup-py-dir nta.utils" +
                       " --setup-py-dir htmengine" +
                       " --setup-py-dir infrastructure" +
                       " --extend-pythonpath grok/lib/python2.7/site-packages" +
                       " --sha " + grokSha +
                       " --artifact opt" +
                       " --git-url " + grokRemote)
            # Due to some environment issue's I have used local here,
            # we can change this later.
            # fixme https://jira.numenta.com/browse/TAUR-797
            from fabric.api import local
            local(command)
            # getting name of the RPM created
            nameOfRpmCreated = glob.glob("nta-products-grok-*.rpm").pop()
            if not nameOfRpmCreated:
              raise exceptions.MissingRPMError("%s rpm name not found exiting"
                                               % rpmName)
            # Creating artifact
            with open("%s.txt" % rpmName, "w") as fp:
              fp.write(nameOfRpmCreated)

            logger.info("\n\n######### %s RPM created #########\n\n"
                        % rpmName)
          except:
            raise exceptions.RPMBuildingError("Failed while creating %s RPM."
                                              % rpmName)
          else:
            syncRpm = True

        filename = os.path.join(OPERATIONS_SCRIPTS, "%s.txt" % rpmName)
        # updating rpm details
        rpmNameDetails.update({rpmName:nameOfRpmCreated})
        # moving the rpms name to artifacts directory
        move(filename, artifactsDir)
        shaFileName = createShaFile(nameOfRpmCreated, sha)
        # move rpmname to rpmbuild
        status = moveRpmsToRpmbuild(nameOfRpmCreated, config, logger)
        if status:
          uploadShaFiletoBucket(rpmName, shaFileName, logger)
          # deleting the rpm after copying to rpmbuild
          os.remove("%s/%s" %  (OPERATIONS_SCRIPTS, nameOfRpmCreated))
        else:
          raise exceptions.FailedToMoveRPM("Failed to move rpms to "
                                           "rpmbuilder machine")
      else:
        logger.info("RPM for %s with %s sha already exists,"
                    "skipping creation of rpm!!", rpmName, sha)
    return syncRpm, rpmNameDetails
  except Exception:
    logger.exception("RPM building failed.")
    raise