Exemplo n.º 1
0
    def constructPreBuiltGrokFakeroot(self):
        """
    Construct fakeroot from prebuilt grok

    :returns: SHA of the products repo in the fakeroot
    :rtype: tuple
    """

        config = self.config
        logger = self.logger
        productsDirectory = self.productsDirectory
        logger.debug("Creating %s", productsDirectory)
        mkpath(productsDirectory)
        copy_tree(config.productsDir, productsDirectory)
        iteration = git.getCommitCount(productsDirectory)

        with changeToWorkingDir(productsDirectory):
            actualSHA = git.getCurrentSha()

        # Set extra python path
        self.setPythonPath()

        # Clean Grok Scripts
        self.cleanScripts()

        # Purge anything not whitelisted
        self.purgeBlacklistedStuff()

        return (iteration, actualSHA)
Exemplo n.º 2
0
  def constructPreBuiltHTMITFakeroot(self):
    """
    Construct fakeroot from prebuilt htm-it

    :returns: SHA of the products repo in the fakeroot
    :rtype: tuple
    """

    config = self.config
    logger = self.logger
    productsDirectory = self.productsDirectory
    logger.debug("Creating %s", productsDirectory)
    mkpath(productsDirectory)
    copy_tree(config.productsDir, productsDirectory)
    iteration = git.getCommitCount(productsDirectory, logger=logger)

    with changeToWorkingDir(productsDirectory):
      actualSHA = git.getCurrentSha(logger=logger)

    # Set extra python path
    self.setPythonPath()

    # Clean HTM-IT Scripts
    self.cleanScripts()

    # Purge anything not whitelisted
    self.purgeBlacklistedStuff()

    return (iteration, actualSHA)
Exemplo n.º 3
0
  def constructInfrastructureFakeroot(self):
    """
    Construct our fakeroot directory tree

    :returns: (iteration, fakerootSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

    config = self.config
    fakeroot = self.fakeroot
    logger = self.logger
    productsDirectory = self.productsDirectory
    srvPath = os.path.join(fakeroot, "opt", "numenta")
    logger.debug("Creating %s", srvPath)
    mkpath(srvPath)

    logger.debug("Cloning %s into %s...", fakeroot, config.gitURL)

    # Collect the SHA from the fakeroot. This way we can put the SHA into
    # the RPM information even if we are packaging tip of a branch and not
    # a specific SHA
    installDirectory = os.path.join("opt", "numenta")
    fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot,
                                           installDirectory=installDirectory,
                                           repoDirectory="products",
                                           gitURL=config.gitURL,
                                           logger=logger,
                                           sha=config.sha)

    # Capture the commit count since we're going to trash products once we pull
    # out the saltcellar
    iteration = git.getCommitCount(productsDirectory, logger=logger)
    logger.debug("Commit count in %s is %s", productsDirectory, iteration)
    logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA)

    # Clean everything not whitelisted out of products so we don't conflict
    # with htm-it or taurus rpms
    purgeDirectory(path=productsDirectory,
                   whitelist=["__init__.py",
                              "infrastructure" ],
                   logger=logger)

    # Clean out infrastructure, too - we only want the utilities
    infraPath = os.path.join(productsDirectory, "infrastructure")
    purgeDirectory(path=infraPath,
                   whitelist=["__init__.py",
                              "DEPENDENCIES.md",
                              "infrastructure",
                              "LICENSE",
                              "README.md",
                              "requirements.txt",
                              "setup.py"],
                   logger=logger)

    return (iteration, fakerootSHA)
Exemplo n.º 4
0
    def constructInfrastructureFakeroot(self):
        """
    Construct our fakeroot directory tree

    :returns: (iteration, fakerootSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

        config = self.config
        fakeroot = self.fakeroot
        logger = self.logger
        productsDirectory = self.productsDirectory
        srvPath = os.path.join(fakeroot, "opt", "numenta")
        logger.debug("Creating %s", srvPath)
        mkpath(srvPath)

        logger.debug("Cloning %s into %s...", fakeroot, config.gitURL)

        # Collect the SHA from the fakeroot. This way we can put the SHA into
        # the RPM information even if we are packaging tip of a branch and not
        # a specific SHA
        installDirectory = os.path.join("opt", "numenta")
        fakerootSHA = rpm.gitCloneIntoFakeroot(
            fakeroot=fakeroot,
            installDirectory=installDirectory,
            repoDirectory="products",
            gitURL=config.gitURL,
            logger=logger,
            sha=config.sha)

        # Capture the commit count since we're going to trash products once we pull
        # out the saltcellar
        iteration = git.getCommitCount(productsDirectory)
        logger.debug("Commit count in %s is %s", productsDirectory, iteration)
        logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA)

        # Clean everything not whitelisted out of products so we don't conflict
        # with grok or taurus rpms
        purgeDirectory(path=productsDirectory,
                       whitelist=["__init__.py", "infrastructure"],
                       logger=logger)

        # Clean out infrastructure, too - we only want the utilities
        infraPath = os.path.join(productsDirectory, "infrastructure")
        purgeDirectory(path=infraPath,
                       whitelist=[
                           "__init__.py", "DEPENDENCIES.md", "infrastructure",
                           "LICENSE", "README.md", "requirements.txt",
                           "setup.py"
                       ],
                       logger=logger)

        return (iteration, fakerootSHA)
Exemplo n.º 5
0
    def constructGrokFakeroot(self):
        """
    Construct a Grok fakeroot directory tree.

    1. Add any directories specified with --extend-pythonpath to the PYTHONPATH
       we will be using for setup.py, build scripts and the cleanup scripts.

    2. Install any wheels that have been specied by --use-wheel

    3. Run setup.py in any directories that have been specified with
       --setup-py-dir. Uses the arguments specfied by --setup-py-arguments.

    4. Run any build scripts specified by --build-script

    5. Run any cleanup scripts specified by --cleanup-script

    6. Purge any files or directories at the top level of the checkout that were
       not whitelisted with --whitelist.

    :returns: (iteration, actualSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

        config = self.config
        fakeroot = self.fakeroot
        logger = self.logger

        logger.info("Preparing Grok fakeroot in %s\n", fakeroot)

        actualSHA = self.installProductsIntoGrokFakeroot()

        productsDirectory = self.productsDirectory
        grokPath = os.path.join(productsDirectory, "grok")
        iteration = git.getCommitCount(productsDirectory)

        # Extend PYTHONPATH for setup.py, build & cleanup scripts
        # pythonpathExtensions
        logger.debug("**************************************************")
        logger.info("Phase 1: Preparing PYTHONPATH and installing wheels")
        # Set extra python path
        self.setPythonPath()
        environment = self.environment
        sitePackagesDirectory = "%s/grok/lib/python2.7/site-packages" % \
                                productsDirectory

        # Install wheels if any have been specified
        with changeToWorkingDir(grokPath):
            for wheel in config.wheels:
                logger.info("Installing %s", os.path.basename(wheel))
                if not os.path.exists(wheel):
                    raise InvalidParametersError("%s does not exist!" % wheel)
                pipCommand = "pip install %s --no-deps --target=%s" % \
                  (wheel, sitePackagesDirectory)
                logger.debug("pip command: %s", pipCommand)
                runWithOutput(pipCommand)
                logger.debug("wheel install complete")

        # Run setup.py if specified
        logger.info("Phase 2: Running setup.py commands")

        for pyDir in config.setupPyDirs:
            pyDirPath = "%s/%s" % (productsDirectory, pyDir)
            logger.debug("Changing to %s", pyDirPath)
            with changeToWorkingDir(pyDirPath):
                setupCommand = "python setup.py develop --prefix=%s/grok" % \
                               productsDirectory
                logger.debug("Running %s", setupCommand)
                runWithOutput(setupCommand, env=environment)

        # Run any build scripts. We assume that they should be run in the
        # directory they're in.
        logger.info("Phase 3: Running build scripts...")
        for builder in config.buildScripts:
            builderPath = "%s/%s" % (fakeroot, builder)
            logger.debug("Attempting to run %s", builderPath)
            if not os.path.exists(builderPath):
                raise InvalidParametersError("%s does not exist!" %
                                             builderPath)
            workDirectory = os.path.dirname(builderPath)
            logger.debug("Changing to %s", workDirectory)
            with changeToWorkingDir(workDirectory):
                runWithOutput(builderPath, env=environment)

        # Run any cleanup scripts. We assume that they should be run in the
        # directory they're in.
        logger.info("Phase 4: Running cleanup scripts...")
        # Clean Scripts
        self.cleanScripts()

        logger.info("Phase 5: Purge anything not whitelisted.")
        # Purge anything not whitelisted
        self.purgeBlacklistedStuff()

        return (iteration, actualSHA)
Exemplo n.º 6
0
    def constructSaltcellarFakeroot(self):
        """
    Make a saltcellar fakeroot

    :returns: (iteration, fakerootSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

        config = self.config
        fakeroot = self.fakeroot
        logger = self.logger
        srvPath = os.path.join(fakeroot, "srv")
        logger.debug("Creating saltcellar fakeroot in %s", srvPath)
        productsPath = os.path.join(fakeroot, "products")
        mkpath(srvPath)

        logger.debug("Cloning...")

        # Collect the SHA from the fakeroot. This way we can put the SHA into
        # the RPM information even if we are packaging tip of a branch and not
        # a specific SHA
        fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot,
                                               installDirectory="/",
                                               repoDirectory="products",
                                               gitURL=config.gitURL,
                                               logger=logger,
                                               sha=config.sha)

        # Capture the commit count since we're going to trash products once we pull
        # out the saltcellar
        iteration = git.getCommitCount(productsPath)
        logger.debug("Commit count in %s is %s", productsPath, iteration)

        # Move the saltcellar to /srv/salt
        logger.debug("Moving saltcellar to %s/salt", srvPath)
        logger.debug("srvPath: %s", srvPath)
        logger.debug("productsPath: %s", productsPath)
        logger.debug("%s/infrastructure/saltcellar", productsPath)

        logger.debug("Checking for %s/infrastructure/saltcellar", productsPath)
        logger.debug(
            os.path.exists("%s/infrastructure/saltcellar" % productsPath))

        os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"),
                  os.path.join(srvPath, "salt"))

        # Now that we have the salt formulas, nuke the rest of products out of
        # the fakeroot
        logger.debug("Deleting products from fakeroot")
        rmrf(productsPath)

        # Finally, scrub the private data out of /srv/salt
        if not config.numenta_internal_only:
            logger.debug("Sanitizing /srv/salt")
            self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
        else:
            logger.critical(
                "Baking numenta-internal rpm, not sanitizing /srv/salt")
        return (iteration, fakerootSHA)
Exemplo n.º 7
0
  def constructSaltcellarFakeroot(self):
    """
    Make a saltcellar fakeroot

    :returns: (iteration, fakerootSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

    config = self.config
    fakeroot = self.fakeroot
    logger = self.logger
    srvPath = os.path.join(fakeroot, "srv")
    logger.debug("Creating saltcellar fakeroot in %s", srvPath)
    productsPath = os.path.join(fakeroot, "products")
    mkpath(srvPath)

    logger.debug("Cloning...")

    # Collect the SHA from the fakeroot. This way we can put the SHA into
    # the RPM information even if we are packaging tip of a branch and not
    # a specific SHA
    fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot,
                                           installDirectory="/",
                                           repoDirectory="products",
                                           gitURL=config.gitURL,
                                           logger=logger,
                                           sha=config.sha)

    # Capture the commit count since we're going to trash products once we pull
    # out the saltcellar
    iteration = git.getCommitCount(productsPath, logger=logger)
    logger.debug("Commit count in %s is %s", productsPath, iteration)

    # Move the saltcellar to /srv/salt
    logger.debug("Moving saltcellar to %s/salt", srvPath)
    logger.debug("srvPath: %s", srvPath)
    logger.debug("productsPath: %s", productsPath)
    logger.debug("%s/infrastructure/saltcellar", productsPath)

    logger.debug("Checking for %s/infrastructure/saltcellar",
                 productsPath)
    logger.debug(os.path.exists("%s/infrastructure/saltcellar" %
                                productsPath))

    os.rename(os.path.join(productsPath, "infrastructure",
                           "saltcellar"),
              os.path.join(srvPath, "salt"))

    # Now that we have the salt formulas, nuke the rest of products out of
    # the fakeroot
    logger.debug("Deleting products from fakeroot")
    rmrf(productsPath)

    # Finally, scrub the private data out of /srv/salt
    if not config.numenta_internal_only:
      logger.debug("Sanitizing /srv/salt")
      self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
    else:
      logger.critical("Baking numenta-internal rpm, not sanitizing /srv/salt")
    return (iteration, fakerootSHA)
Exemplo n.º 8
0
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

    config = self.config
    fakeroot = self.fakeroot
    logger = self.logger

    logger.info("Preparing HTM-IT fakeroot in %s\n", fakeroot)

    actualSHA = self.installProductsIntoHTM-ITFakeroot()

    productsDirectory = self.productsDirectory
    htm-itPath = os.path.join(productsDirectory, "htm-it")
    iteration = git.getCommitCount(productsDirectory, logger=logger)

    # Extend PYTHONPATH for setup.py, build & cleanup scripts
    # pythonpathExtensions
    logger.debug("**************************************************")
    logger.info("Phase 1: Preparing PYTHONPATH and installing wheels")
    # Set extra python path
    self.setPythonPath()
    environment = self.environment
    sitePackagesDirectory = "%s/htm-it/lib/python2.7/site-packages" % \
                            productsDirectory

    # Install wheels if any have been specified
    with changeToWorkingDir(htm-itPath):
      for wheel in config.wheels:
        logger.info("Installing %s", os.path.basename(wheel))
Exemplo n.º 9
0
  def constructHTMITFakeroot(self):
    """
    Construct a HTM-IT fakeroot directory tree.

    1. Add any directories specified with --extend-pythonpath to the PYTHONPATH
       we will be using for setup.py, build scripts and the cleanup scripts.

    2. Install any wheels that have been specied by --use-wheel

    3. Run setup.py in any directories that have been specified with
       --setup-py-dir. Uses the arguments specfied by --setup-py-arguments.

    4. Run any build scripts specified by --build-script

    5. Run any cleanup scripts specified by --cleanup-script

    6. Purge any files or directories at the top level of the checkout that were
       not whitelisted with --whitelist.

    :returns: (iteration, actualSHA) where iteration is the total commit count
    in the repository and fakerootSHA is the SHA in the fakeroot. If we're
    packaging a branch or tip of master, we're still going to want to know what
    the SHA was so we can include it in the RPM description.

    :rtype: tuple
    """

    config = self.config
    fakeroot = self.fakeroot
    logger = self.logger

    logger.info("Preparing HTM-IT fakeroot in %s\n", fakeroot)

    actualSHA = self.installProductsIntoHTMITFakeroot()

    productsDirectory = self.productsDirectory
    htm-itPath = os.path.join(productsDirectory, "htm-it")
    iteration = git.getCommitCount(productsDirectory, logger=logger)

    # Extend PYTHONPATH for setup.py, build & cleanup scripts
    # pythonpathExtensions
    logger.debug("**************************************************")
    logger.info("Phase 1: Preparing PYTHONPATH and installing wheels")
    # Set extra python path
    self.setPythonPath()
    environment = self.environment
    sitePackagesDirectory = "%s/htm-it/lib/python2.7/site-packages" % \
                            productsDirectory

    # Install wheels if any have been specified
    with changeToWorkingDir(htm-itPath):
      for wheel in config.wheels:
        logger.info("Installing %s", os.path.basename(wheel))
        if not os.path.exists(wheel):
          raise InvalidParametersError("%s does not exist!" % wheel)
        pipCommand = "pip install %s --no-deps --target=%s" % \
          (wheel, sitePackagesDirectory)
        logger.debug("pip command: %s", pipCommand)
        runWithOutput(pipCommand)
        logger.debug("wheel install complete")

    # Run setup.py if specified
    logger.info("Phase 2: Running setup.py commands")

    for pyDir in config.setupPyDirs:
      pyDirPath = "%s/%s" % (productsDirectory, pyDir)
      logger.debug("Changing to %s", pyDirPath)
      with changeToWorkingDir(pyDirPath):
        setupCommand = "python setup.py develop --prefix=%s/htm-it" % \
                       productsDirectory
        logger.debug("Running %s", setupCommand)
        runWithOutput(setupCommand, env=environment)

    # Run any build scripts. We assume that they should be run in the
    # directory they're in.
    logger.info("Phase 3: Running build scripts...")
    for builder in config.buildScripts:
      builderPath = "%s/%s" % (fakeroot, builder)
      logger.debug("Attempting to run %s", builderPath)
      if not os.path.exists(builderPath):
        raise InvalidParametersError("%s does not exist!" % builderPath)
      workDirectory = os.path.dirname(builderPath)
      logger.debug("Changing to %s", workDirectory)
      with changeToWorkingDir(workDirectory):
        runWithOutput(builderPath, env=environment)

    # Run any cleanup scripts. We assume that they should be run in the
    # directory they're in.
    logger.info("Phase 4: Running cleanup scripts...")
    # Clean Scripts
    self.cleanScripts()

    logger.info("Phase 5: Purge anything not whitelisted.")
    # Purge anything not whitelisted
    self.purgeBlacklistedStuff()

    return (iteration, actualSHA)