def constructPreBuiltGrokFakeroot(self): """ Construct fakeroot from prebuilt grok :returns: SHA of the products repo in the fakeroot :rtype: tuple """ config = self.config logger = self.logger productsDirectory = self.productsDirectory logger.debug("Creating %s", productsDirectory) mkpath(productsDirectory) copy_tree(config.productsDir, productsDirectory) iteration = git.getCommitCount(productsDirectory) with changeToWorkingDir(productsDirectory): actualSHA = git.getCurrentSha() # Set extra python path self.setPythonPath() # Clean Grok Scripts self.cleanScripts() # Purge anything not whitelisted self.purgeBlacklistedStuff() return (iteration, actualSHA)
def constructPreBuiltHTMITFakeroot(self): """ Construct fakeroot from prebuilt htm-it :returns: SHA of the products repo in the fakeroot :rtype: tuple """ config = self.config logger = self.logger productsDirectory = self.productsDirectory logger.debug("Creating %s", productsDirectory) mkpath(productsDirectory) copy_tree(config.productsDir, productsDirectory) iteration = git.getCommitCount(productsDirectory, logger=logger) with changeToWorkingDir(productsDirectory): actualSHA = git.getCurrentSha(logger=logger) # Set extra python path self.setPythonPath() # Clean HTM-IT Scripts self.cleanScripts() # Purge anything not whitelisted self.purgeBlacklistedStuff() return (iteration, actualSHA)
def constructInfrastructureFakeroot(self): """ Construct our fakeroot directory tree :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger productsDirectory = self.productsDirectory srvPath = os.path.join(fakeroot, "opt", "numenta") logger.debug("Creating %s", srvPath) mkpath(srvPath) logger.debug("Cloning %s into %s...", fakeroot, config.gitURL) # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA installDirectory = os.path.join("opt", "numenta") fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot, installDirectory=installDirectory, repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsDirectory, logger=logger) logger.debug("Commit count in %s is %s", productsDirectory, iteration) logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA) # Clean everything not whitelisted out of products so we don't conflict # with htm-it or taurus rpms purgeDirectory(path=productsDirectory, whitelist=["__init__.py", "infrastructure" ], logger=logger) # Clean out infrastructure, too - we only want the utilities infraPath = os.path.join(productsDirectory, "infrastructure") purgeDirectory(path=infraPath, whitelist=["__init__.py", "DEPENDENCIES.md", "infrastructure", "LICENSE", "README.md", "requirements.txt", "setup.py"], logger=logger) return (iteration, fakerootSHA)
def constructInfrastructureFakeroot(self): """ Construct our fakeroot directory tree :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger productsDirectory = self.productsDirectory srvPath = os.path.join(fakeroot, "opt", "numenta") logger.debug("Creating %s", srvPath) mkpath(srvPath) logger.debug("Cloning %s into %s...", fakeroot, config.gitURL) # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA installDirectory = os.path.join("opt", "numenta") fakerootSHA = rpm.gitCloneIntoFakeroot( fakeroot=fakeroot, installDirectory=installDirectory, repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsDirectory) logger.debug("Commit count in %s is %s", productsDirectory, iteration) logger.debug("SHA in %s is %s", productsDirectory, fakerootSHA) # Clean everything not whitelisted out of products so we don't conflict # with grok or taurus rpms purgeDirectory(path=productsDirectory, whitelist=["__init__.py", "infrastructure"], logger=logger) # Clean out infrastructure, too - we only want the utilities infraPath = os.path.join(productsDirectory, "infrastructure") purgeDirectory(path=infraPath, whitelist=[ "__init__.py", "DEPENDENCIES.md", "infrastructure", "LICENSE", "README.md", "requirements.txt", "setup.py" ], logger=logger) return (iteration, fakerootSHA)
def constructGrokFakeroot(self): """ Construct a Grok fakeroot directory tree. 1. Add any directories specified with --extend-pythonpath to the PYTHONPATH we will be using for setup.py, build scripts and the cleanup scripts. 2. Install any wheels that have been specied by --use-wheel 3. Run setup.py in any directories that have been specified with --setup-py-dir. Uses the arguments specfied by --setup-py-arguments. 4. Run any build scripts specified by --build-script 5. Run any cleanup scripts specified by --cleanup-script 6. Purge any files or directories at the top level of the checkout that were not whitelisted with --whitelist. :returns: (iteration, actualSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger logger.info("Preparing Grok fakeroot in %s\n", fakeroot) actualSHA = self.installProductsIntoGrokFakeroot() productsDirectory = self.productsDirectory grokPath = os.path.join(productsDirectory, "grok") iteration = git.getCommitCount(productsDirectory) # Extend PYTHONPATH for setup.py, build & cleanup scripts # pythonpathExtensions logger.debug("**************************************************") logger.info("Phase 1: Preparing PYTHONPATH and installing wheels") # Set extra python path self.setPythonPath() environment = self.environment sitePackagesDirectory = "%s/grok/lib/python2.7/site-packages" % \ productsDirectory # Install wheels if any have been specified with changeToWorkingDir(grokPath): for wheel in config.wheels: logger.info("Installing %s", os.path.basename(wheel)) if not os.path.exists(wheel): raise InvalidParametersError("%s does not exist!" % wheel) pipCommand = "pip install %s --no-deps --target=%s" % \ (wheel, sitePackagesDirectory) logger.debug("pip command: %s", pipCommand) runWithOutput(pipCommand) logger.debug("wheel install complete") # Run setup.py if specified logger.info("Phase 2: Running setup.py commands") for pyDir in config.setupPyDirs: pyDirPath = "%s/%s" % (productsDirectory, pyDir) logger.debug("Changing to %s", pyDirPath) with changeToWorkingDir(pyDirPath): setupCommand = "python setup.py develop --prefix=%s/grok" % \ productsDirectory logger.debug("Running %s", setupCommand) runWithOutput(setupCommand, env=environment) # Run any build scripts. We assume that they should be run in the # directory they're in. logger.info("Phase 3: Running build scripts...") for builder in config.buildScripts: builderPath = "%s/%s" % (fakeroot, builder) logger.debug("Attempting to run %s", builderPath) if not os.path.exists(builderPath): raise InvalidParametersError("%s does not exist!" % builderPath) workDirectory = os.path.dirname(builderPath) logger.debug("Changing to %s", workDirectory) with changeToWorkingDir(workDirectory): runWithOutput(builderPath, env=environment) # Run any cleanup scripts. We assume that they should be run in the # directory they're in. logger.info("Phase 4: Running cleanup scripts...") # Clean Scripts self.cleanScripts() logger.info("Phase 5: Purge anything not whitelisted.") # Purge anything not whitelisted self.purgeBlacklistedStuff() return (iteration, actualSHA)
def constructSaltcellarFakeroot(self): """ Make a saltcellar fakeroot :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger srvPath = os.path.join(fakeroot, "srv") logger.debug("Creating saltcellar fakeroot in %s", srvPath) productsPath = os.path.join(fakeroot, "products") mkpath(srvPath) logger.debug("Cloning...") # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot, installDirectory="/", repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsPath) logger.debug("Commit count in %s is %s", productsPath, iteration) # Move the saltcellar to /srv/salt logger.debug("Moving saltcellar to %s/salt", srvPath) logger.debug("srvPath: %s", srvPath) logger.debug("productsPath: %s", productsPath) logger.debug("%s/infrastructure/saltcellar", productsPath) logger.debug("Checking for %s/infrastructure/saltcellar", productsPath) logger.debug( os.path.exists("%s/infrastructure/saltcellar" % productsPath)) os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"), os.path.join(srvPath, "salt")) # Now that we have the salt formulas, nuke the rest of products out of # the fakeroot logger.debug("Deleting products from fakeroot") rmrf(productsPath) # Finally, scrub the private data out of /srv/salt if not config.numenta_internal_only: logger.debug("Sanitizing /srv/salt") self.sanitizeSrvSalt("%s/srv/salt" % fakeroot) else: logger.critical( "Baking numenta-internal rpm, not sanitizing /srv/salt") return (iteration, fakerootSHA)
def constructSaltcellarFakeroot(self): """ Make a saltcellar fakeroot :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger srvPath = os.path.join(fakeroot, "srv") logger.debug("Creating saltcellar fakeroot in %s", srvPath) productsPath = os.path.join(fakeroot, "products") mkpath(srvPath) logger.debug("Cloning...") # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot, installDirectory="/", repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsPath, logger=logger) logger.debug("Commit count in %s is %s", productsPath, iteration) # Move the saltcellar to /srv/salt logger.debug("Moving saltcellar to %s/salt", srvPath) logger.debug("srvPath: %s", srvPath) logger.debug("productsPath: %s", productsPath) logger.debug("%s/infrastructure/saltcellar", productsPath) logger.debug("Checking for %s/infrastructure/saltcellar", productsPath) logger.debug(os.path.exists("%s/infrastructure/saltcellar" % productsPath)) os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"), os.path.join(srvPath, "salt")) # Now that we have the salt formulas, nuke the rest of products out of # the fakeroot logger.debug("Deleting products from fakeroot") rmrf(productsPath) # Finally, scrub the private data out of /srv/salt if not config.numenta_internal_only: logger.debug("Sanitizing /srv/salt") self.sanitizeSrvSalt("%s/srv/salt" % fakeroot) else: logger.critical("Baking numenta-internal rpm, not sanitizing /srv/salt") return (iteration, fakerootSHA)
the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger logger.info("Preparing HTM-IT fakeroot in %s\n", fakeroot) actualSHA = self.installProductsIntoHTM-ITFakeroot() productsDirectory = self.productsDirectory htm-itPath = os.path.join(productsDirectory, "htm-it") iteration = git.getCommitCount(productsDirectory, logger=logger) # Extend PYTHONPATH for setup.py, build & cleanup scripts # pythonpathExtensions logger.debug("**************************************************") logger.info("Phase 1: Preparing PYTHONPATH and installing wheels") # Set extra python path self.setPythonPath() environment = self.environment sitePackagesDirectory = "%s/htm-it/lib/python2.7/site-packages" % \ productsDirectory # Install wheels if any have been specified with changeToWorkingDir(htm-itPath): for wheel in config.wheels: logger.info("Installing %s", os.path.basename(wheel))
def constructHTMITFakeroot(self): """ Construct a HTM-IT fakeroot directory tree. 1. Add any directories specified with --extend-pythonpath to the PYTHONPATH we will be using for setup.py, build scripts and the cleanup scripts. 2. Install any wheels that have been specied by --use-wheel 3. Run setup.py in any directories that have been specified with --setup-py-dir. Uses the arguments specfied by --setup-py-arguments. 4. Run any build scripts specified by --build-script 5. Run any cleanup scripts specified by --cleanup-script 6. Purge any files or directories at the top level of the checkout that were not whitelisted with --whitelist. :returns: (iteration, actualSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger logger.info("Preparing HTM-IT fakeroot in %s\n", fakeroot) actualSHA = self.installProductsIntoHTMITFakeroot() productsDirectory = self.productsDirectory htm-itPath = os.path.join(productsDirectory, "htm-it") iteration = git.getCommitCount(productsDirectory, logger=logger) # Extend PYTHONPATH for setup.py, build & cleanup scripts # pythonpathExtensions logger.debug("**************************************************") logger.info("Phase 1: Preparing PYTHONPATH and installing wheels") # Set extra python path self.setPythonPath() environment = self.environment sitePackagesDirectory = "%s/htm-it/lib/python2.7/site-packages" % \ productsDirectory # Install wheels if any have been specified with changeToWorkingDir(htm-itPath): for wheel in config.wheels: logger.info("Installing %s", os.path.basename(wheel)) if not os.path.exists(wheel): raise InvalidParametersError("%s does not exist!" % wheel) pipCommand = "pip install %s --no-deps --target=%s" % \ (wheel, sitePackagesDirectory) logger.debug("pip command: %s", pipCommand) runWithOutput(pipCommand) logger.debug("wheel install complete") # Run setup.py if specified logger.info("Phase 2: Running setup.py commands") for pyDir in config.setupPyDirs: pyDirPath = "%s/%s" % (productsDirectory, pyDir) logger.debug("Changing to %s", pyDirPath) with changeToWorkingDir(pyDirPath): setupCommand = "python setup.py develop --prefix=%s/htm-it" % \ productsDirectory logger.debug("Running %s", setupCommand) runWithOutput(setupCommand, env=environment) # Run any build scripts. We assume that they should be run in the # directory they're in. logger.info("Phase 3: Running build scripts...") for builder in config.buildScripts: builderPath = "%s/%s" % (fakeroot, builder) logger.debug("Attempting to run %s", builderPath) if not os.path.exists(builderPath): raise InvalidParametersError("%s does not exist!" % builderPath) workDirectory = os.path.dirname(builderPath) logger.debug("Changing to %s", workDirectory) with changeToWorkingDir(workDirectory): runWithOutput(builderPath, env=environment) # Run any cleanup scripts. We assume that they should be run in the # directory they're in. logger.info("Phase 4: Running cleanup scripts...") # Clean Scripts self.cleanScripts() logger.info("Phase 5: Purge anything not whitelisted.") # Purge anything not whitelisted self.purgeBlacklistedStuff() return (iteration, actualSHA)