def purgeBlacklistedStuff(self): """ Purges anything not whitelisted. """ config = self.config logger = self.logger productsDirectory = self.productsDirectory logger.info("Purge anything not whitelisted.") for thing in os.listdir(productsDirectory): if thing not in config.whitelisted: rmrf("%s/%s" % (productsDirectory, thing))
def sanitizeSrvSalt(self, saltpath): """ Ensure only whitelisted files & directories are installed to /srv/salt by the RPM. Numenta convention is to only include explicitly whitelisted formulas and files in RPMs deployed to customer machines. We add a PUBLIC file at the top level of a formula's directory tree to add it to the whitelist. This prevents us from accidentally publishing internal-only files to customer machines. :param saltpath: Path to /srv/salt in the fakeroot """ logger = self.logger fileWhitelist = ["bootstrap.sh", "top.sls" ] logger.debug("Sanitizing %s", saltpath) for artifact in os.listdir(saltpath): artifactPath = "%s/%s" % (saltpath, artifact) if os.path.isfile(artifactPath): if artifact not in fileWhitelist: logger.debug("Purging %s", artifact) rmrf(artifactPath) if os.path.isdir(artifactPath): # Formula directories have to be explicitly whitelisted by having # a PUBLIC file or they will be purged from the salt tree. if not os.path.isfile("%s/PUBLIC" % artifactPath): logger.debug("Purging %s", artifact) rmrf(artifactPath) else: logger.info("packaging formula %s", artifact) # AWS requires that we don't include keys in marketplace AMIs. # Purge any pubkeys in the salt tree # Note that we _don't_ quote the wildcard here so that check_call # passes it to find correctly when it is called by runWithOutput. # Same for the {} and ; findPubkeys = """find %s -name *.pub -exec rm -fv {} ;""" % saltpath logger.debug("**************************************************") logger.debug("Sanitizing %s with %s", saltpath, findPubkeys) runWithOutput(findPubkeys, logger=logger) # Purge pemfiles findPemFiles = """find %s -name *.pem -exec rm -fv {} ;""" % saltpath logger.debug("**************************************************") logger.debug("Sanitizing %s with %s", saltpath, findPubkeys) runWithOutput(findPemFiles, logger=logger)
def sanitizeSrvSalt(self, saltpath): """ Ensure only whitelisted files & directories are installed to /srv/salt by the RPM. Numenta convention is to only include explicitly whitelisted formulas and files in RPMs deployed to customer machines. We add a PUBLIC file at the top level of a formula's directory tree to add it to the whitelist. This prevents us from accidentally publishing internal-only files to customer machines. :param saltpath: Path to /srv/salt in the fakeroot """ logger = self.logger fileWhitelist = ["bootstrap.sh", "top.sls"] logger.debug("Sanitizing %s", saltpath) for artifact in os.listdir(saltpath): artifactPath = "%s/%s" % (saltpath, artifact) if os.path.isfile(artifactPath): if artifact not in fileWhitelist: logger.debug("Purging %s", artifact) rmrf(artifactPath) if os.path.isdir(artifactPath): # Formula directories have to be explicitly whitelisted by having # a PUBLIC file or they will be purged from the salt tree. if not os.path.isfile("%s/PUBLIC" % artifactPath): logger.debug("Purging %s", artifact) rmrf(artifactPath) else: logger.info("packaging formula %s", artifact) # AWS requires that we don't include keys in marketplace AMIs. # Purge any pubkeys in the salt tree # Note that we _don't_ quote the wildcard here so that check_call # passes it to find correctly when it is called by runWithOutput. # Same for the {} and ; findPubkeys = """find %s -name *.pub -exec rm -fv {} ;""" % saltpath logger.debug("**************************************************") logger.debug("Sanitizing %s with %s", saltpath, findPubkeys) runWithOutput(findPubkeys, logger=logger) # Purge pemfiles findPemFiles = """find %s -name *.pem -exec rm -fv {} ;""" % saltpath logger.debug("**************************************************") logger.debug("Sanitizing %s with %s", saltpath, findPubkeys) runWithOutput(findPemFiles, logger=logger)
def cleanupDirectories(self): """ Nuke any temp files unless preserveFakeroot is set in the configuration. """ config = self.config logger = self.logger fakeroot = self.fakeroot if not config.preserveFakeroot: if logger: logger.debug("Scrubbing fakeroot in %s", fakeroot) rmrf(fakeroot, logger=logger) else: if logger: logger.debug("Skipping fakeroot scrub, leaving %s intact.", fakeroot)
def constructSaltcellarFakeroot(self): """ Make a saltcellar fakeroot :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger srvPath = os.path.join(fakeroot, "srv") logger.debug("Creating saltcellar fakeroot in %s", srvPath) productsPath = os.path.join(fakeroot, "products") mkpath(srvPath) logger.debug("Cloning...") # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot, installDirectory="/", repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsPath) logger.debug("Commit count in %s is %s", productsPath, iteration) # Move the saltcellar to /srv/salt logger.debug("Moving saltcellar to %s/salt", srvPath) logger.debug("srvPath: %s", srvPath) logger.debug("productsPath: %s", productsPath) logger.debug("%s/infrastructure/saltcellar", productsPath) logger.debug("Checking for %s/infrastructure/saltcellar", productsPath) logger.debug( os.path.exists("%s/infrastructure/saltcellar" % productsPath)) os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"), os.path.join(srvPath, "salt")) # Now that we have the salt formulas, nuke the rest of products out of # the fakeroot logger.debug("Deleting products from fakeroot") rmrf(productsPath) # Finally, scrub the private data out of /srv/salt if not config.numenta_internal_only: logger.debug("Sanitizing /srv/salt") self.sanitizeSrvSalt("%s/srv/salt" % fakeroot) else: logger.critical( "Baking numenta-internal rpm, not sanitizing /srv/salt") return (iteration, fakerootSHA)
def constructSaltcellarFakeroot(self): """ Make a saltcellar fakeroot :returns: (iteration, fakerootSHA) where iteration is the total commit count in the repository and fakerootSHA is the SHA in the fakeroot. If we're packaging a branch or tip of master, we're still going to want to know what the SHA was so we can include it in the RPM description. :rtype: tuple """ config = self.config fakeroot = self.fakeroot logger = self.logger srvPath = os.path.join(fakeroot, "srv") logger.debug("Creating saltcellar fakeroot in %s", srvPath) productsPath = os.path.join(fakeroot, "products") mkpath(srvPath) logger.debug("Cloning...") # Collect the SHA from the fakeroot. This way we can put the SHA into # the RPM information even if we are packaging tip of a branch and not # a specific SHA fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot, installDirectory="/", repoDirectory="products", gitURL=config.gitURL, logger=logger, sha=config.sha) # Capture the commit count since we're going to trash products once we pull # out the saltcellar iteration = git.getCommitCount(productsPath, logger=logger) logger.debug("Commit count in %s is %s", productsPath, iteration) # Move the saltcellar to /srv/salt logger.debug("Moving saltcellar to %s/salt", srvPath) logger.debug("srvPath: %s", srvPath) logger.debug("productsPath: %s", productsPath) logger.debug("%s/infrastructure/saltcellar", productsPath) logger.debug("Checking for %s/infrastructure/saltcellar", productsPath) logger.debug(os.path.exists("%s/infrastructure/saltcellar" % productsPath)) os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"), os.path.join(srvPath, "salt")) # Now that we have the salt formulas, nuke the rest of products out of # the fakeroot logger.debug("Deleting products from fakeroot") rmrf(productsPath) # Finally, scrub the private data out of /srv/salt if not config.numenta_internal_only: logger.debug("Sanitizing /srv/salt") self.sanitizeSrvSalt("%s/srv/salt" % fakeroot) else: logger.critical("Baking numenta-internal rpm, not sanitizing /srv/salt") return (iteration, fakerootSHA)