def purgeBlacklistedStuff(self):
"""
Purges anything not whitelisted.
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.info("Purge anything not whitelisted.")
for thing in os.listdir(productsDirectory):
if thing not in config.whitelisted:
rmrf("%s/%s" % (productsDirectory, thing))
def sanitizeSrvSalt(self, saltpath):
"""
Ensure only whitelisted files & directories are installed to /srv/salt by
the RPM.
Numenta convention is to only include explicitly whitelisted formulas
and files in RPMs deployed to customer machines.
We add a PUBLIC file at the top level of a formula's directory tree
to add it to the whitelist.
This prevents us from accidentally publishing internal-only files to
customer machines.
:param saltpath: Path to /srv/salt in the fakeroot
"""
logger = self.logger
fileWhitelist = ["bootstrap.sh",
"top.sls"
]
logger.debug("Sanitizing %s", saltpath)
for artifact in os.listdir(saltpath):
artifactPath = "%s/%s" % (saltpath, artifact)
if os.path.isfile(artifactPath):
if artifact not in fileWhitelist:
logger.debug("Purging %s", artifact)
rmrf(artifactPath)
if os.path.isdir(artifactPath):
# Formula directories have to be explicitly whitelisted by having
# a PUBLIC file or they will be purged from the salt tree.
if not os.path.isfile("%s/PUBLIC" % artifactPath):
logger.debug("Purging %s", artifact)
rmrf(artifactPath)
else:
logger.info("packaging formula %s", artifact)
# AWS requires that we don't include keys in marketplace AMIs.
# Purge any pubkeys in the salt tree
# Note that we _don't_ quote the wildcard here so that check_call
# passes it to find correctly when it is called by runWithOutput.
# Same for the {} and ;
findPubkeys = """find %s -name *.pub -exec rm -fv {} ;""" % saltpath
logger.debug("**************************************************")
logger.debug("Sanitizing %s with %s", saltpath, findPubkeys)
runWithOutput(findPubkeys, logger=logger)
# Purge pemfiles
findPemFiles = """find %s -name *.pem -exec rm -fv {} ;""" % saltpath
logger.debug("**************************************************")
logger.debug("Sanitizing %s with %s", saltpath, findPubkeys)
runWithOutput(findPemFiles, logger=logger)
def purgeBlacklistedStuff(self):
"""
Purges anything not whitelisted.
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.info("Purge anything not whitelisted.")
for thing in os.listdir(productsDirectory):
if thing not in config.whitelisted:
rmrf("%s/%s" % (productsDirectory, thing))
def sanitizeSrvSalt(self, saltpath):
"""
Ensure only whitelisted files & directories are installed to /srv/salt by
the RPM.
Numenta convention is to only include explicitly whitelisted formulas
and files in RPMs deployed to customer machines.
We add a PUBLIC file at the top level of a formula's directory tree
to add it to the whitelist.
This prevents us from accidentally publishing internal-only files to
customer machines.
:param saltpath: Path to /srv/salt in the fakeroot
"""
logger = self.logger
fileWhitelist = ["bootstrap.sh", "top.sls"]
logger.debug("Sanitizing %s", saltpath)
for artifact in os.listdir(saltpath):
artifactPath = "%s/%s" % (saltpath, artifact)
if os.path.isfile(artifactPath):
if artifact not in fileWhitelist:
logger.debug("Purging %s", artifact)
rmrf(artifactPath)
if os.path.isdir(artifactPath):
# Formula directories have to be explicitly whitelisted by having
# a PUBLIC file or they will be purged from the salt tree.
if not os.path.isfile("%s/PUBLIC" % artifactPath):
logger.debug("Purging %s", artifact)
rmrf(artifactPath)
else:
logger.info("packaging formula %s", artifact)
# AWS requires that we don't include keys in marketplace AMIs.
# Purge any pubkeys in the salt tree
# Note that we _don't_ quote the wildcard here so that check_call
# passes it to find correctly when it is called by runWithOutput.
# Same for the {} and ;
findPubkeys = """find %s -name *.pub -exec rm -fv {} ;""" % saltpath
logger.debug("**************************************************")
logger.debug("Sanitizing %s with %s", saltpath, findPubkeys)
runWithOutput(findPubkeys, logger=logger)
# Purge pemfiles
findPemFiles = """find %s -name *.pem -exec rm -fv {} ;""" % saltpath
logger.debug("**************************************************")
logger.debug("Sanitizing %s with %s", saltpath, findPubkeys)
runWithOutput(findPemFiles, logger=logger)
def cleanupDirectories(self):
"""
Nuke any temp files unless preserveFakeroot is set in the configuration.
"""
config = self.config
logger = self.logger
fakeroot = self.fakeroot
if not config.preserveFakeroot:
if logger:
logger.debug("Scrubbing fakeroot in %s", fakeroot)
rmrf(fakeroot, logger=logger)
else:
if logger:
logger.debug("Skipping fakeroot scrub, leaving %s intact.", fakeroot)
def cleanupDirectories(self):
"""
Nuke any temp files unless preserveFakeroot is set in the configuration.
"""
config = self.config
logger = self.logger
fakeroot = self.fakeroot
if not config.preserveFakeroot:
if logger:
logger.debug("Scrubbing fakeroot in %s", fakeroot)
rmrf(fakeroot, logger=logger)
else:
if logger:
logger.debug("Skipping fakeroot scrub, leaving %s intact.",
fakeroot)
def constructSaltcellarFakeroot(self):
"""
Make a saltcellar fakeroot
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
srvPath = os.path.join(fakeroot, "srv")
logger.debug("Creating saltcellar fakeroot in %s", srvPath)
productsPath = os.path.join(fakeroot, "products")
mkpath(srvPath)
logger.debug("Cloning...")
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot,
installDirectory="/",
repoDirectory="products",
gitURL=config.gitURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = git.getCommitCount(productsPath)
logger.debug("Commit count in %s is %s", productsPath, iteration)
# Move the saltcellar to /srv/salt
logger.debug("Moving saltcellar to %s/salt", srvPath)
logger.debug("srvPath: %s", srvPath)
logger.debug("productsPath: %s", productsPath)
logger.debug("%s/infrastructure/saltcellar", productsPath)
logger.debug("Checking for %s/infrastructure/saltcellar", productsPath)
logger.debug(
os.path.exists("%s/infrastructure/saltcellar" % productsPath))
os.rename(os.path.join(productsPath, "infrastructure", "saltcellar"),
os.path.join(srvPath, "salt"))
# Now that we have the salt formulas, nuke the rest of products out of
# the fakeroot
logger.debug("Deleting products from fakeroot")
rmrf(productsPath)
# Finally, scrub the private data out of /srv/salt
if not config.numenta_internal_only:
logger.debug("Sanitizing /srv/salt")
self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
else:
logger.critical(
"Baking numenta-internal rpm, not sanitizing /srv/salt")
return (iteration, fakerootSHA)
def constructSaltcellarFakeroot(self):
"""
Make a saltcellar fakeroot
:returns: (iteration, fakerootSHA) where iteration is the total commit count
in the repository and fakerootSHA is the SHA in the fakeroot. If we're
packaging a branch or tip of master, we're still going to want to know what
the SHA was so we can include it in the RPM description.
:rtype: tuple
"""
config = self.config
fakeroot = self.fakeroot
logger = self.logger
srvPath = os.path.join(fakeroot, "srv")
logger.debug("Creating saltcellar fakeroot in %s", srvPath)
productsPath = os.path.join(fakeroot, "products")
mkpath(srvPath)
logger.debug("Cloning...")
# Collect the SHA from the fakeroot. This way we can put the SHA into
# the RPM information even if we are packaging tip of a branch and not
# a specific SHA
fakerootSHA = rpm.gitCloneIntoFakeroot(fakeroot=fakeroot,
installDirectory="/",
repoDirectory="products",
gitURL=config.gitURL,
logger=logger,
sha=config.sha)
# Capture the commit count since we're going to trash products once we pull
# out the saltcellar
iteration = git.getCommitCount(productsPath, logger=logger)
logger.debug("Commit count in %s is %s", productsPath, iteration)
# Move the saltcellar to /srv/salt
logger.debug("Moving saltcellar to %s/salt", srvPath)
logger.debug("srvPath: %s", srvPath)
logger.debug("productsPath: %s", productsPath)
logger.debug("%s/infrastructure/saltcellar", productsPath)
logger.debug("Checking for %s/infrastructure/saltcellar",
productsPath)
logger.debug(os.path.exists("%s/infrastructure/saltcellar" %
productsPath))
os.rename(os.path.join(productsPath, "infrastructure",
"saltcellar"),
os.path.join(srvPath, "salt"))
# Now that we have the salt formulas, nuke the rest of products out of
# the fakeroot
logger.debug("Deleting products from fakeroot")
rmrf(productsPath)
# Finally, scrub the private data out of /srv/salt
if not config.numenta_internal_only:
logger.debug("Sanitizing /srv/salt")
self.sanitizeSrvSalt("%s/srv/salt" % fakeroot)
else:
logger.critical("Baking numenta-internal rpm, not sanitizing /srv/salt")
return (iteration, fakerootSHA)
