def testBzip2(self):
fn = "%s/foo" % self.tmpdir
open(fn, "w").write("hello")
bzip2(fn)
proc = subprocess.Popen(["bzcat", fn], stdout=subprocess.PIPE)
self.assertEquals("hello", proc.stdout.read())
bunzip2(fn)
self.assertEquals("hello", open(fn, 'rb').read())
def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
"""Sign `pkgfile`, putting the results into `dstdir`.
If `remember` is True, then cache the newly signed files into our
cache.
If `compressed` is True, then the contents of pkgfile are bz2
compressed (e.g. in a mar file), and should be decompressed before
signing.
"""
log.info("Processing %s", pkgfile)
basename = os.path.basename(pkgfile)
dstfile = convertPath(pkgfile, dstdir)
# Keep track of our output in a list here, and we can output everything
# when we're done This is to avoid interleaving the output from
# multiple processes.
logs = []
logs.append("Repacking %s to %s" % (pkgfile, dstfile))
parentdir = os.path.dirname(dstfile)
if not os.path.exists(parentdir):
os.makedirs(parentdir, 0755)
nFiles = 0
cacheHits = 0
nSigned = 0
tmpdir = tempfile.mkdtemp()
try:
# Unpack it
logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
unpackfile(pkgfile, tmpdir)
# Swap in files we have already signed
for f in findfiles(tmpdir):
# We don't need to do anything to files we're not going to sign
if not shouldSign(f):
continue
h = sha1sum(f)
basename = os.path.basename(f)
nFiles += 1
chk = getChkFile(f)
# Look in the cache for another file with the same original
# hash
cachedFile = self.getFile(h, f)
if cachedFile:
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from %s" % (basename, cachedFile))
# Preserve the original file's mode; don't use the cached mode
# We usually process installer .exe's first, and 7z doesn't
# preserve the file mode, so the cached copies of the files
# are mode 0666. In the mar files, executables have mode
# 0777, so we want to preserve that.
copyfile(cachedFile, f, copymode=False)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in cache
cachedChk = self.getFile(h, chk)
logs.append("Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
else:
# We need to sign this file
# If this file is compressed, check if we have a cached copy that
# is uncompressed
if compressed:
bunzip2(f)
h2 = sha1sum(f)
cachedFile = self.getFile(h2, f)
if cachedFile:
# We have a cached copy of this file that is uncompressed.
# So copy it into our dstdir, and recompress it, and
# save it for future use.
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from uncompressed %s" %
(basename, cachedFile))
# See note above about not copying the file's mode
copyfile(cachedFile, f, copymode=False)
bzip2(f)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in
# cache
cachedChk = self.getFile(h2, chk)
logs.append(
"Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
bzip2(chk)
if remember:
logs.append("Caching compressed %s as %s" %
(f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
continue
nSigned += 1
logs.append("Signing %s" % f)
signfile(f, self.keydir, self.fake)
if compressed:
bzip2(f)
# If we have a chk file, compress that too
if chk:
bzip2(chk)
if remember:
logs.append("Caching %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
# Repack it
logs.append("Packing %s" % dstfile)
packfile(dstfile, tmpdir)
# Sign installer
if dstfile.endswith('.exe') and not self.unsignedInstallers:
logs.append("Signing %s" % dstfile)
signfile(dstfile, self.keydir, self.fake)
return nFiles, cacheHits, nSigned
except:
log.exception("Error signing %s", pkgfile)
return False
finally:
# Clean up after ourselves, and output our logs
shutil.rmtree(tmpdir)
log.info("\n ".join(logs))
def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
"""Sign `pkgfile`, putting the results into `dstdir`.
If `remember` is True, then cache the newly signed files into our
cache.
If `compressed` is True, then the contents of pkgfile are bz2
compressed (e.g. in a mar file), and should be decompressed before
signing.
"""
log.info("Processing %s", pkgfile)
basename = os.path.basename(pkgfile)
dstfile = convertPath(pkgfile, dstdir)
# Keep track of our output in a list here, and we can output everything
# when we're done This is to avoid interleaving the output from
# multiple processes.
logs = []
logs.append("Repacking %s to %s" % (pkgfile, dstfile))
parentdir = os.path.dirname(dstfile)
if not os.path.exists(parentdir):
os.makedirs(parentdir, 0755)
nFiles = 0
cacheHits = 0
nSigned = 0
tmpdir = tempfile.mkdtemp()
try:
# Unpack it
logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
unpackfile(pkgfile, tmpdir)
# Swap in files we have already signed
for f in findfiles(tmpdir):
# We don't need to do anything to files we're not going to sign
if not shouldSign(f):
continue
h = sha1sum(f)
basename = os.path.basename(f)
nFiles += 1
chk = getChkFile(f)
# Look in the cache for another file with the same original
# hash
cachedFile = self.getFile(h, f)
if cachedFile:
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from %s" % (basename, cachedFile))
# Preserve the original file's mode; don't use the cached mode
# We usually process installer .exe's first, and 7z doesn't
# preserve the file mode, so the cached copies of the files
# are mode 0666. In the mar files, executables have mode
# 0777, so we want to preserve that.
copyfile(cachedFile, f, copymode=False)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in cache
cachedChk = self.getFile(h, chk)
logs.append("Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
else:
# We need to sign this file
# If this file is compressed, check if we have a cached copy that
# is uncompressed
if compressed:
bunzip2(f)
h2 = sha1sum(f)
cachedFile = self.getFile(h2, f)
if cachedFile:
# We have a cached copy of this file that is uncompressed.
# So copy it into our dstdir, and recompress it, and
# save it for future use.
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from uncompressed %s" %
(basename, cachedFile))
# See note above about not copying the file's mode
copyfile(cachedFile, f, copymode=False)
bzip2(f)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in
# cache
cachedChk = self.getFile(h2, chk)
logs.append("Copying %s from %s" % (
os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
bzip2(chk)
if remember:
logs.append(
"Caching compressed %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
continue
nSigned += 1
logs.append("Signing %s" % f)
signfile(f, self.keydir, self.fake)
if compressed:
bzip2(f)
# If we have a chk file, compress that too
if chk:
bzip2(chk)
if remember:
logs.append("Caching %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
# Repack it
logs.append("Packing %s" % dstfile)
packfile(dstfile, tmpdir)
# Sign installer
if dstfile.endswith('.exe') and not self.unsignedInstallers:
logs.append("Signing %s" % dstfile)
signfile(dstfile, self.keydir, self.fake)
return nFiles, cacheHits, nSigned
except:
log.exception("Error signing %s", pkgfile)
return False
finally:
# Clean up after ourselves, and output our logs
shutil.rmtree(tmpdir)
log.info("\n ".join(logs))
