def testCopyFile(self):
tmp = os.path.join(self.tmpdir, "t")
copyfile(__file__, tmp)
self.assertEquals(sha1sum(__file__), sha1sum(tmp))
self.assertEquals(os.stat(__file__).st_mode, os.stat(tmp).st_mode)
self.assertEquals(
int(os.stat(__file__).st_mtime), int(os.stat(tmp).st_mtime))
def uploadfile(baseurl, filename, format_, token, nonce):
"""Uploads file (given by `filename`) to server at `baseurl`.
`sesson_key` and `nonce` are string values that get passed as POST
parameters.
"""
from poster.encode import multipart_encode
filehash = sha1sum(filename)
try:
fp = open(filename, 'rb')
params = {
'filedata': fp,
'sha1': filehash,
'filename': os.path.basename(filename),
'token': token,
'nonce': nonce,
}
datagen, headers = multipart_encode(params)
r = urllib2.Request(
"%s/sign/%s" % (baseurl, format_), datagen, headers)
return urllib2.urlopen(r)
finally:
fp.close()
def uploadfile(baseurl, filename, format_, token, nonce):
"""Uploads file (given by `filename`) to server at `baseurl`.
`sesson_key` and `nonce` are string values that get passed as POST
parameters.
"""
from poster.encode import multipart_encode
filehash = sha1sum(filename)
try:
fp = open(filename, 'rb')
params = {
'filedata': fp,
'sha1': filehash,
'filename': os.path.basename(filename),
'token': token,
'nonce': nonce,
}
datagen, headers = multipart_encode(params)
r = urllib2.Request(
"%s/sign/%s" % (baseurl, format_), datagen, headers)
return urllib2.urlopen(r)
finally:
fp.close()
def remote_signfile(options, urls, filename, fmt, token, dest=None):
filehash = sha1sum(filename)
if dest is None:
dest = filename
if fmt == 'gpg':
dest += '.asc'
parent_dir = os.path.dirname(os.path.abspath(dest))
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
# Check the cache
cached_fn = None
if options.cachedir:
log.debug("%s: checking cache", filehash)
cached_fn = os.path.join(options.cachedir, fmt, filehash)
if os.path.exists(cached_fn):
log.info("%s: exists in the cache; copying to %s", filehash, dest)
cached_fp = open(cached_fn, 'rb')
tmpfile = dest + '.tmp'
fp = open(tmpfile, 'wb')
hsh = hashlib.new('sha1')
while True:
data = cached_fp.read(1024 ** 2)
if not data:
break
hsh.update(data)
fp.write(data)
fp.close()
newhash = hsh.hexdigest()
if os.path.exists(dest):
os.unlink(dest)
os.rename(tmpfile, dest)
log.info("%s: OK", filehash)
# See if we should re-sign NSS
if options.nsscmd and filehash != newhash and os.path.exists(os.path.splitext(filename)[0] + ".chk"):
cmd = '%s "%s"' % (options.nsscmd, dest)
log.info("Regenerating .chk file")
log.debug("Running %s", cmd)
check_call(cmd, shell=True)
return True
errors = 0
pendings = 0
max_errors = 20
max_pending_tries = 300
while True:
if pendings >= max_pending_tries:
log.error("%s: giving up after %i tries", filehash, pendings)
return False
if errors >= max_errors:
log.error("%s: giving up after %i tries", filehash, errors)
return False
# Try to get a previously signed copy of this file
try:
url = urls[0]
log.info("%s: processing %s on %s", filehash, filename, url)
req = getfile(url, filehash, fmt)
headers = req.info()
responsehash = headers['X-SHA1-Digest']
tmpfile = dest + '.tmp'
fp = open(tmpfile, 'wb')
while True:
data = req.read(1024 ** 2)
if not data:
break
fp.write(data)
fp.close()
newhash = sha1sum(tmpfile)
if newhash != responsehash:
log.warn(
"%s: hash mismatch; trying to download again", filehash)
os.unlink(tmpfile)
errors += 1
continue
if os.path.exists(dest):
os.unlink(dest)
os.rename(tmpfile, dest)
log.info("%s: OK", filehash)
# See if we should re-sign NSS
if options.nsscmd and filehash != responsehash and os.path.exists(os.path.splitext(filename)[0] + ".chk"):
cmd = '%s "%s"' % (options.nsscmd, dest)
log.info("Regenerating .chk file")
log.debug("Running %s", cmd)
check_call(cmd, shell=True)
# Possibly write to our cache
if cached_fn:
cached_dir = os.path.dirname(cached_fn)
if not os.path.exists(cached_dir):
log.debug("Creating %s", cached_dir)
os.makedirs(cached_dir)
log.info("Copying %s to cache %s", dest, cached_fn)
copyfile(dest, cached_fn)
break
except urllib2.HTTPError, e:
try:
if 'X-Pending' in e.headers:
log.debug("%s: pending; try again in a bit", filehash)
time.sleep(1)
pendings += 1
continue
except:
raise
errors += 1
# That didn't work...so let's upload it
log.info("%s: uploading for signing", filehash)
req = None
try:
try:
nonce = open(options.noncefile, 'rb').read()
except IOError:
nonce = ""
req = uploadfile(url, filename, fmt, token, nonce=nonce)
nonce = req.info()['X-Nonce']
open(options.noncefile, 'wb').write(nonce)
except urllib2.HTTPError, e:
# python2.5 doesn't think 202 is ok...but really it is!
if 'X-Nonce' in e.headers:
log.debug("updating nonce")
nonce = e.headers['X-Nonce']
open(options.noncefile, 'wb').write(nonce)
if e.code != 202:
log.info("%s: error uploading file for signing: %s %s",
filehash, e.code, e.msg)
urls.pop(0)
urls.append(url)
except (urllib2.URLError, socket.error, httplib.BadStatusLine):
# Try again in a little while
log.info("%s: connection error; trying again soon", filehash)
# Move the current url to the back
urls.pop(0)
urls.append(url)
def handle_upload(self, environ, start_response, values, rest, next_nonce):
format_ = rest[0]
assert format_ in self.formats
filehash = values['sha1']
filename = values['filename']
log.info("Request to %s sign %s (%s) from %s", format_, filename,
filehash, environ['REMOTE_ADDR'])
fn = os.path.join(self.unsigned_dir, filehash)
headers = [('X-Nonce', next_nonce)]
if os.path.exists(fn):
# Validate the file
mydigest = sha1sum(fn)
if mydigest != filehash:
log.warning("%s is corrupt; deleting (%s != %s)", fn, mydigest,
filehash)
safe_unlink(fn)
elif os.path.exists(os.path.join(self.signed_dir, filehash)):
# Everything looks ok
log.info("File already exists")
start_response("202 File already exists", headers)
return ""
elif (filehash, format_) in self.pending:
log.info("File is pending")
start_response("202 File is pending", headers)
return ""
log.info("Not pending or already signed, re-queue")
# Validate filename
if not any(exp.match(filename) for exp in self.allowed_filenames):
log.info("%s forbidden due to invalid filename: %s",
environ['REMOTE_ADDR'], filename)
start_response("403 Unacceptable filename", headers)
return ""
try:
fd, tmpname = tempfile.mkstemp(dir=self.unsigned_dir)
fp = os.fdopen(fd, 'wb')
h = hashlib.new('sha1')
s = 0
while True:
data = values['filedata'].file.read(1024**2)
if not data:
break
s += len(data)
h.update(data)
fp.write(data)
fp.close()
except:
log.exception("Error downloading data")
if os.path.exists(tmpname):
os.unlink(tmpname)
if s < self.min_filesize:
if os.path.exists(tmpname):
os.unlink(tmpname)
start_response("400 File too small", headers)
return ""
if self.max_filesize[format_] and s > self.max_filesize[format_]:
if os.path.exists(tmpname):
os.unlink(tmpname)
start_response("400 File too large", headers)
return ""
if h.hexdigest() != filehash:
if os.path.exists(tmpname):
os.unlink(tmpname)
log.warn("Hash mismatch. Bad upload?")
start_response("400 Hash mismatch", headers)
return ""
# Good to go! Rename the temporary filename to the real filename
self.save_filename(filehash, filename)
os.rename(tmpname, fn)
self.submit_file(filehash, filename, format_)
start_response("202 Accepted", headers)
self.uploads += 1
return ""
def do_GET(self, environ, start_response):
"""
GET /sign/<format>/<hash>
"""
try:
_, magic, format_, filehash = environ['PATH_INFO'].split('/')
assert magic == 'sign'
assert format_ in self.formats
except:
log.debug("bad request: %s", environ['PATH_INFO'])
start_response("400 Bad Request", [])
yield ""
return
filehash = os.path.basename(environ['PATH_INFO'])
try:
pending = self.pending.get((filehash, format_))
if pending:
log.debug("Waiting for pending job")
# Wait up to a minute for this to finish
pending.wait(timeout=60)
log.debug("Pending job finished!")
fn = self.get_path(filehash, format_)
filename = self.get_filename(filehash)
if filename:
log.debug("Looking for %s (%s)", fn, filename)
else:
log.debug("Looking for %s", fn)
checksum = sha1sum(fn)
headers = [
('X-SHA1-Digest', checksum),
('Content-Length', os.path.getsize(fn)),
]
fp = open(fn, 'rb')
os.utime(fn, None)
log.debug("%s is OK", fn)
start_response("200 OK", headers)
while True:
data = fp.read(1024**2)
if not data:
break
yield data
self.hits += 1
except IOError:
log.debug("%s is missing", fn)
headers = []
fn = os.path.join(self.unsigned_dir, filehash)
if (filehash, format_) in self.pending:
log.info("File is pending, come back soon!")
log.debug("Pending: %s", self.pending)
headers.append(('X-Pending', 'True'))
# Maybe we have the file, but not for this format
# If so, queue it up and return a pending response
# This prevents the client from having to upload the file again
elif os.path.exists(fn):
log.debug(
"GET for file we already have, but not for the right format"
)
# Validate the file
myhash = sha1sum(fn)
if myhash != filehash:
log.warning("%s is corrupt; deleting (%s != %s)", fn,
filehash, myhash)
safe_unlink(fn)
else:
filename = self.get_filename(filehash)
if filename:
self.submit_file(filehash, filename, format_)
log.info("File is pending, come back soon!")
headers.append(('X-Pending', 'True'))
else:
log.debug(
"I don't remember the filename; re-submit please!")
else:
self.misses += 1
start_response("404 Not Found", headers)
yield ""
def _worker(self):
# Main worker process
# We pop items off the queue and process them
# How many jobs to process before exiting
max_jobs = 10
jobs = 0
while True:
# Event to signal when we're done
e = None
try:
jobs += 1
# Fall on our sword if we're too old
if jobs >= max_jobs:
break
try:
item = self.queue.get(block=False)
if not item:
break
except queue.Empty:
log.debug("no items, exiting")
break
filehash, filename, format_, e = item
log.info("Signing %s (%s - %s)", filename, format_, filehash)
inputfile = os.path.join(self.inputdir, filehash)
outputfile = os.path.join(self.outputdir, format_, filehash)
logfile = outputfile + ".out"
if not os.path.exists(os.path.join(self.outputdir, format_)):
os.makedirs(os.path.join(self.outputdir, format_))
retval = run_signscript(self.signcmd, inputfile, outputfile,
filename, format_,
self.passphrases.get(format_))
if retval != 0:
if os.path.exists(logfile):
logoutput = open(logfile).read()
else:
logoutput = None
log.warning("Signing failed %s (%s - %s)", filename,
format_, filehash)
log.warning("Signing log: %s", logoutput)
safe_unlink(outputfile)
self.app.messages.put(
('errors', item, 'signing script returned non-zero'))
continue
# Copy our signed result into unsigned and signed so if
# somebody wants to get this file signed again, they get the
# same results.
outputhash = sha1sum(outputfile)
log.debug("Copying result to %s", outputhash)
copied_input = os.path.join(self.inputdir, outputhash)
if not os.path.exists(copied_input):
safe_copyfile(outputfile, copied_input)
copied_output = os.path.join(self.outputdir, format_,
outputhash)
if not os.path.exists(copied_output):
safe_copyfile(outputfile, copied_output)
self.app.messages.put(('done', item, outputhash))
except:
# Inconceivable! Something went wrong!
# Remove our output, it might be corrupted
safe_unlink(outputfile)
if os.path.exists(logfile):
logoutput = open(logfile).read()
else:
logoutput = None
log.exception("Exception signing file %s; output: %s ", item,
logoutput)
self.app.messages.put(
('errors', item, 'worker hit an exception while signing'))
finally:
if e:
e.set()
log.debug("Worker exiting")
def signFiles(self,
files,
dstdir,
product,
firstLocale='en-US',
firstLocaleSigned=False):
"""Sign `files`, putting the results into `dstdir`. If `files` has a
single entry, and refers to a directory, then all files under that
directory are signed.
`product` is the product name, e.g. 'firefox', and is used to filename
pattern matching
`firstLocale` is the first locale that should be signed, and will have
the results cached. Other locales will use the cached versions of
these signed files where appropriate.
If `firstLocaleSigned` is True, then all `firstLocale` files must
already be signed. The unsigned and signed copies of `firstLocale`
will be unpacked, and the results cached so that other locales will use
the signed copies of `firstLocale` files where appropriate.
"""
start = time.time()
if len(files) == 1 and os.path.isdir(files[0]):
files = findfiles(files[0])
files = sortFiles(filterFiles(files, product), product, firstLocale)
nfiles = len(files)
# Split the files up into locales
locales = {}
for f in files:
info = fileInfo(f, product)
locale = info['locale']
if not locale in locales:
locales[locale] = []
locales[locale].append(f)
# This is required to be global because localeFinished is called via a
# callback when child processes finish signing locales, and the
# callback doesn't have access to this local scope
global doneLocales
doneLocales = 0
# pool_start records when the pool of children has started processing
# files. We initialize it to start here because we haven't started the
# pool yet!
pool_start = start
# This is called when we finish signing a locale. We update some
# stats, and report on our progress
def localeFinished(locale):
global doneLocales
doneLocales += 1
now = int(time.time())
n = len(locales)
t_per_locale = (now - pool_start) / doneLocales
remaining = (n - doneLocales) * t_per_locale
eta_s = remaining % 60
eta_m = (remaining / 60) % 60
eta_h = (remaining / 3600)
percent = 100.0 * doneLocales / float(n)
eta_time = time.strftime("%H:%M:%S",
time.localtime(int(now + remaining)))
log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)
# Sign the firstLocale first. If we don't have any firstLocale files,
# then something is wrong!
if firstLocale not in locales:
log.error("No files found with locale %s", firstLocale)
sys.exit(1)
if not firstLocaleSigned:
if not _signLocale(
self, dstdir, locales[firstLocale], remember=True):
log.error("Error signing %s", firstLocale)
sys.exit(1)
else:
# If the first locale is already signed, we need to unpack both the
# signed and unsigned copies, and then make sure the signed
# versions are cached
for uf in locales[firstLocale]:
sf = convertPath(uf, dstdir)
if not os.path.exists(sf):
log.error(
"Signed version of %s doesn't exist as expected at %s",
uf, sf)
sys.exit(1)
unsigned_dir = tempfile.mkdtemp()
signed_dir = tempfile.mkdtemp()
try:
log.info("Unpacking %s into %s", uf, unsigned_dir)
unpackfile(uf, unsigned_dir)
log.info("Unpacking %s into %s", sf, signed_dir)
unpackfile(sf, signed_dir)
for f in findfiles(unsigned_dir):
# We don't need to cache things that aren't signed
if not shouldSign(f):
continue
# Calculate the hash of the original, unsigned file
h = sha1sum(f)
sf = signed_dir + f[len(unsigned_dir):]
# Cache the signed version
log.info("Caching %s as %s" % (f, h))
self.rememberFile(h, sf)
chk = getChkFile(sf)
if chk:
log.info("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
finally:
shutil.rmtree(unsigned_dir)
shutil.rmtree(signed_dir)
localeFinished(firstLocale)
results = [0, 0, 0]
# We're going to start our pool of children for processing the other
# locales, so reset our pool_start time to the current time.
# We do this because the time to sign the firstLocale isn't
# representative of the time to sign future locales, due to caching and
# parallelization. Using pool_start gives more accurate ETA
# calculations.
pool_start = time.time()
# Setup before signing
# We define different addLocale functions depending on if we're signing
# things concurrently or serially
if self.concurrency > 1:
# If we're doing signing in parallel, start up our pool of children
# using the processing module
import processing.pool
pool = processing.pool.Pool(self.concurrency)
result_list = []
def addLocale(locale):
def cb(r):
if not r:
log.error("Signing locale %s failed", locale)
return
for i in range(3):
results[i] += r[i]
localeFinished(locale)
files = locales[locale]
r = pool.apply_async(_signLocale, (self, dstdir, files),
callback=cb)
result_list.append(r)
else:
pool = None
def addLocale(locale):
files = locales[locale]
r = _signLocale(self, dstdir, files)
if not r:
log.error("Signing locale %s failed", locale)
sys.exit(1)
localeFinished(locale)
for i in range(3):
results[i] += r[i]
# Now go through our locales and call addLocale to start processing
# them
for locale in sorted(locales.keys()):
if locale == firstLocale:
continue
addLocale(locale)
error = False
if pool:
# Clean up the pool of child processes afterwards
for r in result_list:
try:
if not r.get():
log.error("Signing locale %s failed", locale)
error = True
pool.terminate()
break
except:
log.exception("Error")
pool.terminate()
sys.exit(1)
try:
pool.close()
pool.join()
except:
log.exception("Error")
pool.terminate()
raise
# Report on our stats
end = time.time()
nTotalFiles, cacheHits, nSigned = results
if nTotalFiles > 0:
hitrate = cacheHits / float(nTotalFiles) * 100.0
else:
hitrate = 0
if nfiles > 0:
time_per_file = (end - start) / float(nfiles)
else:
time_per_file = 0
log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
nfiles, end - start, time_per_file)
if error:
sys.exit(1)
def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
"""Sign `pkgfile`, putting the results into `dstdir`.
If `remember` is True, then cache the newly signed files into our
cache.
If `compressed` is True, then the contents of pkgfile are bz2
compressed (e.g. in a mar file), and should be decompressed before
signing.
"""
log.info("Processing %s", pkgfile)
basename = os.path.basename(pkgfile)
dstfile = convertPath(pkgfile, dstdir)
# Keep track of our output in a list here, and we can output everything
# when we're done This is to avoid interleaving the output from
# multiple processes.
logs = []
logs.append("Repacking %s to %s" % (pkgfile, dstfile))
parentdir = os.path.dirname(dstfile)
if not os.path.exists(parentdir):
os.makedirs(parentdir, 0755)
nFiles = 0
cacheHits = 0
nSigned = 0
tmpdir = tempfile.mkdtemp()
try:
# Unpack it
logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
unpackfile(pkgfile, tmpdir)
# Swap in files we have already signed
for f in findfiles(tmpdir):
# We don't need to do anything to files we're not going to sign
if not shouldSign(f):
continue
h = sha1sum(f)
basename = os.path.basename(f)
nFiles += 1
chk = getChkFile(f)
# Look in the cache for another file with the same original
# hash
cachedFile = self.getFile(h, f)
if cachedFile:
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from %s" % (basename, cachedFile))
# Preserve the original file's mode; don't use the cached mode
# We usually process installer .exe's first, and 7z doesn't
# preserve the file mode, so the cached copies of the files
# are mode 0666. In the mar files, executables have mode
# 0777, so we want to preserve that.
copyfile(cachedFile, f, copymode=False)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in cache
cachedChk = self.getFile(h, chk)
logs.append("Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
else:
# We need to sign this file
# If this file is compressed, check if we have a cached copy that
# is uncompressed
if compressed:
bunzip2(f)
h2 = sha1sum(f)
cachedFile = self.getFile(h2, f)
if cachedFile:
# We have a cached copy of this file that is uncompressed.
# So copy it into our dstdir, and recompress it, and
# save it for future use.
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from uncompressed %s" %
(basename, cachedFile))
# See note above about not copying the file's mode
copyfile(cachedFile, f, copymode=False)
bzip2(f)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in
# cache
cachedChk = self.getFile(h2, chk)
logs.append(
"Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
bzip2(chk)
if remember:
logs.append("Caching compressed %s as %s" %
(f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
continue
nSigned += 1
logs.append("Signing %s" % f)
signfile(f, self.keydir, self.fake)
if compressed:
bzip2(f)
# If we have a chk file, compress that too
if chk:
bzip2(chk)
if remember:
logs.append("Caching %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
# Repack it
logs.append("Packing %s" % dstfile)
packfile(dstfile, tmpdir)
# Sign installer
if dstfile.endswith('.exe') and not self.unsignedInstallers:
logs.append("Signing %s" % dstfile)
signfile(dstfile, self.keydir, self.fake)
return nFiles, cacheHits, nSigned
except:
log.exception("Error signing %s", pkgfile)
return False
finally:
# Clean up after ourselves, and output our logs
shutil.rmtree(tmpdir)
log.info("\n ".join(logs))
def signFiles(self, files, dstdir, product, firstLocale='en-US', firstLocaleSigned=False):
"""Sign `files`, putting the results into `dstdir`. If `files` has a
single entry, and refers to a directory, then all files under that
directory are signed.
`product` is the product name, e.g. 'firefox', and is used to filename
pattern matching
`firstLocale` is the first locale that should be signed, and will have
the results cached. Other locales will use the cached versions of
these signed files where appropriate.
If `firstLocaleSigned` is True, then all `firstLocale` files must
already be signed. The unsigned and signed copies of `firstLocale`
will be unpacked, and the results cached so that other locales will use
the signed copies of `firstLocale` files where appropriate.
"""
start = time.time()
if len(files) == 1 and os.path.isdir(files[0]):
files = findfiles(files[0])
files = sortFiles(filterFiles(files, product), product, firstLocale)
nfiles = len(files)
# Split the files up into locales
locales = {}
for f in files:
info = fileInfo(f, product)
locale = info['locale']
if not locale in locales:
locales[locale] = []
locales[locale].append(f)
# This is required to be global because localeFinished is called via a
# callback when child processes finish signing locales, and the
# callback doesn't have access to this local scope
global doneLocales
doneLocales = 0
# pool_start records when the pool of children has started processing
# files. We initialize it to start here because we haven't started the
# pool yet!
pool_start = start
# This is called when we finish signing a locale. We update some
# stats, and report on our progress
def localeFinished(locale):
global doneLocales
doneLocales += 1
now = int(time.time())
n = len(locales)
t_per_locale = (now - pool_start) / doneLocales
remaining = (n - doneLocales) * t_per_locale
eta_s = remaining % 60
eta_m = (remaining / 60) % 60
eta_h = (remaining / 3600)
percent = 100.0 * doneLocales / float(n)
eta_time = time.strftime(
"%H:%M:%S", time.localtime(int(now + remaining)))
log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)
# Sign the firstLocale first. If we don't have any firstLocale files,
# then something is wrong!
if firstLocale not in locales:
log.error("No files found with locale %s", firstLocale)
sys.exit(1)
if not firstLocaleSigned:
if not _signLocale(self, dstdir, locales[firstLocale], remember=True):
log.error("Error signing %s", firstLocale)
sys.exit(1)
else:
# If the first locale is already signed, we need to unpack both the
# signed and unsigned copies, and then make sure the signed
# versions are cached
for uf in locales[firstLocale]:
sf = convertPath(uf, dstdir)
if not os.path.exists(sf):
log.error("Signed version of %s doesn't exist as expected at %s", uf, sf)
sys.exit(1)
unsigned_dir = tempfile.mkdtemp()
signed_dir = tempfile.mkdtemp()
try:
log.info("Unpacking %s into %s", uf, unsigned_dir)
unpackfile(uf, unsigned_dir)
log.info("Unpacking %s into %s", sf, signed_dir)
unpackfile(sf, signed_dir)
for f in findfiles(unsigned_dir):
# We don't need to cache things that aren't signed
if not shouldSign(f):
continue
# Calculate the hash of the original, unsigned file
h = sha1sum(f)
sf = signed_dir + f[len(unsigned_dir):]
# Cache the signed version
log.info("Caching %s as %s" % (f, h))
self.rememberFile(h, sf)
chk = getChkFile(sf)
if chk:
log.info("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
finally:
shutil.rmtree(unsigned_dir)
shutil.rmtree(signed_dir)
localeFinished(firstLocale)
results = [0, 0, 0]
# We're going to start our pool of children for processing the other
# locales, so reset our pool_start time to the current time.
# We do this because the time to sign the firstLocale isn't
# representative of the time to sign future locales, due to caching and
# parallelization. Using pool_start gives more accurate ETA
# calculations.
pool_start = time.time()
# Setup before signing
# We define different addLocale functions depending on if we're signing
# things concurrently or serially
if self.concurrency > 1:
# If we're doing signing in parallel, start up our pool of children
# using the processing module
import processing.pool
pool = processing.pool.Pool(self.concurrency)
result_list = []
def addLocale(locale):
def cb(r):
if not r:
log.error("Signing locale %s failed", locale)
return
for i in range(3):
results[i] += r[i]
localeFinished(locale)
files = locales[locale]
r = pool.apply_async(
_signLocale, (self, dstdir, files), callback=cb)
result_list.append(r)
else:
pool = None
def addLocale(locale):
files = locales[locale]
r = _signLocale(self, dstdir, files)
if not r:
log.error("Signing locale %s failed", locale)
sys.exit(1)
localeFinished(locale)
for i in range(3):
results[i] += r[i]
# Now go through our locales and call addLocale to start processing
# them
for locale in sorted(locales.keys()):
if locale == firstLocale:
continue
addLocale(locale)
error = False
if pool:
# Clean up the pool of child processes afterwards
for r in result_list:
try:
if not r.get():
log.error("Signing locale %s failed", locale)
error = True
pool.terminate()
break
except:
log.exception("Error")
pool.terminate()
sys.exit(1)
try:
pool.close()
pool.join()
except:
log.exception("Error")
pool.terminate()
raise
# Report on our stats
end = time.time()
nTotalFiles, cacheHits, nSigned = results
if nTotalFiles > 0:
hitrate = cacheHits / float(nTotalFiles) * 100.0
else:
hitrate = 0
if nfiles > 0:
time_per_file = (end - start) / float(nfiles)
else:
time_per_file = 0
log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
nfiles, end - start, time_per_file)
if error:
sys.exit(1)
def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
"""Sign `pkgfile`, putting the results into `dstdir`.
If `remember` is True, then cache the newly signed files into our
cache.
If `compressed` is True, then the contents of pkgfile are bz2
compressed (e.g. in a mar file), and should be decompressed before
signing.
"""
log.info("Processing %s", pkgfile)
basename = os.path.basename(pkgfile)
dstfile = convertPath(pkgfile, dstdir)
# Keep track of our output in a list here, and we can output everything
# when we're done This is to avoid interleaving the output from
# multiple processes.
logs = []
logs.append("Repacking %s to %s" % (pkgfile, dstfile))
parentdir = os.path.dirname(dstfile)
if not os.path.exists(parentdir):
os.makedirs(parentdir, 0755)
nFiles = 0
cacheHits = 0
nSigned = 0
tmpdir = tempfile.mkdtemp()
try:
# Unpack it
logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
unpackfile(pkgfile, tmpdir)
# Swap in files we have already signed
for f in findfiles(tmpdir):
# We don't need to do anything to files we're not going to sign
if not shouldSign(f):
continue
h = sha1sum(f)
basename = os.path.basename(f)
nFiles += 1
chk = getChkFile(f)
# Look in the cache for another file with the same original
# hash
cachedFile = self.getFile(h, f)
if cachedFile:
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from %s" % (basename, cachedFile))
# Preserve the original file's mode; don't use the cached mode
# We usually process installer .exe's first, and 7z doesn't
# preserve the file mode, so the cached copies of the files
# are mode 0666. In the mar files, executables have mode
# 0777, so we want to preserve that.
copyfile(cachedFile, f, copymode=False)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in cache
cachedChk = self.getFile(h, chk)
logs.append("Copying %s from %s" %
(os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
else:
# We need to sign this file
# If this file is compressed, check if we have a cached copy that
# is uncompressed
if compressed:
bunzip2(f)
h2 = sha1sum(f)
cachedFile = self.getFile(h2, f)
if cachedFile:
# We have a cached copy of this file that is uncompressed.
# So copy it into our dstdir, and recompress it, and
# save it for future use.
cacheHits += 1
assert os.path.basename(cachedFile) == basename
logs.append("Copying %s from uncompressed %s" %
(basename, cachedFile))
# See note above about not copying the file's mode
copyfile(cachedFile, f, copymode=False)
bzip2(f)
if chk:
# If there's a .chk file for this file, copy that out of cache
# It's an error if this file doesn't exist in
# cache
cachedChk = self.getFile(h2, chk)
logs.append("Copying %s from %s" % (
os.path.basename(cachedChk), cachedChk))
copyfile(cachedChk, chk, copymode=False)
bzip2(chk)
if remember:
logs.append(
"Caching compressed %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
continue
nSigned += 1
logs.append("Signing %s" % f)
signfile(f, self.keydir, self.fake)
if compressed:
bzip2(f)
# If we have a chk file, compress that too
if chk:
bzip2(chk)
if remember:
logs.append("Caching %s as %s" % (f, h))
self.rememberFile(h, f)
# Remember any regenerated chk files
if chk:
logs.append("Caching %s as %s" % (chk, h))
self.rememberFile(h, chk)
# Repack it
logs.append("Packing %s" % dstfile)
packfile(dstfile, tmpdir)
# Sign installer
if dstfile.endswith('.exe') and not self.unsignedInstallers:
logs.append("Signing %s" % dstfile)
signfile(dstfile, self.keydir, self.fake)
return nFiles, cacheHits, nSigned
except:
log.exception("Error signing %s", pkgfile)
return False
finally:
# Clean up after ourselves, and output our logs
shutil.rmtree(tmpdir)
log.info("\n ".join(logs))
def remote_signfile(options, urls, filename, fmt, token, dest=None):
filehash = sha1sum(filename)
if dest is None:
dest = filename
if fmt == 'gpg':
dest += '.asc'
elif fmt in ('widevine', 'widevine_blessed'):
dest += '.sig'
parent_dir = os.path.dirname(os.path.abspath(dest))
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
# Check the cache
cached_fn = None
if options.cachedir:
log.debug("%s: checking cache", filehash)
cached_fn = os.path.join(options.cachedir, fmt, filehash)
if os.path.exists(cached_fn):
log.info("%s: exists in the cache; copying to %s", filehash, dest)
cached_fp = open(cached_fn, 'rb')
tmpfile = dest + '.tmp'
fp = open(tmpfile, 'wb')
hsh = hashlib.new('sha1')
while True:
data = cached_fp.read(1024 ** 2)
if not data:
break
hsh.update(data)
fp.write(data)
fp.close()
newhash = hsh.hexdigest()
if os.path.exists(dest):
os.unlink(dest)
os.rename(tmpfile, dest)
log.info("%s: OK", filehash)
# See if we should re-sign NSS
if options.nsscmd and filehash != newhash and os.path.exists(os.path.splitext(filename)[0] + ".chk"):
cmd = '%s "%s"' % (options.nsscmd, dest)
log.info("Regenerating .chk file")
log.debug("Running %s", cmd)
check_call(cmd, shell=True)
return True
errors = 0
pendings = 0
max_errors = 5
# It takes the server ~60s to respond to an attempting to get a signed file
# We want to give up after about 5 minutes, so 60*5 = 5 tries.
max_pending_tries = 5
while True:
if pendings >= max_pending_tries:
log.error("%s: giving up after %i tries", filehash, pendings)
# If we've given up on the current server, try a different one!
urls.pop(0)
urls.append(url)
errors += 1
# Pendings needs to be reset to give the next server a fair shake.
pendings = 0
if errors >= max_errors:
log.error("%s: giving up after %i tries", filehash, errors)
return False
# Try to get a previously signed copy of this file
try:
url = urls[0]
log.info("%s: processing %s on %s", filehash, filename, url)
req = getfile(url, filehash, fmt)
headers = req.info()
responsehash = headers['X-SHA1-Digest']
tmpfile = dest + '.tmp'
fp = open(tmpfile, 'wb')
while True:
data = req.read(1024 ** 2)
if not data:
break
fp.write(data)
fp.close()
newhash = sha1sum(tmpfile)
if newhash != responsehash:
log.warn(
"%s: hash mismatch; trying to download again", filehash)
os.unlink(tmpfile)
errors += 1
continue
if os.path.exists(dest):
os.unlink(dest)
os.rename(tmpfile, dest)
log.info("%s: OK", filehash)
# See if we should re-sign NSS
if options.nsscmd and filehash != responsehash and os.path.exists(os.path.splitext(filename)[0] + ".chk"):
cmd = '%s "%s"' % (options.nsscmd, dest)
log.info("Regenerating .chk file")
log.debug("Running %s", cmd)
check_call(cmd, shell=True)
# Possibly write to our cache
if cached_fn:
cached_dir = os.path.dirname(cached_fn)
if not os.path.exists(cached_dir):
log.debug("Creating %s", cached_dir)
os.makedirs(cached_dir)
log.info("Copying %s to cache %s", dest, cached_fn)
copyfile(dest, cached_fn)
break
except urllib2.HTTPError, e:
try:
if 'X-Pending' in e.headers:
log.debug("%s: pending; try again in a bit", filehash)
time.sleep(15)
pendings += 1
continue
except:
raise
errors += 1
# That didn't work...so let's upload it
log.info("%s: uploading for signing", filehash)
req = None
try:
try:
nonce = open(options.noncefile, 'rb').read()
except IOError:
nonce = ""
req = uploadfile(url, filename, fmt, token, nonce=nonce)
nonce = req.info()['X-Nonce']
open(options.noncefile, 'wb').write(nonce)
except urllib2.HTTPError, e:
# python2.5 doesn't think 202 is ok...but really it is!
if 'X-Nonce' in e.headers:
log.debug("updating nonce")
nonce = e.headers['X-Nonce']
open(options.noncefile, 'wb').write(nonce)
if e.code != 202:
log.exception("%s: error uploading file for signing: %s %s",
filehash, e.code, e.msg)
urls.pop(0)
urls.append(url)
except (urllib2.URLError, socket.error, httplib.BadStatusLine):
# Try again in a little while
log.exception("%s: connection error; trying again soon", filehash)
# Move the current url to the back
urls.pop(0)
urls.append(url)
def handle_upload(self, environ, start_response, values, rest, next_nonce):
format_ = rest[0]
assert format_ in self.formats
filehash = values['sha1']
filename = values['filename']
log.info("Request to %s sign %s (%s) from %s", format_,
filename, filehash, environ['REMOTE_ADDR'])
fn = os.path.join(self.unsigned_dir, filehash)
headers = [('X-Nonce', next_nonce)]
if os.path.exists(fn):
# Validate the file
mydigest = sha1sum(fn)
if mydigest != filehash:
log.warning("%s is corrupt; deleting (%s != %s)",
fn, mydigest, filehash)
safe_unlink(fn)
elif os.path.exists(os.path.join(self.signed_dir, filehash)):
# Everything looks ok
log.info("File already exists")
start_response("202 File already exists", headers)
return ""
elif (filehash, format_) in self.pending:
log.info("File is pending")
start_response("202 File is pending", headers)
return ""
log.info("Not pending or already signed, re-queue")
# Validate filename
if not any(exp.match(filename) for exp in self.allowed_filenames):
log.info("%s forbidden due to invalid filename: %s",
environ['REMOTE_ADDR'], filename)
start_response("403 Unacceptable filename", headers)
return ""
try:
fd, tmpname = tempfile.mkstemp(dir=self.unsigned_dir)
fp = os.fdopen(fd, 'wb')
h = hashlib.new('sha1')
s = 0
while True:
data = values['filedata'].file.read(1024 ** 2)
if not data:
break
s += len(data)
h.update(data)
fp.write(data)
fp.close()
except:
log.exception("Error downloading data")
if os.path.exists(tmpname):
os.unlink(tmpname)
if s < self.min_filesize:
if os.path.exists(tmpname):
os.unlink(tmpname)
start_response("400 File too small", headers)
return ""
if self.max_filesize[format_] and s > self.max_filesize[format_]:
if os.path.exists(tmpname):
os.unlink(tmpname)
start_response("400 File too large", headers)
return ""
if h.hexdigest() != filehash:
if os.path.exists(tmpname):
os.unlink(tmpname)
log.warn("Hash mismatch. Bad upload?")
start_response("400 Hash mismatch", headers)
return ""
# Good to go! Rename the temporary filename to the real filename
self.save_filename(filehash, filename)
os.rename(tmpname, fn)
self.submit_file(filehash, filename, format_)
start_response("202 Accepted", headers)
self.uploads += 1
return ""
def do_GET(self, environ, start_response):
"""
GET /sign/<format>/<hash>
"""
try:
_, magic, format_, filehash = environ['PATH_INFO'].split('/')
assert magic == 'sign'
assert format_ in self.formats
except:
log.debug("bad request: %s", environ['PATH_INFO'])
start_response("400 Bad Request", [])
yield ""
return
filehash = os.path.basename(environ['PATH_INFO'])
try:
pending = self.pending.get((filehash, format_))
if pending:
log.debug("Waiting for pending job")
# Wait up to a minute for this to finish
pending.wait(timeout=60)
log.debug("Pending job finished!")
fn = self.get_path(filehash, format_)
filename = self.get_filename(filehash)
if filename:
log.debug("Looking for %s (%s)", fn, filename)
else:
log.debug("Looking for %s", fn)
checksum = sha1sum(fn)
headers = [
('X-SHA1-Digest', checksum),
('Content-Length', str(os.path.getsize(fn))),
]
fp = open(fn, 'rb')
os.utime(fn, None)
log.debug("%s is OK", fn)
start_response("200 OK", headers)
while True:
data = fp.read(1024 ** 2)
if not data:
break
yield data
self.hits += 1
except IOError:
log.debug("%s is missing", fn)
headers = []
fn = os.path.join(self.unsigned_dir, filehash)
if (filehash, format_) in self.pending:
log.info("File is pending, come back soon!")
log.debug("Pending: %s", self.pending)
headers.append(('X-Pending', 'True'))
# Maybe we have the file, but not for this format
# If so, queue it up and return a pending response
# This prevents the client from having to upload the file again
elif os.path.exists(fn):
log.debug("GET for file we already have, but not for the right format")
# Validate the file
myhash = sha1sum(fn)
if myhash != filehash:
log.warning("%s is corrupt; deleting (%s != %s)",
fn, filehash, myhash)
safe_unlink(fn)
else:
filename = self.get_filename(filehash)
if filename:
self.submit_file(filehash, filename, format_)
log.info("File is pending, come back soon!")
headers.append(('X-Pending', 'True'))
else:
log.debug("I don't remember the filename; re-submit please!")
else:
self.misses += 1
start_response("404 Not Found", headers)
yield ""
except:
log.exception("ISE")
start_response("500 Internal Server Error", headers)
yield ""
def _worker(self):
# Main worker process
# We pop items off the queue and process them
# How many jobs to process before exiting
max_jobs = 10
jobs = 0
while True:
# Event to signal when we're done
e = None
try:
jobs += 1
# Fall on our sword if we're too old
if jobs >= max_jobs:
break
try:
item = self.queue.get(block=False)
if not item:
break
except queue.Empty:
log.debug("no items, exiting")
break
filehash, filename, format_, e = item
log.info("Signing %s (%s - %s)", filename, format_, filehash)
inputfile = os.path.join(self.inputdir, filehash)
outputfile = os.path.join(self.outputdir, format_, filehash)
logfile = outputfile + ".out"
if not os.path.exists(os.path.join(self.outputdir, format_)):
os.makedirs(os.path.join(self.outputdir, format_))
retval = run_signscript(self.signcmd, inputfile, outputfile,
filename, format_, self.passphrases.get(format_))
if retval != 0:
if os.path.exists(logfile):
logoutput = open(logfile).read()
else:
logoutput = None
log.warning("Signing failed %s (%s - %s)",
filename, format_, filehash)
log.warning("Signing log: %s", logoutput)
safe_unlink(outputfile)
self.app.messages.put(
('errors', item, 'signing script returned non-zero'))
continue
# Copy our signed result into unsigned and signed so if
# somebody wants to get this file signed again, they get the
# same results.
outputhash = sha1sum(outputfile)
log.debug("Copying result to %s", outputhash)
copied_input = os.path.join(self.inputdir, outputhash)
if not os.path.exists(copied_input):
safe_copyfile(outputfile, copied_input)
copied_output = os.path.join(
self.outputdir, format_, outputhash)
if not os.path.exists(copied_output):
safe_copyfile(outputfile, copied_output)
self.app.messages.put(('done', item, outputhash))
except:
# Inconceivable! Something went wrong!
# Remove our output, it might be corrupted
safe_unlink(outputfile)
if os.path.exists(logfile):
logoutput = open(logfile).read()
else:
logoutput = None
log.exception(
"Exception signing file %s; output: %s ", item, logoutput)
self.app.messages.put((
'errors', item, 'worker hit an exception while signing'))
finally:
if e:
e.set()
log.debug("Worker exiting")
def testSha1sum(self):
h = hashlib.new('sha1')
h.update(open(__file__, 'rb').read())
self.assertEquals(sha1sum(__file__), h.hexdigest())