def testSorting(self): """ Test that the expected sort order is maintained with plain exes, followed by partner exes, followed by MARs""" files = [ 'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe', 'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar', 'unsigned/win32/en-US/Firefox Setup 3.6.13.exe', 'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe', 'unsigned/update/win32/ar/firefox-3.6.13.complete.mar', 'unsigned/win32/ar/Firefox Setup 3.6.13.exe', ] sorted_files = [ 'unsigned/win32/en-US/Firefox Setup 3.6.13.exe', 'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar', 'unsigned/win32/ar/Firefox Setup 3.6.13.exe', 'unsigned/update/win32/ar/firefox-3.6.13.complete.mar', 'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe', 'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe', ] results = sortFiles(files, self.product, self.firstLocale) self.assertEquals(results, sorted_files)
def signFiles(self, files, dstdir, product, firstLocale='en-US', firstLocaleSigned=False): """Sign `files`, putting the results into `dstdir`. If `files` has a single entry, and refers to a directory, then all files under that directory are signed. `product` is the product name, e.g. 'firefox', and is used to filename pattern matching `firstLocale` is the first locale that should be signed, and will have the results cached. Other locales will use the cached versions of these signed files where appropriate. If `firstLocaleSigned` is True, then all `firstLocale` files must already be signed. The unsigned and signed copies of `firstLocale` will be unpacked, and the results cached so that other locales will use the signed copies of `firstLocale` files where appropriate. """ start = time.time() if len(files) == 1 and os.path.isdir(files[0]): files = findfiles(files[0]) files = sortFiles(filterFiles(files, product), product, firstLocale) nfiles = len(files) # Split the files up into locales locales = {} for f in files: info = fileInfo(f, product) locale = info['locale'] if not locale in locales: locales[locale] = [] locales[locale].append(f) # This is required to be global because localeFinished is called via a # callback when child processes finish signing locales, and the # callback doesn't have access to this local scope global doneLocales doneLocales = 0 # pool_start records when the pool of children has started processing # files. We initialize it to start here because we haven't started the # pool yet! pool_start = start # This is called when we finish signing a locale. We update some # stats, and report on our progress def localeFinished(locale): global doneLocales doneLocales += 1 now = int(time.time()) n = len(locales) t_per_locale = (now - pool_start) / doneLocales remaining = (n - doneLocales) * t_per_locale eta_s = remaining % 60 eta_m = (remaining / 60) % 60 eta_h = (remaining / 3600) percent = 100.0 * doneLocales / float(n) eta_time = time.strftime("%H:%M:%S", time.localtime(int(now + remaining))) log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s", doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time) # Sign the firstLocale first. If we don't have any firstLocale files, # then something is wrong! if firstLocale not in locales: log.error("No files found with locale %s", firstLocale) sys.exit(1) if not firstLocaleSigned: if not _signLocale( self, dstdir, locales[firstLocale], remember=True): log.error("Error signing %s", firstLocale) sys.exit(1) else: # If the first locale is already signed, we need to unpack both the # signed and unsigned copies, and then make sure the signed # versions are cached for uf in locales[firstLocale]: sf = convertPath(uf, dstdir) if not os.path.exists(sf): log.error( "Signed version of %s doesn't exist as expected at %s", uf, sf) sys.exit(1) unsigned_dir = tempfile.mkdtemp() signed_dir = tempfile.mkdtemp() try: log.info("Unpacking %s into %s", uf, unsigned_dir) unpackfile(uf, unsigned_dir) log.info("Unpacking %s into %s", sf, signed_dir) unpackfile(sf, signed_dir) for f in findfiles(unsigned_dir): # We don't need to cache things that aren't signed if not shouldSign(f): continue # Calculate the hash of the original, unsigned file h = sha1sum(f) sf = signed_dir + f[len(unsigned_dir):] # Cache the signed version log.info("Caching %s as %s" % (f, h)) self.rememberFile(h, sf) chk = getChkFile(sf) if chk: log.info("Caching %s as %s" % (chk, h)) self.rememberFile(h, chk) finally: shutil.rmtree(unsigned_dir) shutil.rmtree(signed_dir) localeFinished(firstLocale) results = [0, 0, 0] # We're going to start our pool of children for processing the other # locales, so reset our pool_start time to the current time. # We do this because the time to sign the firstLocale isn't # representative of the time to sign future locales, due to caching and # parallelization. Using pool_start gives more accurate ETA # calculations. pool_start = time.time() # Setup before signing # We define different addLocale functions depending on if we're signing # things concurrently or serially if self.concurrency > 1: # If we're doing signing in parallel, start up our pool of children # using the processing module import processing.pool pool = processing.pool.Pool(self.concurrency) result_list = [] def addLocale(locale): def cb(r): if not r: log.error("Signing locale %s failed", locale) return for i in range(3): results[i] += r[i] localeFinished(locale) files = locales[locale] r = pool.apply_async(_signLocale, (self, dstdir, files), callback=cb) result_list.append(r) else: pool = None def addLocale(locale): files = locales[locale] r = _signLocale(self, dstdir, files) if not r: log.error("Signing locale %s failed", locale) sys.exit(1) localeFinished(locale) for i in range(3): results[i] += r[i] # Now go through our locales and call addLocale to start processing # them for locale in sorted(locales.keys()): if locale == firstLocale: continue addLocale(locale) error = False if pool: # Clean up the pool of child processes afterwards for r in result_list: try: if not r.get(): log.error("Signing locale %s failed", locale) error = True pool.terminate() break except: log.exception("Error") pool.terminate() sys.exit(1) try: pool.close() pool.join() except: log.exception("Error") pool.terminate() raise # Report on our stats end = time.time() nTotalFiles, cacheHits, nSigned = results if nTotalFiles > 0: hitrate = cacheHits / float(nTotalFiles) * 100.0 else: hitrate = 0 if nfiles > 0: time_per_file = (end - start) / float(nfiles) else: time_per_file = 0 log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned) log.info("%s files repacked in %.0f seconds (%.2f seconds per file)", nfiles, end - start, time_per_file) if error: sys.exit(1)
def signFiles(self, files, dstdir, product, firstLocale='en-US', firstLocaleSigned=False): """Sign `files`, putting the results into `dstdir`. If `files` has a single entry, and refers to a directory, then all files under that directory are signed. `product` is the product name, e.g. 'firefox', and is used to filename pattern matching `firstLocale` is the first locale that should be signed, and will have the results cached. Other locales will use the cached versions of these signed files where appropriate. If `firstLocaleSigned` is True, then all `firstLocale` files must already be signed. The unsigned and signed copies of `firstLocale` will be unpacked, and the results cached so that other locales will use the signed copies of `firstLocale` files where appropriate. """ start = time.time() if len(files) == 1 and os.path.isdir(files[0]): files = findfiles(files[0]) files = sortFiles(filterFiles(files, product), product, firstLocale) nfiles = len(files) # Split the files up into locales locales = {} for f in files: info = fileInfo(f, product) locale = info['locale'] if not locale in locales: locales[locale] = [] locales[locale].append(f) # This is required to be global because localeFinished is called via a # callback when child processes finish signing locales, and the # callback doesn't have access to this local scope global doneLocales doneLocales = 0 # pool_start records when the pool of children has started processing # files. We initialize it to start here because we haven't started the # pool yet! pool_start = start # This is called when we finish signing a locale. We update some # stats, and report on our progress def localeFinished(locale): global doneLocales doneLocales += 1 now = int(time.time()) n = len(locales) t_per_locale = (now - pool_start) / doneLocales remaining = (n - doneLocales) * t_per_locale eta_s = remaining % 60 eta_m = (remaining / 60) % 60 eta_h = (remaining / 3600) percent = 100.0 * doneLocales / float(n) eta_time = time.strftime( "%H:%M:%S", time.localtime(int(now + remaining))) log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s", doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time) # Sign the firstLocale first. If we don't have any firstLocale files, # then something is wrong! if firstLocale not in locales: log.error("No files found with locale %s", firstLocale) sys.exit(1) if not firstLocaleSigned: if not _signLocale(self, dstdir, locales[firstLocale], remember=True): log.error("Error signing %s", firstLocale) sys.exit(1) else: # If the first locale is already signed, we need to unpack both the # signed and unsigned copies, and then make sure the signed # versions are cached for uf in locales[firstLocale]: sf = convertPath(uf, dstdir) if not os.path.exists(sf): log.error("Signed version of %s doesn't exist as expected at %s", uf, sf) sys.exit(1) unsigned_dir = tempfile.mkdtemp() signed_dir = tempfile.mkdtemp() try: log.info("Unpacking %s into %s", uf, unsigned_dir) unpackfile(uf, unsigned_dir) log.info("Unpacking %s into %s", sf, signed_dir) unpackfile(sf, signed_dir) for f in findfiles(unsigned_dir): # We don't need to cache things that aren't signed if not shouldSign(f): continue # Calculate the hash of the original, unsigned file h = sha1sum(f) sf = signed_dir + f[len(unsigned_dir):] # Cache the signed version log.info("Caching %s as %s" % (f, h)) self.rememberFile(h, sf) chk = getChkFile(sf) if chk: log.info("Caching %s as %s" % (chk, h)) self.rememberFile(h, chk) finally: shutil.rmtree(unsigned_dir) shutil.rmtree(signed_dir) localeFinished(firstLocale) results = [0, 0, 0] # We're going to start our pool of children for processing the other # locales, so reset our pool_start time to the current time. # We do this because the time to sign the firstLocale isn't # representative of the time to sign future locales, due to caching and # parallelization. Using pool_start gives more accurate ETA # calculations. pool_start = time.time() # Setup before signing # We define different addLocale functions depending on if we're signing # things concurrently or serially if self.concurrency > 1: # If we're doing signing in parallel, start up our pool of children # using the processing module import processing.pool pool = processing.pool.Pool(self.concurrency) result_list = [] def addLocale(locale): def cb(r): if not r: log.error("Signing locale %s failed", locale) return for i in range(3): results[i] += r[i] localeFinished(locale) files = locales[locale] r = pool.apply_async( _signLocale, (self, dstdir, files), callback=cb) result_list.append(r) else: pool = None def addLocale(locale): files = locales[locale] r = _signLocale(self, dstdir, files) if not r: log.error("Signing locale %s failed", locale) sys.exit(1) localeFinished(locale) for i in range(3): results[i] += r[i] # Now go through our locales and call addLocale to start processing # them for locale in sorted(locales.keys()): if locale == firstLocale: continue addLocale(locale) error = False if pool: # Clean up the pool of child processes afterwards for r in result_list: try: if not r.get(): log.error("Signing locale %s failed", locale) error = True pool.terminate() break except: log.exception("Error") pool.terminate() sys.exit(1) try: pool.close() pool.join() except: log.exception("Error") pool.terminate() raise # Report on our stats end = time.time() nTotalFiles, cacheHits, nSigned = results if nTotalFiles > 0: hitrate = cacheHits / float(nTotalFiles) * 100.0 else: hitrate = 0 if nfiles > 0: time_per_file = (end - start) / float(nfiles) else: time_per_file = 0 log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned) log.info("%s files repacked in %.0f seconds (%.2f seconds per file)", nfiles, end - start, time_per_file) if error: sys.exit(1)