Example #1
0
    def testSorting(self):
        """ Test that the expected sort order is maintained with plain
           exes, followed by partner exes, followed by MARs"""
        files = [
            'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar',
            'unsigned/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/ar/firefox-3.6.13.complete.mar',
            'unsigned/win32/ar/Firefox Setup 3.6.13.exe',
        ]
        sorted_files = [
            'unsigned/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar',
            'unsigned/win32/ar/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/ar/firefox-3.6.13.complete.mar',
            'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe',
        ]

        results = sortFiles(files, self.product, self.firstLocale)
        self.assertEquals(results, sorted_files)
Example #2
0
    def testSorting(self):
        """ Test that the expected sort order is maintained with plain
           exes, followed by partner exes, followed by MARs"""
        files = [
            'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar',
            'unsigned/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/ar/firefox-3.6.13.complete.mar',
            'unsigned/win32/ar/Firefox Setup 3.6.13.exe',
        ]
        sorted_files = [
            'unsigned/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/en-US/firefox-3.6.13.complete.mar',
            'unsigned/win32/ar/Firefox Setup 3.6.13.exe',
            'unsigned/update/win32/ar/firefox-3.6.13.complete.mar',
            'unsigned/partner-repacks/chinapack-win32/win32/en-US/Firefox Setup 3.6.13.exe',
            'unsigned/partner-repacks/google/win32/ar/Firefox Setup 3.6.13.exe',
        ]

        results = sortFiles(files, self.product, self.firstLocale)
        self.assertEquals(results, sorted_files)
Example #3
0
    def signFiles(self,
                  files,
                  dstdir,
                  product,
                  firstLocale='en-US',
                  firstLocaleSigned=False):
        """Sign `files`, putting the results into `dstdir`.  If `files` has a
        single entry, and refers to a directory, then all files under that
        directory are signed.

        `product` is the product name, e.g. 'firefox', and is used to filename
        pattern matching

        `firstLocale` is the first locale that should be signed, and will have
        the results cached.  Other locales will use the cached versions of
        these signed files where appropriate.

        If `firstLocaleSigned` is True, then all `firstLocale` files must
        already be signed.  The unsigned and signed copies of `firstLocale`
        will be unpacked, and the results cached so that other locales will use
        the signed copies of `firstLocale` files where appropriate.
        """
        start = time.time()

        if len(files) == 1 and os.path.isdir(files[0]):
            files = findfiles(files[0])

        files = sortFiles(filterFiles(files, product), product, firstLocale)
        nfiles = len(files)

        # Split the files up into locales
        locales = {}
        for f in files:
            info = fileInfo(f, product)
            locale = info['locale']
            if not locale in locales:
                locales[locale] = []
            locales[locale].append(f)

        # This is required to be global because localeFinished is called via a
        # callback when child processes finish signing locales, and the
        # callback doesn't have access to this local scope
        global doneLocales
        doneLocales = 0

        # pool_start records when the pool of children has started processing
        # files.  We initialize it to start here because we haven't started the
        # pool yet!
        pool_start = start

        # This is called when we finish signing a locale.  We update some
        # stats, and report on our progress
        def localeFinished(locale):
            global doneLocales
            doneLocales += 1
            now = int(time.time())
            n = len(locales)
            t_per_locale = (now - pool_start) / doneLocales
            remaining = (n - doneLocales) * t_per_locale
            eta_s = remaining % 60
            eta_m = (remaining / 60) % 60
            eta_h = (remaining / 3600)
            percent = 100.0 * doneLocales / float(n)
            eta_time = time.strftime("%H:%M:%S",
                                     time.localtime(int(now + remaining)))
            log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
                     doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)

        # Sign the firstLocale first.  If we don't have any firstLocale files,
        # then something is wrong!
        if firstLocale not in locales:
            log.error("No files found with locale %s", firstLocale)
            sys.exit(1)

        if not firstLocaleSigned:
            if not _signLocale(
                    self, dstdir, locales[firstLocale], remember=True):
                log.error("Error signing %s", firstLocale)
                sys.exit(1)
        else:
            # If the first locale is already signed, we need to unpack both the
            # signed and unsigned copies, and then make sure the signed
            # versions are cached
            for uf in locales[firstLocale]:
                sf = convertPath(uf, dstdir)
                if not os.path.exists(sf):
                    log.error(
                        "Signed version of %s doesn't exist as expected at %s",
                        uf, sf)
                    sys.exit(1)

                unsigned_dir = tempfile.mkdtemp()
                signed_dir = tempfile.mkdtemp()
                try:
                    log.info("Unpacking %s into %s", uf, unsigned_dir)
                    unpackfile(uf, unsigned_dir)
                    log.info("Unpacking %s into %s", sf, signed_dir)
                    unpackfile(sf, signed_dir)
                    for f in findfiles(unsigned_dir):
                        # We don't need to cache things that aren't signed
                        if not shouldSign(f):
                            continue
                        # Calculate the hash of the original, unsigned file
                        h = sha1sum(f)
                        sf = signed_dir + f[len(unsigned_dir):]
                        # Cache the signed version
                        log.info("Caching %s as %s" % (f, h))
                        self.rememberFile(h, sf)
                        chk = getChkFile(sf)
                        if chk:
                            log.info("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)
                finally:
                    shutil.rmtree(unsigned_dir)
                    shutil.rmtree(signed_dir)
        localeFinished(firstLocale)

        results = [0, 0, 0]
        # We're going to start our pool of children for processing the other
        # locales, so reset our pool_start time to the current time.
        # We do this because the time to sign the firstLocale isn't
        # representative of the time to sign future locales, due to caching and
        # parallelization.  Using pool_start gives more accurate ETA
        # calculations.
        pool_start = time.time()

        # Setup before signing
        # We define different addLocale functions depending on if we're signing
        # things concurrently or serially
        if self.concurrency > 1:
            # If we're doing signing in parallel, start up our pool of children
            # using the processing module
            import processing.pool
            pool = processing.pool.Pool(self.concurrency)
            result_list = []

            def addLocale(locale):
                def cb(r):
                    if not r:
                        log.error("Signing locale %s failed", locale)
                        return

                    for i in range(3):
                        results[i] += r[i]
                    localeFinished(locale)

                files = locales[locale]
                r = pool.apply_async(_signLocale, (self, dstdir, files),
                                     callback=cb)
                result_list.append(r)
        else:
            pool = None

            def addLocale(locale):
                files = locales[locale]
                r = _signLocale(self, dstdir, files)
                if not r:
                    log.error("Signing locale %s failed", locale)
                    sys.exit(1)
                localeFinished(locale)
                for i in range(3):
                    results[i] += r[i]

        # Now go through our locales and call addLocale to start processing
        # them
        for locale in sorted(locales.keys()):
            if locale == firstLocale:
                continue
            addLocale(locale)

        error = False
        if pool:
            # Clean up the pool of child processes afterwards
            for r in result_list:
                try:
                    if not r.get():
                        log.error("Signing locale %s failed", locale)
                        error = True
                        pool.terminate()
                        break
                except:
                    log.exception("Error")
                    pool.terminate()
                    sys.exit(1)
            try:
                pool.close()
                pool.join()
            except:
                log.exception("Error")
                pool.terminate()
                raise

        # Report on our stats
        end = time.time()
        nTotalFiles, cacheHits, nSigned = results
        if nTotalFiles > 0:
            hitrate = cacheHits / float(nTotalFiles) * 100.0
        else:
            hitrate = 0
        if nfiles > 0:
            time_per_file = (end - start) / float(nfiles)
        else:
            time_per_file = 0
        log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
        log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
                 nfiles, end - start, time_per_file)

        if error:
            sys.exit(1)
Example #4
0
    def signFiles(self, files, dstdir, product, firstLocale='en-US', firstLocaleSigned=False):
        """Sign `files`, putting the results into `dstdir`.  If `files` has a
        single entry, and refers to a directory, then all files under that
        directory are signed.

        `product` is the product name, e.g. 'firefox', and is used to filename
        pattern matching

        `firstLocale` is the first locale that should be signed, and will have
        the results cached.  Other locales will use the cached versions of
        these signed files where appropriate.

        If `firstLocaleSigned` is True, then all `firstLocale` files must
        already be signed.  The unsigned and signed copies of `firstLocale`
        will be unpacked, and the results cached so that other locales will use
        the signed copies of `firstLocale` files where appropriate.
        """
        start = time.time()

        if len(files) == 1 and os.path.isdir(files[0]):
            files = findfiles(files[0])

        files = sortFiles(filterFiles(files, product), product, firstLocale)
        nfiles = len(files)

        # Split the files up into locales
        locales = {}
        for f in files:
            info = fileInfo(f, product)
            locale = info['locale']
            if not locale in locales:
                locales[locale] = []
            locales[locale].append(f)

        # This is required to be global because localeFinished is called via a
        # callback when child processes finish signing locales, and the
        # callback doesn't have access to this local scope
        global doneLocales
        doneLocales = 0

        # pool_start records when the pool of children has started processing
        # files.  We initialize it to start here because we haven't started the
        # pool yet!
        pool_start = start

        # This is called when we finish signing a locale.  We update some
        # stats, and report on our progress
        def localeFinished(locale):
            global doneLocales
            doneLocales += 1
            now = int(time.time())
            n = len(locales)
            t_per_locale = (now - pool_start) / doneLocales
            remaining = (n - doneLocales) * t_per_locale
            eta_s = remaining % 60
            eta_m = (remaining / 60) % 60
            eta_h = (remaining / 3600)
            percent = 100.0 * doneLocales / float(n)
            eta_time = time.strftime(
                "%H:%M:%S", time.localtime(int(now + remaining)))
            log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
                     doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)

        # Sign the firstLocale first.  If we don't have any firstLocale files,
        # then something is wrong!
        if firstLocale not in locales:
            log.error("No files found with locale %s", firstLocale)
            sys.exit(1)

        if not firstLocaleSigned:
            if not _signLocale(self, dstdir, locales[firstLocale], remember=True):
                log.error("Error signing %s", firstLocale)
                sys.exit(1)
        else:
            # If the first locale is already signed, we need to unpack both the
            # signed and unsigned copies, and then make sure the signed
            # versions are cached
            for uf in locales[firstLocale]:
                sf = convertPath(uf, dstdir)
                if not os.path.exists(sf):
                    log.error("Signed version of %s doesn't exist as expected at %s", uf, sf)
                    sys.exit(1)

                unsigned_dir = tempfile.mkdtemp()
                signed_dir = tempfile.mkdtemp()
                try:
                    log.info("Unpacking %s into %s", uf, unsigned_dir)
                    unpackfile(uf, unsigned_dir)
                    log.info("Unpacking %s into %s", sf, signed_dir)
                    unpackfile(sf, signed_dir)
                    for f in findfiles(unsigned_dir):
                        # We don't need to cache things that aren't signed
                        if not shouldSign(f):
                            continue
                        # Calculate the hash of the original, unsigned file
                        h = sha1sum(f)
                        sf = signed_dir + f[len(unsigned_dir):]
                        # Cache the signed version
                        log.info("Caching %s as %s" % (f, h))
                        self.rememberFile(h, sf)
                        chk = getChkFile(sf)
                        if chk:
                            log.info("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)
                finally:
                    shutil.rmtree(unsigned_dir)
                    shutil.rmtree(signed_dir)
        localeFinished(firstLocale)

        results = [0, 0, 0]
        # We're going to start our pool of children for processing the other
        # locales, so reset our pool_start time to the current time.
        # We do this because the time to sign the firstLocale isn't
        # representative of the time to sign future locales, due to caching and
        # parallelization.  Using pool_start gives more accurate ETA
        # calculations.
        pool_start = time.time()

        # Setup before signing
        # We define different addLocale functions depending on if we're signing
        # things concurrently or serially
        if self.concurrency > 1:
            # If we're doing signing in parallel, start up our pool of children
            # using the processing module
            import processing.pool
            pool = processing.pool.Pool(self.concurrency)
            result_list = []

            def addLocale(locale):
                def cb(r):
                    if not r:
                        log.error("Signing locale %s failed", locale)
                        return

                    for i in range(3):
                        results[i] += r[i]
                    localeFinished(locale)

                files = locales[locale]
                r = pool.apply_async(
                    _signLocale, (self, dstdir, files), callback=cb)
                result_list.append(r)
        else:
            pool = None

            def addLocale(locale):
                files = locales[locale]
                r = _signLocale(self, dstdir, files)
                if not r:
                    log.error("Signing locale %s failed", locale)
                    sys.exit(1)
                localeFinished(locale)
                for i in range(3):
                    results[i] += r[i]

        # Now go through our locales and call addLocale to start processing
        # them
        for locale in sorted(locales.keys()):
            if locale == firstLocale:
                continue
            addLocale(locale)

        error = False
        if pool:
            # Clean up the pool of child processes afterwards
            for r in result_list:
                try:
                    if not r.get():
                        log.error("Signing locale %s failed", locale)
                        error = True
                        pool.terminate()
                        break
                except:
                    log.exception("Error")
                    pool.terminate()
                    sys.exit(1)
            try:
                pool.close()
                pool.join()
            except:
                log.exception("Error")
                pool.terminate()
                raise

        # Report on our stats
        end = time.time()
        nTotalFiles, cacheHits, nSigned = results
        if nTotalFiles > 0:
            hitrate = cacheHits / float(nTotalFiles) * 100.0
        else:
            hitrate = 0
        if nfiles > 0:
            time_per_file = (end - start) / float(nfiles)
        else:
            time_per_file = 0
        log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
        log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
                 nfiles, end - start, time_per_file)

        if error:
            sys.exit(1)