Example #1
0
    def signFiles(self,
                  files,
                  dstdir,
                  product,
                  firstLocale='en-US',
                  firstLocaleSigned=False):
        """Sign `files`, putting the results into `dstdir`.  If `files` has a
        single entry, and refers to a directory, then all files under that
        directory are signed.

        `product` is the product name, e.g. 'firefox', and is used to filename
        pattern matching

        `firstLocale` is the first locale that should be signed, and will have
        the results cached.  Other locales will use the cached versions of
        these signed files where appropriate.

        If `firstLocaleSigned` is True, then all `firstLocale` files must
        already be signed.  The unsigned and signed copies of `firstLocale`
        will be unpacked, and the results cached so that other locales will use
        the signed copies of `firstLocale` files where appropriate.
        """
        start = time.time()

        if len(files) == 1 and os.path.isdir(files[0]):
            files = findfiles(files[0])

        files = sortFiles(filterFiles(files, product), product, firstLocale)
        nfiles = len(files)

        # Split the files up into locales
        locales = {}
        for f in files:
            info = fileInfo(f, product)
            locale = info['locale']
            if not locale in locales:
                locales[locale] = []
            locales[locale].append(f)

        # This is required to be global because localeFinished is called via a
        # callback when child processes finish signing locales, and the
        # callback doesn't have access to this local scope
        global doneLocales
        doneLocales = 0

        # pool_start records when the pool of children has started processing
        # files.  We initialize it to start here because we haven't started the
        # pool yet!
        pool_start = start

        # This is called when we finish signing a locale.  We update some
        # stats, and report on our progress
        def localeFinished(locale):
            global doneLocales
            doneLocales += 1
            now = int(time.time())
            n = len(locales)
            t_per_locale = (now - pool_start) / doneLocales
            remaining = (n - doneLocales) * t_per_locale
            eta_s = remaining % 60
            eta_m = (remaining / 60) % 60
            eta_h = (remaining / 3600)
            percent = 100.0 * doneLocales / float(n)
            eta_time = time.strftime("%H:%M:%S",
                                     time.localtime(int(now + remaining)))
            log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
                     doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)

        # Sign the firstLocale first.  If we don't have any firstLocale files,
        # then something is wrong!
        if firstLocale not in locales:
            log.error("No files found with locale %s", firstLocale)
            sys.exit(1)

        if not firstLocaleSigned:
            if not _signLocale(
                    self, dstdir, locales[firstLocale], remember=True):
                log.error("Error signing %s", firstLocale)
                sys.exit(1)
        else:
            # If the first locale is already signed, we need to unpack both the
            # signed and unsigned copies, and then make sure the signed
            # versions are cached
            for uf in locales[firstLocale]:
                sf = convertPath(uf, dstdir)
                if not os.path.exists(sf):
                    log.error(
                        "Signed version of %s doesn't exist as expected at %s",
                        uf, sf)
                    sys.exit(1)

                unsigned_dir = tempfile.mkdtemp()
                signed_dir = tempfile.mkdtemp()
                try:
                    log.info("Unpacking %s into %s", uf, unsigned_dir)
                    unpackfile(uf, unsigned_dir)
                    log.info("Unpacking %s into %s", sf, signed_dir)
                    unpackfile(sf, signed_dir)
                    for f in findfiles(unsigned_dir):
                        # We don't need to cache things that aren't signed
                        if not shouldSign(f):
                            continue
                        # Calculate the hash of the original, unsigned file
                        h = sha1sum(f)
                        sf = signed_dir + f[len(unsigned_dir):]
                        # Cache the signed version
                        log.info("Caching %s as %s" % (f, h))
                        self.rememberFile(h, sf)
                        chk = getChkFile(sf)
                        if chk:
                            log.info("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)
                finally:
                    shutil.rmtree(unsigned_dir)
                    shutil.rmtree(signed_dir)
        localeFinished(firstLocale)

        results = [0, 0, 0]
        # We're going to start our pool of children for processing the other
        # locales, so reset our pool_start time to the current time.
        # We do this because the time to sign the firstLocale isn't
        # representative of the time to sign future locales, due to caching and
        # parallelization.  Using pool_start gives more accurate ETA
        # calculations.
        pool_start = time.time()

        # Setup before signing
        # We define different addLocale functions depending on if we're signing
        # things concurrently or serially
        if self.concurrency > 1:
            # If we're doing signing in parallel, start up our pool of children
            # using the processing module
            import processing.pool
            pool = processing.pool.Pool(self.concurrency)
            result_list = []

            def addLocale(locale):
                def cb(r):
                    if not r:
                        log.error("Signing locale %s failed", locale)
                        return

                    for i in range(3):
                        results[i] += r[i]
                    localeFinished(locale)

                files = locales[locale]
                r = pool.apply_async(_signLocale, (self, dstdir, files),
                                     callback=cb)
                result_list.append(r)
        else:
            pool = None

            def addLocale(locale):
                files = locales[locale]
                r = _signLocale(self, dstdir, files)
                if not r:
                    log.error("Signing locale %s failed", locale)
                    sys.exit(1)
                localeFinished(locale)
                for i in range(3):
                    results[i] += r[i]

        # Now go through our locales and call addLocale to start processing
        # them
        for locale in sorted(locales.keys()):
            if locale == firstLocale:
                continue
            addLocale(locale)

        error = False
        if pool:
            # Clean up the pool of child processes afterwards
            for r in result_list:
                try:
                    if not r.get():
                        log.error("Signing locale %s failed", locale)
                        error = True
                        pool.terminate()
                        break
                except:
                    log.exception("Error")
                    pool.terminate()
                    sys.exit(1)
            try:
                pool.close()
                pool.join()
            except:
                log.exception("Error")
                pool.terminate()
                raise

        # Report on our stats
        end = time.time()
        nTotalFiles, cacheHits, nSigned = results
        if nTotalFiles > 0:
            hitrate = cacheHits / float(nTotalFiles) * 100.0
        else:
            hitrate = 0
        if nfiles > 0:
            time_per_file = (end - start) / float(nfiles)
        else:
            time_per_file = 0
        log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
        log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
                 nfiles, end - start, time_per_file)

        if error:
            sys.exit(1)
Example #2
0
    def signFiles(self, files, dstdir, product, firstLocale='en-US', firstLocaleSigned=False):
        """Sign `files`, putting the results into `dstdir`.  If `files` has a
        single entry, and refers to a directory, then all files under that
        directory are signed.

        `product` is the product name, e.g. 'firefox', and is used to filename
        pattern matching

        `firstLocale` is the first locale that should be signed, and will have
        the results cached.  Other locales will use the cached versions of
        these signed files where appropriate.

        If `firstLocaleSigned` is True, then all `firstLocale` files must
        already be signed.  The unsigned and signed copies of `firstLocale`
        will be unpacked, and the results cached so that other locales will use
        the signed copies of `firstLocale` files where appropriate.
        """
        start = time.time()

        if len(files) == 1 and os.path.isdir(files[0]):
            files = findfiles(files[0])

        files = sortFiles(filterFiles(files, product), product, firstLocale)
        nfiles = len(files)

        # Split the files up into locales
        locales = {}
        for f in files:
            info = fileInfo(f, product)
            locale = info['locale']
            if not locale in locales:
                locales[locale] = []
            locales[locale].append(f)

        # This is required to be global because localeFinished is called via a
        # callback when child processes finish signing locales, and the
        # callback doesn't have access to this local scope
        global doneLocales
        doneLocales = 0

        # pool_start records when the pool of children has started processing
        # files.  We initialize it to start here because we haven't started the
        # pool yet!
        pool_start = start

        # This is called when we finish signing a locale.  We update some
        # stats, and report on our progress
        def localeFinished(locale):
            global doneLocales
            doneLocales += 1
            now = int(time.time())
            n = len(locales)
            t_per_locale = (now - pool_start) / doneLocales
            remaining = (n - doneLocales) * t_per_locale
            eta_s = remaining % 60
            eta_m = (remaining / 60) % 60
            eta_h = (remaining / 3600)
            percent = 100.0 * doneLocales / float(n)
            eta_time = time.strftime(
                "%H:%M:%S", time.localtime(int(now + remaining)))
            log.info("%i/%i (%.2f%%) complete, ETA in %i:%02i:%02i at %s",
                     doneLocales, n, percent, eta_h, eta_m, eta_s, eta_time)

        # Sign the firstLocale first.  If we don't have any firstLocale files,
        # then something is wrong!
        if firstLocale not in locales:
            log.error("No files found with locale %s", firstLocale)
            sys.exit(1)

        if not firstLocaleSigned:
            if not _signLocale(self, dstdir, locales[firstLocale], remember=True):
                log.error("Error signing %s", firstLocale)
                sys.exit(1)
        else:
            # If the first locale is already signed, we need to unpack both the
            # signed and unsigned copies, and then make sure the signed
            # versions are cached
            for uf in locales[firstLocale]:
                sf = convertPath(uf, dstdir)
                if not os.path.exists(sf):
                    log.error("Signed version of %s doesn't exist as expected at %s", uf, sf)
                    sys.exit(1)

                unsigned_dir = tempfile.mkdtemp()
                signed_dir = tempfile.mkdtemp()
                try:
                    log.info("Unpacking %s into %s", uf, unsigned_dir)
                    unpackfile(uf, unsigned_dir)
                    log.info("Unpacking %s into %s", sf, signed_dir)
                    unpackfile(sf, signed_dir)
                    for f in findfiles(unsigned_dir):
                        # We don't need to cache things that aren't signed
                        if not shouldSign(f):
                            continue
                        # Calculate the hash of the original, unsigned file
                        h = sha1sum(f)
                        sf = signed_dir + f[len(unsigned_dir):]
                        # Cache the signed version
                        log.info("Caching %s as %s" % (f, h))
                        self.rememberFile(h, sf)
                        chk = getChkFile(sf)
                        if chk:
                            log.info("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)
                finally:
                    shutil.rmtree(unsigned_dir)
                    shutil.rmtree(signed_dir)
        localeFinished(firstLocale)

        results = [0, 0, 0]
        # We're going to start our pool of children for processing the other
        # locales, so reset our pool_start time to the current time.
        # We do this because the time to sign the firstLocale isn't
        # representative of the time to sign future locales, due to caching and
        # parallelization.  Using pool_start gives more accurate ETA
        # calculations.
        pool_start = time.time()

        # Setup before signing
        # We define different addLocale functions depending on if we're signing
        # things concurrently or serially
        if self.concurrency > 1:
            # If we're doing signing in parallel, start up our pool of children
            # using the processing module
            import processing.pool
            pool = processing.pool.Pool(self.concurrency)
            result_list = []

            def addLocale(locale):
                def cb(r):
                    if not r:
                        log.error("Signing locale %s failed", locale)
                        return

                    for i in range(3):
                        results[i] += r[i]
                    localeFinished(locale)

                files = locales[locale]
                r = pool.apply_async(
                    _signLocale, (self, dstdir, files), callback=cb)
                result_list.append(r)
        else:
            pool = None

            def addLocale(locale):
                files = locales[locale]
                r = _signLocale(self, dstdir, files)
                if not r:
                    log.error("Signing locale %s failed", locale)
                    sys.exit(1)
                localeFinished(locale)
                for i in range(3):
                    results[i] += r[i]

        # Now go through our locales and call addLocale to start processing
        # them
        for locale in sorted(locales.keys()):
            if locale == firstLocale:
                continue
            addLocale(locale)

        error = False
        if pool:
            # Clean up the pool of child processes afterwards
            for r in result_list:
                try:
                    if not r.get():
                        log.error("Signing locale %s failed", locale)
                        error = True
                        pool.terminate()
                        break
                except:
                    log.exception("Error")
                    pool.terminate()
                    sys.exit(1)
            try:
                pool.close()
                pool.join()
            except:
                log.exception("Error")
                pool.terminate()
                raise

        # Report on our stats
        end = time.time()
        nTotalFiles, cacheHits, nSigned = results
        if nTotalFiles > 0:
            hitrate = cacheHits / float(nTotalFiles) * 100.0
        else:
            hitrate = 0
        if nfiles > 0:
            time_per_file = (end - start) / float(nfiles)
        else:
            time_per_file = 0
        log.info("%.2f%% hit rate, %i files signed", hitrate, nSigned)
        log.info("%s files repacked in %.0f seconds (%.2f seconds per file)",
                 nfiles, end - start, time_per_file)

        if error:
            sys.exit(1)
Example #3
0
    def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
        """Sign `pkgfile`, putting the results into `dstdir`.

        If `remember` is True, then cache the newly signed files into our
        cache.

        If `compressed` is True, then the contents of pkgfile are bz2
        compressed (e.g. in a mar file), and should be decompressed before
        signing.
        """
        log.info("Processing %s", pkgfile)
        basename = os.path.basename(pkgfile)
        dstfile = convertPath(pkgfile, dstdir)

        # Keep track of our output in a list here, and we can output everything
        # when we're done This is to avoid interleaving the output from
        # multiple processes.
        logs = []
        logs.append("Repacking %s to %s" % (pkgfile, dstfile))
        parentdir = os.path.dirname(dstfile)
        if not os.path.exists(parentdir):
            os.makedirs(parentdir, 0755)

        nFiles = 0
        cacheHits = 0
        nSigned = 0
        tmpdir = tempfile.mkdtemp()
        try:
            # Unpack it
            logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
            unpackfile(pkgfile, tmpdir)
            # Swap in files we have already signed
            for f in findfiles(tmpdir):
                # We don't need to do anything to files we're not going to sign
                if not shouldSign(f):
                    continue

                h = sha1sum(f)
                basename = os.path.basename(f)
                nFiles += 1
                chk = getChkFile(f)

                # Look in the cache for another file with the same original
                # hash
                cachedFile = self.getFile(h, f)
                if cachedFile:
                    cacheHits += 1
                    assert os.path.basename(cachedFile) == basename
                    logs.append("Copying %s from %s" % (basename, cachedFile))
                    # Preserve the original file's mode; don't use the cached mode
                    # We usually process installer .exe's first, and 7z doesn't
                    # preserve the file mode, so the cached copies of the files
                    # are mode 0666.  In the mar files, executables have mode
                    # 0777, so we want to preserve that.
                    copyfile(cachedFile, f, copymode=False)
                    if chk:
                        # If there's a .chk file for this file, copy that out of cache
                        # It's an error if this file doesn't exist in cache
                        cachedChk = self.getFile(h, chk)
                        logs.append("Copying %s from %s" %
                                    (os.path.basename(cachedChk), cachedChk))
                        copyfile(cachedChk, chk, copymode=False)
                else:
                    # We need to sign this file
                    # If this file is compressed, check if we have a cached copy that
                    # is uncompressed
                    if compressed:
                        bunzip2(f)
                        h2 = sha1sum(f)
                        cachedFile = self.getFile(h2, f)
                        if cachedFile:
                            # We have a cached copy of this file that is uncompressed.
                            # So copy it into our dstdir, and recompress it, and
                            # save it for future use.
                            cacheHits += 1
                            assert os.path.basename(cachedFile) == basename
                            logs.append("Copying %s from uncompressed %s" %
                                        (basename, cachedFile))
                            # See note above about not copying the file's mode
                            copyfile(cachedFile, f, copymode=False)
                            bzip2(f)
                            if chk:
                                # If there's a .chk file for this file, copy that out of cache
                                # It's an error if this file doesn't exist in
                                # cache
                                cachedChk = self.getFile(h2, chk)
                                logs.append(
                                    "Copying %s from %s" %
                                    (os.path.basename(cachedChk), cachedChk))
                                copyfile(cachedChk, chk, copymode=False)
                                bzip2(chk)
                            if remember:
                                logs.append("Caching compressed %s as %s" %
                                            (f, h))
                                self.rememberFile(h, f)
                                # Remember any regenerated chk files
                                if chk:
                                    logs.append("Caching %s as %s" % (chk, h))
                                    self.rememberFile(h, chk)
                            continue

                    nSigned += 1
                    logs.append("Signing %s" % f)
                    signfile(f, self.keydir, self.fake)
                    if compressed:
                        bzip2(f)
                        # If we have a chk file, compress that too
                        if chk:
                            bzip2(chk)
                    if remember:
                        logs.append("Caching %s as %s" % (f, h))
                        self.rememberFile(h, f)
                        # Remember any regenerated chk files
                        if chk:
                            logs.append("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)

            # Repack it
            logs.append("Packing %s" % dstfile)
            packfile(dstfile, tmpdir)
            # Sign installer
            if dstfile.endswith('.exe') and not self.unsignedInstallers:
                logs.append("Signing %s" % dstfile)
                signfile(dstfile, self.keydir, self.fake)
            return nFiles, cacheHits, nSigned
        except:
            log.exception("Error signing %s", pkgfile)
            return False
        finally:
            # Clean up after ourselves, and output our logs
            shutil.rmtree(tmpdir)
            log.info("\n  ".join(logs))
Example #4
0
    def signPackage(self, pkgfile, dstdir, remember=False, compressed=False):
        """Sign `pkgfile`, putting the results into `dstdir`.

        If `remember` is True, then cache the newly signed files into our
        cache.

        If `compressed` is True, then the contents of pkgfile are bz2
        compressed (e.g. in a mar file), and should be decompressed before
        signing.
        """
        log.info("Processing %s", pkgfile)
        basename = os.path.basename(pkgfile)
        dstfile = convertPath(pkgfile, dstdir)

        # Keep track of our output in a list here, and we can output everything
        # when we're done This is to avoid interleaving the output from
        # multiple processes.
        logs = []
        logs.append("Repacking %s to %s" % (pkgfile, dstfile))
        parentdir = os.path.dirname(dstfile)
        if not os.path.exists(parentdir):
            os.makedirs(parentdir, 0755)

        nFiles = 0
        cacheHits = 0
        nSigned = 0
        tmpdir = tempfile.mkdtemp()
        try:
            # Unpack it
            logs.append("Unpacking %s to %s" % (pkgfile, tmpdir))
            unpackfile(pkgfile, tmpdir)
            # Swap in files we have already signed
            for f in findfiles(tmpdir):
                # We don't need to do anything to files we're not going to sign
                if not shouldSign(f):
                    continue

                h = sha1sum(f)
                basename = os.path.basename(f)
                nFiles += 1
                chk = getChkFile(f)

                # Look in the cache for another file with the same original
                # hash
                cachedFile = self.getFile(h, f)
                if cachedFile:
                    cacheHits += 1
                    assert os.path.basename(cachedFile) == basename
                    logs.append("Copying %s from %s" % (basename, cachedFile))
                    # Preserve the original file's mode; don't use the cached mode
                    # We usually process installer .exe's first, and 7z doesn't
                    # preserve the file mode, so the cached copies of the files
                    # are mode 0666.  In the mar files, executables have mode
                    # 0777, so we want to preserve that.
                    copyfile(cachedFile, f, copymode=False)
                    if chk:
                        # If there's a .chk file for this file, copy that out of cache
                        # It's an error if this file doesn't exist in cache
                        cachedChk = self.getFile(h, chk)
                        logs.append("Copying %s from %s" %
                                    (os.path.basename(cachedChk), cachedChk))
                        copyfile(cachedChk, chk, copymode=False)
                else:
                    # We need to sign this file
                    # If this file is compressed, check if we have a cached copy that
                    # is uncompressed
                    if compressed:
                        bunzip2(f)
                        h2 = sha1sum(f)
                        cachedFile = self.getFile(h2, f)
                        if cachedFile:
                            # We have a cached copy of this file that is uncompressed.
                            # So copy it into our dstdir, and recompress it, and
                            # save it for future use.
                            cacheHits += 1
                            assert os.path.basename(cachedFile) == basename
                            logs.append("Copying %s from uncompressed %s" %
                                        (basename, cachedFile))
                            # See note above about not copying the file's mode
                            copyfile(cachedFile, f, copymode=False)
                            bzip2(f)
                            if chk:
                                # If there's a .chk file for this file, copy that out of cache
                                # It's an error if this file doesn't exist in
                                # cache
                                cachedChk = self.getFile(h2, chk)
                                logs.append("Copying %s from %s" % (
                                    os.path.basename(cachedChk), cachedChk))
                                copyfile(cachedChk, chk, copymode=False)
                                bzip2(chk)
                            if remember:
                                logs.append(
                                    "Caching compressed %s as %s" % (f, h))
                                self.rememberFile(h, f)
                                # Remember any regenerated chk files
                                if chk:
                                    logs.append("Caching %s as %s" % (chk, h))
                                    self.rememberFile(h, chk)
                            continue

                    nSigned += 1
                    logs.append("Signing %s" % f)
                    signfile(f, self.keydir, self.fake)
                    if compressed:
                        bzip2(f)
                        # If we have a chk file, compress that too
                        if chk:
                            bzip2(chk)
                    if remember:
                        logs.append("Caching %s as %s" % (f, h))
                        self.rememberFile(h, f)
                        # Remember any regenerated chk files
                        if chk:
                            logs.append("Caching %s as %s" % (chk, h))
                            self.rememberFile(h, chk)

            # Repack it
            logs.append("Packing %s" % dstfile)
            packfile(dstfile, tmpdir)
            # Sign installer
            if dstfile.endswith('.exe') and not self.unsignedInstallers:
                logs.append("Signing %s" % dstfile)
                signfile(dstfile, self.keydir, self.fake)
            return nFiles, cacheHits, nSigned
        except:
            log.exception("Error signing %s", pkgfile)
            return False
        finally:
            # Clean up after ourselves, and output our logs
            shutil.rmtree(tmpdir)
            log.info("\n  ".join(logs))