def testMultipleFiles(self):
        expected = {'a': {'size': 1, 'hashes': {'sha512': 'd'}},
                    'b': {'size': 2, 'hashes': {'sha512': 'e'}}}
        got = parseChecksumsFile("""\
d sha512 1 a
e sha512 2 b
""")
        self.assertEquals(got, expected)
    def testMultipleHashes(self):
        expected = {'yyy': {'size': 5, 'hashes': {'sha512': 'aaa',
                                                  'md5': 'bbb'}}}
        got = parseChecksumsFile("""\
aaa sha512 5 yyy
bbb md5 5 yyy
""")
        self.assertEquals(got, expected)
 def getChecksum(platform, locale, checksumsFile):
     if checksums.get(platform, {}).get(locale):
         log.debug(
             "Using in-memory checksums for %s %s" % (platform, locale))
     else:
         try:
             cacheChecksums(platform, locale,
                            parseChecksumsFile(open(checksumsFile).read()))
             log.debug(
                 "Using on-disk checksums for %s %s" % (platform, locale))
         except (IOError, ValueError):
             contents = requests.get(
                 checksumsUrl, config={'danger_mode': True}).content
             log.debug("Using newly downloaded checksums for %s %s" %
                       (platform, locale))
             # Cache the sums in-memory and on-disk.
             cacheChecksums(platform, locale, parseChecksumsFile(contents))
             with open(checksumsFile, 'w') as f:
                 f.write(contents)
     return checksums[platform][locale]
    def testMultipleHashes(self):
        expected = {
            'yyy': {
                'size': 5,
                'hashes': {
                    'sha512': 'aaa',
                    'md5': 'bbb'
                }
            }
        }
        got = parseChecksumsFile("""\
aaa sha512 5 yyy
bbb md5 5 yyy
""")
        self.assertEquals(got, expected)
    def testMultipleFiles(self):
        expected = {
            'a': {
                'size': 1,
                'hashes': {
                    'sha512': 'd'
                }
            },
            'b': {
                'size': 2,
                'hashes': {
                    'sha512': 'e'
                }
            }
        }
        got = parseChecksumsFile("""\
d sha512 1 a
e sha512 2 b
""")
        self.assertEquals(got, expected)
def createRepacks(sourceRepo,
                  revision,
                  l10nRepoDir,
                  l10nBaseRepo,
                  mozconfigPath,
                  srcMozconfigPath,
                  objdir,
                  makeDirs,
                  appName,
                  locales,
                  product,
                  version,
                  buildNumber,
                  stageServer,
                  stageUsername,
                  stageSshKey,
                  ftpServer,
                  compareLocalesRepo,
                  merge,
                  platform,
                  brand,
                  appVersion,
                  generatePartials=False,
                  partialUpdates=None,
                  usePymake=False,
                  tooltoolManifest=None,
                  tooltool_script=None,
                  tooltool_urls=None,
                  balrog_submitter=None,
                  balrog_hash="sha512",
                  mozillaDir=None,
                  mozillaSrcDir=None,
                  bucket_prefix=None):
    buildid = retry(getBuildID,
                    args=(platform, product, version, buildNumber,
                          'candidates', ftpServer))
    log.info('Got buildid: %s' % buildid)
    sourceRepoName = path.split(sourceRepo)[-1]
    absObjdir = path.abspath(path.join(sourceRepoName, objdir))
    localeSrcDir = path.join(absObjdir, appName, "locales")
    # Even on Windows we need to use "/" as a separator for this because
    # compare-locales doesn"t work any other way
    l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
    env = {
        "MOZ_OBJDIR": objdir,
        "MOZ_MAKE_COMPLETE_MAR": "1",
        "DOWNLOAD_HOST": ftpServer,
        "UPLOAD_HOST": stageServer,
        "UPLOAD_USER": stageUsername,
        "UPLOAD_SSH_KEY": stageSshKey,
        "UPLOAD_TO_TEMP": "1",
        "MOZ_PKG_PRETTYNAMES": "1",
        "MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
        "COMM_REV": os.getenv('COMM_REV', ''),
        "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
        "MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
    }
    if appVersion is None or version != appVersion:
        env["MOZ_PKG_VERSION"] = version
    signed = False
    if os.environ.get('MOZ_SIGN_CMD'):
        env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
        signed = True
    env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
        to_candidates=True,
        product=product,
        version=version,
        buildNumber=buildNumber,
        signed=signed,
        bucket_prefix=bucket_prefix,
    )
    if usePymake:
        env['USE_PYMAKE'] = "1"
        env['MOZILLA_OFFICIAL'] = "1"
        env["MOZ_SIGN_CMD"] = "python " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
            " --cachedir " + \
            path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
            " -t " + \
            path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
            " -n " + \
            path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
            " -c " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
        signingServers = os.environ["MOZ_SIGN_CMD"].split("-H",
                                                          1)[1].split("-H")
        for s in signingServers:
            env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
    build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
    mercurial(sourceRepo, sourceRepoName)
    update(sourceRepoName, revision=revision)
    l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
                   l10nRepoDir, makeDirs, env, tooltoolManifest,
                   tooltool_script, tooltool_urls)
    input_env = retry(downloadReleaseBuilds,
                      args=(ftpServer, product, brand, version, buildNumber,
                            platform),
                      kwargs={
                          'signed': signed,
                          'usePymake': usePymake
                      })
    env.update(input_env)

    if product == "thunderbird" and platform == "macosx64":
        # TODO: FIXME: HACK: KILLME:
        # Terrible, terrible, terrible hack to work around bug 1234935 and make
        # the build system happier
        absMozillaSrcDir = path.abspath(
            path.join(sourceRepoName, mozillaSrcDir))
        run_cmd(['ln', '-sf', '../obj-l10n', absMozillaSrcDir])

    failed = []
    for l in locales:
        try:
            if generatePartials:
                for oldVersion in partialUpdates:
                    oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
                    partialUpdates[oldVersion]['mar'] = retry(
                        downloadUpdateIgnore404,
                        args=(ftpServer, product, oldVersion, oldBuildNumber,
                              platform, l))
            checksums_file = repackLocale(
                locale=l,
                l10nRepoDir=l10nRepoDir,
                l10nBaseRepo=l10nBaseRepo,
                revision=revision,
                localeSrcDir=localeSrcDir,
                l10nIni=l10nIni,
                compareLocalesRepo=compareLocalesRepo,
                env=env,
                absObjdir=absObjdir,
                merge=merge,
                productName=product,
                platform=platform,
                version=version,
                partialUpdates=partialUpdates,
                buildNumber=buildNumber,
                stageServer=ftpServer,
                mozillaDir=mozillaDir,
                mozillaSrcDir=mozillaSrcDir)

            if balrog_submitter:
                # TODO: partials, after bug 797033 is fixed
                checksums = parseChecksumsFile(open(checksums_file).read())
                completeInfo = []
                partialInfo = []
                for f, info in checksums.iteritems():
                    if f.endswith('.complete.mar'):
                        completeInfo.append({
                            "size":
                            info["size"],
                            "hash":
                            info["hashes"][balrog_hash],
                        })
                    if f.endswith('.partial.mar'):
                        pathInfo = fileInfo(f, product.lower())
                        previousVersion = pathInfo["previousVersion"]
                        partialInfo.append({
                            "previousVersion":
                            previousVersion,
                            "previousBuildNumber":
                            partialUpdates[previousVersion]['buildNumber'],
                            "size":
                            info["size"],
                            "hash":
                            info["hashes"][balrog_hash],
                        })
                if not completeInfo:
                    raise Exception("Couldn't find complete mar info")
                retry(balrog_submitter.run,
                      kwargs={
                          'platform': platform,
                          'productName': product.capitalize(),
                          'appVersion': appVersion,
                          'version': version,
                          'build_number': buildNumber,
                          'locale': l,
                          'hashFunction': balrog_hash,
                          'extVersion': appVersion,
                          'buildID': buildid,
                          'completeInfo': completeInfo,
                          'partialInfo': partialInfo,
                      })
        except Exception, e:
            print_exc()
            failed.append((l, format_exc()))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
                  mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
                  locales, product, version, buildNumber,
                  stageServer, stageUsername, stageSshKey, ftpServer,
                  compareLocalesRepo, merge, platform, brand, appVersion,
                  generatePartials=False, partialUpdates=None,
                  usePymake=False, tooltoolManifest=None,
                  tooltool_script=None, tooltool_urls=None,
                  balrog_submitter=None, balrog_hash="sha512",
                  mozillaDir=None, mozillaSrcDir=None, bucket_prefix=None):
    buildid = retry(getBuildID, args=(platform, product, version,
                                      buildNumber, 'candidates', ftpServer))
    log.info('Got buildid: %s' % buildid)
    sourceRepoName = path.split(sourceRepo)[-1]
    absObjdir = path.abspath(path.join(sourceRepoName, objdir))
    localeSrcDir = path.join(absObjdir, appName, "locales")
    # Even on Windows we need to use "/" as a separator for this because
    # compare-locales doesn"t work any other way
    l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
    env = {
        "MOZ_OBJDIR": objdir,
        "MOZ_MAKE_COMPLETE_MAR": "1",
        "DOWNLOAD_HOST": ftpServer,
        "UPLOAD_HOST": stageServer,
        "UPLOAD_USER": stageUsername,
        "UPLOAD_SSH_KEY": stageSshKey,
        "UPLOAD_TO_TEMP": "1",
        "MOZ_PKG_PRETTYNAMES": "1",
        "MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
        "COMM_REV": os.getenv('COMM_REV', ''),
        "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
        "MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
    }
    if appVersion is None or version != appVersion:
        env["MOZ_PKG_VERSION"] = version
    signed = False
    if os.environ.get('MOZ_SIGN_CMD'):
        env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
        signed = True
    env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
        to_candidates=True,
        product=product,
        version=version,
        buildNumber=buildNumber,
        signed=signed,
        bucket_prefix=bucket_prefix,
    )
    if usePymake:
        env['USE_PYMAKE'] = "1"
        env['MOZILLA_OFFICIAL'] = "1"
        env["MOZ_SIGN_CMD"] = "python " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
            " --cachedir " + \
            path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
            " -t " + \
            path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
            " -n " + \
            path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
            " -c " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
        signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H")
        for s in signingServers:
            env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
    build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
    mercurial(sourceRepo, sourceRepoName)
    update(sourceRepoName, revision=revision)
    l10nRepackPrep(
        sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
        makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls)
    input_env = retry(downloadReleaseBuilds,
                      args=(ftpServer, product, brand, version, buildNumber,
                            platform),
                      kwargs={'signed': signed,
                              'usePymake': usePymake})
    env.update(input_env)

    failed = []
    for l in locales:
        try:
            if generatePartials:
                for oldVersion in partialUpdates:
                    oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
                    partialUpdates[oldVersion]['mar'] = retry(
                        downloadUpdateIgnore404,
                        args=(ftpServer, product, oldVersion, oldBuildNumber,
                              platform, l)
                    )
            checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
                                          l10nBaseRepo=l10nBaseRepo, revision=revision,
                                          localeSrcDir=localeSrcDir, l10nIni=l10nIni,
                                          compareLocalesRepo=compareLocalesRepo, env=env,
                                          absObjdir=absObjdir, merge=merge,
                                          productName=product, platform=platform,
                                          version=version, partialUpdates=partialUpdates,
                                          buildNumber=buildNumber, stageServer=ftpServer,
                                          mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir)

            if balrog_submitter:
                # TODO: partials, after bug 797033 is fixed
                checksums = parseChecksumsFile(open(checksums_file).read())
                completeInfo = []
                partialInfo = []
                for f, info in checksums.iteritems():
                    if f.endswith('.complete.mar'):
                        completeInfo.append({
                            "size": info["size"],
                            "hash": info["hashes"][balrog_hash],
                        })
                    if f.endswith('.partial.mar'):
                        pathInfo = fileInfo(f, product.lower())
                        previousVersion = pathInfo["previousVersion"]
                        partialInfo.append({
                            "previousVersion": previousVersion,
                            "previousBuildNumber": partialUpdates[previousVersion]['buildNumber'],
                            "size": info["size"],
                            "hash": info["hashes"][balrog_hash],
                        })
                if not completeInfo:
                    raise Exception("Couldn't find complete mar info")
                retry(balrog_submitter.run,
                    kwargs={
                        'platform': platform,
                        'productName': product.capitalize(),
                        'appVersion': appVersion,
                        'version': version,
                        'build_number': buildNumber,
                        'locale': l,
                        'hashFunction': balrog_hash,
                        'extVersion': appVersion,
                        'buildID': buildid,
                        'completeInfo': completeInfo,
                        'partialInfo': partialInfo,
                    }
                )
        except Exception, e:
            print_exc()
            failed.append((l, format_exc()))
 def testSimple(self):
     expected = {'xxxxx': {'size': 1, 'hashes': {'sha512': 'aaa'}}}
     got = parseChecksumsFile("aaa sha512 1 xxxxx")
     self.assertEquals(got, expected)
    def testSpacesInFilename(self):
        expected = {'b d': {'size': 2, 'hashes': {'sha512': 'g'}}}
        got = parseChecksumsFile("""\
g sha512 2 b d
""")
        self.assertEquals(got, expected)
def createRepacks(sourceRepo,
                  revision,
                  l10nRepoDir,
                  l10nBaseRepo,
                  mozconfigPath,
                  srcMozconfigPath,
                  objdir,
                  makeDirs,
                  appName,
                  locales,
                  product,
                  version,
                  buildNumber,
                  stageServer,
                  stageUsername,
                  stageSshKey,
                  compareLocalesRepo,
                  merge,
                  platform,
                  brand,
                  appVersion,
                  generatePartials=False,
                  partialUpdates=None,
                  usePymake=False,
                  tooltoolManifest=None,
                  tooltool_script=None,
                  tooltool_urls=None,
                  balrog_submitter=None,
                  balrog_hash="sha512",
                  buildid=None):
    sourceRepoName = path.split(sourceRepo)[-1]
    absObjdir = path.abspath(path.join(sourceRepoName, objdir))
    localeSrcDir = path.join(absObjdir, appName, "locales")
    # Even on Windows we need to use "/" as a separator for this because
    # compare-locales doesn"t work any other way
    l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])

    env = {
        "MOZ_OBJDIR": objdir,
        "MOZ_MAKE_COMPLETE_MAR": "1",
        "UPLOAD_HOST": stageServer,
        "UPLOAD_USER": stageUsername,
        "UPLOAD_SSH_KEY": stageSshKey,
        "UPLOAD_TO_TEMP": "1",
        "MOZ_PKG_PRETTYNAMES": "1",
        "MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
        "COMM_REV": os.getenv('COMM_REV', ''),
        "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "")
    }
    if appVersion is None or version != appVersion:
        env["MOZ_PKG_VERSION"] = version
    signed = False
    if os.environ.get('MOZ_SIGN_CMD'):
        env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
        signed = True
    env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
        to_candidates=True,
        product=product,
        version=version,
        buildNumber=buildNumber,
        signed=signed,
    )
    if usePymake:
        env['USE_PYMAKE'] = "1"
        env['MOZILLA_OFFICIAL'] = "1"
        env["MOZ_SIGN_CMD"] = "python " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
            " --cachedir " + \
            path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
            " -t " + \
            path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
            " -n " + \
            path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
            " -c " + \
            path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') + \
            " -H " + \
            os.environ['MOZ_SIGN_CMD'].split(' ')[-1]
    build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
    retry(mercurial, args=(sourceRepo, sourceRepoName))
    update(sourceRepoName, revision=revision)
    l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
                   l10nRepoDir, makeDirs, env, tooltoolManifest,
                   tooltool_script, tooltool_urls)
    input_env = retry(downloadReleaseBuilds,
                      args=(stageServer, product, brand, version, buildNumber,
                            platform),
                      kwargs={
                          'signed': signed,
                          'usePymake': usePymake
                      })
    env.update(input_env)

    failed = []
    for l in locales:
        try:
            if generatePartials:
                for oldVersion in partialUpdates:
                    oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
                    partialUpdates[oldVersion]['mar'] = retry(
                        downloadUpdateIgnore404,
                        args=(stageServer, product, oldVersion, oldBuildNumber,
                              platform, l))
            checksums_file = repackLocale(
                locale=l,
                l10nRepoDir=l10nRepoDir,
                l10nBaseRepo=l10nBaseRepo,
                revision=revision,
                localeSrcDir=localeSrcDir,
                l10nIni=l10nIni,
                compareLocalesRepo=compareLocalesRepo,
                env=env,
                absObjdir=absObjdir,
                merge=merge,
                productName=product,
                platform=platform,
                version=version,
                partialUpdates=partialUpdates,
                buildNumber=buildNumber,
                stageServer=stageServer)

            if balrog_submitter:
                # TODO: partials, after bug 797033 is fixed
                checksums = parseChecksumsFile(open(checksums_file).read())
                marInfo = defaultdict(dict)
                for f, info in checksums.iteritems():
                    if f.endswith('.complete.mar'):
                        marInfo['complete']['hash'] = info['hashes'][
                            balrog_hash]
                        marInfo['complete']['size'] = info['size']
                if not marInfo['complete']:
                    raise Exception("Couldn't find complete mar info")
                retry(balrog_submitter.run,
                      kwargs={
                          'platform': platform,
                          'productName': product.capitalize(),
                          'appVersion': appVersion,
                          'version': version,
                          'build_number': buildNumber,
                          'locale': l,
                          'hashFunction': balrog_hash,
                          'extVersion': appVersion,
                          'buildID': buildid,
                          'completeMarSize': marInfo['complete']['size'],
                          'completeMarHash': marInfo['complete']['hash'],
                      })
        except Exception, e:
            print_exc()
            failed.append((l, format_exc()))
 def testSimple(self):
     expected = {'xxxxx': {'size': 1, 'hashes': {'sha512': 'aaa'}}}
     got = parseChecksumsFile("aaa sha512 1 xxxxx")
     self.assertEquals(got, expected)
    def testSpacesInFilename(self):
        expected = {'b d': {'size': 2, 'hashes': {'sha512': 'g'}}}
        got = parseChecksumsFile("""\
g sha512 2 b d
""")
        self.assertEquals(got, expected)