Example #1
0
def updateSymlink(productName, version, stageServer, stageUsername,
                  stageSshKey, target):
    # step 1 check if we have already pushed to mirrors (bug 1083208)
    push_dir = makeReleasesDir(productName, version)
    try:
        # check if the remote dir exists
        run_remote_cmd(['test', '-d', push_dir],
                       server=stageServer,
                       username=stageUsername,
                       sshKey=stageSshKey)
    except CalledProcessError:
        log.error(
            'ERROR: push to mirrors directory, %s, does not exist on %s' %
            (push_dir, stageServer))
        log.error(
            'ERROR: Did you push to mirrors before running post release?')
        raise

    releases_dir = makeReleasesDir(productName)
    run_remote_cmd([
        'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
        dict(rd=releases_dir, version=version, target=target)
    ],
                   server=stageServer,
                   username=stageUsername,
                   sshKey=stageSshKey)
Example #2
0
def downloadRelease(productName,
                    version,
                    buildNumber,
                    stageServer,
                    stageUsername=None,
                    stageSshKey=None,
                    stageUrlPrefix='http://stage.mozilla.org'):

    candidatesDir = makeCandidatesDir(productName, version,
                                      buildNumber).rstrip('/')
    releasesDir = makeReleasesDir(productName, version).rstrip('/')
    commands = [
        'rm -rf %s' % candidatesDir,
        'rm -rf %s' % releasesDir,
        'mkdir -p %s' % candidatesDir,
        'cd %(candidatesDir)s && \
          wget -nv -r -np -nH --cut-dirs=6 -R index.html* \
          -X %(candidatesDir)s/unsigned \
          -X %(candidatesDir)s/contrib* \
          -X %(candidatesDir)s/partner-repacks \
          -X %(candidatesDir)s/win32-EUballot \
          %(stageUrlPrefix)s%(candidatesDir)s/'                                                % \
          (dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)),
        'ln -s %s %s' % (candidatesDir, releasesDir),
    ]

    for command in commands:
        run_remote_cmd(command,
                       server=stageServer,
                       username=stageUsername,
                       sshKey=stageSshKey)
Example #3
0
def downloadRelease(productName, version, buildNumber, stageServer,
                    stageUsername=None, stageSshKey=None,
                    stageUrlPrefix='http://stage.mozilla.org'):

    candidatesDir = makeCandidatesDir(productName, version,
                                      buildNumber).rstrip('/')
    releasesDir = makeReleasesDir(productName, version).rstrip('/')
    commands = [
        'rm -rf %s' % candidatesDir,
        'rm -rf %s' % releasesDir,
        'mkdir -p %s' % candidatesDir,
        'cd %(candidatesDir)s && \
          wget -nv -r -np -nH --cut-dirs=6 -R index.html* \
          -X %(candidatesDir)s/unsigned \
          -X %(candidatesDir)s/contrib* \
          -X %(candidatesDir)s/partner-repacks \
          -X %(candidatesDir)s/win32-EUballot \
          %(stageUrlPrefix)s%(candidatesDir)s/' % \
          (dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)),
        'ln -s %s %s' % (candidatesDir, releasesDir),
    ]

    for command in commands:
        run_remote_cmd(command, server=stageServer, username=stageUsername,
                       sshKey=stageSshKey)
Example #4
0
def pushToMirrors(productName, version, buildNumber, stageServer,
                  stageUsername=None, stageSshKey=None, excludes=None,
                  extra_excludes=None, dryRun=False):
    """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
    appended to DEFAULT_RSYNC_EXCLUDES. """

    source_dir = makeCandidatesDir(productName, version, buildNumber)
    target_dir = makeReleasesDir(productName, version)

    if not excludes:
        excludes = DEFAULT_RSYNC_EXCLUDES
    if extra_excludes:
        excludes += extra_excludes

    # fail/warn if target directory exists depending on dry run mode
    try:
        run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer,
                       username=stageUsername, sshKey=stageSshKey)
    except CalledProcessError:
        if not dryRun:
            raise
        else:
            log.warning('WARN: target directory %s exists', target_dir)

    if not dryRun:
        run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer,
                       username=stageUsername, sshKey=stageSshKey)
        run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer,
                       username=stageUsername, sshKey=stageSshKey)
    rsync_cmd = ['rsync', '-av' ]
    if dryRun:
        rsync_cmd.append('-n')
    run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir],
                   server=stageServer, username=stageUsername,
                   sshKey=stageSshKey)
Example #5
0
def updateSymlink(productName, version, stageServer, stageUsername,
                  stageSshKey, target):
    releases_dir = makeReleasesDir(productName)

    run_remote_cmd([
        'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
        dict(rd=releases_dir, version=version, target=target)],
        server=stageServer, username=stageUsername, sshKey=stageSshKey)
Example #6
0
def pushToMirrors(productName,
                  version,
                  buildNumber,
                  stageServer,
                  stageUsername=None,
                  stageSshKey=None,
                  excludes=None,
                  extra_excludes=None,
                  dryRun=False,
                  overwrite=False):
    """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
    appended to DEFAULT_RSYNC_EXCLUDES. """

    source_dir = makeCandidatesDir(productName, version, buildNumber)
    target_dir = makeReleasesDir(productName, version)

    if not excludes:
        excludes = DEFAULT_RSYNC_EXCLUDES
    if extra_excludes:
        excludes += ['--exclude=%s' % ex for ex in extra_excludes]

    # fail/warn if target directory exists depending on dry run mode
    try:
        run_remote_cmd(['test', '!', '-d', target_dir],
                       server=stageServer,
                       username=stageUsername,
                       sshKey=stageSshKey)
    except CalledProcessError:
        if overwrite:
            log.info(
                'target directory %s exists, but overwriting files as requested'
                % target_dir)
        elif dryRun:
            log.warning('WARN: target directory %s exists', target_dir)
        else:
            raise

    if not dryRun:
        run_remote_cmd(['mkdir', '-p', target_dir],
                       server=stageServer,
                       username=stageUsername,
                       sshKey=stageSshKey)
        run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir],
                       server=stageServer,
                       username=stageUsername,
                       sshKey=stageSshKey)
    rsync_cmd = ['rsync', '-av']
    if dryRun:
        rsync_cmd.append('-n')
    # use hardlinks
    rsync_cmd.append('--link-dest=%s' % source_dir)
    run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir],
                   server=stageServer,
                   username=stageUsername,
                   sshKey=stageSshKey)
Example #7
0
def updateSymlink(productName, version, stageServer, stageUsername,
                  stageSshKey, target):
    # step 1 check if we have already pushed to mirrors (bug 1083208)
    push_dir = makeReleasesDir(productName, version)
    try:
        # check if the remote dir exists
        run_remote_cmd(['test', '-d', push_dir],
                       server=stageServer, username=stageUsername,
                       sshKey=stageSshKey)
    except CalledProcessError:
            log.error('ERROR: push to mirrors directory, %s, does not exist on %s'
                      % (push_dir, stageServer))
            log.error('ERROR: Did you push to mirrors before running post release?')
            raise

    releases_dir = makeReleasesDir(productName)
    run_remote_cmd([
        'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
        dict(rd=releases_dir, version=version, target=target)],
        server=stageServer, username=stageUsername, sshKey=stageSshKey)
Example #8
0
def updateSymlink(productName, version, stageServer, stageUsername,
                  stageSshKey, target):
    releases_dir = makeReleasesDir(productName)

    run_remote_cmd([
        'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
        dict(rd=releases_dir, version=version, target=target)
    ],
                   server=stageServer,
                   username=stageUsername,
                   sshKey=stageSshKey)
Example #9
0
def pushToMirrors(
    productName,
    version,
    buildNumber,
    stageServer,
    stageUsername=None,
    stageSshKey=None,
    excludes=None,
    extra_excludes=None,
    dryRun=False,
    overwrite=False,
):
    """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
    appended to DEFAULT_RSYNC_EXCLUDES. """

    source_dir = makeCandidatesDir(productName, version, buildNumber)
    target_dir = makeReleasesDir(productName, version)

    if not excludes:
        excludes = DEFAULT_RSYNC_EXCLUDES
    if extra_excludes:
        excludes += ["--exclude=%s" % ex for ex in extra_excludes]

    # fail/warn if target directory exists depending on dry run mode
    try:
        run_remote_cmd(["test", "!", "-d", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
    except CalledProcessError:
        if overwrite:
            log.info("target directory %s exists, but overwriting files as requested" % target_dir)
        elif dryRun:
            log.warning("WARN: target directory %s exists", target_dir)
        else:
            raise

    if not dryRun:
        run_remote_cmd(["mkdir", "-p", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
        run_remote_cmd(
            ["chmod", "u=rwx,g=rxs,o=rx", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey
        )
    rsync_cmd = ["rsync", "-av"]
    if dryRun:
        rsync_cmd.append("-n")
    # use hardlinks
    rsync_cmd.append("--link-dest=%s" % source_dir)
    run_remote_cmd(
        rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey
    )
Example #10
0
 def testRemoteAndVersioned(self):
     got = makeReleasesDir('yx', '1.0', protocol='https', server='cee.dee')
     self.assertEquals('https://cee.dee/pub/yx/releases/1.0/', got)
Example #11
0
 def testRemote(self):
     got = makeReleasesDir('yy', protocol='http', server='foo.bar')
     self.assertEquals('http://foo.bar/pub/yy/releases/', got)
Example #12
0
 def testVersioned(self):
     got = makeReleasesDir('aa', '15.1')
     self.assertEquals('/pub/aa/releases/15.1/', got)
 def testRemoteAndVersioned(self):
     got = makeReleasesDir('yx', '1.0', protocol='https', server='cee.dee')
     self.assertEquals(
         'https://cee.dee/pub/mozilla.org/yx/releases/1.0/', got)
 def testVersioned(self):
     got = makeReleasesDir('aa', '15.1')
     self.assertEquals('/pub/mozilla.org/aa/releases/15.1/', got)
Example #15
0
                      extra_excludes=args.extra_excludes,
                      buildNumber=buildNumber,
                      overwrite=args.overwrite)
        if createIndexFiles:
            deleteIndexFiles(stageServer=stageServer,
                             stageUsername=stageUsername,
                             stageSshKey=stageSshKey,
                             cleanup_dir=makeCandidatesDir(
                                 productName, version, buildNumber))

    if 'postrelease' in actions:
        if createIndexFiles:
            deleteIndexFiles(stageServer=stageServer,
                             stageUsername=stageUsername,
                             stageSshKey=stageSshKey,
                             cleanup_dir=makeReleasesDir(productName, version))
        if ftpSymlinkName:
            updateSymlink(stageServer=stageServer,
                          stageUsername=stageUsername,
                          stageSshKey=stageSshKey,
                          productName=productName,
                          version=version,
                          target=ftpSymlinkName)
        if syncPartnerBundles:
            doSyncPartnerBundles(stageServer=stageServer,
                                 stageUsername=stageUsername,
                                 stageSshKey=stageSshKey,
                                 productName=productName,
                                 version=version,
                                 buildNumber=buildNumber)
        if bouncer_aliases and productName != 'xulrunner':
Example #16
0
 def testBaseReleases(self):
     got = makeReleasesDir('bbb')
     self.assertEquals('/pub/bbb/releases/', got)
Example #17
0
                      version=version,
                      extra_excludes=options.extra_excludes,
                      buildNumber=buildNumber,
                      overwrite=options.overwrite)
        if createIndexFiles:
            deleteIndexFiles(stageServer=stageServer,
                             stageUsername=stageUsername,
                             stageSshKey=stageSshKey,
                             cleanup_dir=makeCandidatesDir(productName, version, buildNumber))

    if 'postrelease' in args:
        if createIndexFiles:
            deleteIndexFiles(stageServer=stageServer,
                             stageUsername=stageUsername,
                             stageSshKey=stageSshKey,
                             cleanup_dir=makeReleasesDir(productName, version))
        if ftpSymlinkName:
            updateSymlink(stageServer=stageServer,
                          stageUsername=stageUsername,
                          stageSshKey=stageSshKey,
                          productName=productName,
                          version=version,
                          target=ftpSymlinkName)
        if syncPartnerBundles:
            doSyncPartnerBundles(stageServer=stageServer,
                                 stageUsername=stageUsername,
                                 stageSshKey=stageSshKey,
                                 productName=productName,
                                 version=version,
                                 buildNumber=buildNumber)
            if ftp_platform not in platforms and locale in locales:
                log.info("Removing %s locale from %s platform for %s" % (
                         locale, ftp_platform, v))
                locales.remove(locale)
        # Exclude locales being full checked
        quick_check_locales = [l for l in locales
                               if l not in full_check_locales]
        # Get the intersection of from and to full_check_locales
        this_full_check_locales = [l for l in full_check_locales
                                   if l in locales]

        from_ = makeReleaseRepackUrls(
            product_name, app_name, v, options.platform,
            locale='%locale%', signed=True, exclude_secondary=True
        ).values()[0]
        release_dir = makeReleasesDir(product_name, v, ftp_root='/')
        from_path = "%s%s" % (release_dir, from_)

        if v in partials:
            # Full test for all locales
            # "from" and "to" to be downloaded from the same staging
            # server in dev environment
            if len(locales) > 0:
                log.info("Generating configs for partial update checks for %s" % v)
                uvc.addRelease(release=appVersion, build_id=build_id,
                               locales=locales,
                               patch_types=['complete', 'partial'],
                               from_path=from_path, ftp_server_from=staging_server,
                               ftp_server_to=staging_server,
                               mar_channel_IDs=mar_channel_IDs)
        else:
        build_id = from_["platforms"][ftp_platform]
        mar_channel_IDs = from_.get('mar-channel-ids')

        # Use new build targets for Windows, but only on compatible
        #  versions (42+). See bug 1185456 for additional context.
        if args.platform not in ("win32", "win64") or \
                LooseVersion(fromVersion) < LooseVersion("42.0"):
            update_platform = buildbot2updatePlatforms(args.platform)[0]
        else:
            update_platform = buildbot2updatePlatforms(args.platform)[1]

        path_ = makeReleaseRepackUrls(
            product_name, app_name, fromVersion, args.platform,
            locale='%locale%', signed=True, exclude_secondary=True
        ).values()[0]
        release_dir = makeReleasesDir(stage_product_name, fromVersion, ftp_root='/')
        from_path = "%s%s" % (release_dir, path_)

        # Exclude locales being full checked
        quick_check_locales = [l for l in locales
                               if l not in full_check_locales]
        # Get the intersection of from and to full_check_locales
        this_full_check_locales = [l for l in full_check_locales
                                   if l in locales]

        if fromVersion in partials:
            log.info("Generating configs for partial update checks for %s" %
                     fromVersion)
            uvc.addRelease(release=appVersion, build_id=build_id,
                           locales=locales,
                           patch_types=["complete", "partial"],
            if ftp_platform not in platforms and locale in locales:
                log.info("Removing %s locale from %s platform for %s" % (
                         locale, ftp_platform, v))
                locales.remove(locale)
        # Exclude locales being full checked
        quick_check_locales = [l for l in locales
                               if l not in full_check_locales]
        # Get the intersection of from and to full_check_locales
        this_full_check_locales = [l for l in full_check_locales
                                   if l in locales]

        from_ = makeReleaseRepackUrls(
            product_name, app_name, v, options.platform,
            locale='%locale%', signed=True, exclude_secondary=True
        ).values()[0]
        release_dir = makeReleasesDir(product_name, v, ftp_root='/')
        from_path = "%s%s" % (release_dir, from_)

        if v in partials:
            # Full test for all locales
            # "from" and "to" to be downloaded from the same staging
            # server in dev environment
            if len(locales) > 0:
                log.info("Generating configs for partial update checks for %s" % v)
                uvc.addRelease(release=appVersion, build_id=build_id,
                               locales=locales,
                               patch_types=['complete', 'partial'],
                               from_path=from_path, ftp_server_from=staging_server,
                               ftp_server_to=staging_server,
                               mar_channel_IDs=mar_channel_IDs)
        else:
 def testRemote(self):
     got = makeReleasesDir('yy', protocol='http', server='foo.bar')
     self.assertEquals('http://foo.bar/pub/mozilla.org/yy/releases/', got)
        # See bug 1185456 for additional context.
        if options.platform not in (
                "win32",
                "win64") or LooseVersion(fromVersion) < LooseVersion("42.0"):
            update_platform = buildbot2updatePlatforms(options.platform)[0]
        else:
            update_platform = buildbot2updatePlatforms(options.platform)[1]

        path_ = makeReleaseRepackUrls(product_name,
                                      app_name,
                                      fromVersion,
                                      options.platform,
                                      locale='%locale%',
                                      signed=True,
                                      exclude_secondary=True).values()[0]
        release_dir = makeReleasesDir(product_name, fromVersion, ftp_root='/')
        from_path = "%s%s" % (release_dir, path_)

        # Exclude locales being full checked
        quick_check_locales = [
            l for l in locales if l not in full_check_locales
        ]
        # Get the intersection of from and to full_check_locales
        this_full_check_locales = [
            l for l in full_check_locales if l in locales
        ]

        if fromVersion in partials:
            log.info("Generating configs for partial update checks for %s" %
                     fromVersion)
            uvc.addRelease(release=appVersion,
 def testBaseReleases(self):
     got = makeReleasesDir('bbb')
     self.assertEquals('/pub/mozilla.org/bbb/releases/', got)
        # See bug 1185456 for additional context.
        if options.platform not in ("win32", "win64") or LooseVersion(fromVersion) < LooseVersion("42.0"):
            update_platform = buildbot2updatePlatforms(options.platform)[0]
        else:
            update_platform = buildbot2updatePlatforms(options.platform)[1]

        path_ = makeReleaseRepackUrls(
            product_name,
            app_name,
            fromVersion,
            options.platform,
            locale="%locale%",
            signed=True,
            exclude_secondary=True,
        ).values()[0]
        release_dir = makeReleasesDir(product_name, fromVersion, ftp_root="/")
        from_path = "%s%s" % (release_dir, path_)

        # Exclude locales being full checked
        quick_check_locales = [l for l in locales if l not in full_check_locales]
        # Get the intersection of from and to full_check_locales
        this_full_check_locales = [l for l in full_check_locales if l in locales]

        if fromVersion in partials:
            log.info("Generating configs for partial update checks for %s" % fromVersion)
            uvc.addRelease(
                release=appVersion,
                build_id=build_id,
                locales=locales,
                patch_types=["complete", "partial"],
                from_path=from_path,