Ejemplo n.º 1
0
def master_stuff(dest_dir):
    """If we're operating on a master branch, do some extra weird stuff."""
    repos = {
        'extensions': 'mediawiki/extensions',
        'vendor': 'mediawiki/vendor',
        'skins': 'mediawiki/skins',
    }

    for dest, upstream in repos.items():
        path = os.path.join(dest_dir, dest)
        url = SOURCE_URL + upstream

        if os.path.exists(path):
            with utils.cd(path):
                subprocess.check_call(
                    ['/usr/bin/git', 'init']
                )
                subprocess.check_call(
                    ['/usr/bin/git', 'remote', 'add', 'origin', url]
                )

        git.fetch(path, url)
        git.checkout(path, 'master')
        git.update_submodules(path, use_upstream=True)
        update_update_strategy(path)
Ejemplo n.º 2
0
    def master_stuff(self):
        repos = {"extensions": "mediawiki/extensions", "vendor": "mediawiki/vendor", "skins": "mediawiki/skins"}

        for dest, upstream in repos.items():
            path = os.path.join(self.dest_dir, dest)
            url = self.gerrit + upstream

            if os.path.exists(path):
                with utils.cd(path):
                    subprocess.check_call(["/usr/bin/git", "init"], shell=True)
                    subprocess.check_call(["/usr/bin/git", "remote", "add", "origin", url], shell=True)

            git.fetch(path, url)
            git.checkout(path, "master")
            git.update_submodules(path, use_upstream=True)
            self.update_submodule_update_strategy(path)
Ejemplo n.º 3
0
    def main(self, *extra_args):
        """ Checkout next MediaWiki """

        self.branch = self.arguments.branch
        self.dest_dir = os.path.join(
            self.config['stage_dir'],
            '{}{}'.format(self.arguments.prefix, self.branch)
        )
        old_branch = self.active_wikiversions().keys()[0]
        copy_dir = os.path.join(
            self.config['stage_dir'],
            '{}{}'.format(self.arguments.prefix, old_branch)
        )

        if os.path.isdir(self.dest_dir):
            self.get_logger().info('Version already checked out')
            return 0

        git.fetch(self.dest_dir, self.gerrit + 'mediawiki/core', copy_dir)

        with utils.cd(self.dest_dir):
            if subprocess.call(['/usr/bin/git', 'config',
                                'branch.autosetuprebase', 'always']) != 0:
                self.get_logger().warn('Unable to setup auto-rebase')

        checkout_version = 'master'
        if self.branch != 'master':
            checkout_version = 'wmf/%s' % self.branch

        git.checkout(self.dest_dir, checkout_version)

        if checkout_version == 'master':
            self.master_stuff()
        else:
            git.update_submodules(self.dest_dir, use_upstream=True)
            self.update_submodule_update_strategy(self.dest_dir)

        self.write_localsettings()
        self.create_startprofiler_symlink()

        cache_dir = os.path.join(self.dest_dir, 'cache')
        os.chmod(cache_dir, 0o777)
        utils.sudo_check_call('l10nupdate',
                              'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))

        self.get_logger().info('MediaWiki %s successfully checked out.' %
                               checkout_version)
Ejemplo n.º 4
0
    def master_stuff(self):
        repos = {
            'extensions': 'mediawiki/extensions',
            'vendor': 'mediawiki/vendor',
            'skins': 'mediawiki/skins',
        }

        for dest, upstream in repos.items():
            path = os.path.join(self.dest_dir, dest)
            url = self.gerrit + upstream

            if os.path.exists(path):
                with utils.cd(path):
                    subprocess.check_call(['/usr/bin/git', 'init'])
                    subprocess.check_call(
                        ['/usr/bin/git', 'remote', 'add', 'origin', url])

            git.fetch(path, url)
            git.checkout(path, 'master')
            git.update_submodules(path, use_upstream=True)
            self.update_submodule_update_strategy(path)
Ejemplo n.º 5
0
def master_stuff(dest_dir):
    """If we're operating on a master branch, do some extra weird stuff."""
    repos = {
        'extensions': 'mediawiki/extensions',
        'vendor': 'mediawiki/vendor',
        'skins': 'mediawiki/skins',
    }

    for dest, upstream in repos.items():
        path = os.path.join(dest_dir, dest)
        url = SOURCE_URL + upstream

        if os.path.exists(path):
            with utils.cd(path):
                subprocess.check_call(['/usr/bin/git', 'init'])
                subprocess.check_call(
                    ['/usr/bin/git', 'remote', 'add', 'origin', url])

        git.fetch(path, url)
        git.checkout(path, 'master')
        git.update_submodules(path, use_upstream=True)
        update_update_strategy(path)
Ejemplo n.º 6
0
    def main(self, *extra_args):
        """ Checkout next MediaWiki """

        self.branch = self.arguments.branch
        self.dest_dir = os.path.join(self.config["stage_dir"], "{}{}".format(self.arguments.prefix, self.branch))

        if os.path.isdir(self.dest_dir):
            self.get_logger().info("Version already checked out")
            return 0

        git.fetch(self.dest_dir, self.gerrit + "mediawiki/core")

        with utils.cd(self.dest_dir):
            if subprocess.call(["/usr/bin/git", "config", "branch.autosetuprebase", "always"]) != 0:
                self.get_logger().warn("Unable to setup auto-rebase")

        checkout_version = "master"
        if self.branch != "master":
            checkout_version = "wmf/%s" % self.branch

        git.checkout(self.dest_dir, checkout_version)

        if checkout_version == "master":
            self.master_stuff()
        else:
            git.update_submodules(self.dest_dir, use_upstream=True)
            self.update_submodule_update_strategy(self.dest_dir)

        self.write_localsettings()
        self.create_startprofiler_symlink()

        cache_dir = os.path.join(self.dest_dir, "cache")
        os.chmod(cache_dir, 0o777)
        utils.sudo_check_call("l10nupdate", 'mkdir "%s"' % os.path.join(cache_dir, "l10n"))

        self.get_logger().info("MediaWiki %s successfully checked out." % checkout_version)
Ejemplo n.º 7
0
    def fetch(self):
        """
        Fetch the specified revision of the remote repo.

        The given repo is cloned into the cache directory and a new working
        directory for the given revision is created under revs/{rev}.

        At the end of this stage, the .in-progress link is created to signal
        the possibility for future rollback.
        """
        logger = self.get_logger()

        has_submodules = self.config['git_submodules']
        git_binary_manager = None
        if self.config['git_fat']:
            logger.warning(
                'Using deprecated git_fat config, swap to git_binary_manager')
            git_binary_manager = [git.FAT]
        elif self.config['git_binary_manager']:
            git_binary_manager = config.multi_value(
                self.config['git_binary_manager'])

        rev_dir = self.context.rev_path(self.rev)

        git_remote = os.path.join(self.server_url, '.git')
        logger.info('Fetch from: {}'.format(git_remote))

        # clone/fetch from the repo to the cache directory
        git.fetch(self.context.cache_dir, git_remote)

        if has_submodules:
            upstream_submodules = self.config['git_upstream_submodules']
            logger.info('Update submodules')
            git.update_submodules(location=self.context.cache_dir,
                                  git_remote=git_remote,
                                  use_upstream=upstream_submodules)

        # If the rev_dir already exists AND the currently checked-out HEAD is
        # already at the revision specified by ``self.rev`` then you can assume
        #
        # 1. If there is a config deploy, the config is inside the rev_dir
        # 2. The code represented by the SHA1 to be deployed is inside
        #    the rev_dir
        #
        # Set the noop flag and return
        if os.path.isdir(rev_dir) and not self.arguments.force:
            rev = git.sha(rev_dir, 'HEAD')
            if rev == self.rev:
                logger.info('Revision directory already exists '
                            '(use --force to override)')
                self.noop = True
                self.context.mark_rev_in_progress(self.rev)
                return

        if not os.path.isdir(rev_dir):
            # if lfs is enabled and this is the first time cloning this repo,
            # then we need to run `git lfs install`` before `git clone`
            if git_binary_manager and git.LFS in git_binary_manager:
                git.lfs_install('--global')

        # clone/fetch from the local cache directory to the revision directory
        git.fetch(rev_dir,
                  self.context.cache_dir,
                  reference=self.context.cache_dir,
                  dissociate=False,
                  recurse_submodules=False)

        logger.info('Checkout rev: {}'.format(self.rev))

        # checkout the given revision
        git.checkout(rev_dir, self.rev)

        if has_submodules:
            git.update_submodules(location=rev_dir,
                                  git_remote=git_remote,
                                  use_upstream=upstream_submodules,
                                  reference=self.context.cache_dir)

        if git_binary_manager:
            for manager in git_binary_manager:
                logger.info("Pulling large objects [using %s]", manager)
                if manager in [git.FAT, git.LFS]:
                    git.largefile_pull(rev_dir, manager)
                else:
                    logger.warning("Passed unrecognized binary manager %s",
                                   manager)

        self.context.mark_rev_in_progress(self.rev)
Ejemplo n.º 8
0
    def main(self, *extra_args):
        """Checkout next MediaWiki."""
        dest_dir = os.path.join(
            self.config['stage_dir'], '{}{}'.format(self.arguments.prefix,
                                                    self.arguments.branch))

        checkout_version = 'master'
        if self.arguments.branch != 'master':
            checkout_version = 'wmf/%s' % self.arguments.branch

        reference_dir = None
        if checkout_version != 'master':
            old_branch = self.active_wikiversions().keys()[0]
            reference_dir = os.path.join(
                self.config['stage_dir'],
                '{}{}'.format(self.arguments.prefix, old_branch))
            patch_path = os.path.join('/srv/patches', self.arguments.branch)
            if not os.path.exists(patch_path):
                shutil.copytree(os.path.join('/srv/patches', old_branch),
                                os.path.join(patch_path))

        if os.path.isdir(dest_dir):
            self.get_logger().info('Version already checked out')
            return 0

        git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)

        with utils.cd(dest_dir):
            if subprocess.call(
                ['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always'
                 ]) != 0:
                self.get_logger().warn('Unable to setup auto-rebase')

            num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
            if subprocess.call([
                    '/usr/bin/git', 'config', 'submodule.fetchJobs', num_procs
            ]) != 0:
                self.get_logger().warn('Unable to setup submodule fetch jobs')

        git.checkout(dest_dir, checkout_version)

        if checkout_version == 'master':
            # Specific to Beta Cluster
            master_stuff(dest_dir)

            write_settings_stub(
                os.path.join(dest_dir, 'StartProfiler.php'),
                os.path.join(self.config['deploy_dir'], 'wmf-config',
                             'StartProfiler-labs.php'))
        else:
            # Specific to production
            git.update_submodules(dest_dir, use_upstream=True)
            update_update_strategy(dest_dir)

            write_settings_stub(
                os.path.join(dest_dir, 'StartProfiler.php'),
                os.path.join(self.config['deploy_dir'], 'wmf-config',
                             'StartProfiler.php'))

        write_settings_stub(
            os.path.join(dest_dir, 'LocalSettings.php'),
            os.path.join(self.config['deploy_dir'], 'wmf-config',
                         'CommonSettings.php'))

        cache_dir = os.path.join(dest_dir, 'cache')
        os.chmod(cache_dir, 0o777)
        utils.sudo_check_call('l10nupdate',
                              'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))

        self.get_logger().info('MediaWiki %s successfully checked out.' %
                               checkout_version)
Ejemplo n.º 9
0
    def fetch(self):
        """
        Fetch the specified revision of the remote repo.

        The given repo is cloned into the cache directory and a new working
        directory for the given revision is created under revs/{rev}.

        At the end of this stage, the .in-progress link is created to signal
        the possibility for future rollback.
        """
        logger = self.get_logger()

        has_submodules = self.config['git_submodules']
        git_binary_manager = None
        if self.config['git_fat']:
            logger.warning(
                'Using deprecated git_fat config, swap to git_binary_manager')
            git_binary_manager = [git.FAT]
        elif self.config['git_binary_manager']:
            git_binary_manager = config.multi_value(
                self.config['git_binary_manager'])

        rev_dir = self.context.rev_path(self.rev)

        git_remote = os.path.join(self.server_url, '.git')
        logger.info('Fetch from: {}'.format(git_remote))

        # clone/fetch from the repo to the cache directory
        git.fetch(self.context.cache_dir, git_remote)

        if has_submodules:
            upstream_submodules = self.config['git_upstream_submodules']
            logger.info('Update submodules')
            git.update_submodules(location=self.context.cache_dir,
                                  git_remote=git_remote,
                                  use_upstream=upstream_submodules)

        # If the rev_dir already exists AND the currently checked-out HEAD is
        # already at the revision specified by ``self.rev`` then you can assume
        #
        # 1. If there is a config deploy, the config is inside the rev_dir
        # 2. The code represented by the SHA1 to be deployed is inside
        #    the rev_dir
        #
        # Set the noop flag and return
        if os.path.isdir(rev_dir) and not self.arguments.force:
            rev = git.sha(rev_dir, 'HEAD')
            if rev == self.rev:
                logger.info(
                    'Revision directory already exists '
                    '(use --force to override)')
                self.noop = True
                self.context.mark_rev_in_progress(self.rev)
                return

        if not os.path.isdir(rev_dir):
            # if lfs is enabled and this is the first time cloning this repo,
            # then we need to run `git lfs install`` before `git clone`
            if git_binary_manager and git.LFS in git_binary_manager:
                git.lfs_install('--global')

        # clone/fetch from the local cache directory to the revision directory
        git.fetch(rev_dir, self.context.cache_dir,
                  reference=self.context.cache_dir,
                  dissociate=False,
                  recurse_submodules=False)

        logger.info('Checkout rev: {}'.format(self.rev))

        # checkout the given revision
        git.checkout(rev_dir, self.rev)

        if has_submodules:
            git.update_submodules(location=rev_dir, git_remote=git_remote,
                                  use_upstream=upstream_submodules,
                                  reference=self.context.cache_dir)

        if git_binary_manager:
            for manager in git_binary_manager:
                logger.info("Pulling large objects [using %s]", manager)
                if manager in [git.FAT, git.LFS]:
                    git.largefile_pull(rev_dir, manager)
                else:
                    logger.warning("Passed unrecognized binary manager %s",
                                   manager)

        self.context.mark_rev_in_progress(self.rev)
Ejemplo n.º 10
0
    def main(self, *extra_args):
        """Checkout next MediaWiki."""
        dest_dir = os.path.join(
            self.config['stage_dir'], '{}{}'.format(self.arguments.prefix,
                                                    self.arguments.branch))

        checkout_version = 'master'
        if self.arguments.branch != 'master':
            checkout_version = 'wmf/%s' % self.arguments.branch

        reference_dir = None
        if checkout_version != 'master':
            # active_wikiversions() is already sorted by loose-version number,
            # we want the latest version if there's more than 1
            old_branch = self.active_wikiversions().keys()[-1]
            old_branch_dir = os.path.join(
                self.config['stage_dir'],
                '{}{}'.format(self.arguments.prefix, old_branch))
            reference_dir = None
            if (os.path.exists(old_branch_dir)):
                reference_dir = old_branch_dir
            patch_base_dir = '/srv/patches'
            patch_path = os.path.join(patch_base_dir, self.arguments.branch)
            if not os.path.exists(patch_path):
                if os.path.exists(os.path.join(patch_base_dir, old_branch)):
                    shutil.copytree(os.path.join(patch_base_dir, old_branch),
                                    os.path.join(patch_path))

                    srv_patches_git_message = 'Scap prep for "{}"'.format(
                        self.arguments.branch)
                    git.add_all(patch_base_dir,
                                message=srv_patches_git_message)

        if os.path.isdir(dest_dir):
            self.get_logger().info('Version already checked out')
            return 0

        self.get_logger().info('Fetching core to {}'.format(dest_dir))
        git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)

        with utils.cd(dest_dir):
            if subprocess.call(
                ['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always'
                 ]) != 0:
                self.get_logger().warn('Unable to setup auto-rebase')

            num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
            if subprocess.call([
                    '/usr/bin/git', 'config', 'submodule.fetchJobs', num_procs
            ]) != 0:
                self.get_logger().warn('Unable to setup submodule fetch jobs')

        self.get_logger().info('Checkout {} in {}'.format(
            checkout_version, dest_dir))
        git.checkout(dest_dir, checkout_version)

        if checkout_version == 'master':
            # Specific to Beta Cluster
            master_stuff(dest_dir)
        else:
            # Specific to production
            self.get_logger().info('Update submodules for {}'.format(dest_dir))
            git.update_submodules(dest_dir, use_upstream=True)
            update_update_strategy(dest_dir)

        self.get_logger().info('Creating LocalSettings.php stub')
        write_settings_stub(os.path.join(dest_dir, 'LocalSettings.php'))

        self.get_logger().info('Creating l10n cache dir')
        cache_dir = os.path.join(dest_dir, 'cache')
        os.chmod(cache_dir, 0o777)
        utils.sudo_check_call('l10nupdate',
                              'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))

        self.get_logger().info('MediaWiki %s successfully checked out.' %
                               checkout_version)
Ejemplo n.º 11
0
    def main(self, *extra_args):
        """Checkout next MediaWiki."""
        dest_dir = os.path.join(
            self.config['stage_dir'],
            '{}{}'.format(self.arguments.prefix, self.arguments.branch)
        )

        checkout_version = 'master'
        if self.arguments.branch != 'master':
            checkout_version = 'wmf/%s' % self.arguments.branch

        reference_dir = None
        if checkout_version != 'master':
            # active_wikiversions() is already sorted by loose-version number,
            # we want the latest version if there's more than 1
            old_branch = self.active_wikiversions().keys()[-1]
            old_branch_dir = os.path.join(
                self.config['stage_dir'],
                '{}{}'.format(self.arguments.prefix, old_branch)
            )
            reference_dir = None
            if (os.path.exists(old_branch_dir)):
                reference_dir = old_branch_dir
            patch_path = os.path.join('/srv/patches', self.arguments.branch)
            if not os.path.exists(patch_path):
                if os.path.exists(os.path.join('/srv/patches', old_branch)):
                    shutil.copytree(
                        os.path.join('/srv/patches', old_branch),
                        os.path.join(patch_path)
                    )

        if os.path.isdir(dest_dir):
            self.get_logger().info('Version already checked out')
            return 0

        self.get_logger().info('Fetching core to {}'.format(dest_dir))
        git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)

        with utils.cd(dest_dir):
            if subprocess.call(['/usr/bin/git', 'config',
                                'branch.autosetuprebase', 'always']) != 0:
                self.get_logger().warn('Unable to setup auto-rebase')

            num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
            if subprocess.call(['/usr/bin/git', 'config',
                                'submodule.fetchJobs', num_procs]) != 0:
                self.get_logger().warn('Unable to setup submodule fetch jobs')

        self.get_logger().info('Checkout {} in {}'.format(checkout_version, dest_dir))
        git.checkout(dest_dir, checkout_version)

        if checkout_version == 'master':
            # Specific to Beta Cluster
            master_stuff(dest_dir)
        else:
            # Specific to production
            self.get_logger().info('Update submodules for {}'.format(dest_dir))
            git.update_submodules(dest_dir, use_upstream=True)
            update_update_strategy(dest_dir)

        self.get_logger().info('Creating LocalSettings.php stub')
        write_settings_stub(os.path.join(dest_dir, 'LocalSettings.php'))

        self.get_logger().info('Creating l10n cache dir')
        cache_dir = os.path.join(dest_dir, 'cache')
        os.chmod(cache_dir, 0o777)
        utils.sudo_check_call('l10nupdate',
                              'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))

        self.get_logger().info('MediaWiki %s successfully checked out.' %
                               checkout_version)