def master_stuff(dest_dir):
"""If we're operating on a master branch, do some extra weird stuff."""
repos = {
'extensions': 'mediawiki/extensions',
'vendor': 'mediawiki/vendor',
'skins': 'mediawiki/skins',
}
for dest, upstream in repos.items():
path = os.path.join(dest_dir, dest)
url = SOURCE_URL + upstream
if os.path.exists(path):
with utils.cd(path):
subprocess.check_call(
['/usr/bin/git', 'init']
)
subprocess.check_call(
['/usr/bin/git', 'remote', 'add', 'origin', url]
)
git.fetch(path, url)
git.checkout(path, 'master')
git.update_submodules(path, use_upstream=True)
update_update_strategy(path)
def master_stuff(self):
repos = {"extensions": "mediawiki/extensions", "vendor": "mediawiki/vendor", "skins": "mediawiki/skins"}
for dest, upstream in repos.items():
path = os.path.join(self.dest_dir, dest)
url = self.gerrit + upstream
if os.path.exists(path):
with utils.cd(path):
subprocess.check_call(["/usr/bin/git", "init"], shell=True)
subprocess.check_call(["/usr/bin/git", "remote", "add", "origin", url], shell=True)
git.fetch(path, url)
git.checkout(path, "master")
git.update_submodules(path, use_upstream=True)
self.update_submodule_update_strategy(path)
def main(self, *extra_args):
""" Checkout next MediaWiki """
self.branch = self.arguments.branch
self.dest_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, self.branch)
)
old_branch = self.active_wikiversions().keys()[0]
copy_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, old_branch)
)
if os.path.isdir(self.dest_dir):
self.get_logger().info('Version already checked out')
return 0
git.fetch(self.dest_dir, self.gerrit + 'mediawiki/core', copy_dir)
with utils.cd(self.dest_dir):
if subprocess.call(['/usr/bin/git', 'config',
'branch.autosetuprebase', 'always']) != 0:
self.get_logger().warn('Unable to setup auto-rebase')
checkout_version = 'master'
if self.branch != 'master':
checkout_version = 'wmf/%s' % self.branch
git.checkout(self.dest_dir, checkout_version)
if checkout_version == 'master':
self.master_stuff()
else:
git.update_submodules(self.dest_dir, use_upstream=True)
self.update_submodule_update_strategy(self.dest_dir)
self.write_localsettings()
self.create_startprofiler_symlink()
cache_dir = os.path.join(self.dest_dir, 'cache')
os.chmod(cache_dir, 0o777)
utils.sudo_check_call('l10nupdate',
'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))
self.get_logger().info('MediaWiki %s successfully checked out.' %
checkout_version)
def master_stuff(self):
repos = {
'extensions': 'mediawiki/extensions',
'vendor': 'mediawiki/vendor',
'skins': 'mediawiki/skins',
}
for dest, upstream in repos.items():
path = os.path.join(self.dest_dir, dest)
url = self.gerrit + upstream
if os.path.exists(path):
with utils.cd(path):
subprocess.check_call(['/usr/bin/git', 'init'])
subprocess.check_call(
['/usr/bin/git', 'remote', 'add', 'origin', url])
git.fetch(path, url)
git.checkout(path, 'master')
git.update_submodules(path, use_upstream=True)
self.update_submodule_update_strategy(path)
def master_stuff(dest_dir):
"""If we're operating on a master branch, do some extra weird stuff."""
repos = {
'extensions': 'mediawiki/extensions',
'vendor': 'mediawiki/vendor',
'skins': 'mediawiki/skins',
}
for dest, upstream in repos.items():
path = os.path.join(dest_dir, dest)
url = SOURCE_URL + upstream
if os.path.exists(path):
with utils.cd(path):
subprocess.check_call(['/usr/bin/git', 'init'])
subprocess.check_call(
['/usr/bin/git', 'remote', 'add', 'origin', url])
git.fetch(path, url)
git.checkout(path, 'master')
git.update_submodules(path, use_upstream=True)
update_update_strategy(path)
def main(self, *extra_args):
""" Checkout next MediaWiki """
self.branch = self.arguments.branch
self.dest_dir = os.path.join(self.config["stage_dir"], "{}{}".format(self.arguments.prefix, self.branch))
if os.path.isdir(self.dest_dir):
self.get_logger().info("Version already checked out")
return 0
git.fetch(self.dest_dir, self.gerrit + "mediawiki/core")
with utils.cd(self.dest_dir):
if subprocess.call(["/usr/bin/git", "config", "branch.autosetuprebase", "always"]) != 0:
self.get_logger().warn("Unable to setup auto-rebase")
checkout_version = "master"
if self.branch != "master":
checkout_version = "wmf/%s" % self.branch
git.checkout(self.dest_dir, checkout_version)
if checkout_version == "master":
self.master_stuff()
else:
git.update_submodules(self.dest_dir, use_upstream=True)
self.update_submodule_update_strategy(self.dest_dir)
self.write_localsettings()
self.create_startprofiler_symlink()
cache_dir = os.path.join(self.dest_dir, "cache")
os.chmod(cache_dir, 0o777)
utils.sudo_check_call("l10nupdate", 'mkdir "%s"' % os.path.join(cache_dir, "l10n"))
self.get_logger().info("MediaWiki %s successfully checked out." % checkout_version)
def fetch(self):
"""
Fetch the specified revision of the remote repo.
The given repo is cloned into the cache directory and a new working
directory for the given revision is created under revs/{rev}.
At the end of this stage, the .in-progress link is created to signal
the possibility for future rollback.
"""
logger = self.get_logger()
has_submodules = self.config['git_submodules']
git_binary_manager = None
if self.config['git_fat']:
logger.warning(
'Using deprecated git_fat config, swap to git_binary_manager')
git_binary_manager = [git.FAT]
elif self.config['git_binary_manager']:
git_binary_manager = config.multi_value(
self.config['git_binary_manager'])
rev_dir = self.context.rev_path(self.rev)
git_remote = os.path.join(self.server_url, '.git')
logger.info('Fetch from: {}'.format(git_remote))
# clone/fetch from the repo to the cache directory
git.fetch(self.context.cache_dir, git_remote)
if has_submodules:
upstream_submodules = self.config['git_upstream_submodules']
logger.info('Update submodules')
git.update_submodules(location=self.context.cache_dir,
git_remote=git_remote,
use_upstream=upstream_submodules)
# If the rev_dir already exists AND the currently checked-out HEAD is
# already at the revision specified by ``self.rev`` then you can assume
#
# 1. If there is a config deploy, the config is inside the rev_dir
# 2. The code represented by the SHA1 to be deployed is inside
# the rev_dir
#
# Set the noop flag and return
if os.path.isdir(rev_dir) and not self.arguments.force:
rev = git.sha(rev_dir, 'HEAD')
if rev == self.rev:
logger.info('Revision directory already exists '
'(use --force to override)')
self.noop = True
self.context.mark_rev_in_progress(self.rev)
return
if not os.path.isdir(rev_dir):
# if lfs is enabled and this is the first time cloning this repo,
# then we need to run `git lfs install`` before `git clone`
if git_binary_manager and git.LFS in git_binary_manager:
git.lfs_install('--global')
# clone/fetch from the local cache directory to the revision directory
git.fetch(rev_dir,
self.context.cache_dir,
reference=self.context.cache_dir,
dissociate=False,
recurse_submodules=False)
logger.info('Checkout rev: {}'.format(self.rev))
# checkout the given revision
git.checkout(rev_dir, self.rev)
if has_submodules:
git.update_submodules(location=rev_dir,
git_remote=git_remote,
use_upstream=upstream_submodules,
reference=self.context.cache_dir)
if git_binary_manager:
for manager in git_binary_manager:
logger.info("Pulling large objects [using %s]", manager)
if manager in [git.FAT, git.LFS]:
git.largefile_pull(rev_dir, manager)
else:
logger.warning("Passed unrecognized binary manager %s",
manager)
self.context.mark_rev_in_progress(self.rev)
def main(self, *extra_args):
"""Checkout next MediaWiki."""
dest_dir = os.path.join(
self.config['stage_dir'], '{}{}'.format(self.arguments.prefix,
self.arguments.branch))
checkout_version = 'master'
if self.arguments.branch != 'master':
checkout_version = 'wmf/%s' % self.arguments.branch
reference_dir = None
if checkout_version != 'master':
old_branch = self.active_wikiversions().keys()[0]
reference_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, old_branch))
patch_path = os.path.join('/srv/patches', self.arguments.branch)
if not os.path.exists(patch_path):
shutil.copytree(os.path.join('/srv/patches', old_branch),
os.path.join(patch_path))
if os.path.isdir(dest_dir):
self.get_logger().info('Version already checked out')
return 0
git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)
with utils.cd(dest_dir):
if subprocess.call(
['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always'
]) != 0:
self.get_logger().warn('Unable to setup auto-rebase')
num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
if subprocess.call([
'/usr/bin/git', 'config', 'submodule.fetchJobs', num_procs
]) != 0:
self.get_logger().warn('Unable to setup submodule fetch jobs')
git.checkout(dest_dir, checkout_version)
if checkout_version == 'master':
# Specific to Beta Cluster
master_stuff(dest_dir)
write_settings_stub(
os.path.join(dest_dir, 'StartProfiler.php'),
os.path.join(self.config['deploy_dir'], 'wmf-config',
'StartProfiler-labs.php'))
else:
# Specific to production
git.update_submodules(dest_dir, use_upstream=True)
update_update_strategy(dest_dir)
write_settings_stub(
os.path.join(dest_dir, 'StartProfiler.php'),
os.path.join(self.config['deploy_dir'], 'wmf-config',
'StartProfiler.php'))
write_settings_stub(
os.path.join(dest_dir, 'LocalSettings.php'),
os.path.join(self.config['deploy_dir'], 'wmf-config',
'CommonSettings.php'))
cache_dir = os.path.join(dest_dir, 'cache')
os.chmod(cache_dir, 0o777)
utils.sudo_check_call('l10nupdate',
'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))
self.get_logger().info('MediaWiki %s successfully checked out.' %
checkout_version)
def fetch(self):
"""
Fetch the specified revision of the remote repo.
The given repo is cloned into the cache directory and a new working
directory for the given revision is created under revs/{rev}.
At the end of this stage, the .in-progress link is created to signal
the possibility for future rollback.
"""
logger = self.get_logger()
has_submodules = self.config['git_submodules']
git_binary_manager = None
if self.config['git_fat']:
logger.warning(
'Using deprecated git_fat config, swap to git_binary_manager')
git_binary_manager = [git.FAT]
elif self.config['git_binary_manager']:
git_binary_manager = config.multi_value(
self.config['git_binary_manager'])
rev_dir = self.context.rev_path(self.rev)
git_remote = os.path.join(self.server_url, '.git')
logger.info('Fetch from: {}'.format(git_remote))
# clone/fetch from the repo to the cache directory
git.fetch(self.context.cache_dir, git_remote)
if has_submodules:
upstream_submodules = self.config['git_upstream_submodules']
logger.info('Update submodules')
git.update_submodules(location=self.context.cache_dir,
git_remote=git_remote,
use_upstream=upstream_submodules)
# If the rev_dir already exists AND the currently checked-out HEAD is
# already at the revision specified by ``self.rev`` then you can assume
#
# 1. If there is a config deploy, the config is inside the rev_dir
# 2. The code represented by the SHA1 to be deployed is inside
# the rev_dir
#
# Set the noop flag and return
if os.path.isdir(rev_dir) and not self.arguments.force:
rev = git.sha(rev_dir, 'HEAD')
if rev == self.rev:
logger.info(
'Revision directory already exists '
'(use --force to override)')
self.noop = True
self.context.mark_rev_in_progress(self.rev)
return
if not os.path.isdir(rev_dir):
# if lfs is enabled and this is the first time cloning this repo,
# then we need to run `git lfs install`` before `git clone`
if git_binary_manager and git.LFS in git_binary_manager:
git.lfs_install('--global')
# clone/fetch from the local cache directory to the revision directory
git.fetch(rev_dir, self.context.cache_dir,
reference=self.context.cache_dir,
dissociate=False,
recurse_submodules=False)
logger.info('Checkout rev: {}'.format(self.rev))
# checkout the given revision
git.checkout(rev_dir, self.rev)
if has_submodules:
git.update_submodules(location=rev_dir, git_remote=git_remote,
use_upstream=upstream_submodules,
reference=self.context.cache_dir)
if git_binary_manager:
for manager in git_binary_manager:
logger.info("Pulling large objects [using %s]", manager)
if manager in [git.FAT, git.LFS]:
git.largefile_pull(rev_dir, manager)
else:
logger.warning("Passed unrecognized binary manager %s",
manager)
self.context.mark_rev_in_progress(self.rev)
def main(self, *extra_args):
"""Checkout next MediaWiki."""
dest_dir = os.path.join(
self.config['stage_dir'], '{}{}'.format(self.arguments.prefix,
self.arguments.branch))
checkout_version = 'master'
if self.arguments.branch != 'master':
checkout_version = 'wmf/%s' % self.arguments.branch
reference_dir = None
if checkout_version != 'master':
# active_wikiversions() is already sorted by loose-version number,
# we want the latest version if there's more than 1
old_branch = self.active_wikiversions().keys()[-1]
old_branch_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, old_branch))
reference_dir = None
if (os.path.exists(old_branch_dir)):
reference_dir = old_branch_dir
patch_base_dir = '/srv/patches'
patch_path = os.path.join(patch_base_dir, self.arguments.branch)
if not os.path.exists(patch_path):
if os.path.exists(os.path.join(patch_base_dir, old_branch)):
shutil.copytree(os.path.join(patch_base_dir, old_branch),
os.path.join(patch_path))
srv_patches_git_message = 'Scap prep for "{}"'.format(
self.arguments.branch)
git.add_all(patch_base_dir,
message=srv_patches_git_message)
if os.path.isdir(dest_dir):
self.get_logger().info('Version already checked out')
return 0
self.get_logger().info('Fetching core to {}'.format(dest_dir))
git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)
with utils.cd(dest_dir):
if subprocess.call(
['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always'
]) != 0:
self.get_logger().warn('Unable to setup auto-rebase')
num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
if subprocess.call([
'/usr/bin/git', 'config', 'submodule.fetchJobs', num_procs
]) != 0:
self.get_logger().warn('Unable to setup submodule fetch jobs')
self.get_logger().info('Checkout {} in {}'.format(
checkout_version, dest_dir))
git.checkout(dest_dir, checkout_version)
if checkout_version == 'master':
# Specific to Beta Cluster
master_stuff(dest_dir)
else:
# Specific to production
self.get_logger().info('Update submodules for {}'.format(dest_dir))
git.update_submodules(dest_dir, use_upstream=True)
update_update_strategy(dest_dir)
self.get_logger().info('Creating LocalSettings.php stub')
write_settings_stub(os.path.join(dest_dir, 'LocalSettings.php'))
self.get_logger().info('Creating l10n cache dir')
cache_dir = os.path.join(dest_dir, 'cache')
os.chmod(cache_dir, 0o777)
utils.sudo_check_call('l10nupdate',
'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))
self.get_logger().info('MediaWiki %s successfully checked out.' %
checkout_version)
def main(self, *extra_args):
"""Checkout next MediaWiki."""
dest_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, self.arguments.branch)
)
checkout_version = 'master'
if self.arguments.branch != 'master':
checkout_version = 'wmf/%s' % self.arguments.branch
reference_dir = None
if checkout_version != 'master':
# active_wikiversions() is already sorted by loose-version number,
# we want the latest version if there's more than 1
old_branch = self.active_wikiversions().keys()[-1]
old_branch_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, old_branch)
)
reference_dir = None
if (os.path.exists(old_branch_dir)):
reference_dir = old_branch_dir
patch_path = os.path.join('/srv/patches', self.arguments.branch)
if not os.path.exists(patch_path):
if os.path.exists(os.path.join('/srv/patches', old_branch)):
shutil.copytree(
os.path.join('/srv/patches', old_branch),
os.path.join(patch_path)
)
if os.path.isdir(dest_dir):
self.get_logger().info('Version already checked out')
return 0
self.get_logger().info('Fetching core to {}'.format(dest_dir))
git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)
with utils.cd(dest_dir):
if subprocess.call(['/usr/bin/git', 'config',
'branch.autosetuprebase', 'always']) != 0:
self.get_logger().warn('Unable to setup auto-rebase')
num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
if subprocess.call(['/usr/bin/git', 'config',
'submodule.fetchJobs', num_procs]) != 0:
self.get_logger().warn('Unable to setup submodule fetch jobs')
self.get_logger().info('Checkout {} in {}'.format(checkout_version, dest_dir))
git.checkout(dest_dir, checkout_version)
if checkout_version == 'master':
# Specific to Beta Cluster
master_stuff(dest_dir)
else:
# Specific to production
self.get_logger().info('Update submodules for {}'.format(dest_dir))
git.update_submodules(dest_dir, use_upstream=True)
update_update_strategy(dest_dir)
self.get_logger().info('Creating LocalSettings.php stub')
write_settings_stub(os.path.join(dest_dir, 'LocalSettings.php'))
self.get_logger().info('Creating l10n cache dir')
cache_dir = os.path.join(dest_dir, 'cache')
os.chmod(cache_dir, 0o777)
utils.sudo_check_call('l10nupdate',
'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))
self.get_logger().info('MediaWiki %s successfully checked out.' %
checkout_version)