def cleanup_branch(self, branch, delete):
"""
Given a branch, go through the cleanup proccess on the master.
(1) Prune git branches [if deletion]
(2) Remove l10nupdate cache
(3) Remove l10n cache
(4) Remove l10n bootstrap file
(5) Remove some branch files [all if deletion]
(6) Remove security patches [if deletion]
"""
if not os.path.isdir(self.branch_stage_dir):
raise ValueError('No such branch exists, aborting')
with log.Timer('clean-l10nupdate-cache', self.get_stats()):
utils.sudo_check_call(
'www-data',
'rm -fR /var/lib/l10nupdate/caches/cache-%s' % branch)
with log.Timer('clean-l10nupdate-owned-files', self.get_stats()):
utils.sudo_check_call(
'l10nupdate',
'find %s -user l10nupdate -delete' % self.branch_stage_dir)
with log.Timer('clean-ExtensionMessages'):
ext_msg = os.path.join(self.config['stage_dir'], 'wmf-config',
'ExtensionMessages-%s.php' % branch)
self._maybe_delete(ext_msg)
logger = self.get_logger()
if delete:
# Moved behind a feature flag until T218750 is resolved
if self.arguments.delete_gerrit_branch:
git_prune_cmd = [
'git', 'push', 'origin', '--quiet', '--delete',
'wmf/%s' % branch
]
with log.Timer('prune-git-branches', self.get_stats()):
# Prune all the submodules' remote branches
with utils.cd(self.branch_stage_dir):
submodule_cmd = 'git submodule foreach "{} ||:"'.format(
' '.join(git_prune_cmd))
subprocess.check_output(submodule_cmd, shell=True)
if subprocess.call(git_prune_cmd) != 0:
logger.info('Failed to prune core branch')
with log.Timer('removing-local-copy'):
self._maybe_delete(self.branch_stage_dir)
with log.Timer('cleaning-unused-patches', self.get_stats()):
patch_base_dir = '/srv/patches'
self._maybe_delete(os.path.join(patch_base_dir, branch))
srv_patches_git_message = 'Scap clean for "{}"'.format(branch)
git.add_all(patch_base_dir, message=srv_patches_git_message)
else:
with log.Timer('cleaning-unused-files', self.get_stats()):
for rmdir in DELETABLE_DIRS:
self._maybe_delete(
os.path.join(self.branch_stage_dir, rmdir))
def main(self, *extra_args):
"""Run deploy-mediawiki."""
# Flatten local into git repo
self.get_logger().info('scap deploy-mediawiki')
git.default_ignore(self.config['deploy_dir'])
git.add_all(self.config['deploy_dir'], message=self.arguments.message)
git.garbage_collect(self.config['deploy_dir'])
scap = self.get_script_path()
options = {
'git_repo': self.config['deploy_dir'],
}
option_list = ['-D{}:{}'.format(x, y) for x, y in options.items()]
cmd = [scap, 'deploy', '-v']
cmd += option_list
cmd += ['--init']
with utils.cd(self.config['deploy_dir']):
subprocess.check_call(cmd)
def main(self, *extra_args):
"""Run deploy-mediawiki."""
# Flatten local into git repo
self.get_logger().info('scap deploy-mediawiki')
git.default_ignore(self.config['deploy_dir'])
git.add_all(self.config['deploy_dir'], message=self.arguments.message)
git.garbage_collect(self.config['deploy_dir'])
scap = self.get_script_path()
options = {
'git_repo': self.config['deploy_dir'],
}
option_list = ['-D{}:{}'.format(x, y) for x, y in options.items()]
cmd = [scap, 'deploy', '-v']
cmd += option_list
cmd += ['--init']
with utils.cd(self.config['deploy_dir']):
subprocess.check_call(cmd)
def setUp(self):
git.init(TEMPDIR)
git.default_ignore(TEMPDIR)
with sh.pushd(TEMPDIR):
_TOUCH('testfile')
git.add_all(TEMPDIR, 'first commit')
def main(self, *extra_args):
"""Checkout next MediaWiki."""
dest_dir = os.path.join(
self.config['stage_dir'], '{}{}'.format(self.arguments.prefix,
self.arguments.branch))
checkout_version = 'master'
if self.arguments.branch != 'master':
checkout_version = 'wmf/%s' % self.arguments.branch
reference_dir = None
if checkout_version != 'master':
# active_wikiversions() is already sorted by loose-version number,
# we want the latest version if there's more than 1
old_branch = self.active_wikiversions().keys()[-1]
old_branch_dir = os.path.join(
self.config['stage_dir'],
'{}{}'.format(self.arguments.prefix, old_branch))
reference_dir = None
if (os.path.exists(old_branch_dir)):
reference_dir = old_branch_dir
patch_base_dir = '/srv/patches'
patch_path = os.path.join(patch_base_dir, self.arguments.branch)
if not os.path.exists(patch_path):
if os.path.exists(os.path.join(patch_base_dir, old_branch)):
shutil.copytree(os.path.join(patch_base_dir, old_branch),
os.path.join(patch_path))
srv_patches_git_message = 'Scap prep for "{}"'.format(
self.arguments.branch)
git.add_all(patch_base_dir,
message=srv_patches_git_message)
if os.path.isdir(dest_dir):
self.get_logger().info('Version already checked out')
return 0
self.get_logger().info('Fetching core to {}'.format(dest_dir))
git.fetch(dest_dir, SOURCE_URL + 'mediawiki/core', reference_dir)
with utils.cd(dest_dir):
if subprocess.call(
['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always'
]) != 0:
self.get_logger().warn('Unable to setup auto-rebase')
num_procs = str(max(multiprocessing.cpu_count() / 2, 1))
if subprocess.call([
'/usr/bin/git', 'config', 'submodule.fetchJobs', num_procs
]) != 0:
self.get_logger().warn('Unable to setup submodule fetch jobs')
self.get_logger().info('Checkout {} in {}'.format(
checkout_version, dest_dir))
git.checkout(dest_dir, checkout_version)
if checkout_version == 'master':
# Specific to Beta Cluster
master_stuff(dest_dir)
else:
# Specific to production
self.get_logger().info('Update submodules for {}'.format(dest_dir))
git.update_submodules(dest_dir, use_upstream=True)
update_update_strategy(dest_dir)
self.get_logger().info('Creating LocalSettings.php stub')
write_settings_stub(os.path.join(dest_dir, 'LocalSettings.php'))
self.get_logger().info('Creating l10n cache dir')
cache_dir = os.path.join(dest_dir, 'cache')
os.chmod(cache_dir, 0o777)
utils.sudo_check_call('l10nupdate',
'mkdir "%s"' % os.path.join(cache_dir, 'l10n'))
self.get_logger().info('MediaWiki %s successfully checked out.' %
checkout_version)
def setUp(self):
git.init(TEMPDIR)
git.default_ignore(TEMPDIR)
with sh.pushd(TEMPDIR):
_TOUCH('testfile')
git.add_all(TEMPDIR, 'first commit')