def sync_master(cfg, master, verbose=False, logger=None):
"""
Sync local staging dir with upstream rsync server's copy.
Rsync from ``server::common`` to the local staging directory.
:param cfg: Dict of global configuration values.
:param master: Master server to sync with
:param verbose: Enable verbose logging?
"""
if not os.path.isdir(cfg['stage_dir']):
raise IOError(
('rsync target directory %s not found. Ask root to create it '
'(should belong to root:wikidev).') % cfg['stage_dir'])
# Execute rsync fetch locally via sudo and wrapper script
rsync = ['sudo', '-n', '--', '/usr/local/bin/scap-master-sync', master]
logger.info('Copying from %s to %s', master, socket.getfqdn())
logger.debug('Running rsync command: `%s`', ' '.join(rsync))
stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port']))
with log.Timer('rsync master', stats):
subprocess.check_call(rsync)
# Rebuild the CDB files from the JSON versions
with log.Timer('rebuild CDB staging files', stats):
subprocess.check_call([
'sudo', '-u', 'l10nupdate', '-n', '--',
os.path.join(os.path.dirname(sys.argv[0]), 'scap'), 'cdb-rebuild',
'--no-progress', '--staging', '--verbose'
])
def _after_cluster_sync(self):
"""
Need to remove cache dir manually after sync
"""
branch_cache = os.path.join(self.branch_deploy_dir, 'cache')
target_hosts = self._get_target_list()
cmd = ['/bin/rm', '-rf', branch_cache]
with log.Timer('clean-remote-caches', self.get_stats()):
remove_remote_dirs = ssh.Job(
target_hosts,
user=self.config['ssh_user'],
key=self.get_keyholder_key()
)
remove_remote_dirs.command(cmd)
remove_remote_dirs.progress(
log.reporter(
'clean-remote-caches',
self.config['fancy_progress']
)
)
success, fail = remove_remote_dirs.run()
if fail:
self.get_logger().warning(
'%d hosts failed to remove cache', fail
)
self.soft_errors = True
def main(self, *extra_args):
rsync_args = ['--delete-excluded'
] if self.arguments.delete_excluded else []
tasks.sync_common(self.config,
include=self.arguments.include,
sync_from=self.arguments.servers,
verbose=self.verbose,
rsync_args=rsync_args)
if self.arguments.update_l10n:
with log.Timer('scap-cdb-rebuild', self.get_stats()):
utils.sudo_check_call(
'mwdeploy',
self.get_script_path() + ' cdb-rebuild --no-progress')
# Invalidate opcache
# TODO deduplicate this from AbstractSync._invalidate_opcache()
php7_admin_port = self.config.get('php7-admin-port')
if php7_admin_port:
om = opcache_manager.OpcacheManager(php7_admin_port)
failed = om.invalidate([socket.gethostname()], None)
if failed:
self.get_logger().warning('Opcache invalidation failed. '
'Consider performing it manually.')
if (self.arguments.php_restart):
fpm = php_fpm.PHPRestart(self.config)
self.get_logger().info('Checking if php-fpm restart needed')
failed = fpm.restart_self()
if failed:
self.get_logger().warning('php-fpm restart failed!')
return 0
def _after_cluster_sync(self):
target_hosts = self._get_target_list()
# Ask apaches to rebuild l10n CDB files
with log.Timer('scap-cdb-rebuild', self.get_stats()):
rebuild_cdbs = ssh.Job(target_hosts,
user=self.config['ssh_user'],
key=self.get_keyholder_key())
rebuild_cdbs.shuffle()
rebuild_cdbs.command('sudo -u mwdeploy -n -- %s cdb-rebuild' %
self.get_script_path())
rebuild_cdbs.progress(
log.reporter('scap-cdb-rebuild',
self.config['fancy_progress']))
succeeded, failed = rebuild_cdbs.run()
if failed:
self.get_logger().warning(
'%d hosts had scap-cdb-rebuild errors', failed)
self.soft_errors = True
# Update and sync wikiversions.php
succeeded, failed = tasks.sync_wikiversions(
target_hosts, self.config, key=self.get_keyholder_key())
if failed:
self.get_logger().warning('%d hosts had sync_wikiversions errors',
failed)
self.soft_errors = True
tasks.clear_message_blobs()
self._invalidate_opcache()
self._restart_php()
def _after_sync_common(self):
self._git_repo()
# Compute git version information
with log.Timer('cache_git_info', self.get_stats()):
for version, wikidb in self.active_wikiversions().items():
tasks.cache_git_info(version, self.config)
def execute_remote(self, description, targets, command):
with log.Timer(description, self.get_stats()):
clean_job = ssh.Job(targets, user=self.config['ssh_user'])
clean_job.shuffle()
clean_job.command(command)
clean_job.progress(
log.reporter(description, self.config['fancy_progress']))
succeeded, failed = clean_job.run()
if failed:
self.get_logger().warning('%d had clean errors', failed)
def cleanup_branch(self, branch, delete):
"""
Given a branch, go through the cleanup proccess on the master.
(1) Prune git branches [if deletion]
(2) Remove l10nupdate cache
(3) Remove l10n cache
(4) Remove l10n bootstrap file
(5) Remove some branch files [all if deletion]
(6) Remove security patches [if deletion]
"""
if not os.path.isdir(self.branch_stage_dir):
raise ValueError('No such branch exists, aborting')
with log.Timer('clean-l10nupdate-cache', self.get_stats()):
utils.sudo_check_call(
'www-data',
'rm -fR /var/lib/l10nupdate/caches/cache-%s' % branch
)
with log.Timer('clean-l10nupdate-owned-files', self.get_stats()):
utils.sudo_check_call(
'l10nupdate',
'find %s -user l10nupdate -delete' % self.branch_stage_dir
)
with log.Timer('clean-ExtensionMessages'):
ext_msg = os.path.join(self.config['stage_dir'], 'wmf-config',
'ExtensionMessages-%s.php' % branch)
self._maybe_delete(ext_msg)
logger = self.get_logger()
if delete:
# Moved behind a feature flag until T218750 is resolved
if self.arguments.delete_gerrit_branch:
git_prune_cmd = [
'git', 'push', 'origin', '--quiet', '--delete',
'wmf/%s' % branch
]
with log.Timer('prune-git-branches', self.get_stats()):
# Prune all the submodules' remote branches
with utils.cd(self.branch_stage_dir):
submodule_cmd = 'git submodule foreach "{} ||:"'.format(
' '.join(git_prune_cmd))
subprocess.check_output(submodule_cmd, shell=True)
if subprocess.call(git_prune_cmd) != 0:
logger.info('Failed to prune core branch')
with log.Timer('removing-local-copy'):
self._maybe_delete(self.branch_stage_dir)
with log.Timer('cleaning-unused-patches', self.get_stats()):
self._maybe_delete(os.path.join('/srv/patches', branch))
else:
with log.Timer('cleaning-unused-files', self.get_stats()):
for rmdir in DELETABLE_DIRS:
self._maybe_delete(
os.path.join(self.branch_stage_dir, rmdir)
)
def _after_sync_common(self):
super(Scap, self)._after_sync_common()
# Bug 63659: Compile deploy_dir/wikiversions.json to cdb
cmd = '{} wikiversions-compile'.format(self.get_script_path())
utils.sudo_check_call('mwdeploy', cmd)
# Update list of extension message files and regenerate the
# localisation cache.
with log.Timer('l10n-update', self.get_stats()):
for version, wikidb in self.active_wikiversions().items():
tasks.update_localization_cache(version, wikidb, self.verbose,
self.config)
def cleanup_branch(self, branch, keep_static):
stage_dir = os.path.join(self.config['stage_dir'], 'php-%s' % branch)
deploy_dir = os.path.join(self.config['deploy_dir'], 'php-%s' % branch)
if not keep_static:
gerrit_prune_cmd = [
'git', 'push', 'origin', '--delete',
'wmf/%s' % branch
]
logger = self.get_logger()
with log.Timer('prune-git-branches', self.get_stats()):
# Prune all the submodules' remote branches
for submodule in git.list_submodules(stage_dir):
submodule_path = submodule.lstrip(' ').split(' ')[1]
with utils.cd(os.path.join(stage_dir, submodule_path)):
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info(
'Failed to prune submodule branch for %s' %
submodule)
# Prune core last
with utils.cd(stage_dir):
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info('Failed to prune core branch')
# Prune cache junk from masters used by l10nupdate
self.execute_remote('clean-masters-l10nupdate-cache',
self._get_master_list(), [
'sudo', '-u', 'l10nupdate', 'rm', '-fR',
'/var/lib/l10nupdate/caches/cache-%s' % branch
])
# Prune junk from masters owned by l10nupdate
self.execute_remote('clean-masters-l10nupdate',
self._get_master_list(), [
'sudo', '-u', 'l10nupdate', 'find', stage_dir,
'-user', 'l10nupdate', '-delete'
])
# Update masters
self.execute_remote('clean-masters', self._get_master_list(),
self.clean_command(stage_dir, keep_static))
# Update apaches
self.execute_remote('clean-apaches', self._get_target_list(),
self.clean_command(deploy_dir, keep_static))
def sync_wikiversions(hosts, cfg, key=None):
"""
Rebuild and sync wikiversions.php to the cluster.
:param hosts: List of hosts to sync to
:param cfg: Dict of global configuration values
"""
stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port']))
with log.Timer('sync_wikiversions', stats):
compile_wikiversions('stage', cfg)
rsync = ssh.Job(hosts, user=cfg['ssh_user'], key=key).shuffle()
rsync.command('sudo -u mwdeploy -n -- /usr/bin/rsync -l '
'%(master_rsync)s::common/wikiversions*.{json,php} '
'%(deploy_dir)s' % cfg)
return rsync.progress(
log.reporter('sync_wikiversions', cfg['fancy_progress'])).run()
def master_only_cmd(self, timer, cmd):
"""
Run a command on all other master servers than the one we're on
:param timer: String name to use in timer/logging
:param cmd: List of command/parameters to be executed
"""
masters = self.get_master_list()
with log.Timer(timer, self.get_stats()):
update_masters = ssh.Job(masters,
user=self.config['ssh_user'],
key=self.get_keyholder_key())
update_masters.exclude_hosts([socket.getfqdn()])
update_masters.command(cmd)
update_masters.progress(
log.reporter(timer, self.config['fancy_progress']))
succeeded, failed = update_masters.run()
if failed:
self.get_logger().warning('%d masters had sync errors', failed)
self.soft_errors = True
def _after_cluster_sync(self):
# Rebuild l10n CDB files
target_hosts = self._get_target_list()
with log.Timer('scap-cdb-rebuild', self.get_stats()):
rebuild_cdbs = ssh.Job(target_hosts,
user=self.config['ssh_user'],
key=self.get_keyholder_key())
rebuild_cdbs.shuffle()
cdb_cmd = 'sudo -u mwdeploy -n -- {} cdb-rebuild --version {}'
cdb_cmd = cdb_cmd.format(self.get_script_path(),
self.arguments.version)
rebuild_cdbs.command(cdb_cmd)
rebuild_cdbs.progress(
log.reporter('scap-cdb-rebuild',
self.config['fancy_progress']))
succeeded, failed = rebuild_cdbs.run()
if failed:
self.get_logger().warning(
'%d hosts had scap-cdb-rebuild errors', failed)
self.soft_errors = True
tasks.clear_message_blobs()
# Globally invalidate opcache. TODO: is this needed?
self._invalidate_opcache(target_hosts)
self._restart_php()
def main(self, *extra_args):
logger = self.get_logger()
self.repo = self.config['git_repo']
if self.arguments.stages:
stages = self.arguments.stages.split(',')
else:
stages = STAGES
if self.arguments.service_restart:
stages = [RESTART]
if not self.config.get('service_name'):
raise RuntimeError(
'--service-restart flag requires a `service_name` in '
'the config')
if self.arguments.dry_run:
stages = ['config_diff']
restart_only = False
if len(stages) == 1 and stages[0] == RESTART:
restart_only = True
if not git.is_dir(self.context.root):
raise RuntimeError(errno.EPERM, 'Script must be run from git repo')
self._build_deploy_groups()
if not self.all_targets:
logger.warning('No targets selected, check limits and dsh_targets')
return 1
short_sha1 = git.info(self.context.root)['headSHA1'][:7]
if not short_sha1:
short_sha1 = 'UNKNOWN'
deploy_name = 'deploy'
if self.arguments.init:
deploy_name = 'setup'
elif restart_only:
deploy_name = 'restart'
display_name = '{} [{}@{}]'.format(deploy_name, self.repo, short_sha1)
environment_name = self.config['environment']
if environment_name is not None:
display_name = '{} ({})'.format(display_name, environment_name)
rev = self.arguments.rev
# No revision passed on the command line, let's check the config
if not rev:
rev = self.config.get('git_rev')
if rev:
use_upstream = self.config.get('git_upstream_submodules',
False)
if rev.startswith('origin/') and not use_upstream:
logger.warning(
'You have set `git_rev` to "%s" without ' +
'setting `git_upstream_submodules=True`. ' +
'This could lead to unexpected behavior.', rev)
# No revision from the config or cli
# AND we're not running a stage that we want to deploy new code
if not rev and not self._needs_latest_sha1(stages):
last_deploy_tag = git.last_deploy_tag(self.context.root)
if last_deploy_tag is not None:
# This is what we want to pass to rev-parse to get the sha1 of
# the annotated commit, rather than the sha1 of the annotation.
rev = '%s^{}' % last_deploy_tag
# We must be trying to deploy the latest and greatest
if not rev:
rev = 'HEAD'
with lock.Lock(self.get_lock_file(), self.arguments.message):
with log.Timer(display_name):
timestamp = datetime.utcnow()
tag = git.next_deploy_tag(location=self.context.root)
commit = git.sha(location=self.context.root, rev=rev)
logger.info('Deploying Rev: {} = {}'.format(rev, commit))
self.config_deploy_setup(commit)
self.checks_setup()
self.config['git_rev'] = commit
self.deploy_info.update({
'tag': tag,
'commit': commit,
'user': utils.get_username(),
'timestamp': timestamp.isoformat(),
})
# Handle JSON output from deploy-local
self.deploy_info['log_json'] = True
self.get_logger().debug('Update DEPLOY_HEAD')
self.deploy_info.update({
key: self.config.get(key)
for key in self.DEPLOY_CONF
if self.config.get(key) is not None
})
git.update_deploy_head(self.deploy_info, self.context.root)
git.tag_repo(self.deploy_info, location=self.context.root)
# Remove old tags
git.clean_tags(self.context.root, self.config['tags_to_keep'])
# Run git update-server-info because git repo is a dumb
# apache server
git.update_server_info(self.config['git_submodules'])
if self.arguments.init:
return 0
self.announce('Started %s: %s', display_name,
self.arguments.message)
exec_result = self._execute_for_groups(stages)
if not restart_only:
self.announce('Finished %s: %s (duration: %s)',
display_name, self.arguments.message,
utils.human_duration(self.get_duration()))
return exec_result
return 0
def cleanup_branch(self, branch, delete):
"""
Given a branch, go through the cleanup proccess on the master:
(1) Prune git branches [if deletion]
(2) Remove l10nupdate cache
(3) Remove l10n cache
(4) Remove l10n bootstrap file
(5) Remove some branch files [all if deletion]
(6) Remove security patches [if deletion]
"""
stage_dir = os.path.join(self.config['stage_dir'], 'php-%s' % branch)
if not os.path.isdir(stage_dir):
raise ValueError('No such branch exists, aborting')
command_list = []
command_list.append([
'clean-l10nupdate-cache',
[
'sudo', '-u', 'www-data', 'rm', '-fR',
'/var/lib/l10nupdate/caches/cache-%s' % branch
]
])
command_list.append([
'clean-l10nupdate-owned-files',
[
'sudo', '-u', 'l10nupdate', 'find', stage_dir, '-user',
'l10nupdate', '-delete'
]
])
command_list.append([
'clean-l10n-bootstrap',
[
'rm', '-fR',
os.path.join(self.config['stage_dir'], 'wmf-config',
'ExtensionMessages-%s.php' % branch)
]
])
logger = self.get_logger()
if delete:
gerrit_prune_cmd = [
'git', 'push', 'origin', '--quiet', '--delete',
'wmf/%s' % branch
]
with log.Timer('prune-git-branches', self.get_stats()):
# Prune all the submodules' remote branches
for submodule in git.list_submodules(stage_dir):
submodule_path = submodule.lstrip(' ').split(' ')[1]
with utils.cd(os.path.join(stage_dir, submodule_path)):
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info(
'Failed to prune submodule branch for %s' %
submodule)
# Prune core last
with utils.cd(stage_dir):
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info('Failed to prune core branch')
command_list.append(['cleaning-branch', ['rm', '-fR', stage_dir]])
command_list.append([
'cleaning-patches',
['rm', '-fR',
os.path.join('/srv/patches', branch)]
])
else:
regex = r'".*\.?({0})$"'.format('|'.join(DELETABLE_TYPES))
command_list.append([
'cleaning-branch',
[
'find', stage_dir, '-type', 'f', '-regextype',
'posix-extended', '-regex', regex, '-delete'
]
])
for command_signature in command_list:
name = command_signature[0]
command = command_signature[1]
with log.Timer(name + '-' + branch, self.get_stats()):
try:
subprocess.check_call(command)
except (subprocess.CalledProcessError, OSError):
logger.warning('Command failed [%s]: %s' %
(name, ' '.join(command)))
def sync_common(cfg,
include=None,
sync_from=None,
verbose=False,
logger=None,
rsync_args=None):
"""
Sync local deploy dir with upstream rsync server's copy.
Rsync from ``server::common`` to the local deploy directory.
If a list of servers is given in ``sync_from`` we will attempt to select
the "best" one to sync from. If no servers are given or all servers given
have issues we will fall back to using the server named by
``master_rsync`` in the configuration data.
:param cfg: Dict of global configuration values.
:param include: List of rsync include patterns to limit the sync to. If
``None`` is given the entire ``common`` module on the target rsync
server will be transferred. Rsync syntax for syncing a directory is
``<dirname>/***``.
:param sync_from: List of rsync servers to fetch from.
"""
if not os.path.isdir(cfg['deploy_dir']):
raise IOError(
('rsync target directory %s not found. Ask root to create it '
'(should belong to mwdeploy:mwdeploy).') % cfg['deploy_dir'])
server = None
if sync_from:
server = utils.find_nearest_host(sync_from)
if server is None:
server = cfg['master_rsync']
server = server.strip()
# Execute rsync fetch locally via sudo
rsync = ['sudo', '-u', 'mwdeploy', '-n', '--'] + DEFAULT_RSYNC_ARGS
# Exclude .git metadata
rsync.append('--exclude=**/.git')
if verbose:
rsync.append('--verbose')
if include:
for path in include:
rsync.append('--include=/%s' % path)
# Exclude everything not explicitly included
rsync.append('--exclude=*')
if rsync_args:
rsync += rsync_args
rsync.append('%s::common' % server)
rsync.append(cfg['deploy_dir'])
logger.info('Copying from %s to %s', server, socket.getfqdn())
logger.debug('Running rsync command: `%s`', ' '.join(rsync))
stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port']))
with log.Timer('rsync common', stats):
subprocess.check_call(rsync)
# Bug 58618: Invalidate local configuration cache by updating the
# timestamp of wmf-config/InitialiseSettings.php
settings_path = os.path.join(cfg['deploy_dir'], 'wmf-config',
'InitialiseSettings.php')
logger.debug('Touching %s', settings_path)
subprocess.check_call(('sudo', '-u', 'mwdeploy', '-n', '--',
'/usr/bin/touch', settings_path))
def main(self, *extra_args):
"""Perform a sync operation to the cluster."""
print(ansi.logo())
self._assert_auth_sock()
with lock.Lock(self.get_lock_file(), self.arguments.message):
self._check_sync_flag()
if not self.arguments.force:
self.get_logger().info(
'Checking for new runtime errors locally')
self._check_fatals()
else:
self.get_logger().warning('check_fatals Skipped by --force')
self._before_cluster_sync()
self._sync_common()
self._after_sync_common()
self._sync_masters()
full_target_list = self._get_target_list()
if not self.arguments.force:
canaries = [
node for node in self._get_canary_list()
if node in full_target_list
]
with log.Timer('sync-check-canaries',
self.get_stats()) as timer:
self.sync_canary(canaries)
timer.mark('Canaries Synced')
self._invalidate_opcache(canaries)
self.canary_checks(canaries, timer)
else:
self.get_logger().warning('Canaries Skipped by --force')
# Update proxies
proxies = [
node for node in self._get_proxy_list()
if node in full_target_list
]
with log.Timer('sync-proxies', self.get_stats()):
sync_cmd = self._apache_sync_command(self.get_master_list())
# Proxies should always use the current host as their sync
# origin server.
sync_cmd.append(socket.getfqdn())
update_proxies = ssh.Job(proxies,
user=self.config['ssh_user'],
key=self.get_keyholder_key())
update_proxies.command(sync_cmd)
update_proxies.progress(
log.reporter('sync-proxies',
self.config['fancy_progress']))
succeeded, failed = update_proxies.run()
if failed:
self.get_logger().warning('%d proxies had sync errors',
failed)
self.soft_errors = True
# Update apaches
with log.Timer('sync-apaches', self.get_stats()):
update_apaches = ssh.Job(full_target_list,
user=self.config['ssh_user'],
key=self.get_keyholder_key())
update_apaches.exclude_hosts(proxies)
update_apaches.exclude_hosts(self.get_master_list())
if not self.arguments.force:
update_apaches.exclude_hosts(canaries)
update_apaches.shuffle()
update_apaches.command(self._apache_sync_command(proxies))
update_apaches.progress(
log.reporter('sync-apaches',
self.config['fancy_progress']))
succeeded, failed = update_apaches.run()
if failed:
self.get_logger().warning('%d apaches had sync errors',
failed)
self.soft_errors = True
self._after_cluster_sync()
self._after_lock_release()
if self.soft_errors:
return 1
return 0
