def test_checksum_dir(self):
"""Test checksum'ing of directory structure.
"""
# Setup directory
# /.git
os.makedirs(os.path.join(self.root, '.git'))
# /common/foo/
os.makedirs(os.path.join(self.root, 'common'))
os.makedirs(os.path.join(self.root, 'common', 'foo'))
# /common/foo/bar
with open(os.path.join(self.root, 'common', 'foo', 'bar'), 'w'):
pass
# /common/foo/baz -> bar
os.symlink('bar',
os.path.join(self.root, 'common', 'foo', 'baz'))
# /foodir -> common/foo
os.symlink(os.path.join('common', 'foo'),
os.path.join(self.root, 'foodir'))
# /foo/bar -> ../common/foo/bar
os.makedirs(os.path.join(self.root, 'foo'))
os.symlink(os.path.join('..', 'common', 'foo', 'bar'),
os.path.join(self.root, 'foo', 'bar'))
# /otherdir -> /dir
os.symlink('/dir',
os.path.join(self.root, 'otherdir'))
mock_checksum = hashlib.sha256.return_value
versionmgr.checksum_dir(self.root)
mock_checksum.update.assert_has_calls(
[
# XXX(boysson): Figure out how to test this
# mock.call('/common/foo/bar <mode> <size> <mtime> <ctime>'),
mock.call(b'/common/foo/baz -> bar'),
mock.call(b'/foodir -> common/foo'),
mock.call(b'/foo/bar -> ../common/foo/bar'),
mock.call(b'/otherdir -> /dir'),
],
any_order=True,
)
self.assertEqual(mock_checksum.update.call_count, 5)
def upgrade(cell, ldap, ldap_search_base, batch, timeout, treadmill_root,
continue_on_error, dry_run, force, servers):
"""Upgrade the supplied cell"""
context.GLOBAL.ldap.url = ldap
context.GLOBAL.ldap.search_base = ldap_search_base
servers = []
for server_list in servers:
servers.extend(server_list.split(','))
if not treadmill_root:
admin_cell = admin.Cell(context.GLOBAL.ldap.conn)
cell_info = admin_cell.get(cell)
treadmill_root = cell_info.get('treadmill_root')
_LOGGER.info('Treadmill root: %s', treadmill_root)
digest = versionmgr.checksum_dir(treadmill_root).hexdigest()
_LOGGER.info('Checksum: %s', digest)
context.GLOBAL.resolve(cell)
zkclient = context.GLOBAL.zk.conn
if not servers:
# pylint: disable=R0204
servers = zkutils.get(zkclient, zkutils.SERVERS)
if dry_run:
failed = versionmgr.verify(zkclient, digest, servers)
else:
failed = versionmgr.upgrade(
zkclient,
digest,
servers,
batch,
timeout,
stop_on_error=(not continue_on_error),
force_upgrade=force,
)
if not failed:
_LOGGER.info('All servers are up to date.')
else:
_LOGGER.error('Upgrade failed.')
utils.print_yaml(failed)
def version_monitor(approot, command):
"""Runs node version monitor."""
cli_cmd = list(command)
_LOGGER.info('Initializing code monitor: %r', cli_cmd)
watchdogs = watchdog.Watchdog(
os.path.join(
approot,
appenv.AppEnvironment.WATCHDOG_DIR,
))
context.GLOBAL.zk.conn.add_listener(zkutils.exit_on_lost)
while not context.GLOBAL.zk.conn.exists(z.VERSION):
_LOGGER.warn('%r node not created yet. Cell masters running?',
z.VERSION)
time.sleep(30)
hostname = sysinfo.hostname()
version_path = z.path.version(hostname)
codepath = os.path.realpath(utils.rootdir())
digest = versionmgr.checksum_dir(codepath).hexdigest()
_LOGGER.info('codepath: %s, digest: %s', codepath, digest)
info = {
'codepath': codepath,
'since': int(time.time()),
'digest': digest,
}
zkutils.put(context.GLOBAL.zk.conn, version_path, info)
@context.GLOBAL.zk.conn.DataWatch(version_path)
@exc.exit_on_unhandled
def _watch_version(_data, _stat, event):
"""Force exit if server node is deleted."""
# If the node is deleted, we exit to pick up new version code.
if event is not None and event.type == 'DELETED':
# The version info not present, restart services and register
# new checksum.
_LOGGER.info('Upgrade requested, running: %s', cli_cmd)
if cli_cmd:
try:
subproc.check_call(cli_cmd)
# Record successful upgrade.
except subprocess.CalledProcessError:
_LOGGER.exception('Upgrade failed.')
# Immediately trigger a watchdog timeout
watchdogs.create(
name='version_monitor',
timeout='0s',
content='Upgrade to '
'{code!r}({digest}) failed'.format(code=codepath,
digest=digest),
).heartbeat()
del info['digest']
_LOGGER.info('Upgrade complete.')
utils.sys_exit(0)
return True
while True:
time.sleep(100000)
_LOGGER.info('service shutdown.')