def upgrade_machine(app_name, unit, machine, machine_num):
"""Run the upgrade process for a single machine"""
cmd = [
kiki.cmd(), 'run', '--unit', unit, 'status-set', 'maintenance',
'Upgrading series'
]
subprocess.call(cmd)
if not do_release_upgrade(unit):
return False
if machine["series"] == "trusty":
upstart_to_systemd(machine_num)
cmd = [kiki.cmd(), 'run', '--unit', unit, 'status-set', 'active']
subprocess.call(cmd)
logging.debug("Rebooting")
reboot(unit)
cmd = [kiki.cmd(), "ssh", unit, "exit"]
while (True):
try:
subprocess.check_call(cmd)
break
except subprocess.CalledProcessError:
logging.debug("Waiting 2 more seconds")
time.sleep(2)
update_machine_series(app_name, machine_num)
return True
def update_machine_series(app_name, machine_num):
cmd = [
kiki.cmd(), 'run', '--machine', machine_num, 'lsb_release', '-c', '-s'
]
codename = subprocess.check_output(cmd)
if six.PY3:
codename = codename.decode('utf-8')
codename = codename.strip()
logging.debug("Telling juju that {} series is {}".format(
machine_num, codename))
cmd = [kiki.cmd(), 'update-series', str(machine_num), codename]
subprocess.call(cmd)
cmd = [kiki.cmd(), 'update-series', app_name, codename]
subprocess.call(cmd)
def juju_set(service, option, wait=None):
if wait is None:
wait = True
logging.info('Setting %s to %s' % (service, option))
subprocess.check_call([kiki.cmd(), kiki.set_config(), service, option])
if wait:
juju_wait_finished()
def remote_upload(unit, script, remote_dir=None):
if remote_dir:
dst = unit + ':' + remote_dir
else:
dst = unit + ':/tmp/'
cmd = [kiki.cmd(), 'scp', script, dst]
return subprocess.check_call(cmd)
def remote_shell_check(unit, timeout=None):
cmd = [kiki.cmd(), 'run']
if timeout:
cmd.extend(['--timeout', str(timeout)])
cmd.extend(['--unit', unit, 'uname -a'])
FNULL = open(os.devnull, 'w')
return not subprocess.call(cmd, stdout=FNULL, stderr=subprocess.STDOUT)
def reboot(unit):
"""Reboot machine"""
cmd = [kiki.cmd(), 'run', '--unit', unit, 'sudo', 'reboot', '&&', 'exit']
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
logging.info(e)
pass
def get_juju_status(service=None, unit=None):
cmd = [kiki.cmd(), 'status', '--format=yaml']
if service:
cmd.append(service)
if unit:
cmd.append(unit)
status_file = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
return yaml.load(status_file)
def juju_get(service, option):
cmd = [kiki.cmd(), kiki.get_config(), service]
juju_get_output = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
service_config = yaml.load(juju_get_output)
if (option in service_config['settings']
and 'value' in service_config['settings'][option]):
return service_config['settings'][option]['value']
else:
# Stable charms may not yet have the same config keys as next charms
return None
def do_release_upgrade(unit):
"""Runs do-release-upgrade noninteractive"""
logging.info('Upgrading ' + unit)
subprocess.call([
kiki.cmd(), 'run', '--unit', unit, 'status-set', 'maintenance',
'Doing release upgrade'
])
cmd = [
kiki.cmd(), 'ssh', unit, 'sudo', 'do-release-upgrade', '-f',
'DistUpgradeViewNonInteractive'
]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
logging.warn("Failed do-release-upgrade for {}".format(unit))
logging.warn(e)
return False
finally:
subprocess.call(
[kiki.cmd(), 'run', '--unit', unit, 'status-set', 'active'])
return True
def units_upstart_to_systemd_commands(machine_number):
"""Upgrade a specific application unit from Upstart to Systemd"""
units = get_juju_status(unit=str(machine_number))["applications"]
base_command = [kiki.cmd(), 'run', '--machine', str(machine_number), '--']
commands = []
for (name, app_unit) in units.iteritems():
for (unit_name, unit) in app_unit["units"].iteritems():
logging.debug("Updating {} [{}]".format(name, unit_name))
app_number = unit_name.split("/")[-1]
systemd_file_name = ("jujud-unit-{app_name}"
"-{app_number}.service").format(
app_name=name, app_number=app_number)
systemd_file_path = ('/var/lib/juju/init/jujud-unit-{app_name}'
'-{app_number}/{file_name}').format(
app_name=name,
app_number=app_number,
file_name=systemd_file_name)
commands += [
base_command + [
"sudo", "mkdir", "-p",
"/var/lib/juju/init/jujud-unit-{}-{}".format(
name, app_number)
], base_command + [
'echo',
SYSTEMD_JUJU_UNIT_AGENT_SCRIPT.format(
application=unit_name,
application_name=name,
application_number=app_number), '|', 'sudo', 'tee',
('/var/lib/juju/init/jujud-unit-{app_name}-'
'{app_number}/exec-start.sh').format(
app_name=name, app_number=app_number)
], base_command + [
'echo',
SYSTEMD_JUJU_UNIT_INIT_FILE.format(
application_name=name, application_number=app_number),
'|', 'sudo', 'tee', systemd_file_path
], base_command + [
'sudo', 'chmod', '755',
('/var/lib/juju/init/jujud-unit-{app_name}-'
'{app_number}/exec-start.sh').format(
app_name=name, app_number=app_number)
], base_command + [
'sudo', 'ln', '-s', systemd_file_path,
'/etc/systemd/system/'
], base_command + [
'sudo', 'ln', '-s', systemd_file_path,
('/etc/systemd/system/multi-user.target.wants/'
'{file_name}').format(machine_id=machine_number,
file_name=systemd_file_name)
]
]
return commands
def get_provider_type():
""" Get the type of the undercloud
@returns String name of the undercloud type
"""
juju_env = subprocess.check_output([kiki.cmd(), 'switch']).strip('\n')
if kiki.version() < 2:
juju_env_contents = get_juju_environments_yaml()
return juju_env_contents['environments'][juju_env]['type']
else:
cloud = get_cloud_from_controller()
if cloud:
# If the controller was deployed from this system with
# the cloud configured in ~/.local/share/juju/clouds.yaml
# Determine the cloud type directly
cmd = [kiki.cmd(), 'show-cloud', cloud, '--format=yaml']
return yaml.load(subprocess.check_output(cmd))['type']
else:
# If the controller was deployed elsewhere
# show-controllers unhelpfully returns an empty string for cloud
# For now assume openstack
return 'openstack'
def delete_unit_juju(unit):
service = unit.split('/')[0]
unit_count = len(get_juju_units(service=service))
logging.info('Removing unit ' + unit)
cmd = [kiki.cmd(), kiki.remove_unit(), unit]
subprocess.check_call(cmd)
target_num = unit_count - 1
# Wait for the unit to disappear from juju status
while len(get_juju_units(service=service)) > target_num:
# Check no hooks are in error state
juju_status_check_and_wait()
time.sleep(5)
juju_wait_finished()
def get_cloud_from_controller():
""" Get the cloud name from the Juju 2.x controller
@returns String name of the cloud for the current Juju 2.x controller
"""
cmd = [kiki.cmd(), 'show-controller', '--format=yaml']
cloud_config = yaml.load(subprocess.check_output(cmd))
# There will only be one top level controller from show-controller,
# but we do not know its name.
assert len(cloud_config) == 1
try:
return cloud_config.values()[0]['details']['cloud']
except KeyError:
raise KeyError("Failed to get cloud information from the controller")
def add_unit(service, unit_num=None):
unit_count = len(get_juju_units(service=service))
if unit_num:
additional_units = int(unit_num)
else:
additional_units = 1
logging.info('Adding %i unit(s) to %s' % (additional_units, service))
cmd = [kiki.cmd(), 'add-unit', service, '-n', str(additional_units)]
subprocess.check_call(cmd)
target_num = unit_count + additional_units
# Wait for the new unit to appear in juju status
while len(get_juju_units(service=service)) < target_num:
time.sleep(5)
juju_wait_finished()
def upgrade_service(svc, charm_name=None, switch=None):
if charm_name and os.path.exists(os.path.join(get_charm_dir(),
charm_name)):
charm_dir = os.path.join(get_charm_dir(), charm_name)
else:
charm_dir = os.path.join(get_charm_dir(), svc)
logging.info('Upgrading ' + svc)
cmd = [kiki.cmd(), 'upgrade-charm']
# Switch and path are now mutually exclusive
if switch and switch.get(svc):
cmd.extend(['--switch', charm_dir, svc])
else:
cmd.extend(['--path', charm_dir, svc])
subprocess.check_call(cmd)
def remote_run(unit, remote_cmd=None, timeout=None, fatal=None):
if fatal is None:
fatal = True
cmd = [kiki.cmd(), 'run', '--unit', unit]
if timeout:
cmd.extend(['--timeout', str(timeout)])
if remote_cmd:
cmd.append(remote_cmd)
else:
cmd.append('uname -a')
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
output = p.communicate()
if p.returncode != 0 and fatal:
raise Exception('Error running remote command')
return output
def upstart_to_systemd(machine_number):
"""Upgrade upstart scripts to Systemd after upgrade from Trusty"""
base_command = [kiki.cmd(), 'run', '--machine', str(machine_number), '--']
commands = [
base_command + [
"sudo", "mkdir", "-p",
"/var/lib/juju/init/jujud-machine-{}".format(machine_number)
], base_command + [
'echo',
SYSTEMD_JUJU_MACHINE_AGENT_SCRIPT.format(
machine_id=machine_number), '|', 'sudo', 'tee',
('/var/lib/juju/init/jujud-machine-{machine_id}'
'/exec-start.sh').format(machine_id=machine_number)
], base_command + [
'echo',
SYSTEMD_JUJU_MACHINE_INIT_FILE.format(name=machine_number), '|',
'sudo', 'tee',
('/var/lib/juju/init/jujud-machine-{name}'
'/jujud-machine-{name}.service').format(name=machine_number)
], base_command + [
'sudo', 'chmod', '755',
('/var/lib/juju/init/jujud-machine-{machine_id}/'
'exec-start.sh').format(machine_id=machine_number)
], base_command + [
'sudo', 'ln', '-s',
('/var/lib/juju/init/jujud-machine-{machine_id}/'
'jujud-machine-{machine_id}.service').format(
machine_id=machine_number), '/etc/systemd/system/'
], base_command + [
'sudo', 'ln', '-s',
('/var/lib/juju/init/jujud-machine-{machine_id}/'
'jujud-machine-{machine_id}.service').format(
machine_id=machine_number),
('/etc/systemd/system/multi-user.target.wants/'
'jujud-machine-{machine_id}.service').format(
machine_id=machine_number)
]
]
commands += units_upstart_to_systemd_commands(machine_number)
for cmd in commands:
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
logging.warn(e)
return False
def delete_application(application, wait=True):
logging.info('Removing application ' + application)
cmd = [kiki.cmd(), kiki.remove_application(), application]
subprocess.check_call(cmd)
def juju_get_config_keys(service):
cmd = [kiki.cmd(), kiki.get_config(), service]
juju_get_output = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
service_config = yaml.load(juju_get_output)
return service_config['settings'].keys()
