def test_json_generator(self):
self.run_cli_cmd('setdeploy local')
host1 = 'host_test1'
self.run_cli_cmd('host add %s' % host1)
inventory = Inventory.load()
path = inventory.create_json_gen_file()
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.assertNotEqual('', msg,
'json generator returned no data: %s' % msg)
self.assertIn(host1, msg,
'%s not in json_gen output: %s' % (host1, msg))
ansible_inventory = AnsibleInventory()
services = ansible_inventory.services
for servicename, service in services.items():
self.assertIn(servicename, msg,
'%s not in json_gen output: %s' % (servicename, msg))
# verify that json output is valid. This will throw if invalid json
try:
json.loads(msg)
except Exception:
self.assertTrue(False, 'invalid json: %s' % msg)
remote_msg = '"ansible_ssh_user": "******"'
local_msg = '"ansible_connection": "local"'
# verify that setdeploy local worked:
self.assertIn(local_msg, msg,
'%s not in local json_gen output: %s' % (local_msg, msg))
self.assertNotIn(remote_msg, msg,
'%s in local json_gen output: %s' % (remote_msg, msg))
# verify that setdeploy remote worked:
CLIENT.set_deploy_mode(remote_mode=True)
inventory = Inventory.load()
path = inventory.create_json_gen_file()
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.assertIn(
remote_msg, msg,
'%s not in remote json_gen output: %s' % (remote_msg, msg))
self.assertNotIn(local_msg, msg,
'%s in remote json_gen output: %s' % (local_msg, msg))
def _add_cmd_info(tar):
# run all the kollacli list commands
cmds = [
'kolla-cli --version', 'kolla-cli service listgroups',
'kolla-cli service list', 'kolla-cli group listservices',
'kolla-cli group listhosts', 'kolla-cli host list',
'kolla-cli property list', 'kolla-cli password list'
]
# collect the json inventory output
inventory = Inventory.load()
inv_path = inventory.create_json_gen_file()
cmds.append(inv_path)
try:
fd, path = tempfile.mkstemp(suffix='.tmp')
os.close(fd)
with open(path, 'w') as tmp_file:
for cmd in cmds:
err_msg, output = run_cmd(cmd, False)
tmp_file.write('\n\n$ %s\n' % cmd)
if err_msg:
tmp_file.write('Error message: %s\n' % err_msg)
for line in output.split('\n'):
tmp_file.write(line + '\n')
tar.add(path, arcname=os.path.join('kolla', 'cmds_output'))
except Exception as e:
raise e
finally:
if path:
os.remove(path)
inventory.remove_json_gen_file(inv_path)
return
def _get_services(self, servicenames, get_all=False):
# type: (List[str], bool) -> List[Service]
services = []
inventory = Inventory.load()
if get_all:
inv_services = inventory.get_services(client_filter=True)
for inv_service in inv_services:
service = Service(inv_service.name,
inv_service.get_parentnames(),
inv_service.get_childnames(),
inv_service.get_groupnames())
services.append(service)
else:
inventory.validate_servicenames(servicenames, client_filter=True)
for servicename in servicenames:
inv_service = inventory.get_service(servicename,
client_filter=True)
if inv_service:
service = Service(inv_service.name,
inv_service.get_parentnames(),
inv_service.get_childnames(),
inv_service.get_groupnames())
services.append(service)
return services
def get_logs(servicenames, hostname, dirname):
inventory = Inventory.load()
inventory.validate_hostnames([hostname])
inventory.validate_servicenames(servicenames, client_filter=True)
logs = HostLogs(hostname, inventory, servicenames)
logs.load_container_info()
logs.filter_services()
logs.write_logs(dirname)
def check_kolla_args(hostnames=[], servicenames=[]):
if not any([hostnames, servicenames]):
return
inventory = Inventory.load()
if hostnames:
inventory.validate_hostnames(hostnames)
if servicenames:
inventory.validate_servicenames(servicenames)
def test_ceph(self):
# ceph has an odd structure in the upstream all-in-one file.
# It was changed in 3.0.1 of kolla. This test is to check that
# the kolla change was not overwritten by a kolla update from upstream.
# If the upstream file is used, ceph will default to having no
# groups.
inventory = Inventory.load()
ceph = inventory.get_service('ceph')
groups = ceph.get_groupnames()
self.assertIsNot(groups, 'ceph has no groups, could be due to '
'using an unpatched upstream all-in-one file.')
def host_get_all():
# type: () -> List[Host]
"""Get all hosts in the inventory
:return: Hosts
:rtype: list of Host objects
"""
inventory = Inventory.load()
hosts = []
host_groups = inventory.get_host_groups()
for hostname, groupnames in host_groups.items():
hosts.append(Host(hostname, groupnames))
return hosts
def test_json_filtering(self):
hosts = ['host_test1', 'host_test2', 'host_test3']
groups = ['control', 'network', 'storage']
for host in hosts:
self.run_cli_cmd('host add %s' % host)
for groupname in groups:
group = CLIENT.group_get([groupname])[0]
group.add_host(host)
inventory = Inventory.load()
# filter by host- include all hosts
inv_filter = {'deploy_hosts': hosts}
path = inventory.create_json_gen_file(inv_filter)
self.log.info('run command: %s' % path)
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.check_json(msg, groups, hosts, groups, hosts)
# filter by host- to first host
included_host = hosts[0]
inv_filter = {'deploy_hosts': [included_host]}
path = inventory.create_json_gen_file(inv_filter)
self.log.info('run command: %s' % path)
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.check_json(msg, groups, hosts, groups, [included_host])
# filter by group- include all groups
inv_filter = {'deploy_groups': groups}
path = inventory.create_json_gen_file(inv_filter)
self.log.info('run command: %s' % path)
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.check_json(msg, groups, hosts, groups, hosts)
# filter by group- to first group
included_group = groups[0]
inv_filter = {'deploy_groups': [included_group]}
path = inventory.create_json_gen_file(inv_filter)
self.log.info('run command: %s' % path)
(retval, msg) = self.run_command(path)
inventory.remove_json_gen_file(path)
self.assertEqual(0, retval, 'json generator command failed: %s' % msg)
self.check_json(msg, groups, hosts, [included_group], hosts)
def get_deploy_mode():
"""Get deploy mode from either local or remote.
Local indicates that the openstack deployment will be
to the local host. Remote means that the deployment is
executed via ssh.
NOTE: local mode is not supported and should never be used
in production environments.
"""
inventory = Inventory.load()
remote_mode = inventory.remote_mode
deploy_mode = 'remote' if remote_mode else 'local'
return deploy_mode
def group_remove(self, groupnames):
# type: (List[str]) -> None
"""Remove groups from the inventory
:param groupnames: names of the groups to remove from the inventory
:type groupnames: list of strings
"""
check_arg(groupnames, u._('Group names'), list)
groupnames = safe_decode(groupnames)
inventory = Inventory.load()
for groupname in groupnames:
inventory.remove_group(groupname)
Inventory.save(inventory)
def group_add(self, groupnames):
# type: (List[str]) -> None
"""Add groups to the inventory
:param groupnames: names of the groups to add to the inventory
:type groupnames: list of strings
"""
check_arg(groupnames, u._('Group names'), list)
groupnames = safe_decode(groupnames)
inventory = Inventory.load()
for groupname in groupnames:
inventory.add_group(groupname)
Inventory.save(inventory)
def run(self):
try:
self.inventory = Inventory.load()
inventory_path = self._make_temp_inventory()
cmd = self._get_playbook_cmd(inventory_path)
self._log_ansible_cmd(cmd, inventory_path)
# create and run the job
job = AnsibleJob(cmd, self.deploy_id, self.print_output,
inventory_path)
job._ignore_error_strings = self.ignore_error_strings
job.run()
return job
except Exception:
raise Exception(traceback.format_exc())
def _get_groups(self, groupnames, get_all=False):
# type: (List[str], bool) -> List[Group]
groups = []
inventory = Inventory.load()
if get_all:
groupnames = inventory.get_groupnames()
else:
inventory.validate_groupnames(groupnames)
group_services = inventory.get_group_services()
for groupname in groupnames:
inv_group = inventory.get_group(groupname)
group = Group(groupname, group_services[groupname],
inv_group.get_hostnames())
groups.append(group)
return groups
def host_remove(hostnames):
# type: (List[str]) -> None
"""Remove hosts from the inventory
:param hostnames: list of strings
"""
check_arg(hostnames, u._('Host names'), list)
hostnames = safe_decode(hostnames)
inventory = Inventory.load()
any_changed = False
for hostname in hostnames:
changed = inventory.remove_host(hostname)
if changed:
any_changed = True
if any_changed:
Inventory.save(inventory)
def host_get(hostnames):
# type: (List[str]) -> List[Host]
"""Get selected hosts in the inventory
:param hostnames: list of strings
:return: hosts
:rtype: list of Host objects
"""
check_arg(hostnames, u._('Host names'), list)
hostnames = safe_decode(hostnames)
inventory = Inventory.load()
inventory.validate_hostnames(hostnames)
hosts = []
host_groups = inventory.get_host_groups()
for hostname in hostnames:
hosts.append(Host(hostname, host_groups[hostname]))
return hosts
def set_deploy_mode(remote_mode):
# type: (bool) -> None
"""Set deploy mode.
Set deploy mode to either local or remote. Local indicates
that the openstack deployment will be to the local host.
Remote means that the deployment is executed via ssh.
NOTE: local mode is not supported and should never be used
in production environments.
:param remote_mode: if remote mode is True deployment is done via ssh
:type remote_mode: bool
"""
check_arg(remote_mode, u._('Remote mode'), bool)
inventory = Inventory.load()
inventory.set_deploy_mode(remote_mode)
Inventory.save(inventory)
def _run_deploy_rules(self, playbook):
properties = AnsibleProperties()
inventory = Inventory.load()
# cannot have both groups and hosts
if playbook.hosts and playbook.groups:
raise InvalidArgument(
u._('Hosts and Groups arguments cannot '
'both be present at the same time.'))
# verify that all services exists
if playbook.services:
for service in playbook.services:
valid_service = inventory.get_service(service)
if not valid_service:
raise NotInInventory(u._('Service'), service)
# check that every group with enabled services
# has hosts associated to it
group_services = inventory.get_group_services()
failed_groups = []
failed_services = []
if group_services:
for (groupname, servicenames) in group_services.items():
group = inventory.get_group(groupname)
hosts = group.get_hostnames()
group_needs_host = False
if not hosts:
for servicename in servicenames:
if self._is_service_enabled(servicename, inventory,
properties):
group_needs_host = True
failed_services.append(servicename)
if group_needs_host:
failed_groups.append(groupname)
if len(failed_groups) > 0:
raise InvalidConfiguration(
u._('Deploy failed. '
'Groups: {groups} with enabled '
'services : {services} '
'have no associated hosts').format(
groups=failed_groups, services=failed_services))
def add_service(self, servicename):
# type: (str) -> None
"""Add service to group
:param servicename: name of the service to add to the group
:type servicename: string
"""
check_arg(servicename, u._('Service name'), str)
servicename = safe_decode(servicename)
inventory = Inventory.load()
inventory.validate_servicenames([servicename], client_filter=True)
group_services = inventory.get_group_services()
self._servicenames = group_services[self.name]
if servicename not in self._servicenames:
# service not associated with group, add it
inventory.add_group_to_service(self.name, servicename)
self._servicenames.append(servicename)
Inventory.save(inventory)
def host_setup(hosts_info):
# type: (Dict[str,Dict[str,object]]) -> None
"""Setup multiple hosts for ssh access
hosts_info is a dictionary of form:
- {hostname': {
'password': password
'uname': user_name},
...
}
The uname entry is optional.
:param hosts_info: dictionary
"""
check_arg(hosts_info, u._('Hosts info'), dict)
inventory = Inventory.load()
inventory.validate_hostnames(hosts_info.keys())
inventory.setup_hosts(hosts_info)
def host_destroy(hostnames,
destroy_type,
verbose_level=1,
include_data=False,
remove_images=False):
# type: (List[str], str, int, bool, bool) -> Job
"""Destroy Hosts.
Stops and removes all kolla related docker containers on the
specified hosts.
:param hostnames: host names
:type hostnames: list
:param destroy_type: either 'kill' or 'stop'
:type destroy_type: string
:param verbose_level: the higher the number, the more verbose
:type verbose_level: integer
:param include_data: if true, destroy data containers too.
:type include_data: boolean
:param remove_images: if true, destroy will remove the docker images
:type remove_images: boolean
:return: Job object
:rtype: Job
"""
check_arg(hostnames, u._('Host names'), list)
check_arg(destroy_type, u._('Destroy type'), str)
check_arg(verbose_level, u._('Verbose level'), int)
check_arg(include_data, u._('Include data'), bool)
check_arg(remove_images, u._('Remove images'), bool)
if destroy_type not in ['stop', 'kill']:
raise InvalidArgument(
u._('Invalid destroy type ({type}). Must be either '
'"stop" or "kill".').format(type=destroy_type))
hostnames = safe_decode(hostnames)
inventory = Inventory.load()
inventory.validate_hostnames(hostnames)
action = KollaAction(verbose_level=verbose_level,
playbook_name='destroy.yml')
ansible_job = action.destroy_hosts(hostnames, destroy_type,
include_data, remove_images)
return Job(ansible_job)
def remove_service(self, servicename):
# type: (str) -> None
"""Remove service from group
:param servicename: name of the service to remove from the group
:type servicename: string
"""
check_arg(servicename, u._('Service name'), str)
servicename = safe_decode(servicename)
inventory = Inventory.load()
inventory.validate_servicenames([servicename], client_filter=True)
group_services = inventory.get_group_services()
self._servicenames = group_services[self.name]
if servicename in self._servicenames:
# service is associated with group, remove it
inventory.remove_group_from_service(self.name, servicename)
self._servicenames.remove(servicename)
Inventory.save(inventory)
def remove_host(self, hostname):
# type: (str) -> None
"""Remove host from group
:param hostname: name of the host to remove from the group
:type hostname: string
"""
check_arg(hostname, u._('Host name'), str)
hostname = safe_decode(hostname)
inventory = Inventory.load()
inventory.validate_hostnames([hostname])
group = inventory.get_group(self.name)
self._hostnames = group.get_hostnames()
if hostname in self._hostnames:
# host is associated with group, remove it
inventory.remove_host(hostname, self.name)
self._hostnames.remove(hostname)
Inventory.save(inventory)
def add_host(self, hostname):
# type: (str) -> None
"""Add host to group
:param hostname: name of the host to add to the group
:type hostname: string
"""
check_arg(hostname, u._('Host name'), str)
hostname = safe_decode(hostname)
inventory = Inventory.load()
inventory.validate_hostnames([hostname])
group = inventory.get_group(self.name)
self._hostnames = group.get_hostnames()
if hostname not in self._hostnames:
# host not associated with group, add it
inventory.add_host(hostname, self.name)
self._hostnames.append(hostname)
Inventory.save(inventory)
def host_ssh_check(hostnames):
# type: (List[str]) -> Dict[str,Dict[str,object]]
"""Check hosts for ssh connectivity
Check status is a dictionary of form:
- {hostname: {
'success':<True|False>,
'msg':message_string},
...
}
:param hostnames: list of strings
:return: check status
:rtype: dictionary
"""
check_arg(hostnames, u._('Host names'), list)
inventory = Inventory.load()
hostnames = safe_decode(hostnames)
inventory.validate_hostnames(hostnames)
summary = inventory.ssh_check_hosts(hostnames)
return summary
def host_precheck(hostnames, verbose_level=1):
# type: (List[str], int) -> Job
"""Check pre-deployment configuration of hosts.
Check if host is ready for a new deployment. This will fail if
any of the hosts are not configured correctly or if they have
already been deployed to.
:param hostnames: host names
:type hostnames: list
:param verbose_level: the higher the number, the more verbose
:type verbose_level: integer
:return: Job object
:rtype: Job
"""
check_arg(hostnames, u._('Host names'), list)
check_arg(verbose_level, u._('Verbose level'), int)
hostnames = safe_decode(hostnames)
inventory = Inventory.load()
inventory.validate_hostnames(hostnames)
ansible_job = actions.precheck(hostnames, verbose_level)
return Job(ansible_job)
def run(self):
try:
if self.local_only:
# Create a temporary local inventory with only localhost
self.inventory = Inventory()
self.inventory.set_deploy_mode(False)
self.inventory.add_host('localhost')
else:
self.inventory = Inventory.load()
inventory_path = self._make_temp_inventory()
cmd = self._get_playbook_cmd(inventory_path)
self._log_ansible_cmd(cmd, inventory_path)
# create and run the job
job = AnsibleJob(cmd, self.deploy_id, self.print_output,
inventory_path)
job._ignore_error_strings = self.ignore_error_strings
job.run()
return job
except Exception:
raise Exception(traceback.format_exc())
def host_stop(hostnames, verbose_level=1):
# type: (List[str], int) -> Job
"""Stop Hosts.
Stops all kolla related docker containers on the specified hosts.
:param hostnames: host names
:type hostnames: list
:param verbose_level: the higher the number, the more verbose
:type verbose_level: integer
:return: Job object
:rtype: Job
"""
check_arg(hostnames, u._('Host names'), list)
check_arg(verbose_level, u._('Verbose level'), int)
hostnames = safe_decode(hostnames)
inventory = Inventory.load()
inventory.validate_hostnames(hostnames)
ansible_job = actions.stop_hosts(hostnames, verbose_level)
return Job(ansible_job)
def _load_inventory(self):
if not self._inventory:
self._inventory = Inventory.load() # nosec
def test_service_add_group(self):
servicename = 'cinder'
new_group = 'network'
inventory = Inventory.load()
service = inventory.get_service(servicename)
groupnames = service.get_groupnames()
# add new group to a service
self.run_cli_cmd('service addgroup %s %s' % (servicename, new_group))
msg = self.run_cli_cmd('service listgroups -f json')
cli_services = json.loads(msg)
cli_service = ''
for svc in cli_services:
if svc['Service'] == servicename:
cli_service = svc
break
self.assertNotEqual(cli_service, '',
'service: %s, ' % servicename +
'not found in cli_services: \n%s'
% cli_service)
cli_groups = cli_service['Groups']
groupnames.append(new_group)
self.assertEqual(groupnames, cli_groups,
'service: %s, ' % service +
'expected groups: %s, ' % groupnames +
'cli_groups: %s' % cli_groups)
# remove that group
self.run_cli_cmd('service removegroup %s %s' % (servicename,
new_group))
groupnames.remove(new_group)
msg = self.run_cli_cmd('service listgroups -f json')
cli_services = json.loads(msg)
for svc in cli_services:
if svc['Service'] == servicename:
cli_service = svc
break
self.assertNotEqual(cli_service, '',
'service: %s, ' % servicename +
'not found in cli_services: \n%s'
% cli_service)
cli_groups = cli_service['Groups']
expected_groups = groupnames
self.assertEqual(expected_groups, cli_groups,
'service: %s, ' % service +
'expected groups: %s, ' % expected_groups +
'cli_groups: %s' % cli_groups)
# add new group to a service which has a parent
servicename = 'glance-api'
new_group = 'control'
self.run_cli_cmd('service addgroup %s %s' % (servicename, new_group))
msg = self.run_cli_cmd('service listgroups -f json')
cli_services = json.loads(msg)
for svc in cli_services:
if svc['Service'] == servicename:
cli_service = svc
break
self.assertNotEqual(cli_service, '',
'service: %s, ' % servicename +
'not found in cli_services: \n%s'
% cli_service)
cli_groups = cli_service['Groups']
expected_groups = ['%s' % new_group]
self.assertEqual(expected_groups, cli_groups,
'service: %s, ' % service +
'expected groups: %s, ' % expected_groups +
'cli_groups: %s' % cli_groups)
# remove that group
self.run_cli_cmd('service removegroup %s %s' % (servicename,
new_group))
msg = self.run_cli_cmd('service listgroups -f json')
cli_services = json.loads(msg)
for svc in cli_services:
if svc['Service'] == servicename:
cli_service = svc
break
self.assertNotEqual(cli_service, '',
'service: %s, ' % service +
'not found in cli_services: \n%s'
% cli_service)
cli_groups = cli_service['Groups']
expected_groups = []
self.assertEqual(expected_groups, cli_groups,
'service: %s, ' % servicename +
'expected groups: %s, ' % expected_groups +
'cli_groups: %s' % cli_groups)
test_group = 'testgroup'
self.run_cli_cmd('group add %s' % test_group)
self.run_cli_cmd('service addgroup cinder %s' % test_group)
self.run_cli_cmd('group remove %s' % test_group)
msg = self.run_cli_cmd('service listgroups -f json')
self.assertNotIn(test_group, msg,
'Group: %s, still listed in services: %s'
% (test_group, msg))
def _properties_test(self, groups=[], hosts=[]):
switch = ''
if groups:
switch = '--groups'
dir_name = 'group_vars'
elif hosts:
switch = '--hosts'
dir_name = 'host_vars'
key = 'TeStKeY'
value = 'TeStVaLuE:123:abc'
# initialize keys
targets_csv = ''
targets = groups + hosts
if 'all' in groups:
inv = Inventory.load()
targets = inv.get_groupnames()
targets_csv = 'all'
elif 'all' in hosts:
inv = Inventory.load()
targets = inv.get_hostnames()
targets_csv = 'all'
comma = ''
sizes = {} # key = path, value = [size1, size2, etc]
for target in targets:
self.run_cli_cmd('property clear %s %s %s' % (switch, target, key))
if targets_csv != 'all':
targets_csv += comma + target
comma = ','
path = os.path.join(get_kolla_ansible_home(), 'ansible', dir_name,
target)
sizes[path] = [os.path.getsize(path)]
if not switch:
self.run_cli_cmd('property clear %s' % key)
path = os.path.join(get_kolla_ansible_home(),
'ansible/group_vars/__GLOBAL__')
sizes[path] = [os.path.getsize(path)]
# test append
self.run_cli_cmd('property set %s %s %s %s' %
(switch, targets_csv, key, value))
if switch:
msg = self.run_cli_cmd('property list -f json %s all' % (switch))
else:
msg = self.run_cli_cmd('property list -f json')
err_msg = self._check_property_values(key, value, msg, targets)
self.assertEqual(
err_msg, '', 'set failed property not in output: %s, %s (%s %s)' %
(key, value, switch, targets_csv))
bad_path = self._is_size_ok(sizes, 0, '<', 1)
self.assertIsNone(
bad_path, 'Size of file %s did not ' % bad_path +
'increase after append (%s %s)' % (switch, targets_csv))
# test modify existing
value += '2'
self.run_cli_cmd('property set %s %s %s %s' %
(switch, targets_csv, key, value))
msg = self.run_cli_cmd('property list --all -f json %s %s' %
(switch, targets_csv))
err_msg = self._check_property_values(key, value, msg, targets)
self.assertEqual(
err_msg, '', 'set failed property not in output: %s, %s (%s %s)' %
(key, value, switch, targets_csv))
bad_path = self._is_size_ok(sizes, 1, '<', 2)
self.assertIsNone(
bad_path, 'Size of file %s did not ' % bad_path +
'increase after modify (%s %s)' % (switch, targets_csv))
# test clear
self.run_cli_cmd('property clear %s %s %s' %
(switch, targets_csv, key))
msg = self.run_cli_cmd('property list --long -f json %s %s' %
(switch, targets_csv))
err_msg = self._check_property_values(key, value, msg, targets)
self.assertTrue(
'missing' in err_msg, 'clear failed, property still in output: ' +
'%s, %s (%s %s)' % (key, value, switch, targets_csv))
bad_path = self._is_size_ok(sizes, 0, '=', 3)
self.assertIsNone(
bad_path,
'Size of file %s is ' % bad_path + 'different from initial size '
'(%s %s %s)' % (switch, targets_csv, str(sizes)))
# test setting empty string
value = '""'
self.run_cli_cmd('property set %s %s %s %s' %
(switch, targets_csv, key, value))
msg = self.run_cli_cmd('property list --all -f json %s %s' %
(switch, targets_csv))
err_msg = self._check_property_values(key, value, msg, targets)
self.assertTrue(
'missing' in err_msg, 'clear failed, property still in output: ' +
'%s, %s (%s %s)' % (key, value, switch, targets_csv))
self.run_cli_cmd('property clear %s %s %s' %
(switch, targets_csv, key))