def sync_all(environment):
logger.info(
'Syncing all salt systems for environment: {0!r}'.format(environment))
client = salt.client.LocalClient(
settings.STACKDIO_CONFIG.salt_master_config)
ret = client.cmd_iter('env:environments.{}'.format(environment.name),
'saltutil.sync_all',
expr_form='grain')
result = {}
for res in ret:
for host, data in res.items():
result[host] = data
for host, data in result.items():
if 'retcode' not in data:
logger.warning(
'Host {0} missing a retcode... assuming failure'.format(host))
if data.get('retcode', 1) != 0:
err_msg = six.text_type(data['ret'])
raise EnvironmentTaskException(
'Error syncing salt data: {0!r}'.format(err_msg))
def query(hyper=None, quiet=False):
"""
Query the virtual machines. When called without options all hypervisors
are detected and a full query is returned. A single hypervisor can be
passed in to specify an individual hypervisor to query.
"""
ret = {}
client = salt.client.get_local_client(__opts__["conf_file"])
for info in client.cmd_iter("virtual:physical", "virt.full_info", expr_form="grain"):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.iterkeys().next()
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if "ret" not in info[id_]:
continue
if not isinstance(info[id_]["ret"], dict):
continue
chunk[id_] = info[id_]["ret"]
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, "virt_query", __opts__)
return ret
def cmd(tgt,
fun,
arg=(),
timeout=None,
expr_form='glob',
ret='',
kwarg=None,
ssh=False,
**kwargs):
'''
Assuming this minion is a master, execute a salt command
CLI Example:
.. code-block:: bash
salt '*' saltutil.cmd
'''
if ssh:
client = salt.client.SSHClient(os.path.dirname(__opts__['conf_file']))
else:
client = salt.client.LocalClient(os.path.dirname(
__opts__['conf_file']))
ret = {}
for ret_comp in client.cmd_iter(tgt, fun, arg, timeout, expr_form, ret,
kwarg, **kwargs):
ret.update(ret_comp)
return ret
def query(hyper=None, quiet=False):
'''
Query the virtual machines. When called without options all hypervisors
are detected and a full query is returned. A single hypervisor can be
passed in to specify an individual hypervisor to query.
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.full_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'virt_query', __opts__)
return ret
def __virtual__():
'''
Provides vmadm runner
'''
have_computenodes = False
## check if there are computenode minions
client = salt.client.get_local_client(__opts__['conf_file'])
try:
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'test.ping',
expr_form='compound'):
if not cn:
continue
else:
have_computenodes = True
except SaltClientError as client_error:
pass
if have_computenodes:
return __virtualname__
else:
return (
False,
'{0} runner can only be loaded when there are SmartOS computed nodes'
.format(__virtualname__))
def resume(name):
'''
Resume a paused vm
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'error': 'Failed to find VM {0} to pause'.format(name)}, 'progress')
return 'not found'
hyper = next(data.iterkeys())
if data[hyper][name]['state'] != 'paused':
__jid_event__.fire_event({'error': 'VM {0} is not paused'.format(name)}, 'progress')
return 'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.resume',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be resumed: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Resumed VM {0}'.format(name)}, 'progress')
return 'good'
def query(hyper=None, quiet=False):
'''
Query the virtual machines
'''
ret = {}
client = salt.client.LocalClient(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.full_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'virt_query', __opts__)
return ret
def pause(name):
'''
Pause the named VM
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event(
{'error': 'Failed to find VM {0} to pause'.format(name)},
'progress')
return 'fail'
host = next(six.iterkeys(data))
if data[host][name]['state'] == 'paused':
__jid_event__.fire_event(
{'error': 'VM {0} is already paused'.format(name)}, 'progress')
return 'bad state'
try:
cmd_ret = client.cmd_iter(host, 'virt.pause', [name], timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be pasued: {1}'.format(
name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Paused VM {0}'.format(name)},
'progress')
return 'good'
def query(hyper=None, quiet=False):
"""
Query the virtual machines
"""
ret = {}
client = salt.client.LocalClient(__opts__["conf_file"])
for info in client.cmd_iter("virtual:physical", "virt.full_info", expr_form="grain"):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if not "ret" in info[id_]:
continue
chunk[id_] = info[id_]["ret"]
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, "virt_query", __opts__)
return ret
def query(hyper=None, quiet=False):
'''
Query the virtual machines
'''
ret = {}
client = salt.client.LocalClient(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.full_info',
expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'virt_query', __opts__)
return ret
def purge(name, delete_key=True):
"""
Destroy the named VM
"""
ret = {}
client = salt.client.get_local_client(__opts__["conf_file"])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event(
{"error": "Failed to find VM {0} to purge".format(name)}, "progress"
)
return "fail"
host = next(six.iterkeys(data))
try:
cmd_ret = client.cmd_iter(host, "virt.purge", [name, True], timeout=600)
except SaltClientError as client_error:
return "Virtual machine {0} could not be purged: {1}".format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
if delete_key:
log.debug("Deleting key %s", name)
skey = salt.key.Key(__opts__)
skey.delete_key(name)
__jid_event__.fire_event({"message": "Purged VM {0}".format(name)}, "progress")
return "good"
def resume(name):
"""
Resume a paused VM
"""
ret = {}
client = salt.client.get_local_client(__opts__["conf_file"])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event(
{"error": "Failed to find VM {0} to pause".format(name)}, "progress"
)
return "not found"
host = next(six.iterkeys(data))
if data[host][name]["state"] != "paused":
__jid_event__.fire_event(
{"error": "VM {0} is not paused".format(name)}, "progress"
)
return "bad state"
try:
cmd_ret = client.cmd_iter(host, "virt.resume", [name], timeout=600)
except SaltClientError as client_error:
return "Virtual machine {0} could not be resumed: {1}".format(
name, client_error
)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({"message": "Resumed VM {0}".format(name)}, "progress")
return "good"
def _list_iter(host=None, path=None):
"""
Return a generator iterating over hosts
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: 2015.8.0
"""
tgt = host or "*"
client = salt.client.get_local_client(__opts__["conf_file"])
for container_info in client.cmd_iter(tgt, "lxc.list", kwarg={"path": path}):
if not container_info:
continue
if not isinstance(container_info, dict):
continue
chunk = {}
id_ = next(iter(container_info.keys()))
if host and host != id_:
continue
if not isinstance(container_info[id_], dict):
continue
if "ret" not in container_info[id_]:
continue
if not isinstance(container_info[id_]["ret"], dict):
continue
chunk[id_] = container_info[id_]["ret"]
yield chunk
def query(hyper=None, quiet=False):
'''
Query the virtual machines. When called without options all hypervisors
are detected and a full query is returned. A single hypervisor can be
passed in to specify an individual hypervisor to query.
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.full_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'virt_query', __opts__)
return ret
def list_(host=None, quiet=False):
'''
List defined containers (running, stopped, and frozen) for the named
(or all) host(s).
.. code-block:: bash
salt-run lxc.list [host=minion_id]
'''
tgt = host or '*'
ret = {}
client = salt.client.LocalClient(__opts__['conf_file'])
for container_info in client.cmd_iter(tgt, 'lxc.list'):
if not container_info:
continue
if not isinstance(container_info, dict):
continue
chunk = {}
id_ = container_info.keys()[0]
if host and host != id_:
continue
if not isinstance(container_info[id_], dict):
continue
if 'ret' not in container_info[id_]:
continue
if not isinstance(container_info[id_]['ret'], dict):
continue
chunk[id_] = container_info[id_]['ret']
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'lxc_list', __opts__)
return ret
def execution():
"""
Collect all the sys.doc output from each minion and return the aggregate
CLI Example:
.. code-block:: bash
salt-run doc.execution
"""
client = salt.client.get_local_client(__opts__["conf_file"])
docs = {}
try:
for ret in client.cmd_iter("*", "sys.doc",
timeout=__opts__["timeout"]):
for v in six.itervalues(ret):
docs.update(v)
except SaltClientError as exc:
print(exc)
return []
i = itertools.chain.from_iterable([six.iteritems(docs["ret"])])
ret = dict(list(i))
return ret
def cmd_iter(tgt,
fun,
arg=(),
timeout=None,
expr_form='glob',
ret='',
kwarg=None,
ssh=False,
**kwargs):
'''
Assuming this minion is a master, execute a salt command
CLI Example:
.. code-block:: bash
salt '*' saltutil.cmd_iter
'''
if ssh:
client = salt.client.SSHClient(__opts__['conf_file'])
else:
client = salt.client.get_local_client(__opts__['conf_file'])
for ret in client.cmd_iter(tgt, fun, arg, timeout, expr_form, ret, kwarg,
**kwargs):
yield ret
def pause(name):
"""
Pause the named VM
"""
ret = {}
client = salt.client.get_local_client(__opts__["conf_file"])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event(
{"error": "Failed to find VM {} to pause".format(name)},
"progress")
return "fail"
host = next(iter(data.keys()))
if data[host][name]["state"] == "paused":
__jid_event__.fire_event(
{"error": "VM {} is already paused".format(name)}, "progress")
return "bad state"
try:
cmd_ret = client.cmd_iter(host, "virt.pause", [name], timeout=600)
except SaltClientError as client_error:
return "Virtual machine {} could not be pasued: {}".format(
name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({"message": "Paused VM {}".format(name)},
"progress")
return "good"
def query(hyper=None, quiet=False):
'''
Query the virtual machines. When called without options all hypervisors
are detected and a full query is returned. A single hypervisor can be
passed in to specify an individual hypervisor to query.
'''
if quiet:
log.warn('\'quiet\' is deprecated. Please migrate to --quiet')
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
try:
for info in client.cmd_iter('virtual:physical',
'virt.full_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = next(info.iterkeys())
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
__jid_event__.fire_event({'data': chunk, 'outputter': 'virt_query'}, 'progress')
except SaltClientError as client_error:
print(client_error)
return ret
def start(name):
'''
Start a named virtual machine
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'message': 'Failed to find vm {0} to start'.format(name)}, 'progress')
return 'fail'
hyper = next(data.iterkeys())
if data[hyper][name]['state'] == 'running':
print('VM {0} is already running'.format(name))
return 'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.start',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} not started: {1}'. format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Started VM {0}'.format(name)}, 'progress')
return 'good'
def create_zombies(stack, n):
"""
For the given stack, will randomly select `n` hosts and kill the
salt-minion service that should already be running on the host.
@param (stacks.models.Stack) stack - the stack we're targeting
@param (int) n - the number of randomly selected hosts to zombify
@returns (None)
"""
# Random sampling of n hosts
hosts = random.sample(stack.hosts.all(), n)
if not hosts:
return
client = salt.client.LocalClient(
settings.STACKDIO_CONFIG.salt_master_config
)
result = list(client.cmd_iter(
[h.hostname for h in hosts],
'service.stop',
arg=('salt-minion',),
expr_form='list'
))
logger.info(result)
def force_off(name):
'''
Force power down the named virtual machine
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
print('Failed to find vm {0} to destroy'.format(name))
return 'fail'
hyper = next(six.iterkeys(data))
if data[hyper][name]['state'] == 'shutdown':
print('VM {0} is already shutdown'.format(name))
return'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.destroy',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be forced off: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Powered off VM {0}'.format(name)}, 'progress')
return 'good'
def start(name):
'''
Start a named virtual machine
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'message': 'Failed to find vm {0} to start'.format(name)}, 'progress')
return 'fail'
hyper = next(six.iterkeys(data))
if data[hyper][name]['state'] == 'running':
print('VM {0} is already running'.format(name))
return 'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.start',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} not started: {1}'. format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Started VM {0}'.format(name)}, 'progress')
return 'good'
def _do(name, fun, path=None):
'''
Invoke a function in the lxc module with no args
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: Beryllium
'''
host = find_guest(name, quiet=True, path=path)
if not host:
return False
client = salt.client.get_local_client(__opts__['conf_file'])
cmd_ret = client.cmd_iter(
host,
'lxc.{0}'.format(fun),
[name],
kwarg={'path': path},
timeout=60)
data = next(cmd_ret)
data = data.get(host, {}).get('ret', None)
if data:
data = {host: data}
return data
def _do_names(names, fun, path=None):
'''
Invoke a function in the lxc module with no args
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: Beryllium
'''
ret = {}
hosts = find_guests(names, path=path)
if not hosts:
return False
client = salt.client.get_local_client(__opts__['conf_file'])
for host, sub_names in six.iteritems(hosts):
cmds = []
for name in sub_names:
cmds.append(client.cmd_iter(
host,
'lxc.{0}'.format(fun),
[name],
kwarg={'path': path},
timeout=60))
for cmd in cmds:
data = next(cmd)
data = data.get(host, {}).get('ret', None)
if data:
ret.update({host: data})
return ret
def _do_names(names, fun, path=None):
'''
Invoke a function in the lxc module with no args
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: 2015.8.0
'''
ret = {}
hosts = find_guests(names, path=path)
if not hosts:
return False
client = salt.client.get_local_client(__opts__['conf_file'])
for host, sub_names in six.iteritems(hosts):
cmds = []
for name in sub_names:
cmds.append(
client.cmd_iter(host,
'lxc.{0}'.format(fun), [name],
kwarg={'path': path},
timeout=60))
for cmd in cmds:
data = next(cmd)
data = data.get(host, {}).get('ret', None)
if data:
ret.update({host: data})
return ret
def _do(name, fun, path=None):
'''
Invoke a function in the lxc module with no args
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: 2015.8.0
'''
host = find_guest(name, quiet=True, path=path)
if not host:
return False
client = salt.client.get_local_client(__opts__['conf_file'])
cmd_ret = client.cmd_iter(
host,
'lxc.{0}'.format(fun),
[name],
kwarg={'path': path},
timeout=60)
data = next(cmd_ret)
data = data.get(host, {}).get('ret', None)
if data:
data = {host: data}
return data
def query(hyper=None, quiet=False):
'''
Query the virtual machines. When called without options all hypervisors
are detected and a full query is returned. A single hypervisor can be
passed in to specify an individual hypervisor to query.
'''
if quiet:
log.warn('\'quiet\' is deprecated. Please migrate to --quiet')
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
try:
for info in client.cmd_iter('virtual:physical',
'virt.full_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = next(info.iterkeys())
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
chunk[id_] = info[id_]['ret']
ret.update(chunk)
if not quiet:
__jid_event__.fire_event({'data': chunk, 'outputter': 'virt_query'}, 'progress')
except SaltClientError as client_error:
print(client_error)
return ret
def _list_iter(host=None, path=None):
'''
Return a generator iterating over hosts
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: 2015.8.0
'''
tgt = host or '*'
client = salt.client.get_local_client(__opts__['conf_file'])
for container_info in client.cmd_iter(tgt,
'lxc.list',
kwarg={'path': path}):
if not container_info:
continue
if not isinstance(container_info, dict):
continue
chunk = {}
id_ = next(six.iterkeys(container_info))
if host and host != id_:
continue
if not isinstance(container_info[id_], dict):
continue
if 'ret' not in container_info[id_]:
continue
if not isinstance(container_info[id_]['ret'], dict):
continue
chunk[id_] = container_info[id_]['ret']
yield chunk
def pause(name):
'''
Pause the named VM
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'error': 'Failed to find VM {0} to pause'.format(name)}, 'progress')
return 'fail'
host = next(six.iterkeys(data))
if data[host][name]['state'] == 'paused':
__jid_event__.fire_event({'error': 'VM {0} is already paused'.format(name)}, 'progress')
return 'bad state'
try:
cmd_ret = client.cmd_iter(
host,
'virt.pause',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be pasued: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Paused VM {0}'.format(name)}, 'progress')
return 'good'
def purge(name, delete_key=True):
'''
Destroy the named vm
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'error': 'Failed to find vm {0} to purge'.format(name)}, 'progress')
return 'fail'
hyper = next(data.iterkeys())
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.purge',
[name, True],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be purged: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
if delete_key:
skey = salt.key.Key(__opts__)
skey.delete_key(name)
__jid_event__.fire_event({'message': 'Purged VM {0}'.format(name)}, 'progress')
return 'good'
def force_off(name):
'''
Force power down the named virtual machine
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
print('Failed to find vm {0} to destroy'.format(name))
return 'fail'
hyper = next(data.iterkeys())
if data[hyper][name]['state'] == 'shutdown':
print('VM {0} is already shutdown'.format(name))
return'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.destroy',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be forced off: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Powered off VM {0}'.format(name)}, 'progress')
return 'good'
def resume(name):
'''
Resume a paused vm
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'error': 'Failed to find VM {0} to pause'.format(name)}, 'progress')
return 'not found'
hyper = next(six.iterkeys(data))
if data[hyper][name]['state'] != 'paused':
__jid_event__.fire_event({'error': 'VM {0} is not paused'.format(name)}, 'progress')
return 'bad state'
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.resume',
[name],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be resumed: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({'message': 'Resumed VM {0}'.format(name)}, 'progress')
return 'good'
def cmd_iter(tgt,
fun,
arg=(),
timeout=None,
expr_form='glob',
ret='',
kwarg=None,
ssh=False,
**kwargs):
'''
Assuming this minion is a master, execute a salt command
CLI Example:
.. code-block:: bash
salt '*' saltutil.cmd
'''
if ssh:
client = salt.client.SSHClient(
os.path.dirname(__opts__['conf_file']))
else:
client = salt.client.LocalClient(
os.path.dirname(__opts__['conf_file']))
for ret in client.cmd_iter(
tgt,
fun,
arg,
timeout,
expr_form,
ret,
kwarg,
**kwargs):
yield ret
def purge(name, delete_key=True):
'''
Destroy the named vm
'''
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event({'error': 'Failed to find vm {0} to purge'.format(name)}, 'progress')
return 'fail'
hyper = next(six.iterkeys(data))
try:
cmd_ret = client.cmd_iter(
hyper,
'virt.purge',
[name, True],
timeout=600)
except SaltClientError as client_error:
return 'Virtual machine {0} could not be purged: {1}'.format(name, client_error)
for comp in cmd_ret:
ret.update(comp)
if delete_key:
skey = salt.key.Key(__opts__)
skey.delete_key(name)
__jid_event__.fire_event({'message': 'Purged VM {0}'.format(name)}, 'progress')
return 'good'
def list(hyper=None, quiet=False):
'''
List the virtual machines on each hyper
'''
ret = {}
client = salt.client.LocalClient(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.vm_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = info.keys()[0]
if hyper:
if hyper != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
data = {}
for k, v in info[id_]['ret'].items():
if v['state'] in data:
data[v['state']].append(k)
else:
data[v['state']] = [k]
chunk[id_] = data
ret.update(chunk)
if not quiet:
salt.output.display_output(chunk, 'virt_list', __opts__)
return ret
def _list_iter(host=None, path=None):
'''
Return a generator iterating over hosts
path
path to the container parent
default: /var/lib/lxc (system default)
.. versionadded:: Beryllium
'''
tgt = host or '*'
client = salt.client.get_local_client(__opts__['conf_file'])
for container_info in client.cmd_iter(
tgt, 'lxc.list', kwarg={'path': path}
):
if not container_info:
continue
if not isinstance(container_info, dict):
continue
chunk = {}
id_ = next(six.iterkeys(container_info))
if host and host != id_:
continue
if not isinstance(container_info[id_], dict):
continue
if 'ret' not in container_info[id_]:
continue
if not isinstance(container_info[id_]['ret'], dict):
continue
chunk[id_] = container_info[id_]['ret']
yield chunk
def start(name):
"""
Start a named virtual machine
"""
ret = {}
client = salt.client.get_local_client(__opts__["conf_file"])
data = vm_info(name, quiet=True)
if not data:
__jid_event__.fire_event(
{"message": "Failed to find VM {0} to start".format(name)},
"progress")
return "fail"
host = next(six.iterkeys(data))
if data[host][name]["state"] == "running":
print("VM {0} is already running".format(name))
return "bad state"
try:
cmd_ret = client.cmd_iter(host, "virt.start", [name], timeout=600)
except SaltClientError as client_error:
return "Virtual machine {0} not started: {1}".format(
name, client_error)
for comp in cmd_ret:
ret.update(comp)
__jid_event__.fire_event({"message": "Started VM {0}".format(name)},
"progress")
return "good"
def cmd(tgt,
fun,
arg=(),
timeout=None,
expr_form='glob',
ret='',
kwarg=None,
ssh=False,
**kwargs):
'''
Assuming this minion is a master, execute a salt command
CLI Example:
.. code-block:: bash
salt '*' saltutil.cmd
'''
if ssh:
client = salt.client.SSHClient(__opts__['conf_file'])
else:
client = salt.client.LocalClient(__opts__['conf_file'])
ret = {}
for ret_comp in client.cmd_iter(
tgt,
fun,
arg,
timeout,
expr_form,
ret,
kwarg,
**kwargs):
ret.update(ret_comp)
return ret
def ping(stack, activity, interval=5, max_failures=10):
"""
Attempts to use salt's test.ping module to ping the entire stack
and confirm that all hosts are reachable by salt.
@stack_id: The id of the stack to ping. We will use salt's grain
system to target the hosts with this stack id
@interval: The looping interval, ie, the amount of time to sleep
before the next iteration.
@max_failures: Number of ping failures before giving up completely.
The timeout does not affect this parameter.
@raises StackTaskException
"""
stack.log_history('Attempting to ping all hosts.', activity)
required_hosts = [h.hostname for h in stack.get_hosts()]
client = salt.client.LocalClient(settings.STACKDIO_CONFIG.salt_master_config)
# Execute until successful, failing after a few attempts
failures = 0
while True:
ret = client.cmd_iter(required_hosts, 'test.ping', expr_form='list')
result = {}
for res in ret:
for host, data in res.items():
result[host] = data
# check that we got a report back for all hosts
pinged_hosts = set(result.keys())
missing_hosts = set(required_hosts).difference(pinged_hosts)
if missing_hosts:
failures += 1
logger.debug('The following hosts did not respond to '
'the ping request: {0}; Total failures: '
'{1}'.format(missing_hosts,
failures))
false_hosts = []
for host, data in result.items():
if data['ret'] is not True or data['retcode'] != 0:
failures += 1
false_hosts.append(host)
if not missing_hosts and not false_hosts:
# Successful ping.
break
if failures > max_failures:
err_msg = 'Max failures ({0}) reached while pinging hosts.'.format(max_failures)
raise StackTaskException(err_msg)
time.sleep(interval)
if false_hosts:
err_msg = 'Unable to ping hosts: {0}'.format(', '.join(false_hosts))
raise StackTaskException(err_msg)
stack.log_history('All hosts pinged successfully.')
def execution():
'''
Collect all the sys.doc output from each minion and return the aggregate
CLI Example:
.. code-block:: bash
salt-run doc.execution
'''
client = salt.client.get_local_client(__opts__['conf_file'])
docs = {}
try:
for ret in client.cmd_iter('*', 'sys.doc', timeout=__opts__['timeout']):
for v in six.itervalues(ret):
docs.update(v)
except SaltClientError as exc:
print(exc)
return []
i = itertools.chain.from_iterable([six.iteritems(docs['ret'])])
ret = dict(list(i))
return ret
def sync_all(stack):
# Update status
stack.log_history('Synchronizing salt systems on all hosts.', Activity.PROVISIONING)
logger.info('Syncing all salt systems for stack: {0!r}'.format(stack))
# Generate all the files before we sync
stack.generate_orchestrate_file()
stack.generate_global_orchestrate_file()
target = [h.hostname for h in stack.get_hosts()]
client = salt.client.LocalClient(settings.STACKDIO_CONFIG.salt_master_config)
ret = client.cmd_iter(target, 'saltutil.sync_all', kwarg={'saltenv': 'base'}, expr_form='list')
result = {}
for res in ret:
for host, data in res.items():
result[host] = data
for host, data in result.items():
if 'retcode' not in data:
logger.warning('Host {0} missing a retcode... assuming failure'.format(host))
if data.get('retcode', 1) != 0:
err_msg = six.text_type(data['ret'])
raise StackTaskException('Error syncing salt data: {0!r}'.format(err_msg))
stack.log_history('Finished synchronizing salt systems on all hosts.')
def _process_hostkeys(client, pillars, cache):
log.debug("Retriving host keys for minions: %s", pillars.keys())
try:
if use_certs_param:
cmd_run = client.cmd_iter(pillars.keys(),
'ssh.host_keys',
kwarg={
'private': False,
'certs': False
},
**{expr_keyname: 'list'})
else:
cmd_run = client.cmd_iter(pillars.keys(),
'ssh.host_keys',
kwarg={'private': False},
**{expr_keyname: 'list'})
for rets in cmd_run:
for minion_id, resp in rets.iteritems():
log.trace("Minion '%s' returned: %s", minion_id, resp)
try:
if resp['retcode'] != 0:
log.warn(
"Minion '%s' returned an error running"
" 'ssh.host_keys': %s", minion_id, resp['ret'])
continue
if not use_certs_param:
for key in resp['ret'].keys():
if '-cert.pub' in key or '-cert-' in resp['ret'][
key]:
del resp['ret'][key]
log.trace("Found host keys for minion '%s'", minion_id)
try:
cache.store('sshpki/hostkeys', minion_id, resp['ret'])
log.debug("Stored host keys for minion '%s'",
minion_id)
except:
log.warn("Failed to store host keys for minion '%s'",
minion_id,
exc_info=True)
except:
log.warn("Error processing return data for minion '%s'",
minion_id,
exc_info=True)
log.debug("Host key processing complete")
except:
log.warn("Error retriving host keys for minions", exc_info=True)
def nodes(verbose=False):
"""
List all compute nodes
verbose : boolean
print additional information about the node
e.g. platform version, hvm capable, ...
CLI Example:
.. code-block:: bash
salt-run vmadm.nodes
salt-run vmadm.nodes verbose=True
"""
ret = {} if verbose else []
with salt.client.get_local_client(__opts__["conf_file"]) as client:
## get list of nodes
try:
for cn in client.cmd_iter(
"G@virtual:physical and G@os:smartos",
"grains.items",
tgt_type="compound",
):
if not cn:
continue
node = next(iter(cn.keys()))
if (not isinstance(cn[node], dict) or "ret" not in cn[node]
or not isinstance(cn[node]["ret"], dict)):
continue
if verbose:
ret[node] = {}
ret[node]["version"] = {}
ret[node]["version"]["platform"] = cn[node]["ret"][
"osrelease"]
if "computenode_sdc_version" in cn[node]["ret"]:
ret[node]["version"]["sdc"] = cn[node]["ret"][
"computenode_sdc_version"]
ret[node]["vms"] = {}
if ("computenode_vm_capable" in cn[node]["ret"]
and cn[node]["ret"]["computenode_vm_capable"]
and "computenode_vm_hw_virt" in cn[node]["ret"]):
ret[node]["vms"]["hw_cap"] = cn[node]["ret"][
"computenode_vm_hw_virt"]
else:
ret[node]["vms"]["hw_cap"] = False
if "computenode_vms_running" in cn[node]["ret"]:
ret[node]["vms"]["running"] = cn[node]["ret"][
"computenode_vms_running"]
else:
ret.append(node)
except SaltClientError as client_error:
return "{}".format(client_error)
if not verbose:
ret.sort()
return ret
def list_vms(search=None, verbose=False):
'''
List all vms
search : string
filter vms, see the execution module
verbose : boolean
print additional information about the vm
CLI Example:
.. code-block:: bash
salt-run vmadm.list
salt-run vmadm.list search='type=KVM'
salt-run vmadm.list verbose=True
'''
ret = OrderedDict() if verbose else []
client = salt.client.get_local_client(__opts__['conf_file'])
try:
vmadm_args = {}
vmadm_args['order'] = 'uuid,alias,hostname,state,type,cpu_cap,vcpus,ram'
if search:
vmadm_args['search'] = search
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'vmadm.list', kwarg=vmadm_args,
tgt_type='compound'):
if not cn:
continue
node = next(cn.iterkeys())
if not isinstance(cn[node], dict) or \
'ret' not in cn[node] or \
not isinstance(cn[node]['ret'], dict):
continue
for vm in cn[node]['ret'].keys():
vmcfg = cn[node]['ret'][vm]
if verbose:
ret[vm] = OrderedDict()
ret[vm]['hostname'] = vmcfg['hostname']
ret[vm]['alias'] = vmcfg['alias']
ret[vm]['computenode'] = node
ret[vm]['state'] = vmcfg['state']
ret[vm]['resources'] = OrderedDict()
ret[vm]['resources']['memory'] = vmcfg['ram']
if vmcfg['type'] == 'KVM':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['vcpus']))
else:
if vmcfg['cpu_cap'] != '':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['cpu_cap'])/100)
else:
ret.append(vm)
except SaltClientError as client_error:
return "{0}".format(client_error)
if not verbose:
ret = sorted(ret)
return ret
def list_vms(search=None, verbose=False):
'''
List all vms
search : string
filter vms, see the execution module
verbose : boolean
print additional information about the vm
CLI Example:
.. code-block:: bash
salt-run vmadm.list
salt-run vmadm.list search='type=KVM'
salt-run vmadm.list verbose=True
'''
ret = OrderedDict() if verbose else []
client = salt.client.get_local_client(__opts__['conf_file'])
try:
vmadm_args = {}
vmadm_args['order'] = 'uuid,alias,hostname,state,type,cpu_cap,vcpus,ram'
if search:
vmadm_args['search'] = search
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'vmadm.list', kwarg=vmadm_args,
expr_form='compound'):
if not cn:
continue
node = next(cn.iterkeys())
if not isinstance(cn[node], dict) or \
'ret' not in cn[node] or \
not isinstance(cn[node]['ret'], dict):
continue
for vm in cn[node]['ret'].keys():
vmcfg = cn[node]['ret'][vm]
if verbose:
ret[vm] = OrderedDict()
ret[vm]['hostname'] = vmcfg['hostname']
ret[vm]['alias'] = vmcfg['alias']
ret[vm]['computenode'] = node
ret[vm]['state'] = vmcfg['state']
ret[vm]['resources'] = OrderedDict()
ret[vm]['resources']['memory'] = vmcfg['ram']
if vmcfg['type'] == 'KVM':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['vcpus']))
else:
if vmcfg['cpu_cap'] != '':
ret[vm]['resources']['cpu'] = "{0:.2f}".format(int(vmcfg['cpu_cap'])/100)
else:
ret.append(vm)
except SaltClientError as client_error:
return "{0}".format(client_error)
if not verbose:
ret = sorted(ret)
return ret
def init(
name,
cpu,
mem,
image,
hyper=None,
seed=True,
nic='default',
install=True):
'''
Initialize a new vm
'''
print('Searching for Hypervisors')
data = query(hyper, quiet=True)
# Check if the name is already deployed
for hyper in data:
if 'vm_info' in data[hyper]:
if name in data[hyper]['vm_info']:
print('Virtual machine {0} is already deployed'.format(name))
return 'fail'
if hyper is None:
hyper = _determine_hyper(data)
if hyper not in data or not hyper:
print('Hypervisor {0} was not found'.format(hyper))
return 'fail'
if seed:
print('Minion will be preseeded')
kv = salt.utils.virt.VirtKey(hyper, name, __opts__)
kv.authorize()
client = salt.client.LocalClient(__opts__['conf_file'])
print('Creating VM {0} on hypervisor {1}'.format(name, hyper))
cmd_ret = client.cmd_iter(
hyper,
'virt.init',
[
name,
cpu,
mem,
image,
'seed={0}'.format(seed),
'nic={0}'.format(nic),
'install={0}'.format(install),
],
timeout=600)
ret = next(cmd_ret)
if not ret:
print('VM {0} was not initialized.'.format(name))
return 'fail'
print('VM {0} initialized on hypervisor {1}'.format(name, hyper))
return 'good'
def nodes(verbose=False):
'''
List all compute nodes
verbose : boolean
print additional information about the node
e.g. platform version, hvm capable, ...
CLI Example:
.. code-block:: bash
salt-run vmadm.nodes
salt-run vmadm.nodes verbose=True
'''
ret = {} if verbose else []
client = salt.client.get_local_client(__opts__['conf_file'])
## get list of nodes
try:
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'grains.items',
tgt_type='compound'):
if not cn:
continue
node = next(six.iterkeys(cn))
if not isinstance(cn[node], dict) or \
'ret' not in cn[node] or \
not isinstance(cn[node]['ret'], dict):
continue
if verbose:
ret[node] = {}
ret[node]['version'] = {}
ret[node]['version']['platform'] = cn[node]['ret']['osrelease']
if 'computenode_sdc_version' in cn[node]['ret']:
ret[node]['version']['sdc'] = cn[node]['ret'][
'computenode_sdc_version']
ret[node]['vms'] = {}
if 'computenode_vm_capable' in cn[node]['ret'] and \
cn[node]['ret']['computenode_vm_capable'] and \
'computenode_vm_hw_virt' in cn[node]['ret']:
ret[node]['vms']['hw_cap'] = cn[node]['ret'][
'computenode_vm_hw_virt']
else:
ret[node]['vms']['hw_cap'] = False
if 'computenode_vms_running' in cn[node]['ret']:
ret[node]['vms']['running'] = cn[node]['ret'][
'computenode_vms_running']
else:
ret.append(node)
except SaltClientError as client_error:
return "{0}".format(client_error)
if not verbose:
ret.sort()
return ret
def nodes(verbose=False):
'''
List all compute nodes
verbose : boolean
print additional information about the node
e.g. platform version, hvm capable, ...
CLI Example:
.. code-block:: bash
salt-run vmadm.nodes
salt-run vmadm.nodes verbose=True
'''
ret = {} if verbose else []
client = salt.client.get_local_client(__opts__['conf_file'])
## get list of nodes
try:
for cn in client.cmd_iter('G@virtual:physical and G@os:smartos',
'grains.items', tgt_type='compound'):
if not cn:
continue
node = next(cn.iterkeys())
if not isinstance(cn[node], dict) or \
'ret' not in cn[node] or \
not isinstance(cn[node]['ret'], dict):
continue
if verbose:
ret[node] = {}
ret[node]['version'] = {}
ret[node]['version']['platform'] = cn[node]['ret']['osrelease']
if 'computenode_sdc_version' in cn[node]['ret']:
ret[node]['version']['sdc'] = cn[node]['ret']['computenode_sdc_version']
ret[node]['vms'] = {}
if 'computenode_vm_capable' in cn[node]['ret'] and \
cn[node]['ret']['computenode_vm_capable'] and \
'computenode_vm_hw_virt' in cn[node]['ret']:
ret[node]['vms']['hw_cap'] = cn[node]['ret']['computenode_vm_hw_virt']
else:
ret[node]['vms']['hw_cap'] = False
if 'computenode_vms_running' in cn[node]['ret']:
ret[node]['vms']['running'] = cn[node]['ret']['computenode_vms_running']
else:
ret.append(node)
except SaltClientError as client_error:
return "{0}".format(client_error)
if not verbose:
ret.sort()
return ret
def _exec(client, tgt, fun, arg, timeout, expr_form, ret, kwarg, **kwargs):
ret = {}
seen = 0
for ret_comp in client.cmd_iter(
tgt, fun, arg, timeout, expr_form, ret, kwarg, **kwargs):
ret.update(ret_comp)
seen += 1
# ret can be empty, so we cannot len the whole return dict
if expr_form == 'list' and len(tgt) == seen:
# do not wait for timeout when explicit list matching
# and all results are there
break
return ret
def _exec(client, tgt, fun, arg, timeout, expr_form, ret, kwarg, **kwargs):
ret = {}
seen = 0
for ret_comp in client.cmd_iter(
tgt, fun, arg, timeout, expr_form, ret, kwarg, **kwargs):
ret.update(ret_comp)
seen += 1
# ret can be empty, so we cannot len the whole return dict
if expr_form == 'list' and len(tgt) == seen:
# do not wait for timeout when explicit list matching
# and all results are there
break
return ret
def list(host=None, quiet=False, hyper=None): # pylint: disable=redefined-builtin
'''
List the virtual machines on each host, this is a simplified query,
showing only the virtual machine names belonging to each host.
A single host can be passed in to specify an individual host
to list.
'''
if hyper is not None:
salt.utils.warn_until(
'Carbon',
'Please use "host" instead of "hyper". The "hyper" argument will '
'be removed in the Carbon release of Salt')
host = hyper
if quiet:
log.warn('\'quiet\' is deprecated. Please migrate to --quiet')
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.vm_info',
expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = next(six.iterkeys(info))
if host:
if host != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
data = {}
for key, val in six.iteritems(info[id_]['ret']):
if val['state'] in data:
data[val['state']].append(key)
else:
data[val['state']] = [key]
chunk[id_] = data
ret.update(chunk)
if not quiet:
__jid_event__.fire_event({
'data': chunk,
'outputter': 'virt_list'
}, 'progress')
return ret
def get_current_hosts(self):
client = salt.client.LocalClient(settings.STACKDIO_CONFIG.salt_master_config)
result = client.cmd_iter('env:environments.{}'.format(self.name),
'grains.items',
expr_form='grain')
ret = []
for res in result:
for data in res.values():
if data.get('ret', False):
ret.append(data['ret'])
return ret
def ping_stack_hosts(stack):
"""
Returns a set of hostnames in the stack that were reachable via
a ping request.
NOTE: This specifically targets the hosts in a stack instead of
pinging all available hosts that salt is managing.
"""
client = salt.client.LocalClient(settings.STACKDIO_CONFIG.salt_master_config)
target = [host.hostname for host in stack.hosts.all()]
ret = set()
for val in client.cmd_iter(target, 'test.ping', expr_form='list'):
ret.update(val.keys())
return ret
def _do(name, fun):
'''
Invoke a function in the lxc module with no args
'''
host = find_guest(name, quiet=True)
if not host:
return False
client = salt.client.get_local_client(__opts__['conf_file'])
cmd_ret = client.cmd_iter(host, 'lxc.{0}'.format(fun), [name], timeout=60)
data = next(cmd_ret)
data = data.get(host, {}).get('ret', None)
if data:
data = {host: data}
return data
def list(host=None, quiet=False, hyper=None): # pylint: disable=redefined-builtin
'''
List the virtual machines on each host, this is a simplified query,
showing only the virtual machine names belonging to each host.
A single host can be passed in to specify an individual host
to list.
'''
if hyper is not None:
salt.utils.warn_until(
'Carbon',
'Please use "host" instead of "hyper". The "hyper" argument will '
'be removed in the Carbon release of Salt'
)
host = hyper
if quiet:
log.warn('\'quiet\' is deprecated. Please migrate to --quiet')
ret = {}
client = salt.client.get_local_client(__opts__['conf_file'])
for info in client.cmd_iter('virtual:physical',
'virt.vm_info', expr_form='grain'):
if not info:
continue
if not isinstance(info, dict):
continue
chunk = {}
id_ = next(six.iterkeys(info))
if host:
if host != id_:
continue
if not isinstance(info[id_], dict):
continue
if 'ret' not in info[id_]:
continue
if not isinstance(info[id_]['ret'], dict):
continue
data = {}
for key, val in six.iteritems(info[id_]['ret']):
if val['state'] in data:
data[val['state']].append(key)
else:
data[val['state']] = [key]
chunk[id_] = data
ret.update(chunk)
if not quiet:
__jid_event__.fire_event({'data': chunk, 'outputter': 'virt_list'}, 'progress')
return ret
def doc():
'''
Collect all the sys.doc output from each minion and return the aggregate
'''
client = salt.client.LocalClient(__opts__['conf_file'])
docs = {}
for ret in client.cmd_iter('*', 'sys.doc', timeout=__opts__['timeout']):
for k,v in ret.items():
docs.update(v)
i = itertools.chain.from_iterable([i.items() for i in docs.values()])
ret = dict(list(i))
salt.output.display_output(ret, '', __opts__)
return ret
def cmd_iter(tgt, fun, arg=(), timeout=None, expr_form="glob", ret="", kwarg=None, ssh=False, **kwargs):
"""
Assuming this minion is a master, execute a salt command
CLI Example:
.. code-block:: bash
salt '*' saltutil.cmd
"""
if ssh:
client = salt.client.SSHClient(__opts__["conf_file"])
else:
client = salt.client.LocalClient(__opts__["conf_file"])
for ret in client.cmd_iter(tgt, fun, arg, timeout, expr_form, ret, kwarg, **kwargs):
yield ret
