def display_output(data, out=None, opts=None):
'''
Print the passed data using the desired output
'''
if opts is None:
opts = {}
display_data = try_printout(data, out, opts)
output_filename = opts.get('output_file', None)
log.trace('data = {0}'.format(data))
try:
# output filename can be either '' or None
if output_filename:
with salt.utils.fopen(output_filename, 'a') as ofh:
fdata = display_data
if isinstance(fdata, six.text_type):
try:
fdata = fdata.encode('utf-8')
except (UnicodeDecodeError, UnicodeEncodeError):
# try to let the stream write
# even if we didn't encode it
pass
ofh.write(fdata)
ofh.write('\n')
return
if display_data:
print_cli(display_data)
except IOError as exc:
# Only raise if it's NOT a broken pipe
if exc.errno != errno.EPIPE:
raise exc
def _print_errors_summary(self, errors):
if errors:
print_cli('\n')
print_cli('---------------------------')
print_cli('Errors')
print_cli('---------------------------')
for minion in errors:
print_cli(self._format_error(minion))
def _print_errors_summary(self, errors):
if errors:
print_cli("\n")
print_cli("---------------------------")
print_cli("Errors")
print_cli("---------------------------")
for error in errors:
print_cli(self._format_error(error))
def print_docs(self):
'''
Pick up the documentation for all of the modules and print it out.
'''
docs = {}
for name, func in six.iteritems(self.minion.functions):
if name not in docs:
if func.__doc__:
docs[name] = func.__doc__
for name in sorted(docs):
if name.startswith(self.opts.get('fun', '')):
print_cli('{0}:\n{1}\n'.format(name, docs[name]))
def _load_files(self):
'''
Parse the files indicated in opts['src'] and load them into a python
object for transport
'''
files = {}
for fn_ in self.opts['src']:
if os.path.isfile(fn_):
files.update(self._file_dict(fn_))
elif os.path.isdir(fn_):
print_cli(fn_ + ' is a directory, only files are supported.')
#files.update(self._recurse_dir(fn_))
return files
def display_output(data, out=None, opts=None, **kwargs):
'''
Print the passed data using the desired output
'''
if opts is None:
opts = {}
display_data = try_printout(data, out, opts, **kwargs)
output_filename = opts.get('output_file', None)
log.trace('data = {0}'.format(data))
try:
# output filename can be either '' or None
if output_filename:
if not hasattr(output_filename, 'write'):
ofh = salt.utils.fopen(output_filename, 'a')
fh_opened = True
else:
# Filehandle/file-like object
ofh = output_filename
fh_opened = False
try:
fdata = display_data
if isinstance(fdata, six.text_type):
try:
fdata = fdata.encode('utf-8')
except (UnicodeDecodeError, UnicodeEncodeError):
# try to let the stream write
# even if we didn't encode it
pass
if fdata:
if six.PY3:
ofh.write(fdata.decode())
else:
ofh.write(fdata)
ofh.write('\n')
finally:
if fh_opened:
ofh.close()
return
if display_data:
print_cli(display_data)
except IOError as exc:
# Only raise if it's NOT a broken pipe
if exc.errno != errno.EPIPE:
raise exc
def get_bnum(self):
'''
Return the active number of minions to maintain
'''
partition = lambda x: float(x) / 100.0 * len(self.minions)
try:
if '%' in self.opts['batch']:
res = partition(float(self.opts['batch'].strip('%')))
if res < 1:
return int(math.ceil(res))
else:
return int(res)
else:
return int(self.opts['batch'])
except ValueError:
if not self.quiet:
print_cli('Invalid batch data sent: {0}\nData must be in the '
'form of %10, 10% or 3'.format(self.opts['batch']))
def _print_docs(self, ret):
'''
Print out the docstrings for all of the functions on the minions
'''
docs = {}
if not ret:
self.exit(2, 'No minions found to gather docs from\n')
if isinstance(ret, str):
self.exit(2, '{0}\n'.format(ret))
for host in ret:
for fun in ret[host]:
if fun not in docs:
if ret[host][fun]:
docs[fun] = ret[host][fun]
for fun in sorted(docs):
salt.output.display_output(fun + ':', 'text', self.config)
print_cli(docs[fun])
print_cli('')
def _print_docs(self, ret):
"""
Print out the docstrings for all of the functions on the minions
"""
docs = {}
if not ret:
self.exit(2, "No minions found to gather docs from\n")
if isinstance(ret, str):
self.exit(2, "{0}\n".format(ret))
for host in ret:
if ret[host] == "Minion did not return. [Not connected]":
continue
for fun in ret[host]:
if fun not in docs:
if ret[host][fun]:
docs[fun] = ret[host][fun]
for fun in sorted(docs):
salt.output.display_output(fun + ":", "text", self.config)
print_cli(docs[fun])
print_cli("")
def _print_docs(self, ret):
'''
Print out the docstrings for all of the functions on the minions
'''
import salt.output
docs = {}
if not ret:
self.exit(2, 'No minions found to gather docs from\n')
if isinstance(ret, str):
self.exit(2, '{0}\n'.format(ret))
for host in ret:
if ret[host] == 'Minion did not return. [Not connected]':
continue
for fun in ret[host]:
if fun not in docs:
if ret[host][fun]:
docs[fun] = ret[host][fun]
for fun in sorted(docs):
salt.output.display_output(fun + ':', 'text', self.config)
print_cli(docs[fun])
print_cli('')
def _print_docs(self, ret):
'''
Print out the docstrings for all of the functions on the minions
'''
import salt.output
docs = {}
if not ret:
self.exit(2, 'No minions found to gather docs from\n')
if isinstance(ret, str):
self.exit(2, '{0}\n'.format(ret))
for host in ret:
if isinstance(ret[host], string_types) and ret[host].startswith(
"Minion did not return"):
continue
for fun in ret[host]:
if fun not in docs and ret[host][fun]:
docs[fun] = ret[host][fun]
if self.options.output:
for fun in sorted(docs):
salt.output.display_output({fun: docs[fun]}, 'nested',
self.config)
else:
for fun in sorted(docs):
print_cli('{0}:'.format(fun))
print_cli(docs[fun])
print_cli('')
def _print_docs(self, ret):
"""
Print out the docstrings for all of the functions on the minions
"""
import salt.output
docs = {}
if not ret:
self.exit(2, "No minions found to gather docs from\n")
if isinstance(ret, str):
self.exit(2, "{0}\n".format(ret))
for host in ret:
if isinstance(ret[host], string_types) and ret[host].startswith("Minion did not return"):
continue
for fun in ret[host]:
if fun not in docs and ret[host][fun]:
docs[fun] = ret[host][fun]
if self.options.output:
for fun in sorted(docs):
salt.output.display_output({fun: docs[fun]}, "nested", self.config)
else:
for fun in sorted(docs):
print_cli("{0}:".format(fun))
print_cli(docs[fun])
print_cli("")
def __gather_minions(self):
'''
Return a list of minions to use for the batch run
'''
args = [self.opts['tgt'],
'test.ping',
[],
self.opts['timeout'],
]
selected_target_option = self.opts.get('selected_target_option', None)
if selected_target_option is not None:
args.append(selected_target_option)
else:
args.append(self.opts.get('tgt_type', 'glob'))
self.pub_kwargs['yield_pub_data'] = True
ping_gen = self.local.cmd_iter(*args,
gather_job_timeout=self.opts['gather_job_timeout'],
**self.pub_kwargs)
# Broadcast to targets
fret = set()
nret = set()
for ret in ping_gen:
if ('minions' and 'jid') in ret:
for minion in ret['minions']:
nret.add(minion)
continue
else:
try:
m = next(six.iterkeys(ret))
except StopIteration:
if not self.quiet:
print_cli('No minions matched the target.')
break
if m is not None:
fret.add(m)
return (list(fret), ping_gen, nret.difference(fret))
def __gather_minions(self):
'''
Return a list of minions to use for the batch run
'''
args = [self.opts['tgt'],
'test.ping',
[],
self.opts['timeout'],
]
selected_target_option = self.opts.get('selected_target_option', None)
if selected_target_option is not None:
args.append(selected_target_option)
else:
args.append(self.opts.get('expr_form', 'glob'))
fret = []
for ret in self.local.cmd_iter(*args, **self.eauth):
for minion in ret:
if not self.quiet:
print_cli('{0} Detected for this batch run'.format(minion))
fret.append(minion)
return sorted(fret)
def __gather_minions(self):
'''
Return a list of minions to use for the batch run
'''
args = [self.opts['tgt'],
'test.ping',
[],
self.opts['timeout'],
]
selected_target_option = self.opts.get('selected_target_option', None)
if selected_target_option is not None:
args.append(selected_target_option)
else:
args.append(self.opts.get('tgt_type', 'glob'))
self.pub_kwargs['yield_pub_data'] = True
ping_gen = self.local.cmd_iter(*args, **self.pub_kwargs)
# Broadcast to targets
fret = set()
nret = set()
try:
for ret in ping_gen:
if ('minions' and 'jid') in ret:
for minion in ret['minions']:
nret.add(minion)
continue
else:
m = next(six.iterkeys(ret))
if m is not None:
fret.add(m)
return (list(fret), ping_gen, nret.difference(fret))
except StopIteration:
if not self.quiet:
print_cli('No minions matched the target.')
return list(fret), ping_gen
def __gather_minions(self):
'''
Return a list of minions to use for the batch run
'''
args = [
self.opts['tgt'],
'test.ping',
[],
self.opts['timeout'],
]
selected_target_option = self.opts.get('selected_target_option', None)
if selected_target_option is not None:
args.append(selected_target_option)
else:
args.append(self.opts.get('expr_form', 'glob'))
fret = []
for ret in self.local.cmd_iter(*args, **self.eauth):
for minion in ret:
if not self.quiet:
print_cli('{0} Detected for this batch run'.format(minion))
fret.append(minion)
return sorted(fret)
def display_output(data, out=None, opts=None):
'''
Print the passed data using the desired output
'''
if opts is None:
opts = {}
try:
display_data = get_printout(out, opts)(data).rstrip()
except (KeyError, AttributeError):
log.debug(traceback.format_exc())
opts.pop('output', None)
display_data = get_printout('nested', opts)(data).rstrip()
output_filename = opts.get('output_file', None)
log.trace('data = {0}'.format(data))
try:
# output filename can be either '' or None
if output_filename:
with salt.utils.fopen(output_filename, 'a') as ofh:
fdata = display_data
if isinstance(fdata, six.text_type):
try:
fdata = fdata.encode('utf-8')
except (UnicodeDecodeError, UnicodeEncodeError):
# try to let the stream write
# even if we didn't encode it
pass
ofh.write(fdata)
ofh.write('\n')
return
if display_data:
print_cli(display_data)
except IOError as exc:
# Only raise if it's NOT a broken pipe
if exc.errno != errno.EPIPE:
raise exc
def run(self):
# '''
# Execute the salt command line
# '''
import salt.client
# self.parse_args()
# Setup file logging!
self.setup_logfile_logger()
verify_log(self.config)
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
self.local_client = salt.client.get_local_client(
self.get_config_file_path(),
skip_perm_errors=skip_perm_errors,
auto_reconnect=True)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.batch or self.options.static:
# _run_batch() will handle all output and
# exit with the appropriate error condition
# Execution will not continue past this point
# in batch mode.
self._run_batch()
return
if self.options.preview_target:
self._preview_target()
return
if self.options.timeout <= 0:
self.options.timeout = self.local_client.opts['timeout']
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid
}
if 'token' in self.config:
try:
with salt.utils.fopen(
os.path.join(self.config['cachedir'], '.root_key'),
'r') as fp_:
kwargs['key'] = fp_.readline()
except IOError:
kwargs['token'] = self.config['token']
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['tgt_type'] = self.selected_target_option
else:
kwargs['tgt_type'] = 'glob'
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'return_kwargs'):
kwargs['ret_kwargs'] = yamlify_arg(
getattr(self.options, 'return_kwargs'))
if getattr(self.options, 'module_executors'):
kwargs['module_executors'] = yamlify_arg(
getattr(self.options, 'module_executors'))
if getattr(self.options, 'metadata'):
kwargs['metadata'] = yamlify_arg(getattr(self.options, 'metadata'))
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and 'key' not in kwargs and self.options.eauth:
# This is expensive. Don't do it unless we need to.
import salt.auth
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
if self.config['async']:
jid = self.local_client.cmd_async(**kwargs)
print_cli('Executed command with job ID: {0}'.format(jid))
return
# local will be None when there was an error
if not self.local_client:
return
retcodes = []
errors = []
try:
if self.options.subset:
cmd_func = self.local_client.cmd_subset
kwargs['sub'] = self.options.subset
kwargs['cli'] = True
else:
cmd_func = self.local_client.cmd_cli
if self.options.progress:
kwargs['progress'] = True
self.config['progress'] = True
ret = {}
for progress in cmd_func(**kwargs):
out = 'progress'
try:
self._progress_ret(progress, out)
except salt.exceptions.LoaderError as exc:
raise salt.exceptions.SaltSystemExit(exc)
if 'return_count' not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config['fun'] == 'sys.doc':
ret = {}
out = ''
for full_ret in self.local_client.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs['verbose'] = True
ret = {}
# TODO: delete debug code start=============================
# import re
# import subprocess
# from salt.newrun import (json, byteify, MessageType)
# oriArr = []
# executeArr = []
# def executeSaltCmd(cmd, msg_in=''):
# try:
# proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
# stdout_value, stderr_value = proc.communicate(msg_in)
# return stdout_value, stderr_value
# except Exception, e:
# log.error(traceback.format_exc())
# #print('traceback.format_exc():\n%s' % traceback.format_exc())
# def getAcceptIp():
# stdout_val, stderr_val = executeSaltCmd("salt-key -l acc")
# syndicList = re.findall(
# r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", stdout_val, re.M)
# return syndicList
# oriArr = getAcceptIp()
# loopi = 0
# TODO: delete debug code end=============================
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
# print('full_ret=: %s, ret_=: %s, out=: %s' % (full_ret, ret_, out))
retcodes.append(retcode)
self._output_ret(ret_, out)
# TODO: delete debug code start=============================
# ret_ = byteify(ret_)
# ssList = re.findall(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", json.dumps(ret_), re.M)
# executeArr = executeArr + ssList
# loopi = loopi +1
# if loopi > 2516:
# eprint = [i for i in oriArr if i not in executeArr]
# print(eprint)
# TODO: delete debug code end=============================
ret.update(full_ret)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config['cli_summary'] is True:
if self.config['fun'] != 'sys.doc':
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.stderr.write(
'ERROR: Minions returned with non-zero exit code\n')
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError,
SaltClientError) as exc:
ret = str(exc)
self._output_ret(ret, '')
def run(self):
"""
Execute the salt command line
"""
import salt.client
self.parse_args()
# Setup file logging!
self.setup_logfile_logger()
verify_log(self.config)
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ""
self.local_client = salt.client.get_local_client(
self.get_config_file_path(), skip_perm_errors=skip_perm_errors, auto_reconnect=True
)
except SaltClientError as exc:
self.exit(2, "{0}\n".format(exc))
return
if self.options.batch or self.options.static:
# _run_batch() will handle all output and
# exit with the appropriate error condition
# Execution will not continue past this point
# in batch mode.
self._run_batch()
return
if self.options.timeout <= 0:
self.options.timeout = self.local_client.opts["timeout"]
kwargs = {
"tgt": self.config["tgt"],
"fun": self.config["fun"],
"arg": self.config["arg"],
"timeout": self.options.timeout,
"show_timeout": self.options.show_timeout,
"show_jid": self.options.show_jid,
}
if "token" in self.config:
try:
with salt.utils.fopen(os.path.join(self.config["cachedir"], ".root_key"), "r") as fp_:
kwargs["key"] = fp_.readline()
except IOError:
kwargs["token"] = self.config["token"]
kwargs["delimiter"] = self.options.delimiter
if self.selected_target_option:
kwargs["tgt_type"] = self.selected_target_option
else:
kwargs["tgt_type"] = "glob"
if getattr(self.options, "return"):
kwargs["ret"] = getattr(self.options, "return")
if getattr(self.options, "return_config"):
kwargs["ret_config"] = getattr(self.options, "return_config")
if getattr(self.options, "return_kwargs"):
kwargs["ret_kwargs"] = yamlify_arg(getattr(self.options, "return_kwargs"))
if getattr(self.options, "module_executors"):
kwargs["module_executors"] = yamlify_arg(getattr(self.options, "module_executors"))
if getattr(self.options, "metadata"):
kwargs["metadata"] = yamlify_arg(getattr(self.options, "metadata"))
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if "token" not in kwargs and "key" not in kwargs and self.options.eauth:
# This is expensive. Don't do it unless we need to.
import salt.auth
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs["token"] = tok.get("token", "")
if not res:
sys.stderr.write("ERROR: Authentication failed\n")
sys.exit(2)
kwargs.update(res)
kwargs["eauth"] = self.options.eauth
if self.config["async"]:
jid = self.local_client.cmd_async(**kwargs)
print_cli("Executed command with job ID: {0}".format(jid))
return
# local will be None when there was an error
if not self.local_client:
return
retcodes = []
errors = []
try:
if self.options.subset:
cmd_func = self.local_client.cmd_subset
kwargs["sub"] = True
kwargs["cli"] = True
else:
cmd_func = self.local_client.cmd_cli
if self.options.progress:
kwargs["progress"] = True
self.config["progress"] = True
ret = {}
for progress in cmd_func(**kwargs):
out = "progress"
try:
self._progress_ret(progress, out)
except salt.exceptions.LoaderError as exc:
raise salt.exceptions.SaltSystemExit(exc)
if "return_count" not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config["fun"] == "sys.doc":
ret = {}
out = ""
for full_ret in self.local_client.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs["verbose"] = True
ret = {}
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(full_ret)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config["cli_summary"] is True:
if self.config["fun"] != "sys.doc":
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.stderr.write("ERROR: Minions returned with non-zero exit code\n")
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc:
ret = str(exc)
self._output_ret(ret, "")
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
for each_minion in ret:
minion_ret = ret[each_minion]
if (isinstance(minion_ret, string_types)
and minion_ret.startswith("Minion did not return")):
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of Minions Targeted: {0}'.format(return_counter +
not_return_counter))
print_cli('# of Minions Returned: {0}'.format(return_counter))
print_cli(
'# of Minions Did Not Return: {0}'.format(not_return_counter))
if self.options.verbose:
print_cli('Minions Which Did Not Return: {0}'.format(
" ".join(not_return_minions)))
print_cli('-------------------------------------------')
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
for each_minion in ret:
minion_ret = ret[each_minion]
if (
isinstance(minion_ret, string_types)
and minion_ret.startswith("Minion did not return")
):
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of Minions Targeted: {0}'.format(return_counter + not_return_counter))
print_cli('# of Minions Returned: {0}'.format(return_counter))
print_cli('# of Minions Did Not Return: {0}'.format(not_return_counter))
if self.options.verbose:
print_cli('Minions Which Did Not Return: {0}'.format(" ".join(not_return_minions)))
print_cli('-------------------------------------------')
def run(self):
'''
Execute the batch run
'''
args = [[],
self.opts['fun'],
self.opts['arg'],
self.opts['timeout'],
'list',
]
bnum = self.get_bnum()
to_run = copy.deepcopy(self.minions)
active = []
ret = {}
iters = []
# the minion tracker keeps track of responses and iterators
# - it removes finished iterators from iters[]
# - if a previously detected minion does not respond, its
# added with an empty answer to ret{} once the timeout is reached
# - unresponsive minions are removed from active[] to make
# sure that the main while loop finishes even with unresp minions
minion_tracker = {}
# Iterate while we still have things to execute
while len(ret) < len(self.minions):
next_ = []
if len(to_run) <= bnum and not active:
# last bit of them, add them all to next iterator
while to_run:
next_.append(to_run.pop())
else:
for i in range(bnum - len(active)):
if to_run:
next_.append(to_run.pop())
active += next_
args[0] = next_
if next_:
if not self.quiet:
print_cli('\nExecuting run on {0}\n'.format(next_))
# create a new iterator for this batch of minions
new_iter = self.local.cmd_iter_no_block(
*args,
raw=self.opts.get('raw', False),
ret=self.opts.get('return', ''),
**self.eauth)
# add it to our iterators and to the minion_tracker
iters.append(new_iter)
minion_tracker[new_iter] = {}
# every iterator added is 'active' and has its set of minions
minion_tracker[new_iter]['minions'] = next_
minion_tracker[new_iter]['active'] = True
else:
time.sleep(0.02)
parts = {}
for queue in iters:
try:
# Gather returns until we get to the bottom
ncnt = 0
while True:
part = next(queue)
if part is None:
time.sleep(0.01)
ncnt += 1
if ncnt > 5:
break
continue
if self.opts.get('raw'):
parts.update({part['id']: part})
else:
parts.update(part)
except StopIteration:
# if a iterator is done:
# - set it to inactive
# - add minions that have not responded to parts{}
# check if the tracker contains the iterator
if queue in minion_tracker:
minion_tracker[queue]['active'] = False
# add all minions that belong to this iterator and
# that have not responded to parts{} with an empty response
for minion in minion_tracker[queue]['minions']:
if minion not in parts:
parts[minion] = {}
parts[minion]['ret'] = {}
for minion, data in parts.items():
active.remove(minion)
if self.opts.get('raw'):
yield data
else:
ret[minion] = data['ret']
yield {minion: data['ret']}
if not self.quiet:
ret[minion] = data['ret']
data[minion] = data.pop('ret')
if 'out' in data:
out = data.pop('out')
else:
out = None
salt.output.display_output(
data,
out,
self.opts)
# remove inactive iterators from the iters list
for queue in minion_tracker:
# only remove inactive queues
if not minion_tracker[queue]['active'] and queue in iters:
iters.remove(queue)
# also remove the iterator's minions from the active list
for minion in minion_tracker[queue]['minions']:
if minion in active:
active.remove(minion)
def run(self):
'''
Execute the batch run
'''
args = [[],
self.opts['fun'],
self.opts['arg'],
self.opts['timeout'],
'list',
]
bnum = self.get_bnum()
# No targets to run
if not self.minions:
return
to_run = copy.deepcopy(self.minions)
active = []
ret = {}
iters = []
# wait the specified time before decide a job is actually done
bwait = self.opts.get('batch_wait', 0)
wait = []
if self.options:
show_jid = self.options.show_jid
show_verbose = self.options.verbose
else:
show_jid = False
show_verbose = False
# the minion tracker keeps track of responses and iterators
# - it removes finished iterators from iters[]
# - if a previously detected minion does not respond, its
# added with an empty answer to ret{} once the timeout is reached
# - unresponsive minions are removed from active[] to make
# sure that the main while loop finishes even with unresp minions
minion_tracker = {}
# We already know some minions didn't respond to the ping, so inform
# the user we won't be attempting to run a job on them
for down_minion in self.down_minions:
print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion))
# Iterate while we still have things to execute
while len(ret) < len(self.minions):
next_ = []
if bwait and wait:
self.__update_wait(wait)
if len(to_run) <= bnum - len(wait) and not active:
# last bit of them, add them all to next iterator
while to_run:
next_.append(to_run.pop())
else:
for i in range(bnum - len(active) - len(wait)):
if to_run:
minion_id = to_run.pop()
if isinstance(minion_id, dict):
next_.append(minion_id.keys()[0])
else:
next_.append(minion_id)
active += next_
args[0] = next_
if next_:
if not self.quiet:
print_cli('\nExecuting run on {0}\n'.format(next_))
# create a new iterator for this batch of minions
new_iter = self.local.cmd_iter_no_block(
*args,
raw=self.opts.get('raw', False),
ret=self.opts.get('return', ''),
show_jid=show_jid,
verbose=show_verbose,
**self.eauth)
# add it to our iterators and to the minion_tracker
iters.append(new_iter)
minion_tracker[new_iter] = {}
# every iterator added is 'active' and has its set of minions
minion_tracker[new_iter]['minions'] = next_
minion_tracker[new_iter]['active'] = True
else:
time.sleep(0.02)
parts = {}
# see if we found more minions
for ping_ret in self.ping_gen:
if ping_ret is None:
break
m = next(ping_ret.iterkeys())
if m not in self.minions:
self.minions.append(m)
to_run.append(m)
for queue in iters:
try:
# Gather returns until we get to the bottom
ncnt = 0
while True:
part = next(queue)
if part is None:
time.sleep(0.01)
ncnt += 1
if ncnt > 5:
break
continue
if self.opts.get('raw'):
parts.update({part['data']['id']: part})
if part['data']['id'] in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(part['data']['id'])
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id']))
else:
parts.update(part)
for id in part.keys():
if id in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(id)
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id))
except StopIteration:
# if a iterator is done:
# - set it to inactive
# - add minions that have not responded to parts{}
# check if the tracker contains the iterator
if queue in minion_tracker:
minion_tracker[queue]['active'] = False
# add all minions that belong to this iterator and
# that have not responded to parts{} with an empty response
for minion in minion_tracker[queue]['minions']:
if minion not in parts:
parts[minion] = {}
parts[minion]['ret'] = {}
for minion, data in six.iteritems(parts):
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
if self.opts.get('raw'):
ret[minion] = data
yield data
else:
ret[minion] = data
yield {minion: data}
if not self.quiet:
ret[minion] = data['ret']
data[minion] = data.pop('ret')
if 'out' in data:
out = data.pop('out')
else:
out = None
salt.output.display_output(
data,
out,
self.opts)
# remove inactive iterators from the iters list
for queue in minion_tracker:
# only remove inactive queues
if not minion_tracker[queue]['active'] and queue in iters:
iters.remove(queue)
# also remove the iterator's minions from the active list
for minion in minion_tracker[queue]['minions']:
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
def call(self):
'''
Call the module
'''
ret = {}
fun = self.opts['fun']
ret['jid'] = salt.utils.jid.gen_jid()
proc_fn = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid']
)
if fun not in self.minion.functions:
sys.stderr.write(self.minion.functions.missing_fun_string(fun))
mod_name = fun.split('.')[0]
if mod_name in self.minion.function_errors:
sys.stderr.write(' Possible reasons: {0}\n'.format(self.minion.function_errors[mod_name]))
else:
sys.stderr.write('\n')
sys.exit(-1)
try:
sdata = {
'fun': fun,
'pid': os.getpid(),
'jid': ret['jid'],
'tgt': 'salt-call'}
args, kwargs = salt.minion.load_args_and_kwargs(
self.minion.functions[fun],
salt.utils.args.parse_input(self.opts['arg']),
data=sdata)
try:
with salt.utils.fopen(proc_fn, 'w+b') as fp_:
fp_.write(self.serial.dumps(sdata))
except NameError:
# Don't require msgpack with local
pass
except IOError:
sys.stderr.write(
'Cannot write to process directory. '
'Do you have permissions to '
'write to {0} ?\n'.format(proc_fn))
func = self.minion.functions[fun]
try:
ret['return'] = func(*args, **kwargs)
except TypeError as exc:
sys.stderr.write('\nPassed invalid arguments: {0}.\n\nUsage:\n'.format(exc))
print_cli(func.__doc__)
active_level = LOG_LEVELS.get(
self.opts['log_level'].lower(), logging.ERROR)
if active_level <= logging.DEBUG:
trace = traceback.format_exc()
sys.stderr.write(trace)
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
try:
ret['retcode'] = sys.modules[
func.__module__].__context__.get('retcode', 0)
except AttributeError:
ret['retcode'] = 1
except (CommandExecutionError) as exc:
msg = 'Error running \'{0}\': {1}\n'
active_level = LOG_LEVELS.get(
self.opts['log_level'].lower(), logging.ERROR)
if active_level <= logging.DEBUG:
sys.stderr.write(traceback.format_exc())
sys.stderr.write(msg.format(fun, str(exc)))
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
except CommandNotFoundError as exc:
msg = 'Command required for \'{0}\' not found: {1}\n'
sys.stderr.write(msg.format(fun, str(exc)))
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
try:
os.remove(proc_fn)
except (IOError, OSError):
pass
if hasattr(self.minion.functions[fun], '__outputter__'):
oput = self.minion.functions[fun].__outputter__
if isinstance(oput, six.string_types):
ret['out'] = oput
is_local = self.opts['local'] or self.opts.get(
'file_client', False) == 'local'
returners = self.opts.get('return', '').split(',')
if (not is_local) or returners:
ret['id'] = self.opts['id']
ret['fun'] = fun
ret['fun_args'] = self.opts['arg']
for returner in returners:
if not returner: # if we got an empty returner somehow, skip
continue
try:
ret['success'] = True
self.minion.returners['{0}.returner'.format(returner)](ret)
except Exception:
pass
# return the job infos back up to the respective minion's master
if not is_local:
try:
mret = ret.copy()
mret['jid'] = 'req'
self.return_pub(mret)
except Exception:
pass
# close raet channel here
return ret
def run(self):
'''
Execute the salt command line
'''
import salt.client
self.parse_args()
# Setup file logging!
self.setup_logfile_logger()
verify_log(self.config)
try:
# We don't need to bail on config file permission errors
# if the CLI process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
self.local_client = salt.client.get_local_client(
self.get_config_file_path(),
skip_perm_errors=skip_perm_errors,
auto_reconnect=True)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.batch or self.options.static:
# _run_batch() will handle all output and
# exit with the appropriate error condition
# Execution will not continue past this point
# in batch mode.
self._run_batch()
return
if self.options.preview_target:
minion_list = self._preview_target()
self._output_ret(minion_list, self.config.get('output', 'nested'))
return
if self.options.timeout <= 0:
self.options.timeout = self.local_client.opts['timeout']
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid
}
if 'token' in self.config:
try:
with salt.utils.fopen(
os.path.join(self.config['cachedir'], '.root_key'),
'r') as fp_:
kwargs['key'] = fp_.readline()
except IOError:
kwargs['token'] = self.config['token']
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['tgt_type'] = self.selected_target_option
else:
kwargs['tgt_type'] = 'glob'
# If batch_safe_limit is set, check minions matching target and
# potentially switch to batch execution
if self.options.batch_safe_limit > 1:
if len(self._preview_target()) >= self.options.batch_safe_limit:
print_cli(
'\nNOTICE: Too many minions targeted, switching to batch execution.'
)
self.options.batch = self.options.batch_safe_size
self._run_batch()
return
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'return_kwargs'):
kwargs['ret_kwargs'] = yamlify_arg(
getattr(self.options, 'return_kwargs'))
if getattr(self.options, 'module_executors'):
kwargs['module_executors'] = yamlify_arg(
getattr(self.options, 'module_executors'))
if getattr(self.options, 'executor_opts'):
kwargs['executor_opts'] = yamlify_arg(
getattr(self.options, 'executor_opts'))
if getattr(self.options, 'metadata'):
kwargs['metadata'] = yamlify_arg(getattr(self.options, 'metadata'))
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and 'key' not in kwargs and self.options.eauth:
# This is expensive. Don't do it unless we need to.
import salt.auth
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
if self.config['async']:
jid = self.local_client.cmd_async(**kwargs)
print_cli('Executed command with job ID: {0}'.format(jid))
return
# local will be None when there was an error
if not self.local_client:
return
retcodes = []
errors = []
try:
if self.options.subset:
cmd_func = self.local_client.cmd_subset
kwargs['sub'] = self.options.subset
kwargs['cli'] = True
else:
cmd_func = self.local_client.cmd_cli
if self.options.progress:
kwargs['progress'] = True
self.config['progress'] = True
ret = {}
for progress in cmd_func(**kwargs):
out = 'progress'
try:
self._progress_ret(progress, out)
except salt.exceptions.LoaderError as exc:
raise salt.exceptions.SaltSystemExit(exc)
if 'return_count' not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config['fun'] == 'sys.doc':
ret = {}
out = ''
for full_ret in self.local_client.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs['verbose'] = True
ret = {}
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(full_ret)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config['cli_summary'] is True:
if self.config['fun'] != 'sys.doc':
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.stderr.write(
'ERROR: Minions returned with non-zero exit code\n')
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError,
SaltClientError) as exc:
ret = str(exc)
self._output_ret(ret, '')
def run(self):
"""
Execute the salt command line
"""
self.parse_args()
if self.config["verify_env"]:
if not self.config["log_file"].startswith(("tcp://", "udp://", "file://")):
# Logfile is not using Syslog, verify
verify_files([self.config["log_file"]], self.config["user"])
# Setup file logging!
self.setup_logfile_logger()
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ""
local = salt.client.get_local_client(self.get_config_file_path(), skip_perm_errors=skip_perm_errors)
except SaltClientError as exc:
self.exit(2, "{0}\n".format(exc))
return
if self.options.batch:
eauth = {}
if "token" in self.config:
eauth["token"] = self.config["token"]
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if "token" not in eauth and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
eauth["token"] = tok.get("token", "")
if not res:
sys.exit(2)
eauth.update(res)
eauth["eauth"] = self.options.eauth
if self.options.static:
batch = salt.cli.batch.Batch(self.config, eauth=eauth, quiet=True)
ret = {}
for res in batch.run():
ret.update(res)
self._output_ret(ret, "")
else:
batch = salt.cli.batch.Batch(self.config, eauth=eauth)
# Printing the output is already taken care of in run() itself
for res in batch.run():
pass
else:
if self.options.timeout <= 0:
self.options.timeout = local.opts["timeout"]
kwargs = {
"tgt": self.config["tgt"],
"fun": self.config["fun"],
"arg": self.config["arg"],
"timeout": self.options.timeout,
"show_timeout": self.options.show_timeout,
"show_jid": self.options.show_jid,
}
if "token" in self.config:
try:
with salt.utils.fopen(os.path.join(self.config["cachedir"], ".root_key"), "r") as fp_:
kwargs["key"] = fp_.readline()
except IOError:
kwargs["token"] = self.config["token"]
if self.selected_target_option:
kwargs["expr_form"] = self.selected_target_option
else:
kwargs["expr_form"] = "glob"
if getattr(self.options, "return"):
kwargs["ret"] = getattr(self.options, "return")
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if "token" not in kwargs and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs["token"] = tok.get("token", "")
if not res:
sys.exit(2)
kwargs.update(res)
kwargs["eauth"] = self.options.eauth
if self.config["async"]:
jid = local.cmd_async(**kwargs)
print_cli("Executed command with job ID: {0}".format(jid))
return
retcodes = []
try:
# local will be None when there was an error
if local:
if self.options.subset:
cmd_func = local.cmd_subset
kwargs["sub"] = self.options.subset
kwargs["cli"] = True
else:
cmd_func = local.cmd_cli
if self.options.static:
if self.options.verbose:
kwargs["verbose"] = True
full_ret = local.cmd_full_return(**kwargs)
ret, out, retcode = self._format_ret(full_ret)
self._output_ret(ret, out)
elif self.config["fun"] == "sys.doc":
ret = {}
out = ""
for full_ret in local.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs["verbose"] = True
ret = {}
for full_ret in cmd_func(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(ret_)
# Returns summary
if self.config["cli_summary"] is True:
if self.config["fun"] != "sys.doc":
if self.options.output is None:
self._print_returns_summary(ret)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError) as exc:
ret = str(exc)
out = ""
self._output_ret(ret, out)
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
for each_minion in ret:
if ret[each_minion] == "Minion did not return":
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of Minions Targeted: {0}'.format(return_counter + not_return_counter))
print_cli('# of Minions Returned: {0}'.format(return_counter))
print_cli('# of Minions Did Not Return: {0}'.format(not_return_counter))
if self.options.verbose:
print_cli('Minions Which Did Not Return: {0}'.format(" ".join(not_return_minions)))
print_cli('-------------------------------------------')
def executeCallback(selfIp):
redisChannel = clientPub.getRedisInstance().pubsub()
redisChannel.subscribe(wrapMesage['tempTopic'])
noResponseRet = []
noConnectRet = []
emptyRet = []
# retcodes = []
comeSubList = clientPub.pullAccept()
# handle special syndic
if selfIp in comeSubList:
comeSubList.remove(selfIp)
syndic_count = len(comeSubList)
resultCount = 0
pingCount = 0
resultPingSet = set()
sucset = set()
resultExeSet = set()
debugSet = set()
repeatet = set()
lossSyndic = []
executeStart = time.time()
normalDone = False
# NOTE: must publish cmd after registered the redis listen
# else we will miss ping message
clientPub.publishToSyndicSub(
salt.newrun.json.dumps(wrapMesage))
from salt.newrun import (json, byteify, MessageType)
normalsize = 0
for message in redisChannel.listen():
try:
messageJson = byteify(message)
if messageJson['type'] == 'message':
resultMessage = messageJson['data']
try:
callResult = json.loads(resultMessage,
encoding='utf-8')
callResult = byteify(callResult)
if isinstance(callResult, dict):
if 'type' in callResult:
messageType = callResult['type']
messageIp = callResult['sub_ip']
if messageType == MessageType.PING and messageIp in comeSubList:
resultPingSet.add(messageIp)
pingCount += 1
elif messageType == MessageType.WORK or messageType == MessageType.INTERRUPT:
# main_log.info('work or interurupt: %s' % (messageIp))
resultExeSet.add(messageIp)
resultCount += 1
else:
main_log.info(
'invalid callresult: %s' %
callResult)
else:
# filter no return received of sub node
retJsonObj = callResult['ret_']
if retJsonObj:
# reset start time
executeStart = time.time()
if callResult[
'out'] == 'no_return':
if '[No response]' in json.dumps(
retJsonObj):
noResponseRet.append(
callResult)
else:
noConnectRet.append(
callResult)
# add to missed list
for k, v in retJsonObj.items():
missedList.add(k)
else:
# put successed ip to tmp set
for k, v in retJsonObj.items():
sucset.add(k)
# NOTE: debug
if k in debugSet:
repeatet.add(
json.dumps(
retJsonObj))
else:
debugSet.add(k)
if callResult['retcode'] == 0:
isnil = False
for k in retJsonObj.keys():
v = retJsonObj[k]
if v == "":
isnil = True
break
if isnil:
emptyRet.append(
callResult)
else:
normalsize += 1
self._output_ret(
callResult['ret_'],
callResult['out'])
else:
emptyRet.append(callResult)
else:
# TODO handle other messages?
pass
except:
resultCount += 1
print_cli(traceback.format_exc())
pass
##check sub timeout, if no node running again
#lossSyndic = [item for item in comeSubList if item not in resultExeSet]
#print(lossSyndic)
losePingCount = syndic_count - pingCount
runningCount = syndic_count - resultCount - losePingCount
#lossPing = [item for item in comeSubList if item not in resultPingSet]
#main_log.info("%s,%s,%s,%s,%s,%s" % (pingCount, syndic_count, runningCount, sub_timeout, executeStart, lossPing))
if pingCount < syndic_count and runningCount <= 0:
if (time.time() - executeStart) > sub_timeout:
main_log.info("---T0 stop")
break
elif syndic_count == pingCount and runningCount > 0:
if (time.time() - executeStart) > sub_timeout:
# main_log.info("---T1 stop")
break
elif syndic_count == pingCount and resultCount == syndic_count:
# main_log.info("---T2 stop")
break
except:
main_log.info(traceback.format_exc())
pass
redisChannel.unsubscribe(wrapMesage['tempTopic'])
redisChannel.connection_pool.disconnect()
# main_log.info('---T: %s, %s, %s' % (emptyRet, noResponseRet, noConnectRet))
##begin print error returns
for result in emptyRet:
if result['ret_']:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noResponseRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noConnectRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
disconnectedSyndic = set(comeSubList).difference(resultPingSet)
if disconnectedSyndic:
print_cli('With disconnected syndic: %s' %
list(disconnectedSyndic))
if len(missedList) > 0 or len(lossSyndic) > 0:
print('missed maids: {}\nmissed minions: {}'.format(
",".join(lossSyndic), ",".join(missedList)))
if len(repeatet) > 0:
print('Find some minion run repeated: {}'.format(repeatet))
print(
'normal size: {}\nmissed size: {}\nempty size: {}'.format(
normalsize, len(missedList), len(emptyRet)))
def run(self):
'''
Execute the salt command line
'''
import salt.auth
import salt.client
self.parse_args()
# Setup file logging!
self.setup_logfile_logger()
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
local = salt.client.get_local_client(
self.get_config_file_path(), skip_perm_errors=skip_perm_errors)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.batch or self.options.static:
import salt.cli.batch
eauth = {}
if 'token' in self.config:
eauth['token'] = self.config['token']
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in eauth and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
eauth['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
eauth.update(res)
eauth['eauth'] = self.options.eauth
if self.options.static:
if not self.options.batch:
self.config['batch'] = '100%'
batch = salt.cli.batch.Batch(self.config,
eauth=eauth,
quiet=True)
ret = {}
for res in batch.run():
ret.update(res)
self._output_ret(ret, '')
else:
batch = salt.cli.batch.Batch(self.config, eauth=eauth)
# Printing the output is already taken care of in run() itself
for res in batch.run():
if self.options.failhard:
for ret in six.itervalues(res):
retcode = salt.utils.job.get_retcode(ret)
if retcode != 0:
sys.stderr.write(
'ERROR: Minions returned with non-zero exit code\n'
)
sys.exit(retcode)
else:
if self.options.timeout <= 0:
self.options.timeout = local.opts['timeout']
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid
}
if 'token' in self.config:
try:
with salt.utils.fopen(
os.path.join(self.config['cachedir'], '.root_key'),
'r') as fp_:
kwargs['key'] = fp_.readline()
except IOError:
kwargs['token'] = self.config['token']
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['expr_form'] = self.selected_target_option
else:
kwargs['expr_form'] = 'glob'
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'metadata'):
kwargs['metadata'] = getattr(self.options, 'metadata')
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
if self.config['async']:
jid = local.cmd_async(**kwargs)
print_cli('Executed command with job ID: {0}'.format(jid))
return
retcodes = []
try:
# local will be None when there was an error
errors = []
if local:
if self.options.subset:
cmd_func = local.cmd_subset
kwargs['sub'] = self.options.subset
kwargs['cli'] = True
else:
cmd_func = local.cmd_cli
if self.options.progress:
kwargs['progress'] = True
self.config['progress'] = True
ret = {}
for progress in cmd_func(**kwargs):
out = 'progress'
try:
self._progress_ret(progress, out)
except salt.exceptions.LoaderError as exc:
raise salt.exceptions.SaltSystemExit(exc)
if 'return_count' not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config['fun'] == 'sys.doc':
ret = {}
out = ''
for full_ret in local.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs['verbose'] = True
ret = {}
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(ret_)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config['cli_summary'] is True:
if self.config['fun'] != 'sys.doc':
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.stderr.write(
'ERROR: Minions returned with non-zero exit code\n'
)
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError,
SaltClientError) as exc:
ret = str(exc)
out = ''
self._output_ret(ret, out)
def run(self):
'''
Execute the batch run
'''
args = [
[],
self.opts['fun'],
self.opts['arg'],
self.opts['timeout'],
'list',
]
bnum = self.get_bnum()
to_run = copy.deepcopy(self.minions)
active = []
ret = {}
iters = []
# the minion tracker keeps track of responses and iterators
# - it removes finished iterators from iters[]
# - if a previously detected minion does not respond, its
# added with an empty answer to ret{} once the timeout is reached
# - unresponsive minions are removed from active[] to make
# sure that the main while loop finishes even with unresp minions
minion_tracker = {}
# Iterate while we still have things to execute
while len(ret) < len(self.minions):
next_ = []
if len(to_run) <= bnum and not active:
# last bit of them, add them all to next iterator
while to_run:
next_.append(to_run.pop())
else:
for i in range(bnum - len(active)):
if to_run:
minion_id = to_run.pop()
if isinstance(minion_id, dict):
next_.append(minion_id.keys()[0])
else:
next_.append(minion_id)
active += next_
args[0] = next_
if next_:
if not self.quiet:
print_cli('\nExecuting run on {0}\n'.format(next_))
# create a new iterator for this batch of minions
new_iter = self.local.cmd_iter_no_block(
*args,
raw=self.opts.get('raw', False),
ret=self.opts.get('return', ''),
**self.eauth)
# add it to our iterators and to the minion_tracker
iters.append(new_iter)
minion_tracker[new_iter] = {}
# every iterator added is 'active' and has its set of minions
minion_tracker[new_iter]['minions'] = next_
minion_tracker[new_iter]['active'] = True
else:
time.sleep(0.02)
parts = {}
# see if we found more minions
for ping_ret in self.ping_gen:
if ping_ret is None:
break
m = next(ping_ret.iterkeys())
if m not in self.minions:
self.minions.append(m)
to_run.append(m)
for queue in iters:
try:
# Gather returns until we get to the bottom
ncnt = 0
while True:
part = next(queue)
if part is None:
time.sleep(0.01)
ncnt += 1
if ncnt > 5:
break
continue
if self.opts.get('raw'):
parts.update({part['id']: part})
else:
parts.update(part)
except StopIteration:
# if a iterator is done:
# - set it to inactive
# - add minions that have not responded to parts{}
# check if the tracker contains the iterator
if queue in minion_tracker:
minion_tracker[queue]['active'] = False
# add all minions that belong to this iterator and
# that have not responded to parts{} with an empty response
for minion in minion_tracker[queue]['minions']:
if minion not in parts:
parts[minion] = {}
parts[minion]['ret'] = {}
for minion, data in six.iteritems(parts):
if minion in active:
active.remove(minion)
if self.opts.get('raw'):
yield data
else:
ret[minion] = data['ret']
yield {minion: data['ret']}
if not self.quiet:
ret[minion] = data['ret']
data[minion] = data.pop('ret')
if 'out' in data:
out = data.pop('out')
else:
out = None
salt.output.display_output(data, out, self.opts)
# remove inactive iterators from the iters list
for queue in minion_tracker:
# only remove inactive queues
if not minion_tracker[queue]['active'] and queue in iters:
iters.remove(queue)
# also remove the iterator's minions from the active list
for minion in minion_tracker[queue]['minions']:
if minion in active:
active.remove(minion)
def _print_returns_summary(self, ret):
"""
Display returns summary
"""
return_counter = 0
not_return_counter = 0
not_return_minions = []
not_response_minions = []
not_connected_minions = []
failed_minions = []
for each_minion in ret:
minion_ret = ret[each_minion].get("ret")
if isinstance(minion_ret, string_types) and minion_ret.startswith("Minion did not return"):
if "Not connected" in minion_ret:
not_connected_minions.append(each_minion)
elif "No response" in minion_ret:
not_response_minions.append(each_minion)
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
if self._get_retcode(ret[each_minion]):
failed_minions.append(each_minion)
print_cli("\n")
print_cli("-------------------------------------------")
print_cli("Summary")
print_cli("-------------------------------------------")
print_cli("# of minions targeted: {0}".format(return_counter + not_return_counter))
print_cli("# of minions returned: {0}".format(return_counter))
print_cli("# of minions that did not return: {0}".format(not_return_counter))
print_cli("# of minions with errors: {0}".format(len(failed_minions)))
if self.options.verbose:
if not_connected_minions:
print_cli("Minions not connected: {0}".format(" ".join(not_connected_minions)))
if not_response_minions:
print_cli("Minions not responding: {0}".format(" ".join(not_response_minions)))
if failed_minions:
print_cli("Minions with failures: {0}".format(" ".join(failed_minions)))
print_cli("-------------------------------------------")
def new_run(self):
# '''
# Execute the salt command line
# '''
import salt.client
# self.parse_args()
# print('################:%s' % (self.config['order_masters']==True))
signal.signal(signal.SIGINT, self.quit)
signal.signal(signal.SIGTERM, self.quit)
if self.config['log_level'] not in ('quiet', ):
# Setup file logging!
self.setup_logfile_logger()
verify_log(self.config)
try:
# We don't need to bail on config file permission errors
# if the CLI process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
self.local_client = salt.client.get_local_client(
self.get_config_file_path(),
skip_perm_errors=skip_perm_errors,
auto_reconnect=True)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.preview_target:
minion_list = self._preview_target()
self._output_ret(minion_list, self.config.get('output', 'nested'))
return
if self.options.timeout <= 0:
self.options.timeout = self.local_client.opts['timeout']
# read tgt list from file
if self.options.file_target:
try:
with open(self.config['tgt']) as xf:
xfContent = xf.read().strip("\n").strip(' ')
if xfContent == '':
self.exit(
2,
'Find empty ip list from {0}, pls check.\n'.format(
self.config['tgt']))
return
if ',' in xfContent:
self.config['tgt'] = xfContent.split(",")
self.selected_target_option = 'list'
elif '\n' in xfContent:
self.config['tgt'] = xfContent.split("\n")
self.selected_target_option = 'list'
else:
print('Find invalid args with -X.')
return
except IOError as exc:
self.exit(2, '{0}\n'.format(exc))
return
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid
}
# kwargs = self.config
# kwargs['timeout'] = self.options.timeout
# kwargs['show_timeout'] = self.options.show_timeout
# kwargs['show_jid'] = self.options.show_jid
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['tgt_type'] = self.selected_target_option
else:
kwargs['tgt_type'] = 'glob'
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'return_kwargs'):
kwargs['ret_kwargs'] = yamlify_arg(
getattr(self.options, 'return_kwargs'))
if getattr(self.options, 'module_executors'):
kwargs['module_executors'] = yamlify_arg(
getattr(self.options, 'module_executors'))
if getattr(self.options, 'metadata'):
kwargs['metadata'] = yamlify_arg(getattr(self.options, 'metadata'))
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and 'key' not in kwargs and self.options.eauth:
# This is expensive. Don't do it unless we need to.
import salt.auth
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(self.options.eauth, res)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
self.newopt = self.config
sub_timeout = self.newopt['channel_sub_timeout']
if self.options.timeout > sub_timeout:
sub_timeout = self.options.timeout
self.bootConfig = {
'_sub_timeout': sub_timeout,
'_sub_node': '',
'_channel_redis_sentinel': self.newopt['channel_redis_sentinel'],
'_channel_redis_password': self.newopt['channel_redis_password'],
'_master_pub_topic': self.newopt['id']
}
# NOTE: Only in super master, filter no-response ip, when use saltx
def getPassedIp():
import numpy
numpy.warnings.filterwarnings('ignore')
passed_ip = numpy.loadtxt('/data0/md/ip.md', dtype=numpy.str)
return passed_ip
missedList = set()
runAllminion = False
if isinstance(kwargs['tgt'], list):
passed_ip = getPassedIp()
kwargs['tgt'] = [i for i in kwargs['tgt'] if i not in passed_ip]
if len(kwargs['tgt']) == 0:
print_cli('There are nothing iplist to be apply.')
return
else:
if kwargs['tgt'] != '*':
passed_ip = getPassedIp()
if kwargs['tgt'] in passed_ip:
print_cli('There are nothing iplist to be apply.')
return
else:
runAllminion = True
import salt.newrun
clientPub = salt.newrun.MasterPub(**self.bootConfig)
if self.config['async']:
# NOTE: generate a jid for saltx
jid = salt.utils.jid.gen_jid()
wrapMesage = {
'type': salt.newrun.FunctionType.ASYNC_RUN,
'jid': jid,
'kwargs': kwargs,
'tempTopic': str(salt.newrun.uuid.uuid1())
}
clientPub.publishToSyndicSub(salt.newrun.json.dumps(wrapMesage))
print_cli('Executed command with master job ID: {0}'.format(jid))
return
else:
wrapMesage = {
'type': salt.newrun.FunctionType.SYNC_RUN,
'kwargs': kwargs,
'tempTopic': str(salt.newrun.uuid.uuid1())
}
batch_hold = 0
lossSyndic = []
repeatet = set()
emptyRet = []
noResponseRet = []
noConnectRet = []
# running, make sure to be batch count
global batch_running
batch_running = set()
# init batch ip
batch_init = set()
comeSubList = clientPub.pullAccept()
resultPingSet = []
resultExeSet = []
global normalsize
normalsize = 0
sucset = set()
debugSet = set()
def batchExecuteCallback(selfIp, clientPub, redisChannel):
while len(batch_running) <= 0:
tmpKwargs = wrapMesage['kwargs']
try:
for i in range(batch_hold):
batch_running.add(batch_init.pop())
# trigger to sub run
tmpKwargs['tgt'] = list(batch_running)
wrapMesage['kwargs'] = tmpKwargs
batchRun(wrapMesage, selfIp, clientPub, redisChannel)
except:
if len(batch_running) > 0:
# trigger to sub run
tmpKwargs['tgt'] = list(batch_running)
wrapMesage['kwargs'] = tmpKwargs
batchRun(wrapMesage, selfIp, clientPub,
redisChannel)
else:
break
##begin print error returns
for result in emptyRet:
if result['ret_']:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noResponseRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noConnectRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
disconnectedSyndic = set(comeSubList).difference(resultPingSet)
if disconnectedSyndic:
print_cli('With disconnected syndic: %s' %
list(disconnectedSyndic))
if len(missedList) > 0 or len(lossSyndic) > 0:
print('missed maids: {}\nmissed minions: {}'.format(
",".join(lossSyndic), ",".join(missedList)))
if len(repeatet) > 0:
print('Find some minion run repeated: {}'.format(repeatet))
global normalsize
print(
'normal size: {}\nmissed size: {}\nempty size: {}'.format(
normalsize, len(missedList), len(emptyRet)))
redisChannel.unsubscribe(wrapMesage['tempTopic'])
redisChannel.connection_pool.disconnect()
def batchRun(wrapMesage, selfIp, clientPub, redisChannel):
# NOTE: batch running mode
# handle special syndic
if selfIp in comeSubList:
comeSubList.remove(selfIp)
syndic_count = len(comeSubList)
resultCount = 0
pingCount = 0
executeStart = time.time()
normalDone = False
# NOTE: must publish cmd after registered the redis listen
# else we will miss ping message
# tmpKwargs1 = wrapMesage['kwargs']
# batch_running = set(tmpKwargs1['tgt'])
#print('publish wrapMesage: %s' % wrapMesage)
clientPub.publishToSyndicSub(
salt.newrun.json.dumps(wrapMesage))
from salt.newrun import (json, byteify, MessageType)
for message in redisChannel.listen():
try:
messageJson = byteify(message)
if messageJson['type'] == 'message':
resultMessage = messageJson['data']
try:
callResult = json.loads(resultMessage,
encoding='utf-8')
callResult = byteify(callResult)
if isinstance(callResult, dict):
if 'type' in callResult:
messageType = callResult['type']
messageIp = callResult['sub_ip']
if messageType == MessageType.PING and messageIp in comeSubList:
resultPingSet.append(messageIp)
elif messageType == MessageType.WORK or messageType == MessageType.INTERRUPT:
resultExeSet.append(messageIp)
else:
main_log.info(
'invalid callresult: %s' %
callResult)
else:
# filter no return received of sub node
retJsonObj = callResult['ret_']
if retJsonObj:
# reset start time
executeStart = time.time()
for k, v in retJsonObj.items():
# reset running and wait node
batch_running.discard(k)
if callResult[
'out'] == 'no_return':
if '[No response]' in json.dumps(
retJsonObj):
noResponseRet.append(
callResult)
else:
noConnectRet.append(
callResult)
else:
# put successed ip to tmp set
for k, v in retJsonObj.items():
sucset.add(k)
# NOTE: debug
if k in debugSet:
repeatet.add(
json.dumps(
retJsonObj))
else:
debugSet.add(k)
if callResult['retcode'] == 0:
isnil = False
for k in retJsonObj.keys():
v = retJsonObj[k]
if v == "":
isnil = True
break
if isnil:
emptyRet.append(
callResult)
else:
global normalsize
normalsize += 1
self._output_ret(
callResult['ret_'],
callResult['out'])
else:
emptyRet.append(callResult)
else:
# TODO handle other messages?
pass
except:
resultCount += 1
print_cli(traceback.format_exc())
pass
pingCount = len(resultPingSet)
resultCount = len(resultExeSet)
#from collections import Counter
#main_log.info("%s, %s, %s, %s" % (pingCount, resultCount, Counter(resultPingSet), Counter(resultExeSet)))
# if len(batch_init) <= 0:
# break
if len(batch_running) == 0:
break
if pingCount != resultCount:
if (time.time() - executeStart) > sub_timeout:
# main_log.info("---T0 stop")
break
except:
main_log.info(traceback.format_exc())
pass
def executeCallback(selfIp):
redisChannel = clientPub.getRedisInstance().pubsub()
redisChannel.subscribe(wrapMesage['tempTopic'])
noResponseRet = []
noConnectRet = []
emptyRet = []
# retcodes = []
comeSubList = clientPub.pullAccept()
# handle special syndic
if selfIp in comeSubList:
comeSubList.remove(selfIp)
syndic_count = len(comeSubList)
resultCount = 0
pingCount = 0
resultPingSet = set()
sucset = set()
resultExeSet = set()
debugSet = set()
repeatet = set()
lossSyndic = []
executeStart = time.time()
normalDone = False
# NOTE: must publish cmd after registered the redis listen
# else we will miss ping message
clientPub.publishToSyndicSub(
salt.newrun.json.dumps(wrapMesage))
from salt.newrun import (json, byteify, MessageType)
normalsize = 0
for message in redisChannel.listen():
try:
messageJson = byteify(message)
if messageJson['type'] == 'message':
resultMessage = messageJson['data']
try:
callResult = json.loads(resultMessage,
encoding='utf-8')
callResult = byteify(callResult)
if isinstance(callResult, dict):
if 'type' in callResult:
messageType = callResult['type']
messageIp = callResult['sub_ip']
if messageType == MessageType.PING and messageIp in comeSubList:
resultPingSet.add(messageIp)
pingCount += 1
elif messageType == MessageType.WORK or messageType == MessageType.INTERRUPT:
# main_log.info('work or interurupt: %s' % (messageIp))
resultExeSet.add(messageIp)
resultCount += 1
else:
main_log.info(
'invalid callresult: %s' %
callResult)
else:
# filter no return received of sub node
retJsonObj = callResult['ret_']
if retJsonObj:
# reset start time
executeStart = time.time()
if callResult[
'out'] == 'no_return':
if '[No response]' in json.dumps(
retJsonObj):
noResponseRet.append(
callResult)
else:
noConnectRet.append(
callResult)
# add to missed list
for k, v in retJsonObj.items():
missedList.add(k)
else:
# put successed ip to tmp set
for k, v in retJsonObj.items():
sucset.add(k)
# NOTE: debug
if k in debugSet:
repeatet.add(
json.dumps(
retJsonObj))
else:
debugSet.add(k)
if callResult['retcode'] == 0:
isnil = False
for k in retJsonObj.keys():
v = retJsonObj[k]
if v == "":
isnil = True
break
if isnil:
emptyRet.append(
callResult)
else:
normalsize += 1
self._output_ret(
callResult['ret_'],
callResult['out'])
else:
emptyRet.append(callResult)
else:
# TODO handle other messages?
pass
except:
resultCount += 1
print_cli(traceback.format_exc())
pass
##check sub timeout, if no node running again
#lossSyndic = [item for item in comeSubList if item not in resultExeSet]
#print(lossSyndic)
losePingCount = syndic_count - pingCount
runningCount = syndic_count - resultCount - losePingCount
#lossPing = [item for item in comeSubList if item not in resultPingSet]
#main_log.info("%s,%s,%s,%s,%s,%s" % (pingCount, syndic_count, runningCount, sub_timeout, executeStart, lossPing))
if pingCount < syndic_count and runningCount <= 0:
if (time.time() - executeStart) > sub_timeout:
main_log.info("---T0 stop")
break
elif syndic_count == pingCount and runningCount > 0:
if (time.time() - executeStart) > sub_timeout:
# main_log.info("---T1 stop")
break
elif syndic_count == pingCount and resultCount == syndic_count:
# main_log.info("---T2 stop")
break
except:
main_log.info(traceback.format_exc())
pass
redisChannel.unsubscribe(wrapMesage['tempTopic'])
redisChannel.connection_pool.disconnect()
# main_log.info('---T: %s, %s, %s' % (emptyRet, noResponseRet, noConnectRet))
##begin print error returns
for result in emptyRet:
if result['ret_']:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noResponseRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
for result in noConnectRet:
if result['ret_']:
for k, v in result['ret_'].items():
if k not in sucset:
# begin print in client console
self._output_ret(result['ret_'], result['out'])
disconnectedSyndic = set(comeSubList).difference(resultPingSet)
if disconnectedSyndic:
print_cli('With disconnected syndic: %s' %
list(disconnectedSyndic))
if len(missedList) > 0 or len(lossSyndic) > 0:
print('missed maids: {}\nmissed minions: {}'.format(
",".join(lossSyndic), ",".join(missedList)))
if len(repeatet) > 0:
print('Find some minion run repeated: {}'.format(repeatet))
print(
'normal size: {}\nmissed size: {}\nempty size: {}'.format(
normalsize, len(missedList), len(emptyRet)))
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
# if retcodes.count(0) < len(retcodes):
# sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
# sys.exit(11)
if self.options.batch:
bwait = self.config.get('batch_wait', 0)
redisChannel = clientPub.getRedisInstance().pubsub()
percentBatch = 0.0
try:
if self.options.batch.endswith('%'):
stripBatch = float(self.options.batch.strip('%'))
percentBatch = stripBatch / 100
else:
batch_hold = int(self.options.batch)
except:
print('An Int or Percent can be used for batch.')
return
# find all ip list
if kwargs['tgt'] == '*':
reGetAllMinionList = []
wrapFindAcceptMesage = {
'type': salt.newrun.FunctionType.FIND_ACCEPT,
'tempTopic': ('fa_%s' % str(salt.newrun.uuid.uuid1()))
}
redisChannel.subscribe(wrapFindAcceptMesage['tempTopic'])
clientPub.publishToSyndicSub(
salt.newrun.json.dumps(wrapFindAcceptMesage))
from salt.newrun import (json, byteify, MessageType)
ping1stCount = 0
work1stcount = 0
for message in redisChannel.listen():
try:
messageJson = byteify(message)
if messageJson['type'] == 'message':
resultMessage = messageJson['data']
try:
callResult = json.loads(resultMessage,
encoding='utf-8')
callResult = byteify(callResult)
if isinstance(callResult, dict):
if 'type' in callResult:
messageType = callResult['type']
messageIp = callResult['sub_ip']
if messageType == MessageType.PING and messageIp in comeSubList:
ping1stCount += 1
elif messageType == MessageType.WORK or messageType == MessageType.INTERRUPT:
work1stcount += 1
if ping1stCount == work1stcount and work1stcount == len(
comeSubList):
break
else:
main_log.info(
'invalid callresult: %s' %
callResult)
else:
# filter no return received of sub node
retJsonObj = callResult['ip_list']
if retJsonObj:
reGetAllMinionList = reGetAllMinionList + retJsonObj
else:
pass
#print('callResult: %s' % callResult)
except:
main_log.info(traceback.format_exc())
pass
except:
main_log.info(traceback.format_exc())
pass
kwargs['tgt'] = reGetAllMinionList
batch_init = set(kwargs['tgt'])
redisChannel.unsubscribe(wrapFindAcceptMesage['tempTopic'])
redisChannel.connection_pool.disconnect()
else:
if kwargs['tgt_type'] == 'glob':
batch_init.add(kwargs['tgt'])
else:
batch_init = set(kwargs['tgt'])
kwargs['tgt_type'] = 'list'
wrapMesage['kwargs'] = kwargs
if percentBatch > 0:
batch_hold = percentBatch * len(batch_init)
redisChannel.subscribe(wrapMesage['tempTopic'])
batchExecuteCallback(self.newopt['id'], clientPub,
redisChannel)
else:
executeCallback(self.newopt['id'])
def run(self):
'''
Execute the salt command line
'''
import salt.auth
import salt.client
self.parse_args()
if self.config['verify_env']:
if not self.config['log_file'].startswith(('tcp://',
'udp://',
'file://')):
# Logfile is not using Syslog, verify
verify_files(
[self.config['log_file']],
self.config['user']
)
# Setup file logging!
self.setup_logfile_logger()
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
local = salt.client.get_local_client(
self.get_config_file_path(),
skip_perm_errors=skip_perm_errors)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.batch or self.options.static:
import salt.cli.batch
eauth = {}
if 'token' in self.config:
eauth['token'] = self.config['token']
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in eauth and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(
self.options.eauth,
res
)
if tok:
eauth['token'] = tok.get('token', '')
if not res:
sys.exit(2)
eauth.update(res)
eauth['eauth'] = self.options.eauth
if self.options.static:
if not self.options.batch:
self.config['batch'] = '100%'
batch = salt.cli.batch.Batch(self.config, eauth=eauth, quiet=True)
ret = {}
for res in batch.run():
ret.update(res)
self._output_ret(ret, '')
else:
batch = salt.cli.batch.Batch(self.config, eauth=eauth)
# Printing the output is already taken care of in run() itself
for res in batch.run():
if self.options.failhard:
for ret in res.itervalues():
retcode = salt.utils.job.get_retcode(ret)
if retcode != 0:
sys.exit(retcode)
else:
if self.options.timeout <= 0:
self.options.timeout = local.opts['timeout']
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid}
if 'token' in self.config:
try:
with salt.utils.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_:
kwargs['key'] = fp_.readline()
except IOError:
kwargs['token'] = self.config['token']
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['expr_form'] = self.selected_target_option
else:
kwargs['expr_form'] = 'glob'
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'metadata'):
kwargs['metadata'] = getattr(self.options, 'metadata')
if self.config['fun']:
import os
import urllib2
user = getattr(self.options, 'salt_user') or os.getenv('SALT_USER', '')
auth_url = 'http://auth.salt.4399api.net/request?tgt={0}&user={1}'.format(self.config['tgt'], user)
urllib2.urlopen(auth_url)
kwargs['xcj_code'] = raw_input('Enter xcj_code:')
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(
self.options.eauth,
res
)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
if self.config['async']:
jid = local.cmd_async(**kwargs)
print_cli('Executed command with job ID: {0}'.format(jid))
return
retcodes = []
try:
# local will be None when there was an error
errors = []
if local:
if self.options.subset:
cmd_func = local.cmd_subset
kwargs['sub'] = self.options.subset
kwargs['cli'] = True
else:
cmd_func = local.cmd_cli
if self.options.progress:
kwargs['progress'] = True
self.config['progress'] = True
ret = {}
for progress in cmd_func(**kwargs):
out = 'progress'
self._progress_ret(progress, out)
if 'return_count' not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config['fun'] == 'sys.doc':
ret = {}
out = ''
for full_ret in local.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs['verbose'] = True
ret = {}
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(ret_)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config['cli_summary'] is True:
if self.config['fun'] != 'sys.doc':
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc:
ret = str(exc)
out = ''
self._output_ret(ret, out)
def run(self):
'''
Execute the batch run
'''
args = [[],
self.opts['fun'],
self.opts['arg'],
self.opts['timeout'],
'list',
]
bnum = self.get_bnum()
# No targets to run
if not self.minions:
return
to_run = copy.deepcopy(self.minions)
active = []
ret = {}
iters = []
# wait the specified time before decide a job is actually done
bwait = self.opts.get('batch_wait', 0)
wait = []
if self.options:
show_jid = self.options.show_jid
show_verbose = self.options.verbose
else:
show_jid = False
show_verbose = False
# the minion tracker keeps track of responses and iterators
# - it removes finished iterators from iters[]
# - if a previously detected minion does not respond, its
# added with an empty answer to ret{} once the timeout is reached
# - unresponsive minions are removed from active[] to make
# sure that the main while loop finishes even with unresp minions
minion_tracker = {}
# We already know some minions didn't respond to the ping, so inform
# the user we won't be attempting to run a job on them
for down_minion in self.down_minions:
print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion))
# Iterate while we still have things to execute
while len(ret) < len(self.minions):
next_ = []
if bwait and wait:
self.__update_wait(wait)
if len(to_run) <= bnum - len(wait) and not active:
# last bit of them, add them all to next iterator
while to_run:
next_.append(to_run.pop())
else:
for i in range(bnum - len(active) - len(wait)):
if to_run:
minion_id = to_run.pop()
if isinstance(minion_id, dict):
next_.append(minion_id.keys()[0])
else:
next_.append(minion_id)
active += next_
args[0] = next_
if next_:
if not self.quiet:
print_cli('\nExecuting run on {0}\n'.format(sorted(next_)))
# create a new iterator for this batch of minions
new_iter = self.local.cmd_iter_no_block(
*args,
raw=self.opts.get('raw', False),
ret=self.opts.get('return', ''),
show_jid=show_jid,
verbose=show_verbose,
gather_job_timeout=self.opts['gather_job_timeout'],
**self.eauth)
# add it to our iterators and to the minion_tracker
iters.append(new_iter)
minion_tracker[new_iter] = {}
# every iterator added is 'active' and has its set of minions
minion_tracker[new_iter]['minions'] = next_
minion_tracker[new_iter]['active'] = True
else:
time.sleep(0.02)
parts = {}
# see if we found more minions
for ping_ret in self.ping_gen:
if ping_ret is None:
break
m = next(six.iterkeys(ping_ret))
if m not in self.minions:
self.minions.append(m)
to_run.append(m)
for queue in iters:
try:
# Gather returns until we get to the bottom
ncnt = 0
while True:
part = next(queue)
if part is None:
time.sleep(0.01)
ncnt += 1
if ncnt > 5:
break
continue
if self.opts.get('raw'):
parts.update({part['data']['id']: part})
if part['data']['id'] in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(part['data']['id'])
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id']))
else:
parts.update(part)
for id in part:
if id in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(id)
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id))
except StopIteration:
# if a iterator is done:
# - set it to inactive
# - add minions that have not responded to parts{}
# check if the tracker contains the iterator
if queue in minion_tracker:
minion_tracker[queue]['active'] = False
# add all minions that belong to this iterator and
# that have not responded to parts{} with an empty response
for minion in minion_tracker[queue]['minions']:
if minion not in parts:
parts[minion] = {}
parts[minion]['ret'] = {}
for minion, data in six.iteritems(parts):
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
# Munge retcode into return data
if 'retcode' in data and isinstance(data['ret'], dict) and 'retcode' not in data['ret']:
data['ret']['retcode'] = data['retcode']
if self.opts.get('raw'):
ret[minion] = data
yield data
else:
ret[minion] = data['ret']
yield {minion: data['ret']}
if not self.quiet:
ret[minion] = data['ret']
data[minion] = data.pop('ret')
if 'out' in data:
out = data.pop('out')
else:
out = None
salt.output.display_output(
data,
out,
self.opts)
# remove inactive iterators from the iters list
for queue in minion_tracker:
# only remove inactive queues
if not minion_tracker[queue]['active'] and queue in iters:
iters.remove(queue)
# also remove the iterator's minions from the active list
for minion in minion_tracker[queue]['minions']:
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
def call(self):
'''
Call the module
'''
ret = {}
fun = self.opts['fun']
ret['jid'] = salt.utils.jid.gen_jid()
proc_fn = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid']
)
if fun not in self.minion.functions:
sys.stderr.write(self.minion.functions.missing_fun_string(fun))
mod_name = fun.split('.')[0]
if mod_name in self.minion.function_errors:
sys.stderr.write(' Possible reasons: {0}\n'.format(self.minion.function_errors[mod_name]))
else:
sys.stderr.write('\n')
sys.exit(-1)
try:
sdata = {
'fun': fun,
'pid': os.getpid(),
'jid': ret['jid'],
'tgt': 'salt-call'}
args, kwargs = salt.minion.load_args_and_kwargs(
self.minion.functions[fun],
salt.utils.args.parse_input(self.opts['arg']),
data=sdata)
try:
with salt.utils.fopen(proc_fn, 'w+b') as fp_:
fp_.write(self.serial.dumps(sdata))
except NameError:
# Don't require msgpack with local
pass
except IOError:
sys.stderr.write(
'Cannot write to process directory. '
'Do you have permissions to '
'write to {0} ?\n'.format(proc_fn))
func = self.minion.functions[fun]
try:
ret['return'] = func(*args, **kwargs)
except TypeError as exc:
sys.stderr.write('\nPassed invalid arguments: {0}.\n\nUsage:\n'.format(exc))
print_cli(func.__doc__)
active_level = LOG_LEVELS.get(
self.opts['log_level'].lower(), logging.ERROR)
if active_level <= logging.DEBUG:
trace = traceback.format_exc()
sys.stderr.write(trace)
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
try:
ret['retcode'] = sys.modules[
func.__module__].__context__.get('retcode', 0)
except AttributeError:
ret['retcode'] = 1
except (CommandExecutionError) as exc:
msg = 'Error running \'{0}\': {1}\n'
active_level = LOG_LEVELS.get(
self.opts['log_level'].lower(), logging.ERROR)
if active_level <= logging.DEBUG:
sys.stderr.write(traceback.format_exc())
sys.stderr.write(msg.format(fun, str(exc)))
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
except CommandNotFoundError as exc:
msg = 'Command required for \'{0}\' not found: {1}\n'
sys.stderr.write(msg.format(fun, str(exc)))
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
try:
os.remove(proc_fn)
except (IOError, OSError):
pass
if hasattr(self.minion.functions[fun], '__outputter__'):
oput = self.minion.functions[fun].__outputter__
if isinstance(oput, six.string_types):
ret['out'] = oput
is_local = self.opts['local'] or self.opts.get(
'file_client', False) == 'local'
returners = self.opts.get('return', '').split(',')
if (not is_local) or returners:
ret['id'] = self.opts['id']
ret['fun'] = fun
ret['fun_args'] = self.opts['arg']
for returner in returners:
if not returner: # if we got an empty returner somehow, skip
continue
try:
ret['success'] = True
self.minion.returners['{0}.returner'.format(returner)](ret)
except Exception:
pass
# return the job infos back up to the respective minion's master
if not is_local:
try:
mret = ret.copy()
mret['jid'] = 'req'
self.return_pub(mret)
except Exception:
pass
# close raet channel here
return ret
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
for each_minion in ret:
if ret[each_minion] == "Minion did not return":
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of Minions Targeted: {0}'.format(return_counter +
not_return_counter))
print_cli('# of Minions Returned: {0}'.format(return_counter))
print_cli(
'# of Minions Did Not Return: {0}'.format(not_return_counter))
if self.options.verbose:
print_cli('Minions Which Did Not Return: {0}'.format(
" ".join(not_return_minions)))
print_cli('-------------------------------------------')
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
not_response_minions = []
not_connected_minions = []
failed_minions = []
for each_minion in ret:
minion_ret = ret[each_minion].get('ret')
if (
isinstance(minion_ret, string_types)
and minion_ret.startswith("Minion did not return")
):
if "Not connected" in minion_ret:
not_connected_minions.append(each_minion)
elif "No response" in minion_ret:
not_response_minions.append(each_minion)
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
if salt.utils.job.get_retcode(ret[each_minion]):
failed_minions.append(each_minion)
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of minions targeted: {0}'.format(return_counter + not_return_counter))
print_cli('# of minions returned: {0}'.format(return_counter))
print_cli('# of minions that did not return: {0}'.format(not_return_counter))
print_cli('# of minions with errors: {0}'.format(len(failed_minions)))
if self.options.verbose:
if not_connected_minions:
print_cli('Minions not connected: {0}'.format(" ".join(not_connected_minions)))
if not_response_minions:
print_cli('Minions not responding: {0}'.format(" ".join(not_response_minions)))
if failed_minions:
print_cli('Minions with failures: {0}'.format(" ".join(failed_minions)))
print_cli('-------------------------------------------')
def run(self):
'''
Execute the salt command line
'''
import salt.auth
import salt.client
self.parse_args()
# Setup file logging!
self.setup_logfile_logger()
verify_log(self.config)
try:
# We don't need to bail on config file permission errors
# if the CLI
# process is run with the -a flag
skip_perm_errors = self.options.eauth != ''
local = salt.client.get_local_client(
self.get_config_file_path(),
skip_perm_errors=skip_perm_errors)
except SaltClientError as exc:
self.exit(2, '{0}\n'.format(exc))
return
if self.options.batch or self.options.static:
self._run_batch()
else:
if self.options.timeout <= 0:
self.options.timeout = local.opts['timeout']
kwargs = {
'tgt': self.config['tgt'],
'fun': self.config['fun'],
'arg': self.config['arg'],
'timeout': self.options.timeout,
'show_timeout': self.options.show_timeout,
'show_jid': self.options.show_jid}
if 'token' in self.config:
try:
with salt.utils.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_:
kwargs['key'] = fp_.readline()
except IOError:
kwargs['token'] = self.config['token']
kwargs['delimiter'] = self.options.delimiter
if self.selected_target_option:
kwargs['expr_form'] = self.selected_target_option
else:
kwargs['expr_form'] = 'glob'
if getattr(self.options, 'return'):
kwargs['ret'] = getattr(self.options, 'return')
if getattr(self.options, 'return_config'):
kwargs['ret_config'] = getattr(self.options, 'return_config')
if getattr(self.options, 'return_kwargs'):
kwargs['ret_kwargs'] = yamlify_arg(
getattr(self.options, 'return_kwargs'))
if getattr(self.options, 'module_executors'):
kwargs['module_executors'] = yamlify_arg(getattr(self.options, 'module_executors'))
if getattr(self.options, 'metadata'):
kwargs['metadata'] = yamlify_arg(
getattr(self.options, 'metadata'))
# If using eauth and a token hasn't already been loaded into
# kwargs, prompt the user to enter auth credentials
if 'token' not in kwargs and 'key' not in kwargs and self.options.eauth:
resolver = salt.auth.Resolver(self.config)
res = resolver.cli(self.options.eauth)
if self.options.mktoken and res:
tok = resolver.token_cli(
self.options.eauth,
res
)
if tok:
kwargs['token'] = tok.get('token', '')
if not res:
sys.stderr.write('ERROR: Authentication failed\n')
sys.exit(2)
kwargs.update(res)
kwargs['eauth'] = self.options.eauth
if self.config['async']:
jid = local.cmd_async(**kwargs)
print_cli('Executed command with job ID: {0}'.format(jid))
return
retcodes = []
try:
# local will be None when there was an error
errors = []
if local:
if self.options.subset:
cmd_func = local.cmd_subset
kwargs['sub'] = self.options.subset
kwargs['cli'] = True
else:
cmd_func = local.cmd_cli
if self.options.progress:
kwargs['progress'] = True
self.config['progress'] = True
ret = {}
for progress in cmd_func(**kwargs):
out = 'progress'
try:
self._progress_ret(progress, out)
except salt.exceptions.LoaderError as exc:
raise salt.exceptions.SaltSystemExit(exc)
if 'return_count' not in progress:
ret.update(progress)
self._progress_end(out)
self._print_returns_summary(ret)
elif self.config['fun'] == 'sys.doc':
ret = {}
out = ''
for full_ret in local.cmd_cli(**kwargs):
ret_, out, retcode = self._format_ret(full_ret)
ret.update(ret_)
self._output_ret(ret, out)
else:
if self.options.verbose:
kwargs['verbose'] = True
ret = {}
for full_ret in cmd_func(**kwargs):
try:
ret_, out, retcode = self._format_ret(full_ret)
retcodes.append(retcode)
self._output_ret(ret_, out)
ret.update(full_ret)
except KeyError:
errors.append(full_ret)
# Returns summary
if self.config['cli_summary'] is True:
if self.config['fun'] != 'sys.doc':
if self.options.output is None:
self._print_returns_summary(ret)
self._print_errors_summary(errors)
# NOTE: Return code is set here based on if all minions
# returned 'ok' with a retcode of 0.
# This is the final point before the 'salt' cmd returns,
# which is why we set the retcode here.
if retcodes.count(0) < len(retcodes):
sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
sys.exit(11)
except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc:
ret = str(exc)
out = ''
self._output_ret(ret, out)
def _print_returns_summary(self, ret):
"""
Display returns summary
"""
return_counter = 0
not_return_counter = 0
not_return_minions = []
for each_minion in ret:
if ret[each_minion] == "Minion did not return":
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli("\n")
print_cli("-------------------------------------------")
print_cli("Summary")
print_cli("-------------------------------------------")
print_cli("# of Minions Targeted: {0}".format(return_counter + not_return_counter))
print_cli("# of Minions Returned: {0}".format(return_counter))
print_cli("# of Minions Did Not Return: {0}".format(not_return_counter))
if self.options.verbose:
print_cli("Minions Which Did Not Return: {0}".format(" ".join(not_return_minions)))
print_cli("-------------------------------------------")
def _print_returns_summary(self, ret):
'''
Display returns summary
'''
return_counter = 0
not_return_counter = 0
not_return_minions = []
not_response_minions = []
not_connected_minions = []
for each_minion in ret:
minion_ret = ret[each_minion]
if (isinstance(minion_ret, string_types)
and minion_ret.startswith("Minion did not return")):
if "Not connected" in ret[each_minion]:
not_connected_minions.append(each_minion)
elif "No response" in ret[each_minion]:
not_response_minions.append(each_minion)
not_return_counter += 1
not_return_minions.append(each_minion)
else:
return_counter += 1
print_cli('\n')
print_cli('-------------------------------------------')
print_cli('Summary')
print_cli('-------------------------------------------')
print_cli('# of minions targeted: {0}'.format(return_counter +
not_return_counter))
print_cli('# of minions returned: {0}'.format(return_counter))
print_cli(
'# of minions that did not return: {0}'.format(not_return_counter))
if self.options.verbose:
if not_connected_minions:
print_cli('Minions not connected: {0}'.format(
" ".join(not_connected_minions)))
if not_response_minions:
print_cli('Minions not responding: {0}'.format(
" ".join(not_response_minions)))
print_cli('-------------------------------------------')
def batchRun(wrapMesage, selfIp, clientPub, redisChannel):
# NOTE: batch running mode
# handle special syndic
if selfIp in comeSubList:
comeSubList.remove(selfIp)
syndic_count = len(comeSubList)
resultCount = 0
pingCount = 0
executeStart = time.time()
normalDone = False
# NOTE: must publish cmd after registered the redis listen
# else we will miss ping message
# tmpKwargs1 = wrapMesage['kwargs']
# batch_running = set(tmpKwargs1['tgt'])
#print('publish wrapMesage: %s' % wrapMesage)
clientPub.publishToSyndicSub(
salt.newrun.json.dumps(wrapMesage))
from salt.newrun import (json, byteify, MessageType)
for message in redisChannel.listen():
try:
messageJson = byteify(message)
if messageJson['type'] == 'message':
resultMessage = messageJson['data']
try:
callResult = json.loads(resultMessage,
encoding='utf-8')
callResult = byteify(callResult)
if isinstance(callResult, dict):
if 'type' in callResult:
messageType = callResult['type']
messageIp = callResult['sub_ip']
if messageType == MessageType.PING and messageIp in comeSubList:
resultPingSet.append(messageIp)
elif messageType == MessageType.WORK or messageType == MessageType.INTERRUPT:
resultExeSet.append(messageIp)
else:
main_log.info(
'invalid callresult: %s' %
callResult)
else:
# filter no return received of sub node
retJsonObj = callResult['ret_']
if retJsonObj:
# reset start time
executeStart = time.time()
for k, v in retJsonObj.items():
# reset running and wait node
batch_running.discard(k)
if callResult[
'out'] == 'no_return':
if '[No response]' in json.dumps(
retJsonObj):
noResponseRet.append(
callResult)
else:
noConnectRet.append(
callResult)
else:
# put successed ip to tmp set
for k, v in retJsonObj.items():
sucset.add(k)
# NOTE: debug
if k in debugSet:
repeatet.add(
json.dumps(
retJsonObj))
else:
debugSet.add(k)
if callResult['retcode'] == 0:
isnil = False
for k in retJsonObj.keys():
v = retJsonObj[k]
if v == "":
isnil = True
break
if isnil:
emptyRet.append(
callResult)
else:
global normalsize
normalsize += 1
self._output_ret(
callResult['ret_'],
callResult['out'])
else:
emptyRet.append(callResult)
else:
# TODO handle other messages?
pass
except:
resultCount += 1
print_cli(traceback.format_exc())
pass
pingCount = len(resultPingSet)
resultCount = len(resultExeSet)
#from collections import Counter
#main_log.info("%s, %s, %s, %s" % (pingCount, resultCount, Counter(resultPingSet), Counter(resultExeSet)))
# if len(batch_init) <= 0:
# break
if len(batch_running) == 0:
break
if pingCount != resultCount:
if (time.time() - executeStart) > sub_timeout:
# main_log.info("---T0 stop")
break
except:
main_log.info(traceback.format_exc())
pass
