def _thread_multi_return(class_, minion_instance, opts, data):
'''
This method should be used as a threading target, start the actual
minion side execution.
'''
# this seems awkward at first, but it's a workaround for Windows
# multiprocessing communication.
if not minion_instance:
minion_instance = class_(opts)
ret = {
'return': {},
'success': {},
}
for ind in range(0, len(data['fun'])):
for index in range(0, len(data['arg'][ind])):
try:
arg = eval(data['arg'][ind][index])
if isinstance(arg, bool):
data['arg'][ind][index] = str(data['arg'][ind][index])
elif isinstance(arg, (dict, int, list, string_types)):
data['arg'][ind][index] = arg
else:
data['arg'][ind][index] = str(data['arg'][ind][index])
except Exception:
pass
ret['success'][data['fun'][ind]] = False
try:
func = minion_instance.functions[data['fun'][ind]]
args, kwargs = detect_kwargs(func, data['arg'][ind], data)
ret['return'][data['fun'][ind]] = func(*args, **kwargs)
ret['success'][data['fun'][ind]] = True
except Exception as exc:
trb = traceback.format_exc()
log.warning(
'The minion function caused an exception: {0}'.format(
exc
)
)
ret['return'][data['fun'][ind]] = trb
ret['jid'] = data['jid']
minion_instance._return_pub(ret)
if data['ret']:
for returner in set(data['ret'].split(',')):
ret['id'] = opts['id']
try:
minion_instance.returners['{0}.returner'.format(
returner
)](ret)
except Exception as exc:
log.error(
'The return failed for job {0} {1}'.format(
data['jid'],
exc
)
)
def _thread_multi_return(class_, minion_instance, opts, data):
'''
This method should be used as a threading target, start the actual
minion side execution.
'''
# this seems awkward at first, but it's a workaround for Windows
# multiprocessing communication.
if not minion_instance:
minion_instance = class_(opts)
ret = {
'return': {},
'success': {},
}
for ind in range(0, len(data['fun'])):
for index in range(0, len(data['arg'][ind])):
try:
arg = eval(data['arg'][ind][index])
if isinstance(arg, bool):
data['arg'][ind][index] = str(data['arg'][ind][index])
elif isinstance(arg, (dict, int, list, string_types)):
data['arg'][ind][index] = arg
else:
data['arg'][ind][index] = str(data['arg'][ind][index])
except Exception:
pass
ret['success'][data['fun'][ind]] = False
try:
func = minion_instance.functions[data['fun'][ind]]
args, kwargs = detect_kwargs(func, data['arg'][ind], data)
ret['return'][data['fun'][ind]] = func(*args, **kwargs)
ret['success'][data['fun'][ind]] = True
except Exception as exc:
trb = traceback.format_exc()
log.warning(
'The minion function caused an exception: {0}'.format(exc))
ret['return'][data['fun'][ind]] = trb
ret['jid'] = data['jid']
minion_instance._return_pub(ret)
if data['ret']:
for returner in set(data['ret'].split(',')):
ret['id'] = opts['id']
try:
minion_instance.returners['{0}.returner'.format(returner)](
ret)
except Exception as exc:
log.error('The return failed for job {0} {1}'.format(
data['jid'], exc))
def detect_kwargs(func, args, data=None):
'''
Detect the args and kwargs that need to be passed to a function call
'''
spec_args, _, has_kwargs, defaults = salt.state._getargs(func)
defaults = [] if defaults is None else defaults
starti = len(spec_args) - len(defaults)
kwarg_spec = set()
for ind in range(len(defaults)):
kwarg_spec.add(spec_args[starti])
starti += 1
_args = []
kwargs = {}
for arg in args:
if isinstance(arg, string_types):
if '=' in arg:
comps = arg.split('=')
if has_kwargs:
kwargs[comps[0]] = '='.join(comps[1:])
continue
if comps[0] in kwarg_spec:
kwargs[comps[0]] = '='.join(comps[1:])
continue
_args.append(arg)
if has_kwargs and isinstance(data, dict):
# this function accepts kwargs, pack in the publish data
for key, val in data.items():
kwargs['__pub_{0}'.format(key)] = val
return _args, kwargs
def detect_kwargs(func, args, data=None):
'''
Detect the args and kwargs that need to be passed to a function call
'''
spec_args, _, has_kwargs, defaults = salt.state._getargs(func)
defaults = [] if defaults is None else defaults
starti = len(spec_args) - len(defaults)
kwarg_spec = set()
for ind in range(len(defaults)):
kwarg_spec.add(spec_args[starti])
starti += 1
_args = []
kwargs = {}
for arg in args:
if isinstance(arg, string_types):
if '=' in arg:
comps = arg.split('=')
if has_kwargs:
kwargs[comps[0]] = '='.join(comps[1:])
continue
if comps[0] in kwarg_spec:
kwargs[comps[0]] = '='.join(comps[1:])
continue
_args.append(arg)
if has_kwargs and isinstance(data, dict):
# this function accepts kwargs, pack in the publish data
for key, val in data.items():
kwargs['__pub_{0}'.format(key)] = val
return _args, kwargs
def _traverse_dict(self, data, target, delim=':'):
'''
Traverse a dict using a colon-delimited (or otherwise delimited, using
the "delim" param) target string. The target 'foo:bar:baz' will return
data['foo']['bar']['baz'] if this value exists, and will otherwise
return an empty dict.
'''
target = target.split(delim)
ptr = data
try:
for index in range(len(target)):
ptr = ptr.get(target[index], {})
except (SyntaxError, AttributeError):
# Encountered a non-dict value in the middle of traversing
return {}
return ptr
def _traverse_dict(self, data, target, delim=':'):
'''
Traverse a dict using a colon-delimited (or otherwise delimited, using
the "delim" param) target string. The target 'foo:bar:baz' will return
data['foo']['bar']['baz'] if this value exists, and will otherwise
return an empty dict.
'''
target = target.split(delim)
ptr = data
try:
for index in range(len(target)):
ptr = ptr.get(target[index], {})
except (SyntaxError, AttributeError):
# Encountered a non-dict value in the middle of traversing
return {}
return ptr
def pub(self,
tgt,
fun,
arg=(),
expr_form='glob',
ret='',
jid='',
timeout=5):
'''
Take the required arguments and publish the given command.
Arguments:
tgt:
The tgt is a regex or a glob used to match up the ids on
the minions. Salt works by always publishing every command
to all of the minions and then the minions determine if
the command is for them based on the tgt value.
fun:
The function name to be called on the remote host(s), this
must be a string in the format "<modulename>.<function name>"
arg:
The arg option needs to be a tuple of arguments to pass
to the calling function, if left blank
Returns:
jid:
A string, as returned by the publisher, which is the job
id, this will inform the client where to get the job results
minions:
A set, the targets that the tgt passed should match.
'''
if expr_form == 'nodegroup':
if tgt not in self.opts['nodegroups']:
conf_file = self.opts.get('conf_file',
'the master config file')
err = 'Node group {0} unavailable in {1}'.format(
tgt, conf_file)
raise SaltInvocationError(err)
tgt = self.opts['nodegroups'][tgt]
expr_form = 'compound'
# Convert a range expression to a list of nodes and change expression
# form to list
if expr_form == 'range' and RANGE:
tgt = self._convert_range_to_list(tgt)
expr_form = 'list'
# Run a check_minions, if no minions match return False
# format the payload - make a function that does this in the payload
# module
# make the zmq client
# connect to the req server
# send!
# return what we get back
minions = self.check_minions(tgt, expr_form)
if self.opts['order_masters']:
# If we're a master of masters, ignore the check_minion and
# set the minions to the target. This speeds up wait time
# for lists and ranges and makes regex and other expression
# forms possible
minions = tgt
elif not minions:
return {'jid': '', 'minions': minions}
# Generate the standard keyword args to feed to format_payload
payload_kwargs = {
'cmd': 'publish',
'tgt': tgt,
'fun': fun,
'arg': arg,
'key': self.key,
'tgt_type': expr_form,
'ret': ret,
'jid': jid
}
# If we have a salt user, add it to the payload
if self.salt_user:
payload_kwargs['user'] = self.salt_user
# If we're a syndication master, pass the timeout
if self.opts['order_masters']:
payload_kwargs['to'] = timeout
package = salt.payload.format_payload('clear', **payload_kwargs)
# Prep zmq
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.linger = 0
socket.connect('tcp://{0[interface]}:{0[ret_port]}'.format(self.opts))
socket.send(package)
payload = None
for ind in range(100):
try:
payload = self.serial.loads(socket.recv(zmq.NOBLOCK))
break
except zmq.core.error.ZMQError:
time.sleep(0.01)
if not payload:
return {'jid': '0', 'minions': []}
return {'jid': payload['load']['jid'], 'minions': minions}
def pub(self, tgt, fun, arg=(), expr_form='glob',
ret='', jid='', timeout=5):
'''
Take the required arguments and publish the given command.
Arguments:
tgt:
The tgt is a regex or a glob used to match up the ids on
the minions. Salt works by always publishing every command
to all of the minions and then the minions determine if
the command is for them based on the tgt value.
fun:
The function name to be called on the remote host(s), this
must be a string in the format "<modulename>.<function name>"
arg:
The arg option needs to be a tuple of arguments to pass
to the calling function, if left blank
Returns:
jid:
A string, as returned by the publisher, which is the job
id, this will inform the client where to get the job results
minions:
A set, the targets that the tgt passed should match.
'''
if expr_form == 'nodegroup':
if tgt not in self.opts['nodegroups']:
conf_file = self.opts.get('conf_file', 'the master config file')
err = 'Node group {0} unavailable in {1}'.format(tgt, conf_file)
raise SaltInvocationError(err)
tgt = self.opts['nodegroups'][tgt]
expr_form = 'compound'
# Convert a range expression to a list of nodes and change expression
# form to list
if expr_form == 'range' and RANGE:
tgt = self._convert_range_to_list(tgt)
expr_form = 'list'
# Run a check_minions, if no minions match return False
# format the payload - make a function that does this in the payload
# module
# make the zmq client
# connect to the req server
# send!
# return what we get back
minions = self.check_minions(tgt, expr_form)
if not minions:
return {'jid': '',
'minions': minions}
# Generate the standard keyword args to feed to format_payload
payload_kwargs = { 'cmd': 'publish',
'tgt': tgt,
'fun': fun,
'arg': arg,
'key': self.key,
'tgt_type': expr_form,
'ret': ret,
'jid': jid }
# If we have a salt user, add it to the payload
if self.salt_user:
payload_kwargs['user'] = self.salt_user
# If we're a syndication master, pass the timeout
if self.opts['order_masters']:
payload_kwargs['to'] = timeout
package = salt.payload.format_payload( 'clear', **payload_kwargs)
# Prep zmq
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.linger = 0
socket.connect(
'tcp://{0[interface]}:{0[ret_port]}'.format(
self.opts
)
)
socket.send(package)
payload = None
for ind in range(100):
try:
payload = self.serial.loads(
socket.recv(
zmq.NOBLOCK
)
)
break
except zmq.core.error.ZMQError:
time.sleep(0.01)
if not payload:
return {'jid': '0', 'minions': []}
return {'jid': payload['load']['jid'],
'minions': minions}
def _thread_return(class_, minion_instance, opts, data):
'''
This method should be used as a threading target, start the actual
minion side execution.
'''
# this seems awkward at first, but it's a workaround for Windows
# multiprocessing communication.
if not minion_instance:
minion_instance = class_(opts)
if opts['multiprocessing']:
fn_ = os.path.join(minion_instance.proc_dir, data['jid'])
sdata = {'pid': os.getpid()}
sdata.update(data)
with salt.utils.fopen(fn_, 'w+') as fp_:
fp_.write(minion_instance.serial.dumps(sdata))
ret = {}
for ind in range(0, len(data['arg'])):
try:
arg = eval(data['arg'][ind])
if isinstance(arg, bool):
data['arg'][ind] = str(data['arg'][ind])
elif isinstance(arg, (dict, int, list, string_types)):
data['arg'][ind] = arg
else:
data['arg'][ind] = str(data['arg'][ind])
except Exception:
pass
function_name = data['fun']
if function_name in minion_instance.functions:
ret['success'] = False
try:
func = minion_instance.functions[data['fun']]
args, kwargs = detect_kwargs(func, data['arg'], data)
ret['return'] = func(*args, **kwargs)
ret['success'] = True
except CommandNotFoundError as exc:
msg = 'Command required for \'{0}\' not found: {1}'
log.debug(msg.format(function_name, str(exc)))
ret['return'] = msg.format(function_name, str(exc))
except CommandExecutionError as exc:
msg = 'A command in {0} had a problem: {1}'
log.error(msg.format(function_name, str(exc)))
ret['return'] = 'ERROR: {0}'.format(str(exc))
except SaltInvocationError as exc:
msg = 'Problem executing "{0}": {1}'
log.error(msg.format(function_name, str(exc)))
ret['return'] = 'ERROR executing {0}: {1}'.format(
function_name, exc)
except Exception:
trb = traceback.format_exc()
msg = 'The minion function caused an exception: {0}'
log.warning(msg.format(trb))
ret['return'] = trb
else:
ret['return'] = '"{0}" is not available.'.format(function_name)
ret['jid'] = data['jid']
ret['fun'] = data['fun']
minion_instance._return_pub(ret)
if data['ret']:
for returner in set(data['ret'].split(',')):
ret['id'] = opts['id']
try:
minion_instance.returners['{0}.returner'.format(returner)](
ret)
except Exception as exc:
log.error('The return failed for job {0} {1}'.format(
data['jid'], exc))
options.contents = True
# binger is now a dictionary with each item being a key of a unique md5sum. perform a CacheExtractor search and compare with its friends.
# once we reach 1 item, it doesn't make sense to compare it against itself, so bail.
# If we start off with only 1 item, then everything is identical.
if len(binger) == 1:
first_key, (first_key_count, first_key_md5sum, first_key_hostlist) = binger.popitem()
print "All machines with the key " + first_key + " had the same md5sum " + first_key_md5sum
if options.hostlist:
for host in first_key_hostlist:
print host
sys.exit(0)
# Calculate how many actual comparisons will be executed.
total_comparisons = 0
for x in range(0, (len(binger) + 1)):
total_comparisons = total_comparisons + (x -1)
total_comparisons += 1
attention_whore = "***********************************************************************************"
print attention_whore.center(150)
print attention_whore.center(150)
print attention_whore.center(150)
print "Comparison Summary: " + str(len(binger)) + " unique objects exist in the cache."
print "A total of " + str(total_comparisons) + " comparisons will be executed below"
for key in binger.iterkeys():
(key_count, key_md5sum, key_hostlist) = binger[key]
print "Unique sample object " + key + " with md5sum " + key_md5sum + " found " + str(key_count) + " times."
print attention_whore.center(150)
print attention_whore.center(150)
print attention_whore.center(150)
# binger is now a dictionary with each item being a key of a unique md5sum. perform a sysopsapi.cache_extractor search and compare with its friends.
# once we reach 1 item, it doesn't make sense to compare it against itself, so bail.
# If we start off with only 1 item, then everything is identical.
if len(binger) == 1:
first_key, (first_key_count, first_key_md5sum,
first_key_hostlist) = binger.popitem()
print "All machines with the key " + first_key + " had the same md5sum " + first_key_md5sum
if options.hostlist:
for host in first_key_hostlist:
print host
sys.exit(0)
# Calculate how many actual comparisons will be executed.
total_comparisons = 0
for x in range(0, (len(binger) + 1)):
total_comparisons = total_comparisons + (x - 1)
total_comparisons += 1
attention_whore = "***********************************************************************************"
print attention_whore.center(150)
print attention_whore.center(150)
print attention_whore.center(150)
print "Comparison Summary: " + str(
len(binger)) + " unique objects exist in the cache."
print "A total of " + str(
total_comparisons) + " comparisons will be executed below"
for key in binger.iterkeys():
(key_count, key_md5sum, key_hostlist) = binger[key]
print "Unique sample object " + key + " with md5sum " + key_md5sum + " found " + str(
key_count) + " times."
def _thread_return(class_, minion_instance, opts, data):
'''
This method should be used as a threading target, start the actual
minion side execution.
'''
# this seems awkward at first, but it's a workaround for Windows
# multiprocessing communication.
if not minion_instance:
minion_instance = class_(opts)
if opts['multiprocessing']:
fn_ = os.path.join(minion_instance.proc_dir, data['jid'])
sdata = {'pid': os.getpid()}
sdata.update(data)
with salt.utils.fopen(fn_, 'w+') as fp_:
fp_.write(minion_instance.serial.dumps(sdata))
ret = {}
for ind in range(0, len(data['arg'])):
try:
arg = eval(data['arg'][ind])
if isinstance(arg, bool):
data['arg'][ind] = str(data['arg'][ind])
elif isinstance(arg, (dict, int, list, string_types)):
data['arg'][ind] = arg
else:
data['arg'][ind] = str(data['arg'][ind])
except Exception:
pass
function_name = data['fun']
if function_name in minion_instance.functions:
ret['success'] = False
try:
func = minion_instance.functions[data['fun']]
args, kwargs = detect_kwargs(func, data['arg'], data)
ret['return'] = func(*args, **kwargs)
ret['success'] = True
except CommandNotFoundError as exc:
msg = 'Command required for \'{0}\' not found: {1}'
log.debug(msg.format(function_name, str(exc)))
ret['return'] = msg.format(function_name, str(exc))
except CommandExecutionError as exc:
msg = 'A command in {0} had a problem: {1}'
log.error(msg.format(function_name, str(exc)))
ret['return'] = 'ERROR: {0}'.format(str(exc))
except SaltInvocationError as exc:
msg = 'Problem executing "{0}": {1}'
log.error(msg.format(function_name, str(exc)))
ret['return'] = 'ERROR executing {0}: {1}'.format(
function_name, exc
)
except Exception:
trb = traceback.format_exc()
msg = 'The minion function caused an exception: {0}'
log.warning(msg.format(trb))
ret['return'] = trb
else:
ret['return'] = '"{0}" is not available.'.format(function_name)
ret['jid'] = data['jid']
ret['fun'] = data['fun']
minion_instance._return_pub(ret)
if data['ret']:
for returner in set(data['ret'].split(',')):
ret['id'] = opts['id']
try:
minion_instance.returners['{0}.returner'.format(
returner
)](ret)
except Exception as exc:
log.error(
'The return failed for job {0} {1}'.format(
data['jid'],
exc
)
)
