Ejemplo n.º 1
0
    def _thread_multi_return(class_, minion_instance, opts, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        # this seems awkward at first, but it's a workaround for Windows
        # multiprocessing communication.
        if not minion_instance:
            minion_instance = class_(opts)
        ret = {
            'return': {},
            'success': {},
        }
        for ind in range(0, len(data['fun'])):
            for index in range(0, len(data['arg'][ind])):
                try:
                    arg = eval(data['arg'][ind][index])
                    if isinstance(arg, bool):
                        data['arg'][ind][index] = str(data['arg'][ind][index])
                    elif isinstance(arg, (dict, int, list, string_types)):
                        data['arg'][ind][index] = arg
                    else:
                        data['arg'][ind][index] = str(data['arg'][ind][index])
                except Exception:
                    pass

            ret['success'][data['fun'][ind]] = False
            try:
                func = minion_instance.functions[data['fun'][ind]]
                args, kwargs = detect_kwargs(func, data['arg'][ind], data)
                ret['return'][data['fun'][ind]] = func(*args, **kwargs)
                ret['success'][data['fun'][ind]] = True
            except Exception as exc:
                trb = traceback.format_exc()
                log.warning(
                    'The minion function caused an exception: {0}'.format(
                    exc
                    )
                )
                ret['return'][data['fun'][ind]] = trb
            ret['jid'] = data['jid']
        minion_instance._return_pub(ret)
        if data['ret']:
            for returner in set(data['ret'].split(',')):
                ret['id'] = opts['id']
                try:
                    minion_instance.returners['{0}.returner'.format(
                        returner
                    )](ret)
                except Exception as exc:
                    log.error(
                        'The return failed for job {0} {1}'.format(
                        data['jid'],
                        exc
                        )
                    )
Ejemplo n.º 2
0
    def _thread_multi_return(class_, minion_instance, opts, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        # this seems awkward at first, but it's a workaround for Windows
        # multiprocessing communication.
        if not minion_instance:
            minion_instance = class_(opts)
        ret = {
            'return': {},
            'success': {},
        }
        for ind in range(0, len(data['fun'])):
            for index in range(0, len(data['arg'][ind])):
                try:
                    arg = eval(data['arg'][ind][index])
                    if isinstance(arg, bool):
                        data['arg'][ind][index] = str(data['arg'][ind][index])
                    elif isinstance(arg, (dict, int, list, string_types)):
                        data['arg'][ind][index] = arg
                    else:
                        data['arg'][ind][index] = str(data['arg'][ind][index])
                except Exception:
                    pass

            ret['success'][data['fun'][ind]] = False
            try:
                func = minion_instance.functions[data['fun'][ind]]
                args, kwargs = detect_kwargs(func, data['arg'][ind], data)
                ret['return'][data['fun'][ind]] = func(*args, **kwargs)
                ret['success'][data['fun'][ind]] = True
            except Exception as exc:
                trb = traceback.format_exc()
                log.warning(
                    'The minion function caused an exception: {0}'.format(exc))
                ret['return'][data['fun'][ind]] = trb
            ret['jid'] = data['jid']
        minion_instance._return_pub(ret)
        if data['ret']:
            for returner in set(data['ret'].split(',')):
                ret['id'] = opts['id']
                try:
                    minion_instance.returners['{0}.returner'.format(returner)](
                        ret)
                except Exception as exc:
                    log.error('The return failed for job {0} {1}'.format(
                        data['jid'], exc))
Ejemplo n.º 3
0
def detect_kwargs(func, args, data=None):
    '''
    Detect the args and kwargs that need to be passed to a function call
    '''
    spec_args, _, has_kwargs, defaults = salt.state._getargs(func)
    defaults = [] if defaults is None else defaults
    starti = len(spec_args) - len(defaults)
    kwarg_spec = set()
    for ind in range(len(defaults)):
        kwarg_spec.add(spec_args[starti])
        starti += 1
    _args = []
    kwargs = {}
    for arg in args:
        if isinstance(arg, string_types):
            if '=' in arg:
                comps = arg.split('=')
                if has_kwargs:
                    kwargs[comps[0]] = '='.join(comps[1:])
                    continue
                if comps[0] in kwarg_spec:
                    kwargs[comps[0]] = '='.join(comps[1:])
                    continue
        _args.append(arg)
    if has_kwargs and isinstance(data, dict):
        # this function accepts kwargs, pack in the publish data
        for key, val in data.items():
            kwargs['__pub_{0}'.format(key)] = val
    return _args, kwargs
Ejemplo n.º 4
0
def detect_kwargs(func, args, data=None):
    '''
    Detect the args and kwargs that need to be passed to a function call
    '''
    spec_args, _, has_kwargs, defaults = salt.state._getargs(func)
    defaults = [] if defaults is None else defaults
    starti = len(spec_args) - len(defaults)
    kwarg_spec = set()
    for ind in range(len(defaults)):
        kwarg_spec.add(spec_args[starti])
        starti += 1
    _args = []
    kwargs = {}
    for arg in args:
        if isinstance(arg, string_types):
            if '=' in arg:
                comps = arg.split('=')
                if has_kwargs:
                    kwargs[comps[0]] = '='.join(comps[1:])
                    continue
                if comps[0] in kwarg_spec:
                    kwargs[comps[0]] = '='.join(comps[1:])
                    continue
        _args.append(arg)
    if has_kwargs and isinstance(data, dict):
        # this function accepts kwargs, pack in the publish data
        for key, val in data.items():
            kwargs['__pub_{0}'.format(key)] = val
    return _args, kwargs
Ejemplo n.º 5
0
 def _traverse_dict(self, data, target, delim=':'):
     '''
     Traverse a dict using a colon-delimited (or otherwise delimited, using
     the "delim" param) target string. The target 'foo:bar:baz' will return
     data['foo']['bar']['baz'] if this value exists, and will otherwise
     return an empty dict.
     '''
     target = target.split(delim)
     ptr = data
     try:
         for index in range(len(target)):
             ptr = ptr.get(target[index], {})
     except (SyntaxError, AttributeError):
         # Encountered a non-dict value in the middle of traversing
         return {}
     return ptr
Ejemplo n.º 6
0
 def _traverse_dict(self, data, target, delim=':'):
     '''
     Traverse a dict using a colon-delimited (or otherwise delimited, using
     the "delim" param) target string. The target 'foo:bar:baz' will return
     data['foo']['bar']['baz'] if this value exists, and will otherwise
     return an empty dict.
     '''
     target = target.split(delim)
     ptr = data
     try:
         for index in range(len(target)):
             ptr = ptr.get(target[index], {})
     except (SyntaxError, AttributeError):
         # Encountered a non-dict value in the middle of traversing
         return {}
     return ptr
Ejemplo n.º 7
0
    def pub(self,
            tgt,
            fun,
            arg=(),
            expr_form='glob',
            ret='',
            jid='',
            timeout=5):
        '''
        Take the required arguments and publish the given command.
        Arguments:
            tgt:
                The tgt is a regex or a glob used to match up the ids on
                the minions. Salt works by always publishing every command
                to all of the minions and then the minions determine if
                the command is for them based on the tgt value.
            fun:
                The function name to be called on the remote host(s), this
                must be a string in the format "<modulename>.<function name>"
            arg:
                The arg option needs to be a tuple of arguments to pass
                to the calling function, if left blank
        Returns:
            jid:
                A string, as returned by the publisher, which is the job
                id, this will inform the client where to get the job results
            minions:
                A set, the targets that the tgt passed should match.
        '''
        if expr_form == 'nodegroup':
            if tgt not in self.opts['nodegroups']:
                conf_file = self.opts.get('conf_file',
                                          'the master config file')
                err = 'Node group {0} unavailable in {1}'.format(
                    tgt, conf_file)
                raise SaltInvocationError(err)
            tgt = self.opts['nodegroups'][tgt]
            expr_form = 'compound'

        # Convert a range expression to a list of nodes and change expression
        # form to list
        if expr_form == 'range' and RANGE:
            tgt = self._convert_range_to_list(tgt)
            expr_form = 'list'

        # Run a check_minions, if no minions match return False
        # format the payload - make a function that does this in the payload
        #   module
        # make the zmq client
        # connect to the req server
        # send!
        # return what we get back
        minions = self.check_minions(tgt, expr_form)

        if self.opts['order_masters']:
            # If we're a master of masters, ignore the check_minion and
            # set the minions to the target.  This speeds up wait time
            # for lists and ranges and makes regex and other expression
            # forms possible
            minions = tgt
        elif not minions:
            return {'jid': '', 'minions': minions}

        # Generate the standard keyword args to feed to format_payload
        payload_kwargs = {
            'cmd': 'publish',
            'tgt': tgt,
            'fun': fun,
            'arg': arg,
            'key': self.key,
            'tgt_type': expr_form,
            'ret': ret,
            'jid': jid
        }

        # If we have a salt user, add it to the payload
        if self.salt_user:
            payload_kwargs['user'] = self.salt_user

        # If we're a syndication master, pass the timeout
        if self.opts['order_masters']:
            payload_kwargs['to'] = timeout

        package = salt.payload.format_payload('clear', **payload_kwargs)

        # Prep zmq
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.linger = 0
        socket.connect('tcp://{0[interface]}:{0[ret_port]}'.format(self.opts))
        socket.send(package)
        payload = None
        for ind in range(100):
            try:
                payload = self.serial.loads(socket.recv(zmq.NOBLOCK))
                break
            except zmq.core.error.ZMQError:
                time.sleep(0.01)
        if not payload:
            return {'jid': '0', 'minions': []}
        return {'jid': payload['load']['jid'], 'minions': minions}
Ejemplo n.º 8
0
Archivo: client.py Proyecto: cmek/salt
    def pub(self, tgt, fun, arg=(), expr_form='glob',
            ret='', jid='', timeout=5):
        '''
        Take the required arguments and publish the given command.
        Arguments:
            tgt:
                The tgt is a regex or a glob used to match up the ids on
                the minions. Salt works by always publishing every command
                to all of the minions and then the minions determine if
                the command is for them based on the tgt value.
            fun:
                The function name to be called on the remote host(s), this
                must be a string in the format "<modulename>.<function name>"
            arg:
                The arg option needs to be a tuple of arguments to pass
                to the calling function, if left blank
        Returns:
            jid:
                A string, as returned by the publisher, which is the job
                id, this will inform the client where to get the job results
            minions:
                A set, the targets that the tgt passed should match.
        '''
        if expr_form == 'nodegroup':
            if tgt not in self.opts['nodegroups']:
                conf_file = self.opts.get('conf_file', 'the master config file')
                err = 'Node group {0} unavailable in {1}'.format(tgt, conf_file)
                raise SaltInvocationError(err)
            tgt = self.opts['nodegroups'][tgt]
            expr_form = 'compound'

        # Convert a range expression to a list of nodes and change expression
        # form to list
        if expr_form == 'range' and RANGE:
          tgt = self._convert_range_to_list(tgt)
          expr_form = 'list'

        # Run a check_minions, if no minions match return False
        # format the payload - make a function that does this in the payload
        #   module
        # make the zmq client
        # connect to the req server
        # send!
        # return what we get back
        minions = self.check_minions(tgt, expr_form)

        if not minions:
            return {'jid': '',
                    'minions': minions}

        # Generate the standard keyword args to feed to format_payload
        payload_kwargs = { 'cmd': 'publish',
                           'tgt': tgt,
                           'fun': fun,
                           'arg': arg,
                           'key': self.key,
                           'tgt_type': expr_form,
                           'ret': ret,
                           'jid': jid }

        # If we have a salt user, add it to the payload
        if self.salt_user:
            payload_kwargs['user'] = self.salt_user

        # If we're a syndication master, pass the timeout
        if self.opts['order_masters']:
            payload_kwargs['to'] = timeout

        package = salt.payload.format_payload( 'clear', **payload_kwargs)

        # Prep zmq
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.linger = 0
        socket.connect(
                'tcp://{0[interface]}:{0[ret_port]}'.format(
                    self.opts
                    )
                )
        socket.send(package)
        payload = None
        for ind in range(100):
            try:
                payload = self.serial.loads(
                        socket.recv(
                            zmq.NOBLOCK
                            )
                        )
                break
            except zmq.core.error.ZMQError:
                time.sleep(0.01)
        if not payload:
            return {'jid': '0', 'minions': []}
        return {'jid': payload['load']['jid'],
                'minions': minions}
Ejemplo n.º 9
0
    def _thread_return(class_, minion_instance, opts, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        # this seems awkward at first, but it's a workaround for Windows
        # multiprocessing communication.
        if not minion_instance:
            minion_instance = class_(opts)
        if opts['multiprocessing']:
            fn_ = os.path.join(minion_instance.proc_dir, data['jid'])
            sdata = {'pid': os.getpid()}
            sdata.update(data)
            with salt.utils.fopen(fn_, 'w+') as fp_:
                fp_.write(minion_instance.serial.dumps(sdata))
        ret = {}
        for ind in range(0, len(data['arg'])):
            try:
                arg = eval(data['arg'][ind])
                if isinstance(arg, bool):
                    data['arg'][ind] = str(data['arg'][ind])
                elif isinstance(arg, (dict, int, list, string_types)):
                    data['arg'][ind] = arg
                else:
                    data['arg'][ind] = str(data['arg'][ind])
            except Exception:
                pass

        function_name = data['fun']
        if function_name in minion_instance.functions:
            ret['success'] = False
            try:
                func = minion_instance.functions[data['fun']]
                args, kwargs = detect_kwargs(func, data['arg'], data)
                ret['return'] = func(*args, **kwargs)
                ret['success'] = True
            except CommandNotFoundError as exc:
                msg = 'Command required for \'{0}\' not found: {1}'
                log.debug(msg.format(function_name, str(exc)))
                ret['return'] = msg.format(function_name, str(exc))
            except CommandExecutionError as exc:
                msg = 'A command in {0} had a problem: {1}'
                log.error(msg.format(function_name, str(exc)))
                ret['return'] = 'ERROR: {0}'.format(str(exc))
            except SaltInvocationError as exc:
                msg = 'Problem executing "{0}": {1}'
                log.error(msg.format(function_name, str(exc)))
                ret['return'] = 'ERROR executing {0}: {1}'.format(
                    function_name, exc)
            except Exception:
                trb = traceback.format_exc()
                msg = 'The minion function caused an exception: {0}'
                log.warning(msg.format(trb))
                ret['return'] = trb
        else:
            ret['return'] = '"{0}" is not available.'.format(function_name)

        ret['jid'] = data['jid']
        ret['fun'] = data['fun']
        minion_instance._return_pub(ret)
        if data['ret']:
            for returner in set(data['ret'].split(',')):
                ret['id'] = opts['id']
                try:
                    minion_instance.returners['{0}.returner'.format(returner)](
                        ret)
                except Exception as exc:
                    log.error('The return failed for job {0} {1}'.format(
                        data['jid'], exc))
Ejemplo n.º 10
0
      options.contents = True

    # binger is now a dictionary with each item being a key of a unique md5sum.  perform a CacheExtractor search and compare with its friends.
    # once we reach 1 item, it doesn't make sense to compare it against itself, so bail. 
    # If we start off with only 1 item, then everything is identical.
    if len(binger) == 1:
      first_key, (first_key_count, first_key_md5sum, first_key_hostlist) = binger.popitem()
      print "All machines with the key " + first_key + " had the same md5sum " + first_key_md5sum
      if options.hostlist:
        for host in first_key_hostlist:
          print host
      sys.exit(0)

    # Calculate how many actual comparisons will be executed.
    total_comparisons = 0
    for x in range(0, (len(binger) + 1)):
      total_comparisons = total_comparisons + (x -1)
    total_comparisons += 1
    
    attention_whore = "***********************************************************************************"
    print attention_whore.center(150)
    print attention_whore.center(150)
    print attention_whore.center(150)
    print "Comparison Summary: " + str(len(binger)) + " unique objects exist in the cache."
    print "A total of " + str(total_comparisons) + " comparisons will be executed below"
    for key in binger.iterkeys():
      (key_count, key_md5sum, key_hostlist) = binger[key]
      print "Unique sample object " + key + " with md5sum " + key_md5sum + " found " + str(key_count) + " times."
    print attention_whore.center(150)
    print attention_whore.center(150)
    print attention_whore.center(150)
Ejemplo n.º 11
0
        # binger is now a dictionary with each item being a key of a unique md5sum.  perform a sysopsapi.cache_extractor search and compare with its friends.
        # once we reach 1 item, it doesn't make sense to compare it against itself, so bail.
        # If we start off with only 1 item, then everything is identical.
        if len(binger) == 1:
            first_key, (first_key_count, first_key_md5sum,
                        first_key_hostlist) = binger.popitem()
            print "All machines with the key " + first_key + " had the same md5sum " + first_key_md5sum
            if options.hostlist:
                for host in first_key_hostlist:
                    print host
            sys.exit(0)

        # Calculate how many actual comparisons will be executed.
        total_comparisons = 0
        for x in range(0, (len(binger) + 1)):
            total_comparisons = total_comparisons + (x - 1)
        total_comparisons += 1

        attention_whore = "***********************************************************************************"
        print attention_whore.center(150)
        print attention_whore.center(150)
        print attention_whore.center(150)
        print "Comparison Summary: " + str(
            len(binger)) + " unique objects exist in the cache."
        print "A total of " + str(
            total_comparisons) + " comparisons will be executed below"
        for key in binger.iterkeys():
            (key_count, key_md5sum, key_hostlist) = binger[key]
            print "Unique sample object " + key + " with md5sum " + key_md5sum + " found " + str(
                key_count) + " times."
Ejemplo n.º 12
0
    def _thread_return(class_, minion_instance, opts, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        # this seems awkward at first, but it's a workaround for Windows
        # multiprocessing communication.
        if not minion_instance:
            minion_instance = class_(opts)
        if opts['multiprocessing']:
            fn_ = os.path.join(minion_instance.proc_dir, data['jid'])
            sdata = {'pid': os.getpid()}
            sdata.update(data)
            with salt.utils.fopen(fn_, 'w+') as fp_:
                fp_.write(minion_instance.serial.dumps(sdata))
        ret = {}
        for ind in range(0, len(data['arg'])):
            try:
                arg = eval(data['arg'][ind])
                if isinstance(arg, bool):
                    data['arg'][ind] = str(data['arg'][ind])
                elif isinstance(arg, (dict, int, list, string_types)):
                    data['arg'][ind] = arg
                else:
                    data['arg'][ind] = str(data['arg'][ind])
            except Exception:
                pass

        function_name = data['fun']
        if function_name in minion_instance.functions:
            ret['success'] = False
            try:
                func = minion_instance.functions[data['fun']]
                args, kwargs = detect_kwargs(func, data['arg'], data)
                ret['return'] = func(*args, **kwargs)
                ret['success'] = True
            except CommandNotFoundError as exc:
                msg = 'Command required for \'{0}\' not found: {1}'
                log.debug(msg.format(function_name, str(exc)))
                ret['return'] = msg.format(function_name, str(exc))
            except CommandExecutionError as exc:
                msg = 'A command in {0} had a problem: {1}'
                log.error(msg.format(function_name, str(exc)))
                ret['return'] = 'ERROR: {0}'.format(str(exc))
            except SaltInvocationError as exc:
                msg = 'Problem executing "{0}": {1}'
                log.error(msg.format(function_name, str(exc)))
                ret['return'] = 'ERROR executing {0}: {1}'.format(
                    function_name, exc
                )
            except Exception:
                trb = traceback.format_exc()
                msg = 'The minion function caused an exception: {0}'
                log.warning(msg.format(trb))
                ret['return'] = trb
        else:
            ret['return'] = '"{0}" is not available.'.format(function_name)

        ret['jid'] = data['jid']
        ret['fun'] = data['fun']
        minion_instance._return_pub(ret)
        if data['ret']:
            for returner in set(data['ret'].split(',')):
                ret['id'] = opts['id']
                try:
                    minion_instance.returners['{0}.returner'.format(
                        returner
                    )](ret)
                except Exception as exc:
                    log.error(
                        'The return failed for job {0} {1}'.format(
                        data['jid'],
                        exc
                        )
                    )