예제 #1
0
 def _handle_aes(self, load):
     '''
     Takes the aes encrypted load, decrypts is and runs the encapsulated
     instructions
     '''
     try:
         data = self.crypticle.loads(load)
     except AuthenticationError:
         self.authenticate()
         data = self.crypticle.loads(load)
     # Verify that the publication is valid
     if 'tgt' not in data or 'jid' not in data or 'fun' not in data \
        or 'arg' not in data:
         return
     # Verify that the publication applies to this minion
     if 'tgt_type' in data:
         if not getattr(self.matcher, '{0}_match'.format(data['tgt_type']))(
                 data['tgt']):
             return
     else:
         if not self.matcher.glob_match(data['tgt']):
             return
     # If the minion does not have the function, don't execute,
     # this prevents minions that could not load a minion module
     # from returning a predictable exception
     #if data['fun'] not in self.functions:
     #    return
     if 'user' in data:
         log.info(('User {0[user]} Executing command {0[fun]} with jid '
                   '{0[jid]}'.format(data)))
     else:
         log.info(
             ('Executing command {0[fun]} with jid {0[jid]}'.format(data)))
     log.debug('Command details {0}'.format(data))
     self._handle_decoded_payload(data)
예제 #2
0
def ec2_tags():

    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("%s: installed boto version %s < %s, can't find ec2_tags",
                  __name__, boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("%s: not an EC2 instance, skipping", __name__)
        return None

    (instance_id, region) = _get_instance_info()
    credentials = _get_credentials()
    if not credentials:
        log.error(
            "%s: no AWS credentials found, see documentation for how to provide them.",
            __name__)
        return None

    # Connect to EC2 and parse the Roles tags for this instance
    conn = boto.ec2.connect_to_region(
        region,
        aws_access_key_id=credentials['access_key'],
        aws_secret_access_key=credentials['secret_key'])

    tags = {}
    try:
        reservation = conn.get_all_instances(instance_ids=[instance_id])[0]
        instance = reservation.instances[0]
        tags = instance.tags
    except IndexError, e:
        log.error("Couldn't retrieve instance information: %s", e)
        return None
예제 #3
0
def ec2_tags():
    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("Installed boto version %s < %s, can't find ec2_tags",
                  boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("Not an EC2 instance, skipping")
        return None

    instance_id, region = _get_instance_info()
    credentials = _get_credentials()

    # Connect to EC2 and parse the Roles tags for this instance
    if not (credentials['access_key'] and credentials['secret_key']):
        log.error("No AWS credentials found, see documentation for how to provide them.")
        return None

    try:
        conn = boto.ec2.connect_to_region(
            region,
            aws_access_key_id=credentials['access_key'],
            aws_secret_access_key=credentials['secret_key'],
        )
    except Exception, e:
        log.error("Could not get AWS connection: %s", e)
        return None
예제 #4
0
def ec2_tags():

    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("%s: installed boto version %s < %s, can't find ec2_tags",
                __name__, boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("%s: not an EC2 instance, skipping", __name__)
        return None

    (instance_id, region) = _get_instance_info()
    credentials = _get_credentials()
    if not credentials:
        log.error("%s: no AWS credentials found, see documentation for how to provide them.", __name__)
        return None

    # Connect to EC2 and parse the Roles tags for this instance
    conn = boto.ec2.connect_to_region(region,
            aws_access_key_id=credentials['access_key'],
            aws_secret_access_key=credentials['secret_key'])

    tags = {}
    try:
        reservation = conn.get_all_instances(instance_ids=[ instance_id ])[0]
        instance = reservation.instances[0]
        tags = instance.tags
    except IndexError, e:
        log.error("Couldn't retrieve instance information: %s", e)
        return None
예제 #5
0
파일: payload.py 프로젝트: steverweber/salt
 def send(self, enc, load, tries=1, timeout=60):
     """
     Takes two arguments, the encryption type and the base payload
     """
     payload = {"enc": enc}
     payload["load"] = load
     pkg = self.serial.dumps(payload)
     self.socket.send(pkg)
     self.poller.register(self.socket, zmq.POLLIN)
     tried = 0
     while True:
         polled = self.poller.poll(timeout * 1000)
         tried += 1
         if polled:
             break
         if tries > 1:
             log.info(
                 "SaltReqTimeoutError: after %s seconds. (Try %s of %s)",
                 timeout,
                 tried,
                 tries,
             )
         if tried >= tries:
             self.clear_socket()
             raise SaltReqTimeoutError(
                 "SaltReqTimeoutError: after {} seconds, ran {} "
                 "tries".format(timeout * tried, tried))
     return self.serial.loads(self.socket.recv())
예제 #6
0
파일: minion.py 프로젝트: 11craft/salt
 def _handle_aes(self, load):
     '''
     Takes the aes encrypted load, decrypts is and runs the encapsulated
     instructions
     '''
     try:
         data = self.crypticle.loads(load)
     except AuthenticationError:
         self.authenticate()
         data = self.crypticle.loads(load)
     # Verify that the publication is valid
     if 'tgt' not in data or 'jid' not in data or 'fun' not in data \
        or 'arg' not in data:
         return
     # Verify that the publication applies to this minion
     if 'tgt_type' in data:
         if not getattr(self.matcher,
                        '{0}_match'.format(data['tgt_type']))(data['tgt']):
             return
     else:
         if not self.matcher.glob_match(data['tgt']):
             return
     # If the minion does not have the function, don't execute,
     # this prevents minions that could not load a minion module
     # from returning a predictable exception
     #if data['fun'] not in self.functions:
     #    return
     if 'user' in data:
         log.info(('User {0[user]} Executing command {0[fun]} with jid '
                   '{0[jid]}'.format(data)))
     else:
         log.info(('Executing command {0[fun]} with jid {0[jid]}'
                   .format(data)))
     log.debug('Command details {0}'.format(data))
     self._handle_decoded_payload(data)
예제 #7
0
 def send(self, enc, load, tries=1, timeout=60):
     '''
     Takes two arguments, the encryption type and the base payload
     '''
     payload = {'enc': enc}
     payload['load'] = load
     pkg = self.serial.dumps(payload)
     self.socket.send(pkg)
     self.poller.register(self.socket, zmq.POLLIN)
     tried = 0
     while True:
         polled = self.poller.poll(timeout * 1000)
         tried += 1
         if polled:
             break
         if tries > 1:
             log.info(
                 'SaltReqTimeoutError: after {0} seconds. (Try {1} of {2})'.
                 format(timeout, tried, tries))
         if tried >= tries:
             self.clear_socket()
             raise SaltReqTimeoutError(
                 'SaltReqTimeoutError: after {0} seconds, ran {1} tries'.
                 format(timeout * tried, tried))
     return self.serial.loads(self.socket.recv())
예제 #8
0
 def start(self):
     '''
     we override start() just for our log message
     '''
     log.info("starting salt-eventsd daemon")
     # leave the startup to the supers daemon, thats where all 
     # the daemonizing and double-forking takes place
     super(SaltEventsDaemon, self).start()
예제 #9
0
 def run(self):
     '''
     the method automatically called by start() from
     our parent class 
     '''
     log.info("initializing event listener")
     self.pid = self._get_pid()
     self._write_state()
     self.listen()
예제 #10
0
파일: minion.py 프로젝트: 11craft/salt
 def _return_pub(self, ret, ret_cmd='_return'):
     '''
     Return the data from the executed command to the master server
     '''
     if self.opts['multiprocessing']:
         fn_ = os.path.join(self.proc_dir, ret['jid'])
         if os.path.isfile(fn_):
             try:
                 os.remove(fn_)
             except (OSError, IOError):
                 # The file is gone already
                 pass
     log.info('Returning information for job: {0}'.format(ret['jid']))
     sreq = salt.payload.SREQ(self.opts['master_uri'])
     if ret_cmd == '_syndic_return':
         load = {'cmd': ret_cmd,
                 'jid': ret['jid'],
                 'id': self.opts['id']}
         load['return'] = {}
         for key, value in ret.items():
             if key == 'jid' or key == 'fun':
                 continue
             load['return'][key] = value
     else:
         load = {'return': ret['return'],
                 'cmd': ret_cmd,
                 'jid': ret['jid'],
                 'id': self.opts['id']}
     try:
         if hasattr(self.functions[ret['fun']], '__outputter__'):
             oput = self.functions[ret['fun']].__outputter__
             if isinstance(oput, string_types):
                 load['out'] = oput
     except KeyError:
         pass
     try:
         ret_val = sreq.send('aes', self.crypticle.dumps(load))
     except SaltReqTimeoutError:
         ret_val = ''
     if isinstance(ret_val, string_types) and not ret_val:
         # The master AES key has changed, reauth
         self.authenticate()
         ret_val = sreq.send('aes', self.crypticle.dumps(load))
     if self.opts['cache_jobs']:
         # Local job cache has been enabled
         fn_ = os.path.join(
             self.opts['cachedir'],
             'minion_jobs',
             load['jid'],
             'return.p')
         jdir = os.path.dirname(fn_)
         if not os.path.isdir(jdir):
             os.makedirs(jdir)
         salt.utils.fopen(fn_, 'w+').write(self.serial.dumps(ret))
     return ret_val
예제 #11
0
    def __init__(self, config='/etc/salt/eventsd'):

        # retrieve current settings from the config file
        self.opts = None
        self._read_yaml(config)

        # make sure we have a 'general' section
        if 'general' in self.opts.keys():
            self.gen_opts = self.opts['general']

        self._init_logger()
        log.info("loaded config from {0}".format(config))
예제 #12
0
def _dmidecode_data(regex_dict):
    '''
    Parse the output of dmidecode in a generic fashion that can
    be used for the multiple system types which have dmidecode.
    '''
    ret = {}

    if 'proxyminion' in __opts__:
        return {}

    # No use running if dmidecode/smbios isn't in the path
    if salt.utils.which('dmidecode'):
        out = __salt__['cmd.run']('dmidecode')
    elif salt.utils.which('smbios'):
        out = __salt__['cmd.run']('smbios')
    else:
        log.info(
            'The `dmidecode` binary is not available on the system. GPU grains '
            'will not be available.')
        return ret

    for section in regex_dict:
        section_found = False

        # Look at every line for the right section
        for line in out.splitlines():
            if not line:
                continue
            # We've found it, woohoo!
            if re.match(section, line):
                section_found = True
                continue
            if not section_found:
                continue

            # Now that a section has been found, find the data
            for item in regex_dict[section]:
                # Examples:
                #    Product Name: 64639SU
                #    Version: 7LETC1WW (2.21 )
                regex = re.compile(r'\s+{0}\s+(.*)$'.format(item))
                grain = regex_dict[section][item]
                # Skip to the next iteration if this grain
                # has been found in the dmidecode output.
                if grain in ret:
                    continue

                match = regex.match(line)

                # Finally, add the matched data to the grains returned
                if match:
                    ret[grain] = match.group(1).strip()
    return ret
예제 #13
0
 def _return_pub(self, ret, ret_cmd='_return'):
     '''
     Return the data from the executed command to the master server
     '''
     if self.opts['multiprocessing']:
         fn_ = os.path.join(self.proc_dir, ret['jid'])
         if os.path.isfile(fn_):
             try:
                 os.remove(fn_)
             except (OSError, IOError):
                 # The file is gone already
                 pass
     log.info('Returning information for job: {0}'.format(ret['jid']))
     sreq = salt.payload.SREQ(self.opts['master_uri'])
     if ret_cmd == '_syndic_return':
         load = {'cmd': ret_cmd, 'jid': ret['jid'], 'id': self.opts['id']}
         load['return'] = {}
         for key, value in ret.items():
             if key == 'jid' or key == 'fun':
                 continue
             load['return'][key] = value
     else:
         load = {
             'return': ret['return'],
             'cmd': ret_cmd,
             'jid': ret['jid'],
             'id': self.opts['id']
         }
     try:
         if hasattr(self.functions[ret['fun']], '__outputter__'):
             oput = self.functions[ret['fun']].__outputter__
             if isinstance(oput, string_types):
                 load['out'] = oput
     except KeyError:
         pass
     try:
         ret_val = sreq.send('aes', self.crypticle.dumps(load))
     except SaltReqTimeoutError:
         ret_val = ''
     if isinstance(ret_val, string_types) and not ret_val:
         # The master AES key has changed, reauth
         self.authenticate()
         ret_val = sreq.send('aes', self.crypticle.dumps(load))
     if self.opts['cache_jobs']:
         # Local job cache has been enabled
         fn_ = os.path.join(self.opts['cachedir'], 'minion_jobs',
                            load['jid'], 'return.p')
         jdir = os.path.dirname(fn_)
         if not os.path.isdir(jdir):
             os.makedirs(jdir)
         salt.utils.fopen(fn_, 'w+').write(self.serial.dumps(ret))
     return ret_val
예제 #14
0
    def __init__(self,
                 config='/etc/salt/eventsd'):

        # retrieve current settings from the config file
        self.opts = None
        self._read_yaml(config)

        # make sure we have a 'general' section
        if 'general' in self.opts.keys():
            self.gen_opts = self.opts['general']

        self._init_logger()
        log.info("loaded config from {0}".format(config))
예제 #15
0
    def _init_worker(self, qdata):
        '''
        The method dumps the data into a worker thread which
        handles pushing the data into different backends.
        '''
        self.threads_cre += 1

        log.info("Starting worker #{0}".format(self.threads_cre))

        # make sure we pass a copy of the list
        worker = SaltEventsdWorker(list(qdata), self.threads_cre,
                                   self.event_map, self.backends, **self.opts)

        worker.start()
        self.running_workers.append(worker)
예제 #16
0
def ec2_tags():

    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("%s: installed boto version %s < %s, can't find ec2_tags",
                  __name__, boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("%s: not an EC2 instance, skipping", __name__)
        return None

    (instance_id, region) = _get_instance_info()
    credentials = _get_credentials()

    # Connect to EC2 and parse the Roles tags for this instance
    try:
        conn = boto.ec2.connect_to_region(
            region,
            aws_access_key_id=credentials['access_key'],
            aws_secret_access_key=credentials['secret_key'])
    except:
        if not (credentials['access_key'] and credentials['secret_key']):
            log.error(
                "%s: no AWS credentials found, see documentation for how to provide them.",
                __name__)
            return None
        else:
            log.error(
                "%s: invalid AWS credentials found, see documentation for how to provide them.",
                __name__)
            return None

    tags = {}
    try:
        _tags = conn.get_all_tags(filters={
            'resource-type': 'instance',
            'resource-id': instance_id
        })
        for tag in _tags:
            tags[tag.name] = tag.value
    except IndexError, e:
        log.error("Couldn't retrieve instance information: %s", e)
        return None
예제 #17
0
    def _init_worker(self, qdata):
        '''
        write a collection of events to the database. every invocation of
        this methoed creates its own thread that writes into the database
        '''
        self.threads_cre += 1

        log.info("starting worker #{0}".format(self.threads_cre))

        # make sure we pass a copy of the list
        worker = SaltEventsdWorker(list(qdata),
                                   self.threads_cre,
                                   self.event_map,
                                   self.backends,
                                   **self.opts)

        worker.start()
        self.running_workers.append(worker)
예제 #18
0
파일: config.py 프로젝트: herlo/salt
def get_id():
    '''
    Guess the id of the minion.

    - If socket.getfqdn() returns us something other than localhost, use it
    - Check /etc/hosts for something that isn't localhost that maps to 127.*
    - Look for a routeable / public IP
    - A private IP is better than a loopback IP
    - localhost may be better than killing the minion
    '''

    log.debug('Guessing ID. The id can be explicitly in set {0}'
              .format('/etc/salt/minion'))
    fqdn = socket.getfqdn()
    if 'localhost' != fqdn:
        log.info('Found minion id from getfqdn(): {0}'.format(fqdn))
        return fqdn, False

    # Can /etc/hosts help us?
    try:
        # TODO Add Windows host file support
        with open('/etc/hosts') as f:
            line = f.readline()
            while line:
                names = line.split()
                ip = names.pop(0)
                if ip.startswith('127.'):
                    for name in names:
                        if name != 'localhost':
                            log.info('Found minion id in hosts file: {0}'
                                     .format(name))
                            return name, False
                line = f.readline()
    except Exception:
        pass

    # What IP addresses do we have?
    ip_addresses = [salt.utils.socket_util.IPv4Address(a) for a
                    in salt.utils.socket_util.ip4_addrs()
                    if not a.startswith('127.')]

    for a in ip_addresses:
        if not a.is_private:
            log.info('Using public ip address for id: {0}'.format(a))
            return str(a), True

    if ip_addresses:
        a = ip_addresses.pop(0)
        log.info('Using private ip address for id: {0}'.format(a))
        return str(a), True

    log.error('No id found, falling back to localhost')
    return 'localhost', False
예제 #19
0
def ec2_tags():
    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("Installed boto version %s < %s, can't find ec2_tags",
                  boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("Not an EC2 instance, skipping")
        return None

    instance_id, region = _get_instance_info()
    credentials = _get_credentials()

    # Connect to EC2 and parse the Roles tags for this instance
    try:
        conn = boto.ec2.connect_to_region(
            region,
            aws_access_key_id=credentials['access_key'],
            aws_secret_access_key=credentials['secret_key'],
        )
    except Exception as e:
        log.error("Could not get AWS connection: %s", e)
        return None

    ec2_tags = {}
    try:
        tags = conn.get_all_tags(filters={'resource-type': 'instance',
                                          'resource-id': instance_id})
        for tag in tags:
            ec2_tags[tag.name] = tag.value
    except Exception as e:
        log.error("Couldn't retrieve instance tags: %s", e)
        return None

    ret = dict(ec2_tags=ec2_tags)

    # Provide ec2_tags_roles functionality
    if 'Roles' in ec2_tags:
        ret['ec2_roles'] = ec2_tags['Roles'].split(',')

    return ret
예제 #20
0
    def _init_worker(self, qdata):
        '''
        The method dumps the data into a worker thread which
        handles pushing the data into different backends.
        '''
        self.threads_cre += 1

        log.info("Starting worker #{0}".format(self.threads_cre))

        # make sure we pass a copy of the list
        worker = SaltEventsdWorker(
            list(qdata),
            self.threads_cre,
            self.event_map,
            self.backends,
            **self.opts
        )

        worker.start()
        self.running_workers.append(worker)
예제 #21
0
 def authenticate(self):
     '''
     Authenticate with the master, this method breaks the functional
     paradigm, it will update the master information from a fresh sign
     in, signing in can occur as often as needed to keep up with the
     revolving master aes key.
     '''
     log.debug(
         'Attempting to authenticate with the Salt Master at {0}'.format(
             self.opts['master_ip']))
     auth = salt.crypt.Auth(self.opts)
     while True:
         creds = auth.sign_in()
         if creds != 'retry':
             log.info('Authentication with master successful!')
             break
         log.info('Waiting for minion key to be accepted by the master.')
         time.sleep(self.opts['acceptance_wait_time'])
     self.aes = creds['aes']
     self.publish_port = creds['publish_port']
     self.crypticle = salt.crypt.Crypticle(self.opts, self.aes)
예제 #22
0
    def _init_events(self, events={}):
        '''
        Creates a dict of precompiled regexes for all defined events
        from config for maximum performance.
        '''
        self.event_map = events
        # we precompile all regexes
        log.info("Initialising events...")

        for key in events.keys():
            # we compile the regex configured in the config
            self.event_map[key]['tag'] = compile(events[key]['tag'])
            log.info("Added event '{0}'".format(key))

            # if subevents are configured, also update them with
            # regex-matching object
            if 'subs' in events[key]:
                for sub_ev in events[key]['subs'].keys():
                    try:
                        self.event_map[key]['subs'][sub_ev]['fun'] = compile(events[key]['subs'][sub_ev]['fun'])
                    except KeyError:
                        pass

                    try:
                        self.event_map[key]['subs'][sub_ev]['tag'] = compile(events[key]['subs'][sub_ev]['tag'])
                    except KeyError:
                        pass

                    log.info("Added sub-event '{0}->{1}'".format(key, sub_ev))
예제 #23
0
    def stop(self,
             signal,
             frame):
        '''
        we override stop() to brake our main loop
        and have a pretty log message
        '''
        log.info("received signal {0}".format(signal))

        # if we have running workers, run through all and join() the ones
        # that have finished. if we still have running workers after that,
        # wait 5 secs for the rest and then exit. Maybe we should improv
        # this a litte bit more
        if( len(self.running_workers) > 0 ):
            clean_workers = []

            for count in range(0, 2):
                for worker in self.running_workers:
                    if worker.isAlive():
                        clean_workers.append(worker)
                    else:
                        worker.join()
                        log.debug("joined worker #{0}".format(worker.getName()))

                if( len(clean_workers) > 0 ):
                    log.info("waiting 5secs for remaining workers..")
                    time.sleep(5)
                else:
                    break

        log.info("salt-eventsd has shut down")

        # leave the cleanup to the supers stop
        super(SaltEventsDaemon, self).stop(signal, frame)
예제 #24
0
    def _init_events(self, events={}):
        '''
        Creates a dict of precompiled regexes for all defined events
        from config for maximum performance.
        '''
        self.event_map = events
        # we precompile all regexes
        log.info("Initialising events...")

        for key in events.keys():
            # we compile the regex configured in the config
            self.event_map[key]['tag'] = compile(events[key]['tag'])
            log.info("Added event '{0}'".format(key))

            # if subevents are configured, also update them with
            # regex-matching object
            if 'subs' in events[key]:
                for sub_ev in events[key]['subs'].keys():
                    try:
                        self.event_map[key]['subs'][sub_ev]['fun'] = compile(
                            events[key]['subs'][sub_ev]['fun'])
                    except KeyError:
                        pass

                    try:
                        self.event_map[key]['subs'][sub_ev]['tag'] = compile(
                            events[key]['subs'][sub_ev]['tag'])
                    except KeyError:
                        pass

                    log.info("Added sub-event '{0}->{1}'".format(key, sub_ev))
예제 #25
0
    def _init_events(self, events={}):
        '''
        this is used to tell the class about the events it should handle.
        it has to be a dictionary with appropriate mappings in it. see the
        config file for examples on how to compose the dict. each entry is
        converted to a precompiled regex for maximum flexibility
        '''
        self.event_map = events
        # we precompile all regexes
        log.info("initialising events...")
        for key in events.keys():
            # we compile the regex configured in the config
            self.event_map[key]['tag'] = compile( events[key]['tag'] )
            log.info("Added event '{0}'".format(key))

            # if subevents are configured, also update them with 
            # regex-macthing object
            if( events[key].has_key('subs') ):
                for sub_ev in events[key]['subs'].keys():
                    try:
                        self.event_map[key]['subs'][sub_ev]['fun'] = compile(events[key]['subs'][sub_ev]['fun'])
                    except KeyError:
                        pass

                    try:
                        self.event_map[key]['subs'][sub_ev]['tag'] = compile(events[key]['subs'][sub_ev]['tag'])
                    except KeyError:
                        pass

                    log.info("Added sub-event '{0}->{1}'".format(key,
                                                                 sub_ev))
예제 #26
0
    def _write_state(self):
        '''
        writes a current status to the defined status-file
        this includes the current pid, events received/handled
        and threads created/joined
        '''
        try:
            # write the info to the specified log
            statf = open(self.state_file, 'w')
            statf.writelines(simplejson.dumps({'events_received':self.events_rec,
                                               'events_handled':self.events_han,
                                               'threads_created':self.threads_cre,
                                               'threads_joined':self.threads_join}
                                             ))
            # if we have the same pid as the pidfile, we are the running daemon
            # and also print the current counters to the logfile with 'info'
            if( os.getpid() == self.pid ):
                log.info("running with pid {0}".format(self.pid))
                log.info("events (han/recv): {0}/{1}".format(self.events_han,
                                                             self.events_rec))
                log.info("threads (cre/joi):{0}/{1}".format(self.threads_cre,
                                                            self.threads_join))


            statf.write("\n")
            statf.close()
            sys.stdout.flush()
        except IOError as ioerr:
            log.critical("Failed to write state to {0}".format(self.state_file))
            log.exception(ioerr)
        except OSError as oserr:
            log.critical("Failed to write state to {0}".format(self.state_file))
            log.exception(oserr)
예제 #27
0
def ec2_tags():

    boto_version = StrictVersion(boto.__version__)
    required_boto_version = StrictVersion('2.8.0')
    if boto_version < required_boto_version:
        log.error("%s: installed boto version %s < %s, can't find ec2_tags",
                __name__, boto_version, required_boto_version)
        return None

    if not _on_ec2():
        log.info("%s: not an EC2 instance, skipping", __name__)
        return None

    (instance_id, region) = _get_instance_info()
    credentials = _get_credentials()

    # Connect to EC2 and parse the Roles tags for this instance
    try:
        conn = boto.ec2.connect_to_region(region,
                aws_access_key_id=credentials['access_key'],
                aws_secret_access_key=credentials['secret_key'])
    except:
        if not (credentials['access_key'] and credentials['secret_key']):
            log.error("%s: no AWS credentials found, see documentation for how to provide them.", __name__)
            return None
        else:
            log.error("%s: invalid AWS credentials found, see documentation for how to provide them.", __name__)
            return None

    tags = {}
    try:
        _tags = conn.get_all_tags(filters={'resource-type': 'instance',
                'resource-id': instance_id})
        for tag in _tags:
            tags[tag.name] = tag.value
    except IndexError, e:
        log.error("Couldn't retrieve instance information: %s", e)
        return None
예제 #28
0
파일: minion.py 프로젝트: 11craft/salt
 def authenticate(self):
     '''
     Authenticate with the master, this method breaks the functional
     paradigm, it will update the master information from a fresh sign
     in, signing in can occur as often as needed to keep up with the
     revolving master aes key.
     '''
     log.debug(
         'Attempting to authenticate with the Salt Master at {0}'.format(
             self.opts['master_ip']
         )
     )
     auth = salt.crypt.Auth(self.opts)
     while True:
         creds = auth.sign_in()
         if creds != 'retry':
             log.info('Authentication with master successful!')
             break
         log.info('Waiting for minion key to be accepted by the master.')
         time.sleep(self.opts['acceptance_wait_time'])
     self.aes = creds['aes']
     self.publish_port = creds['publish_port']
     self.crypticle = salt.crypt.Crypticle(self.opts, self.aes)
예제 #29
0
 def send(self, enc, load, tries=1, timeout=60):
     '''
     Takes two arguments, the encryption type and the base payload
     '''
     payload = {'enc': enc}
     payload['load'] = load
     pkg = self.serial.dumps(payload)
     self.socket.send(pkg)
     self.poller.register(self.socket, zmq.POLLIN)
     tried = 0
     while True:
         polled = self.poller.poll(timeout * 1000)
         tried += 1
         if polled:
             break
         if tries > 1:
             log.info('SaltReqTimeoutError: after {0} seconds. (Try {1} of {2})'.format(
               timeout, tried, tries))
         if tried >= tries:
             self.clear_socket()
             raise SaltReqTimeoutError(
                 'SaltReqTimeoutError: after {0} seconds, ran {1} tries'.format(timeout * tried, tried)
             )
     return self.serial.loads(self.socket.recv())
예제 #30
0
파일: master.py 프로젝트: bryson/salt
    def run(self):
        '''
        Main loop of the ConCache, starts updates in intervals and
        answers requests from the MWorkers
        '''
        context = zmq.Context()
        # the socket for incoming cache requests
        creq_in = context.socket(zmq.REP)
        creq_in.setsockopt(zmq.LINGER, 100)
        creq_in.bind('ipc://' + self.cache_sock)

        # the socket for incoming cache-updates from workers
        cupd_in = context.socket(zmq.SUB)
        cupd_in.setsockopt(zmq.SUBSCRIBE, '')
        cupd_in.setsockopt(zmq.LINGER, 100)
        cupd_in.bind('ipc://' + self.update_sock)

        # the socket for the timer-event
        timer_in = context.socket(zmq.SUB)
        timer_in.setsockopt(zmq.SUBSCRIBE, '')
        timer_in.setsockopt(zmq.LINGER, 100)
        timer_in.connect('ipc://' + self.upd_t_sock)

        poller = zmq.Poller()
        poller.register(creq_in, zmq.POLLIN)
        poller.register(cupd_in, zmq.POLLIN)
        poller.register(timer_in, zmq.POLLIN)

        # our serializer
        serial = salt.payload.Serial(self.opts.get('serial', ''))

        # register a signal handler
        signal.signal(signal.SIGINT, self.signal_handler)

        # secure the sockets from the world
        self.secure()

        log.info('ConCache started')

        while self.running:

            # we check for new events with the poller
            try:
                socks = dict(poller.poll(1))
            except KeyboardInterrupt:
                self.stop()
            except zmq.ZMQError as zmq_err:
                log.error('ConCache ZeroMQ-Error occurred')
                log.exception(zmq_err)
                self.stop()

            # check for next cache-request
            if socks.get(creq_in) == zmq.POLLIN:
                msg = serial.loads(creq_in.recv())
                log.debug('ConCache Received request: {0}'.format(msg))

                # requests to the minion list are send as str's
                if isinstance(msg, str):
                    if msg == 'minions':
                        # Send reply back to client
                        reply = serial.dumps(self.minions)
                        creq_in.send(reply)

            # check for next cache-update from workers
            if socks.get(cupd_in) == zmq.POLLIN:
                new_c_data = serial.loads(cupd_in.recv())
                # tell the worker to exit
                #cupd_in.send(serial.dumps('ACK'))

                # check if the returned data is usable
                if not isinstance(new_c_data, list):
                    log.error('ConCache Worker returned unusable result')
                    del new_c_data
                    continue

                # the cache will receive lists of minions
                # 1. if the list only has 1 item, its from an MWorker, we append it
                # 2. if the list contains another list, its from a CacheWorker and
                #    the currently cached minions are replaced with that list
                # 3. anything else is considered malformed

                try:

                    if len(new_c_data) == 0:
                        log.debug('ConCache Got empty update from worker')
                        continue

                    data = new_c_data[0]

                    if isinstance(data, str):
                        if data not in self.minions:
                            log.debug('ConCache Adding minion {0} to cache'.format(new_c_data[0]))
                            self.minions.append(data)

                    elif isinstance(data, list):
                        log.debug('ConCache Replacing minion list from worker')
                        self.minions = data

                except IndexError:
                    log.debug('ConCache Got malformed result dict from worker')
                    del new_c_data

                log.info('ConCache {0} entries in cache'.format(len(self.minions)))

            # check for next timer-event to start new jobs
            if socks.get(timer_in) == zmq.POLLIN:
                sec_event = serial.loads(timer_in.recv())

                # update the list every 30 seconds
                if int(sec_event % 30) == 0:
                    cw = CacheWorker(self.opts)
                    cw.start()

        self.stop()
        creq_in.close()
        cupd_in.close()
        timer_in.close()
        context.term()
        log.debug('ConCache Shutting down')
예제 #31
0
파일: minion.py 프로젝트: 11craft/salt
    def tune_in(self):
        '''
        Lock onto the publisher. This is the main event loop for the minion
        '''
        log.info(
            '{0} is starting as user \'{1}\''.format(
                self.__class__.__name__,
                getpass.getuser()
            )
        )
        log.debug('Minion "{0}" trying to tune in'.format(self.opts['id']))
        self.context = zmq.Context()

        # Prepare the minion event system
        #
        # Start with the publish socket
        id_hash = hashlib.md5(self.opts['id']).hexdigest()
        epub_sock_path = os.path.join(
            self.opts['sock_dir'],
            'minion_event_{0}_pub.ipc'.format(id_hash)
        )
        epull_sock_path = os.path.join(
            self.opts['sock_dir'],
            'minion_event_{0}_pull.ipc'.format(id_hash)
        )
        self.epub_sock = self.context.socket(zmq.PUB)
        if self.opts.get('ipc_mode', '') == 'tcp':
            epub_uri = 'tcp://127.0.0.1:{0}'.format(
                self.opts['tcp_pub_port']
            )
            epull_uri = 'tcp://127.0.0.1:{0}'.format(
                self.opts['tcp_pull_port']
            )
        else:
            epub_uri = 'ipc://{0}'.format(epub_sock_path)
            salt.utils.check_ipc_path_max_len(epub_uri)
            epull_uri = 'ipc://{0}'.format(epull_sock_path)
            salt.utils.check_ipc_path_max_len(epull_uri)
        log.debug(
            '{0} PUB socket URI: {1}'.format(
                self.__class__.__name__, epub_uri
            )
        )
        log.debug(
            '{0} PULL socket URI: {1}'.format(
                self.__class__.__name__, epull_uri
            )
        )

        # Create the pull socket
        self.epull_sock = self.context.socket(zmq.PULL)
        # Bind the event sockets
        self.epub_sock.bind(epub_uri)
        self.epull_sock.bind(epull_uri)
        # Restrict access to the sockets
        if not self.opts.get('ipc_mode', '') == 'tcp':
            os.chmod(
                epub_sock_path,
                448
            )
            os.chmod(
                epull_sock_path,
                448
            )

        self.poller = zmq.Poller()
        self.epoller = zmq.Poller()
        self.socket = self.context.socket(zmq.SUB)
        self.socket.setsockopt(zmq.SUBSCRIBE, '')
        self.socket.setsockopt(zmq.IDENTITY, self.opts['id'])
        if hasattr(zmq, 'RECONNECT_IVL_MAX'):
            self.socket.setsockopt(
                zmq.RECONNECT_IVL_MAX, self.opts['recon_max']
            )
        if hasattr(zmq, 'TCP_KEEPALIVE'):
            self.socket.setsockopt(
                zmq.TCP_KEEPALIVE, self.opts['tcp_keepalive']
            )
            self.socket.setsockopt(
                zmq.TCP_KEEPALIVE_IDLE, self.opts['tcp_keepalive_idle']
            )
            self.socket.setsockopt(
                zmq.TCP_KEEPALIVE_CNT, self.opts['tcp_keepalive_cnt']
            )
            self.socket.setsockopt(
                zmq.TCP_KEEPALIVE_INTVL, self.opts['tcp_keepalive_intvl']
            )
        if hasattr(zmq, 'IPV4ONLY'):
            self.socket.setsockopt(
                zmq.IPV4ONLY, int(not int(self.opts.get('ipv6_enable', False)))
            )
        self.socket.connect(self.master_pub)
        self.poller.register(self.socket, zmq.POLLIN)
        self.epoller.register(self.epull_sock, zmq.POLLIN)
        # Send an event to the master that the minion is live
        self._fire_master(
            'Minion {0} started at {1}'.format(
            self.opts['id'],
            time.asctime()
            ),
            'minion_start'
        )

        if self.opts['multiprocessing'] and not salt.utils.is_windows():
            signal.signal(signal.SIGCHLD, self.handle_sigchld)
        # Make sure to gracefully handle SIGUSR1
        enable_sigusr1_handler()

        # On first startup execute a state run if configured to do so
        self._state_run()

        while True:
            try:
                self.schedule.eval()
                socks = dict(self.poller.poll(
                    self.opts['loop_interval'] * 1000)
                )
                if self.socket in socks and socks[self.socket] == zmq.POLLIN:
                    payload = self.serial.loads(self.socket.recv())
                    self._handle_payload(payload)
                time.sleep(0.05)
                # Clean up the minion processes which have been executed and
                # have finished
                # Check if modules and grains need to be refreshed
                self.passive_refresh()
                # Check the event system
                if self.epoller.poll(1):
                    try:
                        package = self.epull_sock.recv(zmq.NOBLOCK)
                        self.epub_sock.send(package)
                    except Exception:
                        pass
            except zmq.ZMQError:
                # This is thrown by the inturupt caused by python handling the
                # SIGCHLD. This is a safe error and we just start the poll
                # again
                continue
            except Exception:
                log.critical(traceback.format_exc())
예제 #32
0
def _linux_gpu_data():
    '''
    num_gpus: int
    gpus:
      - vendor: nvidia|amd|ati|...
        model: string
    '''
    lspci = salt.utils.which('lspci')
    if not lspci:
        log.info(
            'The `lspci` binary is not available on the system. GPU grains '
            'will not be available.')
        return {}

    elif __opts__.get('enable_gpu_grains', None) is False:
        log.info(
            'Skipping lspci call because enable_gpu_grains was set to False '
            'in the config. GPU grains will not be available.')
        return {}

    # dominant gpu vendors to search for (MUST be lowercase for matching below)
    known_vendors = ['nvidia', 'amd', 'ati', 'intel']

    devs = []
    try:
        lspci_out = __salt__['cmd.run']('lspci -vmm')

        cur_dev = {}
        error = False
        # Add a blank element to the lspci_out.splitlines() list,
        # otherwise the last device is not evaluated as a cur_dev and ignored.
        lspci_list = lspci_out.splitlines()
        lspci_list.append('')
        for line in lspci_list:
            # check for record-separating empty lines
            if line == '':
                if cur_dev.get('Class', '') == 'VGA compatible controller':
                    devs.append(cur_dev)
                # XXX; may also need to search for "3D controller"
                cur_dev = {}
                continue
            if re.match(r'^\w+:\s+.*', line):
                key, val = line.split(':', 1)
                cur_dev[key.strip()] = val.strip()
            else:
                error = True
                log.debug('Unexpected lspci output: \'{0}\''.format(line))

        if error:
            log.warn('Error loading grains, unexpected linux_gpu_data output, '
                     'check that you have a valid shell configured and '
                     'permissions to run lspci command')
    except OSError:
        pass

    gpus = []
    for gpu in devs:
        vendor_strings = gpu['Vendor'].lower().split()
        # default vendor to 'unknown', overwrite if we match a known one
        vendor = 'unknown'
        for name in known_vendors:
            # search for an 'expected' vendor name in the list of strings
            if name in vendor_strings:
                vendor = name
                break
        gpus.append({'vendor': vendor, 'model': gpu['Device']})

    grains = {}
    grains['num_gpus'] = len(gpus)
    grains['gpus'] = gpus
    return grains
예제 #33
0
파일: fscache.py 프로젝트: sylphid8/salt
    def run(self):
        '''
        Main loop of the FSCache, checks schedule, retrieves result-data
        from the workers and answer requests with data from the cache
        '''
        context = zmq.Context()
        # the socket for incoming cache requests
        creq_in = context.socket(zmq.REP)
        creq_in.setsockopt(zmq.LINGER, 100)
        creq_in.bind('ipc:///' + self.cache_sock)

        # the socket for incoming cache-updates from workers
        cupd_in = context.socket(zmq.REP)
        cupd_in.setsockopt(zmq.LINGER, 100)
        cupd_in.bind('ipc:///' + self.update_sock)

        # wait for the timer to bind to its socket
        log.debug('wait 2 secs for the timer')
        time.sleep(2)

        # the socket for the timer-event
        timer_in = context.socket(zmq.PULL)
        timer_in.setsockopt(zmq.LINGER, 100)
        timer_in.connect('ipc:///' + self.upd_t_sock)

        poller = zmq.Poller()
        poller.register(creq_in, zmq.POLLIN)
        poller.register(cupd_in, zmq.POLLIN)
        poller.register(timer_in, zmq.POLLIN)

        # our serializer
        serial = salt.payload.Serial(self.opts.get('serial', ''))

        # register a signal handler
        signal.signal(signal.SIGINT, self.signal_handler)

        # secure the sockets from the world
        self.secure()

        log.info('FSCache started')
        log.debug('FSCache started')

        while self.running:

            # we check for new events with the poller
            try:
                socks = dict(poller.poll())
            except KeyboardInterrupt:
                self.stop()
            except zmq.ZMQError as t:
                self.stop()

            # check for next cache-request
            if socks.get(creq_in) == zmq.POLLIN:
                msg = serial.loads(creq_in.recv())
                log.debug('Received request: {0}'.format(msg))

                # we only accept requests as lists [req_id, <path>]
                if isinstance(msg, list):
                    # for now only one item is assumed to be requested
                    msgid, file_n = msg[:]
                    log.debug('Looking for {0}:{1}'.format(msgid, file_n))

                    fdata = self.path_data.get(file_n, None)

                    if fdata is not None:
                        log.debug('Cache HIT')
                    else:
                        log.debug('Cache MISS')

                    # simulate slow caches
                    #randsleep = random.randint(0,3)
                    #time.sleep(randsleep)

                    # Send reply back to client
                    reply = serial.dumps([msgid, fdata])
                    creq_in.send(reply)

                # wrong format, item not cached
                else:
                    reply = serial.dumps([msgid, None])
                    creq_in.send(reply)

            # check for next cache-update from workers
            elif socks.get(cupd_in) == zmq.POLLIN:
                new_c_data = serial.loads(cupd_in.recv())
                # tell the worker to exit
                cupd_in.send(serial.dumps('OK'))

                # check if the returned data is usable
                if not isinstance(new_c_data, dict):
                    log.error('Worker returned unusable result')
                    del new_c_data
                    continue

                # the workers will return differing data:
                # 1. '{'file1': <data1>, 'file2': <data2>,...}' - a cache update
                # 2. '{search-path: None}' -  job was not run, pre-checks failed
                # 3. '{}' - no files found, check the pattern if defined?
                # 4. anything else is considered malformed

                if len(new_c_data) == 0:
                    log.debug('Got empty update from worker')
                elif new_c_data.values()[0] is not None:
                    log.debug('Got cache update with {0} item(s)'.format(
                        len(new_c_data)))
                    self.path_data.update(new_c_data)
                else:
                    log.debug('Got malformed result dict from worker')

                log.info('{0} entries in cache'.format(len(self.path_data)))

            # check for next timer-event to start new jobs
            elif socks.get(timer_in) == zmq.POLLIN:
                sec_event = serial.loads(timer_in.recv())

                log.debug('Timer event: #{0}'.format(sec_event))

                # loop through the jobs and start if a jobs ival matches
                for item in self.jobs:
                    if sec_event in self.jobs[item]['ival']:
                        self.run_job(item)
        self.stop()
        creq_in.close()
        cupd_in.close()
        timer_in.close()
        context.term()
        log.debug('Shutting down')\
예제 #34
0
    def _write_state(self):
        '''
        Writes a current status to the defined status-file
        this includes the current pid, events received/handled
        and threads created/joined
        '''
        ev_hdl_per_s = float((float(self.events_han - self.stat_hdl_count)) / float(self.state_timer_intrvl))
        ev_tot_per_s = float((float(self.events_rec - self.stat_rec_count)) / float(self.state_timer_intrvl))

        if self.config['stat_worker']:
            stat_data = {
                'events_rec': self.events_rec,
                'events_hdl': self.events_han,
                'events_hdl_sec': round(ev_hdl_per_s, 2),
                'events_tot_sec': round(ev_tot_per_s, 2),
                'threads_created': self.threads_cre,
                'threads_joined': self.threads_join
            }

            self.threads_cre += 1

            st_worker = SaltEventsdWorker(
                stat_data,
                self.threads_cre,
                None,
                self.backends,
                **self.opts
            )
            st_worker.start()

            try:
                self.running_workers.append(st_worker)
            except AttributeError:
                log.error('self is missing running_workers')
                try:
                    log.info(self)
                    log.info(dir(self))
                except Exception:
                    log.error('Failed to dump dir(self)')

        try:
            # write the info to the specified log
            statf = open(self.state_file, 'w')
            statf.writelines(
                json.dumps({
                    'events_rec': self.events_rec,
                    'events_hdl': self.events_han,
                    'events_hdl_sec': round(ev_hdl_per_s, 2),
                    'events_tot_sec': round(ev_tot_per_s, 2),
                    'threads_created': self.threads_cre,
                    'threads_joined': self.threads_join
                })
            )

            # if we have the same pid as the pidfile, we are the running daemon
            # and also print the current counters to the logfile with 'info'
            if os.getpid() == self.pid:
                log.info("Running with pid {0}".format(self.pid))
                log.info("Events (han/recv): {0}/{1}".format(
                    self.events_han,
                    self.events_rec,
                ))
                log.info("Threads (cre/joi):{0}/{1}".format(
                    self.threads_cre,
                    self.threads_join,
                ))

            statf.write("\n")
            statf.close()
            sys.stdout.flush()
        except IOError as ioerr:
            log.critical("Failed to write state to {0}".format(self.state_file))
            log.exception(ioerr)
        except OSError as oserr:
            log.critical("Failed to write state to {0}".format(self.state_file))
            log.exception(oserr)
        self.stat_rec_count = self.events_rec
        self.stat_hdl_count = self.events_han
예제 #35
0
    def tune_in(self):
        '''
        Lock onto the publisher. This is the main event loop for the minion
        '''
        log.info('{0} is starting as user \'{1}\''.format(
            self.__class__.__name__, getpass.getuser()))
        log.debug('Minion "{0}" trying to tune in'.format(self.opts['id']))
        self.context = zmq.Context()

        # Prepare the minion event system
        #
        # Start with the publish socket
        id_hash = hashlib.md5(self.opts['id']).hexdigest()
        epub_sock_path = os.path.join(
            self.opts['sock_dir'], 'minion_event_{0}_pub.ipc'.format(id_hash))
        epull_sock_path = os.path.join(
            self.opts['sock_dir'], 'minion_event_{0}_pull.ipc'.format(id_hash))
        self.epub_sock = self.context.socket(zmq.PUB)
        if self.opts.get('ipc_mode', '') == 'tcp':
            epub_uri = 'tcp://127.0.0.1:{0}'.format(self.opts['tcp_pub_port'])
            epull_uri = 'tcp://127.0.0.1:{0}'.format(
                self.opts['tcp_pull_port'])
        else:
            epub_uri = 'ipc://{0}'.format(epub_sock_path)
            salt.utils.check_ipc_path_max_len(epub_uri)
            epull_uri = 'ipc://{0}'.format(epull_sock_path)
            salt.utils.check_ipc_path_max_len(epull_uri)
        log.debug('{0} PUB socket URI: {1}'.format(self.__class__.__name__,
                                                   epub_uri))
        log.debug('{0} PULL socket URI: {1}'.format(self.__class__.__name__,
                                                    epull_uri))

        # Create the pull socket
        self.epull_sock = self.context.socket(zmq.PULL)
        # Bind the event sockets
        self.epub_sock.bind(epub_uri)
        self.epull_sock.bind(epull_uri)
        # Restrict access to the sockets
        if not self.opts.get('ipc_mode', '') == 'tcp':
            os.chmod(epub_sock_path, 448)
            os.chmod(epull_sock_path, 448)

        self.poller = zmq.Poller()
        self.epoller = zmq.Poller()
        self.socket = self.context.socket(zmq.SUB)
        self.socket.setsockopt(zmq.SUBSCRIBE, '')
        self.socket.setsockopt(zmq.IDENTITY, self.opts['id'])
        if hasattr(zmq, 'RECONNECT_IVL_MAX'):
            self.socket.setsockopt(zmq.RECONNECT_IVL_MAX,
                                   self.opts['recon_max'])
        if hasattr(zmq, 'TCP_KEEPALIVE'):
            self.socket.setsockopt(zmq.TCP_KEEPALIVE,
                                   self.opts['tcp_keepalive'])
            self.socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE,
                                   self.opts['tcp_keepalive_idle'])
            self.socket.setsockopt(zmq.TCP_KEEPALIVE_CNT,
                                   self.opts['tcp_keepalive_cnt'])
            self.socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL,
                                   self.opts['tcp_keepalive_intvl'])
        if hasattr(zmq, 'IPV4ONLY'):
            self.socket.setsockopt(
                zmq.IPV4ONLY,
                int(not int(self.opts.get('ipv6_enable', False))))
        self.socket.connect(self.master_pub)
        self.poller.register(self.socket, zmq.POLLIN)
        self.epoller.register(self.epull_sock, zmq.POLLIN)
        # Send an event to the master that the minion is live
        self._fire_master(
            'Minion {0} started at {1}'.format(self.opts['id'],
                                               time.asctime()), 'minion_start')

        if self.opts['multiprocessing'] and not salt.utils.is_windows():
            signal.signal(signal.SIGCHLD, self.handle_sigchld)
        # Make sure to gracefully handle SIGUSR1
        enable_sigusr1_handler()

        # On first startup execute a state run if configured to do so
        self._state_run()

        while True:
            try:
                self.schedule.eval()
                socks = dict(
                    self.poller.poll(self.opts['loop_interval'] * 1000))
                if self.socket in socks and socks[self.socket] == zmq.POLLIN:
                    payload = self.serial.loads(self.socket.recv())
                    self._handle_payload(payload)
                time.sleep(0.05)
                # Clean up the minion processes which have been executed and
                # have finished
                # Check if modules and grains need to be refreshed
                self.passive_refresh()
                # Check the event system
                if self.epoller.poll(1):
                    try:
                        package = self.epull_sock.recv(zmq.NOBLOCK)
                        self.epub_sock.send(package)
                    except Exception:
                        pass
            except zmq.ZMQError:
                # This is thrown by the inturupt caused by python handling the
                # SIGCHLD. This is a safe error and we just start the poll
                # again
                continue
            except Exception:
                log.critical(traceback.format_exc())
예제 #36
0
파일: core.py 프로젝트: penta-srl/salt
def _linux_gpu_data():
    """
    num_gpus: int
    gpus:
      - vendor: nvidia|amd|ati|...
        model: string
    """
    lspci = salt.utils.which("lspci")
    if not lspci:
        log.info("The `lspci` binary is not available on the system. GPU grains " "will not be available.")
        return {}

    elif __opts__.get("enable_gpu_grains", None) is False:
        log.info(
            "Skipping lspci call because enable_gpu_grains was set to False "
            "in the config. GPU grains will not be available."
        )
        return {}

    # dominant gpu vendors to search for (MUST be lowercase for matching below)
    known_vendors = ["nvidia", "amd", "ati", "intel"]

    devs = []
    try:
        lspci_out = __salt__["cmd.run"]("lspci -vmm")

        cur_dev = {}
        error = False
        # Add a blank element to the lspci_out.splitlines() list,
        # otherwise the last device is not evaluated as a cur_dev and ignored.
        lspci_list = lspci_out.splitlines()
        lspci_list.append("")
        for line in lspci_list:
            # check for record-separating empty lines
            if line == "":
                if cur_dev.get("Class", "") == "VGA compatible controller":
                    devs.append(cur_dev)
                # XXX; may also need to search for "3D controller"
                cur_dev = {}
                continue
            if re.match(r"^\w+:\s+.*", line):
                key, val = line.split(":", 1)
                cur_dev[key.strip()] = val.strip()
            else:
                error = True
                log.debug("Unexpected lspci output: '{0}'".format(line))

        if error:
            log.warn(
                "Error loading grains, unexpected linux_gpu_data output, "
                "check that you have a valid shell configured and "
                "permissions to run lspci command"
            )
    except OSError:
        pass

    gpus = []
    for gpu in devs:
        vendor_strings = gpu["Vendor"].lower().split()
        # default vendor to 'unknown', overwrite if we match a known one
        vendor = "unknown"
        for name in known_vendors:
            # search for an 'expected' vendor name in the list of strings
            if name in vendor_strings:
                vendor = name
                break
        gpus.append({"vendor": vendor, "model": gpu["Device"]})

    grains = {}
    grains["num_gpus"] = len(gpus)
    grains["gpus"] = gpus
    return grains
예제 #37
0
    def __init__(self):

        self.opts = SaltEventsdLoader().getopts()

        self._pre_startup(self.opts)

        if type(self.opts) is not dict:
            log.info("Received invalid configdata, startup cancelled")
            sys.exit(1)

        self.config = self.opts['general']
        super(SaltEventsDaemon, self).__init__(self.config['pidfile'])

	# the map of events are stored here, loaded in _init_events()
	self.event_map = None
        self._init_events(self.opts['events'])

        self.backends = self._init_backends( self.config['backends'] )
        log.info(self.backends)

        # the socket to listen on for the events
        self.sock_dir = self.config['sock_dir']

        # two possible values for 'node': master and minion
        # they do the same thing, just on different sockets
        self.node = self.config['node']

        # the id, usually 'master'
        self.nodeid = self.config['id']

        # the statefile where we write the daemon status
        self.state_file = self.config['state_file']
        # how many events to handle before updating the status
        self.state_upd = self.config['state_upd']
        # we dont know our pid (yet)
        self.pid = None

        # how many parallel workers to start max
        self.max_workers = self.config['max_workers']

        # the number of events to collect before starting a worker
        self.event_limit = self.config['event_limit']

        # a list to keep track of the currently running workers
        # this is mainly for debugging to check wether all started
        # workers are correctly joined over time so we dont leak memory
        self.running_workers = []

        # setup some counters used for the status
        self.events_han = 0
        self.events_rec = 0
        self.threads_cre = 0
        self.threads_join = 0

        # the timer thats write data to the database every x seconds
        # this is used to push data into the database even if 
        # self.event_limit is not reached regularly
        self.ev_timer_ev = False
        self.ev_timer_intrvl = self.config['dump_timer']
        self.ev_timer = ResetTimer(self.ev_timer_intrvl, 
                                 self)
예제 #38
0
    def listen(self):
        '''
        the main event loop where we receive the events and
        start the workers that dump our data into the database
        '''
        # log on to saltstacks event-bus
        event = salt.utils.event.SaltEvent(self.node,
                                           self.sock_dir)

        # we store our events in a list, we dont really care about an order
        # or what kind of data is put in there. all that is configured with the
        # sql-template configured in the configfile
        event_queue = []

        # start our dump_timer
        self.ev_timer.start()

        # this is for logline chronology so the timer-message always comes
        # _before_ the actual startup-message of the listening loop below :-)
        time.sleep(1)

        log.info("entering main event loop")
        log.info("listening on: {0}".format(event.puburi))

        # read everything we can get our hands on
        while True:

            # the zmq-socket does not like ^C very much, make the error
            # a little more graceful. alright, alright, ignore the damn thing,
            # we're exiting anyways...
            try:
                ret = event.get_event(full=True)
            except zmq.ZMQError:
                pass

            if ret is None:
                continue

            # if the timer has expired, we may have not received enough
            # events in the queue to reach event_limit, in that case we dump
            # the data anyway to have it in the database
            if(self.ev_timer_ev):
                if (len(self.running_workers) < self.max_workers) and \
                   (len(event_queue) > 0):

                    self._init_worker(event_queue)

                    # reset our queue to prevent duplicate entries
                    del event_queue[:]

                    # we reset the timer.ev_timer_ev  at the end of the loop 
                    # so we can update the stats that are logged


            # filter only the events we're interested in. all events have a tag 
            # we can filter them by. we match with a precompiled regex
            if( 'tag' in ret ):

                # filter out events with an empty tag. those are special
                if( ret['tag'] != '' ):
                       
                    # run through our configured events and try to match the 
                    # current events tag against the ones we're interested in
                    for key in self.event_map.keys():
                        if( self.event_map[key]['tag'].match(ret['tag'])):
                            log.debug("matching on {0}:{1}".format(key, 
                                                                   ret['tag']))

                            prio = self.event_map[key].get('prio', 0)

                            # push prio1-events directly into a worker
                            if prio > 0:
                                log.debug('Prio1 event found, pushing immediately!')
                                self.events_han += 1
                                self._init_worker([ret])
                            else:
                                event_queue.append(ret)
                                self.events_han += 1

            # once we reach the event_limit, start a worker that
            # writes that data in to the database
            if len(event_queue) >= self.event_limit:

                # only start a worker if not too many workers are running
                if len(self.running_workers) < self.max_workers:
                    self._init_worker(event_queue)
                    # reset the timer
                    self.ev_timer.reset()

                    # reset our queue to prevent duplicate entries
                    del event_queue[:]

                else:
                    # FIXME: we need to handle this situation somehow if
                    # too many workers are running. just flush the events?
                    # there really is no sane way except queueing more and more
                    # until some sort of limit is reached and we care more about
                    # our saltmaster than about the collected events!
                    log.critical("too many workers running, loosing data!!!")
                   
            # a list for the workers that are still running
            clean_workers = []

            # run through all the workers and join() the ones
            # that have finished dumping their data and keep
            # the running ones on our list
            for worker in self.running_workers:
                if worker.isAlive():
                    clean_workers.append(worker)
                else:
                    worker.join()
                    log.debug("joined worker #{0}".format(worker.getName()))
                    self.threads_join += 1

            # get rid of the old reference  and set a new one
            # FIXME: is this really neccessary?
            del self.running_workers

            self.running_workers = clean_workers
            self.events_rec += 1

            # we update the stats every 'received div handled == 0'
            # or if we recevied a timer event from our ResetTimer
            if( (self.events_rec % self.state_upd) == 0 ):
                self._write_state()
            elif(self.ev_timer_ev):
                self._write_state()
                self.ev_timer_ev = False

        log.info("listen loop ended...")                
예제 #39
0
    def run(self):
        '''
        Main loop of the ConCache, starts updates in intervals and
        answers requests from the MWorkers
        '''
        context = zmq.Context()
        # the socket for incoming cache requests
        creq_in = context.socket(zmq.REP)
        creq_in.setsockopt(zmq.LINGER, 100)
        creq_in.bind('ipc://' + self.cache_sock)

        # the socket for incoming cache-updates from workers
        cupd_in = context.socket(zmq.SUB)
        cupd_in.setsockopt(zmq.SUBSCRIBE, b'')
        cupd_in.setsockopt(zmq.LINGER, 100)
        cupd_in.bind('ipc://' + self.update_sock)

        # the socket for the timer-event
        timer_in = context.socket(zmq.SUB)
        timer_in.setsockopt(zmq.SUBSCRIBE, b'')
        timer_in.setsockopt(zmq.LINGER, 100)
        timer_in.connect('ipc://' + self.upd_t_sock)

        poller = zmq.Poller()
        poller.register(creq_in, zmq.POLLIN)
        poller.register(cupd_in, zmq.POLLIN)
        poller.register(timer_in, zmq.POLLIN)

        # our serializer
        serial = salt.payload.Serial(self.opts.get('serial', ''))

        # register a signal handler
        signal.signal(signal.SIGINT, self.signal_handler)

        # secure the sockets from the world
        self.secure()

        log.info('ConCache started')

        while self.running:

            # we check for new events with the poller
            try:
                socks = dict(poller.poll(1))
            except KeyboardInterrupt:
                self.stop()
            except zmq.ZMQError as zmq_err:
                log.error('ConCache ZeroMQ-Error occurred')
                log.exception(zmq_err)
                self.stop()

            # check for next cache-request
            if socks.get(creq_in) == zmq.POLLIN:
                msg = serial.loads(creq_in.recv())
                log.debug('ConCache Received request: %s', msg)

                # requests to the minion list are send as str's
                if isinstance(msg, six.string_types):
                    if msg == 'minions':
                        # Send reply back to client
                        reply = serial.dumps(self.minions)
                        creq_in.send(reply)

            # check for next cache-update from workers
            if socks.get(cupd_in) == zmq.POLLIN:
                new_c_data = serial.loads(cupd_in.recv())
                # tell the worker to exit
                #cupd_in.send(serial.dumps('ACK'))

                # check if the returned data is usable
                if not isinstance(new_c_data, list):
                    log.error('ConCache Worker returned unusable result')
                    del new_c_data
                    continue

                # the cache will receive lists of minions
                # 1. if the list only has 1 item, its from an MWorker, we append it
                # 2. if the list contains another list, its from a CacheWorker and
                #    the currently cached minions are replaced with that list
                # 3. anything else is considered malformed

                try:

                    if not new_c_data:
                        log.debug('ConCache Got empty update from worker')
                        continue

                    data = new_c_data[0]

                    if isinstance(data, six.string_types):
                        if data not in self.minions:
                            log.debug('ConCache Adding minion %s to cache',
                                      new_c_data[0])
                            self.minions.append(data)

                    elif isinstance(data, list):
                        log.debug('ConCache Replacing minion list from worker')
                        self.minions = data

                except IndexError:
                    log.debug('ConCache Got malformed result dict from worker')
                    del new_c_data

                log.info('ConCache %s entries in cache', len(self.minions))

            # check for next timer-event to start new jobs
            if socks.get(timer_in) == zmq.POLLIN:
                sec_event = serial.loads(timer_in.recv())

                # update the list every 30 seconds
                if int(sec_event % 30) == 0:
                    cw = CacheWorker(self.opts)
                    cw.start()

        self.stop()
        creq_in.close()
        cupd_in.close()
        timer_in.close()
        context.term()
        log.debug('ConCache Shutting down')
예제 #40
0
파일: config.py 프로젝트: jslatts/salt
def get_id():
    """
    Guess the id of the minion.

    - Check /etc/hostname for a value other than localhost
    - If socket.getfqdn() returns us something other than localhost, use it
    - Check /etc/hosts for something that isn't localhost that maps to 127.*
    - Look for a routeable / public IP
    - A private IP is better than a loopback IP
    - localhost may be better than killing the minion

    Returns two values: the detected ID, and a boolean value noting whether or
    not an IP address is being used for the ID.
    """

    log.debug("Guessing ID. The id can be explicitly in set {0}".format(os.path.join(syspaths.CONFIG_DIR, "minion")))

    # Check /etc/hostname
    try:
        with salt.utils.fopen("/etc/hostname") as hfl:
            name = hfl.read().strip()
        if re.search(r"\s", name):
            log.warning(
                "Whitespace character detected in /etc/hostname. " "This file should not contain any whitespace."
            )
        else:
            if name != "localhost":
                return name, False
    except Exception:
        pass

    # Nothing in /etc/hostname or /etc/hostname not found
    fqdn = socket.getfqdn()
    if fqdn != "localhost":
        log.info("Found minion id from getfqdn(): {0}".format(fqdn))
        return fqdn, False

    # Can /etc/hosts help us?
    try:
        with salt.utils.fopen("/etc/hosts") as hfl:
            for line in hfl:
                names = line.split()
                ip_ = names.pop(0)
                if ip_.startswith("127."):
                    for name in names:
                        if name != "localhost":
                            log.info("Found minion id in hosts file: {0}".format(name))
                            return name, False
    except Exception:
        pass

    # Can Windows 'hosts' file help?
    try:
        windir = os.getenv("WINDIR")
        with salt.utils.fopen(windir + "\\system32\\drivers\\etc\\hosts") as hfl:
            for line in hfl:
                # skip commented or blank lines
                if line[0] == "#" or len(line) <= 1:
                    continue
                # process lines looking for '127.' in first column
                try:
                    entry = line.split()
                    if entry[0].startswith("127."):
                        for name in entry[1:]:  # try each name in the row
                            if name != "localhost":
                                log.info("Found minion id in hosts file: {0}".format(name))
                                return name, False
                except IndexError:
                    pass  # could not split line (malformed entry?)
    except Exception:
        pass

    # What IP addresses do we have?
    ip_addresses = [
        salt.utils.network.IPv4Address(addr)
        for addr in salt.utils.network.ip_addrs(include_loopback=True)
        if not addr.startswith("127.")
    ]

    for addr in ip_addresses:
        if not addr.is_private:
            log.info("Using public ip address for id: {0}".format(addr))
            return str(addr), True

    if ip_addresses:
        addr = ip_addresses.pop(0)
        log.info("Using private ip address for id: {0}".format(addr))
        return str(addr), True

    log.error("No id found, falling back to localhost")
    return "localhost", False
예제 #41
0
파일: config.py 프로젝트: micahhausler/salt
def get_id():
    '''
    Guess the id of the minion.

    - Check /etc/hostname for a value other than localhost
    - If socket.getfqdn() returns us something other than localhost, use it
    - Check /etc/hosts for something that isn't localhost that maps to 127.*
    - Look for a routeable / public IP
    - A private IP is better than a loopback IP
    - localhost may be better than killing the minion

    Returns two values: the detected ID, and a boolean value noting whether or
    not an IP address is being used for the ID.
    '''

    log.debug('Guessing ID. The id can be explicitly in set {0}'.format(
        os.path.join(syspaths.CONFIG_DIR, 'minion')))

    # Check /etc/hostname
    try:
        with salt.utils.fopen('/etc/hostname') as hfl:
            name = hfl.read().strip()
        if re.search(r'\s', name):
            log.warning('Whitespace character detected in /etc/hostname. '
                        'This file should not contain any whitespace.')
        else:
            if name != 'localhost':
                return name, False
    except Exception:
        pass

    # Nothing in /etc/hostname or /etc/hostname not found
    fqdn = socket.getfqdn()
    if fqdn != 'localhost':
        log.info('Found minion id from getfqdn(): {0}'.format(fqdn))
        return fqdn, False

    # Can /etc/hosts help us?
    try:
        with salt.utils.fopen('/etc/hosts') as hfl:
            for line in hfl:
                names = line.split()
                ip_ = names.pop(0)
                if ip_.startswith('127.'):
                    for name in names:
                        if name != 'localhost':
                            log.info(
                                'Found minion id in hosts file: {0}'.format(
                                    name))
                            return name, False
    except Exception:
        pass

    # Can Windows 'hosts' file help?
    try:
        windir = os.getenv("WINDIR")
        with salt.utils.fopen(windir +
                              '\\system32\\drivers\\etc\\hosts') as hfl:
            for line in hfl:
                # skip commented or blank lines
                if line[0] == '#' or len(line) <= 1:
                    continue
                # process lines looking for '127.' in first column
                try:
                    entry = line.split()
                    if entry[0].startswith('127.'):
                        for name in entry[1:]:  # try each name in the row
                            if name != 'localhost':
                                log.info('Found minion id in hosts file: {0}'.
                                         format(name))
                                return name, False
                except IndexError:
                    pass  # could not split line (malformed entry?)
    except Exception:
        pass

    # What IP addresses do we have?
    ip_addresses = [
        salt.utils.network.IPv4Address(addr)
        for addr in salt.utils.network.ip_addrs(include_loopback=True)
        if not addr.startswith('127.')
    ]

    for addr in ip_addresses:
        if not addr.is_private:
            log.info('Using public ip address for id: {0}'.format(addr))
            return str(addr), True

    if ip_addresses:
        addr = ip_addresses.pop(0)
        log.info('Using private ip address for id: {0}'.format(addr))
        return str(addr), True

    log.error('No id found, falling back to localhost')
    return 'localhost', False
예제 #42
0
파일: core.py 프로젝트: 1mentat/salt
def _linux_gpu_data():
    '''
    num_gpus: int
    gpus:
      - vendor: nvidia|amd|ati|...
        model: string
    '''
    lspci = salt.utils.which('lspci')
    if not lspci:
        log.info(
            'The `lspci` binary is not available on the system. GPU grains '
            'will not be available.'
        )
        return {}

    elif __opts__.get('enable_gpu_grains', None) is False:
        log.info(
            'Skipping lspci call because enable_gpu_grains was set to False '
            'in the config. GPU grains will not be available.'
        )
        return {}

    # dominant gpu vendors to search for (MUST be lowercase for matching below)
    known_vendors = ['nvidia', 'amd', 'ati', 'intel']

    devs = []
    try:
        lspci_out = __salt__['cmd.run']('lspci -vmm')

        cur_dev = {}
        error = False
        # Add a blank element to the lspci_out.splitlines() list,
        # otherwise the last device is not evaluated as a cur_dev and ignored.
        lspci_list = lspci_out.splitlines()
        lspci_list.append('')
        for line in lspci_list:
            # check for record-separating empty lines
            if line == '':
                if cur_dev.get('Class', '') == 'VGA compatible controller':
                    devs.append(cur_dev)
                # XXX; may also need to search for "3D controller"
                cur_dev = {}
                continue
            if re.match(r'^\w+:\s+.*', line):
                key, val = line.split(':', 1)
                cur_dev[key.strip()] = val.strip()
            else:
                error = True
                log.debug('Unexpected lspci output: \'{0}\''.format(line))

        if error:
            log.warn(
                'Error loading grains, unexpected linux_gpu_data output, '
                'check that you have a valid shell configured and '
                'permissions to run lspci command'
            )
    except OSError:
        pass

    gpus = []
    for gpu in devs:
        vendor_strings = gpu['Vendor'].lower().split()
        # default vendor to 'unknown', overwrite if we match a known one
        vendor = 'unknown'
        for name in known_vendors:
            # search for an 'expected' vendor name in the list of strings
            if name in vendor_strings:
                vendor = name
                break
        gpus.append({'vendor': vendor, 'model': gpu['Device']})

    grains = {}
    grains['num_gpus'] = len(gpus)
    grains['gpus'] = gpus
    return grains
예제 #43
0
    def _write_state(self):
        '''
        Writes a current status to the defined status-file
        this includes the current pid, events received/handled
        and threads created/joined
        '''
        ev_hdl_per_s = float((float(self.events_han - self.stat_hdl_count)) /
                             float(self.state_timer_intrvl))
        ev_tot_per_s = float((float(self.events_rec - self.stat_rec_count)) /
                             float(self.state_timer_intrvl))

        if self.config['stat_worker']:
            stat_data = {
                'events_rec': self.events_rec,
                'events_hdl': self.events_han,
                'events_hdl_sec': round(ev_hdl_per_s, 2),
                'events_tot_sec': round(ev_tot_per_s, 2),
                'threads_created': self.threads_cre,
                'threads_joined': self.threads_join
            }

            self.threads_cre += 1

            st_worker = SaltEventsdWorker(stat_data, self.threads_cre, None,
                                          self.backends, **self.opts)
            st_worker.start()

            try:
                self.running_workers.append(st_worker)
            except AttributeError:
                log.error('self is missing running_workers')
                try:
                    log.info(self)
                    log.info(dir(self))
                except Exception:
                    log.error('Failed to dump dir(self)')

        try:
            # write the info to the specified log
            statf = open(self.state_file, 'w')
            statf.writelines(
                json.dumps({
                    'events_rec': self.events_rec,
                    'events_hdl': self.events_han,
                    'events_hdl_sec': round(ev_hdl_per_s, 2),
                    'events_tot_sec': round(ev_tot_per_s, 2),
                    'threads_created': self.threads_cre,
                    'threads_joined': self.threads_join
                }))

            # if we have the same pid as the pidfile, we are the running daemon
            # and also print the current counters to the logfile with 'info'
            if os.getpid() == self.pid:
                log.info("Running with pid {0}".format(self.pid))
                log.info("Events (han/recv): {0}/{1}".format(
                    self.events_han,
                    self.events_rec,
                ))
                log.info("Threads (cre/joi):{0}/{1}".format(
                    self.threads_cre,
                    self.threads_join,
                ))

            statf.write("\n")
            statf.close()
            sys.stdout.flush()
        except IOError as ioerr:
            log.critical("Failed to write state to {0}".format(
                self.state_file))
            log.exception(ioerr)
        except OSError as oserr:
            log.critical("Failed to write state to {0}".format(
                self.state_file))
            log.exception(oserr)
        self.stat_rec_count = self.events_rec
        self.stat_hdl_count = self.events_han