def get_msg(self, timestamp=None): ''' Remove and return the first (oldest) message in the recipient's queue. If there are no messages in the queue, return None. If the number of messages in the queue drops to zero, remove the recipient from the pending set. >>> pending = set() >>> r = Recipient('*****@*****.**', pending=pending) >>> r.state = READY >>> r.add_msg('msg 1') >>> r.add_msg('msg 2') # get and send messages >>> while r.msgs: ... print r.get_msg() ... print pending msg 1 set([[email protected]]) msg 2 set([]) ''' msg = None self.expire_msgs(timestamp) if self.msgs: created, msg = self.msgs.popleft() if self.readd_idx > 0: self.readd_idx -= 1 if self.pending is not None and len(self.msgs) == 0: log.trace('remove %s from pending', self.addr) self.pending.discard(self) return msg
def main(): """ Start computer service. .. code-block:: bash hs-computer --server|--node|--all """ salt.log.setup_console_logger() parser = option_parser() shell = Shell(parser) try: shell.exec_command() except Exception as err: log.error("shell command failed: {0!r}".format( err) ) except KeyboardInterrupt: raise SystemExit('\nExiting gracefully on Ctrl-c') finally: # TODO: need cleanup log.trace("TODO cleanup work") return
def add_subscriber(self, regex, addr): ''' ''' log.trace('add %s subscriber: pattern="%s" address="%s"', self.protocol, regex.pattern, addr) if regex not in self.distrib_lists: self.distrib_lists[regex] = set() self.distrib_lists[regex].add(self._parse_subscriber(addr))
def _load_time(self, config): ''' Load the time format and timezone from data in /etc/salt/alert. ''' timecfg = config.get('alert.time', {}) timedefs = (timecfg.get('format', STRFTIME_DEFAULT), timecfg.get('timezone', TIMEZONE_DEFAULT)) log.trace('alert time: format="%s" timezone="%s"', *timedefs) return timedefs
def _load_verbs(self, config): ''' Load the preferred raised and cleared verbs from /etc/salt/alert. ''' verbs = VERBS_DEFAULT.copy() for verb, preferred in config.get('alert.verbs', {}).iteritems(): if preferred: verbs[verb] = preferred log.trace('alert verbs: %s', verbs) return verbs
def __init__(self, protocol, config): ''' Configure the agent from YAML data parsed from /etc/salt/alert. ''' Agent.__init__(self, protocol) user = config.get('user') password = config.get('password') sleekxmpp.ClientXMPP.__init__(self, user, password) self.auto_subscribe = False self.auto_authorize = None server = config.get('host') if server: self.server_addr = (server, config.get('port', DEFAULT_PORT)) else: self.server_addr = (self.boundjid.host, DEFAULT_PORT) self.max_msgs = config.get('max_msgs', DEFAULT_MAX_MSGS) self.max_age = config.get('max_age', DEFAULT_MAX_AGE) log.trace('connect to %s as %s/%s', self.server_addr, user, password) self.connected = False self.message = string.Template(config.get('message', DEFAULT_MESSAGE)) self.pending = set() self.recipients = {} self.service_down = False self.retry_service_wait = 60 self.last_send_time = 0 self.throttle_wait = False msgs_per_sec = config.get('msgs_per_sec', 0) if msgs_per_sec <= 0: self.send_interval = 0 else: self.send_interval = 1.0 / msgs_per_sec self.add_event_handler('presence_subscribe', self.__presence) self.add_event_handler('presence_subscribed', self.__presence) self.add_event_handler('presence_unsubscribe', self.__presence) self.add_event_handler('presence_unsubscribed', self.__presence) self.add_event_handler('presence_available', self.__presence) self.add_event_handler('presence_error', self.__presence) self.add_event_handler('changed_subscription', self.__subscription) self.add_event_handler('message', self.__message) self.add_event_handler('roster_update', self.__roster) self.add_event_handler('session_start', self.__start) self.register_plugin('xep_0030') # Service Discovery self.register_plugin('xep_0199') # XMPP Ping self.register_plugin('xep_0086') # Legacy Errors
def __subscription(self, event): ''' Handle peers subscribing and unsubscribing to us. ''' addr = event.get_from().bare etype = event.get_type() log.trace('_subscription: addr=%s type=%s', addr, etype) recipient = self.recipients.get(addr) if recipient: if etype == 'subscribed': # Send authorization back to peer self.send_presence(pto=addr, ptype='subscribed')
def _deliver(self, subscribers, alert): ''' ''' log.trace('_deliver: %s', alert) if not self.connected: self.__connect() self.connected = True timestamp = time.time() msg = self.message.safe_substitute(alert) for recipient in subscribers: log.trace('queue message to %s: %s message(s) pending', recipient.addr, len(recipient.msgs)) recipient.add_msg(msg, timestamp) self.__pending()
def clear_socket(self): ''' delete socket if you have it ''' if hasattr(self, '_socket'): if isinstance(self.poller.sockets, dict): sockets = list(self.poller.sockets.keys()) for socket in sockets: log.trace('Unregistering socket: %s', socket) self.poller.unregister(socket) else: for socket in self.poller.sockets: log.trace('Unregistering socket: %s', socket) self.poller.unregister(socket[0]) del self._socket
def __pending(self): ''' Send pending messages. ''' log.trace('_pending: %s recipients have msgs to send', len(self.pending)) while self.pending: for recipient in self.recipients.values(): if self.__throttled(): return if len(recipient.msgs) > 0: addr = recipient.addr msg = recipient.get_msg() log.trace('send to %s: %s', addr, msg) self.send_message(mto=addr, mbody=msg, mtype='chat')
def clear_socket(self): ''' delete socket if you have it ''' if hasattr(self, '_socket'): if isinstance(self.poller.sockets, dict): sockets = list(self.poller.sockets.keys()) for socket in sockets: log.trace('Unregistering socket: {0}'.format(socket)) self.poller.unregister(socket) else: for socket in self.poller.sockets: log.trace('Unregistering socket: {0}'.format(socket)) self.poller.unregister(socket[0]) del self._socket
def clear_socket(self): """ delete socket if you have it """ if hasattr(self, "_socket"): if isinstance(self.poller.sockets, dict): sockets = list(self.poller.sockets.keys()) for socket in sockets: log.trace("Unregistering socket: %s", socket) self.poller.unregister(socket) else: for socket in self.poller.sockets: log.trace("Unregistering socket: %s", socket) self.poller.unregister(socket[0]) del self._socket
def __roster(self, event): ''' At startup, set the recipient's state based on the roster. ''' if log.isEnabledFor(salt.log.TRACE): roster = [addr for addr in self.client_roster] log.trace('roster changed: %s', roster) for recipient in self.recipients.values(): if recipient.state == UNKNOWN: if recipient.addr in self.client_roster: roster_item = self.client_roster[recipient.addr] else: roster_item = None self.__set_state(recipient, roster_item) # self.del_event_handler('roster_update', self.__roster) self.__pending()
class MonitorTask(object): ''' A single monitor task. ''' def __init__(self, taskid, pyexe, context, scheduler=None): self.taskid = taskid self.code = pyexe self.context = context self.scheduler = scheduler def run(self): log.trace('start thread for %s', self.taskid) minion = self.context.get('id') collector = self.context.get('collector') while True: try: exec self.code in self.context except Exception, ex: log.error("can't execute %s: %s", self.taskid, ex, exc_info=ex) if collector: jid = datetime.datetime.strftime( datetime.datetime.now(), 'M%Y%m%d%H%M%S%f') try: collector(minion, self.context['cmd'], self.context['result']) except Exception, ex: log.error('monitor error: %s', self.taskid, exc_info=ex) if self.scheduler is None: break duration = self.scheduler.next() log.trace('%s: sleep %s seconds', self.taskid, duration) time.sleep(duration)
def run(self): log.trace('start thread for %s', self.taskid) minion = self.context.get('id') collector = self.context.get('collector') while True: try: exec self.code in self.context except Exception, ex: log.error("can't execute %s: %s", self.taskid, ex, exc_info=ex) if collector: jid = datetime.datetime.strftime( datetime.datetime.now(), 'M%Y%m%d%H%M%S%f') try: collector(minion, self.context['cmd'], self.context['result']) except Exception, ex: log.error('monitor error: %s', self.taskid, exc_info=ex)
def _load_msg_config(self, config): ''' Load the email message configuration from /etc/salt/alert. ''' self.sender = config.get('from', DEFAULT_SENDER) self.headers = config.get('headers', DEFAULT_HEADERS) self.subject = string.Template(config.get('subject', DEFAULT_SUBJECT)) self.body = string.Template(config.get('body', DEFAULT_BODY)) if log.isEnabledFor(salt.log.TRACE): log.trace( '''email alert message: from: %s subject: %s headers: %s body: %s''', self.sender, self.subject.safe_substitute({}), self.headers, '\n '.join(self.body.safe_substitute({}).splitlines()))
def __presence(self, event): ''' Handle peers subscribing and unsubscribing to us. ''' addr = event.get_from().bare etype = event.get_type() log.trace('_presence: addr=%s type=%s', addr, etype) recipient = self.recipients.get(addr) if recipient: if etype == 'unsubscribed': recipient.state = READY self.__set_state(recipient) elif etype == 'subscribe': self.send_presence(pto=addr, ptype='subscribed') elif etype in ['subscribed', 'available']: recipient.state = READY self.__pending()
def _expand_tasks(self, parsed_yaml): ''' Assemble compiled code from the configuration described by python dictionaries and lists. ''' results = [] for tasknum, taskdict in enumerate(parsed_yaml, 1): try: log.trace(taskdict) taskid = taskdict.get('id', 'monitor-{}'.format(tasknum)) pysrc = self._expand_task(taskid, taskdict) log.trace("generated '%s' task source:\n%s", taskid, pysrc) pyexe = compile(pysrc, '<monitor-config>', 'exec') scheduler = self._expand_scheduler(taskdict) results.append( MonitorTask(taskid, pyexe, self.context, scheduler)) except ValueError, ex: log.error('ignore monitor command #{} {!r}: {}'.format( tasknum, taskdict.get('run', '<unknown>'), ex))
def _load_msg_config(self, config): """ Load the email message configuration from /etc/salt/alert. """ self.sender = config.get("from", DEFAULT_SENDER) self.headers = config.get("headers", DEFAULT_HEADERS) self.subject = string.Template(config.get("subject", DEFAULT_SUBJECT)) self.body = string.Template(config.get("body", DEFAULT_BODY)) if log.isEnabledFor(salt.log.TRACE): log.trace( """email alert message: from: %s subject: %s headers: %s body: %s""", self.sender, self.subject.safe_substitute({}), self.headers, "\n ".join(self.body.safe_substitute({}).splitlines()), )
def _expand_tasks(self, parsed_yaml): ''' Assemble compiled code from the configuration described by python dictionaries and lists. ''' results = [] for tasknum, taskdict in enumerate(parsed_yaml, 1): try: log.trace(taskdict) taskid = taskdict.get('id', 'monitor-{}'.format(tasknum)) pysrc = self._expand_task(taskid, taskdict) log.trace("generated '%s' task source:\n%s", taskid, pysrc) pyexe = compile(pysrc, '<monitor-config>', 'exec') scheduler = self._expand_scheduler(taskdict) results.append(MonitorTask(taskid, pyexe, self.context, scheduler)) except ValueError, ex: log.error( 'ignore monitor command #{} {!r}: {}'.format( tasknum, taskdict.get('run', '<unknown>'), ex ) )
def which(exe=None): ''' Python clone of /usr/bin/which ''' if exe: if os.access(exe, os.X_OK): return exe # default path based on busybox's default default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' search_path = os.environ.get('PATH', default_path) for path in search_path.split(os.pathsep): full_path = os.path.join(path, exe) if os.access(full_path, os.X_OK): return full_path log.trace('{0!r} could not be found in the following search ' 'path: {1!r}'.format(exe, search_path)) log.trace('No executable was passed to be searched by which') return None
def _load_smtp_config(self, config): ''' Load the smtp configuration from /etc/salt/alert. ''' smtp_config = config.get('smtp') if not smtp_config: raise ValueError('alert.email config missing "smtp" options') self.server = smtp_config.get('host') self.port = smtp_config.get('port', DEFAULT_PORT) self.user = smtp_config.get('user', DEFAULT_USER) self.password = smtp_config.get('password', DEFAULT_PASSWORD) if not self.server: raise ValueError('alert.email.smtp config missing or ' 'blank "host" option') log.trace( '''email alert smtp: server: %s port: %s user: %s password: %s''', self.server, self.port, self.user, self.password)
def load(self, config): ''' Load the alert agents, subscriptions, and miscellaneous data from structures generated from YAML in /etc/salt/alert. ''' if not isinstance(config, dict): raise ValueError('expected config dict, not %s', type(config)) self.agents = salt.ext.alert.agents.load_agents(config) self.timeformat, timezone = self._load_time(config) self.verbs = self._load_verbs(config) self._load_subscriptions(config, self.agents) log.debug('set timezone to %s', timezone) os.environ['TZ'] = timezone time.tzset() # remove agents that have no subscribers for protocol in list(self.agents.keys()): if not self.agents[protocol].has_subscribers(): log.trace('remove %s agent: no subscribers defined', protocol) del self.agents[protocol]
def get_running_jobs(opts): """ Return the running jobs on this minion """ ret = [] proc_dir = os.path.join(opts["cachedir"], "proc") if not os.path.isdir(proc_dir): return ret for fn_ in os.listdir(proc_dir): path = os.path.join(proc_dir, fn_) try: data = _read_proc_file(path, opts) if data is not None: ret.append(data) except OSError: # proc files may be removed at any time during this process by # the master process that is executing the JID in question, so # we must ignore ENOENT during this process log.trace("%s removed during processing by master process", path) return ret
def load_agents(config): ''' Load the agents specified in /etc/salt/alert from the salt.ext.alert.agents package. Each module must define a load_agents() function that accepts the parsed YAML configuration for the agents. ''' ignore_modules = ['alert.time', 'alert.subscriptions', 'alert.verbs'] agents = {} for key, value in config.iteritems(): if key.startswith('alert.') and key not in ignore_modules: modname = AGENTS_MODULE + '._' + key[6:] log.trace('load %s', modname) try: mod = __import__(modname, fromlist=[AGENTS_MODULE]) except ImportError, ex: log.trace('not an agent module: %s', modname, exc_info=ex) continue try: new_agents = mod.load_agents(value) except AttributeError, ex: log.error('not an agent module: %s', modname, exc_info=ex) continue common = set(agents.keys()) & set(new_agents.keys()) if len(common) != 0: raise ValueError( 'agent name(s) collide in config: {}'.format( ', '.join(["'{}'".format(x) for x in common]))) agents.update(new_agents) log.trace('loaded alert agent(s): %s', new_agents.keys())
def load_agents(config): ''' Load the agents specified in /etc/salt/alert from the salt.ext.alert.agents package. Each module must define a load_agents() function that accepts the parsed YAML configuration for the agents. ''' ignore_modules = ['alert.time', 'alert.subscriptions', 'alert.verbs'] agents = {} for key, value in config.iteritems(): if key.startswith('alert.') and key not in ignore_modules: modname = AGENTS_MODULE + '._' + key[6:] log.trace('load %s', modname) try: mod = __import__(modname, fromlist=[AGENTS_MODULE]) except ImportError, ex: log.trace('not an agent module: %s', modname, exc_info=ex) continue try: new_agents = mod.load_agents(value) except AttributeError, ex: log.error('not an agent module: %s', modname, exc_info=ex) continue common = set(agents.keys()) & set(new_agents.keys()) if len(common) != 0: raise ValueError('agent name(s) collide in config: {}'.format( ', '.join(["'{}'".format(x) for x in common]))) agents.update(new_agents) log.trace('loaded alert agent(s): %s', new_agents.keys())
def add_msg(self, msg, timestamp=None): ''' Add a message to the recipient's outbound queue. If the recipient is ready and has messages to send and wasn't previously in the pending set, add it to the pending set. msg = the message to send, e.g. 'hello world' timestamp = the (optional) message timestamp. It is used to determine a message's age in expire_msgs(). ''' assert isinstance(msg, basestring) if timestamp is None and self.max_age: timestamp = time.time() oldlen = len(self.msgs) self.msgs.append((timestamp, msg)) self.expire_msgs(timestamp) if self.pending is not None and \ self._state == READY and \ oldlen == 0 and \ len(self.msgs) > 0: log.trace('add %s to pending', self.addr) self.pending.add(self)
def which(exe=None): ''' Python clone of /usr/bin/which ''' if exe: if os.access(exe, os.X_OK): return exe # default path based on busybox's default default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' search_path = os.environ.get('PATH', default_path) for path in search_path.split(os.pathsep): full_path = os.path.join(path, exe) if os.access(full_path, os.X_OK): return full_path log.trace( '{0!r} could not be found in the following search ' 'path: {1!r}'.format( exe, search_path ) ) log.trace('No executable was passed to be searched by which') return None
def _load_smtp_config(self, config): """ Load the smtp configuration from /etc/salt/alert. """ smtp_config = config.get("smtp") if not smtp_config: raise ValueError('alert.email config missing "smtp" options') self.server = smtp_config.get("host") self.port = smtp_config.get("port", DEFAULT_PORT) self.user = smtp_config.get("user", DEFAULT_USER) self.password = smtp_config.get("password", DEFAULT_PASSWORD) if not self.server: raise ValueError("alert.email.smtp config missing or " 'blank "host" option') log.trace( """email alert smtp: server: %s port: %s user: %s password: %s""", self.server, self.port, self.user, self.password, )
def state(self, value): ''' Change the recipient's state. If the recipient becomes ready and has messages, add it to the pending set. If the recipient becomes unavailable, remove it from the pending set. ''' log.trace('%s state: %s -> %s', self.addr, self._state, value) self._state = value if self.pending is not None: if value == READY: if self.msgs: log.trace('add %s to pending', self.addr) self.pending.add(self) else: log.trace('remove %s from pending', self.addr) self.pending.discard(self)
def __throttled(self): ''' ''' if self.service_down: log.trace('service is down ... waiting') return True if self.send_interval > 0: if self.throttle_wait: log.trace('throttle messages ... waiting') return True now = time.time() secs_since_last_send = now - self.last_send_time if secs_since_last_send < self.send_interval: sleep_time = self.send_interval - secs_since_last_send log.trace('delay sending for %0.1f seconds', sleep_time) self.throttle_wait = True self.schedule('send-pending', sleep_time, self.__wait) return True self.last_send_time = now return False
def __set_state(self, recipient, roster_item=None): ''' ''' if roster_item: can_send = roster_item['to'] pending_out = roster_item['pending_out'] else: can_send = False pending_out = False log.trace('set_state: addr=%s state=%s can_send=%s pending_out=%s', recipient.addr, recipient.state, can_send, pending_out) if can_send: recipient.state = READY self.__pending() elif recipient.state == UNKNOWN: # always send a subscription request at restart log.trace('send presence to %s (startup)', recipient.addr) recipient.state = WAITING_FOR_AUTHZ self.send_presence(pto=recipient.addr, ptype='subscribe') elif not pending_out and recipient.state != WAITING_FOR_AUTHZ: log.trace('send presence to %s', recipient.addr) recipient.state = WAITING_FOR_AUTHZ self.send_presence(pto=recipient.addr, ptype='subscribe')
def run(self): ''' Main loop of the ConCache, starts updates in intervals and answers requests from the MWorkers ''' context = zmq.Context() # the socket for incoming cache requests creq_in = context.socket(zmq.REP) creq_in.setsockopt(zmq.LINGER, 100) creq_in.bind('ipc://' + self.cache_sock) # the socket for incoming cache-updates from workers cupd_in = context.socket(zmq.SUB) cupd_in.setsockopt(zmq.SUBSCRIBE, '') cupd_in.setsockopt(zmq.LINGER, 100) cupd_in.bind('ipc://' + self.update_sock) # the socket for the timer-event timer_in = context.socket(zmq.SUB) timer_in.setsockopt(zmq.SUBSCRIBE, '') timer_in.setsockopt(zmq.LINGER, 100) timer_in.connect('ipc://' + self.upd_t_sock) poller = zmq.Poller() poller.register(creq_in, zmq.POLLIN) poller.register(cupd_in, zmq.POLLIN) poller.register(timer_in, zmq.POLLIN) # our serializer serial = salt.payload.Serial(self.opts.get('serial', '')) # register a signal handler signal.signal(signal.SIGINT, self.signal_handler) # secure the sockets from the world self.secure() log.info('ConCache started') while self.running: # we check for new events with the poller try: socks = dict(poller.poll(1)) except KeyboardInterrupt: self.stop() except zmq.ZMQError as zmq_err: log.error('ConCache ZeroMQ-Error occured') log.exception(zmq_err) self.stop() # check for next cache-request if socks.get(creq_in) == zmq.POLLIN: msg = serial.loads(creq_in.recv()) log.trace('ConCache Received request: {0}'.format(msg)) # requests to the minion list are send as str's if isinstance(msg, str): if msg == 'minions': # Send reply back to client reply = serial.dumps(self.minions) creq_in.send(reply) else: reply = serial.dumps(False) creq_in.send(reply) # check for next cache-update from workers elif socks.get(cupd_in) == zmq.POLLIN: new_c_data = serial.loads(cupd_in.recv()) # tell the worker to exit #cupd_in.send(serial.dumps('ACK')) # check if the returned data is usable if not isinstance(new_c_data, list): log.error('ConCache Worker returned unusable result') del new_c_data continue # the cache will receive lists of minions # 1. if the list only has 1 item, its from an MWorker, we append it # 2. anything else is considered malformed if len(new_c_data) == 0: log.debug('ConCache Got empty update from worker') elif len(new_c_data) == 1: if new_c_data[0] not in self.minions: log.trace('ConCache Adding minion {0} to cache'.format(new_c_data[0])) self.minions.append(new_c_data[0]) else: log.debug('ConCache Got malformed result dict from worker') del new_c_data log.info('ConCache {0} entries in cache'.format(len(self.minions))) # check for next timer-event to start new jobs elif socks.get(timer_in) == zmq.POLLIN: sec_event = serial.loads(timer_in.recv()) # update the list every 30 seconds if (sec_event == 31) or (sec_event == 1): self.renew() self.stop() creq_in.close() cupd_in.close() timer_in.close() context.term() log.debug('ConCache Shutting down')
def _deliver(self, addrs, alert): ''' Deliver the alert to the specified addresses. This method should only be called by Agent.deliver(). addrs = a list of "To:" recipients. Each recipient can be a plain email address, e.g. "*****@*****.**", or a real name plus an address, e.g. "Super Duper <*****@*****.**>". alert = the alert dict used to expand ${var}s in message subject and body ''' if len(addrs) == 0: return full_addrs = [addr[0] for addr in addrs] email_addrs = [addr[1] for addr in addrs] msg = email.mime.text.MIMEText(self.body.safe_substitute(alert)) msg['Subject'] = self.subject.safe_substitute(alert) msg['From'] = self.sender msg['To'] = ', '.join(full_addrs) msgstr = msg.as_string() log.trace('send email:\n%s', msgstr) log.trace('email: connect to %s port %s', self.server, self.port) try: s = smtplib.SMTP(self.server, self.port) s.ehlo() if s.has_extn('STARTTLS'): log.trace('email: start tls') s.starttls() if self.user and self.password: log.trace('email: login as %s', self.user) s.login(self.user, self.password) log.trace('email: send message to %s', email_addrs) s.sendmail(self.user, email_addrs, msgstr) log.trace('email: disconnect') s.quit() except smtplib.SMTPException, ex: log.error('failed to send email alert:\n%s', msgstr, exc_info=ex)
def cmd_block(self, is_retry=False): ''' Prepare the pre-check command to send to the subsystem 1. execute SHIM + command 2. check if SHIM returns a master request or if it completed 3. handle any master request 4. re-execute SHIM + command 5. split SHIM results from command results 6. return command results ''' self.argv = _convert_args(self.argv) log.debug( 'Performing shimmed, blocking command as follows:\n{0}'.format( ' '.join(self.argv))) cmd_str = self._cmd_str() stdout, stderr, retcode = self.shim_cmd(cmd_str) log.trace('STDOUT {1}\n{0}'.format(stdout, self.target['host'])) log.trace('STDERR {1}\n{0}'.format(stderr, self.target['host'])) log.debug('RETCODE {1}: {0}'.format(retcode, self.target['host'])) error = self.categorize_shim_errors(stdout, stderr, retcode) if error: if error == 'Undefined SHIM state': self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search( RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying thin, undefined state: {0}'.format( stdout), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: return 'ERROR: {0}'.format(error), stderr, retcode # FIXME: this discards output from ssh_shim if the shim succeeds. It should # always save the shim output regardless of shim success or failure. if re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() else: # This is actually an error state prior to the shim but let it fall through pass if re.search(RSTR_RE, stderr): # Found RSTR in stderr which means SHIM completed and only # and remaining output is only from salt. stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: # RSTR was found in stdout but not stderr - which means there # is a SHIM command for the master. shim_command = re.split(r'\r?\n', stdout, 1)[0].strip() log.debug('SHIM retcode({0}) and command: {1}'.format( retcode, shim_command)) if 'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY: self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search( RSTR_RE, stderr): if not self.tty: # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying thin: {0}\n{1}'.format( stdout, stderr), stderr, retcode elif not re.search(RSTR_RE, stdout): # If RSTR is not seen in stdout with tty, then there # was a thin deployment problem. return 'ERROR: Failure deploying thin: {0}\n{1}'.format( stdout, stderr), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() if self.tty: stderr = '' else: stderr = re.split(RSTR_RE, stderr, 1)[1].strip() elif 'ext_mods' == shim_command: self.deploy_ext() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search( RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying ext_mods: {0}'.format( stdout), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() stderr = re.split(RSTR_RE, stderr, 1)[1].strip() return stdout, stderr, retcode
def run(self): ''' Main loop of the ConCache, starts updates in intervals and answers requests from the MWorkers ''' context = zmq.Context() # the socket for incoming cache requests creq_in = context.socket(zmq.REP) creq_in.setsockopt(zmq.LINGER, 100) creq_in.bind('ipc://' + self.cache_sock) # the socket for incoming cache-updates from workers cupd_in = context.socket(zmq.SUB) cupd_in.setsockopt(zmq.SUBSCRIBE, '') cupd_in.setsockopt(zmq.LINGER, 100) cupd_in.bind('ipc://' + self.update_sock) # the socket for the timer-event timer_in = context.socket(zmq.SUB) timer_in.setsockopt(zmq.SUBSCRIBE, '') timer_in.setsockopt(zmq.LINGER, 100) timer_in.connect('ipc://' + self.upd_t_sock) poller = zmq.Poller() poller.register(creq_in, zmq.POLLIN) poller.register(cupd_in, zmq.POLLIN) poller.register(timer_in, zmq.POLLIN) # our serializer serial = salt.payload.Serial(self.opts.get('serial', '')) # register a signal handler signal.signal(signal.SIGINT, self.signal_handler) # secure the sockets from the world self.secure() log.info('ConCache started') while self.running: # we check for new events with the poller try: socks = dict(poller.poll(1)) except KeyboardInterrupt: self.stop() except zmq.ZMQError as zmq_err: log.error('ConCache ZeroMQ-Error occurred') log.exception(zmq_err) self.stop() # check for next cache-request if socks.get(creq_in) == zmq.POLLIN: msg = serial.loads(creq_in.recv()) log.trace('ConCache Received request: {0}'.format(msg)) # requests to the minion list are send as str's if isinstance(msg, str): if msg == 'minions': # Send reply back to client reply = serial.dumps(self.minions) creq_in.send(reply) else: reply = serial.dumps(False) creq_in.send(reply) # check for next cache-update from workers elif socks.get(cupd_in) == zmq.POLLIN: new_c_data = serial.loads(cupd_in.recv()) # tell the worker to exit #cupd_in.send(serial.dumps('ACK')) # check if the returned data is usable if not isinstance(new_c_data, list): log.error('ConCache Worker returned unusable result') del new_c_data continue # the cache will receive lists of minions # 1. if the list only has 1 item, its from an MWorker, we append it # 2. anything else is considered malformed if len(new_c_data) == 0: log.debug('ConCache Got empty update from worker') elif len(new_c_data) == 1: if new_c_data[0] not in self.minions: log.trace('ConCache Adding minion {0} to cache'.format(new_c_data[0])) self.minions.append(new_c_data[0]) else: log.debug('ConCache Got malformed result dict from worker') del new_c_data log.info('ConCache {0} entries in cache'.format(len(self.minions))) # check for next timer-event to start new jobs elif socks.get(timer_in) == zmq.POLLIN: sec_event = serial.loads(timer_in.recv()) # update the list every 30 seconds if (sec_event == 31) or (sec_event == 1): self.renew() self.stop() creq_in.close() cupd_in.close() timer_in.close() context.term() log.debug('ConCache Shutting down')
def __retry_service(self): ''' ''' log.trace('retrying send') self.service_down = False self.__pending()
def _deliver(self, addrs, alert): """ Deliver the alert to the specified addresses. This method should only be called by Agent.deliver(). addrs = a list of "To:" recipients. Each recipient can be a plain email address, e.g. "*****@*****.**", or a real name plus an address, e.g. "Super Duper <*****@*****.**>". alert = the alert dict used to expand ${var}s in message subject and body """ if len(addrs) == 0: return full_addrs = [addr[0] for addr in addrs] email_addrs = [addr[1] for addr in addrs] msg = email.mime.text.MIMEText(self.body.safe_substitute(alert)) msg["Subject"] = self.subject.safe_substitute(alert) msg["From"] = self.sender msg["To"] = ", ".join(full_addrs) msgstr = msg.as_string() log.trace("send email:\n%s", msgstr) log.trace("email: connect to %s port %s", self.server, self.port) try: s = smtplib.SMTP(self.server, self.port) s.ehlo() if s.has_extn("STARTTLS"): log.trace("email: start tls") s.starttls() if self.user and self.password: log.trace("email: login as %s", self.user) s.login(self.user, self.password) log.trace("email: send message to %s", email_addrs) s.sendmail(self.user, email_addrs, msgstr) log.trace("email: disconnect") s.quit() except smtplib.SMTPException, ex: log.error("failed to send email alert:\n%s", msgstr, exc_info=ex)
def readd_msg(self, msg, timestamp=None): ''' Add a previously queued message. This method is used to requeue messages when an asynchronous send fails and we want to retry. The requeued message is placed at the end of the requeued messages and at the front of the messages that have never been removed. If the queue is bounded and full, the message is dropped. If timestamp isn't specified, the oldest unrequeued message is used. If there are only requeued messages, then the youngest requeued timestamp is used. >>> r = Recipient('*****@*****.**') # readd to an empty queue >>> r.readd_msg('msg 1', timestamp=1) >>> print r [email protected] [READY]: 1: msg 1 >>> r.get_msg() 'msg 1' # readd to a queue with no readded messages and not timestamp # specified for the readded message >>> r.add_msg('msg 3', timestamp=3) >>> r.add_msg('msg 4', timestamp=4) >>> print r [email protected] [READY]: 3: msg 3 4: msg 4 >>> r.readd_msg('msg 1') >>> print r [email protected] [READY]: 3: msg 1 3: msg 3 4: msg 4 >>> r.get_msg() 'msg 1' # readd to a queue with no readded messages and a timestamp # specified for the readded message >>> r.readd_msg('msg 1', timestamp=1) >>> print r [email protected] [READY]: 1: msg 1 3: msg 3 4: msg 4 # readd to a queue with readded messages and no timestamp # specified for the readded message >>> r.readd_msg('msg 2') >>> print r [email protected] [READY]: 1: msg 1 3: msg 2 3: msg 3 4: msg 4 # drain the queue >>> while r.msgs: ... r.get_msg() ... assert r.readd_idx >= 0 'msg 1' 'msg 2' 'msg 3' 'msg 4' ''' if self.msgs.maxlen and len(self.msgs) == self.msgs.maxlen: # drop message ... the queue is full of younger messages return if timestamp is None: if len(self.msgs) == 0: # arbitrarily set the message timestamp to now if self.max_age: timestamp = time.time() elif self.readd_idx >= len(self.msgs): # use the time of the youngest *readded* message timestamp = self.msgs[-1][0] else: # use the time of the oldest unreadded message timestamp = self.msgs[self.readd_idx][0] oldlen = len(self.msgs) self.msgs.rotate(-self.readd_idx) self.msgs.appendleft((timestamp, msg)) self.msgs.rotate(self.readd_idx) self.readd_idx += 1 self.expire_msgs(timestamp) if self.pending is not None and \ self._state == READY and \ oldlen == 0 and \ len(self.msgs) > 0: log.trace('add %s to pending', self.addr) self.pending.add(self)
def cmd_block(self, is_retry=False): ''' Prepare the pre-check command to send to the subsystem 1. execute SHIM + command 2. check if SHIM returns a master request or if it completed 3. handle any master request 4. re-execute SHIM + command 5. split SHIM results from command results 6. return command results ''' self.argv = _convert_args(self.argv) log.debug('Performing shimmed, blocking command as follows:\n{0}'.format(' '.join(self.argv))) cmd_str = self._cmd_str() stdout, stderr, retcode = self.shim_cmd(cmd_str) log.trace('STDOUT {1}\n{0}'.format(stdout, self.target['host'])) log.trace('STDERR {1}\n{0}'.format(stderr, self.target['host'])) log.debug('RETCODE {1}: {0}'.format(retcode, self.target['host'])) error = self.categorize_shim_errors(stdout, stderr, retcode) if error: if error == 'Undefined SHIM state': self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying thin, undefined state: {0}'.format(stdout), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: return 'ERROR: {0}'.format(error), stderr, retcode # FIXME: this discards output from ssh_shim if the shim succeeds. It should # always save the shim output regardless of shim success or failure. if re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() else: # This is actually an error state prior to the shim but let it fall through pass if re.search(RSTR_RE, stderr): # Found RSTR in stderr which means SHIM completed and only # and remaining output is only from salt. stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: # RSTR was found in stdout but not stderr - which means there # is a SHIM command for the master. shim_command = re.split(r'\r?\n', stdout, 1)[0].strip() log.debug('SHIM retcode({0}) and command: {1}'.format(retcode, shim_command)) if 'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY: self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): if not self.tty: # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode elif not re.search(RSTR_RE, stdout): # If RSTR is not seen in stdout with tty, then there # was a thin deployment problem. return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() if self.tty: stderr = '' else: stderr = re.split(RSTR_RE, stderr, 1)[1].strip() elif 'ext_mods' == shim_command: self.deploy_ext() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return 'ERROR: Failure deploying ext_mods: {0}'.format(stdout), stderr, retcode stdout = re.split(RSTR_RE, stdout, 1)[1].strip() stderr = re.split(RSTR_RE, stderr, 1)[1].strip() return stdout, stderr, retcode