def get_msg(self, timestamp=None):
'''
Remove and return the first (oldest) message in the recipient's queue.
If there are no messages in the queue, return None. If the number
of messages in the queue drops to zero, remove the recipient from
the pending set.
>>> pending = set()
>>> r = Recipient('*****@*****.**', pending=pending)
>>> r.state = READY
>>> r.add_msg('msg 1')
>>> r.add_msg('msg 2')
# get and send messages
>>> while r.msgs:
... print r.get_msg()
... print pending
msg 1
set([[email protected]])
msg 2
set([])
'''
msg = None
self.expire_msgs(timestamp)
if self.msgs:
created, msg = self.msgs.popleft()
if self.readd_idx > 0:
self.readd_idx -= 1
if self.pending is not None and len(self.msgs) == 0:
log.trace('remove %s from pending', self.addr)
self.pending.discard(self)
return msg
def main():
"""
Start computer service.
.. code-block:: bash
hs-computer --server|--node|--all
"""
salt.log.setup_console_logger()
parser = option_parser()
shell = Shell(parser)
try:
shell.exec_command()
except Exception as err:
log.error("shell command failed: {0!r}".format(
err)
)
except KeyboardInterrupt:
raise SystemExit('\nExiting gracefully on Ctrl-c')
finally:
# TODO: need cleanup
log.trace("TODO cleanup work")
return
def add_subscriber(self, regex, addr):
'''
'''
log.trace('add %s subscriber: pattern="%s" address="%s"',
self.protocol, regex.pattern, addr)
if regex not in self.distrib_lists:
self.distrib_lists[regex] = set()
self.distrib_lists[regex].add(self._parse_subscriber(addr))
def add_subscriber(self, regex, addr):
'''
'''
log.trace('add %s subscriber: pattern="%s" address="%s"',
self.protocol, regex.pattern, addr)
if regex not in self.distrib_lists:
self.distrib_lists[regex] = set()
self.distrib_lists[regex].add(self._parse_subscriber(addr))
def _load_time(self, config):
'''
Load the time format and timezone from data in /etc/salt/alert.
'''
timecfg = config.get('alert.time', {})
timedefs = (timecfg.get('format', STRFTIME_DEFAULT),
timecfg.get('timezone', TIMEZONE_DEFAULT))
log.trace('alert time: format="%s" timezone="%s"', *timedefs)
return timedefs
def _load_time(self, config):
'''
Load the time format and timezone from data in /etc/salt/alert.
'''
timecfg = config.get('alert.time', {})
timedefs = (timecfg.get('format', STRFTIME_DEFAULT),
timecfg.get('timezone', TIMEZONE_DEFAULT))
log.trace('alert time: format="%s" timezone="%s"', *timedefs)
return timedefs
def _load_verbs(self, config):
'''
Load the preferred raised and cleared verbs from /etc/salt/alert.
'''
verbs = VERBS_DEFAULT.copy()
for verb, preferred in config.get('alert.verbs', {}).iteritems():
if preferred:
verbs[verb] = preferred
log.trace('alert verbs: %s', verbs)
return verbs
def _load_verbs(self, config):
'''
Load the preferred raised and cleared verbs from /etc/salt/alert.
'''
verbs = VERBS_DEFAULT.copy()
for verb, preferred in config.get('alert.verbs', {}).iteritems():
if preferred:
verbs[verb] = preferred
log.trace('alert verbs: %s', verbs)
return verbs
def __init__(self, protocol, config):
'''
Configure the agent from YAML data parsed from /etc/salt/alert.
'''
Agent.__init__(self, protocol)
user = config.get('user')
password = config.get('password')
sleekxmpp.ClientXMPP.__init__(self, user, password)
self.auto_subscribe = False
self.auto_authorize = None
server = config.get('host')
if server:
self.server_addr = (server, config.get('port', DEFAULT_PORT))
else:
self.server_addr = (self.boundjid.host, DEFAULT_PORT)
self.max_msgs = config.get('max_msgs', DEFAULT_MAX_MSGS)
self.max_age = config.get('max_age', DEFAULT_MAX_AGE)
log.trace('connect to %s as %s/%s', self.server_addr, user, password)
self.connected = False
self.message = string.Template(config.get('message', DEFAULT_MESSAGE))
self.pending = set()
self.recipients = {}
self.service_down = False
self.retry_service_wait = 60
self.last_send_time = 0
self.throttle_wait = False
msgs_per_sec = config.get('msgs_per_sec', 0)
if msgs_per_sec <= 0:
self.send_interval = 0
else:
self.send_interval = 1.0 / msgs_per_sec
self.add_event_handler('presence_subscribe', self.__presence)
self.add_event_handler('presence_subscribed', self.__presence)
self.add_event_handler('presence_unsubscribe', self.__presence)
self.add_event_handler('presence_unsubscribed', self.__presence)
self.add_event_handler('presence_available', self.__presence)
self.add_event_handler('presence_error', self.__presence)
self.add_event_handler('changed_subscription', self.__subscription)
self.add_event_handler('message', self.__message)
self.add_event_handler('roster_update', self.__roster)
self.add_event_handler('session_start', self.__start)
self.register_plugin('xep_0030') # Service Discovery
self.register_plugin('xep_0199') # XMPP Ping
self.register_plugin('xep_0086') # Legacy Errors
def __init__(self, protocol, config):
'''
Configure the agent from YAML data parsed from /etc/salt/alert.
'''
Agent.__init__(self, protocol)
user = config.get('user')
password = config.get('password')
sleekxmpp.ClientXMPP.__init__(self, user, password)
self.auto_subscribe = False
self.auto_authorize = None
server = config.get('host')
if server:
self.server_addr = (server, config.get('port', DEFAULT_PORT))
else:
self.server_addr = (self.boundjid.host, DEFAULT_PORT)
self.max_msgs = config.get('max_msgs', DEFAULT_MAX_MSGS)
self.max_age = config.get('max_age', DEFAULT_MAX_AGE)
log.trace('connect to %s as %s/%s', self.server_addr, user, password)
self.connected = False
self.message = string.Template(config.get('message', DEFAULT_MESSAGE))
self.pending = set()
self.recipients = {}
self.service_down = False
self.retry_service_wait = 60
self.last_send_time = 0
self.throttle_wait = False
msgs_per_sec = config.get('msgs_per_sec', 0)
if msgs_per_sec <= 0:
self.send_interval = 0
else:
self.send_interval = 1.0 / msgs_per_sec
self.add_event_handler('presence_subscribe', self.__presence)
self.add_event_handler('presence_subscribed', self.__presence)
self.add_event_handler('presence_unsubscribe', self.__presence)
self.add_event_handler('presence_unsubscribed', self.__presence)
self.add_event_handler('presence_available', self.__presence)
self.add_event_handler('presence_error', self.__presence)
self.add_event_handler('changed_subscription', self.__subscription)
self.add_event_handler('message', self.__message)
self.add_event_handler('roster_update', self.__roster)
self.add_event_handler('session_start', self.__start)
self.register_plugin('xep_0030') # Service Discovery
self.register_plugin('xep_0199') # XMPP Ping
self.register_plugin('xep_0086') # Legacy Errors
def __subscription(self, event):
'''
Handle peers subscribing and unsubscribing to us.
'''
addr = event.get_from().bare
etype = event.get_type()
log.trace('_subscription: addr=%s type=%s', addr, etype)
recipient = self.recipients.get(addr)
if recipient:
if etype == 'subscribed':
# Send authorization back to peer
self.send_presence(pto=addr, ptype='subscribed')
def __subscription(self, event):
'''
Handle peers subscribing and unsubscribing to us.
'''
addr = event.get_from().bare
etype = event.get_type()
log.trace('_subscription: addr=%s type=%s', addr, etype)
recipient = self.recipients.get(addr)
if recipient:
if etype == 'subscribed':
# Send authorization back to peer
self.send_presence(pto=addr, ptype='subscribed')
def _deliver(self, subscribers, alert):
'''
'''
log.trace('_deliver: %s', alert)
if not self.connected:
self.__connect()
self.connected = True
timestamp = time.time()
msg = self.message.safe_substitute(alert)
for recipient in subscribers:
log.trace('queue message to %s: %s message(s) pending',
recipient.addr, len(recipient.msgs))
recipient.add_msg(msg, timestamp)
self.__pending()
def _deliver(self, subscribers, alert):
'''
'''
log.trace('_deliver: %s', alert)
if not self.connected:
self.__connect()
self.connected = True
timestamp = time.time()
msg = self.message.safe_substitute(alert)
for recipient in subscribers:
log.trace('queue message to %s: %s message(s) pending',
recipient.addr, len(recipient.msgs))
recipient.add_msg(msg, timestamp)
self.__pending()
def clear_socket(self):
'''
delete socket if you have it
'''
if hasattr(self, '_socket'):
if isinstance(self.poller.sockets, dict):
sockets = list(self.poller.sockets.keys())
for socket in sockets:
log.trace('Unregistering socket: %s', socket)
self.poller.unregister(socket)
else:
for socket in self.poller.sockets:
log.trace('Unregistering socket: %s', socket)
self.poller.unregister(socket[0])
del self._socket
def __pending(self):
'''
Send pending messages.
'''
log.trace('_pending: %s recipients have msgs to send',
len(self.pending))
while self.pending:
for recipient in self.recipients.values():
if self.__throttled():
return
if len(recipient.msgs) > 0:
addr = recipient.addr
msg = recipient.get_msg()
log.trace('send to %s: %s', addr, msg)
self.send_message(mto=addr, mbody=msg, mtype='chat')
def __pending(self):
'''
Send pending messages.
'''
log.trace('_pending: %s recipients have msgs to send',
len(self.pending))
while self.pending:
for recipient in self.recipients.values():
if self.__throttled():
return
if len(recipient.msgs) > 0:
addr = recipient.addr
msg = recipient.get_msg()
log.trace('send to %s: %s', addr, msg)
self.send_message(mto=addr, mbody=msg, mtype='chat')
def clear_socket(self):
'''
delete socket if you have it
'''
if hasattr(self, '_socket'):
if isinstance(self.poller.sockets, dict):
sockets = list(self.poller.sockets.keys())
for socket in sockets:
log.trace('Unregistering socket: {0}'.format(socket))
self.poller.unregister(socket)
else:
for socket in self.poller.sockets:
log.trace('Unregistering socket: {0}'.format(socket))
self.poller.unregister(socket[0])
del self._socket
def clear_socket(self):
"""
delete socket if you have it
"""
if hasattr(self, "_socket"):
if isinstance(self.poller.sockets, dict):
sockets = list(self.poller.sockets.keys())
for socket in sockets:
log.trace("Unregistering socket: %s", socket)
self.poller.unregister(socket)
else:
for socket in self.poller.sockets:
log.trace("Unregistering socket: %s", socket)
self.poller.unregister(socket[0])
del self._socket
def __roster(self, event):
'''
At startup, set the recipient's state based on the roster.
'''
if log.isEnabledFor(salt.log.TRACE):
roster = [addr for addr in self.client_roster]
log.trace('roster changed: %s', roster)
for recipient in self.recipients.values():
if recipient.state == UNKNOWN:
if recipient.addr in self.client_roster:
roster_item = self.client_roster[recipient.addr]
else:
roster_item = None
self.__set_state(recipient, roster_item)
# self.del_event_handler('roster_update', self.__roster)
self.__pending()
class MonitorTask(object):
'''
A single monitor task.
'''
def __init__(self, taskid, pyexe, context, scheduler=None):
self.taskid = taskid
self.code = pyexe
self.context = context
self.scheduler = scheduler
def run(self):
log.trace('start thread for %s', self.taskid)
minion = self.context.get('id')
collector = self.context.get('collector')
while True:
try:
exec self.code in self.context
except Exception, ex:
log.error("can't execute %s: %s", self.taskid, ex, exc_info=ex)
if collector:
jid = datetime.datetime.strftime(
datetime.datetime.now(), 'M%Y%m%d%H%M%S%f')
try:
collector(minion, self.context['cmd'], self.context['result'])
except Exception, ex:
log.error('monitor error: %s', self.taskid, exc_info=ex)
if self.scheduler is None:
break
duration = self.scheduler.next()
log.trace('%s: sleep %s seconds', self.taskid, duration)
time.sleep(duration)
def run(self):
log.trace('start thread for %s', self.taskid)
minion = self.context.get('id')
collector = self.context.get('collector')
while True:
try:
exec self.code in self.context
except Exception, ex:
log.error("can't execute %s: %s", self.taskid, ex, exc_info=ex)
if collector:
jid = datetime.datetime.strftime(
datetime.datetime.now(), 'M%Y%m%d%H%M%S%f')
try:
collector(minion, self.context['cmd'], self.context['result'])
except Exception, ex:
log.error('monitor error: %s', self.taskid, exc_info=ex)
def _load_msg_config(self, config):
'''
Load the email message configuration from /etc/salt/alert.
'''
self.sender = config.get('from', DEFAULT_SENDER)
self.headers = config.get('headers', DEFAULT_HEADERS)
self.subject = string.Template(config.get('subject', DEFAULT_SUBJECT))
self.body = string.Template(config.get('body', DEFAULT_BODY))
if log.isEnabledFor(salt.log.TRACE):
log.trace(
'''email alert message:
from: %s
subject: %s
headers: %s
body: %s''', self.sender, self.subject.safe_substitute({}),
self.headers,
'\n '.join(self.body.safe_substitute({}).splitlines()))
def __presence(self, event):
'''
Handle peers subscribing and unsubscribing to us.
'''
addr = event.get_from().bare
etype = event.get_type()
log.trace('_presence: addr=%s type=%s', addr, etype)
recipient = self.recipients.get(addr)
if recipient:
if etype == 'unsubscribed':
recipient.state = READY
self.__set_state(recipient)
elif etype == 'subscribe':
self.send_presence(pto=addr, ptype='subscribed')
elif etype in ['subscribed', 'available']:
recipient.state = READY
self.__pending()
def __presence(self, event):
'''
Handle peers subscribing and unsubscribing to us.
'''
addr = event.get_from().bare
etype = event.get_type()
log.trace('_presence: addr=%s type=%s', addr, etype)
recipient = self.recipients.get(addr)
if recipient:
if etype == 'unsubscribed':
recipient.state = READY
self.__set_state(recipient)
elif etype == 'subscribe':
self.send_presence(pto=addr, ptype='subscribed')
elif etype in ['subscribed', 'available']:
recipient.state = READY
self.__pending()
def __roster(self, event):
'''
At startup, set the recipient's state based on the roster.
'''
if log.isEnabledFor(salt.log.TRACE):
roster = [addr for addr in self.client_roster]
log.trace('roster changed: %s', roster)
for recipient in self.recipients.values():
if recipient.state == UNKNOWN:
if recipient.addr in self.client_roster:
roster_item = self.client_roster[recipient.addr]
else:
roster_item = None
self.__set_state(recipient, roster_item)
# self.del_event_handler('roster_update', self.__roster)
self.__pending()
def _expand_tasks(self, parsed_yaml):
'''
Assemble compiled code from the configuration described by
python dictionaries and lists.
'''
results = []
for tasknum, taskdict in enumerate(parsed_yaml, 1):
try:
log.trace(taskdict)
taskid = taskdict.get('id', 'monitor-{}'.format(tasknum))
pysrc = self._expand_task(taskid, taskdict)
log.trace("generated '%s' task source:\n%s", taskid, pysrc)
pyexe = compile(pysrc, '<monitor-config>', 'exec')
scheduler = self._expand_scheduler(taskdict)
results.append(
MonitorTask(taskid, pyexe, self.context, scheduler))
except ValueError, ex:
log.error('ignore monitor command #{} {!r}: {}'.format(
tasknum, taskdict.get('run', '<unknown>'), ex))
def _load_msg_config(self, config):
"""
Load the email message configuration from /etc/salt/alert.
"""
self.sender = config.get("from", DEFAULT_SENDER)
self.headers = config.get("headers", DEFAULT_HEADERS)
self.subject = string.Template(config.get("subject", DEFAULT_SUBJECT))
self.body = string.Template(config.get("body", DEFAULT_BODY))
if log.isEnabledFor(salt.log.TRACE):
log.trace(
"""email alert message:
from: %s
subject: %s
headers: %s
body: %s""",
self.sender,
self.subject.safe_substitute({}),
self.headers,
"\n ".join(self.body.safe_substitute({}).splitlines()),
)
def _expand_tasks(self, parsed_yaml):
'''
Assemble compiled code from the configuration described by
python dictionaries and lists.
'''
results = []
for tasknum, taskdict in enumerate(parsed_yaml, 1):
try:
log.trace(taskdict)
taskid = taskdict.get('id', 'monitor-{}'.format(tasknum))
pysrc = self._expand_task(taskid, taskdict)
log.trace("generated '%s' task source:\n%s", taskid, pysrc)
pyexe = compile(pysrc, '<monitor-config>', 'exec')
scheduler = self._expand_scheduler(taskdict)
results.append(MonitorTask(taskid, pyexe, self.context, scheduler))
except ValueError, ex:
log.error( 'ignore monitor command #{} {!r}: {}'.format(
tasknum,
taskdict.get('run', '<unknown>'),
ex ) )
def which(exe=None):
'''
Python clone of /usr/bin/which
'''
if exe:
if os.access(exe, os.X_OK):
return exe
# default path based on busybox's default
default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
search_path = os.environ.get('PATH', default_path)
for path in search_path.split(os.pathsep):
full_path = os.path.join(path, exe)
if os.access(full_path, os.X_OK):
return full_path
log.trace('{0!r} could not be found in the following search '
'path: {1!r}'.format(exe, search_path))
log.trace('No executable was passed to be searched by which')
return None
def _load_smtp_config(self, config):
'''
Load the smtp configuration from /etc/salt/alert.
'''
smtp_config = config.get('smtp')
if not smtp_config:
raise ValueError('alert.email config missing "smtp" options')
self.server = smtp_config.get('host')
self.port = smtp_config.get('port', DEFAULT_PORT)
self.user = smtp_config.get('user', DEFAULT_USER)
self.password = smtp_config.get('password', DEFAULT_PASSWORD)
if not self.server:
raise ValueError('alert.email.smtp config missing or '
'blank "host" option')
log.trace(
'''email alert smtp:
server: %s
port: %s
user: %s
password: %s''', self.server, self.port, self.user, self.password)
def load(self, config):
'''
Load the alert agents, subscriptions, and miscellaneous data
from structures generated from YAML in /etc/salt/alert.
'''
if not isinstance(config, dict):
raise ValueError('expected config dict, not %s', type(config))
self.agents = salt.ext.alert.agents.load_agents(config)
self.timeformat, timezone = self._load_time(config)
self.verbs = self._load_verbs(config)
self._load_subscriptions(config, self.agents)
log.debug('set timezone to %s', timezone)
os.environ['TZ'] = timezone
time.tzset()
# remove agents that have no subscribers
for protocol in list(self.agents.keys()):
if not self.agents[protocol].has_subscribers():
log.trace('remove %s agent: no subscribers defined', protocol)
del self.agents[protocol]
def get_running_jobs(opts):
"""
Return the running jobs on this minion
"""
ret = []
proc_dir = os.path.join(opts["cachedir"], "proc")
if not os.path.isdir(proc_dir):
return ret
for fn_ in os.listdir(proc_dir):
path = os.path.join(proc_dir, fn_)
try:
data = _read_proc_file(path, opts)
if data is not None:
ret.append(data)
except OSError:
# proc files may be removed at any time during this process by
# the master process that is executing the JID in question, so
# we must ignore ENOENT during this process
log.trace("%s removed during processing by master process", path)
return ret
def load(self, config):
'''
Load the alert agents, subscriptions, and miscellaneous data
from structures generated from YAML in /etc/salt/alert.
'''
if not isinstance(config, dict):
raise ValueError('expected config dict, not %s', type(config))
self.agents = salt.ext.alert.agents.load_agents(config)
self.timeformat, timezone = self._load_time(config)
self.verbs = self._load_verbs(config)
self._load_subscriptions(config, self.agents)
log.debug('set timezone to %s', timezone)
os.environ['TZ'] = timezone
time.tzset()
# remove agents that have no subscribers
for protocol in list(self.agents.keys()):
if not self.agents[protocol].has_subscribers():
log.trace('remove %s agent: no subscribers defined', protocol)
del self.agents[protocol]
def load_agents(config):
'''
Load the agents specified in /etc/salt/alert from the
salt.ext.alert.agents package. Each module must define a
load_agents() function that accepts the parsed YAML configuration
for the agents.
'''
ignore_modules = ['alert.time', 'alert.subscriptions', 'alert.verbs']
agents = {}
for key, value in config.iteritems():
if key.startswith('alert.') and key not in ignore_modules:
modname = AGENTS_MODULE + '._' + key[6:]
log.trace('load %s', modname)
try:
mod = __import__(modname, fromlist=[AGENTS_MODULE])
except ImportError, ex:
log.trace('not an agent module: %s', modname, exc_info=ex)
continue
try:
new_agents = mod.load_agents(value)
except AttributeError, ex:
log.error('not an agent module: %s', modname, exc_info=ex)
continue
common = set(agents.keys()) & set(new_agents.keys())
if len(common) != 0:
raise ValueError(
'agent name(s) collide in config: {}'.format(
', '.join(["'{}'".format(x) for x in common])))
agents.update(new_agents)
log.trace('loaded alert agent(s): %s', new_agents.keys())
def load_agents(config):
'''
Load the agents specified in /etc/salt/alert from the
salt.ext.alert.agents package. Each module must define a
load_agents() function that accepts the parsed YAML configuration
for the agents.
'''
ignore_modules = ['alert.time', 'alert.subscriptions', 'alert.verbs']
agents = {}
for key, value in config.iteritems():
if key.startswith('alert.') and key not in ignore_modules:
modname = AGENTS_MODULE + '._' + key[6:]
log.trace('load %s', modname)
try:
mod = __import__(modname, fromlist=[AGENTS_MODULE])
except ImportError, ex:
log.trace('not an agent module: %s', modname, exc_info=ex)
continue
try:
new_agents = mod.load_agents(value)
except AttributeError, ex:
log.error('not an agent module: %s', modname, exc_info=ex)
continue
common = set(agents.keys()) & set(new_agents.keys())
if len(common) != 0:
raise ValueError('agent name(s) collide in config: {}'.format(
', '.join(["'{}'".format(x) for x in common])))
agents.update(new_agents)
log.trace('loaded alert agent(s): %s', new_agents.keys())
def add_msg(self, msg, timestamp=None):
'''
Add a message to the recipient's outbound queue.
If the recipient is ready and has messages to send and wasn't
previously in the pending set, add it to the pending set.
msg = the message to send, e.g. 'hello world'
timestamp = the (optional) message timestamp. It is used to
determine a message's age in expire_msgs().
'''
assert isinstance(msg, basestring)
if timestamp is None and self.max_age:
timestamp = time.time()
oldlen = len(self.msgs)
self.msgs.append((timestamp, msg))
self.expire_msgs(timestamp)
if self.pending is not None and \
self._state == READY and \
oldlen == 0 and \
len(self.msgs) > 0:
log.trace('add %s to pending', self.addr)
self.pending.add(self)
def which(exe=None):
'''
Python clone of /usr/bin/which
'''
if exe:
if os.access(exe, os.X_OK):
return exe
# default path based on busybox's default
default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
search_path = os.environ.get('PATH', default_path)
for path in search_path.split(os.pathsep):
full_path = os.path.join(path, exe)
if os.access(full_path, os.X_OK):
return full_path
log.trace(
'{0!r} could not be found in the following search '
'path: {1!r}'.format(
exe, search_path
)
)
log.trace('No executable was passed to be searched by which')
return None
def _load_smtp_config(self, config):
"""
Load the smtp configuration from /etc/salt/alert.
"""
smtp_config = config.get("smtp")
if not smtp_config:
raise ValueError('alert.email config missing "smtp" options')
self.server = smtp_config.get("host")
self.port = smtp_config.get("port", DEFAULT_PORT)
self.user = smtp_config.get("user", DEFAULT_USER)
self.password = smtp_config.get("password", DEFAULT_PASSWORD)
if not self.server:
raise ValueError("alert.email.smtp config missing or " 'blank "host" option')
log.trace(
"""email alert smtp:
server: %s
port: %s
user: %s
password: %s""",
self.server,
self.port,
self.user,
self.password,
)
def state(self, value):
'''
Change the recipient's state.
If the recipient becomes ready and has messages, add it to
the pending set. If the recipient becomes unavailable,
remove it from the pending set.
'''
log.trace('%s state: %s -> %s', self.addr, self._state, value)
self._state = value
if self.pending is not None:
if value == READY:
if self.msgs:
log.trace('add %s to pending', self.addr)
self.pending.add(self)
else:
log.trace('remove %s from pending', self.addr)
self.pending.discard(self)
def __throttled(self):
'''
'''
if self.service_down:
log.trace('service is down ... waiting')
return True
if self.send_interval > 0:
if self.throttle_wait:
log.trace('throttle messages ... waiting')
return True
now = time.time()
secs_since_last_send = now - self.last_send_time
if secs_since_last_send < self.send_interval:
sleep_time = self.send_interval - secs_since_last_send
log.trace('delay sending for %0.1f seconds', sleep_time)
self.throttle_wait = True
self.schedule('send-pending', sleep_time, self.__wait)
return True
self.last_send_time = now
return False
def __throttled(self):
'''
'''
if self.service_down:
log.trace('service is down ... waiting')
return True
if self.send_interval > 0:
if self.throttle_wait:
log.trace('throttle messages ... waiting')
return True
now = time.time()
secs_since_last_send = now - self.last_send_time
if secs_since_last_send < self.send_interval:
sleep_time = self.send_interval - secs_since_last_send
log.trace('delay sending for %0.1f seconds', sleep_time)
self.throttle_wait = True
self.schedule('send-pending', sleep_time, self.__wait)
return True
self.last_send_time = now
return False
def __set_state(self, recipient, roster_item=None):
'''
'''
if roster_item:
can_send = roster_item['to']
pending_out = roster_item['pending_out']
else:
can_send = False
pending_out = False
log.trace('set_state: addr=%s state=%s can_send=%s pending_out=%s',
recipient.addr, recipient.state, can_send, pending_out)
if can_send:
recipient.state = READY
self.__pending()
elif recipient.state == UNKNOWN:
# always send a subscription request at restart
log.trace('send presence to %s (startup)', recipient.addr)
recipient.state = WAITING_FOR_AUTHZ
self.send_presence(pto=recipient.addr, ptype='subscribe')
elif not pending_out and recipient.state != WAITING_FOR_AUTHZ:
log.trace('send presence to %s', recipient.addr)
recipient.state = WAITING_FOR_AUTHZ
self.send_presence(pto=recipient.addr, ptype='subscribe')
def __set_state(self, recipient, roster_item=None):
'''
'''
if roster_item:
can_send = roster_item['to']
pending_out = roster_item['pending_out']
else:
can_send = False
pending_out = False
log.trace('set_state: addr=%s state=%s can_send=%s pending_out=%s',
recipient.addr, recipient.state, can_send, pending_out)
if can_send:
recipient.state = READY
self.__pending()
elif recipient.state == UNKNOWN:
# always send a subscription request at restart
log.trace('send presence to %s (startup)', recipient.addr)
recipient.state = WAITING_FOR_AUTHZ
self.send_presence(pto=recipient.addr, ptype='subscribe')
elif not pending_out and recipient.state != WAITING_FOR_AUTHZ:
log.trace('send presence to %s', recipient.addr)
recipient.state = WAITING_FOR_AUTHZ
self.send_presence(pto=recipient.addr, ptype='subscribe')
def run(self):
'''
Main loop of the ConCache, starts updates in intervals and
answers requests from the MWorkers
'''
context = zmq.Context()
# the socket for incoming cache requests
creq_in = context.socket(zmq.REP)
creq_in.setsockopt(zmq.LINGER, 100)
creq_in.bind('ipc://' + self.cache_sock)
# the socket for incoming cache-updates from workers
cupd_in = context.socket(zmq.SUB)
cupd_in.setsockopt(zmq.SUBSCRIBE, '')
cupd_in.setsockopt(zmq.LINGER, 100)
cupd_in.bind('ipc://' + self.update_sock)
# the socket for the timer-event
timer_in = context.socket(zmq.SUB)
timer_in.setsockopt(zmq.SUBSCRIBE, '')
timer_in.setsockopt(zmq.LINGER, 100)
timer_in.connect('ipc://' + self.upd_t_sock)
poller = zmq.Poller()
poller.register(creq_in, zmq.POLLIN)
poller.register(cupd_in, zmq.POLLIN)
poller.register(timer_in, zmq.POLLIN)
# our serializer
serial = salt.payload.Serial(self.opts.get('serial', ''))
# register a signal handler
signal.signal(signal.SIGINT, self.signal_handler)
# secure the sockets from the world
self.secure()
log.info('ConCache started')
while self.running:
# we check for new events with the poller
try:
socks = dict(poller.poll(1))
except KeyboardInterrupt:
self.stop()
except zmq.ZMQError as zmq_err:
log.error('ConCache ZeroMQ-Error occured')
log.exception(zmq_err)
self.stop()
# check for next cache-request
if socks.get(creq_in) == zmq.POLLIN:
msg = serial.loads(creq_in.recv())
log.trace('ConCache Received request: {0}'.format(msg))
# requests to the minion list are send as str's
if isinstance(msg, str):
if msg == 'minions':
# Send reply back to client
reply = serial.dumps(self.minions)
creq_in.send(reply)
else:
reply = serial.dumps(False)
creq_in.send(reply)
# check for next cache-update from workers
elif socks.get(cupd_in) == zmq.POLLIN:
new_c_data = serial.loads(cupd_in.recv())
# tell the worker to exit
#cupd_in.send(serial.dumps('ACK'))
# check if the returned data is usable
if not isinstance(new_c_data, list):
log.error('ConCache Worker returned unusable result')
del new_c_data
continue
# the cache will receive lists of minions
# 1. if the list only has 1 item, its from an MWorker, we append it
# 2. anything else is considered malformed
if len(new_c_data) == 0:
log.debug('ConCache Got empty update from worker')
elif len(new_c_data) == 1:
if new_c_data[0] not in self.minions:
log.trace('ConCache Adding minion {0} to cache'.format(new_c_data[0]))
self.minions.append(new_c_data[0])
else:
log.debug('ConCache Got malformed result dict from worker')
del new_c_data
log.info('ConCache {0} entries in cache'.format(len(self.minions)))
# check for next timer-event to start new jobs
elif socks.get(timer_in) == zmq.POLLIN:
sec_event = serial.loads(timer_in.recv())
# update the list every 30 seconds
if (sec_event == 31) or (sec_event == 1):
self.renew()
self.stop()
creq_in.close()
cupd_in.close()
timer_in.close()
context.term()
log.debug('ConCache Shutting down')
def _deliver(self, addrs, alert):
'''
Deliver the alert to the specified addresses.
This method should only be called by Agent.deliver().
addrs = a list of "To:" recipients. Each recipient can be a
plain email address, e.g. "*****@*****.**", or a real name
plus an address, e.g. "Super Duper <*****@*****.**>".
alert = the alert dict used to expand ${var}s in message subject
and body
'''
if len(addrs) == 0:
return
full_addrs = [addr[0] for addr in addrs]
email_addrs = [addr[1] for addr in addrs]
msg = email.mime.text.MIMEText(self.body.safe_substitute(alert))
msg['Subject'] = self.subject.safe_substitute(alert)
msg['From'] = self.sender
msg['To'] = ', '.join(full_addrs)
msgstr = msg.as_string()
log.trace('send email:\n%s', msgstr)
log.trace('email: connect to %s port %s', self.server, self.port)
try:
s = smtplib.SMTP(self.server, self.port)
s.ehlo()
if s.has_extn('STARTTLS'):
log.trace('email: start tls')
s.starttls()
if self.user and self.password:
log.trace('email: login as %s', self.user)
s.login(self.user, self.password)
log.trace('email: send message to %s', email_addrs)
s.sendmail(self.user, email_addrs, msgstr)
log.trace('email: disconnect')
s.quit()
except smtplib.SMTPException, ex:
log.error('failed to send email alert:\n%s', msgstr, exc_info=ex)
def cmd_block(self, is_retry=False):
'''
Prepare the pre-check command to send to the subsystem
1. execute SHIM + command
2. check if SHIM returns a master request or if it completed
3. handle any master request
4. re-execute SHIM + command
5. split SHIM results from command results
6. return command results
'''
self.argv = _convert_args(self.argv)
log.debug(
'Performing shimmed, blocking command as follows:\n{0}'.format(
' '.join(self.argv)))
cmd_str = self._cmd_str()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
log.trace('STDOUT {1}\n{0}'.format(stdout, self.target['host']))
log.trace('STDERR {1}\n{0}'.format(stderr, self.target['host']))
log.debug('RETCODE {1}: {0}'.format(retcode, self.target['host']))
error = self.categorize_shim_errors(stdout, stderr, retcode)
if error:
if error == 'Undefined SHIM state':
self.deploy()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(
RSTR_RE, stderr):
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin, undefined state: {0}'.format(
stdout), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
else:
return 'ERROR: {0}'.format(error), stderr, retcode
# FIXME: this discards output from ssh_shim if the shim succeeds. It should
# always save the shim output regardless of shim success or failure.
if re.search(RSTR_RE, stdout):
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
else:
# This is actually an error state prior to the shim but let it fall through
pass
if re.search(RSTR_RE, stderr):
# Found RSTR in stderr which means SHIM completed and only
# and remaining output is only from salt.
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
else:
# RSTR was found in stdout but not stderr - which means there
# is a SHIM command for the master.
shim_command = re.split(r'\r?\n', stdout, 1)[0].strip()
log.debug('SHIM retcode({0}) and command: {1}'.format(
retcode, shim_command))
if 'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY:
self.deploy()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(
RSTR_RE, stderr):
if not self.tty:
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin: {0}\n{1}'.format(
stdout, stderr), stderr, retcode
elif not re.search(RSTR_RE, stdout):
# If RSTR is not seen in stdout with tty, then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin: {0}\n{1}'.format(
stdout, stderr), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
if self.tty:
stderr = ''
else:
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
elif 'ext_mods' == shim_command:
self.deploy_ext()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(
RSTR_RE, stderr):
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying ext_mods: {0}'.format(
stdout), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
return stdout, stderr, retcode
def run(self):
'''
Main loop of the ConCache, starts updates in intervals and
answers requests from the MWorkers
'''
context = zmq.Context()
# the socket for incoming cache requests
creq_in = context.socket(zmq.REP)
creq_in.setsockopt(zmq.LINGER, 100)
creq_in.bind('ipc://' + self.cache_sock)
# the socket for incoming cache-updates from workers
cupd_in = context.socket(zmq.SUB)
cupd_in.setsockopt(zmq.SUBSCRIBE, '')
cupd_in.setsockopt(zmq.LINGER, 100)
cupd_in.bind('ipc://' + self.update_sock)
# the socket for the timer-event
timer_in = context.socket(zmq.SUB)
timer_in.setsockopt(zmq.SUBSCRIBE, '')
timer_in.setsockopt(zmq.LINGER, 100)
timer_in.connect('ipc://' + self.upd_t_sock)
poller = zmq.Poller()
poller.register(creq_in, zmq.POLLIN)
poller.register(cupd_in, zmq.POLLIN)
poller.register(timer_in, zmq.POLLIN)
# our serializer
serial = salt.payload.Serial(self.opts.get('serial', ''))
# register a signal handler
signal.signal(signal.SIGINT, self.signal_handler)
# secure the sockets from the world
self.secure()
log.info('ConCache started')
while self.running:
# we check for new events with the poller
try:
socks = dict(poller.poll(1))
except KeyboardInterrupt:
self.stop()
except zmq.ZMQError as zmq_err:
log.error('ConCache ZeroMQ-Error occurred')
log.exception(zmq_err)
self.stop()
# check for next cache-request
if socks.get(creq_in) == zmq.POLLIN:
msg = serial.loads(creq_in.recv())
log.trace('ConCache Received request: {0}'.format(msg))
# requests to the minion list are send as str's
if isinstance(msg, str):
if msg == 'minions':
# Send reply back to client
reply = serial.dumps(self.minions)
creq_in.send(reply)
else:
reply = serial.dumps(False)
creq_in.send(reply)
# check for next cache-update from workers
elif socks.get(cupd_in) == zmq.POLLIN:
new_c_data = serial.loads(cupd_in.recv())
# tell the worker to exit
#cupd_in.send(serial.dumps('ACK'))
# check if the returned data is usable
if not isinstance(new_c_data, list):
log.error('ConCache Worker returned unusable result')
del new_c_data
continue
# the cache will receive lists of minions
# 1. if the list only has 1 item, its from an MWorker, we append it
# 2. anything else is considered malformed
if len(new_c_data) == 0:
log.debug('ConCache Got empty update from worker')
elif len(new_c_data) == 1:
if new_c_data[0] not in self.minions:
log.trace('ConCache Adding minion {0} to cache'.format(new_c_data[0]))
self.minions.append(new_c_data[0])
else:
log.debug('ConCache Got malformed result dict from worker')
del new_c_data
log.info('ConCache {0} entries in cache'.format(len(self.minions)))
# check for next timer-event to start new jobs
elif socks.get(timer_in) == zmq.POLLIN:
sec_event = serial.loads(timer_in.recv())
# update the list every 30 seconds
if (sec_event == 31) or (sec_event == 1):
self.renew()
self.stop()
creq_in.close()
cupd_in.close()
timer_in.close()
context.term()
log.debug('ConCache Shutting down')
def __retry_service(self):
'''
'''
log.trace('retrying send')
self.service_down = False
self.__pending()
def _deliver(self, addrs, alert):
"""
Deliver the alert to the specified addresses.
This method should only be called by Agent.deliver().
addrs = a list of "To:" recipients. Each recipient can be a
plain email address, e.g. "*****@*****.**", or a real name
plus an address, e.g. "Super Duper <*****@*****.**>".
alert = the alert dict used to expand ${var}s in message subject
and body
"""
if len(addrs) == 0:
return
full_addrs = [addr[0] for addr in addrs]
email_addrs = [addr[1] for addr in addrs]
msg = email.mime.text.MIMEText(self.body.safe_substitute(alert))
msg["Subject"] = self.subject.safe_substitute(alert)
msg["From"] = self.sender
msg["To"] = ", ".join(full_addrs)
msgstr = msg.as_string()
log.trace("send email:\n%s", msgstr)
log.trace("email: connect to %s port %s", self.server, self.port)
try:
s = smtplib.SMTP(self.server, self.port)
s.ehlo()
if s.has_extn("STARTTLS"):
log.trace("email: start tls")
s.starttls()
if self.user and self.password:
log.trace("email: login as %s", self.user)
s.login(self.user, self.password)
log.trace("email: send message to %s", email_addrs)
s.sendmail(self.user, email_addrs, msgstr)
log.trace("email: disconnect")
s.quit()
except smtplib.SMTPException, ex:
log.error("failed to send email alert:\n%s", msgstr, exc_info=ex)
def __retry_service(self):
'''
'''
log.trace('retrying send')
self.service_down = False
self.__pending()
def readd_msg(self, msg, timestamp=None):
'''
Add a previously queued message.
This method is used to requeue messages when an asynchronous send
fails and we want to retry.
The requeued message is placed at the end of the requeued messages
and at the front of the messages that have never been removed.
If the queue is bounded and full, the message is dropped.
If timestamp isn't specified, the oldest unrequeued message is
used. If there are only requeued messages, then the youngest
requeued timestamp is used.
>>> r = Recipient('*****@*****.**')
# readd to an empty queue
>>> r.readd_msg('msg 1', timestamp=1)
>>> print r
[email protected] [READY]: 1: msg 1
>>> r.get_msg()
'msg 1'
# readd to a queue with no readded messages and not timestamp
# specified for the readded message
>>> r.add_msg('msg 3', timestamp=3)
>>> r.add_msg('msg 4', timestamp=4)
>>> print r
[email protected] [READY]:
3: msg 3
4: msg 4
>>> r.readd_msg('msg 1')
>>> print r
[email protected] [READY]:
3: msg 1
3: msg 3
4: msg 4
>>> r.get_msg()
'msg 1'
# readd to a queue with no readded messages and a timestamp
# specified for the readded message
>>> r.readd_msg('msg 1', timestamp=1)
>>> print r
[email protected] [READY]:
1: msg 1
3: msg 3
4: msg 4
# readd to a queue with readded messages and no timestamp
# specified for the readded message
>>> r.readd_msg('msg 2')
>>> print r
[email protected] [READY]:
1: msg 1
3: msg 2
3: msg 3
4: msg 4
# drain the queue
>>> while r.msgs:
... r.get_msg()
... assert r.readd_idx >= 0
'msg 1'
'msg 2'
'msg 3'
'msg 4'
'''
if self.msgs.maxlen and len(self.msgs) == self.msgs.maxlen:
# drop message ... the queue is full of younger messages
return
if timestamp is None:
if len(self.msgs) == 0:
# arbitrarily set the message timestamp to now
if self.max_age:
timestamp = time.time()
elif self.readd_idx >= len(self.msgs):
# use the time of the youngest *readded* message
timestamp = self.msgs[-1][0]
else:
# use the time of the oldest unreadded message
timestamp = self.msgs[self.readd_idx][0]
oldlen = len(self.msgs)
self.msgs.rotate(-self.readd_idx)
self.msgs.appendleft((timestamp, msg))
self.msgs.rotate(self.readd_idx)
self.readd_idx += 1
self.expire_msgs(timestamp)
if self.pending is not None and \
self._state == READY and \
oldlen == 0 and \
len(self.msgs) > 0:
log.trace('add %s to pending', self.addr)
self.pending.add(self)
def cmd_block(self, is_retry=False):
'''
Prepare the pre-check command to send to the subsystem
1. execute SHIM + command
2. check if SHIM returns a master request or if it completed
3. handle any master request
4. re-execute SHIM + command
5. split SHIM results from command results
6. return command results
'''
self.argv = _convert_args(self.argv)
log.debug('Performing shimmed, blocking command as follows:\n{0}'.format(' '.join(self.argv)))
cmd_str = self._cmd_str()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
log.trace('STDOUT {1}\n{0}'.format(stdout, self.target['host']))
log.trace('STDERR {1}\n{0}'.format(stderr, self.target['host']))
log.debug('RETCODE {1}: {0}'.format(retcode, self.target['host']))
error = self.categorize_shim_errors(stdout, stderr, retcode)
if error:
if error == 'Undefined SHIM state':
self.deploy()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr):
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin, undefined state: {0}'.format(stdout), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
else:
return 'ERROR: {0}'.format(error), stderr, retcode
# FIXME: this discards output from ssh_shim if the shim succeeds. It should
# always save the shim output regardless of shim success or failure.
if re.search(RSTR_RE, stdout):
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
else:
# This is actually an error state prior to the shim but let it fall through
pass
if re.search(RSTR_RE, stderr):
# Found RSTR in stderr which means SHIM completed and only
# and remaining output is only from salt.
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
else:
# RSTR was found in stdout but not stderr - which means there
# is a SHIM command for the master.
shim_command = re.split(r'\r?\n', stdout, 1)[0].strip()
log.debug('SHIM retcode({0}) and command: {1}'.format(retcode, shim_command))
if 'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY:
self.deploy()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr):
if not self.tty:
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode
elif not re.search(RSTR_RE, stdout):
# If RSTR is not seen in stdout with tty, then there
# was a thin deployment problem.
return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
if self.tty:
stderr = ''
else:
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
elif 'ext_mods' == shim_command:
self.deploy_ext()
stdout, stderr, retcode = self.shim_cmd(cmd_str)
if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr):
# If RSTR is not seen in both stdout and stderr then there
# was a thin deployment problem.
return 'ERROR: Failure deploying ext_mods: {0}'.format(stdout), stderr, retcode
stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
stderr = re.split(RSTR_RE, stderr, 1)[1].strip()
return stdout, stderr, retcode
