def subscription_callback(self, message, headers):
#The message body should only contain the event description for now and a standard header: "ION Event SMS"...
"""
This callback is given to all the event subscribers that this user wants notifications for.
If this callback gets called the user in this processor should get an email
"""
log.debug("UserEventProcessor.subscription_callback(): message=" + str(message))
log.debug("event type = " + str(message._get_type()))
log.debug('type of message: %s' % type(message))
time_stamp = str( datetime.fromtimestamp(time.mktime(time.gmtime(float(message.ts_created)/1000))))
event = message.type_
origin = message.origin
description = message.description
log.info("description: %s" % str(description))
# build the email from the event content
msg_body = "Description: %s" % description + '\r\n'
msg_subject = "(SysName: " + get_sys_name() + ") ION event " + event + " from " + origin
msg_sender = ION_NOTIFICATION_EMAIL_ADDRESS
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = msg_sender
msg['To'] = self.msg_recipient
log.debug("UserEventProcessor.subscription_callback(): sending email to %s"\
%self.msg_recipient)
self.smtp_client.sendmail(msg_sender, self.msg_recipient, msg.as_string())
def post_datastore(self, config):
# Make sure to detect that system was already bootstrapped.
# Look in datastore for secret cookie\
cookie_name = get_sys_name() + ".ION_INIT"
try:
res = self.clients.datastore.read_doc(cookie_name)
log.error("System %s already initialized: %s" % (get_sys_name(), res))
return
except iex.NotFound:
pass
# Now set the secret cookie
import time
cookie = dict(container=self.container.id, time=time.time())
cid, _ = self.clients.datastore.create_doc(cookie, cookie_name)
def post_datastore(self, config):
# Make sure to detect that system was already bootstrapped.
# Look in datastore for secret cookie\
cookie_name = get_sys_name() + ".ION_INIT"
try:
res = self.clients.datastore.read_doc(cookie_name)
log.error("System %s already initialized: %s" %
(get_sys_name(), res))
return
except iex.NotFound:
pass
# Now set the secret cookie
import time
cookie = dict(container=self.container.id, time=time.time())
cid, _ = self.clients.datastore.create_doc(cookie, cookie_name)
def send_email(message, msg_recipient, smtp_client):
'''
A common method to send email with formatting
@param message Event
@param msg_recipient str
@param smtp_client fake or real smtp client object
'''
time_stamp = message.ts_created
event = message.type_
origin = message.origin
description = message.description
#------------------------------------------------------------------------------------
# build the email from the event content
#------------------------------------------------------------------------------------
msg_body = string.join(("Event: %s," % event,
"",
"Originator: %s," % origin,
"",
"Description: %s," % description ,
"",
"Time stamp: %s," % time_stamp,
"",
"You received this notification from ION because you asked to be "\
"notified about this event from this source. ",
"To modify or remove notifications about this event, "\
"please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored "\
"and the emails will not be read."),
"\r\n")
msg_subject = "(SysName: " + get_sys_name() + ") ION event " + event + " from " + origin
#------------------------------------------------------------------------------------
# the 'from' email address for notification emails
#------------------------------------------------------------------------------------
ION_NOTIFICATION_EMAIL_ADDRESS = '*****@*****.**'
msg_sender = ION_NOTIFICATION_EMAIL_ADDRESS
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = msg_sender
msg['To'] = msg_recipient
log.debug("UserEventProcessor.subscription_callback(): sending email to %s"\
%msg_recipient)
smtp_sender = CFG.get_safe('server.smtp.sender')
smtp_client.sendmail(smtp_sender, msg_recipient, msg.as_string())
def subscription_callback(self, message, headers):
"""
This callback is given to all the event subscribers that this user wants notifications for.
If this callback gets called the user in this processor should get an email
"""
log.debug("UserEventProcessor.subscription_callback(): message=" +
str(message))
log.debug("event type = " + str(message._get_type()))
log.debug('type of message: %s' % type(message))
time_stamp = str(
datetime.fromtimestamp(
time.mktime(time.gmtime(float(message.ts_created) / 1000))))
event = message.type_
origin = message.origin
description = message.description
# build the email from the event content
msg_body = string.join(("Event: %s" % event,
"",
"Originator: %s" % origin,
"",
"Description: %s" % description ,
"",
"Time stamp: %s" % time_stamp,
"",
"You received this notification from ION because you asked to be "\
"notified about this event from this source. ",
"To modify or remove notifications about this event, "\
"please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored "\
"and the emails will not be read."),
"\r\n")
msg_subject = "(SysName: " + get_sys_name(
) + ") ION event " + event + " from " + origin
msg_sender = ION_NOTIFICATION_EMAIL_ADDRESS
# msg_recipient = self.user_email_addr
msg_recipient = self.notification._res_obj.delivery_config.delivery[
'email']
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = msg_sender
msg['To'] = msg_recipient
log.debug("UserEventProcessor.subscription_callback(): sending email to %s"\
%msg_recipient)
self.smtp_client.sendmail(self.smtp_sender, msg_recipient,
msg.as_string())
def format_and_send_email(self, events_for_message=None, user_id=None, smtp_client=None):
"""
Format the message for a particular user containing information about the events he is to be notified about
@param events_for_message list
@param user_id str
"""
message = str(events_for_message)
log.debug("The user, %s, will get the following events in his batch notification email: %s", user_id, message)
msg = convert_events_to_email_message(events_for_message, self.clients.resource_registry)
msg["Subject"] = "(SysName: " + get_sys_name() + ") ION event "
msg["To"] = self.user_info[user_id]['user_contact'].email
self.send_batch_email(msg, smtp_client)
def subscription_callback(self, message, headers):
"""
This callback is given to all the event subscribers that this user wants notifications for.
If this callback gets called the user in this processor should get an email
"""
log.debug("UserEventProcessor.subscription_callback(): message=" + str(message))
log.debug("event type = " + str(message._get_type()))
log.debug('type of message: %s' % type(message))
time_stamp = str( datetime.fromtimestamp(time.mktime(time.gmtime(float(message.ts_created)/1000))))
event = message.type_
origin = message.origin
description = message.description
# build the email from the event content
msg_body = string.join(("Event: %s" % event,
"",
"Originator: %s" % origin,
"",
"Description: %s" % description ,
"",
"Time stamp: %s" % time_stamp,
"",
"You received this notification from ION because you asked to be "\
"notified about this event from this source. ",
"To modify or remove notifications about this event, "\
"please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored "\
"and the emails will not be read."),
"\r\n")
msg_subject = "(SysName: " + get_sys_name() + ") ION event " + event + " from " + origin
msg_sender = ION_NOTIFICATION_EMAIL_ADDRESS
# msg_recipient = self.user_email_addr
msg_recipient = self.notification._res_obj.delivery_config.delivery['email']
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = msg_sender
msg['To'] = msg_recipient
log.debug("UserEventProcessor.subscription_callback(): sending email to %s"\
%msg_recipient)
self.smtp_client.sendmail(self.smtp_sender, msg_recipient, msg.as_string())
def format_and_send_email(self, events_for_message = None, user_id = None, smtp_client = None):
"""
Format the message for a particular user containing information about the events he is to be notified about
@param events_for_message list
@param user_id str
"""
message = str(events_for_message)
log.debug("The user, %s, will get the following events in his batch notification email: %s", user_id, message)
msg_body = ''
count = 1
for event in events_for_message:
ts_created = _convert_to_human_readable(event.ts_created)
msg_body += string.join(("\r\n",
"Event %s: %s" % (count, event),
"",
"Originator: %s" % event.origin,
"",
"Description: %s" % event.description or "Not provided",
"",
"ts_created: %s" % ts_created,
"\r\n",
"------------------------"
"\r\n"))
count += 1
msg_body += "You received this notification from ION because you asked to be " +\
"notified about this event from this source. " +\
"To modify or remove notifications about this event, " +\
"please access My Notifications Settings in the ION Web UI. " +\
"Do not reply to this email. This email address is not monitored " +\
"and the emails will not be read. \r\n "
log.debug("The email has the following message body: %s", msg_body)
msg_subject = "(SysName: " + get_sys_name() + ") ION event "
self.send_batch_email( msg_body = msg_body,
msg_subject = msg_subject,
msg_recipient=self.user_info[user_id]['user_contact'].email,
smtp_client=smtp_client )
def format_and_send_email(self, events_for_message = None, user_id = None, smtp_client = None):
"""
Format the message for a particular user containing information about the events he is to be notified about
@param events_for_message list
@param user_id str
"""
message = str(events_for_message)
log.debug("The user, %s, will get the following events in his batch notification email: %s", user_id, message)
msg_body = ''
count = 1
for event in events_for_message:
ts_created = _convert_to_human_readable(event.ts_created)
msg_body += string.join(("\r\n",
"Event %s: %s" % (count, event),
"",
"Originator: %s" % event.origin,
"",
"Description: %s" % event.description or "Not provided",
"",
"ts_created: %s" % ts_created,
"\r\n",
"------------------------"
"\r\n"))
count += 1
msg_body += "You received this notification from ION because you asked to be " +\
"notified about this event from this source. " +\
"To modify or remove notifications about this event, " +\
"please access My Notifications Settings in the ION Web UI. " +\
"Do not reply to this email. This email address is not monitored " +\
"and the emails will not be read. \r\n "
log.debug("The email has the following message body: %s", msg_body)
msg_subject = "(SysName: " + get_sys_name() + ") ION event "
self.send_batch_email( msg_body = msg_body,
msg_subject = msg_subject,
msg_recipient=self.user_info[user_id]['user_contact'].email,
smtp_client=smtp_client )
def on_start(self):
super(CacheLauncher, self).on_start()
tms_cli = TransformManagementServiceClient()
pubsub_cli = PubsubManagementServiceClient()
pd_cli = ProcessDispatcherServiceClient()
dname = CACHE_DATASTORE_NAME
number_of_workers = self.CFG.get_safe('process.number_of_workers', 2)
proc_def = ProcessDefinition(
name='last_update_worker_process',
description=
'Worker process for caching the last update from a stream')
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = pd_cli.create_process_definition(
process_definition=proc_def)
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError(
'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure'
% xs_dot_xp)
subscription_id = pubsub_cli.create_subscription(
query=ExchangeQuery(), exchange_name='last_update_cache')
config = {
'couch_storage': {
'datastore_name': dname,
'datastore_profile': 'SCIDATA'
}
}
for i in xrange(number_of_workers):
transform_id = tms_cli.create_transform(
name='last_update_cache%d' % i,
description=
'last_update that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config)
tms_cli.activate_transform(transform_id=transform_id)
def format_and_send_email(self,
events_for_message=None,
user_id=None,
smtp_client=None):
"""
Format the message for a particular user containing information about the events he is to be notified about
@param events_for_message list
@param user_id str
"""
message = str(events_for_message)
log.debug(
"The user, %s, will get the following events in his batch notification email: %s",
user_id, message)
msg = convert_events_to_email_message(events_for_message,
self.clients.resource_registry)
msg["Subject"] = "(SysName: " + get_sys_name() + ") ION event "
msg["To"] = self.user_info[user_id]['user_contact'].email
self.send_batch_email(msg, smtp_client)
def on_start(self):
super(CacheLauncher,self).on_start()
tms_cli = TransformManagementServiceClient()
pubsub_cli = PubsubManagementServiceClient()
pd_cli = ProcessDispatcherServiceClient()
dname = CACHE_DATASTORE_NAME
number_of_workers = self.CFG.get_safe('process.number_of_workers', 2)
proc_def = ProcessDefinition(name='last_update_worker_process',description='Worker process for caching the last update from a stream')
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = pd_cli.create_process_definition(process_definition=proc_def)
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
subscription_id = pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='last_update_cache')
config = {
'couch_storage' : {
'datastore_name' : dname,
'datastore_profile' : 'SCIDATA'
}
}
for i in xrange(number_of_workers):
transform_id = tms_cli.create_transform(
name='last_update_cache%d' % i,
description='last_update that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
tms_cli.activate_transform(transform_id=transform_id)
def format_and_send_email(self, events_for_message, user_name):
'''
Format the message for a particular user containing information about the events he is to be notified about
'''
message = str(events_for_message)
log.info("The user, %s, will get the following events in his batch notification email: %s" % (user_name, message))
msg_body = ''
count = 1
for event in events_for_message:
# build the email from the event content
msg_body += string.join(("\r\n",
"Event %s: %s" % (count, event),
"",
"Originator: %s" % event.origin,
"",
"Description: %s" % event.description ,
"",
"Event time stamp: %s" % event.ts_created,
"\r\n",
"------------------------"
"\r\n"))
count += 1
msg_body += "You received this notification from ION because you asked to be " + \
"notified about this event from this source. " + \
"To modify or remove notifications about this event, " + \
"please access My Notifications Settings in the ION Web UI. " + \
"Do not reply to this email. This email address is not monitored " + \
"and the emails will not be read. \r\n "
log.debug("The email has the following message body: %s" % msg_body)
msg_subject = "(SysName: " + get_sys_name() + ") ION event "
self.send_batch_email( msg_body = msg_body,
msg_subject = msg_subject,
msg_recipient=self.event_processor.user_info[user_name]['user_contact'].email,
smtp_client=self.smtp_client )
def subscription_callback(self, *args, **kwargs):
# this callback is given to all the event subscribers that this user wants notifications for
# if this callback gets called the user in this processor should get an email
log.debug("UserEventProcessor.subscription_callback(): args[0]=" + str(args[0]))
log.debug("event type = " + str(args[0]._get_type()))
origin = args[0].origin
event = str(args[0]._get_type())
description = args[0].description
time_stamp = str( datetime.fromtimestamp(time.mktime(time.gmtime(float(args[0].ts_created)/1000))))
# build the email from the event content
BODY = string.join(("Event: %s" % event,
"",
"Originator: %s" % origin,
"",
"Description: %s" % description,
"",
"Time stamp: %s" % time_stamp,
"",
"You received this notification from ION because you asked to be notified about this event from this source. ",
"To modify or remove notifications about this event, please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored and the emails will not be read."),
"\r\n")
SUBJECT = "(SysName: " + get_sys_name() + ") ION event " + event + " from " + origin
FROM = ION_NOTIFICATION_EMAIL_ADDRESS
TO = self.user_email_addr
msg = MIMEText(BODY)
msg['Subject'] = SUBJECT
msg['From'] = FROM
msg['To'] = TO
log.debug("UserEventProcessor.subscription_callback(): sending email to %s via %s" %(TO, self.smtp_server))
try:
smtp_client = smtplib.SMTP(self.smtp_server)
except Exception as ex:
log.warning("UserEventProcessor.subscription_callback(): failed to connect to SMTP server %s <%s>" %(ION_SMTP_SERVER, ex))
return
try:
smtp_client.sendmail(FROM, TO, msg.as_string())
except Exception as ex:
log.warning("UserEventProcessor.subscription_callback(): failed to send email to %s <%s>" %(TO, ex))
def report(self, stream):
""" all tests have completed but --with-pycc has not yet stopped external container.
request that containers log statistics now
"""
# initialize pyon so we can get system name
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
from pyon.public import get_sys_name, CFG
# make request: bin/pycc --sysname mgmt -x ion.processes.test.manage_system.ReportStats
null = open('/dev/null', 'w')
cmd = ['bin/pycc', '--sysname', get_sys_name(), '-x', 'ion.processes.test.manage_system.ReportStats' ]
status = subprocess.call(cmd, stdout=null, stderr=null)
if status==0:
stream.write('container statistics: a report request has been sent\n')
time.sleep(5) # give time to handle before container shutdown begins
else:
stream.write('container statistics: failed to send report request (logging anyway -- who needs a container?)\n')
from ooi.timer import get_accumulators
for a in get_accumulators().values():
a.log()
def subscription_callback(self, message, headers):
#The message body should only contain the event description for now and a standard header: "ION Event SMS"...
"""
This callback is given to all the event subscribers that this user wants notifications for.
If this callback gets called the user in this processor should get an email
"""
log.debug("UserEventProcessor.subscription_callback(): message=" +
str(message))
log.debug("event type = " + str(message._get_type()))
log.debug('type of message: %s' % type(message))
time_stamp = str(
datetime.fromtimestamp(
time.mktime(time.gmtime(float(message.ts_created) / 1000))))
event = message.type_
origin = message.origin
description = message.description
log.info("description: %s" % str(description))
# build the email from the event content
msg_body = "Description: %s" % description + '\r\n'
msg_subject = "(SysName: " + get_sys_name(
) + ") ION event " + event + " from " + origin
msg_sender = ION_NOTIFICATION_EMAIL_ADDRESS
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = msg_sender
msg['To'] = self.msg_recipient
log.debug("UserEventProcessor.subscription_callback(): sending email to %s"\
%self.msg_recipient)
self.smtp_client.sendmail(msg_sender, self.msg_recipient,
msg.as_string())
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
try:
from pyon.public import get_sys_name
sysname = get_sys_name()
# Force datastore loader to use the same sysname
from ion.processes.bootstrap.datastore_loader import DatastoreLoader
DatastoreLoader.clear_datastore(prefix=sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame = frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write(
'Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write(
'Child pid %d with exit status %d and signum %d\n' %
(pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = [
'bin/pycc', '--noshell', '-sp',
'--system.name=%s' % sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel, 'system.force_clean=False'
]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
DatastoreLoader.dump_datastore(path='res/dd')
debug.write('Dump child container state to file...\n')
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
# Make sure we initialize pyon before anything in this plugin executes
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
try:
from pyon.public import get_sys_name, CFG
self.sysname = get_sys_name()
# Clean exchanges and system queues out there
try:
rmh = RabbitManagementHelper(make_parser(), '-H %s -P 55672 -u %s -p %s -V %s'
% (CFG.server.amqp.host, CFG.server.amqp.username,
CFG.server.amqp.password, CFG.server.amqp.vhost))
exchanges = rmh.list_names('exchanges')
deleted = rmh.delete_names_with_prefix('exchange', exchanges, self.sysname)
debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted))
queues = rmh.list_names('queues')
deleted = rmh.delete_names_with_prefix('queue', queues, self.sysname)
debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted))
except Exception as e:
pass
# Force datastore loader to use the same sysname
from pyon.datastore.datastore_admin import DatastoreAdmin
self.datastore_admin = DatastoreAdmin(config=CFG)
self.datastore_admin.clear_datastore(prefix=self.sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame =frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write('Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = ['bin/pycc', '--noshell', '-sp', '--sysname=%s' % self.sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel,
"--config={'system': {'auto_bootstrap': True}}"]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
self.datastore_admin.dump_datastore(path='res/dd', compact=True)
debug.write('Dump child container state to file...\n')
# Clean again to make sure the first nosetest starts on a clean
# slate
self.datastore_admin.clear_datastore(prefix=self.sysname)
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def convert_events_to_email_message(events=None, rr_client=None):
if events is None: events = []
if 0 == len(events): raise BadRequest("Tried to convert events to email, but none were supplied")
web_ui_url = CFG.get_safe('system.web_ui_url', None)
log.debug("found CFG.system.web_ui_url = %s" % web_ui_url)
if web_ui_url is None:
web_ui_txt = ""
else:
web_ui_txt = ": %s" % web_ui_url
msg_body = ""
resource_human_readable = "<uninitialized string>"
for idx, event in enumerate(events, 1):
ts_created = convert_timestamp_to_human_readable(event.ts_created)
# build human readable resource string
resource_human_readable = "'%s' with ID='%s' (not found)" % (event.origin_type, event.origin)
try:
resource = rr_client.read(event.origin)
resource_human_readable = "%s '%s'" % (type(resource).__name__, resource.name)
except NotFound:
pass
if 1 == len(events):
eventtitle = "Type"
else:
eventtitle = "%s Type" % idx
msg_body += string.join(("\r\n",
"Event %s: %s" % (eventtitle, event.type_),
"",
"Resource: %s" % resource_human_readable,
"",
"Date & Time: %s" % ts_created,
"",
"Description: %s" % get_event_summary(event) or event.description or "Not provided",
# "",
# "Event object as a dictionary: %s," % str(event),
"\r\n",
"------------------------",
"\r\n",
), # necessary!
"\r\n")
msg_body += ("\r\n\r\nAutomated alert from the OOI ION system (%s). " % get_sys_name()) +\
"This notification was received based on " +\
"your current subscription settings for this event type from this resource. To unsubscribe " +\
"from notifications of this event type, please access the actions menu for the resource " +\
("listed above in the ION interface%s. \r\n\r\n" % web_ui_txt) +\
"Do not reply to this email. This email address is not monitored and the emails will not be read.\r\n"
#log.debug("The email has the following message body: %s", msg_body)
msg_subject = ""
if 1 == len(events):
msg_subject += "ION event " + events[0].type_ + " from " + resource_human_readable
else:
msg_subject += "summary of %s ION events" % len(events)
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
# msg['From'] = smtp_sender
# msg['To'] = msg_recipient
return msg
def on_init(self):
if not HighAvailabilityCore:
msg = "HighAvailabilityCore isn't available. Use autolaunch.cfg buildout"
log.error(msg)
return
cfg = self.CFG.get_safe("highavailability")
# use default PD name as the sole PD if none are provided in config
self.pds = self.CFG.get_safe("highavailability.process_dispatchers",
[ProcessDispatcherService.name])
if not len(self.pds) == 1:
raise Exception(
"HA Service doesn't support multiple Process Dispatchers")
self.process_definition_id, self.process_definition = self._get_process_definition(
)
self.process_configuration = self.CFG.get_safe(
"highavailability.process_configuration")
aggregator_config = _get_aggregator_config(self.CFG)
self.service_id, self.service_name = self._register_service()
self.policy_event = Event()
stored_policy = self._stored_policy
if stored_policy != {}:
policy_name = stored_policy.get('name')
policy_parameters = stored_policy.get('parameters')
self._validate_policy_name(policy_name)
self.policy_name = policy_name.lower()
self.policy_parameters = policy_parameters
else:
policy_name = self.CFG.get_safe("highavailability.policy.name")
self._validate_policy_name(policy_name)
self.policy_name = policy_name.lower()
self.policy_parameters = self.CFG.get_safe(
"highavailability.policy.parameters")
self.policy_interval = self.CFG.get_safe(
"highavailability.policy.interval", DEFAULT_INTERVAL)
self.logprefix = "HA Agent (%s): " % self.service_name
self.control = HAProcessControl(self.pds[0],
self.container.resource_registry,
self.service_id,
self.policy_event.set,
logprefix=self.logprefix)
self.core = HighAvailabilityCore(
cfg,
self.control,
self.pds,
self.policy_name,
process_definition_id=self.process_definition_id,
parameters=self.policy_parameters,
process_configuration=self.process_configuration,
aggregator_config=aggregator_config,
name=self.service_name)
dashi_messaging = self.CFG.get_safe("highavailability.dashi_messaging",
False)
if dashi_messaging:
dashi_name = self.CFG.get_safe("highavailability.dashi_name")
if not dashi_name:
raise Exception("dashi_name unknown")
dashi_uri = self.CFG.get_safe("highavailability.dashi_uri")
if not dashi_uri:
rabbit_host = self.CFG.get_safe("server.amqp.host")
rabbit_user = self.CFG.get_safe("server.amqp.username")
rabbit_pass = self.CFG.get_safe("server.amqp.password")
if not (rabbit_host and rabbit_user and rabbit_pass):
raise Exception("cannot form dashi URI")
dashi_uri = "amqp://%s:%s@%s/" % (rabbit_user, rabbit_pass,
rabbit_host)
dashi_exchange = self.CFG.get_safe(
"highavailability.dashi_exchange")
if not dashi_exchange:
dashi_exchange = get_sys_name()
self.dashi_handler = HADashiHandler(self, dashi_name, dashi_uri,
dashi_exchange)
else:
self.dashi_handler = None
def send_email(message, msg_recipient, smtp_client):
'''
A common method to send email with formatting
@param message Event
@param msg_recipient str
@param smtp_client fake or real smtp client object
'''
log.debug("Got type of event to notify on: %s", message.type_)
# Get the diffrent attributes from the event message
event = message.type_
origin = message.origin
description = message.description or "Not provided for this event"
event_obj_as_string = str(message)
ts_created = _convert_to_human_readable(message.ts_created)
#------------------------------------------------------------------------------------
# build the email from the event content
#------------------------------------------------------------------------------------
msg_body = string.join(("Event type: %s," % event,
"",
"Originator: %s," % origin,
"",
"Description: %s," % description,
"",
"ts_created: %s," % ts_created,
"",
"Event object as a dictionary: %s," % event_obj_as_string,
"",
"You received this notification from ION because you asked to be "\
"notified about this event from this source. ",
"To modify or remove notifications about this event, "\
"please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored "\
"and the emails will not be read."),
"\r\n")
msg_subject = "(SysName: " + get_sys_name() + ") ION event " + event + " from " + origin
log.debug("msg_body::: %s", msg_body)
#------------------------------------------------------------------------------------
# the 'from' email address for notification emails
#------------------------------------------------------------------------------------
ION_NOTIFICATION_EMAIL_ADDRESS = '*****@*****.**'
smtp_sender = CFG.get_safe('server.smtp.sender', ION_NOTIFICATION_EMAIL_ADDRESS)
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = smtp_sender
msg['To'] = msg_recipient
log.debug("UNS sending email from %s to %s for event type: %s", smtp_sender,msg_recipient, message.type_)
log.debug("UNS using the smtp client: %s", smtp_client)
try:
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
except: # Can be due to a broken connection... try to create a connection
smtp_client = setting_up_smtp_client()
log.debug("Connect again...message received after ehlo exchange: %s", str(smtp_client.ehlo()))
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
def on_restart(self, process, config, **kwargs):
"""
Handles bootstrapping of system restart for exchange resources and broker state.
- Ensures ExchangePoint and ExchangeSpace resources in system have a properly
declared AMQP exchange
- Ensures ExchangeName resources in system have a properly declared queue
- Logs all exchanges/queues it didn't understand
- Purges all service queues as long as no consumers are attached, or can be
overridden with force=True on pycc command line
"""
ex_manager = process.container.ex_manager
old_use_ems = ex_manager.use_ems
ex_manager.use_ems = False
sys_name = get_sys_name()
# get list of queues from broker with full props that have to do with our sysname
all_queues = ex_manager._list_queues()
queues = {
q['name']: q
for q in all_queues if q['name'].startswith(sys_name)
}
# get list of exchanges from broker with full props
all_exchanges = ex_manager._list_exchanges()
exchanges = {
e['name']: e
for e in all_exchanges if e['name'].startswith(sys_name)
}
# now get list of XOs from RR
xs_objs, _ = process.container.resource_registry.find_resources(
RT.ExchangeSpace)
xp_objs, _ = process.container.resource_registry.find_resources(
RT.ExchangePoint)
xn_objs, _ = process.container.resource_registry.find_resources(
RT.ExchangeName)
xs_by_xp = {}
assocs = process.container.resource_registry.find_associations(
predicate=PRED.hasExchangePoint, id_only=False)
for assoc in assocs:
if assoc.st == RT.ExchangeSpace and assoc.ot == RT.ExchangePoint:
xs_by_xp[assoc.o] = assoc.s
#
# VERIFY XSs have a declared exchange
#
rem_exchanges = set(exchanges)
xs_by_id = {}
for rrxs in xs_objs:
xs = ExchangeSpace(ex_manager, ex_manager._priviledged_transport,
rrxs.name)
xs_by_id[rrxs._id] = xs
if xs.exchange in rem_exchanges:
rem_exchanges.remove(xs.exchange)
else:
log.warn(
"BootstrapExchange restart: RR XS %s, id: %s NOT FOUND in exchanges",
rrxs.name, rrxs._id)
for rrxp in xp_objs:
xs_id = xs_by_xp.get(rrxp._id, None)
if not xs_id or xs_id not in xs_by_id:
log.warn("Inconsistent!! XS for XP %s not found", rrxp.name)
continue
xs = xs_by_id[xs_id]
xp = ExchangePoint(ex_manager, ex_manager._priviledged_transport,
rrxp.name, xs)
if xp.exchange in rem_exchanges:
rem_exchanges.remove(xp.exchange)
else:
log.warn(
"BootstrapExchange restart: RR XP %s, id %s NOT FOUND in exchanges",
rrxp.name, rrxp._id)
# TODO: WARNING this is based on hardcoded names
# events and main service exchange should be left
if sys_name in rem_exchanges:
rem_exchanges.remove(sys_name)
else:
log.warn("BootstrapExchange restart: no main service exchange %s",
sys_name)
event_ex = "%s.pyon.events" % sys_name
if event_ex in rem_exchanges:
rem_exchanges.remove(event_ex)
else:
log.warn("BootstrapExchange restart: no events exchange %s",
event_ex)
# what is left?
for exchange in rem_exchanges:
log.warn(
"BootstrapExchange restart: unknown exchange on broker %s",
exchange)
#
# VERIFY XNs have a declared queue
#
rem_queues = set(queues)
for rrxn in xn_objs:
# can instantiate ExchangeNames, don't need specific types
# @TODO: most queue types have a name instead of anon
"""
# @TODO: except queue type, which needs to be fixed to record declared name type
if rrxn.xn_type == "QUEUE":
log.info("TODO: queue type XNs, %s", rrxn.name)
continue
"""
exchange_space_list, assoc_list = process.container.resource_registry.find_subjects(
RT.ExchangeSpace, PRED.hasExchangeName, rrxn._id)
if not len(exchange_space_list) == 1:
raise StandardError(
"Association from ExchangeSpace to ExchangeName %s does not exist"
% rrxn._id)
rrxs = exchange_space_list[0]
xs = ExchangeSpace(ex_manager, ex_manager._priviledged_transport,
rrxs.name)
xn = ExchangeName(ex_manager, ex_manager._priviledged_transport,
rrxn.name, xs)
if xn.queue in rem_queues:
rem_queues.remove(xn.queue)
else:
log.warn(
"BootstrapExchange restart: RR XN %s, type %s NOT FOUND in queues",
xn.queue, xn.xn_type)
# get list of service name possibilities
svc_objs, _ = process.container.resource_registry.find_resources(
RT.ServiceDefinition)
svc_names = [s.name for s in svc_objs]
# PROCESS QUEUES + SERVICE QUEUES- not yet represented by resource
proc_queues = set()
svc_queues = set()
for queue in list(rem_queues):
# PROCESS QUEUES: proc manager spawned
# pattern "<sysname>.<containerid>.<pid>"
pieces = queue.split(".")
if len(pieces) > 2 and pieces[-1].isdigit():
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# SERVICE QUEUES
# pattern "<sysname>.<service name>"
if len(pieces) == 2:
if pieces[-1] in svc_names:
svc_queues.add(queue)
rem_queues.remove(queue)
# @TODO: PD-spawned process queues
# pattern "<sysname>.<service_name><hex>"
# leftover queues now
for queue in rem_queues:
log.warn("Unknown queue: %s", queue)
#
# EMPTY SERVICE QUEUES
#
for queue in svc_queues:
if int(queues[queue]['consumers']
) > 0 and not process.CFG.get_safe('force', False):
log.warn(
"Refusing to empty service queue %s with consumers (%s), specify force=True to override",
queue, queues[queue]['consumers'])
else:
ex_manager.purge_queue(queue)
log.info("Purged service queue %s of %s messages", queue,
queues[queue]['messages'])
ex_manager.use_ems = old_use_ems
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
# Make sure we initialize pyon before anything in this plugin executes
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
try:
from pyon.public import get_sys_name, CFG
self.sysname = get_sys_name()
# Clean exchanges and system queues out there
try:
connect_str = '-H %s -P 55672 -u %s -p %s -V %s' % (CFG.server.amqp.host,
CFG.server.amqp.username,
CFG.server.amqp.password,
CFG.server.amqp.vhost)
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, self.sysname)
debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted_exchanges))
debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted_queues))
except Exception as e:
pass
# Force datastore loader to use the same sysname
from pyon.datastore.datastore_admin import DatastoreAdmin
self.datastore_admin = DatastoreAdmin(config=CFG)
self.datastore_admin.clear_datastore(prefix=self.sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame =frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write('Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = ['bin/pycc', '-o', '--noshell', '-sp', '--sysname=%s' % self.sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel,
"--config={'system': {'auto_bootstrap': True}}"]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
self.datastore_admin.dump_datastore(path='res/dd', compact=True)
debug.write('Dump child container state to file...\n')
# Clean again to make sure the first nosetest starts on a clean
# slate
self.datastore_admin.clear_datastore(prefix=self.sysname)
# Set PYCC env var in case CEI needs to skip tests in pycc mode
os.environ['PYCC_MODE'] = '1'
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def convert_events_to_email_message(events=None,
notifications_map=None,
rr_client=None):
if events is None: events = []
# map event origins to resource objects to provide additional context in the email
event_origin_to_resource_map = {}
if 0 == len(events):
raise BadRequest(
"Tried to convert events to email, but none were supplied")
web_ui_url = CFG.get_safe('system.web_ui_url', None)
log.debug("found CFG.system.web_ui_url = %s" % web_ui_url)
if web_ui_url is None:
web_ui_txt = ""
else:
web_ui_txt = ": %s" % web_ui_url
msg_body = ""
#collect all the resources from the RR in one call
event_origin_to_resource_map = _collect_resources_from_event_origins(
events=None, rr_client=None)
resource_human_readable = "<uninitialized string>"
for idx, event in enumerate(events, 1):
ts_created = convert_timestamp_to_human_readable(event.ts_created)
# build human readable resource string
resource_human_readable = "'%s' with ID='%s' (not found)" % (
event.origin_type, event.origin)
notification_name = _get_notification_name(
event_id=event._id, notifications_map=notifications_map)
resource = ''
# pull the resource from the map if the origin id was found
if event.origin in event_origin_to_resource_map:
resource = event_origin_to_resource_map[event.origin]
resource_human_readable = "%s '%s'" % (type(resource).__name__,
resource.name)
if 1 == len(events):
eventtitle = "Type"
else:
eventtitle = "%s Type" % idx
msg_body += string.join(
(
"\r\n",
"Event %s: %s" % (eventtitle, event.type_),
"",
"Notification Request Name: %s" % notification_name,
"",
"Resource: %s" % resource_human_readable,
"",
"Date & Time: %s" % ts_created,
"",
"Description: %s" % get_event_summary(event)
or event.description or "Not provided",
# "",
# "Event object as a dictionary: %s," % str(event),
"\r\n",
"------------------------",
"\r\n",
), # necessary!
"\r\n")
msg_body += ("\r\n\r\nAutomated alert from the OOI ION system (%s). " % get_sys_name()) +\
"This notification was received based on " +\
"your current subscription settings for this event type from this resource. To unsubscribe " +\
"from notifications of this event type, please access the actions menu for the resource " +\
("listed above in the ION interface%s. \r\n\r\n" % web_ui_txt) +\
"Do not reply to this email. This email address is not monitored and the emails will not be read.\r\n"
#log.debug("The email has the following message body: %s", msg_body)
msg_subject = ""
if 1 == len(events):
msg_subject += "ION event " + events[
0].type_ + " from " + resource_human_readable
else:
msg_subject += "summary of %s ION events" % len(events)
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
# msg['From'] = smtp_sender
# msg['To'] = msg_recipient
return msg
def on_restart(self, process, config, **kwargs):
"""
Handles bootstrapping of system restart for exchange resources and broker state.
- Ensures ExchangePoint and ExchangeSpace resources in system have a properly
declared AMQP exchange
- Ensures ExchangeName resources in system have a properly declared queue
- Logs all exchanges/queues it didn't understand
- Purges all service queues as long as no consumers are attached, or can be
overridden with force=True on pycc command line
"""
ex_manager = process.container.ex_manager
old_use_ems = ex_manager.use_ems
ex_manager.use_ems = False
sys_name = get_sys_name()
# get list of queues from broker with full props that have to do with our sysname
all_queues = ex_manager._list_queues()
queues = {q["name"]: q for q in all_queues if q["name"].startswith(sys_name)}
# get list of exchanges from broker with full props
all_exchanges = ex_manager._list_exchanges()
exchanges = {e["name"]: e for e in all_exchanges if e["name"].startswith(sys_name)}
# now get list of XOs from RR
xs_objs, _ = process.container.resource_registry.find_resources(RT.ExchangeSpace)
xp_objs, _ = process.container.resource_registry.find_resources(RT.ExchangePoint)
xn_objs, _ = process.container.resource_registry.find_resources(RT.ExchangeName)
xs_by_xp = {}
assocs = process.container.resource_registry.find_associations(predicate=PRED.hasExchangePoint, id_only=False)
for assoc in assocs:
if assoc.st == RT.ExchangeSpace and assoc.ot == RT.ExchangePoint:
xs_by_xp[assoc.o] = assoc.s
#
# VERIFY XSs have a declared exchange
#
rem_exchanges = set(exchanges)
xs_by_id = {}
for rrxs in xs_objs:
xs = ExchangeSpace(ex_manager, ex_manager._priviledged_transport, rrxs.name)
xs_by_id[rrxs._id] = xs
if xs.exchange in rem_exchanges:
rem_exchanges.remove(xs.exchange)
else:
log.warn("BootstrapExchange restart: RR XS %s, id: %s NOT FOUND in exchanges", rrxs.name, rrxs._id)
for rrxp in xp_objs:
xs_id = xs_by_xp.get(rrxp._id, None)
if not xs_id or xs_id not in xs_by_id:
log.warn("Inconsistent!! XS for XP %s not found", rrxp.name)
continue
xs = xs_by_id[xs_id]
xp = ExchangePoint(ex_manager, ex_manager._priviledged_transport, rrxp.name, xs)
if xp.exchange in rem_exchanges:
rem_exchanges.remove(xp.exchange)
else:
log.warn("BootstrapExchange restart: RR XP %s, id %s NOT FOUND in exchanges", rrxp.name, rrxp._id)
# TODO: WARNING this is based on hardcoded names
# events and main service exchange should be left
if sys_name in rem_exchanges:
rem_exchanges.remove(sys_name)
else:
log.warn("BootstrapExchange restart: no main service exchange %s", sys_name)
event_ex = "%s.pyon.events" % sys_name
if event_ex in rem_exchanges:
rem_exchanges.remove(event_ex)
else:
log.warn("BootstrapExchange restart: no events exchange %s", event_ex)
# what is left?
for exchange in rem_exchanges:
log.warn("BootstrapExchange restart: unknown exchange on broker %s", exchange)
#
# VERIFY XNs have a declared queue
#
rem_queues = set(queues)
for rrxn in xn_objs:
# can instantiate ExchangeNames, don't need specific types
# @TODO: most queue types have a name instead of anon
"""
# @TODO: except queue type, which needs to be fixed to record declared name type
if rrxn.xn_type == "QUEUE":
log.info("TODO: queue type XNs, %s", rrxn.name)
continue
"""
exchange_space_list, assoc_list = process.container.resource_registry.find_subjects(
RT.ExchangeSpace, PRED.hasExchangeName, rrxn._id
)
if not len(exchange_space_list) == 1:
raise StandardError("Association from ExchangeSpace to ExchangeName %s does not exist" % rrxn._id)
rrxs = exchange_space_list[0]
xs = ExchangeSpace(ex_manager, ex_manager._priviledged_transport, rrxs.name)
xn = ExchangeName(ex_manager, ex_manager._priviledged_transport, rrxn.name, xs)
if xn.queue in rem_queues:
rem_queues.remove(xn.queue)
else:
log.warn("BootstrapExchange restart: RR XN %s, type %s NOT FOUND in queues", xn.queue, xn.xn_type)
# get list of service name possibilities
svc_objs, _ = process.container.resource_registry.find_resources(RT.ServiceDefinition)
svc_names = [s.name for s in svc_objs]
# PROCESS QUEUES + SERVICE QUEUES- not yet represented by resource
proc_queues = set()
svc_queues = set()
for queue in list(rem_queues):
# PROCESS QUEUES: proc manager spawned
# pattern "<sysname>.<containerid>.<pid>"
pieces = queue.split(".")
if len(pieces) > 2 and pieces[-1].isdigit():
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# SERVICE QUEUES
# pattern "<sysname>.<service name>"
if len(pieces) == 2:
if pieces[-1] in svc_names:
svc_queues.add(queue)
rem_queues.remove(queue)
# @TODO: PD-spawned process queues
# pattern "<sysname>.<service_name><hex>"
# leftover queues now
for queue in rem_queues:
log.warn("Unknown queue: %s", queue)
#
# EMPTY SERVICE QUEUES
#
for queue in svc_queues:
if int(queues[queue]["consumers"]) > 0 and not process.CFG.get_safe("force", False):
log.warn(
"Refusing to empty service queue %s with consumers (%s), specify force=True to override",
queue,
queues[queue]["consumers"],
)
else:
ex_manager.purge_queue(queue)
log.info("Purged service queue %s of %s messages", queue, queues[queue]["messages"])
ex_manager.use_ems = old_use_ems
def _get_dir_entry(self, scenario):
return dict(ts=get_ion_ts(), name=scenario, sys_name=get_sys_name())
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
try:
from pyon.public import get_sys_name
self.sysname = get_sys_name()
# Force datastore loader to use the same sysname
from ion.processes.bootstrap.datastore_loader import DatastoreLoader
DatastoreLoader.clear_datastore(prefix=self.sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame =frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write('Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = ['bin/pycc', '--noshell', '-sp', '--sysname=%s' %
self.sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel,
"--config={'system': {'force_clean': False, 'auto_bootstrap': True}}"]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
DatastoreLoader.dump_datastore(path='res/dd')
debug.write('Dump child container state to file...\n')
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def on_init(self):
if not HighAvailabilityCore:
msg = "HighAvailabilityCore isn't available. Use autolaunch.cfg buildout"
log.error(msg)
return
cfg = self.CFG.get_safe("highavailability")
# use default PD name as the sole PD if none are provided in config
self.pds = self.CFG.get_safe("highavailability.process_dispatchers",
[ProcessDispatcherService.name])
if not len(self.pds) == 1:
raise Exception("HA Service doesn't support multiple Process Dispatchers")
self.process_definition_id, self.process_definition = self._get_process_definition()
self.process_configuration = self.CFG.get_safe("highavailability.process_configuration")
aggregator_config = _get_aggregator_config(self.CFG)
self.service_id, self.service_name = self._register_service()
self.policy_event = Event()
stored_policy = self._stored_policy
if stored_policy != {}:
policy_name = stored_policy.get('name')
policy_parameters = stored_policy.get('parameters')
self._validate_policy_name(policy_name)
self.policy_name = policy_name.lower()
self.policy_parameters = policy_parameters
else:
policy_name = self.CFG.get_safe("highavailability.policy.name")
self._validate_policy_name(policy_name)
self.policy_name = policy_name.lower()
self.policy_parameters = self.CFG.get_safe("highavailability.policy.parameters")
self.policy_interval = self.CFG.get_safe("highavailability.policy.interval",
DEFAULT_INTERVAL)
self.logprefix = "HA Agent (%s): " % self.service_name
self.control = HAProcessControl(self.pds[0],
self.container.resource_registry, self.service_id,
self.policy_event.set, logprefix=self.logprefix)
self.core = HighAvailabilityCore(cfg, self.control,
self.pds, self.policy_name, process_definition_id=self.process_definition_id,
parameters=self.policy_parameters,
process_configuration=self.process_configuration,
aggregator_config=aggregator_config, name=self.service_name)
dashi_messaging = self.CFG.get_safe("highavailability.dashi_messaging", False)
if dashi_messaging:
dashi_name = self.CFG.get_safe("highavailability.dashi_name")
if not dashi_name:
raise Exception("dashi_name unknown")
dashi_uri = self.CFG.get_safe("highavailability.dashi_uri")
if not dashi_uri:
rabbit_host = self.CFG.get_safe("server.amqp.host")
rabbit_user = self.CFG.get_safe("server.amqp.username")
rabbit_pass = self.CFG.get_safe("server.amqp.password")
if not (rabbit_host and rabbit_user and rabbit_pass):
raise Exception("cannot form dashi URI")
dashi_uri = "amqp://%s:%s@%s/" % (rabbit_user, rabbit_pass,
rabbit_host)
dashi_exchange = self.CFG.get_safe("highavailability.dashi_exchange")
if not dashi_exchange:
dashi_exchange = get_sys_name()
self.dashi_handler = HADashiHandler(self, dashi_name, dashi_uri, dashi_exchange)
else:
self.dashi_handler = None
def on_restart(self, process, config, **kwargs):
"""
Handles bootstrapping of system restart for exchange resources and broker state.
- Ensures ExchangePoint and ExchangeSpace resources in system have a properly
declared AMQP exchange
- Ensures ExchangeName resources in system have a properly declared queue
- Logs all exchanges/queues it didn't understand
- Purges all service queues as long as no consumers are attached, or can be
overridden with force=True on pycc command line
"""
rr = process.container.resource_registry
ex_manager = process.container.ex_manager
sys_name = get_sys_name()
# get list of queues from broker with full props that have to do with our sysname
all_queues = ex_manager._list_queues()
queues = {q['name']: q for q in all_queues if q['name'].startswith(sys_name)}
# get list of exchanges from broker with full props
all_exchanges = ex_manager._list_exchanges()
exchanges = {e['name']: e for e in all_exchanges if e['name'].startswith(sys_name)}
# now get list of XOs from RR
xs_objs, _ = rr.find_resources(RT.ExchangeSpace)
xp_objs, _ = rr.find_resources(RT.ExchangePoint)
xn_objs, _ = rr.find_resources(RT.ExchangeName)
xs_by_xp = {}
assocs = rr.find_associations(predicate=PRED.hasExchangePoint, id_only=False)
for assoc in assocs:
if assoc.st == RT.ExchangeSpace and assoc.ot == RT.ExchangePoint:
xs_by_xp[assoc.o] = assoc.s
sys_xs_name = CFG.get_safe("exchange.core.system_xs", DEFAULT_SYSTEM_XS)
sys_node_name, sys_node = ex_manager._get_node_for_xs(sys_xs_name)
#
# VERIFY XSs have a declared exchange
#
rem_exchanges = set(exchanges)
xs_by_id = {}
for rrxs in xs_objs:
xs = ExchangeSpace(ex_manager, ex_manager._get_priv_transport(sys_node_name), sys_node, rrxs.name)
xs_by_id[rrxs._id] = xs
if xs.exchange in rem_exchanges:
rem_exchanges.remove(xs.exchange)
else:
log.warn("BootstrapExchange restart: RR XS %s, id=%s NOT FOUND in exchanges", rrxs.name, rrxs._id)
for rrxp in xp_objs:
xs_id = xs_by_xp.get(rrxp._id, None)
if not xs_id or xs_id not in xs_by_id:
log.warn("Inconsistent!! XS for XP %s not found", rrxp.name)
continue
xs = xs_by_id[xs_id]
xp = ExchangePoint(ex_manager, ex_manager._get_priv_transport(sys_node_name), sys_node, rrxp.name, xs)
if xp.exchange in rem_exchanges:
rem_exchanges.remove(xp.exchange)
else:
log.warn("BootstrapExchange restart: RR XP %s, id=%s NOT FOUND in exchanges", rrxp.name, rrxp._id)
# # events and main service exchange should be left
system_rpc_ex = "%s.%s" % (sys_name, sys_xs_name)
event_ex = "%s.%s.%s" % (sys_name, sys_xs_name, CFG.get_safe("exchange.core.events", DEFAULT_EVENTS_XP))
data_ex = "%s.%s.%s" % (sys_name, sys_xs_name, CFG.get_safe("exchange.core.data_streams", "data"))
if system_rpc_ex in rem_exchanges:
rem_exchanges.remove(system_rpc_ex)
if event_ex in rem_exchanges:
rem_exchanges.remove(event_ex)
if data_ex in rem_exchanges:
rem_exchanges.remove(data_ex)
# what is left?
for exchange in rem_exchanges:
log.warn("BootstrapExchange restart: unknown exchange on broker %s", exchange)
#
# VERIFY XNs have a declared queue
#
rem_queues = set(queues)
for rrxn in xn_objs:
# can instantiate ExchangeNames, don't need specific types
# @TODO: most queue types have a name instead of anon
"""
# @TODO: except queue type, which needs to be fixed to record declared name type
if rrxn.xn_type == "QUEUE":
log.info("TODO: queue type XNs, %s", rrxn.name)
continue
"""
exchange_space_list, _ = rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangeName, rrxn._id)
if not len(exchange_space_list) == 1:
raise StandardError("Association from ExchangeSpace to ExchangeName %s does not exist" % rrxn._id)
rrxs = exchange_space_list[0]
xs = ExchangeSpace(ex_manager, ex_manager._get_priv_transport(sys_node_name), sys_node, rrxs.name)
xn = ExchangeName(ex_manager, ex_manager._get_priv_transport(sys_node_name), sys_node, rrxn.name, xs)
if xn.queue in rem_queues:
rem_queues.remove(xn.queue)
else:
log.warn("BootstrapExchange restart: RR XN %s, type %s NOT FOUND in queues", xn.queue, xn.xn_type)
# get list of service name possibilities
svc_objs, _ = rr.find_resources(RT.ServiceDefinition)
svc_names = [s.name for s in svc_objs]
proc_objs, _ = rr.find_resources(RT.Process, id_only=False)
current_proc_names = [p.name for p in proc_objs]
cont_objs, _ = rr.find_resources(RT.CapabilityContainer, id_only=False)
current_containers = [c.name for c in cont_objs]
from pyon.ion.event import local_event_queues
# PROCESS QUEUES + SERVICE QUEUES - not yet represented by resource
proc_queues = set()
svc_queues = set()
event_queues = set()
for queue in list(rem_queues):
pieces = queue.split(".")
# EVENT QUEUES
if queue.startswith(event_ex) and pieces[-1] in local_event_queues:
event_queues.add(queue)
rem_queues.remove(queue)
continue
# CC AGENT QUEUES
if pieces[-1].startswith("cc_agent_") and pieces[-1][9:] in current_containers:
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# PROCESS QUEUES: proc manager spawned
# pattern "<sysname>.<root_xs>.<containerid>.<pid>"
if len(pieces) > 3 and pieces[-1].isdigit():
if "%s.%s" % (pieces[-2], pieces[-1]) in current_proc_names:
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# SERVICE QUEUES
# pattern "<sysname>.<root_xs>.<service name>"
if len(pieces) == 3:
if pieces[-1] in svc_names:
svc_queues.add(queue)
rem_queues.remove(queue)
# EMPTY LEFTOVER QUEUES - they are unaccounted for
for qn in rem_queues:
if int(queues[qn]['consumers']) == 0:
ex_manager.delete_queue(qn)
log.debug("Deleted unused queue: %s (%s messages)", qn, queues[qn]['messages'])
#
# EMPTY SERVICE QUEUES
#
for queue in svc_queues:
if int(queues[queue]['messages']) > 0:
ex_manager.purge_queue(queue)
log.info("Purged service queue %s (%s messages)", queue, queues[queue]['messages'])
def send_email(message, msg_recipient, smtp_client):
'''
A common method to send email with formatting
@param message Event
@param msg_recipient str
@param smtp_client fake or real smtp client object
'''
log.debug("Got type of event to notify on: %s", message.type_)
# Get the diffrent attributes from the event message
event = message.type_
origin = message.origin
description = message.description or "Not provided for this event"
event_obj_as_string = str(message)
ts_created = _convert_to_human_readable(message.ts_created)
#------------------------------------------------------------------------------------
# build the email from the event content
#------------------------------------------------------------------------------------
msg_body = string.join(("Event type: %s," % event,
"",
"Originator: %s," % origin,
"",
"Description: %s," % description,
"",
"ts_created: %s," % ts_created,
"",
"Event object as a dictionary: %s," % event_obj_as_string,
"",
"You received this notification from ION because you asked to be "\
"notified about this event from this source. ",
"To modify or remove notifications about this event, "\
"please access My Notifications Settings in the ION Web UI.",
"Do not reply to this email. This email address is not monitored "\
"and the emails will not be read."),
"\r\n")
msg_subject = "(SysName: " + get_sys_name(
) + ") ION event " + event + " from " + origin
log.debug("msg_body::: %s", msg_body)
#------------------------------------------------------------------------------------
# the 'from' email address for notification emails
#------------------------------------------------------------------------------------
ION_NOTIFICATION_EMAIL_ADDRESS = '*****@*****.**'
smtp_sender = CFG.get_safe('server.smtp.sender',
ION_NOTIFICATION_EMAIL_ADDRESS)
msg = MIMEText(msg_body)
msg['Subject'] = msg_subject
msg['From'] = smtp_sender
msg['To'] = msg_recipient
log.debug("UNS sending email from %s to %s for event type: %s",
smtp_sender, msg_recipient, message.type_)
log.debug("UNS using the smtp client: %s", smtp_client)
try:
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
except: # Can be due to a broken connection... try to create a connection
smtp_client = setting_up_smtp_client()
log.debug("Connect again...message received after ehlo exchange: %s",
str(smtp_client.ehlo()))
smtp_client.sendmail(smtp_sender, [msg_recipient], msg.as_string())
def on_init(self):
if not HighAvailabilityCore:
msg = "HighAvailabilityCore isn't available. Use autolaunch.cfg buildout"
log.error(msg)
return
log.debug("HighAvailabilityCore Pyon on_init")
policy_name = self.CFG.get_safe("highavailability.policy.name")
if policy_name is None:
msg = "HA service requires a policy name at CFG.highavailability.policy.name"
raise Exception(msg)
try:
self.policy = policy.policy_map[policy_name.lower()]
except KeyError:
raise Exception("HA Service doesn't support '%s' policy" % policy_name)
policy_parameters = self.CFG.get_safe("highavailability.policy.parameters")
self.policy_interval = self.CFG.get_safe("highavailability.policy.interval",
DEFAULT_INTERVAL)
cfg = self.CFG.get_safe("highavailability")
# use default PD name as the sole PD if none are provided in config
self.pds = self.CFG.get_safe("highavailability.process_dispatchers",
[ProcessDispatcherService.name])
self.process_definition_id = self.CFG.get_safe("highavailability.process_definition_id")
self.process_configuration = self.CFG.get_safe("highavailability.process_configuration")
aggregator_config = self.CFG.get_safe("highavailability.aggregator")
self.service_id = self._register_service()
# TODO: Allow other core class?
self.core = HighAvailabilityCore(cfg, ProcessDispatcherSimpleAPIClient,
self.pds, self.policy, process_definition_id=self.process_definition_id,
parameters=policy_parameters,
process_configuration=self.process_configuration,
aggregator_config=aggregator_config,
pd_client_kwargs={'container': self.container,
'service_id': self.service_id})
self.policy_thread = looping_call(self.policy_interval, self.core.apply_policy)
dashi_messaging = self.CFG.get_safe("highavailability.dashi_messaging", False)
if dashi_messaging:
dashi_name = self.CFG.get_safe("highavailability.dashi_name")
if not dashi_name:
raise Exception("dashi_name unknown")
dashi_uri = self.CFG.get_safe("highavailability.dashi_uri")
if not dashi_uri:
rabbit_host = self.CFG.get_safe("server.amqp.host")
rabbit_user = self.CFG.get_safe("server.amqp.username")
rabbit_pass = self.CFG.get_safe("server.amqp.password")
if not (rabbit_host and rabbit_user and rabbit_pass):
raise Exception("cannot form dashi URI")
dashi_uri = "amqp://%s:%s@%s/" % (rabbit_user, rabbit_pass,
rabbit_host)
dashi_exchange = self.CFG.get_safe("highavailability.dashi_exchange")
if not dashi_exchange:
dashi_exchange = get_sys_name()
self.dashi_handler = HADashiHandler(self, dashi_name, dashi_uri, dashi_exchange)
else:
self.dashi_handler = None
def on_restart(self, process, config, **kwargs):
"""
Handles bootstrapping of system restart for exchange resources and broker state.
- Ensures ExchangePoint and ExchangeSpace resources in system have a properly
declared AMQP exchange
- Ensures ExchangeName resources in system have a properly declared queue
- Logs all exchanges/queues it didn't understand
- Purges all service queues as long as no consumers are attached, or can be
overridden with force=True on pycc command line
"""
rr = process.container.resource_registry
ex_manager = process.container.ex_manager
sys_name = get_sys_name()
# get list of queues from broker with full props that have to do with our sysname
all_queues = ex_manager._list_queues()
queues = {
q['name']: q
for q in all_queues if q['name'].startswith(sys_name)
}
# get list of exchanges from broker with full props
all_exchanges = ex_manager._list_exchanges()
exchanges = {
e['name']: e
for e in all_exchanges if e['name'].startswith(sys_name)
}
# now get list of XOs from RR
xs_objs, _ = rr.find_resources(RT.ExchangeSpace)
xp_objs, _ = rr.find_resources(RT.ExchangePoint)
xn_objs, _ = rr.find_resources(RT.ExchangeName)
xs_by_xp = {}
assocs = rr.find_associations(predicate=PRED.hasExchangePoint,
id_only=False)
for assoc in assocs:
if assoc.st == RT.ExchangeSpace and assoc.ot == RT.ExchangePoint:
xs_by_xp[assoc.o] = assoc.s
sys_xs_name = CFG.get_safe("exchange.core.system_xs",
DEFAULT_SYSTEM_XS)
sys_node_name, sys_node = ex_manager._get_node_for_xs(sys_xs_name)
#
# VERIFY XSs have a declared exchange
#
rem_exchanges = set(exchanges)
xs_by_id = {}
for rrxs in xs_objs:
xs = ExchangeSpace(ex_manager,
ex_manager._get_priv_transport(sys_node_name),
sys_node, rrxs.name)
xs_by_id[rrxs._id] = xs
if xs.exchange in rem_exchanges:
rem_exchanges.remove(xs.exchange)
else:
log.warn(
"BootstrapExchange restart: RR XS %s, id=%s NOT FOUND in exchanges",
rrxs.name, rrxs._id)
for rrxp in xp_objs:
xs_id = xs_by_xp.get(rrxp._id, None)
if not xs_id or xs_id not in xs_by_id:
log.warn("Inconsistent!! XS for XP %s not found", rrxp.name)
continue
xs = xs_by_id[xs_id]
xp = ExchangePoint(ex_manager,
ex_manager._get_priv_transport(sys_node_name),
sys_node, rrxp.name, xs)
if xp.exchange in rem_exchanges:
rem_exchanges.remove(xp.exchange)
else:
log.warn(
"BootstrapExchange restart: RR XP %s, id=%s NOT FOUND in exchanges",
rrxp.name, rrxp._id)
# # events and main service exchange should be left
system_rpc_ex = "%s.%s" % (sys_name, sys_xs_name)
event_ex = "%s.%s.%s" % (sys_name, sys_xs_name,
CFG.get_safe("exchange.core.events",
DEFAULT_EVENTS_XP))
data_ex = "%s.%s.%s" % (sys_name, sys_xs_name,
CFG.get_safe("exchange.core.data_streams",
"data"))
if system_rpc_ex in rem_exchanges:
rem_exchanges.remove(system_rpc_ex)
if event_ex in rem_exchanges:
rem_exchanges.remove(event_ex)
if data_ex in rem_exchanges:
rem_exchanges.remove(data_ex)
# what is left?
for exchange in rem_exchanges:
log.warn(
"BootstrapExchange restart: unknown exchange on broker %s",
exchange)
#
# VERIFY XNs have a declared queue
#
rem_queues = set(queues)
for rrxn in xn_objs:
# can instantiate ExchangeNames, don't need specific types
# @TODO: most queue types have a name instead of anon
"""
# @TODO: except queue type, which needs to be fixed to record declared name type
if rrxn.xn_type == "QUEUE":
log.info("TODO: queue type XNs, %s", rrxn.name)
continue
"""
exchange_space_list, _ = rr.find_subjects(RT.ExchangeSpace,
PRED.hasExchangeName,
rrxn._id)
if not len(exchange_space_list) == 1:
raise StandardError(
"Association from ExchangeSpace to ExchangeName %s does not exist"
% rrxn._id)
rrxs = exchange_space_list[0]
xs = ExchangeSpace(ex_manager,
ex_manager._get_priv_transport(sys_node_name),
sys_node, rrxs.name)
xn = ExchangeName(ex_manager,
ex_manager._get_priv_transport(sys_node_name),
sys_node, rrxn.name, xs)
if xn.queue in rem_queues:
rem_queues.remove(xn.queue)
else:
log.warn(
"BootstrapExchange restart: RR XN %s, type %s NOT FOUND in queues",
xn.queue, xn.xn_type)
# get list of service name possibilities
svc_objs, _ = rr.find_resources(RT.ServiceDefinition)
svc_names = [s.name for s in svc_objs]
proc_objs, _ = rr.find_resources(RT.Process, id_only=False)
current_proc_names = [p.name for p in proc_objs]
cont_objs, _ = rr.find_resources(RT.CapabilityContainer, id_only=False)
current_containers = [c.name for c in cont_objs]
from pyon.ion.event import local_event_queues
# PROCESS QUEUES + SERVICE QUEUES - not yet represented by resource
proc_queues = set()
svc_queues = set()
event_queues = set()
for queue in list(rem_queues):
pieces = queue.split(".")
# EVENT QUEUES
if queue.startswith(event_ex) and pieces[-1] in local_event_queues:
event_queues.add(queue)
rem_queues.remove(queue)
continue
# CC AGENT QUEUES
if pieces[-1].startswith(
"cc_agent_") and pieces[-1][9:] in current_containers:
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# PROCESS QUEUES: proc manager spawned
# pattern "<sysname>.<root_xs>.<containerid>.<pid>"
if len(pieces) > 3 and pieces[-1].isdigit():
if "%s.%s" % (pieces[-2], pieces[-1]) in current_proc_names:
proc_queues.add(queue)
rem_queues.remove(queue)
continue
# SERVICE QUEUES
# pattern "<sysname>.<root_xs>.<service name>"
if len(pieces) == 3:
if pieces[-1] in svc_names:
svc_queues.add(queue)
rem_queues.remove(queue)
# LOCAL RPC QUEUES
# pattern "<sysname>.<root_xs>.rpc_<uuid>"
if len(pieces) == 3:
if pieces[-1].startswith("rpc_"):
rem_queues.remove(queue)
# EMPTY LEFTOVER QUEUES - they are unaccounted for
# TODO - current container used queues, e.g. process_dispatcher
for qn in rem_queues:
if int(queues[qn]['consumers']) == 0:
ex_manager.delete_queue(qn)
log.debug("Deleted unused queue: %s (%s messages)", qn,
queues[qn]['messages'])
#
# EMPTY SERVICE QUEUES
#
for queue in svc_queues:
if int(queues[queue]['messages']) > 0:
ex_manager.purge_queue(queue)
log.info("Purged service queue %s (%s messages)", queue,
queues[queue]['messages'])