class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = {}
self._base_url = '/webhooks/'
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return [trigger for trigger in six.itervalues(self._hooks)]
@jsexpose()
def get_one(self, name):
hook = self._hooks.get(name, None)
if not hook:
abort(http_client.NOT_FOUND)
return
return hook
@request_user_has_webhook_permission(permission_type=PermissionType.WEBHOOK_SEND)
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = '/'.join(args) # TODO: There must be a better way to do this.
# Note: For backward compatibility reasons we default to application/json if content
# type is not explicitly provided
content_type = pecan.request.headers.get('Content-Type', 'application/json')
body = pecan.request.body
try:
body = self._parse_request_body(content_type=content_type, body=body)
except Exception as e:
self._log_request('Failed to parse request body: %s.' % (str(e)), pecan.request)
msg = 'Failed to parse request body "%s": %s' % (body, str(e))
return pecan.abort(http_client.BAD_REQUEST, msg)
headers = self._get_headers_as_dict(pecan.request.headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(TRACE_TAG_HEADER, None),
hook=hook)
if hook == 'st2' or hook == 'st2/':
return self._handle_st2_webhook(body, trace_context=trace_context)
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', pecan.request)
msg = 'Webhook %s not registered with st2' % hook
return pecan.abort(http_client.NOT_FOUND, msg)
trigger = self._get_trigger_for_hook(hook)
payload = {}
payload['headers'] = headers
payload['body'] = body
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _parse_request_body(self, content_type, body):
if content_type == 'application/json':
self._log_request('Parsing request body as JSON', request=pecan.request)
body = json.loads(body)
elif content_type in ['application/x-www-form-urlencoded', 'multipart/form-data']:
self._log_request('Parsing request body as form encoded data', request=pecan.request)
body = urlparse.parse_qs(body)
else:
raise ValueError('Unsupported Content-Type: "%s"' % (content_type))
return body
def _handle_st2_webhook(self, body, trace_context):
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _get_trigger_for_hook(self, hook):
return self._hooks[hook]
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = 'webhook-%s-%s' % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = trigger['parameters']['url']
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks[url] = trigger
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = trigger['parameters']['url']
if url in self._hooks:
LOG.info('Stop listening to endpoint: %s', urljoin(self._base_url, url))
del self._hooks[url]
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg, headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class SensorWrapper(object):
def __init__(self, pack, file_path, class_name, trigger_types,
poll_interval=None, parent_args=None):
"""
:param pack: Name of the pack this sensor belongs to.
:type pack: ``str``
:param file_path: Path to the sensor module file.
:type file_path: ``str``
:param class_name: Sensor class name.
:type class_name: ``str``
:param trigger_types: A list of references to trigger types which
belong to this sensor.
:type trigger_types: ``list`` of ``str``
:param poll_interval: Sensor poll interval (in seconds).
:type poll_interval: ``int`` or ``None``
:param parent_args: Command line arguments passed to the parent process.
:type parse_args: ``list``
"""
self._pack = pack
self._file_path = file_path
self._class_name = class_name
self._trigger_types = trigger_types or []
self._poll_interval = poll_interval
self._parent_args = parent_args or []
self._trigger_names = {}
# 1. Parse the config with inherited parent args
try:
config.parse_args(args=self._parent_args)
except Exception:
pass
# 2. Establish DB connection
username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
username=username, password=password)
# 3. Instantiate the watcher
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='sensorwrapper')
# 4. Set up logging
self._logger = logging.getLogger('SensorWrapper.%s' %
(self._class_name))
logging.setup(cfg.CONF.sensorcontainer.logging)
self._sensor_instance = self._get_sensor_instance()
def run(self):
atexit.register(self.stop)
self._trigger_watcher.start()
self._logger.info('Watcher started')
self._logger.info('Running sensor initialization code')
self._sensor_instance.setup()
if self._poll_interval:
message = ('Running sensor in active mode (poll interval=%ss)' %
(self._poll_interval))
else:
message = 'Running sensor in passive mode'
self._logger.info(message)
try:
self._sensor_instance.run()
except Exception as e:
# Include traceback
msg = ('Sensor "%s" run method raised an exception: %s.' %
(self._class_name, str(e)))
self._logger.warn(msg, exc_info=True)
raise Exception(msg)
def stop(self):
# Stop watcher
self._logger.info('Stopping trigger watcher')
self._trigger_watcher.stop()
# Run sensor cleanup code
self._logger.info('Invoking cleanup on sensor')
self._sensor_instance.cleanup()
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
self._logger.debug('Calling sensor "add_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
self._logger.debug('Calling sensor "update_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
trigger_id = str(trigger.id)
if trigger_id not in self._trigger_names:
return
self._logger.debug('Calling sensor "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
del self._trigger_names[trigger_id]
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.remove_trigger(trigger=trigger)
def _get_sensor_instance(self):
"""
Retrieve instance of a sensor class.
"""
_, filename = os.path.split(self._file_path)
module_name, _ = os.path.splitext(filename)
sensor_class = loader.register_plugin_class(base_class=Sensor,
file_path=self._file_path,
class_name=self._class_name)
if not sensor_class:
raise ValueError('Sensor module is missing a class with name "%s"' %
(self._class_name))
sensor_class_kwargs = {}
sensor_class_kwargs['sensor_service'] = SensorService(sensor_wrapper=self)
sensor_config = self._get_sensor_config()
if self._pack not in SYSTEM_PACK_NAMES:
sensor_class_kwargs['config'] = sensor_config
if self._poll_interval and issubclass(sensor_class, PollingSensor):
sensor_class_kwargs['poll_interval'] = self._poll_interval
try:
sensor_instance = sensor_class(**sensor_class_kwargs)
except Exception as e:
raise Exception('Failed to instantiate "%s" sensor class: %s' %
(self._class_name, str(e)))
return sensor_instance
def _get_sensor_config(self):
config_parser = ContentPackConfigParser(pack_name=self._pack)
config = config_parser.get_sensor_config(sensor_file_path=self._file_path)
if config:
self._logger.info('Using config "%s" for sensor "%s"' % (config.file_path,
self._class_name))
return config.config
else:
self._logger.info('No config found for sensor "%s"' % (self._class_name))
return {}
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class SensorWrapper(object):
def __init__(self,
pack,
file_path,
class_name,
trigger_types,
poll_interval=None,
parent_args=None):
"""
:param pack: Name of the pack this sensor belongs to.
:type pack: ``str``
:param file_path: Path to the sensor module file.
:type file_path: ``str``
:param class_name: Sensor class name.
:type class_name: ``str``
:param trigger_types: A list of references to trigger types which
belong to this sensor.
:type trigger_types: ``list`` of ``str``
:param poll_interval: Sensor poll interval (in seconds).
:type poll_interval: ``int`` or ``None``
:param parent_args: Command line arguments passed to the parent process.
:type parse_args: ``list``
"""
self._pack = pack
self._file_path = file_path
self._class_name = class_name
self._trigger_types = trigger_types or []
self._poll_interval = poll_interval
self._parent_args = parent_args or []
self._trigger_names = {}
# 1. Parse the config with inherited parent args
try:
config.parse_args(args=self._parent_args)
except Exception:
pass
# 2. Establish DB connection
username = cfg.CONF.database.username if hasattr(
cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(
cfg.CONF.database, 'password') else None
db_setup_with_retry(
cfg.CONF.database.db_name,
cfg.CONF.database.host,
cfg.CONF.database.port,
username=username,
password=password,
ssl=cfg.CONF.database.ssl,
ssl_keyfile=cfg.CONF.database.ssl_keyfile,
ssl_certfile=cfg.CONF.database.ssl_certfile,
ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
authentication_mechanism=cfg.CONF.database.
authentication_mechanism,
ssl_match_hostname=cfg.CONF.database.ssl_match_hostname)
# 3. Instantiate the watcher
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='sensorwrapper_%s_%s' %
(self._pack, self._class_name),
exclusive=True)
# 4. Set up logging
self._logger = logging.getLogger('SensorWrapper.%s.%s' %
(self._pack, self._class_name))
logging.setup(cfg.CONF.sensorcontainer.logging)
if '--debug' in parent_args:
set_log_level_for_all_loggers()
else:
# NOTE: statsd logger logs everything by default under INFO so we ignore those log
# messages unless verbose / debug mode is used
logging.ignore_statsd_log_messages()
self._sensor_instance = self._get_sensor_instance()
def run(self):
atexit.register(self.stop)
self._trigger_watcher.start()
self._logger.info('Watcher started')
self._logger.info('Running sensor initialization code')
self._sensor_instance.setup()
if self._poll_interval:
message = ('Running sensor in active mode (poll interval=%ss)' %
(self._poll_interval))
else:
message = 'Running sensor in passive mode'
self._logger.info(message)
try:
self._sensor_instance.run()
except Exception as e:
# Include traceback
msg = ('Sensor "%s" run method raised an exception: %s.' %
(self._class_name, six.text_type(e)))
self._logger.warn(msg, exc_info=True)
raise Exception(msg)
def stop(self):
# Stop watcher
self._logger.info('Stopping trigger watcher')
self._trigger_watcher.stop()
# Run sensor cleanup code
self._logger.info('Invoking cleanup on sensor')
self._sensor_instance.cleanup()
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
self._logger.debug(
'Calling sensor "add_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
self._logger.debug(
'Calling sensor "update_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
trigger_id = str(trigger.id)
if trigger_id not in self._trigger_names:
return
self._logger.debug(
'Calling sensor "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
del self._trigger_names[trigger_id]
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.remove_trigger(trigger=trigger)
def _get_sensor_instance(self):
"""
Retrieve instance of a sensor class.
"""
_, filename = os.path.split(self._file_path)
module_name, _ = os.path.splitext(filename)
try:
sensor_class = loader.register_plugin_class(
base_class=Sensor,
file_path=self._file_path,
class_name=self._class_name)
except Exception as e:
tb_msg = traceback.format_exc()
msg = (
'Failed to load sensor class from file "%s" (sensor file most likely doesn\'t '
'exist or contains invalid syntax): %s' %
(self._file_path, six.text_type(e)))
msg += '\n\n' + tb_msg
exc_cls = type(e)
raise exc_cls(msg)
if not sensor_class:
raise ValueError(
'Sensor module is missing a class with name "%s"' %
(self._class_name))
sensor_class_kwargs = {}
sensor_class_kwargs['sensor_service'] = SensorService(
sensor_wrapper=self)
sensor_config = self._get_sensor_config()
sensor_class_kwargs['config'] = sensor_config
if self._poll_interval and issubclass(sensor_class, PollingSensor):
sensor_class_kwargs['poll_interval'] = self._poll_interval
try:
sensor_instance = sensor_class(**sensor_class_kwargs)
except Exception:
self._logger.exception('Failed to instantiate "%s" sensor class' %
(self._class_name))
raise Exception('Failed to instantiate "%s" sensor class' %
(self._class_name))
return sensor_instance
def _get_sensor_config(self):
config_loader = ContentPackConfigLoader(pack_name=self._pack)
config = config_loader.get_config()
if config:
self._logger.info('Found config for sensor "%s"' %
(self._class_name))
else:
self._logger.info('No config found for sensor "%s"' %
(self._class_name))
return config
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = HooksHolder()
self._base_url = '/webhooks/'
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return self._hooks.get_all()
@jsexpose()
def get_one(self, name):
triggers = self._hooks.get_triggers_for_hook(name)
if not triggers:
abort(http_client.NOT_FOUND)
return
# For demonstration purpose return 1st
return triggers[0]
@request_user_has_webhook_permission(
permission_type=PermissionType.WEBHOOK_SEND)
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = '/'.join(args) # TODO: There must be a better way to do this.
# Note: For backward compatibility reasons we default to application/json if content
# type is not explicitly provided
content_type = pecan.request.headers.get('Content-Type',
'application/json')
content_type = parse_content_type_header(content_type=content_type)[0]
body = pecan.request.body
try:
body = self._parse_request_body(content_type=content_type,
body=body)
except Exception as e:
self._log_request('Failed to parse request body: %s.' % (str(e)),
pecan.request)
msg = 'Failed to parse request body "%s": %s' % (body, str(e))
return pecan.abort(http_client.BAD_REQUEST, msg)
headers = self._get_headers_as_dict(pecan.request.headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(
TRACE_TAG_HEADER, None),
hook=hook)
if hook == 'st2' or hook == 'st2/':
return self._handle_st2_webhook(body, trace_context=trace_context)
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', pecan.request)
msg = 'Webhook %s not registered with st2' % hook
return pecan.abort(http_client.NOT_FOUND, msg)
triggers = self._hooks.get_triggers_for_hook(hook)
payload = {}
payload['headers'] = headers
payload['body'] = body
# Dispatch trigger instance for each of the trigger found
for trigger in triggers:
self._trigger_dispatcher.dispatch(trigger,
payload=payload,
trace_context=trace_context)
return body
def _parse_request_body(self, content_type, body):
if content_type == 'application/json':
self._log_request('Parsing request body as JSON',
request=pecan.request)
body = json.loads(body)
elif content_type in [
'application/x-www-form-urlencoded', 'multipart/form-data'
]:
self._log_request('Parsing request body as form encoded data',
request=pecan.request)
body = urlparse.parse_qs(body)
else:
raise ValueError('Unsupported Content-Type: "%s"' % (content_type))
return body
def _handle_st2_webhook(self, body, trace_context):
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger,
payload=payload,
trace_context=trace_context)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = 'webhook-%s-%s' % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks.add_hook(url, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
removed = self._hooks.remove_hook(url, trigger)
if removed:
LOG.info('Stop listening to endpoint: %s',
urljoin(self._base_url, url))
def _get_normalized_url(self, trigger):
"""
remove the trailing and leading / so that the hook url and those coming
from trigger parameters end up being the same.
"""
return trigger['parameters']['url'].strip('/')
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg,
headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class SensorWrapper(object):
def __init__(self, pack, file_path, class_name, trigger_types,
poll_interval=None, parent_args=None):
"""
:param pack: Name of the pack this sensor belongs to.
:type pack: ``str``
:param file_path: Path to the sensor module file.
:type file_path: ``str``
:param class_name: Sensor class name.
:type class_name: ``str``
:param trigger_types: A list of references to trigger types which
belong to this sensor.
:type trigger_types: ``list`` of ``str``
:param poll_interval: Sensor poll interval (in seconds).
:type poll_interval: ``int`` or ``None``
:param parent_args: Command line arguments passed to the parent process.
:type parse_args: ``list``
"""
self._pack = pack
self._file_path = file_path
self._class_name = class_name
self._trigger_types = trigger_types or []
self._poll_interval = poll_interval
self._parent_args = parent_args or []
self._trigger_names = {}
# 1. Parse the config with inherited parent args
try:
config.parse_args(args=self._parent_args)
except Exception:
pass
# 2. Establish DB connection
username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
db_setup_with_retry(cfg.CONF.database.db_name, cfg.CONF.database.host,
cfg.CONF.database.port, username=username, password=password,
ssl=cfg.CONF.database.ssl, ssl_keyfile=cfg.CONF.database.ssl_keyfile,
ssl_certfile=cfg.CONF.database.ssl_certfile,
ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
ssl_match_hostname=cfg.CONF.database.ssl_match_hostname)
# 3. Instantiate the watcher
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='sensorwrapper_%s_%s' %
(self._pack, self._class_name),
exclusive=True)
# 4. Set up logging
self._logger = logging.getLogger('SensorWrapper.%s.%s' %
(self._pack, self._class_name))
logging.setup(cfg.CONF.sensorcontainer.logging)
if '--debug' in parent_args:
set_log_level_for_all_loggers()
self._sensor_instance = self._get_sensor_instance()
def run(self):
atexit.register(self.stop)
self._trigger_watcher.start()
self._logger.info('Watcher started')
self._logger.info('Running sensor initialization code')
self._sensor_instance.setup()
if self._poll_interval:
message = ('Running sensor in active mode (poll interval=%ss)' %
(self._poll_interval))
else:
message = 'Running sensor in passive mode'
self._logger.info(message)
try:
self._sensor_instance.run()
except Exception as e:
# Include traceback
msg = ('Sensor "%s" run method raised an exception: %s.' %
(self._class_name, str(e)))
self._logger.warn(msg, exc_info=True)
raise Exception(msg)
def stop(self):
# Stop watcher
self._logger.info('Stopping trigger watcher')
self._trigger_watcher.stop()
# Run sensor cleanup code
self._logger.info('Invoking cleanup on sensor')
self._sensor_instance.cleanup()
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
self._logger.debug('Calling sensor "add_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
self._logger.debug('Calling sensor "update_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
trigger_id = str(trigger.id)
if trigger_id not in self._trigger_names:
return
self._logger.debug('Calling sensor "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
del self._trigger_names[trigger_id]
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.remove_trigger(trigger=trigger)
def _get_sensor_instance(self):
"""
Retrieve instance of a sensor class.
"""
_, filename = os.path.split(self._file_path)
module_name, _ = os.path.splitext(filename)
try:
sensor_class = loader.register_plugin_class(base_class=Sensor,
file_path=self._file_path,
class_name=self._class_name)
except Exception as e:
tb_msg = traceback.format_exc()
msg = ('Failed to load sensor class from file "%s" (sensor file most likely doesn\'t '
'exist or contains invalid syntax): %s' % (self._file_path, str(e)))
msg += '\n\n' + tb_msg
exc_cls = type(e)
raise exc_cls(msg)
if not sensor_class:
raise ValueError('Sensor module is missing a class with name "%s"' %
(self._class_name))
sensor_class_kwargs = {}
sensor_class_kwargs['sensor_service'] = SensorService(sensor_wrapper=self)
sensor_config = self._get_sensor_config()
sensor_class_kwargs['config'] = sensor_config
if self._poll_interval and issubclass(sensor_class, PollingSensor):
sensor_class_kwargs['poll_interval'] = self._poll_interval
try:
sensor_instance = sensor_class(**sensor_class_kwargs)
except Exception:
self._logger.exception('Failed to instantiate "%s" sensor class' % (self._class_name))
raise Exception('Failed to instantiate "%s" sensor class' % (self._class_name))
return sensor_instance
def _get_sensor_config(self):
config_loader = ContentPackConfigLoader(pack_name=self._pack)
config = config_loader.get_config()
if config:
self._logger.info('Found config for sensor "%s"' % (self._class_name))
else:
self._logger.info('No config found for sensor "%s"' % (self._class_name))
return config
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class WebhooksController(object):
def __init__(self, *args, **kwargs):
self._hooks = HooksHolder()
self._base_url = '/webhooks/'
self._trigger_types = list(WEBHOOK_TRIGGER_TYPES.keys())
self._trigger_dispatcher_service = TriggerDispatcherService(LOG)
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
def get_all(self):
# Return only the hooks known by this controller.
return self._hooks.get_all()
def get_one(self, url, requester_user):
triggers = self._hooks.get_triggers_for_hook(url)
if not triggers:
abort(http_client.NOT_FOUND)
return
permission_type = PermissionType.WEBHOOK_VIEW
rbac_utils.assert_user_has_resource_db_permission(user_db=requester_user,
resource_db=WebhookDB(name=url),
permission_type=permission_type)
# For demonstration purpose return 1st
return triggers[0]
def post(self, hook, webhook_body_api, headers, requester_user):
body = webhook_body_api.data
permission_type = PermissionType.WEBHOOK_SEND
rbac_utils.assert_user_has_resource_db_permission(user_db=requester_user,
resource_db=WebhookDB(name=hook),
permission_type=permission_type)
headers = self._get_headers_as_dict(headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(TRACE_TAG_HEADER, None),
hook=hook)
if hook == 'st2' or hook == 'st2/':
# When using st2 or system webhook, body needs to always be a dict
if not isinstance(body, dict):
type_string = get_json_type_for_python_value(body)
msg = ('Webhook body needs to be an object, got: %s' % (type_string))
raise ValueError(msg)
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher_service.dispatch_with_context(trigger=trigger,
payload=payload,
trace_context=trace_context,
throw_on_validation_error=True)
else:
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', headers, body)
msg = 'Webhook %s not registered with st2' % hook
return abort(http_client.NOT_FOUND, msg)
triggers = self._hooks.get_triggers_for_hook(hook)
payload = {}
payload['headers'] = headers
payload['body'] = body
# Dispatch trigger instance for each of the trigger found
for trigger_dict in triggers:
# TODO: Instead of dispatching the whole dict we should just
# dispatch TriggerDB.ref or similar
self._trigger_dispatcher_service.dispatch_with_context(trigger=trigger_dict,
payload=payload,
trace_context=trace_context,
throw_on_validation_error=True)
return Response(json=body, status=http_client.ACCEPTED)
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = 'webhook-%s-%s' % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# NOTE: trigger is a dictionary
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks.add_hook(url, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
removed = self._hooks.remove_hook(url, trigger)
if removed:
LOG.info('Stop listening to endpoint: %s', urljoin(self._base_url, url))
def _get_normalized_url(self, trigger):
"""
remove the trailing and leading / so that the hook url and those coming
from trigger parameters end up being the same.
"""
return trigger['parameters']['url'].strip('/')
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, headers, body, log_method=LOG.debug):
headers = self._get_headers_as_dict(headers)
body = str(body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg, headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class WebhooksController(pecan.rest.RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = {}
self._base_url = '/webhooks/'
self._trigger_types = [GENERIC_WEBHOOK_TRIGGER_REF]
self._trigger_dispatcher = TriggerDispatcher(LOG)
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types)
self._trigger_watcher.start()
@jsexpose(str, status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = '/'.join(args) # TODO: There must be a better way to do this.
LOG.info('POST /webhooks/ with hook=%s', hook)
if not self._is_valid_hook(hook):
msg = 'Webhook %s not registered with st2' % hook
return pecan.abort(http_client.NOT_FOUND, msg)
body = pecan.request.body
try:
body = json.loads(body)
except ValueError:
msg = 'Invalid JSON body: %s' % (body)
return pecan.abort(http_client.BAD_REQUEST, msg)
trigger = self._get_trigger_for_hook(hook)
payload = {}
payload['headers'] = self._get_headers_as_dict(pecan.request.headers)
payload['body'] = body
self._trigger_dispatcher.dispatch(trigger, payload=payload)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _get_trigger_for_hook(self, hook):
return self._hooks[hook]
def add_trigger(self, trigger):
url = trigger['parameters']['url']
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks[url] = trigger
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
url = trigger['parameters']['url']
if url in self._hooks:
LOG.info('Stop listening to endpoint: %s', urljoin(self._base_url, url))
del self._hooks[url]
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class TimersController(resource.ContentPackResourceController):
model = TriggerAPI
access = Trigger
supported_filters = {
'type': 'type',
}
query_options = {'sort': ['type']}
def __init__(self):
self._timers = TimersHolder()
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_timer_trigger_types()
self._allowed_timer_types = TIMER_TRIGGER_TYPES.keys()
def get_all(self, timer_type=None):
if timer_type and timer_type not in self._allowed_timer_types:
msg = 'Timer type %s not in supported types - %s.' % (
timer_type, self._allowed_timer_types)
abort(http_client.BAD_REQUEST, msg)
t_all = self._timers.get_all(timer_type=timer_type)
LOG.debug('Got timers: %s', t_all)
return t_all
def get_one(self, ref_or_id, requester_user):
try:
trigger_db = self._get_by_ref_or_id(ref_or_id=ref_or_id)
except Exception as e:
LOG.exception(six.text_type(e))
abort(http_client.NOT_FOUND, six.text_type(e))
return
permission_type = PermissionType.TIMER_VIEW
resource_db = TimerDB(pack=trigger_db.pack, name=trigger_db.name)
rbac_utils = get_rbac_backend().get_utils_class()
rbac_utils.assert_user_has_resource_db_permission(
user_db=requester_user,
resource_db=resource_db,
permission_type=permission_type)
result = self.model.from_model(trigger_db)
return result
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
LOG.info('Started timer %s with parameters %s', ref,
trigger['parameters'])
self._timers.add_trigger(ref, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
removed = self._timers.remove_trigger(ref, trigger)
if removed:
LOG.info('Stopped timer %s with parameters %s.', ref,
trigger['parameters'])
def _register_timer_trigger_types(self):
for trigger_type in TIMER_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _get_timer_ref(self, trigger):
return ResourceReference.to_string_reference(pack=trigger['pack'],
name=trigger['name'])
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class TimersController(resource.ContentPackResourceController):
model = TriggerAPI
access = Trigger
supported_filters = {
'type': 'type',
}
query_options = {'sort': ['type']}
def __init__(self, *args, **kwargs):
self._timers = TimersHolder()
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_timer_trigger_types()
self._allowed_timer_types = TIMER_TRIGGER_TYPES.keys()
@jsexpose()
def get_all(self, timer_type=None):
if timer_type and timer_type not in self._allowed_timer_types:
msg = 'Timer type %s not in supported types - %s.' % self._allowed_timer_types
abort(http_client.BAD_REQUEST, msg)
t_all = self._timers.get_all(timer_type=timer_type)
LOG.debug('Got timers: %s', t_all)
return t_all
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
LOG.info('Started timer %s with parameters %s', ref,
trigger['parameters'])
self._timers.add_trigger(ref, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
removed = self._timers.remove_trigger(ref, trigger)
if removed:
LOG.info('Stopped timer %s with parameters %s.', ref,
trigger['parameters'])
def _register_timer_trigger_types(self):
for trigger_type in TIMER_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _get_timer_ref(self, trigger):
return ResourceReference.to_string_reference(pack=trigger['pack'],
name=trigger['name'])
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class TimersController(resource.ContentPackResourceController):
model = TriggerAPI
access = Trigger
supported_filters = {"type": "type"}
query_options = {"sort": ["type"]}
def __init__(self, *args, **kwargs):
self._timers = TimersHolder()
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True,
)
self._trigger_watcher.start()
self._register_timer_trigger_types()
self._allowed_timer_types = TIMER_TRIGGER_TYPES.keys()
@jsexpose()
def get_all(self, timer_type=None):
if timer_type and timer_type not in self._allowed_timer_types:
msg = "Timer type %s not in supported types - %s." % self._allowed_timer_types
abort(http_client.BAD_REQUEST, msg)
t_all = self._timers.get_all(timer_type=timer_type)
LOG.debug("Got timers: %s", t_all)
return t_all
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
LOG.info("Started timer %s with parameters %s", ref, trigger["parameters"])
self._timers.add_trigger(ref, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
removed = self._timers.remove_trigger(ref, trigger)
if removed:
LOG.info("Stopped timer %s with parameters %s.", ref, trigger["parameters"])
def _register_timer_trigger_types(self):
for trigger_type in TIMER_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _get_timer_ref(self, trigger):
return ResourceReference.to_string_reference(pack=trigger["pack"], name=trigger["name"])
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class TimersController(resource.ContentPackResourceController):
model = TriggerAPI
access = Trigger
supported_filters = {
'type': 'type',
}
query_options = {
'sort': ['type']
}
def __init__(self):
self._timers = TimersHolder()
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_timer_trigger_types()
self._allowed_timer_types = TIMER_TRIGGER_TYPES.keys()
def get_all(self, timer_type=None):
if timer_type and timer_type not in self._allowed_timer_types:
msg = 'Timer type %s not in supported types - %s.' % (timer_type,
self._allowed_timer_types)
abort(http_client.BAD_REQUEST, msg)
t_all = self._timers.get_all(timer_type=timer_type)
LOG.debug('Got timers: %s', t_all)
return t_all
def get_one(self, ref_or_id, requester_user):
try:
trigger_db = self._get_by_ref_or_id(ref_or_id=ref_or_id)
except Exception as e:
LOG.exception(six.text_type(e))
abort(http_client.NOT_FOUND, six.text_type(e))
return
permission_type = PermissionType.TIMER_VIEW
resource_db = TimerDB(pack=trigger_db.pack, name=trigger_db.name)
rbac_utils.assert_user_has_resource_db_permission(user_db=requester_user,
resource_db=resource_db,
permission_type=permission_type)
result = self.model.from_model(trigger_db)
return result
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
LOG.info('Started timer %s with parameters %s', ref, trigger['parameters'])
self._timers.add_trigger(ref, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a timer is done during rule
# creation
ref = self._get_timer_ref(trigger)
removed = self._timers.remove_trigger(ref, trigger)
if removed:
LOG.info('Stopped timer %s with parameters %s.', ref, trigger['parameters'])
def _register_timer_trigger_types(self):
for trigger_type in TIMER_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _get_timer_ref(self, trigger):
return ResourceReference.to_string_reference(pack=trigger['pack'], name=trigger['name'])
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class St2Timer(object):
"""
A timer interface that uses APScheduler 3.0.
"""
def __init__(self, local_timezone=None):
self._timezone = local_timezone
self._scheduler = BlockingScheduler(timezone=self._timezone)
self._jobs = {}
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='timers')
self._trigger_dispatcher = TriggerDispatcher(LOG)
def start(self):
self._register_timer_trigger_types()
self._trigger_watcher.start()
self._scheduler.start()
def cleanup(self):
self._scheduler.shutdown(wait=True)
def add_trigger(self, trigger):
self._add_job_to_scheduler(trigger)
def update_trigger(self, trigger):
self.remove_trigger(trigger)
self.add_trigger(trigger)
def remove_trigger(self, trigger):
id = trigger['id']
try:
job_id = self._jobs[id]
except KeyError:
LOG.info('Job not found: %s', id)
return
self._scheduler.remove_job(job_id)
def _add_job_to_scheduler(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
try:
jsonschema.validate(trigger['parameters'],
trigger_type['parameters_schema'])
except jsonschema.ValidationError as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'],
e,
exc_info=True)
raise # Or should we just return?
time_spec = trigger['parameters']
time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone'))
time_type = None
if trigger_type['name'] == 'st2.IntervalTimer':
unit = time_spec.get('unit', None)
value = time_spec.get('delta', None)
time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone})
elif trigger_type['name'] == 'st2.DateTimer':
# Raises an exception if date string isn't a valid one.
dat = date_parser.parse(time_spec.get('date', None))
time_type = DateTrigger(dat, timezone=time_zone)
elif trigger_type['name'] == 'st2.CronTimer':
cron = time_spec.copy()
cron['timezone'] = time_zone
time_type = CronTrigger(**cron)
if hasattr(time_type,
'run_date') and datetime.now(tzutc()) > time_type.run_date:
LOG.warning('Not scheduling expired timer: %s : %s',
trigger['parameters'], time_type.run_date)
else:
self._add_job(trigger, time_type)
def _add_job(self, trigger, time_type, replace=True):
try:
job = self._scheduler.add_job(self._emit_trigger_instance,
trigger=time_type,
args=[trigger],
replace_existing=replace)
LOG.info('Job %s scheduled.', job.id)
self._jobs[trigger['id']] = job.id
except Exception as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'],
e,
exc_info=True)
def _emit_trigger_instance(self, trigger):
LOG.info('Timer fired at: %s. Trigger: %s', str(datetime.utcnow()),
trigger)
payload = {
'executed_at': str(datetime.utcnow()),
'schedule': trigger['parameters'].get('time')
}
self._trigger_dispatcher.dispatch(trigger, payload)
def _register_timer_trigger_types(self):
return container_utils.add_trigger_models(TIMER_TRIGGER_TYPES.values())
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = {}
self._base_url = '/webhooks/'
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='webhooks')
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return [trigger for trigger in six.itervalues(self._hooks)]
@jsexpose()
def get_one(self, name):
hook = self._hooks.get(name, None)
if not hook:
abort(http_client.NOT_FOUND)
return
return hook
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = '/'.join(args) # TODO: There must be a better way to do this.
body = pecan.request.body
try:
body = json.loads(body)
except ValueError:
self._log_request('Invalid JSON body.', pecan.request)
msg = 'Invalid JSON body: %s' % (body)
return pecan.abort(http_client.BAD_REQUEST, msg)
if hook == 'st2' or hook == 'st2/':
return self._handle_st2_webhook(body)
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', pecan.request)
msg = 'Webhook %s not registered with st2' % hook
return pecan.abort(http_client.NOT_FOUND, msg)
trigger = self._get_trigger_for_hook(hook)
payload = {}
payload['headers'] = self._get_headers_as_dict(pecan.request.headers)
payload['body'] = body
self._trigger_dispatcher.dispatch(trigger, payload=payload)
return body
def _handle_st2_webhook(self, body):
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger, payload=payload)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _get_trigger_for_hook(self, hook):
return self._hooks[hook]
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def add_trigger(self, trigger):
url = trigger['parameters']['url']
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks[url] = trigger
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
url = trigger['parameters']['url']
if url in self._hooks:
LOG.info('Stop listening to endpoint: %s', urljoin(self._base_url, url))
del self._hooks[url]
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg, headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = HooksHolder()
self._base_url = "/webhooks/"
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True,
)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return self._hooks.get_all()
@jsexpose()
def get_one(self, name):
triggers = self._hooks.get_triggers_for_hook(name)
if not triggers:
abort(http_client.NOT_FOUND)
return
# For demonstration purpose return 1st
return triggers[0]
@request_user_has_webhook_permission(permission_type=PermissionType.WEBHOOK_SEND)
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = "/".join(args) # TODO: There must be a better way to do this.
# Note: For backward compatibility reasons we default to application/json if content
# type is not explicitly provided
content_type = pecan.request.headers.get("Content-Type", "application/json")
content_type = parse_content_type_header(content_type=content_type)[0]
body = pecan.request.body
try:
body = self._parse_request_body(content_type=content_type, body=body)
except Exception as e:
self._log_request("Failed to parse request body: %s." % (str(e)), pecan.request)
msg = 'Failed to parse request body "%s": %s' % (body, str(e))
return pecan.abort(http_client.BAD_REQUEST, msg)
headers = self._get_headers_as_dict(pecan.request.headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(TRACE_TAG_HEADER, None), hook=hook)
if hook == "st2" or hook == "st2/":
return self._handle_st2_webhook(body, trace_context=trace_context)
if not self._is_valid_hook(hook):
self._log_request("Invalid hook.", pecan.request)
msg = "Webhook %s not registered with st2" % hook
return pecan.abort(http_client.NOT_FOUND, msg)
triggers = self._hooks.get_triggers_for_hook(hook)
payload = {}
payload["headers"] = headers
payload["body"] = body
# Dispatch trigger instance for each of the trigger found
for trigger in triggers:
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _parse_request_body(self, content_type, body):
if content_type == "application/json":
self._log_request("Parsing request body as JSON", request=pecan.request)
body = json.loads(body)
elif content_type in ["application/x-www-form-urlencoded", "multipart/form-data"]:
self._log_request("Parsing request body as form encoded data", request=pecan.request)
body = urlparse.parse_qs(body)
else:
raise ValueError('Unsupported Content-Type: "%s"' % (content_type))
return body
def _handle_st2_webhook(self, body, trace_context):
trigger = body.get("trigger", None)
payload = body.get("payload", None)
if not trigger:
msg = "Trigger not specified."
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = "webhook-%s-%s" % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
LOG.info("Listening to endpoint: %s", urljoin(self._base_url, url))
self._hooks.add_hook(url, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
removed = self._hooks.remove_hook(url, trigger)
if removed:
LOG.info("Stop listening to endpoint: %s", urljoin(self._base_url, url))
def _get_normalized_url(self, trigger):
"""
remove the trailing and leading / so that the hook url and those coming
from trigger parameters end up being the same.
"""
return trigger["parameters"]["url"].strip("/")
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method("%s\n\trequest.header: %s.\n\trequest.body: %s.", msg, headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if "id" in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized["id"] = str(sanitized["id"])
return sanitized
class St2Timer(object):
"""
A timer interface that uses APScheduler 3.0.
"""
def __init__(self, local_timezone=None):
self._timezone = local_timezone
self._scheduler = BlockingScheduler(timezone=self._timezone)
self._jobs = {}
self._trigger_types = list(TIMER_TRIGGER_TYPES.keys())
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=self.__class__.__name__,
exclusive=True)
self._trigger_dispatcher = TriggerDispatcher(LOG)
def start(self):
self._register_timer_trigger_types()
self._trigger_watcher.start()
self._scheduler.start()
def cleanup(self):
self._scheduler.shutdown(wait=True)
def add_trigger(self, trigger):
self._add_job_to_scheduler(trigger)
def update_trigger(self, trigger):
self.remove_trigger(trigger)
self.add_trigger(trigger)
def remove_trigger(self, trigger):
trigger_id = trigger['id']
try:
job_id = self._jobs[trigger_id]
except KeyError:
LOG.info('Job not found: %s', trigger_id)
return
self._scheduler.remove_job(job_id)
del self._jobs[trigger_id]
def _add_job_to_scheduler(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
try:
util_schema.validate(instance=trigger['parameters'],
schema=trigger_type['parameters_schema'],
cls=util_schema.CustomValidator,
use_default=True,
allow_default_none=True)
except jsonschema.ValidationError as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'],
e,
exc_info=True)
raise # Or should we just return?
time_spec = trigger['parameters']
time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone'))
time_type = None
if trigger_type['name'] == 'st2.IntervalTimer':
unit = time_spec.get('unit', None)
value = time_spec.get('delta', None)
time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone})
elif trigger_type['name'] == 'st2.DateTimer':
# Raises an exception if date string isn't a valid one.
dat = date_parser.parse(time_spec.get('date', None))
time_type = DateTrigger(dat, timezone=time_zone)
elif trigger_type['name'] == 'st2.CronTimer':
cron = time_spec.copy()
cron['timezone'] = time_zone
time_type = CronTrigger(**cron)
utc_now = date_utils.get_datetime_utc_now()
if hasattr(time_type, 'run_date') and utc_now > time_type.run_date:
LOG.warning('Not scheduling expired timer: %s : %s',
trigger['parameters'], time_type.run_date)
else:
self._add_job(trigger, time_type)
return time_type
def _add_job(self, trigger, time_type, replace=True):
try:
job = self._scheduler.add_job(self._emit_trigger_instance,
trigger=time_type,
args=[trigger],
replace_existing=replace)
LOG.info('Job %s scheduled.', job.id)
self._jobs[trigger['id']] = job.id
except Exception as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'],
e,
exc_info=True)
def _emit_trigger_instance(self, trigger):
utc_now = date_utils.get_datetime_utc_now()
# debug logging is reasonable for this one. A high resolution timer will end up
# trashing standard logs.
LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger)
payload = {
'executed_at': str(utc_now),
'schedule': trigger['parameters'].get('time')
}
trace_context = TraceContext(trace_tag='%s-%s' %
(self._get_trigger_type_name(trigger),
trigger.get('name',
uuid.uuid4().hex)))
self._trigger_dispatcher.dispatch(trigger,
payload,
trace_context=trace_context)
def _get_trigger_type_name(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
return trigger_type['name']
def _register_timer_trigger_types(self):
return trigger_services.add_trigger_models(
list(TIMER_TRIGGER_TYPES.values()))
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = {}
self._base_url = '/webhooks/'
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='webhooks')
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return [trigger for trigger in six.itervalues(self._hooks)]
@jsexpose()
def get_one(self, name):
hook = self._hooks.get(name, None)
if not hook:
abort(http_client.NOT_FOUND)
return
return hook
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = '/'.join(args) # TODO: There must be a better way to do this.
body = pecan.request.body
try:
body = json.loads(body)
except ValueError:
self._log_request('Invalid JSON body.', pecan.request)
msg = 'Invalid JSON body: %s' % (body)
return pecan.abort(http_client.BAD_REQUEST, msg)
if hook == 'st2' or hook == 'st2/':
return self._handle_st2_webhook(body)
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', pecan.request)
msg = 'Webhook %s not registered with st2' % hook
return pecan.abort(http_client.NOT_FOUND, msg)
trigger = self._get_trigger_for_hook(hook)
payload = {}
payload['headers'] = self._get_headers_as_dict(pecan.request.headers)
payload['body'] = body
self._trigger_dispatcher.dispatch(trigger, payload=payload)
return body
def _handle_st2_webhook(self, body):
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger, payload=payload)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _get_trigger_for_hook(self, hook):
return self._hooks[hook]
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def add_trigger(self, trigger):
url = trigger['parameters']['url']
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks[url] = trigger
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
url = trigger['parameters']['url']
if url in self._hooks:
LOG.info('Stop listening to endpoint: %s',
urljoin(self._base_url, url))
del self._hooks[url]
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg,
headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class SensorWrapper(object):
def __init__(self,
pack,
file_path,
class_name,
trigger_types,
poll_interval=None,
parent_args=None):
"""
:param pack: Name of the pack this sensor belongs to.
:type pack: ``str``
:param file_path: Path to the sensor module file.
:type file_path: ``str``
:param class_name: Sensor class name.
:type class_name: ``str``
:param trigger_types: A list of references to trigger types which
belong to this sensor.
:type trigger_types: ``list`` of ``str``
:param poll_interval: Sensor poll interval (in seconds).
:type poll_interval: ``int`` or ``None``
:param parent_args: Command line arguments passed to the parent process.
:type parse_args: ``list``
"""
self._pack = pack
self._file_path = file_path
self._class_name = class_name
self._trigger_types = trigger_types or []
self._poll_interval = poll_interval
self._parent_args = parent_args or []
self._trigger_names = {}
# 1. Parse the config with inherited parent args
try:
config.parse_args(args=self._parent_args)
except Exception:
pass
# 2. Establish DB connection
username = cfg.CONF.database.username if hasattr(
cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(
cfg.CONF.database, 'password') else None
db_setup_with_retry(cfg.CONF.database.db_name,
cfg.CONF.database.host,
cfg.CONF.database.port,
username=username,
password=password)
# 3. Instantiate the watcher
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix='sensorwrapper_%s_%s' %
(self._pack, self._class_name),
exclusive=True)
# 4. Set up logging
self._logger = logging.getLogger('SensorWrapper.%s' %
(self._class_name))
logging.setup(cfg.CONF.sensorcontainer.logging)
if '--debug' in parent_args:
set_log_level_for_all_loggers()
self._sensor_instance = self._get_sensor_instance()
def run(self):
atexit.register(self.stop)
self._trigger_watcher.start()
self._logger.info('Watcher started')
self._logger.info('Running sensor initialization code')
self._sensor_instance.setup()
if self._poll_interval:
message = ('Running sensor in active mode (poll interval=%ss)' %
(self._poll_interval))
else:
message = 'Running sensor in passive mode'
self._logger.info(message)
try:
self._sensor_instance.run()
except Exception as e:
# Include traceback
msg = ('Sensor "%s" run method raised an exception: %s.' %
(self._class_name, str(e)))
self._logger.warn(msg, exc_info=True)
raise Exception(msg)
def stop(self):
# Stop watcher
self._logger.info('Stopping trigger watcher')
self._trigger_watcher.stop()
# Run sensor cleanup code
self._logger.info('Invoking cleanup on sensor')
self._sensor_instance.cleanup()
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
self._logger.debug(
'Calling sensor "add_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
self._logger.debug(
'Calling sensor "update_trigger" method (trigger.type=%s)' %
(trigger.type))
self._trigger_names[str(trigger.id)] = trigger
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
trigger_id = str(trigger.id)
if trigger_id not in self._trigger_names:
return
self._logger.debug(
'Calling sensor "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
del self._trigger_names[trigger_id]
trigger = self._sanitize_trigger(trigger=trigger)
self._sensor_instance.remove_trigger(trigger=trigger)
def _get_sensor_instance(self):
"""
Retrieve instance of a sensor class.
"""
_, filename = os.path.split(self._file_path)
module_name, _ = os.path.splitext(filename)
sensor_class = loader.register_plugin_class(
base_class=Sensor,
file_path=self._file_path,
class_name=self._class_name)
if not sensor_class:
raise ValueError(
'Sensor module is missing a class with name "%s"' %
(self._class_name))
sensor_class_kwargs = {}
sensor_class_kwargs['sensor_service'] = SensorService(
sensor_wrapper=self)
sensor_config = self._get_sensor_config()
sensor_class_kwargs['config'] = sensor_config
if self._poll_interval and issubclass(sensor_class, PollingSensor):
sensor_class_kwargs['poll_interval'] = self._poll_interval
try:
sensor_instance = sensor_class(**sensor_class_kwargs)
except Exception:
self._logger.exception('Failed to instantiate "%s" sensor class' %
(self._class_name))
raise Exception('Failed to instantiate "%s" sensor class' %
(self._class_name))
return sensor_instance
def _get_sensor_config(self):
config_parser = ContentPackConfigParser(pack_name=self._pack)
config = config_parser.get_sensor_config(
sensor_file_path=self._file_path)
if config:
self._logger.info('Using config "%s" for sensor "%s"' %
(config.file_path, self._class_name))
return config.config
else:
self._logger.info('No config found for sensor "%s"' %
(self._class_name))
return {}
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class St2Timer(object):
"""
A timer interface that uses APScheduler 3.0.
"""
def __init__(self, local_timezone=None):
self._timezone = local_timezone
self._scheduler = BlockingScheduler(timezone=self._timezone)
self._jobs = {}
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=self.__class__.__name__,
exclusive=True)
self._trigger_dispatcher = TriggerDispatcher(LOG)
def start(self):
self._register_timer_trigger_types()
self._trigger_watcher.start()
self._scheduler.start()
def cleanup(self):
self._scheduler.shutdown(wait=True)
def add_trigger(self, trigger):
self._add_job_to_scheduler(trigger)
def update_trigger(self, trigger):
self.remove_trigger(trigger)
self.add_trigger(trigger)
def remove_trigger(self, trigger):
trigger_id = trigger['id']
try:
job_id = self._jobs[trigger_id]
except KeyError:
LOG.info('Job not found: %s', trigger_id)
return
self._scheduler.remove_job(job_id)
del self._jobs[trigger_id]
def _add_job_to_scheduler(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
try:
jsonschema.validate(trigger['parameters'],
trigger_type['parameters_schema'])
except jsonschema.ValidationError as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'], e, exc_info=True)
raise # Or should we just return?
time_spec = trigger['parameters']
time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone'))
time_type = None
if trigger_type['name'] == 'st2.IntervalTimer':
unit = time_spec.get('unit', None)
value = time_spec.get('delta', None)
time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone})
elif trigger_type['name'] == 'st2.DateTimer':
# Raises an exception if date string isn't a valid one.
dat = date_parser.parse(time_spec.get('date', None))
time_type = DateTrigger(dat, timezone=time_zone)
elif trigger_type['name'] == 'st2.CronTimer':
cron = time_spec.copy()
cron['timezone'] = time_zone
time_type = CronTrigger(**cron)
utc_now = date_utils.get_datetime_utc_now()
if hasattr(time_type, 'run_date') and utc_now > time_type.run_date:
LOG.warning('Not scheduling expired timer: %s : %s',
trigger['parameters'], time_type.run_date)
else:
self._add_job(trigger, time_type)
return time_type
def _add_job(self, trigger, time_type, replace=True):
try:
job = self._scheduler.add_job(self._emit_trigger_instance,
trigger=time_type,
args=[trigger],
replace_existing=replace)
LOG.info('Job %s scheduled.', job.id)
self._jobs[trigger['id']] = job.id
except Exception as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'], e, exc_info=True)
def _emit_trigger_instance(self, trigger):
utc_now = date_utils.get_datetime_utc_now()
# debug logging is reasonable for this one. A high resolution timer will end up
# trashing standard logs.
LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger)
payload = {
'executed_at': str(utc_now),
'schedule': trigger['parameters'].get('time')
}
trace_context = TraceContext(trace_tag='%s-%s' % (self._get_trigger_type_name(trigger),
trigger.get('name', uuid.uuid4().hex)))
self._trigger_dispatcher.dispatch(trigger, payload, trace_context=trace_context)
def _get_trigger_type_name(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
return trigger_type['name']
def _register_timer_trigger_types(self):
return trigger_services.add_trigger_models(TIMER_TRIGGER_TYPES.values())
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if 'id' in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized['id'] = str(sanitized['id'])
return sanitized
class WebhooksController(object):
def __init__(self, *args, **kwargs):
self._hooks = HooksHolder()
self._base_url = '/webhooks/'
self._trigger_types = list(WEBHOOK_TRIGGER_TYPES.keys())
self._trigger_dispatcher_service = TriggerDispatcherService(LOG)
queue_suffix = self.__class__.__name__
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=queue_suffix,
exclusive=True)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
def get_all(self):
# Return only the hooks known by this controller.
return self._hooks.get_all()
def get_one(self, url, requester_user):
triggers = self._hooks.get_triggers_for_hook(url)
if not triggers:
abort(http_client.NOT_FOUND)
return
permission_type = PermissionType.WEBHOOK_VIEW
rbac_utils = get_rbac_backend().get_utils_class()
rbac_utils.assert_user_has_resource_db_permission(
user_db=requester_user,
resource_db=WebhookDB(name=url),
permission_type=permission_type)
# For demonstration purpose return 1st
return triggers[0]
def post(self, hook, webhook_body_api, headers, requester_user):
body = webhook_body_api.data
permission_type = PermissionType.WEBHOOK_SEND
rbac_utils = get_rbac_backend().get_utils_class()
rbac_utils.assert_user_has_resource_db_permission(
user_db=requester_user,
resource_db=WebhookDB(name=hook),
permission_type=permission_type)
headers = self._get_headers_as_dict(headers)
headers = self._filter_authentication_headers(headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(
TRACE_TAG_HEADER, None),
hook=hook)
if hook == 'st2' or hook == 'st2/':
# When using st2 or system webhook, body needs to always be a dict
if not isinstance(body, dict):
type_string = get_json_type_for_python_value(body)
msg = ('Webhook body needs to be an object, got: %s' %
(type_string))
raise ValueError(msg)
trigger = body.get('trigger', None)
payload = body.get('payload', None)
if not trigger:
msg = 'Trigger not specified.'
return abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher_service.dispatch_with_context(
trigger=trigger,
payload=payload,
trace_context=trace_context,
throw_on_validation_error=True)
else:
if not self._is_valid_hook(hook):
self._log_request('Invalid hook.', headers, body)
msg = 'Webhook %s not registered with st2' % hook
return abort(http_client.NOT_FOUND, msg)
triggers = self._hooks.get_triggers_for_hook(hook)
payload = {}
payload['headers'] = headers
payload['body'] = body
# Dispatch trigger instance for each of the trigger found
for trigger_dict in triggers:
# TODO: Instead of dispatching the whole dict we should just
# dispatch TriggerDB.ref or similar
self._trigger_dispatcher_service.dispatch_with_context(
trigger=trigger_dict,
payload=payload,
trace_context=trace_context,
throw_on_validation_error=True)
return Response(json=body, status=http_client.ACCEPTED)
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = 'webhook-%s-%s' % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# NOTE: trigger is a dictionary
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
LOG.info('Listening to endpoint: %s', urljoin(self._base_url, url))
self._hooks.add_hook(url, trigger)
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = self._get_normalized_url(trigger)
removed = self._hooks.remove_hook(url, trigger)
if removed:
LOG.info('Stop listening to endpoint: %s',
urljoin(self._base_url, url))
def _get_normalized_url(self, trigger):
"""
remove the trailing and leading / so that the hook url and those coming
from trigger parameters end up being the same.
"""
return trigger['parameters']['url'].strip('/')
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _filter_authentication_headers(self, headers):
auth_headers = [
HEADER_API_KEY_ATTRIBUTE_NAME, HEADER_ATTRIBUTE_NAME, 'Cookie'
]
return {
key: value
for key, value in headers.items() if key not in auth_headers
}
def _log_request(self, msg, headers, body, log_method=LOG.debug):
headers = self._get_headers_as_dict(headers)
body = str(body)
log_method('%s\n\trequest.header: %s.\n\trequest.body: %s.', msg,
headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' %
(trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = TriggerAPI.from_model(trigger).to_dict()
return sanitized
class WebhooksController(RestController):
def __init__(self, *args, **kwargs):
super(WebhooksController, self).__init__(*args, **kwargs)
self._hooks = {}
self._base_url = "/webhooks/"
self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
self._trigger_dispatcher = TriggerDispatcher(LOG)
self._trigger_watcher = TriggerWatcher(
create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix="webhooks",
)
self._trigger_watcher.start()
self._register_webhook_trigger_types()
@jsexpose()
def get_all(self):
# Return only the hooks known by this controller.
return [trigger for trigger in six.itervalues(self._hooks)]
@jsexpose()
def get_one(self, name):
hook = self._hooks.get(name, None)
if not hook:
abort(http_client.NOT_FOUND)
return
return hook
@request_user_has_webhook_permission(permission_type=PermissionType.WEBHOOK_SEND)
@jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
def post(self, *args, **kwargs):
hook = "/".join(args) # TODO: There must be a better way to do this.
body = pecan.request.body
try:
body = json.loads(body)
except ValueError:
self._log_request("Invalid JSON body.", pecan.request)
msg = "Invalid JSON body: %s" % (body)
return pecan.abort(http_client.BAD_REQUEST, msg)
headers = self._get_headers_as_dict(pecan.request.headers)
# If webhook contains a trace-tag use that else create create a unique trace-tag.
trace_context = self._create_trace_context(trace_tag=headers.pop(TRACE_TAG_HEADER, None), hook=hook)
if hook == "st2" or hook == "st2/":
return self._handle_st2_webhook(body, trace_context=trace_context)
if not self._is_valid_hook(hook):
self._log_request("Invalid hook.", pecan.request)
msg = "Webhook %s not registered with st2" % hook
return pecan.abort(http_client.NOT_FOUND, msg)
trigger = self._get_trigger_for_hook(hook)
payload = {}
payload["headers"] = headers
payload["body"] = body
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _handle_st2_webhook(self, body, trace_context):
trigger = body.get("trigger", None)
payload = body.get("payload", None)
if not trigger:
msg = "Trigger not specified."
return pecan.abort(http_client.BAD_REQUEST, msg)
self._trigger_dispatcher.dispatch(trigger, payload=payload, trace_context=trace_context)
return body
def _is_valid_hook(self, hook):
# TODO: Validate hook payload with payload_schema.
return hook in self._hooks
def _get_trigger_for_hook(self, hook):
return self._hooks[hook]
def _register_webhook_trigger_types(self):
for trigger_type in WEBHOOK_TRIGGER_TYPES.values():
trigger_service.create_trigger_type_db(trigger_type)
def _create_trace_context(self, trace_tag, hook):
# if no trace_tag then create a unique one
if not trace_tag:
trace_tag = "webhook-%s-%s" % (hook, uuid.uuid4().hex)
return TraceContext(trace_tag=trace_tag)
def add_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = trigger["parameters"]["url"]
LOG.info("Listening to endpoint: %s", urljoin(self._base_url, url))
self._hooks[url] = trigger
def update_trigger(self, trigger):
pass
def remove_trigger(self, trigger):
# Note: Permission checking for creating and deleting a webhook is done during rule
# creation
url = trigger["parameters"]["url"]
if url in self._hooks:
LOG.info("Stop listening to endpoint: %s", urljoin(self._base_url, url))
del self._hooks[url]
def _get_headers_as_dict(self, headers):
headers_dict = {}
for key, value in headers.items():
headers_dict[key] = value
return headers_dict
def _log_request(self, msg, request, log_method=LOG.debug):
headers = self._get_headers_as_dict(request.headers)
body = str(request.body)
log_method("%s\n\trequest.header: %s.\n\trequest.body: %s.", msg, headers, body)
##############################################
# Event handler methods for the trigger events
##############################################
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
def _handle_update_trigger(self, trigger):
LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.update_trigger(trigger=trigger)
def _handle_delete_trigger(self, trigger):
LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.remove_trigger(trigger=trigger)
def _sanitize_trigger(self, trigger):
sanitized = trigger._data
if "id" in sanitized:
# Friendly objectid rather than the MongoEngine representation.
sanitized["id"] = str(sanitized["id"])
return sanitized