def restart(self, **kwargs):
"""
:param kwargs: passed from zoom.handlers.control_agent_handlers
"""
# if not self._action_is_ready('restart', allow_undefined=True):
# self._log.info('Restart action not ready.')
# return
self._log.info('Running Restart. Queuing stop, unregister, start.')
self._action_queue.clear()
self._action_queue.append_unique(Task('stop', kwargs=kwargs))
self._action_queue.append_unique(Task('unregister'))
self._action_queue.append_unique(Task('start', kwargs=kwargs))
return 0
def on_exist(self, event=None):
try:
if self.zkclient.exists(self._path, watch=self.on_exist):
data, stat = self.zkclient.get(self._path)
task = Task.from_json(data)
self._log.info('Found work to do: {0}'.format(task))
if task.result == ApplicationState.OK:
self._log.debug('Task is already complete: {0}'.format(task))
return # ignore tasks that are already done
if task.name in SENTINEL_METHODS:
if task.target is not None:
self.send_work_single(task)
else:
self.send_work_all(task)
self._log.info("Submitted task {0} for {1}"
.format(task.name, task.target))
else:
err = 'Invalid work submitted: {0}'.format(task.name)
self._log.warning(err)
task.result = ApplicationState.OK
self._log.info(task.to_json())
self.zkclient.set(self._path, task.to_json())
except NoNodeError:
self._log.debug('No Node at {0}'.format(self._path))
def clear_task_queue(self):
self._log.info('Starting, so clearing task queue.')
if self.zkclient.exists(self._path):
data, stat = self.zkclient.get(self._path)
task = Task.from_json(data)
task.result = CommandType.CANCEL
self.zkclient.set(self._path, task.to_json())
def clear_task_queue(self):
self._log.info('Starting, so clearing task queue.')
if self.zkclient.exists(self._path):
data, stat = self.zkclient.get(self._path)
task = Task.from_json(data)
task.result = CommandType.CANCEL
self.zkclient.set(self._path, task.to_json())
def on_exist(self, event=None):
try:
if self.zkclient.exists(self._path, watch=self.on_exist):
data, stat = self.zkclient.get(self._path)
task = Task.from_json(data)
self._log.info('Found work to do: {0}'.format(task))
if task.result == ApplicationState.OK:
self._log.debug(
'Task is already complete: {0}'.format(task))
return # ignore tasks that are already done
if task.name in SENTINEL_METHODS:
if task.target is not None:
self.send_work_single(task)
else:
self.send_work_all(task)
self._log.info("Submitted task {0} for {1}".format(
task.name, task.target))
else:
err = 'Invalid work submitted: {0}'.format(task.name)
self._log.warning(err)
task.result = ApplicationState.OK
self._log.info(task.to_json())
self.zkclient.set(self._path, task.to_json())
except NoNodeError:
self._log.debug('No Node at {0}'.format(self._path))
def _callback(self):
self._log.info('Callback triggered for {0}:\n{1}'.format(
self, self._predicate))
if self.disabled:
self._log.info('Not running action {0}. It is disabled.'.format(
self.name))
return
# ensure all predicates are started
elif not self.started:
self._log.warning('All predicates are not started. '
'Ignoring action {0}'.format(self.name))
return
# ensure all predicates are met
elif not self._predicate.met:
self._log.debug('Not triggering action for {0}. '
'Predicate not met.'.format(self))
# check if there are operational dependencies involved.
# If so, run the operational action. This defaults to 'stop'.
if self._predicate.operationally_relevant and \
self._op_action is not None and \
self._mode != ApplicationMode.MANUAL:
self._log.info('Operational relevancy detected. '
'Triggering operation action.')
self._action_queue.append_unique(Task(self._op_action.__name__,
func=self._op_action),
sender=str(self))
else:
self._log.debug('Operation dep={0}, Mode={1}'.format(
self._predicate.operationally_relevant, self._mode))
return
elif self._action is not None:
self._log.debug('There is a callback and all predicates are met.')
if (self._mode != ApplicationMode.MANUAL
or not self._mode_controlled):
self._action_queue.append_unique(Task(self.name,
func=self._execute,
kwargs=self.kwargs),
sender=str(self))
else:
self._log.info(
'Run mode is set to Manual. Not triggering "{0}"'
' action based on dependency change.'.format(self.name))
else:
self._log.info('Nothing was done with callback')
def start_if_ready(self, **kwargs):
start_action = self._actions.get('start', None)
if start_action is not None and start_action.ready:
start_action.run(**kwargs)
# if start action doesn't exist, a.k.a. read only
elif start_action is None:
self.start(**kwargs)
else:
self._action_queue.append(Task('react', kwargs=kwargs))
return 0
def stop(self):
"""
Stops the Process/Thread
"""
try:
self._log.info('Terminating {0} child process'.format(self.name))
self.cancel_current_task()
self.add_work(Task('terminate', block=True), immediate=True)
except Exception as e:
self._log.warning(
'Exception with stopping {0} child process: {1}'.format(
self.name, e))
def post(self, target=None):
"""
@api {post} /api/v1/status[/:target] Log out and retrieve status of components (plaintext)
@apiDescription This will retrieve the status of all the Sentinel objections
(Components, Actions, Predicates). The POST is best used from curl.
@apiVersion 1.0.0
@apiName CurlGetStatus
@apiGroup Sentinel Agent
"""
tc = self.application.task_client
task = Task('status', target=target, block=True, retval=True)
if target is not None:
result = tc.send_work_single(task, wait=True, immediate=True)
self.write(result.get('result'))
else:
result = tc.send_work_all(task, wait=True, immediate=True)
for i in result.values():
self.write(i.get('result'))
def get(self, target=None):
"""
@api {get} /api/v1/status[/:target] Log out and retrieve status of components (html)
@apiDescription This will retrieve the status of all the Sentinel objections
(Components, Actions, Predicates). The GET is best used from a browser.
@apiVersion 1.0.0
@apiName WebGetStatus
@apiGroup Sentinel Agent
"""
tc = self.application.task_client
task = Task('status', target=target, block=True, retval=True)
if target is not None:
tc.send_work_single(task, wait=True, immediate=True)
else:
tc.send_work_all(task, wait=True, immediate=True)
line_count = 50 * len(self.application.children.keys())
self.render('../templates/log.html', data=get_log(count=line_count))
def post(self, work=None, target=None):
"""
@api {post} /api/v1/:work[/:target] Submit a task to Sentinel
@apiDescription You can use this to mimic the tasks sent from Zoom.
For example: /api/v1/stop/foo will stop the application foo.
@apiVersion 1.0.0
@apiName SubmitTask
@apiGroup Sentinel Agent
"""
tc = self.application.task_client
task = Task(work, target=target, block=True, retval=True)
if target is not None:
result = tc.send_work_single(task, immediate=True)
self.write(json.dumps(result))
else:
result = tc.send_work_all(task, immediate=True)
self.write(json.dumps(result))
def _on_update(self, event):
"""
Callback for the data watch on the task node
:type event: kazoo.protocol.states.WatchedEvent
"""
try:
data, stat = self._zoo_keeper.get(event.path)
task = Task.from_json(data)
if task.result is not None:
self._remove(task, event.path)
self._submit_next(task.host)
else:
logging.info('Task result is {0}. Resetting watch'
.format(task.result))
self._zoo_keeper.get(event.path, watch=self._on_update)
except NoNodeError:
pass
def _on_update(self, event):
"""
Callback for the data watch on the task node
:type event: kazoo.protocol.states.WatchedEvent
"""
try:
data, stat = self._zoo_keeper.get(event.path)
task = Task.from_json(data)
if task.result is None:
logging.info('Task result is {0}. Resetting watch'.format(
task.result))
self._zoo_keeper.get(event.path, watch=self._on_update)
elif task.result == CommandType.CANCEL:
logging.info('Sentinel {0} requested task clear.'.format(
task.host))
self._remove(task, event.path, clear_queue=True)
else:
self._remove(task, event.path)
self._submit_next(task.host)
except NoNodeError:
pass
def post(self):
"""
@api {post} /api/v1/agent/ Create Task from json and add to TaskServer
@apiParam {Boolean} [stay_down=null] Used to signal to Sentinel that the application was brought down on purpose
@apiParam {String} [login_user=null] The user that submitted the task
@apiParam {String} command The function Sentinel should run
@apiParam {String} componentId The app targeted for the task
@apiParam {String} applicationHost The host the app is running on
@apiVersion 1.0.0
@apiName CreateTask
@apiGroup Task
"""
try:
user = self.get_argument("user", default=None)
command = self.get_argument("command")
app = self.get_argument("componentId")
kwarguments = {
'stay_down': self.get_argument("stay_down", default=None),
'login_user': user,
}
task = Task(command,
target=app,
kwargs=kwarguments,
host=self.get_argument("applicationHost"),
submitted=datetime.datetime.now().strftime('%Y%m%d %H:%M:%S'))
logging.info("Received task request from client {0}: {1}"
.format(self.request.remote_ip, task))
cancel = task.name == CommandType.CANCEL
self.task_server.add_task(task, is_cancel=cancel)
self._send_to_chat_server(user, app, command)
except Exception as e:
self.set_status(INTERNAL_SERVER_ERROR)
self.write(json.dumps({'errorText': str(e)}))
logging.exception(e)
def dep_restart(self, **kwargs):
self._run_check_mode = True # only used in self.start()
self._action_queue.append(Task('start_if_ready', kwargs=kwargs))
return 0