def run(self):
data = self.data
input_handler = self.input_handler
output_handler = self.output_handler
# self._worker_signals()
data['output'] = {'result_code': 255,
'result_str': 'unknown error',
'result_data': ''}
try:
self.logger.debug('sending input data to output handler')
data['output'] = output_handler.dispatch(data['input'])
self.logger.debug('got return from output handler')
except KeyboardInterrupt:
raise KeyboardInterrupt
except Exception as e:
etext = detailed_exception()
self.logger.debug('exception in output handler: %s' % etext)
data['output'] = {'result_code': 254,
'result_str': 'dispatch error',
'result_data': etext}
self.logger.debug(
'passing output handler result back to input handler')
input_handler.result(data)
self.logger.debug('dispatch handler terminating')
def _exit(self, exception):
"""Terminate the agent.
:param: exception: whether an exception should be logged. This should
be a boolean value.
"""
self.logger.debug('exiting...')
self._cleanup()
if exception:
etext = detailed_exception()
self.logger.error('exception in initializing opencenter-agent: %s'
% etext)
# wouldn't we rather have a full traceback?
exc_info = sys.exc_info()
if hasattr(exc_info[0], '__name__'):
exc_class, exc, tb = exc_info
tb_path, tb_lineno, tb_func = traceback.extract_tb(tb)[-1][:3]
self.logger.error('%s (%s:%s in %s)', exc_info[1], tb_path,
tb_lineno, tb_func)
else: # string exception
self.logger.error(exc_info[0])
if self.logger.isEnabledFor(logging.DEBUG):
print ''
traceback.print_exception(*exc_info)
sys.exit(1)
sys.exit(0)
def dispatch(self):
output_handler = self.output_handler
input_handler = self.input_handler
# we'll assume non-blocking. we should negotiate this
# with the plugins, I suppose
do_quit = False
try:
while not do_quit:
self.logger.debug('FETCH')
result = input_handler.fetch()
if len(result) == 0:
time.sleep(5)
else:
self.logger.debug('Got input from input handler "%s"'
% (result['plugin']))
self.logger.debug('Data: %s' % result['input'])
# Apply to the pool
worker = OpenCenterAgentDispatchWorker(input_handler,
output_handler,
result)
worker.setDaemon(True)
worker.start()
except KeyboardInterrupt:
self.logger.debug('Got keyboard interrupt.')
self._exit(False)
except Exception, e:
self.logger.debug('Exception: %s' % detailed_exception())
def run(self):
data = self.data
input_handler = self.input_handler
output_handler = self.output_handler
# self._worker_signals()
data['output'] = {
'result_code': 255,
'result_str': 'unknown error',
'result_data': ''
}
try:
self.logger.debug('sending input data to output handler')
data['output'] = output_handler.dispatch(data['input'])
self.logger.debug('got return from output handler')
except KeyboardInterrupt:
raise KeyboardInterrupt
except Exception as e:
etext = detailed_exception()
self.logger.debug('exception in output handler: %s' % etext)
data['output'] = {
'result_code': 254,
'result_str': 'dispatch error',
'result_data': etext
}
self.logger.debug(
'passing output handler result back to input handler')
input_handler.result(data)
self.logger.debug('dispatch handler terminating')
def dispatch(self):
output_handler = self.output_handler
input_handler = self.input_handler
# we'll assume non-blocking. we should negotiate this
# with the plugins, I suppose
do_quit = False
try:
while not do_quit:
self.logger.debug('FETCH')
result = input_handler.fetch()
if len(result) == 0:
time.sleep(5)
else:
self.logger.debug('Got input from input handler "%s"' %
(result['plugin']))
self.logger.debug('Data: %s' % result['input'])
# Apply to the pool
worker = OpenCenterAgentDispatchWorker(
input_handler, output_handler, result)
worker.setDaemon(True)
worker.start()
except KeyboardInterrupt:
self.logger.debug('Got keyboard interrupt.')
self._exit(False)
except Exception, e:
self.logger.debug('Exception: %s' % detailed_exception())
def _exit(self, exception):
"""Terminate the agent.
:param: exception: whether an exception should be logged. This should
be a boolean value.
"""
self.logger.debug('exiting...')
self._cleanup()
if exception:
etext = detailed_exception()
self.logger.error(
'exception in initializing opencenter-agent: %s' % etext)
# wouldn't we rather have a full traceback?
exc_info = sys.exc_info()
if hasattr(exc_info[0], '__name__'):
exc_class, exc, tb = exc_info
tb_path, tb_lineno, tb_func = traceback.extract_tb(tb)[-1][:3]
self.logger.error('%s (%s:%s in %s)', exc_info[1], tb_path,
tb_lineno, tb_func)
else: # string exception
self.logger.error(exc_info[0])
if self.logger.isEnabledFor(logging.DEBUG):
print ''
traceback.print_exception(*exc_info)
sys.exit(1)
sys.exit(0)
def test_detailed_exception(self):
class FakeExceptionForTest(Exception):
pass
def bar():
raise FakeExceptionForTest('testing 123')
def foo():
bar()
try:
foo()
except FakeExceptionForTest:
trace = utils.detailed_exception()
trace_as_string = ''.join(trace)
self.assertNotEqual(trace_as_string.find('bar()'), -1)
self.assertNotEqual(trace_as_string.find('foo()'), -1)
self.assertEqual(trace_as_string.find('banana()'), -1)
self.assertNotEqual(trace_as_string.find('testing 123'), -1)
def backend_wrapper(self, state_data, prim_name, fn, api, *args, **kwargs):
"""
this runs the passed backend primitive function on all the nodes
in the input state node list. Failed nodes are dropped into the
fail bucket to be rolled back.
Right now, this runs all the backend functions in series.
Probably it should be doing this in parallel.
"""
nodelist_length = len(state_data['nodes'])
result_data = {}
# we're serializing on this. when we shift to multi-target
# adventures in the ui, we probably want to do this in parallel,
# _particularly_ in the case of run_task
#
for node in state_data['nodes']:
try:
task_result = fn(state_data, api, node, *args, **kwargs)
except Exception as e:
task_result = {
'result_code': 1,
'result_str': '%s: %s' % (prim_name, str(e)),
'result_data': detailed_exception()
}
result_data[node] = task_result
if task_result['result_code'] != 0:
self._fail_node(state_data, node)
log_entry = 'ran primitive %s: %d/%d completed successfully' % (
prim_name, len(state_data['nodes']), nodelist_length)
if len(state_data['nodes']) > 0:
return self._success(state_data,
result_str=log_entry,
result_data=result_data)
else:
return self._failure(state_data,
result_str=log_entry,
result_data=result_data)
def backend_wrapper(self, state_data, prim_name, fn, api, *args, **kwargs):
"""
this runs the passed backend primitive function on all the nodes
in the input state node list. Failed nodes are dropped into the
fail bucket to be rolled back.
Right now, this runs all the backend functions in series.
Probably it should be doing this in parallel.
"""
nodelist_length = len(state_data["nodes"])
result_data = {}
# we're serializing on this. when we shift to multi-target
# adventures in the ui, we probably want to do this in parallel,
# _particularly_ in the case of run_task
#
for node in state_data["nodes"]:
try:
task_result = fn(state_data, api, node, *args, **kwargs)
except Exception as e:
task_result = {
"result_code": 1,
"result_str": "%s: %s" % (prim_name, str(e)),
"result_data": detailed_exception(),
}
result_data[node] = task_result
if task_result["result_code"] != 0:
self._fail_node(state_data, node)
log_entry = "ran primitive %s: %d/%d completed successfully" % (
prim_name,
len(state_data["nodes"]),
nodelist_length,
)
if len(state_data["nodes"]) > 0:
return self._success(state_data, result_str=log_entry, result_data=result_data)
else:
return self._failure(state_data, result_str=log_entry, result_data=result_data)
def handle_adventurate(input_data):
global endpoint
parent_id = input_data['id']
action = input_data['action']
payload = input_data['payload']
adventure_dsl = None
adventure_id = None
ep = OpenCenterEndpoint(endpoint)
if 'adventure' in payload:
adventure_obj = ep.adventures[int(payload['adventure'])]
adventure_dsl = adventure_obj.dsl
adventure_id = payload['adventure']
elif 'adventure_dsl' in payload:
adventure_dsl = payload['adventure_dsl']
adventure_id = 0
if not adventure_dsl:
return _retval(1,
friendly_str='must specify adventure or adventure_dsl')
if not 'nodes' in payload:
return _retval(1, friendly_str='no "nodes" list in request')
if 'initial_state' in payload:
initial_state = payload['initial_state']
else:
initial_state = {}
if not 'nodes' in initial_state:
initial_state['nodes'] = payload['nodes']
adv_globals = []
if 'globals' in payload:
adv_globals = payload['globals']
LOG.debug('using globals %s' % adv_globals)
ns = {}
ns['LOG'] = LOG
ns['StateMachine'] = StateMachine
ns['StateMachineState'] = StateMachineState
ns['tasks'] = OrchestratorTasks(endpoint=endpoint,
parent_task_id=parent_id,
adventure_globals=adv_globals)
ns['input_data'] = initial_state
ns['result_str'] = 'fail'
ns['result_code'] = 254
ns['result_data'] = {}
ns['sm_description'] = adventure_dsl
LOG.debug('About to run the following dsl: %s' % adventure_dsl)
node_list = {}
ns['input_data']['fails'] = []
for node in initial_state['nodes']:
node_list[int(node)] = 'ok'
attr_obj = ep.attrs.new()
attr_obj.node_id = node
attr_obj.key = 'last_task'
attr_obj.value = 'ok'
attr_obj.save()
try:
exec '(result_data, state_data) = ' \
'tasks.sm_eval(sm_description, input_data)' in ns, ns
except Exception as e:
for node in node_list.keys():
attr_obj = ep.attrs.new()
attr_obj.node_id = node
attr_obj.key = 'last_task'
attr_obj.value = 'failed'
attr_obj.save()
return _retval(1, result_data=detailed_exception())
output_data = {'result_code': 1,
'result_str': 'no return data from adventure',
'result_data': {}}
if 'result_data' in ns:
output_data = ns['result_data']
history = []
if 'state_data' in ns and \
'history' in ns['state_data']:
history = ns['state_data']['history']
# clean up any failed tasks.
LOG.debug('Adventure terminated with state: %s' % ns['state_data'])
rollbacks = {}
# walk through the history and assemble a rollback plan
for entry in history:
# walk through the history and assemble rollback plans
for k, v in entry['result_data'].items():
k = int(k)
if not k in rollbacks:
rollbacks[k] = []
if 'rollback' in v['result_data'] and \
len(v['result_data']['rollback']) > 0:
if isinstance(v['result_data']['rollback'], list):
rollbacks[k] += v['result_data']['rollback']
else:
rollbacks[k].append(v['result_data']['rollback'])
# v['result_data'].pop('history')
state_data = ns['state_data']
output_data['result_data']['history'] = history
output_data['result_data']['rollbacks'] = rollbacks
if 'fails' in state_data:
# we need to walk through all the failed nodes.
for node in map(lambda x: int(x), state_data['fails']):
node_list[node] = 'failed'
if node in rollbacks and len(rollbacks[node]) > 0:
LOG.debug('Running rollback plan for node %d: %s' %
(node, rollbacks[node]))
ns['sm_description'] = rollbacks[node]
ns['input_data'] = {'nodes': [node]}
try:
exec '(rollback_result, rollback_state) = tasks.sm_eval(' \
'sm_description, input_data)' in ns, ns
if 'rollback_result' in ns and \
'result_code' in ns['rollback_result']:
if ns['rollback_result']['result_code'] == 0:
node_list[node] = 'rollback'
else:
LOG.debug('Error in rollback: %s: %s' %
(ns['rollback_result'],
ns['rollback_state']))
except Exception as e:
LOG.debug('Exception running rollback: %s\n%s' %
(str(e), detailed_exception()))
else:
LOG.debug('No rollback plan for failed node %d' % node)
for node in node_list.keys():
attr_obj = ep.attrs.new()
attr_obj.node_id = int(node)
attr_obj.key = 'last_task'
attr_obj.value = node_list[node]
attr_obj.save()
return output_data
