def main_loop():
last_runner_check = 0
while True:
try:
action_cache_lock.acquire()
AnarchyAction.start_actions(runtime)
except Exception as e:
operator_logger.exception("Error in start_actions!")
finally:
action_cache_lock.release()
try:
AnarchySubject.retry_failures(runtime)
except Exception as e:
operator_logger.exception("Error in retry_failures!")
if runner_check_interval < time.time() - last_runner_check:
try:
AnarchyRunner.refresh_all_runner_pods(runtime)
last_runner_check = time.time()
except:
operator_logger.exception(
'Error checking runner pods in main loop')
time.sleep(1)
def handle_action_event(event, **_):
wait_for_init()
try:
action_cache_lock.acquire()
action = AnarchyAction(event['object'])
if event['type'] == 'DELETED':
AnarchyAction.cache_remove(action)
elif event['type'] in ['ADDED', 'MODIFIED', None]:
if not action.has_started:
AnarchyAction.cache_put(action)
else:
operator_logger.warning('Unknown event for AnarchyAction %s', event)
finally:
action_cache_lock.release()
def main_loop():
while True:
with runtime.is_active_condition:
while not runtime.is_active:
runtime.is_active_condition.wait()
if runtime.running_all_in_one:
start_runner_process()
elif not AnarchyRunner.get('default'):
init_default_runner()
while runtime.is_active:
AnarchyAction.start_actions(runtime)
time.sleep(1)
def handle_action_activity(body, logger, **_):
action = AnarchyAction(body)
if not action.has_owner:
action.set_owner(runtime)
elif not action.has_started:
if action.after_datetime <= datetime.utcnow():
action.start(runtime)
else:
AnarchyAction.cache_put(action)
def handle_action_event(event, logger, **_):
obj = event.get('object')
if obj and obj.get('apiVersion') == runtime.api_group_version:
if event['type'] == 'DELETED':
AnarchyAction.cache_remove(obj['metadata']['name'])
else:
action = AnarchyAction(obj)
if not action.has_owner:
action.set_owner(runtime)
elif not action.has_started:
AnarchyAction.cache_put(action)
def run_subject_action_patch(subject_name, action_name):
"""
Callback from runner to update AnarchyAction associated with AnarchySubject assigned to runner.
The only function of this method currently is to pass JSON, `{"successful": true}` or
`{"failed": true}` to mark the action as finished.
"""
anarchy_runner, runner_pod = check_runner_auth(
flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
if not runtime.running_all_in_one \
and subject_name != runner_pod.metadata.labels.get(runtime.subject_label):
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot update actions for AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
anarchy_action = AnarchyAction.get(action_name, runtime)
if not anarchy_action:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchyAction %s!',
anarchy_runner.name, runner_pod.metadata.name, action_name)
flask.abort(400)
anarchy_subject = AnarchySubject.get(subject_name, runtime)
if not anarchy_subject:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
anarchy_governor = anarchy_subject.get_governor(runtime)
if not anarchy_governor:
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot post action to AnarchySubject %s, unable to find AnarchyGovernor %s!',
anarchy_runner.name, runner_pod, subject_name,
anarchy_subject.governor_name)
flask.abort(400)
finished_state = None
if flask.request.json.get('successful', False):
finished_state = 'successful'
elif flask.request.json.get('failed', False):
finished_state = 'failed'
if finished_state != None:
anarchy_subject.remove_active_action(anarchy_action, runtime)
anarchy_action.set_finished(finished_state, runtime)
return flask.jsonify({
'success': True,
'result': anarchy_action.to_dict(runtime)
})
def main_loop():
last_cleanup = 0
last_run_check = 0
last_runner_check = 0
while True:
with runtime.is_active_condition:
while not runtime.is_active:
runtime.is_active_condition.wait()
if runtime.running_all_in_one:
start_runner_process()
elif not AnarchyRunner.get('default'):
init_default_runner()
while runtime.is_active:
AnarchyAction.start_actions(runtime)
if cleanup_interval < time.time() - last_cleanup:
try:
AnarchyGovernor.cleanup(runtime)
last_cleanup = time.time()
except:
operator_logger.exception(
'Error in AnarchyGovernor.cleanup!')
if run_check_interval < time.time() - last_run_check:
try:
AnarchyRun.manage_active_runs(runtime)
last_run_check = time.time()
except:
operator_logger.exception(
'Error in AnarchyRun.manage_active_runs!')
if runner_check_interval < time.time() - last_runner_check:
try:
AnarchyRunner.manage_runners(runtime)
last_runner_check = time.time()
except:
operator_logger.exception(
'Error in AnarchyRunner.manage_runners!')
time.sleep(1)
def handle_action_callback(anarchy_action_name, callback_name):
if not flask.request.json:
flask.abort(400)
return
anarchy_action = AnarchyAction.get(anarchy_action_name, runtime)
if not anarchy_action:
operator_logger.warning("AnarchyAction %s not found for callback", anarchy_action_name)
flask.abort(404)
return
if not anarchy_action.check_callback_token(flask.request.headers.get('Authorization', '')):
operator_logger.warning("Invalid callback token for AnarchyAction %s", anarchy_action_name)
flask.abort(403)
return
anarchy_action.process_callback(runtime, callback_name, flask.request.json)
return flask.jsonify({'status': 'ok'})
def handle_action_event(event, logger, **_):
obj = event.get('object')
if obj and obj.get('apiVersion') == runtime.api_group_version:
if event['type'] == 'DELETED':
AnarchyAction.cache_remove(obj['metadata']['name'])
def post_run(run_name):
anarchy_runner, runner_pod = check_runner_auth(
flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
if not runtime.running_all_in_one:
run_value = runner_pod.metadata.labels.get(runtime.run_label)
runtime.core_v1_api.patch_namespaced_pod(
runner_pod.metadata.name, runner_pod.metadata.namespace, {
'metadata': {
'labels': {
runtime.run_label: '',
runtime.subject_label: ''
}
}
})
if run_value != run_name:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to post run for %s but run label indicates %s',
anarchy_runner.name, runner_pod.metadata.name, run_name,
run_value)
flask.abort(400)
# When an AnarchyRun is handling a delete completion it is normal for the
# AnarchyRun and AnarchySubject to be deleted before the post is received.
anarchy_run = AnarchyRun.get_from_api(run_name, runtime)
if not anarchy_run:
operator_logger.info(
'AnarchyRunner %s pod %s posted result on deleted run %s',
anarchy_runner.name, runner_pod.metadata.name, run_name)
return flask.jsonify({'success': True, 'msg': 'AnarchyRun not found'})
anarchy_subject = anarchy_run.get_subject(runtime)
if not anarchy_subject:
operator_logger.warning(
'AnarchyRun %s post to deleted AnarchySubject %s!',
anarchy_run.name, anarchy_run.subject_name)
return flask.jsonify({
'success': True,
'msg': 'AnarchySubject not found'
})
try:
result = flask.request.json['result']
except KeyError:
flask.abort(
400, flask.jsonify({
'success': False,
'error': 'Invalid run data'
}))
anarchy_run.post_result(result, runner_pod.metadata.name, runtime)
if run_name == anarchy_subject.active_run_name:
if result['status'] == 'successful':
anarchy_subject.remove_active_run_from_status(anarchy_run, runtime)
anarchy_subject.set_active_run_to_pending(runtime)
if anarchy_run.action_name:
anarchy_action = AnarchyAction.get(anarchy_run.action_name,
runtime)
anarchy_governor = anarchy_subject.get_governor(runtime)
action_config = anarchy_governor.action_config(
anarchy_action.action)
if result.get('continue'):
anarchy_action.schedule_continuation(
result['continue'], runtime)
elif action_config.finish_on_successful_run:
if anarchy_action.name == anarchy_subject.active_action_name:
anarchy_subject.remove_active_action(
anarchy_action, runtime)
anarchy_action.set_finished('successful', runtime)
else:
anarchy_subject.set_run_failure_in_status(anarchy_run, runtime)
else:
operator_logger.warning(
'AnarchyRun %s post to AnarchySubject %s, but was not the active run!',
anarchy_run.name, anarchy_run.subject_name)
return flask.jsonify({'success': True})
