def main_loop(): last_runner_check = 0 while True: try: action_cache_lock.acquire() AnarchyAction.start_actions(runtime) except Exception as e: operator_logger.exception("Error in start_actions!") finally: action_cache_lock.release() try: AnarchySubject.retry_failures(runtime) except Exception as e: operator_logger.exception("Error in retry_failures!") if runner_check_interval < time.time() - last_runner_check: try: AnarchyRunner.refresh_all_runner_pods(runtime) last_runner_check = time.time() except: operator_logger.exception( 'Error checking runner pods in main loop') time.sleep(1)
def handle_action_event(event, **_): wait_for_init() try: action_cache_lock.acquire() action = AnarchyAction(event['object']) if event['type'] == 'DELETED': AnarchyAction.cache_remove(action) elif event['type'] in ['ADDED', 'MODIFIED', None]: if not action.has_started: AnarchyAction.cache_put(action) else: operator_logger.warning('Unknown event for AnarchyAction %s', event) finally: action_cache_lock.release()
def main_loop(): while True: with runtime.is_active_condition: while not runtime.is_active: runtime.is_active_condition.wait() if runtime.running_all_in_one: start_runner_process() elif not AnarchyRunner.get('default'): init_default_runner() while runtime.is_active: AnarchyAction.start_actions(runtime) time.sleep(1)
def handle_action_activity(body, logger, **_): action = AnarchyAction(body) if not action.has_owner: action.set_owner(runtime) elif not action.has_started: if action.after_datetime <= datetime.utcnow(): action.start(runtime) else: AnarchyAction.cache_put(action)
def handle_action_event(event, logger, **_): obj = event.get('object') if obj and obj.get('apiVersion') == runtime.api_group_version: if event['type'] == 'DELETED': AnarchyAction.cache_remove(obj['metadata']['name']) else: action = AnarchyAction(obj) if not action.has_owner: action.set_owner(runtime) elif not action.has_started: AnarchyAction.cache_put(action)
def run_subject_action_patch(subject_name, action_name): """ Callback from runner to update AnarchyAction associated with AnarchySubject assigned to runner. The only function of this method currently is to pass JSON, `{"successful": true}` or `{"failed": true}` to mark the action as finished. """ anarchy_runner, runner_pod = check_runner_auth( flask.request.headers.get('Authorization', '')) if not anarchy_runner: flask.abort(400) if not runtime.running_all_in_one \ and subject_name != runner_pod.metadata.labels.get(runtime.subject_label): operator_logger.warning( 'AnarchyRunner %s Pod %s cannot update actions for AnarchySubject %s!', anarchy_runner.name, runner_pod.metadata.name, subject_name) flask.abort(400) anarchy_action = AnarchyAction.get(action_name, runtime) if not anarchy_action: operator_logger.warning( 'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchyAction %s!', anarchy_runner.name, runner_pod.metadata.name, action_name) flask.abort(400) anarchy_subject = AnarchySubject.get(subject_name, runtime) if not anarchy_subject: operator_logger.warning( 'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchySubject %s!', anarchy_runner.name, runner_pod.metadata.name, subject_name) flask.abort(400) anarchy_governor = anarchy_subject.get_governor(runtime) if not anarchy_governor: operator_logger.warning( 'AnarchyRunner %s Pod %s cannot post action to AnarchySubject %s, unable to find AnarchyGovernor %s!', anarchy_runner.name, runner_pod, subject_name, anarchy_subject.governor_name) flask.abort(400) finished_state = None if flask.request.json.get('successful', False): finished_state = 'successful' elif flask.request.json.get('failed', False): finished_state = 'failed' if finished_state != None: anarchy_subject.remove_active_action(anarchy_action, runtime) anarchy_action.set_finished(finished_state, runtime) return flask.jsonify({ 'success': True, 'result': anarchy_action.to_dict(runtime) })
def main_loop(): last_cleanup = 0 last_run_check = 0 last_runner_check = 0 while True: with runtime.is_active_condition: while not runtime.is_active: runtime.is_active_condition.wait() if runtime.running_all_in_one: start_runner_process() elif not AnarchyRunner.get('default'): init_default_runner() while runtime.is_active: AnarchyAction.start_actions(runtime) if cleanup_interval < time.time() - last_cleanup: try: AnarchyGovernor.cleanup(runtime) last_cleanup = time.time() except: operator_logger.exception( 'Error in AnarchyGovernor.cleanup!') if run_check_interval < time.time() - last_run_check: try: AnarchyRun.manage_active_runs(runtime) last_run_check = time.time() except: operator_logger.exception( 'Error in AnarchyRun.manage_active_runs!') if runner_check_interval < time.time() - last_runner_check: try: AnarchyRunner.manage_runners(runtime) last_runner_check = time.time() except: operator_logger.exception( 'Error in AnarchyRunner.manage_runners!') time.sleep(1)
def handle_action_callback(anarchy_action_name, callback_name): if not flask.request.json: flask.abort(400) return anarchy_action = AnarchyAction.get(anarchy_action_name, runtime) if not anarchy_action: operator_logger.warning("AnarchyAction %s not found for callback", anarchy_action_name) flask.abort(404) return if not anarchy_action.check_callback_token(flask.request.headers.get('Authorization', '')): operator_logger.warning("Invalid callback token for AnarchyAction %s", anarchy_action_name) flask.abort(403) return anarchy_action.process_callback(runtime, callback_name, flask.request.json) return flask.jsonify({'status': 'ok'})
def handle_action_event(event, logger, **_): obj = event.get('object') if obj and obj.get('apiVersion') == runtime.api_group_version: if event['type'] == 'DELETED': AnarchyAction.cache_remove(obj['metadata']['name'])
def post_run(run_name): anarchy_runner, runner_pod = check_runner_auth( flask.request.headers.get('Authorization', '')) if not anarchy_runner: flask.abort(400) if not runtime.running_all_in_one: run_value = runner_pod.metadata.labels.get(runtime.run_label) runtime.core_v1_api.patch_namespaced_pod( runner_pod.metadata.name, runner_pod.metadata.namespace, { 'metadata': { 'labels': { runtime.run_label: '', runtime.subject_label: '' } } }) if run_value != run_name: operator_logger.warning( 'AnarchyRunner %s Pod %s attempted to post run for %s but run label indicates %s', anarchy_runner.name, runner_pod.metadata.name, run_name, run_value) flask.abort(400) # When an AnarchyRun is handling a delete completion it is normal for the # AnarchyRun and AnarchySubject to be deleted before the post is received. anarchy_run = AnarchyRun.get_from_api(run_name, runtime) if not anarchy_run: operator_logger.info( 'AnarchyRunner %s pod %s posted result on deleted run %s', anarchy_runner.name, runner_pod.metadata.name, run_name) return flask.jsonify({'success': True, 'msg': 'AnarchyRun not found'}) anarchy_subject = anarchy_run.get_subject(runtime) if not anarchy_subject: operator_logger.warning( 'AnarchyRun %s post to deleted AnarchySubject %s!', anarchy_run.name, anarchy_run.subject_name) return flask.jsonify({ 'success': True, 'msg': 'AnarchySubject not found' }) try: result = flask.request.json['result'] except KeyError: flask.abort( 400, flask.jsonify({ 'success': False, 'error': 'Invalid run data' })) anarchy_run.post_result(result, runner_pod.metadata.name, runtime) if run_name == anarchy_subject.active_run_name: if result['status'] == 'successful': anarchy_subject.remove_active_run_from_status(anarchy_run, runtime) anarchy_subject.set_active_run_to_pending(runtime) if anarchy_run.action_name: anarchy_action = AnarchyAction.get(anarchy_run.action_name, runtime) anarchy_governor = anarchy_subject.get_governor(runtime) action_config = anarchy_governor.action_config( anarchy_action.action) if result.get('continue'): anarchy_action.schedule_continuation( result['continue'], runtime) elif action_config.finish_on_successful_run: if anarchy_action.name == anarchy_subject.active_action_name: anarchy_subject.remove_active_action( anarchy_action, runtime) anarchy_action.set_finished('successful', runtime) else: anarchy_subject.set_run_failure_in_status(anarchy_run, runtime) else: operator_logger.warning( 'AnarchyRun %s post to AnarchySubject %s, but was not the active run!', anarchy_run.name, anarchy_run.subject_name) return flask.jsonify({'success': True})