def handle_subject_create(body, **_):
try:
subject = AnarchySubject(body)
subject.handle_create(runtime)
except AssertionError as e:
operator_logger.warning('AnarchySubject %s invalid: %s',
body['metadata']['name'], e)
def main_loop():
last_runner_check = 0
while True:
try:
action_cache_lock.acquire()
AnarchyAction.start_actions(runtime)
except Exception as e:
operator_logger.exception("Error in start_actions!")
finally:
action_cache_lock.release()
try:
AnarchySubject.retry_failures(runtime)
except Exception as e:
operator_logger.exception("Error in retry_failures!")
if runner_check_interval < time.time() - last_runner_check:
try:
AnarchyRunner.refresh_all_runner_pods(runtime)
last_runner_check = time.time()
except:
operator_logger.exception(
'Error checking runner pods in main loop')
time.sleep(1)
def handle_subject_update(body, old, new, **_):
try:
subject = AnarchySubject(body)
if old['spec'] != new['spec']:
subject.handle_spec_update(runtime)
except AssertionError as e:
operator_logger.warning('AnarchySubject %s invalid: %s',
body['metadata']['name'], e)
def handle_subject_event(event, logger, **_):
'''
Anarchy uses on.event instead of on.delete because Anarchy needs custom
for removing finalizers. The finalizer will be removed immediately if the
AnarchyGovernor does not have a delete subject event handler. If there is
a delete subject event handler then it is up to the governor logic to remove
the finalizer.
'''
obj = event.get('object')
if obj and obj.get('apiVersion') == runtime.api_group_version:
if event['type'] in ['ADDED', 'MODIFIED', None]:
subject = AnarchySubject(obj)
if subject.is_pending_delete:
subject.handle_delete(runtime)
def patch_or_delete_subject(subject_name):
anarchy_runner, runner_pod = check_runner_auth(
flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
return
if subject_name != runner_pod.metadata.labels.get(runtime.subject_label):
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot update AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
return
anarchy_subject = AnarchySubject.get(subject_name, runtime)
if not anarchy_subject:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted %s on deleted AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name,
flask.request.method, subject_name)
flask.abort(400)
return
if flask.request.method == 'PATCH':
if not 'patch' in flask.request.json:
operator_logger.warning('No patch in AnarchySubject %s post',
subject_name)
flask.abort(400)
return
result = anarchy_subject.patch(flask.request.json['patch'], runtime)
elif flask.request.method == 'DELETE':
result = anarchy_subject.delete(
flask.request.json.get('remove_finalizers', False), runtime)
return flask.jsonify({'success': True, 'result': result})
def register(anarchy_run, runtime):
runner_value = anarchy_run.metadata['labels'].get(runtime.runner_label, None)
anarchy_subject_name = anarchy_run.subject_name
anarchy_subject = AnarchySubject.get(anarchy_subject_name, runtime)
if not anarchy_subject:
operator_logger.warn(
'Unable to find AnarchySubject %s for AnarchyRun %s',
anarchy_subject_name, anarchy_run.name
)
return
if not runner_value or runner_value == 'successful':
return
elif runner_value == 'failed':
anarchy_subject.anarchy_run_update(anarchy_run, runtime)
last_attempt = anarchy_run.run_post_datetime
if last_attempt:
if anarchy_run.failures > 9:
retry_delay = timedelta(hours=1)
else:
retry_delay = timedelta(seconds=5 * 2**anarchy_run.failures)
next_attempt = last_attempt + retry_delay
else:
next_attempt = datetime.utcnow()
anarchy_subject.retry_after = next_attempt
elif runner_value == 'lost':
anarchy_subject.anarchy_run_update(anarchy_run, runtime)
anarchy_subject.put_in_job_queue(runtime)
elif runner_value == 'pending':
anarchy_subject.enqueue_anarchy_run(anarchy_run, runtime)
else:
anarchy_subject.anarchy_run_update(anarchy_run, runtime)
def get_run():
anarchy_runner, runner_pod = check_runner_auth(flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
return
anarchy_run = anarchy_runner.runner_pods.get(runner_pod, None)
if anarchy_run:
operator_logger.warning(
'AnarchyRunner %s pod %s get run, but already had run %s!',
anarchy_runner.name, runner_pod, anarchy_run.name
)
anarchy_run.handle_lost_runner(runner_pod, runtime)
anarchy_subject = AnarchySubject.get_pending(anarchy_runner.name, runtime)
if not anarchy_subject:
return flask.jsonify(None)
anarchy_run = anarchy_subject.get_anarchy_run(runtime)
if not anarchy_run:
return flask.jsonify(None)
anarchy_runner.runner_pods[runner_pod] = anarchy_run
anarchy_run.set_runner(anarchy_runner.name + '.' + runner_pod, runtime)
resp = anarchy_run.to_dict(runtime)
resp['subject'] = anarchy_subject.to_dict(runtime)
resp['governor'] = anarchy_subject.get_governor(runtime).to_dict(runtime)
return flask.jsonify(resp)
def handle_subject_update(body, **_):
wait_for_init()
try:
subject = AnarchySubject.get_from_resource(body)
subject.handle_update(runtime)
except AssertionError as e:
operator_logger.warning('AnarchySubject %s invalid: %s', body['metadata']['name'], e)
def run_subject_action_patch(subject_name, action_name):
"""
Callback from runner to update AnarchyAction associated with AnarchySubject assigned to runner.
The only function of this method currently is to pass JSON, `{"successful": true}` or
`{"failed": true}` to mark the action as finished.
"""
anarchy_runner, runner_pod = check_runner_auth(
flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
if not runtime.running_all_in_one \
and subject_name != runner_pod.metadata.labels.get(runtime.subject_label):
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot update actions for AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
anarchy_action = AnarchyAction.get(action_name, runtime)
if not anarchy_action:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchyAction %s!',
anarchy_runner.name, runner_pod.metadata.name, action_name)
flask.abort(400)
anarchy_subject = AnarchySubject.get(subject_name, runtime)
if not anarchy_subject:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to update action on deleted AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
anarchy_governor = anarchy_subject.get_governor(runtime)
if not anarchy_governor:
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot post action to AnarchySubject %s, unable to find AnarchyGovernor %s!',
anarchy_runner.name, runner_pod, subject_name,
anarchy_subject.governor_name)
flask.abort(400)
finished_state = None
if flask.request.json.get('successful', False):
finished_state = 'successful'
elif flask.request.json.get('failed', False):
finished_state = 'failed'
if finished_state != None:
anarchy_subject.remove_active_action(anarchy_action, runtime)
anarchy_action.set_finished(finished_state, runtime)
return flask.jsonify({
'success': True,
'result': anarchy_action.to_dict(runtime)
})
def handle_subject_event(event, **_):
'''
Anarchy uses on.event instead of on.delete because Anarchy needs custom
for removing finalizers. The finalizer will be removed immediately if the
AnarchyGovernor does not have a delete subject event handler. If there is
a delete subject event handler then it is up to the governor logic to remove
the finalizer.
'''
wait_for_init()
resource = event['object']
if event['type'] in ['ADDED', 'MODIFIED', None]:
if 'deletionTimestamp' in resource['metadata']:
subject = AnarchySubject.get_from_resource(resource)
subject.handle_delete(runtime)
def get_subject(self, runtime):
return AnarchySubject.get(self.subject_name, runtime)
def post_subject_action(subject_name):
anarchy_runner, runner_pod = check_runner_auth(
flask.request.headers.get('Authorization', ''))
if not anarchy_runner:
flask.abort(400)
return
if subject_name != runner_pod.metadata.labels.get(runtime.subject_label):
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot update AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
return
anarchy_subject = AnarchySubject.get(subject_name, runtime)
if not anarchy_subject:
operator_logger.warning(
'AnarchyRunner %s Pod %s attempted to create action on deleted AnarchySubject %s!',
anarchy_runner.name, runner_pod.metadata.name, subject_name)
flask.abort(400)
return
anarchy_governor = anarchy_subject.get_governor(runtime)
if not anarchy_governor:
operator_logger.warning(
'AnarchyRunner %s Pod %s cannot post action to AnarchySubject %s, unable to find AnarchyGovernor %s!',
anarchy_runner.name, runner_pod, subject_name,
anarchy_subject.governor_name)
flask.abort(400)
return
action_name = flask.request.json.get('action', None)
after_timestamp = flask.request.json.get('after', None)
cancel_actions = flask.request.json.get('cancel', None)
if not action_name and not cancel_actions:
operator_logger.warning(
'No action or cancel given for scheduling action')
flask.abort(400)
return
if after_timestamp and not re.match(r'\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ',
after_timestamp):
operator_logger.warning(
'Invalide datetime format "%s" given for action after value',
after_timestamp)
flask.abort(400)
return
if action_name not in cancel_actions:
cancel_actions.append(action_name)
for action_resource in runtime.custom_objects_api.list_namespaced_custom_object(
runtime.operator_domain,
runtime.api_version,
runtime.operator_namespace,
'anarchyactions',
label_selector='{}/subject={}'.format(runtime.operator_domain,
anarchy_subject.name)).get(
'items', []):
if action_resource['spec']['action'] in cancel_actions \
and 'status' not in action_resource:
runtime.custom_objects_api.delete_namespaced_custom_object(
runtime.operator_domain, runtime.api_version,
runtime.operator_namespace, 'anarchyactions',
action_resource['metadata']['name'])
if action_name:
result = runtime.custom_objects_api.create_namespaced_custom_object(
runtime.operator_domain, runtime.api_version,
runtime.operator_namespace, 'anarchyactions', {
"apiVersion": runtime.api_group_version,
"kind": "AnarchyAction",
"metadata": {
"generateName":
"%s-%s-" % (anarchy_subject.name, action_name),
"labels": {
runtime.action_label: action_name,
runtime.subject_label: anarchy_subject.name,
runtime.governor_label: anarchy_governor.name
},
"ownerReferences": [{
"apiVersion": runtime.api_group_version,
"controller": True,
"kind": "AnarchySubject",
"name": anarchy_subject.name,
"uid": anarchy_subject.uid
}]
},
"spec": {
"action": action_name,
"after": after_timestamp,
"callbackToken": uuid.uuid4().hex,
"governorRef": {
"apiVersion": runtime.api_group_version,
"kind": "AnarchyGovernor",
"name": anarchy_governor.name,
"namespace": runtime.operator_namespace,
"uid": anarchy_governor.uid
},
"subjectRef": {
"apiVersion": runtime.api_group_version,
"kind": "AnarchySubject",
"name": anarchy_subject.name,
"namespace": runtime.operator_namespace,
"uid": anarchy_subject.uid
}
}
})
else:
result = None
return flask.jsonify({'success': True, 'result': result})
def handle_subject_timer(body, **_):
subject = AnarchySubject(body)
subject.set_active_run_to_pending_from_status(runtime)
def handle_subject_update(body, **_):
wait_for_init()
subject = AnarchySubject.get_from_resource(body)
subject.handle_update(runtime)