def backend_edit(request, object_id=None, api=False):
backend = None
server_form_list = []
header_form_list = []
httpchk_header_form_list = []
api_errors = []
if object_id:
try:
backend = Backend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = BackendForm(request.JSON or None,
instance=backend,
error_class=DivErrorList)
else:
form = BackendForm(request.POST or None,
instance=backend,
error_class=DivErrorList)
def render_form(back, **kwargs):
save_error = kwargs.get('save_error')
if api:
if len(api_errors) > 0 or form.errors:
api_errors.append(form.errors.as_json())
return JsonResponse({"errors": api_errors}, status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
available_sockets = get_darwin_sockets()
if not server_form_list and back:
for l_tmp in back.server_set.all():
server_form_list.append(ServerForm(instance=l_tmp))
if not header_form_list and back:
for h_tmp in back.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
if not httpchk_header_form_list and back:
for k, v in back.http_health_check_headers.items():
httpchk_header_form_list.append(
HttpHealthCheckHeaderForm({
'check_header_name': k,
'check_header_value': v
}))
return render(
request, 'apps/backend_edit.html', {
'form': form,
'servers': server_form_list,
'net_server_form': ServerForm(mode='net'),
'unix_server_form': ServerForm(mode='unix'),
'headers': header_form_list,
'header_form': HeaderForm(),
'sockets_choice': available_sockets,
'http_health_check_headers': httpchk_header_form_list,
'http_health_check_headers_form': HttpHealthCheckHeaderForm(),
**kwargs
})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
server_ids = request.JSON.get('servers', [])
assert isinstance(server_ids,
list), "Servers field must be a list."
else:
server_ids = json_loads(request.POST.get('servers', "[]"))
except Exception as e:
return render_form(backend,
save_error=[
"Error in Servers field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
header_objs = []
httpchk_headers_dict = {}
if form.data.get('mode') == "http":
""" Handle JSON formatted request headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids,
list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(
backend,
save_error=[
"Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
""" Handle JSON formatted request Http-health-check-headers """
try:
if api:
httpchk_header_ids = request.JSON.get(
'http_health_check_headers', [])
assert isinstance(
httpchk_header_ids,
list), "Health check headers field must be a list."
else:
httpchk_header_ids = json_loads(
request.POST.get('http_health_check_headers', "[]"))
except Exception as e:
return render_form(
backend,
save_error=[
"Error in Http-health-check-headers field : {}".format(
e),
str.join('', format_exception(*exc_info()))
])
""" For each Health check header in list """
for header in httpchk_header_ids:
httpchkform = HttpHealthCheckHeaderForm(
header, error_class=DivErrorList)
if not httpchkform.is_valid():
if api:
api_errors.append({
"health_check":
httpchkform.errors.get_json_data()
})
else:
form.add_error('enable_http_health_check',
httpchkform.errors.as_ul())
continue
# Save forms in case we re-print the page
httpchk_header_form_list.append(httpchkform)
httpchk_headers_dict[header.get(
'check_header_name')] = header.get('check_header_value')
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = Header.objects.get(
pk=header['id']) if header.get('id') else None
except ObjectDoesNotExist:
form.add_error(
None,
"Request-header with id {} not found. Injection detected ?"
)
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
api_errors.append(
{"headers": header_f.errors.get_json_data()})
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
server_objs = []
""" For each listener in list """
for server in server_ids:
""" If id is given, retrieve object from mongo """
try:
instance_s = Server.objects.get(
pk=server['id']) if server['id'] else None
except ObjectDoesNotExist:
form.add_error(
None, "Server with id {} not found.".format(server['id']))
continue
""" And instantiate form with the object, or None """
server_f = ServerForm(server, instance=instance_s)
if not server_f.is_valid():
if api:
api_errors.append(
{'server': server_f.errors.get_json_data()})
else:
form.add_error(None, server_f.errors.as_ul())
continue
server_form_list.append(server_f)
server_obj = server_f.save(commit=False)
server_objs.append(server_obj)
# If errors has been added in form
if not form.is_valid():
logger.error("Form errors: {}".format(form.errors.as_json()))
return render_form(backend)
# Save the form to get an id if there is not already one
backend = form.save(commit=False)
backend.configuration = ""
backend.http_health_check_headers = httpchk_headers_dict
# At least one server is required if Frontend enabled
if not server_objs and backend.enabled:
form.add_error(
None, "At least one server is required if backend is enabled.")
return render_form(backend)
""" Generate the non-yet saved object conf """
try:
logger.debug("Generating conf of backend '{}'".format(
backend.name))
backend.configuration = backend.generate_conf(
header_list=header_objs, server_list=server_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of backend '{}'".format(
backend.name))
backend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(backend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(backend,
save_error=[
"No referenced error",
str.join('', format_exception(*exc_info()))
])
""" If the conf is OK, save the Backend object """
# Is that object already in db or not
first_save = not backend.id
try:
logger.debug("Saving backend")
backend.save()
logger.debug("Backend '{}' (id={}) saved in MongoDB.".format(
backend.name, backend.id))
""" And all the listeners created earlier """
for s in server_objs:
s.backend = backend
logger.debug("Saving server {}".format(str(s)))
s.save()
""" Delete listeners deleted in form """
for s in backend.server_set.exclude(
pk__in=[l.id for l in server_objs]):
s.delete()
logger.info("Deleting server {}".format(s))
""" If mode is HTTP """
if backend.mode == "http":
""" Remove request-headers removed """
for header in backend.headers.all():
if header not in header_objs:
backend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
backend.headers.add(header)
logger.debug(
"HTTP Headers {} associated to Frontend {}".format(
header, backend))
""" if the Backend is updated and its name was changed """
if not first_save and "name" in form.changed_data:
logger.info(
"Backend name changed, looking for associated frontends..."
)
workflow_list = backend.workflow_set.all()
for workflow in workflow_list:
logger.info(
"reloading frontend '{}' haproxy configuration".format(
workflow.frontend))
workflow.frontend.reload_conf()
# Re-generate config AFTER save, to get ID
backend.configuration = backend.generate_conf()
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
backend.save_conf()
logger.debug("Write conf of backend '{}' asked on cluster".format(
backend.name))
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = Cluster.api_request(
"services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on cluster\n API request error.",
"haproxy",
traceback=api_res.get('message'))
for node in Node.objects.all():
backend.status[node.name] = "WAITING"
backend.save()
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for server in backend.server_set.all():
server.delete()
backend.delete()
logger.exception(e)
return render_form(backend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(
backend,
save_error=[
"Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))
])
if api:
return build_response(backend.id, "applications.backend.api",
COMMAND_LIST)
return HttpResponseRedirect('/apps/backend/')
return render_form(backend)
def frontend_edit(request, object_id=None, api=False):
frontend = None
listener_form_list = []
header_form_list = []
reputationctx_form_list = []
node_listeners = dict()
if object_id:
try:
frontend = Frontend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")}, status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = FrontendForm(request.JSON or None, instance=frontend, error_class=DivErrorList)
else:
form = FrontendForm(request.POST or None, instance=frontend, error_class=DivErrorList)
def render_form(front, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
logger.error("Frontend api form error : {}".format(form.errors.get_json_data()))
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
logger.error("Frontend api save error : {}".format(save_error))
return JsonResponse({'error': save_error[0]}, status=500)
if not listener_form_list and front:
for l_tmp in front.listener_set.all():
listener_form_list.append(ListenerForm(instance=l_tmp))
if not header_form_list and front:
for h_tmp in front.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
# If it is a new object, add default-example headers
if not front and request.method == "GET":
for header in DEFAULT_FRONTEND_HEADERS:
header_form_list.append(HeaderForm(header))
if not reputationctx_form_list and front:
for r_tmp in front.frontendreputationcontext_set.all():
reputationctx_form_list.append(FrontendReputationContextForm(instance=r_tmp))
return render(request, 'services/frontend_edit.html',
{'form': form, 'listeners': listener_form_list, 'listener_form': ListenerForm(),
'headers': header_form_list, 'header_form': HeaderForm(),
'reputation_contexts': reputationctx_form_list,
'reputationctx_form': FrontendReputationContextForm(),
'log_om_table': LogOMTableForm(auto_id=False),
'redis_forwarder': LogOM.objects.filter(name="Internal_Dashboard").only('id').first().id,
'object_id': (frontend.id if frontend else "") or "", **kwargs})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
listener_ids = request.JSON.get('listeners', [])
assert isinstance(listener_ids, list), "Listeners field must be a list."
else:
listener_ids = json_loads(request.POST.get('listeners', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Listeners field : {}".format(e),
str.join('', format_exception(*exc_info()))])
header_objs = []
reputationctx_objs = []
if form.data.get('mode') == "http":
""" Handle JSON formatted headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids, list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = frontend.headers.get(pk=header['id']) if frontend and header['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Request-header with id {} not found. Injection detected ?")
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
form.add_error(None, header_f.errors.get_json_data())
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
""" Handle JSON formatted ReputationContext """
try:
if api:
reputationctx_ids = request.JSON.get('reputation_contexts', [])
else:
reputationctx_ids = json_loads(request.POST.get('reputation_contexts', "[]"))
assert isinstance(reputationctx_ids, list), "reputation_contexts field must be a list."
except Exception as e:
return render_form(frontend, save_error=["Error in ReputationConext field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each reputation_context in list """
for reputationctx in reputationctx_ids:
""" Instantiate form """
reputationctx_f = FrontendReputationContextForm(reputationctx)
if not reputationctx_f.is_valid():
form.add_error(None if api else "reputation_ctx",
reputationctx_f.errors.get_json_data() if api else reputationctx_f.errors.as_ul())
continue
# Save forms in case we re-print the page
reputationctx_form_list.append(reputationctx_f)
# And save objects list, to save them later, when Frontend will be saved
reputationctx_objs.append(reputationctx_f.save(commit=False))
listener_objs = []
if form.data.get('mode') != "impcap" and form.data.get('listening_mode') != "file":
""" For each listener in list """
for listener in listener_ids:
""" If id is given, retrieve object from mongo """
try:
instance_l = Listener.objects.get(pk=listener['id']) if listener['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Listener with id {} not found.".format(listener['id']))
continue
""" And instantiate form with the object, or None """
listener_f = ListenerForm(listener, instance=instance_l)
if not listener_f.is_valid():
if api:
form.add_error("listeners", listener_f.errors.as_json())
else:
form.add_error("listeners", listener_f.errors.as_ul())
continue
listener_form_list.append(listener_f)
listener_obj = listener_f.save(commit=False)
listener_objs.append(listener_obj)
""" For each listener, get node """
for nic in listener_obj.network_address.nic.all().only('node'):
if not node_listeners.get(nic.node):
node_listeners[nic.node] = list()
node_listeners[nic.node].append(listener_obj)
# Get current Rsyslog configuration filename
old_rsyslog_filename = frontend.get_rsyslog_base_filename() if frontend and frontend.enable_logging else ""
# If errors has been added in form
if not form.is_valid():
logger.error("Frontend form errors: {}".format(form.errors.as_json()))
return render_form(frontend)
# Save the form to get an id if there is not already one
frontend = form.save(commit=False)
frontend.configuration = {}
if frontend.mode == "impcap":
node_listeners[frontend.impcap_intf.node] = []
elif frontend.listening_mode == "file":
node_listeners[frontend.node] = []
# At least one Listener is required if Frontend enabled, except for listener of type "File" and "pcap"
if not listener_objs and frontend.enabled and frontend.mode != "impcap" and frontend.listening_mode != "file":
form.add_error(None, "At least one listener is required if frontend is enabled.")
return render_form(frontend)
try:
""" For each node, the conf differs if listener chosen """
""" Generate the conf """
logger.debug("Generating conf of frontend '{}'".format(frontend.name))
for node, listeners in node_listeners.items():
# HAProxy conf does not use reputationctx_list, Rsyslog conf does
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners,
header_list=header_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of frontend '{}'".format(frontend.name))
frontend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["No referenced error",
str.join('', format_exception(*exc_info()))])
""" If the object already exists """
if frontend.id:
try:
changed_data = form.changed_data
""" If the ruleset has changed, we need to delete the old-named file """
if frontend.mode == "log" and "ruleset" in changed_data and old_rsyslog_filename:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
""" If it is an Rsyslog only conf """
if (frontend.mode == "log" and frontend.listening_mode in ("udp", "file")) \
or frontend.mode == "impcap":
""" And if it was not before saving """
if "mode" in changed_data or "listening_mode" in changed_data:
""" Delete old HAProxy conf """
frontend_filename = frontend.get_base_filename()
# API request deletion of frontend filename
Cluster.api_request('services.haproxy.haproxy.delete_conf', frontend_filename)
logger.info("HAProxy config '{}' deletion asked.".format(frontend_filename))
# And reload of HAProxy service
Cluster.api_request('services.haproxy.haproxy.reload_service')
""" If it is an HAProxy only conf """
elif frontend.mode not in ("log", "impcap") and not frontend.enable_logging:
""" And it was not """
if "mode" in changed_data or "enable_logging" in changed_data:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
# And reload of Rsyslog service
Cluster.api_request('services.rsyslogd.rsyslog.restart_service')
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["Cluster API request failure: {}".format(e),
str.join('', format_exception(*exc_info()))])
if form.errors:
logger.error("Frontend form errors: {}".format(form.errors))
return render_form(frontend)
""" If the conf is OK, save the Frontend object """
# Is that object already in db or not
first_save = not frontend.id
try:
if frontend.mode == "http":
frontend.ruleset = "haproxy"
elif frontend.mode == "tcp":
frontend.ruleset = "haproxy_tcp"
elif frontend.mode == "impcap":
frontend.ruleset = "impcap"
if frontend.enable_logging:
log_forwarders = set()
""" Handle LogForwarders selected objects in log_condition """
for log_om_name in re_findall("{{([^}]+)}}", frontend.log_condition):
try:
# Only id because we do not need any attribute
log_om = LogOM().select_log_om_by_name(log_om_name)
""" If the relation does not already exists : create-it """
log_forwarders.add(log_om.id)
except ObjectDoesNotExist:
form.add_error("log_condition", "LogForwarder not found.")
return render_form(frontend)
frontend.log_forwarders_id = log_forwarders
logger.debug("Saving frontend")
frontend.save()
logger.debug("Frontend '{}' (id={}) saved in MongoDB.".format(frontend.name, frontend.id))
""" And all the listeners early created """
for l in listener_objs:
l.frontend = frontend
logger.debug("Saving listener {}".format(str(l)))
l.save()
""" Delete listeners deleted in form """
for l in frontend.listener_set.exclude(pk__in=[l.id for l in listener_objs]):
l.delete()
logger.info("Deleting listener {}".format(l))
""" If mode is HTTP """
if frontend.mode == "http":
""" Remove request-headers removed """
for header in frontend.headers.all():
if header not in header_objs:
frontend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
frontend.headers.add(header)
logger.debug("HTTP Headers {} associated to Frontend {}".format(header, frontend))
# Delete all intermediary objects
FrontendReputationContext.objects.filter(frontend=frontend).delete()
# And re-create them
for reputationctx in reputationctx_objs:
FrontendReputationContext.objects.create(frontend=frontend,
reputation_ctx=reputationctx.reputation_ctx,
enabled=reputationctx.enabled,
arg_field=reputationctx.arg_field)
# Re-generate config AFTER save, to get ID
for node, listeners in node_listeners.items():
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners)
for node in node_listeners.keys():
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
frontend.save_conf(node)
logger.debug("Write conf of frontend '{}' asked on node '{}'".format(frontend.name, node.name))
""" We need to configure Rsyslog, it will check if conf has changed & restart service if needed """
api_res = node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.id)
if not api_res.get('status'):
raise ServiceConfigError("on node {}\n API request error.".format(node.name), "rsyslog",
traceback=api_res.get('message'))
if not frontend.rsyslog_only_conf:
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = node.api_request("services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on node {}\n API request error.".format(node.name), "haproxy",
traceback=api_res.get('message'))
frontend.status[node.name] = "WAITING"
frontend.save()
# Check if reload logrotate conf if needed
# meaning if there is a log_forwarder file enabled used by this frontend
if frontend.enable_logging and (frontend.log_forwarders.filter(logomfile__enabled=True).count() > 0 or
frontend.log_forwarders_parse_failure.filter(logomfile__enabled=True).count()):
# Reload LogRotate config
Cluster.api_request("services.logrotate.logrotate.reload_conf")
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for listener in frontend.listener_set.all():
listener.delete()
frontend.delete()
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(frontend, save_error=["Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))])
if api:
return build_response(frontend.id, "services.frontend.api", COMMAND_LIST)
return HttpResponseRedirect('/services/frontend')
return render_form(frontend)