def reload_frontend_conf(self):
""" Send API request on each Nodes using the frontends that uses the current CTX
:return The list of concerned nodes
"""
from services.frontend.models import Listener
res = []
# Loop on Nodes
for node in Node.objects.all():
frontends = []
# Get listeners enabled on this node, using the current reputation context
for listener in Listener.objects.filter(
frontend__enabled=True,
frontend__reputation_ctxs=self.id,
network_address__nic__node=node.id).distinct():
if listener.frontend.id not in frontends:
api_res = node.api_request(
"services.rsyslogd.rsyslog.build_conf",
listener.frontend.id)
if not api_res:
raise ServiceConfigError(
"on node '{}' \n API request error.".format(
node.name),
"rsyslog",
traceback=api_res.get('message'))
frontends.append(listener.frontend.id)
res.append(node)
return res
def conf_has_changed(self, new_conf):
"""
Check if configuration has changed
:param new_conf: Conf as text to compare with disk content
:return: conf generated if conf has changed, empty string otherwise
raise ServiceConfigError if error
"""
try:
current_conf = self.read_current_conf()
if current_conf != new_conf:
return new_conf
return False
except ServiceExit:
raise
except Exception as e:
logger.error("Unable to check if conf has changed for {}: ".format(
self.service_name, str(e)))
logger.exception(e)
if "No such file" in str(e):
raise ServiceNoConfigError(str(e), self.service_name)
raise ServiceConfigError(
"Cannot open '{}' : {}".format(self.jinja_template['tpl_path'],
str(e)), self.service_name)
def get_conf(self):
""" Generate conf from mongo object and Jinja template
:return Generated configuration, to write into file
If error, raise ServiceConfigError
"""
try:
path_config = os_path.join(settings.BASE_DIR, 'services', 'config')
jinja2_env = Environment(loader=FileSystemLoader(path_config))
template = jinja2_env.get_template(self.jinja_template['tpl_name'])
return template.render({
'node': Cluster.get_current_node(),
**self.get_dict_conf()
})
except ServiceExit:
raise
except Exception as e:
raise
logger.error(e)
raise ServiceConfigError(
"Failed to generate jinja template: {}".format(str(e)),
self.service_name)
def frontend_edit(request, object_id=None, api=False):
frontend = None
listener_form_list = []
header_form_list = []
reputationctx_form_list = []
node_listeners = dict()
if object_id:
try:
frontend = Frontend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")}, status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = FrontendForm(request.JSON or None, instance=frontend, error_class=DivErrorList)
else:
form = FrontendForm(request.POST or None, instance=frontend, error_class=DivErrorList)
def render_form(front, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
logger.error("Frontend api form error : {}".format(form.errors.get_json_data()))
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
logger.error("Frontend api save error : {}".format(save_error))
return JsonResponse({'error': save_error[0]}, status=500)
if not listener_form_list and front:
for l_tmp in front.listener_set.all():
listener_form_list.append(ListenerForm(instance=l_tmp))
if not header_form_list and front:
for h_tmp in front.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
# If it is a new object, add default-example headers
if not front and request.method == "GET":
for header in DEFAULT_FRONTEND_HEADERS:
header_form_list.append(HeaderForm(header))
if not reputationctx_form_list and front:
for r_tmp in front.frontendreputationcontext_set.all():
reputationctx_form_list.append(FrontendReputationContextForm(instance=r_tmp))
return render(request, 'services/frontend_edit.html',
{'form': form, 'listeners': listener_form_list, 'listener_form': ListenerForm(),
'headers': header_form_list, 'header_form': HeaderForm(),
'reputation_contexts': reputationctx_form_list,
'reputationctx_form': FrontendReputationContextForm(),
'log_om_table': LogOMTableForm(auto_id=False),
'redis_forwarder': LogOM.objects.filter(name="Internal_Dashboard").only('id').first().id,
'object_id': (frontend.id if frontend else "") or "", **kwargs})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
listener_ids = request.JSON.get('listeners', [])
assert isinstance(listener_ids, list), "Listeners field must be a list."
else:
listener_ids = json_loads(request.POST.get('listeners', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Listeners field : {}".format(e),
str.join('', format_exception(*exc_info()))])
header_objs = []
reputationctx_objs = []
if form.data.get('mode') == "http":
""" Handle JSON formatted headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids, list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = frontend.headers.get(pk=header['id']) if frontend and header['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Request-header with id {} not found. Injection detected ?")
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
form.add_error(None, header_f.errors.get_json_data())
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
""" Handle JSON formatted ReputationContext """
try:
if api:
reputationctx_ids = request.JSON.get('reputation_contexts', [])
else:
reputationctx_ids = json_loads(request.POST.get('reputation_contexts', "[]"))
assert isinstance(reputationctx_ids, list), "reputation_contexts field must be a list."
except Exception as e:
return render_form(frontend, save_error=["Error in ReputationConext field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each reputation_context in list """
for reputationctx in reputationctx_ids:
""" Instantiate form """
reputationctx_f = FrontendReputationContextForm(reputationctx)
if not reputationctx_f.is_valid():
form.add_error(None if api else "reputation_ctx",
reputationctx_f.errors.get_json_data() if api else reputationctx_f.errors.as_ul())
continue
# Save forms in case we re-print the page
reputationctx_form_list.append(reputationctx_f)
# And save objects list, to save them later, when Frontend will be saved
reputationctx_objs.append(reputationctx_f.save(commit=False))
listener_objs = []
if form.data.get('mode') != "impcap" and form.data.get('listening_mode') != "file":
""" For each listener in list """
for listener in listener_ids:
""" If id is given, retrieve object from mongo """
try:
instance_l = Listener.objects.get(pk=listener['id']) if listener['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Listener with id {} not found.".format(listener['id']))
continue
""" And instantiate form with the object, or None """
listener_f = ListenerForm(listener, instance=instance_l)
if not listener_f.is_valid():
if api:
form.add_error("listeners", listener_f.errors.as_json())
else:
form.add_error("listeners", listener_f.errors.as_ul())
continue
listener_form_list.append(listener_f)
listener_obj = listener_f.save(commit=False)
listener_objs.append(listener_obj)
""" For each listener, get node """
for nic in listener_obj.network_address.nic.all().only('node'):
if not node_listeners.get(nic.node):
node_listeners[nic.node] = list()
node_listeners[nic.node].append(listener_obj)
# Get current Rsyslog configuration filename
old_rsyslog_filename = frontend.get_rsyslog_base_filename() if frontend and frontend.enable_logging else ""
# If errors has been added in form
if not form.is_valid():
logger.error("Frontend form errors: {}".format(form.errors.as_json()))
return render_form(frontend)
# Save the form to get an id if there is not already one
frontend = form.save(commit=False)
frontend.configuration = {}
if frontend.mode == "impcap":
node_listeners[frontend.impcap_intf.node] = []
elif frontend.listening_mode == "file":
node_listeners[frontend.node] = []
# At least one Listener is required if Frontend enabled, except for listener of type "File" and "pcap"
if not listener_objs and frontend.enabled and frontend.mode != "impcap" and frontend.listening_mode != "file":
form.add_error(None, "At least one listener is required if frontend is enabled.")
return render_form(frontend)
try:
""" For each node, the conf differs if listener chosen """
""" Generate the conf """
logger.debug("Generating conf of frontend '{}'".format(frontend.name))
for node, listeners in node_listeners.items():
# HAProxy conf does not use reputationctx_list, Rsyslog conf does
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners,
header_list=header_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of frontend '{}'".format(frontend.name))
frontend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["No referenced error",
str.join('', format_exception(*exc_info()))])
""" If the object already exists """
if frontend.id:
try:
changed_data = form.changed_data
""" If the ruleset has changed, we need to delete the old-named file """
if frontend.mode == "log" and "ruleset" in changed_data and old_rsyslog_filename:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
""" If it is an Rsyslog only conf """
if (frontend.mode == "log" and frontend.listening_mode in ("udp", "file")) \
or frontend.mode == "impcap":
""" And if it was not before saving """
if "mode" in changed_data or "listening_mode" in changed_data:
""" Delete old HAProxy conf """
frontend_filename = frontend.get_base_filename()
# API request deletion of frontend filename
Cluster.api_request('services.haproxy.haproxy.delete_conf', frontend_filename)
logger.info("HAProxy config '{}' deletion asked.".format(frontend_filename))
# And reload of HAProxy service
Cluster.api_request('services.haproxy.haproxy.reload_service')
""" If it is an HAProxy only conf """
elif frontend.mode not in ("log", "impcap") and not frontend.enable_logging:
""" And it was not """
if "mode" in changed_data or "enable_logging" in changed_data:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
# And reload of Rsyslog service
Cluster.api_request('services.rsyslogd.rsyslog.restart_service')
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["Cluster API request failure: {}".format(e),
str.join('', format_exception(*exc_info()))])
if form.errors:
logger.error("Frontend form errors: {}".format(form.errors))
return render_form(frontend)
""" If the conf is OK, save the Frontend object """
# Is that object already in db or not
first_save = not frontend.id
try:
if frontend.mode == "http":
frontend.ruleset = "haproxy"
elif frontend.mode == "tcp":
frontend.ruleset = "haproxy_tcp"
elif frontend.mode == "impcap":
frontend.ruleset = "impcap"
if frontend.enable_logging:
log_forwarders = set()
""" Handle LogForwarders selected objects in log_condition """
for log_om_name in re_findall("{{([^}]+)}}", frontend.log_condition):
try:
# Only id because we do not need any attribute
log_om = LogOM().select_log_om_by_name(log_om_name)
""" If the relation does not already exists : create-it """
log_forwarders.add(log_om.id)
except ObjectDoesNotExist:
form.add_error("log_condition", "LogForwarder not found.")
return render_form(frontend)
frontend.log_forwarders_id = log_forwarders
logger.debug("Saving frontend")
frontend.save()
logger.debug("Frontend '{}' (id={}) saved in MongoDB.".format(frontend.name, frontend.id))
""" And all the listeners early created """
for l in listener_objs:
l.frontend = frontend
logger.debug("Saving listener {}".format(str(l)))
l.save()
""" Delete listeners deleted in form """
for l in frontend.listener_set.exclude(pk__in=[l.id for l in listener_objs]):
l.delete()
logger.info("Deleting listener {}".format(l))
""" If mode is HTTP """
if frontend.mode == "http":
""" Remove request-headers removed """
for header in frontend.headers.all():
if header not in header_objs:
frontend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
frontend.headers.add(header)
logger.debug("HTTP Headers {} associated to Frontend {}".format(header, frontend))
# Delete all intermediary objects
FrontendReputationContext.objects.filter(frontend=frontend).delete()
# And re-create them
for reputationctx in reputationctx_objs:
FrontendReputationContext.objects.create(frontend=frontend,
reputation_ctx=reputationctx.reputation_ctx,
enabled=reputationctx.enabled,
arg_field=reputationctx.arg_field)
# Re-generate config AFTER save, to get ID
for node, listeners in node_listeners.items():
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners)
for node in node_listeners.keys():
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
frontend.save_conf(node)
logger.debug("Write conf of frontend '{}' asked on node '{}'".format(frontend.name, node.name))
""" We need to configure Rsyslog, it will check if conf has changed & restart service if needed """
api_res = node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.id)
if not api_res.get('status'):
raise ServiceConfigError("on node {}\n API request error.".format(node.name), "rsyslog",
traceback=api_res.get('message'))
if not frontend.rsyslog_only_conf:
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = node.api_request("services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on node {}\n API request error.".format(node.name), "haproxy",
traceback=api_res.get('message'))
frontend.status[node.name] = "WAITING"
frontend.save()
# Check if reload logrotate conf if needed
# meaning if there is a log_forwarder file enabled used by this frontend
if frontend.enable_logging and (frontend.log_forwarders.filter(logomfile__enabled=True).count() > 0 or
frontend.log_forwarders_parse_failure.filter(logomfile__enabled=True).count()):
# Reload LogRotate config
Cluster.api_request("services.logrotate.logrotate.reload_conf")
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for listener in frontend.listener_set.all():
listener.delete()
frontend.delete()
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(frontend, save_error=["Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))])
if api:
return build_response(frontend.id, "services.frontend.api", COMMAND_LIST)
return HttpResponseRedirect('/services/frontend')
return render_form(frontend)