def get(self, request, fw_type=None, object_id=None):
try:
if object_id:
try:
obj = LogOM().select_log_om(object_id).to_dict()
except ObjectDoesNotExist:
return JsonResponse({'error': _('Object does not exist')},
status=404)
elif fw_type:
try:
obj = [
logOM.to_dict()
for logOM in LOGFWD_MODELS[fw_type].objects.all()
]
except KeyError:
return JsonResponse({'error': _('Type does not exist')},
status=404)
else:
obj = [
LogOM().select_log_om(logom.pk).to_dict()
for logom in LogOM.objects.all().only('pk')
]
return JsonResponse({'data': obj})
except Exception as e:
logger.critical(e, exc_info=1)
error = _("An error has occurred")
if settings.DEV_MODE:
error = str(e)
return JsonResponse({'error': error}, status=500)
class LogOMTableForm(Form):
""" Form used to generate table for log_forwarders """
""" First condition word """
condition = ChoiceField(
label=_("Condition"),
choices=CONDITION_CHOICES,
widget=Select(attrs={'class': 'form-control select2'}))
""" Comparison field name """
field_name = ChoiceField(
label=_("Field"),
choices=FIELD_CHOICES,
widget=Select(attrs={'class': 'form-control select2'}))
""" Comparison operator """
operator = ChoiceField(
label=_("Operator"),
choices=OPERATOR_CHOICES,
widget=Select(attrs={'class': 'form-control select2'}))
""" Comparison value """
value = CharField(label=_("Value"),
initial='"plop"',
widget=TextInput(attrs={'class': 'form-control'}))
""" Action - log forwarder """
action = ModelMultipleChoiceField(
label=_("Action"),
queryset=LogOM.objects.all().only(*LogOM.str_attrs()),
widget=SelectMultiple(attrs={'class': 'form-control select2'}))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields['action'].empty_label = None
def as_table_headers(self):
""" Format field names as table head """
result = "<tr>\n"
for field in self:
result += "<th>{}</th>\n".format(field.label)
result += "<th>Delete</th></tr>\n"
return result
def as_table_td(self):
""" Format fields as a table with <td></td> """
result = "<tr>"
for field in self:
result += "<td>{}</td>".format(field)
result += "<td style='text-align:center'><a class='btnDelete'><i style='color:grey' " \
"class='fas fa-trash-alt'></i></a></td></tr>\n"
return result
def clean_log_condition(self):
""" Verify if log_condition contains valids LogOM name into {{}} """
regex = "{{([^}]+)}}"
log_condition = self.cleaned_data.get('log_condition',
"").replace("\r\n", "\n")
for line in log_condition.split('\n'):
if line.count('{') > 2:
raise ValidationError("There cannot be 2 actions on one line.")
match = re_search(regex, line)
if match:
try:
LogOM().select_log_om_by_name(match.group(1))
except ObjectDoesNotExist:
raise ValidationError(
"Log Forwarder named '{}' not found.".format(
match.group(1)))
return log_condition
def frontend_edit(request, object_id=None, api=False):
frontend = None
listener_form_list = []
header_form_list = []
reputationctx_form_list = []
node_listeners = dict()
if object_id:
try:
frontend = Frontend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")}, status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = FrontendForm(request.JSON or None, instance=frontend, error_class=DivErrorList)
else:
form = FrontendForm(request.POST or None, instance=frontend, error_class=DivErrorList)
def render_form(front, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
logger.error("Frontend api form error : {}".format(form.errors.get_json_data()))
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
logger.error("Frontend api save error : {}".format(save_error))
return JsonResponse({'error': save_error[0]}, status=500)
if not listener_form_list and front:
for l_tmp in front.listener_set.all():
listener_form_list.append(ListenerForm(instance=l_tmp))
if not header_form_list and front:
for h_tmp in front.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
# If it is a new object, add default-example headers
if not front and request.method == "GET":
for header in DEFAULT_FRONTEND_HEADERS:
header_form_list.append(HeaderForm(header))
if not reputationctx_form_list and front:
for r_tmp in front.frontendreputationcontext_set.all():
reputationctx_form_list.append(FrontendReputationContextForm(instance=r_tmp))
return render(request, 'services/frontend_edit.html',
{'form': form, 'listeners': listener_form_list, 'listener_form': ListenerForm(),
'headers': header_form_list, 'header_form': HeaderForm(),
'reputation_contexts': reputationctx_form_list,
'reputationctx_form': FrontendReputationContextForm(),
'log_om_table': LogOMTableForm(auto_id=False),
'redis_forwarder': LogOM.objects.filter(name="Internal_Dashboard").only('id').first().id,
'object_id': (frontend.id if frontend else "") or "", **kwargs})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
listener_ids = request.JSON.get('listeners', [])
assert isinstance(listener_ids, list), "Listeners field must be a list."
else:
listener_ids = json_loads(request.POST.get('listeners', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Listeners field : {}".format(e),
str.join('', format_exception(*exc_info()))])
header_objs = []
reputationctx_objs = []
if form.data.get('mode') == "http":
""" Handle JSON formatted headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids, list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = frontend.headers.get(pk=header['id']) if frontend and header['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Request-header with id {} not found. Injection detected ?")
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
form.add_error(None, header_f.errors.get_json_data())
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
""" Handle JSON formatted ReputationContext """
try:
if api:
reputationctx_ids = request.JSON.get('reputation_contexts', [])
else:
reputationctx_ids = json_loads(request.POST.get('reputation_contexts', "[]"))
assert isinstance(reputationctx_ids, list), "reputation_contexts field must be a list."
except Exception as e:
return render_form(frontend, save_error=["Error in ReputationConext field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each reputation_context in list """
for reputationctx in reputationctx_ids:
""" Instantiate form """
reputationctx_f = FrontendReputationContextForm(reputationctx)
if not reputationctx_f.is_valid():
form.add_error(None if api else "reputation_ctx",
reputationctx_f.errors.get_json_data() if api else reputationctx_f.errors.as_ul())
continue
# Save forms in case we re-print the page
reputationctx_form_list.append(reputationctx_f)
# And save objects list, to save them later, when Frontend will be saved
reputationctx_objs.append(reputationctx_f.save(commit=False))
listener_objs = []
if form.data.get('mode') != "impcap" and form.data.get('listening_mode') != "file":
""" For each listener in list """
for listener in listener_ids:
""" If id is given, retrieve object from mongo """
try:
instance_l = Listener.objects.get(pk=listener['id']) if listener['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Listener with id {} not found.".format(listener['id']))
continue
""" And instantiate form with the object, or None """
listener_f = ListenerForm(listener, instance=instance_l)
if not listener_f.is_valid():
if api:
form.add_error("listeners", listener_f.errors.as_json())
else:
form.add_error("listeners", listener_f.errors.as_ul())
continue
listener_form_list.append(listener_f)
listener_obj = listener_f.save(commit=False)
listener_objs.append(listener_obj)
""" For each listener, get node """
for nic in listener_obj.network_address.nic.all().only('node'):
if not node_listeners.get(nic.node):
node_listeners[nic.node] = list()
node_listeners[nic.node].append(listener_obj)
# Get current Rsyslog configuration filename
old_rsyslog_filename = frontend.get_rsyslog_base_filename() if frontend and frontend.enable_logging else ""
# If errors has been added in form
if not form.is_valid():
logger.error("Frontend form errors: {}".format(form.errors.as_json()))
return render_form(frontend)
# Save the form to get an id if there is not already one
frontend = form.save(commit=False)
frontend.configuration = {}
if frontend.mode == "impcap":
node_listeners[frontend.impcap_intf.node] = []
elif frontend.listening_mode == "file":
node_listeners[frontend.node] = []
# At least one Listener is required if Frontend enabled, except for listener of type "File" and "pcap"
if not listener_objs and frontend.enabled and frontend.mode != "impcap" and frontend.listening_mode != "file":
form.add_error(None, "At least one listener is required if frontend is enabled.")
return render_form(frontend)
try:
""" For each node, the conf differs if listener chosen """
""" Generate the conf """
logger.debug("Generating conf of frontend '{}'".format(frontend.name))
for node, listeners in node_listeners.items():
# HAProxy conf does not use reputationctx_list, Rsyslog conf does
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners,
header_list=header_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of frontend '{}'".format(frontend.name))
frontend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["No referenced error",
str.join('', format_exception(*exc_info()))])
""" If the object already exists """
if frontend.id:
try:
changed_data = form.changed_data
""" If the ruleset has changed, we need to delete the old-named file """
if frontend.mode == "log" and "ruleset" in changed_data and old_rsyslog_filename:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
""" If it is an Rsyslog only conf """
if (frontend.mode == "log" and frontend.listening_mode in ("udp", "file")) \
or frontend.mode == "impcap":
""" And if it was not before saving """
if "mode" in changed_data or "listening_mode" in changed_data:
""" Delete old HAProxy conf """
frontend_filename = frontend.get_base_filename()
# API request deletion of frontend filename
Cluster.api_request('services.haproxy.haproxy.delete_conf', frontend_filename)
logger.info("HAProxy config '{}' deletion asked.".format(frontend_filename))
# And reload of HAProxy service
Cluster.api_request('services.haproxy.haproxy.reload_service')
""" If it is an HAProxy only conf """
elif frontend.mode not in ("log", "impcap") and not frontend.enable_logging:
""" And it was not """
if "mode" in changed_data or "enable_logging" in changed_data:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
# And reload of Rsyslog service
Cluster.api_request('services.rsyslogd.rsyslog.restart_service')
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["Cluster API request failure: {}".format(e),
str.join('', format_exception(*exc_info()))])
if form.errors:
logger.error("Frontend form errors: {}".format(form.errors))
return render_form(frontend)
""" If the conf is OK, save the Frontend object """
# Is that object already in db or not
first_save = not frontend.id
try:
if frontend.mode == "http":
frontend.ruleset = "haproxy"
elif frontend.mode == "tcp":
frontend.ruleset = "haproxy_tcp"
elif frontend.mode == "impcap":
frontend.ruleset = "impcap"
if frontend.enable_logging:
log_forwarders = set()
""" Handle LogForwarders selected objects in log_condition """
for log_om_name in re_findall("{{([^}]+)}}", frontend.log_condition):
try:
# Only id because we do not need any attribute
log_om = LogOM().select_log_om_by_name(log_om_name)
""" If the relation does not already exists : create-it """
log_forwarders.add(log_om.id)
except ObjectDoesNotExist:
form.add_error("log_condition", "LogForwarder not found.")
return render_form(frontend)
frontend.log_forwarders_id = log_forwarders
logger.debug("Saving frontend")
frontend.save()
logger.debug("Frontend '{}' (id={}) saved in MongoDB.".format(frontend.name, frontend.id))
""" And all the listeners early created """
for l in listener_objs:
l.frontend = frontend
logger.debug("Saving listener {}".format(str(l)))
l.save()
""" Delete listeners deleted in form """
for l in frontend.listener_set.exclude(pk__in=[l.id for l in listener_objs]):
l.delete()
logger.info("Deleting listener {}".format(l))
""" If mode is HTTP """
if frontend.mode == "http":
""" Remove request-headers removed """
for header in frontend.headers.all():
if header not in header_objs:
frontend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
frontend.headers.add(header)
logger.debug("HTTP Headers {} associated to Frontend {}".format(header, frontend))
# Delete all intermediary objects
FrontendReputationContext.objects.filter(frontend=frontend).delete()
# And re-create them
for reputationctx in reputationctx_objs:
FrontendReputationContext.objects.create(frontend=frontend,
reputation_ctx=reputationctx.reputation_ctx,
enabled=reputationctx.enabled,
arg_field=reputationctx.arg_field)
# Re-generate config AFTER save, to get ID
for node, listeners in node_listeners.items():
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners)
for node in node_listeners.keys():
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
frontend.save_conf(node)
logger.debug("Write conf of frontend '{}' asked on node '{}'".format(frontend.name, node.name))
""" We need to configure Rsyslog, it will check if conf has changed & restart service if needed """
api_res = node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.id)
if not api_res.get('status'):
raise ServiceConfigError("on node {}\n API request error.".format(node.name), "rsyslog",
traceback=api_res.get('message'))
if not frontend.rsyslog_only_conf:
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = node.api_request("services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on node {}\n API request error.".format(node.name), "haproxy",
traceback=api_res.get('message'))
frontend.status[node.name] = "WAITING"
frontend.save()
# Check if reload logrotate conf if needed
# meaning if there is a log_forwarder file enabled used by this frontend
if frontend.enable_logging and (frontend.log_forwarders.filter(logomfile__enabled=True).count() > 0 or
frontend.log_forwarders_parse_failure.filter(logomfile__enabled=True).count()):
# Reload LogRotate config
Cluster.api_request("services.logrotate.logrotate.reload_conf")
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for listener in frontend.listener_set.all():
listener.delete()
frontend.delete()
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(frontend, save_error=["Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))])
if api:
return build_response(frontend.id, "services.frontend.api", COMMAND_LIST)
return HttpResponseRedirect('/services/frontend')
return render_form(frontend)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
""" Impcap/Log Darwin policy """
self.fields['darwin_policy'] = ModelChoiceField(
label=_("Darwin policy"),
queryset=DarwinPolicy.objects.all(),
widget=Select(attrs={'class': 'form-control select2'}),
required=False)
""" Log forwarders """
self.fields['log_forwarders'] = ModelMultipleChoiceField(
label=_("Log forwarders"),
queryset=LogOM.objects.all().only(*LogOM.str_attrs()),
widget=SelectMultiple(attrs={'class': 'form-control select2'}),
required=False)
""" Log forwarders """
self.fields['log_forwarders_parse_failure'] = ModelMultipleChoiceField(
label=_("Log forwarders - parse failure"),
queryset=LogOM.objects.all().only(*LogOM.str_attrs()),
widget=SelectMultiple(attrs={'class': 'form-control select2'}),
required=False)
""" MMDP Reputation database IPv4 """
# Defined here AND in model, to use queryset
self.fields['logging_reputation_database_v4'] = ModelChoiceField(
label=_("Rsyslog IPv4 reputation database"),
# queryset=[(f.get('id'), str(f)) for f in Feed.objects.mongo_find({"filename": {"$regex": r"\.mmdb$"}, # MMDB database
# "label": {"$regex": r"^((?![iI][Pp][Vv]6).)*$"}}, # Excude IPv6
# {"filename": 1, "label": 1})], # .only( label, filename )
# queryset=Feed.objects.filter(filename__iregex="^((?![iI][Pp][Vv]6).)*\.mmdb$"),
# queryset=Feed.objects.exclude(filename__iregex="^((?![iI][Pp][Vv]6).)*$").filter(filename__endswith=".mmdb"),
queryset=ReputationContext.objects.filter(
db_type="ipv4", filename__endswith=".mmdb").only(
*(ReputationContext.str_attrs() +
['filename', 'db_type'])),
widget=Select(attrs={'class': 'form-control select2'}),
empty_label="No IPv4",
required=False)
""" MMDP Reputation database IPv6 """
# Defined here AND in model, to use queryset
self.fields['logging_reputation_database_v6'] = ModelChoiceField(
label=_("Rsyslog IPv6 reputation database"),
queryset=ReputationContext.objects.filter(
db_type="ipv6",
filename__endswith=".mmdb") # MMDP database & IPv6
.only(*(ReputationContext.str_attrs() + ['filename', 'db_type'])),
widget=Select(attrs={'class': 'form-control select2'}),
empty_label="No IPv6",
required=False)
self.fields['logging_geoip_database'] = ModelChoiceField(
label=_("Rsyslog GeoIP database"),
queryset=ReputationContext.objects.filter(db_type="GeoIP").only(
*(ReputationContext.str_attrs() + ['filename', 'db_type'])),
widget=Select(attrs={'class': 'form-control select2'}),
empty_label="No GeoIP",
required=False)
self.fields['node'] = ModelChoiceField(
label=_('Node'),
queryset=Node.objects.all(),
widget=Select(attrs={'class': 'form-control select2'}))
# Remove the blank input generated by django
for field_name in [
'mode', 'ruleset', 'log_level', 'listening_mode',
'compression_algos', 'log_forwarders', 'impcap_intf'
]:
self.fields[field_name].empty_label = None
self.fields['error_template'].empty_label = "No template"
# Set required in POST data to False
for field_name in [
'log_condition', 'ruleset', 'log_level', 'listening_mode',
'headers', 'custom_haproxy_conf', 'cache_total_max_size',
'cache_max_age', 'compression_algos', 'compression_mime_types',
'error_template', 'enable_logging_reputation', 'impcap_filter',
'impcap_filter_type', 'impcap_intf', 'tags', 'timeout_client',
'timeout_connect', 'timeout_keep_alive', 'parser_tag',
'file_path', 'node'
]:
self.fields[field_name].required = False
""" Build choices of "ruleset" field with rsyslog jinja templates names """
# read the entries
try:
with scandir(JINJA_RSYSLOG_PATH) as listOfEntries:
for entry in listOfEntries:
if entry.is_dir():
m = re_search("rsyslog_ruleset_([\w-]+)", entry.name)
if m:
# Do NOT process haproxy - it's an internal log type
if m.group(1) in ("haproxy", "haproxy_tcp"):
continue
self.fields['ruleset'].widget.choices.append(
(m.group(1), m.group(1)))
except Exception as e:
logger.error(
"Cannot build 'ruleset' choices. Seems that path '{}' is not found: "
.format(JINJA_RSYSLOG_PATH))
logger.exception(e)
# Set initial value of compression_algos field,
# convert space separated string into list
if self.initial.get('compression_algos'):
self.initial['compression_algos'] = self.initial.get(
'compression_algos').split(' ')
self.initial['tags'] = ','.join(
self.initial.get('tags', []) or self.fields['tags'].initial)