def edit(request, object_id=None, api=False):
repo = None
if object_id:
try:
repo = OpenIDRepository.objects.get(pk=object_id)
except ObjectDoesNotExist:
return HttpResponseForbidden("Injection detected")
if hasattr(request, "JSON") and api:
form = OpenIDRepositoryForm(request.JSON or None,
instance=repo,
error_class=DivErrorList)
else:
form = OpenIDRepositoryForm(request.POST or None,
instance=repo,
error_class=DivErrorList)
if request.method in ("POST", "PUT") and form.is_valid():
# Save the form to get an id if there is not already one
repo = form.save(commit=False)
# If provider_url changed, force reload of configuration
if "provider_url" in form.changed_data:
repo.last_config_time = None
repo.save()
# If everything succeed, redirect to list view
if api:
return build_response(repo.id, "authentication.openid.api", [])
return HttpResponseRedirect('/authentication/openid/')
return render(request, 'authentication/openid_edit.html', {'form': form})
def otp_edit(request, object_id=None, api=False):
otp = None
if object_id:
try:
otp = OTPRepository.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseNotFound(_("Object not found"))
if hasattr(request, "JSON") and api:
form = OTPRepositoryForm(request.JSON or None,
instance=otp,
error_class=DivErrorList)
else:
form = OTPRepositoryForm(request.POST or None,
instance=otp,
error_class=DivErrorList)
if request.method in ("POST", "PUT") and form.is_valid():
# Save the form to get an id if there is not already one
otp = form.save(commit=False)
otp.save()
# If everything succeed, redirect to list view
if api:
return build_response(otp.id, "api.authentication.otp", [])
return HttpResponseRedirect('/authentication/otp/')
if api:
return JsonResponse({"errors": build_form_errors(form.errors)},
status=400)
return render(request, 'authentication/otp_edit.html', {'form': form})
def ldap_edit(request, object_id=None, api=False):
ldap = None
if object_id:
try:
ldap = LDAPRepository.objects.get(pk=object_id)
except ObjectDoesNotExist:
return HttpResponseNotFound('Object not found')
if hasattr(request, "JSON") and api:
form = LDAPRepositoryForm(request.JSON or None,
instance=ldap,
error_class=DivErrorList)
else:
form = LDAPRepositoryForm(request.POST or None,
instance=ldap,
error_class=DivErrorList)
def render_form(**kwargs):
if api:
logger.error("Frontend api form error : {}".format(
form.errors.get_json_data()))
return JsonResponse({"errors": build_form_errors(form.errors)},
status=400)
return render(request, 'authentication/ldap_edit.html', {
'form': form,
'object_id': object_id,
**kwargs
})
if request.method == "POST":
if request.POST.get('connection_test'):
if form.connection_is_valid():
conf = form.save(commit=False)
ldap_client = conf.get_client()
result = ldap_client.test_ldap_connection()
if not result.get('status'):
return render_form(connection_error=result.get('reason'))
else:
return render_form(success="Successful connection")
else:
return render_form()
if request.method in ("POST", "PUT") and form.is_valid():
# Save the form to get an id if there is not already one
ldap = form.save(commit=False)
ldap.save()
# If everything succeed, redirect to list view
if api:
return build_response(ldap.id, "authentication.api.ldap", [])
return HttpResponseRedirect('/authentication/ldap/')
return render_form()
def reputation_ctx_edit(request, object_id=None, api=False, update=False):
header_form_list = list()
reputation_ctx = None
if object_id:
try:
reputation_ctx = ReputationContext.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if (hasattr(request, "JSON")
and api) or (request.method in ("POST", "PUT") and reputation_ctx
and reputation_ctx.internal):
# If internal objects, only tags is allowed to be modified
if reputation_ctx and reputation_ctx.internal:
request.JSON = {
**reputation_ctx.to_dict(), 'tags': request.POST.get('tags')
}
elif update:
request.JSON = {**reputation_ctx.to_dict(), **request.JSON}
form = ReputationContextForm(request.JSON or None,
instance=reputation_ctx,
error_class=DivErrorList)
else:
form = ReputationContextForm(request.POST or None,
instance=reputation_ctx,
error_class=DivErrorList)
def render_form(ctx, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
if not header_form_list and ctx:
for k, v in ctx.custom_headers.items():
header_form_list.append(
HttpHealthCheckHeaderForm(initial={
'check_header_name': k,
'check_header_value': v
}))
return render(
request, 'apps/reputation_ctx_edit.html', {
'form': form,
'headers': header_form_list,
'header_form': HttpHealthCheckHeaderForm(),
**kwargs
})
if request.method in ("POST", "PUT", "PATCH"):
headers_dict = {}
""" Handle JSON formatted request Http-health-check-headers """
try:
if api:
header_ids = request.JSON.get('custom_headers', [])
assert isinstance(header_ids,
list), "Custom headers field must be a list."
else:
header_ids = json_loads(
request.POST.get('custom_headers') or "[]")
except Exception as e:
return render_form(
reputation_ctx,
save_error=[
"Error in Custom_headers field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
""" For each Health check header in list """
for header in header_ids:
headerform = HttpHealthCheckHeaderForm(header,
error_class=DivErrorList)
if not headerform.is_valid():
if api:
form.add_error(None, headerform.errors.get_json_data())
else:
form.add_error('custom_headers', headerform.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(headerform)
headers_dict[header.get('check_header_name')] = header.get(
'check_header_value')
# If errors has been added in form
if not form.is_valid():
logger.error("Form errors: {}".format(form.errors.as_json()))
return render_form(reputation_ctx)
# Save the form to get an id if there is not already one
reputation_ctx = form.save(commit=False)
reputation_ctx.custom_headers = headers_dict
if not reputation_ctx.internal:
""" Generate the non-yet saved object conf """
try:
logger.info(
"Trying to retrieve MMDB database context '{}'".format(
reputation_ctx.name))
content = reputation_ctx.download_mmdb()
logger.info(
"Reputation context '{}' successfully downloaded".format(
reputation_ctx.name))
except VultureSystemError as e:
logger.exception(e)
return render_form(reputation_ctx,
save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(reputation_ctx,
save_error=[
"No referenced error",
str.join('',
format_exception(*exc_info()))
])
""" If the conf is OK, save the ReputationContext object """
# Is that object already in db or not
first_save = not reputation_ctx.id
try:
logger.debug("Saving reputation context")
reputation_ctx.save()
logger.debug(
"Reputation CTX '{}' (id={}) saved in MongoDB.".format(
reputation_ctx.name, reputation_ctx.id))
# If it is not an internal database, write the file on disk
if not reputation_ctx.internal:
""" API request asynchrone to save conf on node """
# Save conf, to raise if there is an error
reputation_ctx.save_conf()
logger.info(
"Write file of reputationCTX '{}' asked on cluster".format(
reputation_ctx.name))
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
reputation_ctx.delete()
logger.exception(e)
return render_form(reputation_ctx,
save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(
reputation_ctx,
save_error=[
"Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))
])
if api:
return build_response(reputation_ctx.id,
"applications.reputation_ctx.api",
COMMAND_LIST)
return HttpResponseRedirect('/apps/reputation_ctx/')
return render_form(reputation_ctx)
def user_scope_edit(request, object_id=None, api=False):
profile = None
repo_attrs_form_list = []
repo_attrs_objs = []
if object_id:
try:
profile = UserScope.objects.get(pk=object_id)
except ObjectDoesNotExist:
return HttpResponseNotFound(_("Object not found"))
""" Create form with object if exists, and request.POST (or JSON) if exists """
# Do NOT remove this line
empty = {} if api else None
if hasattr(request, "JSON") and api:
form = UserScopeForm(request.JSON or {},
instance=profile,
error_class=DivErrorList)
else:
form = UserScopeForm(request.POST or empty,
instance=profile,
error_class=DivErrorList)
def render_form(profile, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
if not repo_attrs_form_list and profile:
for p in profile.get_repo_attributes():
repo_attrs_form_list.append(RepoAttributeForm(instance=p))
return render(
request, 'authentication/user_scope_edit.html', {
'form': form,
'repo_attributes': repo_attrs_form_list,
'repo_attribute_form': RepoAttributeForm(),
**kwargs
})
if request.method in ("POST", "PUT"):
""" Handle repo attributes (user scope) """
try:
if api and hasattr(request, "JSON"):
repo_attrs = request.JSON.get('repo_attributes', [])
assert isinstance(
repo_attrs, list), "Repo attributes field must be a list."
else:
repo_attrs = json_loads(
request.POST.get('repo_attributes', "[]"))
except Exception as e:
if api:
return JsonResponse(
{"error": "".join(format_exception(*exc_info()))},
status=400)
return render_form(
profile,
save_error=[
"Error in Repo_Attributes field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
""" For each Health check header in list """
for repo_attr in repo_attrs:
repoattrform = RepoAttributeForm(repo_attr,
error_class=DivErrorList)
if not repoattrform.is_valid():
if api:
form.add_error(None, repoattrform.errors.get_json_data())
else:
form.add_error('repo_attributes',
repoattrform.errors.as_ul())
continue
# Save forms in case we re-print the page
repo_attrs_form_list.append(repoattrform)
repo_attrs_objs.append(repoattrform.save(commit=False))
if form.is_valid():
# Save the form to get an id if there is not already one
profile = form.save(commit=False)
for repo_attr in repo_attrs_objs:
repo_attr.save()
profile.repo_attributes = repo_attrs_objs
profile.save()
# If everything succeed, redirect to list view
if api:
return build_response(profile.id,
"api.authentication.user_scope", [])
return HttpResponseRedirect(
reverse("authentication.user_scope.list"))
return render_form(profile)
def backend_edit(request, object_id=None, api=False):
backend = None
server_form_list = []
header_form_list = []
httpchk_header_form_list = []
api_errors = []
if object_id:
try:
backend = Backend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = BackendForm(request.JSON or None,
instance=backend,
error_class=DivErrorList)
else:
form = BackendForm(request.POST or None,
instance=backend,
error_class=DivErrorList)
def render_form(back, **kwargs):
save_error = kwargs.get('save_error')
if api:
if len(api_errors) > 0 or form.errors:
api_errors.append(form.errors.as_json())
return JsonResponse({"errors": api_errors}, status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
available_sockets = get_darwin_sockets()
if not server_form_list and back:
for l_tmp in back.server_set.all():
server_form_list.append(ServerForm(instance=l_tmp))
if not header_form_list and back:
for h_tmp in back.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
if not httpchk_header_form_list and back:
for k, v in back.http_health_check_headers.items():
httpchk_header_form_list.append(
HttpHealthCheckHeaderForm({
'check_header_name': k,
'check_header_value': v
}))
return render(
request, 'apps/backend_edit.html', {
'form': form,
'servers': server_form_list,
'net_server_form': ServerForm(mode='net'),
'unix_server_form': ServerForm(mode='unix'),
'headers': header_form_list,
'header_form': HeaderForm(),
'sockets_choice': available_sockets,
'http_health_check_headers': httpchk_header_form_list,
'http_health_check_headers_form': HttpHealthCheckHeaderForm(),
**kwargs
})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
server_ids = request.JSON.get('servers', [])
assert isinstance(server_ids,
list), "Servers field must be a list."
else:
server_ids = json_loads(request.POST.get('servers', "[]"))
except Exception as e:
return render_form(backend,
save_error=[
"Error in Servers field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
header_objs = []
httpchk_headers_dict = {}
if form.data.get('mode') == "http":
""" Handle JSON formatted request headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids,
list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(
backend,
save_error=[
"Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))
])
""" Handle JSON formatted request Http-health-check-headers """
try:
if api:
httpchk_header_ids = request.JSON.get(
'http_health_check_headers', [])
assert isinstance(
httpchk_header_ids,
list), "Health check headers field must be a list."
else:
httpchk_header_ids = json_loads(
request.POST.get('http_health_check_headers', "[]"))
except Exception as e:
return render_form(
backend,
save_error=[
"Error in Http-health-check-headers field : {}".format(
e),
str.join('', format_exception(*exc_info()))
])
""" For each Health check header in list """
for header in httpchk_header_ids:
httpchkform = HttpHealthCheckHeaderForm(
header, error_class=DivErrorList)
if not httpchkform.is_valid():
if api:
api_errors.append({
"health_check":
httpchkform.errors.get_json_data()
})
else:
form.add_error('enable_http_health_check',
httpchkform.errors.as_ul())
continue
# Save forms in case we re-print the page
httpchk_header_form_list.append(httpchkform)
httpchk_headers_dict[header.get(
'check_header_name')] = header.get('check_header_value')
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = Header.objects.get(
pk=header['id']) if header.get('id') else None
except ObjectDoesNotExist:
form.add_error(
None,
"Request-header with id {} not found. Injection detected ?"
)
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
api_errors.append(
{"headers": header_f.errors.get_json_data()})
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
server_objs = []
""" For each listener in list """
for server in server_ids:
""" If id is given, retrieve object from mongo """
try:
instance_s = Server.objects.get(
pk=server['id']) if server['id'] else None
except ObjectDoesNotExist:
form.add_error(
None, "Server with id {} not found.".format(server['id']))
continue
""" And instantiate form with the object, or None """
server_f = ServerForm(server, instance=instance_s)
if not server_f.is_valid():
if api:
api_errors.append(
{'server': server_f.errors.get_json_data()})
else:
form.add_error(None, server_f.errors.as_ul())
continue
server_form_list.append(server_f)
server_obj = server_f.save(commit=False)
server_objs.append(server_obj)
# If errors has been added in form
if not form.is_valid():
logger.error("Form errors: {}".format(form.errors.as_json()))
return render_form(backend)
# Save the form to get an id if there is not already one
backend = form.save(commit=False)
backend.configuration = ""
backend.http_health_check_headers = httpchk_headers_dict
# At least one server is required if Frontend enabled
if not server_objs and backend.enabled:
form.add_error(
None, "At least one server is required if backend is enabled.")
return render_form(backend)
""" Generate the non-yet saved object conf """
try:
logger.debug("Generating conf of backend '{}'".format(
backend.name))
backend.configuration = backend.generate_conf(
header_list=header_objs, server_list=server_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of backend '{}'".format(
backend.name))
backend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(backend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(backend,
save_error=[
"No referenced error",
str.join('', format_exception(*exc_info()))
])
""" If the conf is OK, save the Backend object """
# Is that object already in db or not
first_save = not backend.id
try:
logger.debug("Saving backend")
backend.save()
logger.debug("Backend '{}' (id={}) saved in MongoDB.".format(
backend.name, backend.id))
""" And all the listeners created earlier """
for s in server_objs:
s.backend = backend
logger.debug("Saving server {}".format(str(s)))
s.save()
""" Delete listeners deleted in form """
for s in backend.server_set.exclude(
pk__in=[l.id for l in server_objs]):
s.delete()
logger.info("Deleting server {}".format(s))
""" If mode is HTTP """
if backend.mode == "http":
""" Remove request-headers removed """
for header in backend.headers.all():
if header not in header_objs:
backend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
backend.headers.add(header)
logger.debug(
"HTTP Headers {} associated to Frontend {}".format(
header, backend))
""" if the Backend is updated and its name was changed """
if not first_save and "name" in form.changed_data:
logger.info(
"Backend name changed, looking for associated frontends..."
)
workflow_list = backend.workflow_set.all()
for workflow in workflow_list:
logger.info(
"reloading frontend '{}' haproxy configuration".format(
workflow.frontend))
workflow.frontend.reload_conf()
# Re-generate config AFTER save, to get ID
backend.configuration = backend.generate_conf()
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
backend.save_conf()
logger.debug("Write conf of backend '{}' asked on cluster".format(
backend.name))
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = Cluster.api_request(
"services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on cluster\n API request error.",
"haproxy",
traceback=api_res.get('message'))
for node in Node.objects.all():
backend.status[node.name] = "WAITING"
backend.save()
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for server in backend.server_set.all():
server.delete()
backend.delete()
logger.exception(e)
return render_form(backend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(
backend,
save_error=[
"Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))
])
if api:
return build_response(backend.id, "applications.backend.api",
COMMAND_LIST)
return HttpResponseRedirect('/apps/backend/')
return render_form(backend)
def cluster_edit(request, object_id, api=False, update=False):
""" View used to edit a node """
# Node MUST exists - It should be created by an API Call from a slave node
# Unlike vulture3 we cannot add a new node from the GUI
# But we can add an existing node from the GUI, if it has previously been removed from replicaset
try:
node_model = Node.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
if hasattr(request, "JSON") and api:
if update:
request.JSON = {**node_model.to_small_dict(), **request.JSON}
form = NodeForm(request.JSON,
instance=node_model,
error_class=DivErrorList)
else:
form = NodeForm(request.POST or None,
instance=node_model,
error_class=DivErrorList)
def render_form(**kwargs):
# if not object_id:
# return render(request, 'system/cluster_add.html', {'form': form, **kwargs})
save_error = kwargs.get('save_error')
if api:
if form.errors:
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
return render(request, 'system/cluster_edit.html', {
'form': form,
**kwargs
})
if request.method in ("POST", "PUT", "PATCH") and form.is_valid():
ip_changed = "management_ip" in form.changed_data
gateway_changed = "gateway" in form.changed_data
gateway_ipv6_changed = "gateway_ipv6" in form.changed_data
static_route_changed = "static_routes" in form.changed_data
pstats_forwarders_changed = "pstats_forwarders" in form.changed_data
node = form.save(commit=False)
node.save()
if ip_changed or gateway_changed or gateway_ipv6_changed or static_route_changed:
node.api_request('toolkit.network.network.write_network_config')
node.api_request('toolkit.network.network.restart_routing')
if ip_changed:
node.api_request("services.apache.apache.reload_conf")
Cluster.api_request(
"toolkit.network.network.make_hostname_resolvable",
(node.name, node.management_ip))
res = node.write_management_ip()
if not res.get('status'):
return render_form(save_error=res.get('message'))
if pstats_forwarders_changed:
node.api_request("services.rsyslogd.rsyslog.configure_pstats")
if api:
return build_response(node.id, "system.node.api", COMMAND_LIST)
return HttpResponseRedirect('/system/cluster/')
return render_form()
def netif_edit(request, object_id=None, api=False):
netif_model = None
if object_id:
try:
netif_model = NetworkAddress.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
if netif_model and netif_model.is_system is True:
form = NetIfSystemForm(request.POST or None, instance=netif_model)
else:
if hasattr(request, 'JSON') and api:
if request.JSON.get('is_system') is True:
form = NetIfSystemForm(request.JSON or None,
instance=netif_model)
else:
form = NetIfForm(request.JSON or None, instance=netif_model)
else:
form = NetIfForm(request.POST or None, instance=netif_model)
if request.method in ("POST", "PUT") and form.is_valid():
netif = form.save(commit=False)
netif.save()
""" CARP settings are defined by default, even if they are not used """
priority = 50
pwd = get_random_string(20)
""" This is a new network address """
if not object_id:
if api:
nics = request.JSON.get('nic')
else:
nics = request.POST.getlist("nic")
for nic in nics:
NetworkAddressNIC.objects.create(nic_id=nic,
network_address=netif,
carp_passwd=pwd,
carp_priority=priority)
priority = priority + 50
else:
""" Add new NIC, if any """
if api:
nic_list = request.JSON.get('nic')
else:
nic_list = request.POST.getlist('nic')
for nic in nic_list:
try:
NetworkAddressNIC.objects.get(network_address=netif,
nic_id=nic)
except NetworkAddressNIC.DoesNotExist:
NetworkAddressNIC.objects.create(nic_id=nic,
network_address=netif,
carp_passwd=pwd,
carp_priority=priority)
priority = priority + 50
""" Delete old NIC, if any """
for current_networkadress_nic in NetworkAddressNIC.objects.filter(
network_address=netif):
""" If the current nic is not in the new config anymore:
Remove it from NetworkAddressNIC """
if str(current_networkadress_nic.nic.pk
) not in request.POST.getlist("nic"):
# NetworkAddressNIC().remove(nicid=current_networkadress_nic.nic.pk,
# network_addressid=netif.id)
current_networkadress_nic.delete()
""" Call ifconfig to setup network IP address right now """
Cluster.api_request('toolkit.network.network.ifconfig_call', netif.id)
""" Write permanent network configuration on disk """
Cluster.api_request('toolkit.network.network.write_network_config')
""" Garbage collector to delete obsolete running ifconfig and configuration """
Cluster.api_request('toolkit.network.network.address_cleanup')
if api:
return build_response(netif.id, "api.system.netaddr", [])
return HttpResponseRedirect('/system/netif/')
# If request POST or PUT & form not valid - return error
if api:
logger.error("NetworkAddress api form error : {}".format(
form.errors.get_json_data()))
return JsonResponse(form.errors.get_json_data(), status=400)
# if save_error:
# logger.error("Frontend api save error : {}".format(save_error))
# return JsonResponse({'error': save_error[0]}, status=500)
if netif_model:
return render(request, 'system/netif_edit.html', {
'form': form,
'is_system': netif_model.is_system
})
else:
return render(request, 'system/netif_edit.html', {
'form': form,
'is_system': False
})
def frontend_edit(request, object_id=None, api=False):
frontend = None
listener_form_list = []
header_form_list = []
reputationctx_form_list = []
node_listeners = dict()
if object_id:
try:
frontend = Frontend.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")}, status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = FrontendForm(request.JSON or None, instance=frontend, error_class=DivErrorList)
else:
form = FrontendForm(request.POST or None, instance=frontend, error_class=DivErrorList)
def render_form(front, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
logger.error("Frontend api form error : {}".format(form.errors.get_json_data()))
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
logger.error("Frontend api save error : {}".format(save_error))
return JsonResponse({'error': save_error[0]}, status=500)
if not listener_form_list and front:
for l_tmp in front.listener_set.all():
listener_form_list.append(ListenerForm(instance=l_tmp))
if not header_form_list and front:
for h_tmp in front.headers.all():
header_form_list.append(HeaderForm(instance=h_tmp))
# If it is a new object, add default-example headers
if not front and request.method == "GET":
for header in DEFAULT_FRONTEND_HEADERS:
header_form_list.append(HeaderForm(header))
if not reputationctx_form_list and front:
for r_tmp in front.frontendreputationcontext_set.all():
reputationctx_form_list.append(FrontendReputationContextForm(instance=r_tmp))
return render(request, 'services/frontend_edit.html',
{'form': form, 'listeners': listener_form_list, 'listener_form': ListenerForm(),
'headers': header_form_list, 'header_form': HeaderForm(),
'reputation_contexts': reputationctx_form_list,
'reputationctx_form': FrontendReputationContextForm(),
'log_om_table': LogOMTableForm(auto_id=False),
'redis_forwarder': LogOM.objects.filter(name="Internal_Dashboard").only('id').first().id,
'object_id': (frontend.id if frontend else "") or "", **kwargs})
if request.method in ("POST", "PUT"):
""" Handle JSON formatted listeners """
try:
if api:
listener_ids = request.JSON.get('listeners', [])
assert isinstance(listener_ids, list), "Listeners field must be a list."
else:
listener_ids = json_loads(request.POST.get('listeners', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Listeners field : {}".format(e),
str.join('', format_exception(*exc_info()))])
header_objs = []
reputationctx_objs = []
if form.data.get('mode') == "http":
""" Handle JSON formatted headers """
try:
if api:
header_ids = request.JSON.get('headers', [])
assert isinstance(header_ids, list), "Headers field must be a list."
else:
header_ids = json_loads(request.POST.get('headers', "[]"))
except Exception as e:
return render_form(frontend, save_error=["Error in Request-headers field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each header in list """
for header in header_ids:
""" If id is given, retrieve object from mongo """
try:
instance_h = frontend.headers.get(pk=header['id']) if frontend and header['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Request-header with id {} not found. Injection detected ?")
continue
""" And instantiate form with the object, or None """
header_f = HeaderForm(header, instance=instance_h)
if not header_f.is_valid():
if api:
form.add_error(None, header_f.errors.get_json_data())
else:
form.add_error('headers', header_f.errors.as_ul())
continue
# Save forms in case we re-print the page
header_form_list.append(header_f)
# And save objects list, to save them later, when Frontend will be saved
header_objs.append(header_f.save(commit=False))
""" Handle JSON formatted ReputationContext """
try:
if api:
reputationctx_ids = request.JSON.get('reputation_contexts', [])
else:
reputationctx_ids = json_loads(request.POST.get('reputation_contexts', "[]"))
assert isinstance(reputationctx_ids, list), "reputation_contexts field must be a list."
except Exception as e:
return render_form(frontend, save_error=["Error in ReputationConext field : {}".format(e),
str.join('', format_exception(*exc_info()))])
""" For each reputation_context in list """
for reputationctx in reputationctx_ids:
""" Instantiate form """
reputationctx_f = FrontendReputationContextForm(reputationctx)
if not reputationctx_f.is_valid():
form.add_error(None if api else "reputation_ctx",
reputationctx_f.errors.get_json_data() if api else reputationctx_f.errors.as_ul())
continue
# Save forms in case we re-print the page
reputationctx_form_list.append(reputationctx_f)
# And save objects list, to save them later, when Frontend will be saved
reputationctx_objs.append(reputationctx_f.save(commit=False))
listener_objs = []
if form.data.get('mode') != "impcap" and form.data.get('listening_mode') != "file":
""" For each listener in list """
for listener in listener_ids:
""" If id is given, retrieve object from mongo """
try:
instance_l = Listener.objects.get(pk=listener['id']) if listener['id'] else None
except ObjectDoesNotExist:
form.add_error(None, "Listener with id {} not found.".format(listener['id']))
continue
""" And instantiate form with the object, or None """
listener_f = ListenerForm(listener, instance=instance_l)
if not listener_f.is_valid():
if api:
form.add_error("listeners", listener_f.errors.as_json())
else:
form.add_error("listeners", listener_f.errors.as_ul())
continue
listener_form_list.append(listener_f)
listener_obj = listener_f.save(commit=False)
listener_objs.append(listener_obj)
""" For each listener, get node """
for nic in listener_obj.network_address.nic.all().only('node'):
if not node_listeners.get(nic.node):
node_listeners[nic.node] = list()
node_listeners[nic.node].append(listener_obj)
# Get current Rsyslog configuration filename
old_rsyslog_filename = frontend.get_rsyslog_base_filename() if frontend and frontend.enable_logging else ""
# If errors has been added in form
if not form.is_valid():
logger.error("Frontend form errors: {}".format(form.errors.as_json()))
return render_form(frontend)
# Save the form to get an id if there is not already one
frontend = form.save(commit=False)
frontend.configuration = {}
if frontend.mode == "impcap":
node_listeners[frontend.impcap_intf.node] = []
elif frontend.listening_mode == "file":
node_listeners[frontend.node] = []
# At least one Listener is required if Frontend enabled, except for listener of type "File" and "pcap"
if not listener_objs and frontend.enabled and frontend.mode != "impcap" and frontend.listening_mode != "file":
form.add_error(None, "At least one listener is required if frontend is enabled.")
return render_form(frontend)
try:
""" For each node, the conf differs if listener chosen """
""" Generate the conf """
logger.debug("Generating conf of frontend '{}'".format(frontend.name))
for node, listeners in node_listeners.items():
# HAProxy conf does not use reputationctx_list, Rsyslog conf does
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners,
header_list=header_objs)
""" Save the conf on disk, and test-it with haproxy -c """
logger.debug("Writing/Testing conf of frontend '{}'".format(frontend.name))
frontend.test_conf()
except ServiceError as e:
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["No referenced error",
str.join('', format_exception(*exc_info()))])
""" If the object already exists """
if frontend.id:
try:
changed_data = form.changed_data
""" If the ruleset has changed, we need to delete the old-named file """
if frontend.mode == "log" and "ruleset" in changed_data and old_rsyslog_filename:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
""" If it is an Rsyslog only conf """
if (frontend.mode == "log" and frontend.listening_mode in ("udp", "file")) \
or frontend.mode == "impcap":
""" And if it was not before saving """
if "mode" in changed_data or "listening_mode" in changed_data:
""" Delete old HAProxy conf """
frontend_filename = frontend.get_base_filename()
# API request deletion of frontend filename
Cluster.api_request('services.haproxy.haproxy.delete_conf', frontend_filename)
logger.info("HAProxy config '{}' deletion asked.".format(frontend_filename))
# And reload of HAProxy service
Cluster.api_request('services.haproxy.haproxy.reload_service')
""" If it is an HAProxy only conf """
elif frontend.mode not in ("log", "impcap") and not frontend.enable_logging:
""" And it was not """
if "mode" in changed_data or "enable_logging" in changed_data:
# API request deletion of rsyslog frontend filename
Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename)
logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename))
# And reload of Rsyslog service
Cluster.api_request('services.rsyslogd.rsyslog.restart_service')
except Exception as e:
logger.exception(e)
return render_form(frontend, save_error=["Cluster API request failure: {}".format(e),
str.join('', format_exception(*exc_info()))])
if form.errors:
logger.error("Frontend form errors: {}".format(form.errors))
return render_form(frontend)
""" If the conf is OK, save the Frontend object """
# Is that object already in db or not
first_save = not frontend.id
try:
if frontend.mode == "http":
frontend.ruleset = "haproxy"
elif frontend.mode == "tcp":
frontend.ruleset = "haproxy_tcp"
elif frontend.mode == "impcap":
frontend.ruleset = "impcap"
if frontend.enable_logging:
log_forwarders = set()
""" Handle LogForwarders selected objects in log_condition """
for log_om_name in re_findall("{{([^}]+)}}", frontend.log_condition):
try:
# Only id because we do not need any attribute
log_om = LogOM().select_log_om_by_name(log_om_name)
""" If the relation does not already exists : create-it """
log_forwarders.add(log_om.id)
except ObjectDoesNotExist:
form.add_error("log_condition", "LogForwarder not found.")
return render_form(frontend)
frontend.log_forwarders_id = log_forwarders
logger.debug("Saving frontend")
frontend.save()
logger.debug("Frontend '{}' (id={}) saved in MongoDB.".format(frontend.name, frontend.id))
""" And all the listeners early created """
for l in listener_objs:
l.frontend = frontend
logger.debug("Saving listener {}".format(str(l)))
l.save()
""" Delete listeners deleted in form """
for l in frontend.listener_set.exclude(pk__in=[l.id for l in listener_objs]):
l.delete()
logger.info("Deleting listener {}".format(l))
""" If mode is HTTP """
if frontend.mode == "http":
""" Remove request-headers removed """
for header in frontend.headers.all():
if header not in header_objs:
frontend.headers.remove(header)
""" Associate added request-headers """
for header in header_objs:
new_object = not header.id
header.save()
if new_object:
frontend.headers.add(header)
logger.debug("HTTP Headers {} associated to Frontend {}".format(header, frontend))
# Delete all intermediary objects
FrontendReputationContext.objects.filter(frontend=frontend).delete()
# And re-create them
for reputationctx in reputationctx_objs:
FrontendReputationContext.objects.create(frontend=frontend,
reputation_ctx=reputationctx.reputation_ctx,
enabled=reputationctx.enabled,
arg_field=reputationctx.arg_field)
# Re-generate config AFTER save, to get ID
for node, listeners in node_listeners.items():
frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners)
for node in node_listeners.keys():
""" asynchronous API request to save conf on node """
# Save conf first, to raise if there is an error
frontend.save_conf(node)
logger.debug("Write conf of frontend '{}' asked on node '{}'".format(frontend.name, node.name))
""" We need to configure Rsyslog, it will check if conf has changed & restart service if needed """
api_res = node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.id)
if not api_res.get('status'):
raise ServiceConfigError("on node {}\n API request error.".format(node.name), "rsyslog",
traceback=api_res.get('message'))
if not frontend.rsyslog_only_conf:
""" Reload HAProxy service - After rsyslog to prevent logging crash """
api_res = node.api_request("services.haproxy.haproxy.reload_service")
if not api_res.get('status'):
raise ServiceReloadError("on node {}\n API request error.".format(node.name), "haproxy",
traceback=api_res.get('message'))
frontend.status[node.name] = "WAITING"
frontend.save()
# Check if reload logrotate conf if needed
# meaning if there is a log_forwarder file enabled used by this frontend
if frontend.enable_logging and (frontend.log_forwarders.filter(logomfile__enabled=True).count() > 0 or
frontend.log_forwarders_parse_failure.filter(logomfile__enabled=True).count()):
# Reload LogRotate config
Cluster.api_request("services.logrotate.logrotate.reload_conf")
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
for listener in frontend.listener_set.all():
listener.delete()
frontend.delete()
logger.exception(e)
return render_form(frontend, save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(frontend, save_error=["Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))])
if api:
return build_response(frontend.id, "services.frontend.api", COMMAND_LIST)
return HttpResponseRedirect('/services/frontend')
return render_form(frontend)
def user_authentication_edit(request, object_id=None, api=False):
profile = None
if object_id:
try:
profile = UserAuthentication.objects.get(pk=object_id)
except ObjectDoesNotExist:
return HttpResponseNotFound(_("Object not found"))
""" Create form with object if exists, and request.POST (or JSON) if exists """
# Do NOT remove this line
empty = {} if api else None
if hasattr(request, "JSON") and api:
form = UserAuthenticationForm(request.JSON or {},
instance=profile,
error_class=DivErrorList)
else:
form = UserAuthenticationForm(request.POST or empty,
instance=profile,
error_class=DivErrorList)
def render_form(profile, **kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
return render(request, 'authentication/user_authentication_edit.html',
{
'form': form,
**kwargs
})
if request.method in ("POST", "PUT"):
# External portal is not compatible with Workflow
# If enable_external has been enabled but a workflow uses this Portal, add error in form
if form.data.get('enable_external'
) and profile and profile.workflow_set.count() > 0:
form.add_error(
'enable_external',
"This portal is used by a Workflow, you can't enable IDP. Please create another Portal or disable this one in Workflow."
)
old_external_frontend = deepcopy(
profile.external_listener
) if profile and profile.enable_external else None
if form.is_valid():
# Check changed attributes before form.save
repo_changed = "repositories" in form.changed_data
disconnect_url_changed = "disconnect_url" in form.changed_data
timeout_changed = "auth_timeout" in form.changed_data or "enable_timeout_restart" in form.changed_data
external_listener_changed = "external_listener" in form.changed_data and profile and profile.enable_external
# Save the form to get an id if there is not already one
profile = form.save(commit=False)
profile.save()
try:
if (repo_changed or disconnect_url_changed or
timeout_changed) and profile.workflow_set.count() > 0:
for workflow in profile.workflow_set.all():
nodes = workflow.frontend.reload_conf()
if timeout_changed:
for node in nodes:
node.api_request(
"services.haproxy.haproxy.configure_node")
if profile.enable_external:
# Automatically create OpenID repo
openid_repo, _ = OpenIDRepository.objects.get_or_create(
client_id=profile.oauth_client_id,
client_secret=profile.oauth_client_secret,
provider="openid",
defaults={
'provider_url': f"https://{profile.external_fqdn}",
'name': "Connector_{}".format(profile.name)
})
openid_repo.provider_url = f"https://{profile.external_fqdn}"
openid_repo.save()
# Reload old external_listener conf if has changed
if external_listener_changed:
old_external_frontend.reload_conf()
profile.external_listener.reload_conf()
Cluster.api_request(
"authentication.user_portal.api.write_templates",
profile.id)
profile.save_conf()
Cluster.api_request("services.haproxy.haproxy.reload_service")
except Exception as e:
if api:
return JsonResponse(
{"error": "".join(format_exception(*exc_info()))},
status=400)
return render_form(
profile,
save_error=[
"Cannot write configuration: {}".format(e),
str.join('', format_exception(*exc_info()))
])
# If everything succeed, redirect to list view
if api:
return build_response(profile.id,
"api.portal.user_authentication", [])
return HttpResponseRedirect(
reverse("portal.user_authentication.list"))
return render_form(profile)
def put(self, request, object_id=None):
try:
if object_id:
# Content should always be JSON here
filters = request.JSON.get('filters', [])
name = request.JSON.get('name', '')
description = request.JSON.get('description', '')
# filters object might be a string when coming from GUI
if isinstance(filters, str):
try:
filters = json.loads(filters)
except json.JsonDecodeError as e:
logger.error(e)
return JsonResponse({
'error': str(e)
}, status=400)
policy, created = DarwinPolicy.objects.get_or_create(pk=object_id)
policy.name = name
policy.description = description
try:
policy.full_clean()
except ValidationError as e:
logger.error(e)
return JsonResponse({
'error': str(e),
}, status=400)
error = DarwinPolicyAPIv1._create_or_update_filters(policy, filters)
if error:
if created:
try:
policy.delete()
except:
pass
return JsonResponse({
"error": error
}, status=400)
# Save once no errors were triggered during filters creation
policy.save()
else:
return JsonResponse({
"error": _("You must provide an id")
}, status=400)
except Exception as e:
logger.critical(e, exc_info=1)
error = _("An error has occurred")
if settings.DEV_MODE:
error = str(e)
return JsonResponse({
'error': error
}, status=500)
for frontend in policy.frontend_set.all():
for node in frontend.get_nodes():
node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.pk)
if DarwinBuffering.objects.filter(destination_filter__policy=policy).exists():
DarwinPolicy.update_buffering()
Cluster.api_request("services.darwin.darwin.write_policy_conf", policy.pk)
Cluster.api_request("services.darwin.darwin.reload_conf")
return build_response(policy.pk, "darwin.policy.api", COMMAND_LIST)
def post(self, request, object_id=None, action=None):
policy = None
try:
if action:
#Trigger action on existing policy
if not object_id:
return JsonResponse({
'error': _('You must specify an ID')
}, status=401)
if action not in list(COMMAND_LIST.keys()):
return JsonResponse({
'error': _('Action not allowed')
}, status=403)
return COMMAND_LIST[action](request, object_id, api=True)
else:
#Create a new policy with filters
# Content could be in POST when coming from GUI
if hasattr(request, "JSON"):
filters_list = request.JSON.get('filters', [])
name = request.JSON.get('name', '')
description = request.JSON.get('description', '')
is_internal = request.JSON.get('is_internal', False)
else:
filters_list = json.loads(request.POST.get('filters', '[]'))
name = request.POST.get('name', '')
description = request.POST.get('description', '')
is_internal = request.POST.get('is_internal', False)
policy = DarwinPolicy(
name=name,
description=description,
is_internal=is_internal
)
try:
policy.full_clean()
policy.save()
except ValidationError as e:
logger.error(e)
return JsonResponse({
'error': str(e),
}, status=400)
error = DarwinPolicyAPIv1._create_or_update_filters(policy, filters_list)
if error:
try:
policy.delete()
except:
pass
return JsonResponse({
"error": error
}, status=400)
except Exception as e:
try:
policy.delete()
except:
pass
logger.critical(e, exc_info=1)
if settings.DEV_MODE:
error = str(e)
else:
error = _("An error has occurred")
return JsonResponse({
'error': error
}, status=500)
if DarwinBuffering.objects.filter(destination_filter__policy=policy).exists():
DarwinPolicy.update_buffering()
for frontend in policy.frontend_set.all():
for node in frontend.get_nodes():
node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.pk)
Cluster.api_request("services.darwin.darwin.write_policy_conf", policy.pk)
Cluster.api_request("services.darwin.darwin.reload_conf")
return build_response(policy.pk, "darwin.policy.api", COMMAND_LIST)
def parser_edit(request, object_id=None, api=False):
parser = None
if object_id:
try:
parser = Parser.objects.get(pk=object_id)
except ObjectDoesNotExist:
if api:
return JsonResponse({'error': _("Object does not exist.")},
status=404)
return HttpResponseForbidden("Injection detected")
""" Create form with object if exists, and request.POST (or JSON) if exists """
if hasattr(request, "JSON") and api:
form = ParserForm(request.JSON or None,
instance=parser,
error_class=DivErrorList)
else:
form = ParserForm(request.POST or None,
instance=parser,
error_class=DivErrorList)
def render_form(**kwargs):
save_error = kwargs.get('save_error')
if api:
if form.errors:
return JsonResponse(form.errors.get_json_data(), status=400)
if save_error:
return JsonResponse({'error': save_error[0]}, status=500)
return render(request, 'apps/parser_edit.html', {
'form': form,
**kwargs
})
if request.method in ("POST", "PUT", "PATCH"):
# If errors has been added in form
if not form.is_valid():
logger.error("Form errors: {}".format(form.errors.as_json()))
if api:
return JsonResponse(form.errors.get_json_data(), status=400)
return render_form()
# Save the form to get an id if there is not already one
parser = form.save(commit=False)
""" Generate the non-yet saved object conf """
try:
logger.debug("Trying '{}'".format(parser.name))
content = parser.test_conf()
logger.info("Parser '{}' conf ok".format(parser.name))
except VultureSystemError as e:
logger.exception(e)
return render_form(save_error=[str(e), e.traceback])
except Exception as e:
logger.exception(e)
return render_form(save_error=[
"No referenced error",
str.join('', format_exception(*exc_info()))
])
""" If the conf is saved, save the Parser object """
logger.debug("Saving parser")
parser.save()
logger.debug("Parser '{}' (id={}) saved in MongoDB.".format(
parser.name, parser.id))
first_save = not object_id
try:
parser.save_conf()
except (VultureSystemError, ServiceError) as e:
""" Error saving configuration file """
""" The object has been saved, delete-it if needed """
if first_save:
parser.delete()
logger.exception(e)
return render_form(save_error=[str(e), e.traceback])
except Exception as e:
""" If we arrive here, the object has not been saved """
logger.exception(e)
return render_form(save_error=[
"Failed to save object in database :\n{}".format(e),
str.join('', format_exception(*exc_info()))
])
if api:
return build_response(parser.id, "applications.parser.api",
COMMAND_LIST)
return HttpResponseRedirect('/apps/parser/')
return render_form()