def reload_frontend_conf(self): """ Send API request on each Nodes using the frontends that uses the current CTX :return The list of concerned nodes """ from services.frontend.models import Listener res = [] # Loop on Nodes for node in Node.objects.all(): frontends = [] # Get listeners enabled on this node, using the current reputation context for listener in Listener.objects.filter( frontend__enabled=True, frontend__reputation_ctxs=self.id, network_address__nic__node=node.id).distinct(): if listener.frontend.id not in frontends: api_res = node.api_request( "services.rsyslogd.rsyslog.build_conf", listener.frontend.id) if not api_res: raise ServiceConfigError( "on node '{}' \n API request error.".format( node.name), "rsyslog", traceback=api_res.get('message')) frontends.append(listener.frontend.id) res.append(node) return res
def conf_has_changed(self, new_conf): """ Check if configuration has changed :param new_conf: Conf as text to compare with disk content :return: conf generated if conf has changed, empty string otherwise raise ServiceConfigError if error """ try: current_conf = self.read_current_conf() if current_conf != new_conf: return new_conf return False except ServiceExit: raise except Exception as e: logger.error("Unable to check if conf has changed for {}: ".format( self.service_name, str(e))) logger.exception(e) if "No such file" in str(e): raise ServiceNoConfigError(str(e), self.service_name) raise ServiceConfigError( "Cannot open '{}' : {}".format(self.jinja_template['tpl_path'], str(e)), self.service_name)
def get_conf(self): """ Generate conf from mongo object and Jinja template :return Generated configuration, to write into file If error, raise ServiceConfigError """ try: path_config = os_path.join(settings.BASE_DIR, 'services', 'config') jinja2_env = Environment(loader=FileSystemLoader(path_config)) template = jinja2_env.get_template(self.jinja_template['tpl_name']) return template.render({ 'node': Cluster.get_current_node(), **self.get_dict_conf() }) except ServiceExit: raise except Exception as e: raise logger.error(e) raise ServiceConfigError( "Failed to generate jinja template: {}".format(str(e)), self.service_name)
def frontend_edit(request, object_id=None, api=False): frontend = None listener_form_list = [] header_form_list = [] reputationctx_form_list = [] node_listeners = dict() if object_id: try: frontend = Frontend.objects.get(pk=object_id) except ObjectDoesNotExist: if api: return JsonResponse({'error': _("Object does not exist.")}, status=404) return HttpResponseForbidden("Injection detected") """ Create form with object if exists, and request.POST (or JSON) if exists """ if hasattr(request, "JSON") and api: form = FrontendForm(request.JSON or None, instance=frontend, error_class=DivErrorList) else: form = FrontendForm(request.POST or None, instance=frontend, error_class=DivErrorList) def render_form(front, **kwargs): save_error = kwargs.get('save_error') if api: if form.errors: logger.error("Frontend api form error : {}".format(form.errors.get_json_data())) return JsonResponse(form.errors.get_json_data(), status=400) if save_error: logger.error("Frontend api save error : {}".format(save_error)) return JsonResponse({'error': save_error[0]}, status=500) if not listener_form_list and front: for l_tmp in front.listener_set.all(): listener_form_list.append(ListenerForm(instance=l_tmp)) if not header_form_list and front: for h_tmp in front.headers.all(): header_form_list.append(HeaderForm(instance=h_tmp)) # If it is a new object, add default-example headers if not front and request.method == "GET": for header in DEFAULT_FRONTEND_HEADERS: header_form_list.append(HeaderForm(header)) if not reputationctx_form_list and front: for r_tmp in front.frontendreputationcontext_set.all(): reputationctx_form_list.append(FrontendReputationContextForm(instance=r_tmp)) return render(request, 'services/frontend_edit.html', {'form': form, 'listeners': listener_form_list, 'listener_form': ListenerForm(), 'headers': header_form_list, 'header_form': HeaderForm(), 'reputation_contexts': reputationctx_form_list, 'reputationctx_form': FrontendReputationContextForm(), 'log_om_table': LogOMTableForm(auto_id=False), 'redis_forwarder': LogOM.objects.filter(name="Internal_Dashboard").only('id').first().id, 'object_id': (frontend.id if frontend else "") or "", **kwargs}) if request.method in ("POST", "PUT"): """ Handle JSON formatted listeners """ try: if api: listener_ids = request.JSON.get('listeners', []) assert isinstance(listener_ids, list), "Listeners field must be a list." else: listener_ids = json_loads(request.POST.get('listeners', "[]")) except Exception as e: return render_form(frontend, save_error=["Error in Listeners field : {}".format(e), str.join('', format_exception(*exc_info()))]) header_objs = [] reputationctx_objs = [] if form.data.get('mode') == "http": """ Handle JSON formatted headers """ try: if api: header_ids = request.JSON.get('headers', []) assert isinstance(header_ids, list), "Headers field must be a list." else: header_ids = json_loads(request.POST.get('headers', "[]")) except Exception as e: return render_form(frontend, save_error=["Error in Request-headers field : {}".format(e), str.join('', format_exception(*exc_info()))]) """ For each header in list """ for header in header_ids: """ If id is given, retrieve object from mongo """ try: instance_h = frontend.headers.get(pk=header['id']) if frontend and header['id'] else None except ObjectDoesNotExist: form.add_error(None, "Request-header with id {} not found. Injection detected ?") continue """ And instantiate form with the object, or None """ header_f = HeaderForm(header, instance=instance_h) if not header_f.is_valid(): if api: form.add_error(None, header_f.errors.get_json_data()) else: form.add_error('headers', header_f.errors.as_ul()) continue # Save forms in case we re-print the page header_form_list.append(header_f) # And save objects list, to save them later, when Frontend will be saved header_objs.append(header_f.save(commit=False)) """ Handle JSON formatted ReputationContext """ try: if api: reputationctx_ids = request.JSON.get('reputation_contexts', []) else: reputationctx_ids = json_loads(request.POST.get('reputation_contexts', "[]")) assert isinstance(reputationctx_ids, list), "reputation_contexts field must be a list." except Exception as e: return render_form(frontend, save_error=["Error in ReputationConext field : {}".format(e), str.join('', format_exception(*exc_info()))]) """ For each reputation_context in list """ for reputationctx in reputationctx_ids: """ Instantiate form """ reputationctx_f = FrontendReputationContextForm(reputationctx) if not reputationctx_f.is_valid(): form.add_error(None if api else "reputation_ctx", reputationctx_f.errors.get_json_data() if api else reputationctx_f.errors.as_ul()) continue # Save forms in case we re-print the page reputationctx_form_list.append(reputationctx_f) # And save objects list, to save them later, when Frontend will be saved reputationctx_objs.append(reputationctx_f.save(commit=False)) listener_objs = [] if form.data.get('mode') != "impcap" and form.data.get('listening_mode') != "file": """ For each listener in list """ for listener in listener_ids: """ If id is given, retrieve object from mongo """ try: instance_l = Listener.objects.get(pk=listener['id']) if listener['id'] else None except ObjectDoesNotExist: form.add_error(None, "Listener with id {} not found.".format(listener['id'])) continue """ And instantiate form with the object, or None """ listener_f = ListenerForm(listener, instance=instance_l) if not listener_f.is_valid(): if api: form.add_error("listeners", listener_f.errors.as_json()) else: form.add_error("listeners", listener_f.errors.as_ul()) continue listener_form_list.append(listener_f) listener_obj = listener_f.save(commit=False) listener_objs.append(listener_obj) """ For each listener, get node """ for nic in listener_obj.network_address.nic.all().only('node'): if not node_listeners.get(nic.node): node_listeners[nic.node] = list() node_listeners[nic.node].append(listener_obj) # Get current Rsyslog configuration filename old_rsyslog_filename = frontend.get_rsyslog_base_filename() if frontend and frontend.enable_logging else "" # If errors has been added in form if not form.is_valid(): logger.error("Frontend form errors: {}".format(form.errors.as_json())) return render_form(frontend) # Save the form to get an id if there is not already one frontend = form.save(commit=False) frontend.configuration = {} if frontend.mode == "impcap": node_listeners[frontend.impcap_intf.node] = [] elif frontend.listening_mode == "file": node_listeners[frontend.node] = [] # At least one Listener is required if Frontend enabled, except for listener of type "File" and "pcap" if not listener_objs and frontend.enabled and frontend.mode != "impcap" and frontend.listening_mode != "file": form.add_error(None, "At least one listener is required if frontend is enabled.") return render_form(frontend) try: """ For each node, the conf differs if listener chosen """ """ Generate the conf """ logger.debug("Generating conf of frontend '{}'".format(frontend.name)) for node, listeners in node_listeners.items(): # HAProxy conf does not use reputationctx_list, Rsyslog conf does frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners, header_list=header_objs) """ Save the conf on disk, and test-it with haproxy -c """ logger.debug("Writing/Testing conf of frontend '{}'".format(frontend.name)) frontend.test_conf() except ServiceError as e: logger.exception(e) return render_form(frontend, save_error=[str(e), e.traceback]) except Exception as e: logger.exception(e) return render_form(frontend, save_error=["No referenced error", str.join('', format_exception(*exc_info()))]) """ If the object already exists """ if frontend.id: try: changed_data = form.changed_data """ If the ruleset has changed, we need to delete the old-named file """ if frontend.mode == "log" and "ruleset" in changed_data and old_rsyslog_filename: # API request deletion of rsyslog frontend filename Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename) logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename)) """ If it is an Rsyslog only conf """ if (frontend.mode == "log" and frontend.listening_mode in ("udp", "file")) \ or frontend.mode == "impcap": """ And if it was not before saving """ if "mode" in changed_data or "listening_mode" in changed_data: """ Delete old HAProxy conf """ frontend_filename = frontend.get_base_filename() # API request deletion of frontend filename Cluster.api_request('services.haproxy.haproxy.delete_conf', frontend_filename) logger.info("HAProxy config '{}' deletion asked.".format(frontend_filename)) # And reload of HAProxy service Cluster.api_request('services.haproxy.haproxy.reload_service') """ If it is an HAProxy only conf """ elif frontend.mode not in ("log", "impcap") and not frontend.enable_logging: """ And it was not """ if "mode" in changed_data or "enable_logging" in changed_data: # API request deletion of rsyslog frontend filename Cluster.api_request('services.rsyslogd.rsyslog.delete_conf', old_rsyslog_filename) logger.info("Rsyslogd config '{}' deletion asked.".format(old_rsyslog_filename)) # And reload of Rsyslog service Cluster.api_request('services.rsyslogd.rsyslog.restart_service') except Exception as e: logger.exception(e) return render_form(frontend, save_error=["Cluster API request failure: {}".format(e), str.join('', format_exception(*exc_info()))]) if form.errors: logger.error("Frontend form errors: {}".format(form.errors)) return render_form(frontend) """ If the conf is OK, save the Frontend object """ # Is that object already in db or not first_save = not frontend.id try: if frontend.mode == "http": frontend.ruleset = "haproxy" elif frontend.mode == "tcp": frontend.ruleset = "haproxy_tcp" elif frontend.mode == "impcap": frontend.ruleset = "impcap" if frontend.enable_logging: log_forwarders = set() """ Handle LogForwarders selected objects in log_condition """ for log_om_name in re_findall("{{([^}]+)}}", frontend.log_condition): try: # Only id because we do not need any attribute log_om = LogOM().select_log_om_by_name(log_om_name) """ If the relation does not already exists : create-it """ log_forwarders.add(log_om.id) except ObjectDoesNotExist: form.add_error("log_condition", "LogForwarder not found.") return render_form(frontend) frontend.log_forwarders_id = log_forwarders logger.debug("Saving frontend") frontend.save() logger.debug("Frontend '{}' (id={}) saved in MongoDB.".format(frontend.name, frontend.id)) """ And all the listeners early created """ for l in listener_objs: l.frontend = frontend logger.debug("Saving listener {}".format(str(l))) l.save() """ Delete listeners deleted in form """ for l in frontend.listener_set.exclude(pk__in=[l.id for l in listener_objs]): l.delete() logger.info("Deleting listener {}".format(l)) """ If mode is HTTP """ if frontend.mode == "http": """ Remove request-headers removed """ for header in frontend.headers.all(): if header not in header_objs: frontend.headers.remove(header) """ Associate added request-headers """ for header in header_objs: new_object = not header.id header.save() if new_object: frontend.headers.add(header) logger.debug("HTTP Headers {} associated to Frontend {}".format(header, frontend)) # Delete all intermediary objects FrontendReputationContext.objects.filter(frontend=frontend).delete() # And re-create them for reputationctx in reputationctx_objs: FrontendReputationContext.objects.create(frontend=frontend, reputation_ctx=reputationctx.reputation_ctx, enabled=reputationctx.enabled, arg_field=reputationctx.arg_field) # Re-generate config AFTER save, to get ID for node, listeners in node_listeners.items(): frontend.configuration[node.name] = frontend.generate_conf(listener_list=listeners) for node in node_listeners.keys(): """ asynchronous API request to save conf on node """ # Save conf first, to raise if there is an error frontend.save_conf(node) logger.debug("Write conf of frontend '{}' asked on node '{}'".format(frontend.name, node.name)) """ We need to configure Rsyslog, it will check if conf has changed & restart service if needed """ api_res = node.api_request("services.rsyslogd.rsyslog.build_conf", frontend.id) if not api_res.get('status'): raise ServiceConfigError("on node {}\n API request error.".format(node.name), "rsyslog", traceback=api_res.get('message')) if not frontend.rsyslog_only_conf: """ Reload HAProxy service - After rsyslog to prevent logging crash """ api_res = node.api_request("services.haproxy.haproxy.reload_service") if not api_res.get('status'): raise ServiceReloadError("on node {}\n API request error.".format(node.name), "haproxy", traceback=api_res.get('message')) frontend.status[node.name] = "WAITING" frontend.save() # Check if reload logrotate conf if needed # meaning if there is a log_forwarder file enabled used by this frontend if frontend.enable_logging and (frontend.log_forwarders.filter(logomfile__enabled=True).count() > 0 or frontend.log_forwarders_parse_failure.filter(logomfile__enabled=True).count()): # Reload LogRotate config Cluster.api_request("services.logrotate.logrotate.reload_conf") except (VultureSystemError, ServiceError) as e: """ Error saving configuration file """ """ The object has been saved, delete-it if needed """ if first_save: for listener in frontend.listener_set.all(): listener.delete() frontend.delete() logger.exception(e) return render_form(frontend, save_error=[str(e), e.traceback]) except Exception as e: """ If we arrive here, the object has not been saved """ logger.exception(e) return render_form(frontend, save_error=["Failed to save object in database :\n{}".format(e), str.join('', format_exception(*exc_info()))]) if api: return build_response(frontend.id, "services.frontend.api", COMMAND_LIST) return HttpResponseRedirect('/services/frontend') return render_form(frontend)