class GenericModel(models.Model): big_int = models.BigIntegerField() bool = models.BooleanField() char = models.CharField(max_length=20) comma_int = models.CommaSeparatedIntegerField() date = models.DateField() date_time = models.DateTimeField() decimal = models.DecimalField(max_digits=10, decimal_places=5) email = models.EmailField() float = models.FloatField() integer = models.IntegerField() null_bool = models.NullBooleanField() pos_int = models.PositiveIntegerField() pos_small_int = models.PositiveSmallIntegerField() slug = models.SlugField() small_int = models.SmallIntegerField() text = models.TextField() time = models.TimeField() url = models.URLField() ip = models.GenericIPAddressField() uuid = models.UUIDField() # TODO: add these # basic_file = models.FileField() # image = models.ImageField() objects = models.DjongoManager()
class Log(models.Model): path_info = models.CharField(max_length=200) event_name = models.CharField(max_length=200) page_title = models.CharField(max_length=200) visited_by = models.CharField(max_length=100) ip_address = models.GenericIPAddressField() datetime = models.DateTimeField() referrer = models.CharField(max_length=500) browser_family = models.CharField(max_length=100) browser_version = models.CharField(max_length=20) os_family = models.CharField(max_length=100) os_version = models.CharField(max_length=20) device_family = models.CharField(max_length=100) device_type = models.CharField(max_length=50) latitude = models.FloatField() longitude = models.FloatField() country = models.CharField(max_length=100) region = models.CharField(max_length=5) city = models.CharField(max_length=100) def __str__(self): return "Website Log Object" class Meta: db_table = 'website_logs' objects = models.DjongoManager()
class VisitorSpot(models.Model): datetime = models.DateTimeField() geom = PointField() ip_address = models.GenericIPAddressField() @property def popupContent(self): return '<span/>{}</span>'.format(self.ip_address)
class Contact(models.Model): first_name = models.CharField(max_length=50) last_name = models.CharField(max_length=50) email = models.EmailField() ip_address = models.GenericIPAddressField(null=True) message = models.TextField() def __str__(self): return f"{self.first_name}.{self.last_name}"
class SourceRemote(djongo_models.Model): systemIP = djongo_models.GenericIPAddressField() username = djongo_models.CharField(max_length=100) password = djongo_models.CharField(max_length=500) file_path = djongo_models.CharField(max_length=200) objects = djongo_models.DjongoManager() def __str__(self): return "remote_source" + '_' + str(self.id) + '_' + self.username
class VisitorPath(models.Model): datetime = models.DateTimeField() city = models.CharField(max_length=100) region = models.CharField(max_length=100) country = models.CharField(max_length=100) ip_address = models.GenericIPAddressField() visit_num = models.IntegerField() browser = models.CharField(max_length=100) os = models.CharField(max_length=100) device = models.CharField(max_length=100) path = models.ArrayField(model_container=Path, ) def __str__(self): return "Visitor Path Model" objects = models.DjongoManager()
class PageViewActivity(models.Model): datetime = models.DateTimeField() browser = models.CharField(max_length=100) os = models.CharField(max_length=100) city = models.CharField(max_length=100) device = models.CharField(max_length=100) region = models.CharField(max_length=100) country = models.CharField(max_length=100) ip_address = models.GenericIPAddressField() page_name = models.CharField(max_length=300) page_url = models.CharField(max_length=300) referrer = models.CharField(max_length=300) def __str__(self): return "Page View Activity Model" objects = models.DjongoManager()
class VisitorActivity(models.Model): datetime = models.DateTimeField() page_views = models.IntegerField() total_visits = models.IntegerField() latest_page_view = models.DateTimeField() city = models.CharField(max_length=100) region = models.CharField(max_length=100) country = models.CharField(max_length=100) visit_length_sec = models.IntegerField() ip_address = models.GenericIPAddressField() browser = models.CharField(max_length=100) os = models.CharField(max_length=100) device = models.CharField(max_length=100) referrer = models.CharField(max_length=300) entry_page = models.CharField(max_length=300) latest_page = models.CharField(max_length=300) def __str__(self): return "Visitor Activity Model" objects = models.DjongoManager()
class Server(models.Model): """ Link class between NetworkAddress and Frontend """ """ NetworkAddress object to listen on """ target = models.TextField(default="1.2.3.4", help_text=_("IP address/hostname of server")) """ Port To listen on """ port = models.PositiveIntegerField( default=80, validators=[MinValueValidator(1), MaxValueValidator(65535)]) """ Backend associated to this server """ backend = models.ForeignKey(to=Backend, on_delete=models.CASCADE) """ TLSProfile to use when listening """ tls_profile = models.ForeignKey(TLSProfile, null=True, default="", on_delete=models.SET_DEFAULT) """ Weight of the server """ weight = models.PositiveIntegerField( validators=[MinValueValidator(0), MaxValueValidator(256)], default=1, help_text=_("")) """ Source """ source = models.GenericIPAddressField( default="", help_text=_("IP address to assign to sources")) def to_template(self): """ Create a JSON representation of the object :return Dictionnary of configuration parameters """ """ Retrieve list/custom objects """ result = { 'id': str(self.id), 'target': self.target, 'port': self.port, 'backend': self.backend, 'weight': self.weight, } if self.source: result['source'] = self.source return result def __str__(self): return "{}:{}".format(self.target, self.port) @staticmethod def str_attrs(): """ List of attributes required by __str__ method """ return ['target', 'port'] def generate_conf(self): """ Generate Listener HAProxy configuration :return A string - Bind directive """ result = "server srv{} {}:{} weight {}".format(self.id, self.target, self.port, self.weight) if self.source: if ":" in self.source: result += " source [{}]".format(self.source) else: result += " source {}".format(self.source) if self.tls_profile: result += self.tls_profile.generate_conf(backend=True) return result
class Backend(models.Model): """ Model used to generated fontends configuration of HAProxy """ """ Is that section enabled or disabled """ enabled = models.BooleanField( default=True, help_text=_("Enable the backend"), ) """ Name of the frontend, unique constraint """ name = models.TextField( unique=True, default="Backend", help_text=_("Name of HAProxy backend"), ) """ Listening mode of Frontend : tcp or http """ mode = models.TextField( default=MODE_CHOICES[0][0], choices=MODE_CHOICES, help_text=_("Proxy mode"), ) timeout_connect = models.PositiveIntegerField( default=2000, validators=[MinValueValidator(1), MaxValueValidator(20000)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) timeout_server = models.PositiveIntegerField( default=60, validators=[MinValueValidator(1), MaxValueValidator(3600)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) """ Save of generated configuration """ configuration = models.TextField(default="{}") """ Status of frontend for each nodes """ status = models.DictField(default={}) """ Headers """ headers = models.ArrayReferenceField(Header, null=True, blank=False, on_delete=models.CASCADE, help_text=_("Header rules")) """ Custom HAProxy Backend directives """ custom_haproxy_conf = models.TextField( default="", help_text=_("Custom HAProxy configuration directives.")) """ HTTP Options """ """ URI of backends """ http_backend_dir = models.TextField( default="/", help_text=_("Servers directory to prefix in front of requests uri"), verbose_name=_("Servers directory")) """ Enable or disable relaxing of HTTP response parsing """ accept_invalid_http_response = models.BooleanField( default=False, help_text=_("Enable relaxing of HTTP response parsing"), verbose_name=_("Accept invalid HTTP response")) """ Enable insertion of the X-Forwarded-For header to requests sent to servers """ http_forwardfor_header = models.TextField( blank=True, null=True, help_text=_("Insertion of the X-Forwarded-For header"), verbose_name=_("Send source ip in ")) """ Except the following IP""" http_forwardfor_except = models.GenericIPAddressField( protocol='IPv4', blank=True, null=True, help_text=_("Except the specified IP address"), verbose_name=_("Except for ")) """ Enable HTTP protocol to check on the servers health """ enable_http_health_check = models.BooleanField( default=False, help_text=_("Enable HTTP protocol health checker"), verbose_name=_("HTTP health check")) """ The optional HTTP method used with the requests """ http_health_check_method = models.TextField( default=HEALTH_CHECK_METHOD_CHOICES[0][0], choices=HEALTH_CHECK_METHOD_CHOICES, help_text=_("HTTP method used"), verbose_name=_("Method")) """ The URI referenced in the HTTP requests """ http_health_check_uri = models.TextField(default='/', blank=True, null=True, help_text=_("URI referenced"), verbose_name=_("URI")) """ The optional HTTP version string """ http_health_check_version = models.TextField( default=HEALTH_CHECK_VERSION_CHOICES[0][0], choices=HEALTH_CHECK_VERSION_CHOICES, help_text=_("HTTP version"), verbose_name=_("Version")) """ """ http_health_check_headers = models.DictField( default={}, help_text=_("HTTP Health Check Headers"), verbose_name=_("HTTP Health Check Headers")) """ Health check expect """ http_health_check_expect_match = models.TextField( choices=HEALTH_CHECK_EXPECT_CHOICES, default=HEALTH_CHECK_EXPECT_CHOICES[0][0], null=True, help_text=_("Type of match to expect"), verbose_name=_("HTTP Health Check expected")) http_health_check_expect_pattern = models.TextField( default="200", help_text=_("Type of pattern to match to expect"), verbose_name=_("HTTP Health Check expected pattern")) """ Enable or disable HTTP keep-alive from client to server """ enable_http_keep_alive = models.BooleanField( default=True, help_text=_("Enable HTTP keep-alive"), verbose_name=_("HTTP Keep alive")) """ Keep-alive Timeout """ http_keep_alive_timeout = models.PositiveIntegerField( default=60, validators=[MinValueValidator(1), MaxValueValidator(20000)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) """ Balancing mode """ balancing_mode = models.TextField( choices=BALANCING_CHOICES, default=BALANCING_CHOICES[0][0], help_text=_("Balancing mode between servers"), verbose_name=_("Balancing mode")) """ Balancing param """ balancing_param = models.TextField( null=True, default="", help_text=_("Balancing param for balancing mode"), verbose_name=_("Balancing parameter")) """ Tags """ tags = models.ListField( models.SlugField(default=""), default=[], help_text=_("Tags to set on this object for search")) @property def balancing(self): if self.balancing_mode == "hdr": result = "hdr({})".format(self.balancing_param) elif self.balancing_mode == "url_param": result = "url_param {}".format(self.balancing_param) elif self.balancing_mode == "rdp-cookie": result = "rdp-cookie({})".format(self.balancing_param) else: result = self.balancing_mode return result @staticmethod def str_attrs(): """ List of attributes required by __str__ method """ return ['name', 'mode'] def __str__(self): return "{} Backend '{}'".format(self.mode.upper(), self.name) def to_dict(self): """ This method MUST be used in API instead of to_template() method to prevent no-serialization of sub-models like Listeners :return A JSON object """ result = { 'id': self.id, 'enable': self.enabled, 'name': self.name, 'mode': self.mode, 'balancing_mode': self.balancing_mode, 'balancing_param': self.balancing_param, 'status': dict(self.status), # It is an OrderedDict 'servers': [], 'custom_haproxy_conf': self.custom_haproxy_conf, 'tags': self.tags } """ Add listeners """ for server in self.server_set.all(): s = server.to_template() # Remove frontend to prevent infinite loop del s['backend'] result['servers'].append(s) """ Other attributes """ if self.mode == "http": result['headers'] = [] for header in self.headers.all(): result['headers'].append(header.to_template()) return result def to_html_template(self): """ Dictionary used to render object as html :return Dictionnary of configuration parameters """ """ Retrieve list/custom objects """ servers_list = [ str(s) for s in self.server_set.all().only(*Server.str_attrs()) ] mode = "UNKNOWN" for m in MODE_CHOICES: if self.mode == m[0]: mode = m[1] balancing_mode = "unknown" for m in BALANCING_CHOICES: if self.balancing_mode == m[0]: balancing_mode = m[1] additional_infos = "Balancing mode : {}".format(balancing_mode) """ And returns the attributes of the class """ return { 'id': str(self.id), 'enabled': self.enabled, 'name': self.name, 'servers': servers_list, 'mode': mode, 'status': self.status, 'additional_infos': additional_infos, 'tags': self.tags } def to_template(self, server_list=None, header_list=None): """ Dictionary used to create configuration file :return Dictionnary of configuration parameters """ workflow_list = [] access_controls_list = [] for workflow in self.workflow_set.filter(enabled=True): tmp = { 'id': str(workflow.pk), 'fqdn': workflow.fqdn, 'public_dir': workflow.public_dir, 'backend': workflow.backend, 'defender_policy': workflow.defender_policy } access_controls_deny = [] access_controls_301 = [] access_controls_302 = [] for acl in workflow.workflowacl_set.filter(before_policy=False): rules, acl_name = acl.access_control.generate_rules() access_controls_list.append(rules) condition = acl.generate_condition(acl_name) redirect_url = None deny = False for type_acl in ('action_satisfy', 'action_not_satisfy'): action = getattr(acl, type_acl) if action != "200": if action in ("301", "302"): redirect_url = getattr( acl, type_acl.replace('action', 'redirect_url')) elif action == "403": deny = True break tmp_acl = { 'before_policy': acl.before_policy, 'redirect_url': redirect_url, 'conditions': condition, 'action': action, 'deny': deny } if action == "403": access_controls_deny.append(tmp_acl) elif action == "301": access_controls_301.append(tmp_acl) elif action == "302": access_controls_302.append(tmp_acl) tmp['access_controls_deny'] = access_controls_deny tmp['access_controls_302'] = access_controls_302 tmp['access_controls_301'] = access_controls_301 workflow_list.append(tmp) """ Simple attributes of the class """ result = { 'id': str(self.id), 'enabled': self.enabled, 'name': self.name, 'mode': self.mode, 'timeout_connect': self.timeout_connect, 'timeout_server': self.timeout_server, 'unix_socket': self.get_unix_socket(), 'custom_haproxy_conf': self.custom_haproxy_conf, 'JAIL_ADDRESSES': JAIL_ADDRESSES, 'accept_invalid_http_response': self.accept_invalid_http_response, 'http_forwardfor_header': self.http_forwardfor_header, 'http_forwardfor_except': self.http_forwardfor_except, 'enable_http_health_check': self.enable_http_health_check, 'http_health_check_method': self.http_health_check_method, 'http_health_check_uri': self.http_health_check_uri, 'http_health_check_version': self.http_health_check_version, 'http_health_check_headers': self.http_health_check_headers, 'enable_http_keep_alive': self.enable_http_keep_alive, 'http_keep_alive_timeout': self.http_keep_alive_timeout, 'access_controls_list': set(access_controls_list), 'http_backend_dir': self.http_backend_dir, 'balancing': self.balancing, 'workflows': workflow_list, 'tags': self.tags } """ Retrieve list/custom objects """ # If facultative arg listener_list is not given if not server_list: # Retrieve all the objects used by the current frontend # No .only ! Used to generated conf, neither str cause we need the object result['servers'] = self.server_set.all( ) # No .only() ! Used to generated conf if self.mode == "http": # Same for headers result['headers'] = self.headers.all( ) if not header_list else header_list if self.enable_http_health_check and self.http_health_check_expect_match: result['http_health_check_expect'] = "{} {}".format( self.http_health_check_expect_match, self.http_health_check_expect_pattern) return result def generate_conf(self, server_list=None, header_list=None): """ Render the conf with Jinja template and self.to_template() method :return The generated configuration as string, or raise """ # The following var is only used by error, do not forget to adapt if needed template_name = JINJA_PATH + JINJA_TEMPLATE try: jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH)) template = jinja2_env.get_template(JINJA_TEMPLATE) return template.render({ 'conf': self.to_template(server_list=server_list, header_list=header_list) }) # In ALL exceptions, associate an error message # The exception instantiation MUST be IN except statement, to retrieve traceback in __init__ except TemplateNotFound: exception = ServiceJinjaError( "The following file cannot be found : '{}'".format( template_name), "haproxy") except TemplatesNotFound: exception = ServiceJinjaError( "The following files cannot be found : '{}'".format( template_name), "haproxy") except (TemplateAssertionError, TemplateRuntimeError): exception = ServiceJinjaError( "Unknown error in template generation: {}".format( template_name), "haproxy") except UndefinedError: exception = ServiceJinjaError( "A variable is undefined while trying to render the following template: " "{}".format(template_name), "haproxy") except TemplateSyntaxError: exception = ServiceJinjaError( "Syntax error in the template: '{}'".format(template_name), "haproxy") # If there was an exception, raise a more general exception with the message and the traceback raise exception def test_conf(self): """ Write the configuration attribute on disk, in test directory, as {id}.conf.new And test the conf with 'haproxy -c' :return True or raise """ """ No need to do API request cause Backends are not relative-to-node objects """ test_filename = self.get_test_filename() conf = self.configuration # NO Node-specific configuration, we can test-it on local node # Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True test_haproxy_conf(test_filename, conf.replace( "backend {}".format(self.name), "backend test_{}".format(self.id or "test")), disabled=True) def get_test_filename(self): """ Return test filename for test conf with haproxy """ """ If object is not already saved : no id so default=test """ return "backend_{}.conf".format(self.id or "test") def get_base_filename(self): """ Return the base filename (without path) """ return "backend_{}.cfg".format(self.id) def get_filename(self): """ Return filename depending on current frontend object """ return "{}/{}".format(HAPROXY_PATH, self.get_base_filename()) def get_unix_socket(self): """ Return filename of unix socket on which HAProxy send data and on which Rsyslog listen """ return "{}/backend_{}.sock".format(UNIX_SOCKET_PATH, self.id) def save_conf(self): """ Write configuration on disk """ if not self.configuration: return params = [ self.get_filename(), self.configuration, BACKEND_OWNER, BACKEND_PERMS ] try: Cluster.api_request('system.config.models.write_conf', config=params) except Exception as e: # e used by VultureSystemConfigError logger.error(e, exc_info=1) raise VultureSystemConfigError( "on cluster.\nRequest failure to write_conf()") def reload_conf(self): """ Generate conf and save it """ self.configuration = self.generate_conf() self.save_conf() self.save() def enable(self): """ Enable backend in HAProxy, by management socket """ """ Cannot enable a disabled frontend ('enable' field) """ if not self.enabled: raise ServiceError( "Cannot start a disabled backend.", "haproxy", "enable backend", traceback="Please edit, enable and save a backend to start-it." ) return hot_action_backend(self.name, "enable") def disable(self): """ Disable frontend in HAProxy, by management socket """ if not self.enabled: return "This frontend is already disabled." """ If it is an Rsyslog only conf, return error """ if self.mode == "log" and self.listening_mode == "udp": raise ServiceError( "Cannot hot disable an Rsyslog only frontend.", "rsyslog", "disable frontend", traceback="Please edit, disable and save the frontend.") return hot_action_backend(self.name, "disable")
class NetworkAddress(models.Model): """ This is a generic IPv[4|6] Address """ name = models.TextField(default=_('Friendly name')) nic = models.ManyToManyField(NetworkInterfaceCard, through='NetworkAddressNIC') ip = models.GenericIPAddressField() prefix_or_netmask = models.TextField() is_system = models.BooleanField(default=False) carp_vhid = models.SmallIntegerField(default=0) # Needed to make alambiquate mongodb queries objects = models.DjongoManager() def to_dict(self): return { "id": str(self.id), "name": self.name, 'ip': self.ip, 'is_system': self.is_system, 'prefix_or_netmask': self.prefix_or_netmask, 'carp_vhid': self.carp_vhid } def to_template(self): """ Dictionary used to create configuration file related to the interface :return Dictionnary of configuration parameters """ nic_list = list() addresses_nic = NetworkAddressNIC.objects.filter(network_address=self) for address_nic in addresses_nic: nic = address_nic.nic nic_list.append(str(nic)) conf = { 'id': str(self.id), 'name': self.name, 'nic': ', '.join(nic_list), 'ip': self.ip, 'is_system': self.is_system, 'prefix_or_netmask': self.prefix_or_netmask, 'carp_vhid': self.carp_vhid } return conf @property def is_carp(self): if self.carp_vhid > 0: return True return False @property def version(self): """ Return the version of the Ip address :return: 4 or 6 """ if ":" in self.ip: return 6 else: return 4 @property def ip_cidr(self): """ Return IP/CIDR notation of Listener :return: String with ip/cidr notation """ ip = ipaddress.ip_address(self.ip) if ip.version == 6: prefix = self.prefix_or_netmask else: prefix = netmask2prefix(self.prefix_or_netmask) if prefix == 0: prefix = self.prefix_or_netmask.replace("/", "") return "{}/{}".format(self.ip, prefix) @property def family(self): """ :return: inet or inet6, depending of the IP address """ ip = ipaddress.ip_address(self.ip) if ip.version == 4: return "inet" else: return "inet6" def rc_config(self, force_dev=None, is_system=False): """ :param: force_dev: Optional NIC device. DO NOT USE except you known what you are doing :param: is_system: If True, "alias" keyword is not present :return: The rc.conf configuration line for this Network Address """ dev = None first = True addresses = NetworkAddressNIC.objects.filter(network_address=self) for address_nic in addresses: if first and address_nic.nic.node == Cluster.get_current_node(): if force_dev: dev = force_dev else: dev = address_nic.nic.dev first = False carp_priority = address_nic.carp_priority carp_passwd = address_nic.carp_passwd elif (not force_dev) and (address_nic.nic.node == Cluster.get_current_node()): logger.error( "Node::NetworkAddress: Inconsistent configuration: SAME IP on MULTIPLE NIC !!! {}" .format(self.ip)) if self.is_carp: if is_system: device = "{}".format(dev) else: device = "{}_alias".format(dev) device += '{}' inet = "{} {} vhid {} advskew {} pass {}".format( self.family, self.ip_cidr, self.carp_vhid, carp_priority, carp_passwd) else: if is_system: device = "{}".format(dev) else: device = "{}_alias".format(dev) device += '{}' inet = "{} {}".format(self.family, self.ip_cidr) if self.family == 'inet6': device += "_ipv6" return 'ifconfig_{}="{}"'.format(device, inet) def __str__(self): return "'{}' : {}/{}".format(self.name, self.ip, self.prefix_or_netmask) @staticmethod def str_attrs(): """ Attributes required by __str__ method """ return ['name', 'ip', 'prefix_or_netmask']
class Node(models.Model): """ A vulture Node """ class Meta: app_label = "system" name = models.TextField(unique=True) pf_limit_states = models.IntegerField(default=500000) pf_limit_frags = models.IntegerField(default=25000) pf_limit_src = models.IntegerField(default=50000) pf_custom_config = models.TextField(blank=True, default="") gateway = models.TextField(blank=True) gateway_ipv6 = models.TextField(blank=True) static_routes = models.TextField( blank=True, default='#static_routes=\"net1 net2\"\n' '#route_net1=\"-net 192.168.0.0/24 192.168.0.1\"\n' '#route_net2=\"-net 192.168.1.0/24 192.168.1.1\"\n') management_ip = models.TextField(unique=True) internet_ip = models.GenericIPAddressField( help_text=_("IP used by jails to contact internet")) scanner_ip = models.ForeignKey(to="NetworkAddress", null=True, on_delete=models.SET_NULL, help_text=_("NAT IP used for scanner"), verbose_name=_("Scanner IP")) def __str__(self): return self.name def to_small_dict(self): """ Return dictionnary of object attributes """ return { "id": str(self.id), "name": self.name, "management_ip": self.management_ip, "internet_ip": self.internet_ip, "pf_limit_states": self.pf_limit_states, "pf_limit_frags": self.pf_limit_frags, "pf_limit_src": self.pf_limit_src, "pf_custom_config": self.pf_custom_config, "gateway": self.gateway, "gateway_ipv6": self.gateway_ipv6, "static_routes": self.static_routes } def to_dict(self): excluded_intf = ("lo0", "lo1", "lo2", "lo3", "lo4", "lo5", "lo6", "pflog0", "vm-public", "tap0", "tun0") intfs = [ n.to_dict() for n in NetworkInterfaceCard.objects.filter( node=self).exclude(dev__in=excluded_intf) ] return { "id": str(self.id), "name": self.name, 'intfs': intfs, "management_ip": self.management_ip, "internet_ip": self.internet_ip, "pf_limit_states": self.pf_limit_states, "pf_limit_frags": self.pf_limit_frags, "pf_limit_src": self.pf_limit_src, "pf_custom_config": self.pf_custom_config, "gateway": self.gateway, "gateway_ipv6": self.gateway_ipv6, "static_routes": self.static_routes, "is_master_mongo": self.is_master_mongo, "is_standalone": self.is_standalone, 'is_configured': self.is_configured, "is_master_redis": self.is_master_redis, 'configured_forwarders': len(self.get_forwarders_enabled ), # TODO : Implement less consuming function 'configured_backends': len(self.get_backends_enabled ), # TODO : Implement less consuming function 'unix_timestamp': time.time() } def api_request(self, action, config=None, internal=False): """ :param action: The requested action :param config: The associated config :param internal: Is this request internal ? Means that it will not be shown to the admin :return: { 'status': True, 'message': 'A meaningfull message' } { 'status': False, 'message': 'A meaningfull message' } """ return Cluster.api_request(action, config, self, internal=internal) def synchronizeNICs(self): """ This call the net2mongo.py script in order to read system's network configuration and put it in mongodb - System wins for "Mains IPs": Mongodb will be overrided :return: True / False """ proc = subprocess.Popen([ '/home/vlt-os/env/bin/python', '/home/vlt-os/scripts/net2mongo.py' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) success, error = proc.communicate() # Logger are inside the script if not error: return True logger.error("[synchronizeNICs] ERROR :: net2mongo : {}".format(error)) return False def to_template(self): """ Dictionary used to create configuration file related to the node :return Dictionnary of configuration parameters """ addresses = list() for nic in NetworkInterfaceCard.objects.filter(node=self): for address in NetworkAddress.objects.filter(nic=nic): data = str(address.ip) + " on " + str(nic.dev) if address.is_carp: data = " (CARP vhid = {})".format(address.carp_vhid) addresses.append(data) conf = { 'id': str(self.id), 'name': self.name, 'pf_limit_states': self.pf_limit_states, 'pf_limit_frags': self.pf_limit_frags, 'pf_limit_src': self.pf_limit_src, 'addresses': addresses, 'is_master_redis': self.is_master_redis, 'is_master_mongo': self.is_master_mongo, 'is_standalone': self.is_standalone } return conf @property def is_standalone(self): """ Check if the current Node is a member of mongoDB :return: True / False, or None in case of a failure """ c = MongoBase() ok = c.connect() if ok: c.connect_primary() config = c.db.admin.command("replSetGetConfig")['config'] return len(config['members']) == 1 return True @property def is_master_mongo(self): """ Check if the current Node is master or not :return: True / False, or None in case of a failure """ c = MongoBase() ok = c.connect() if ok: primary_node = c.get_primary() else: return None if ok and primary_node == self.name + ':9091': return True elif ok: return False return None @property def is_master_redis(self): """ Check if the current Node is master or not :return: True / False, or None in case of a failure """ if self.management_ip: c = RedisBase() master_node = c.get_master(self.name) if master_node == self.name: return True elif master_node: return False return None @property def is_configured(self): if self.management_ip: return True return False def write_management_ip(self): """ Write self.management_ip in management.ip files (jails and host) """ """ First write on host's file """ api_res = self.api_request( "system.config.models.write_conf", [MANAGEMENT_IP_PATH, self.management_ip, "root:wheel", "644"]) """ Then, write into jails same path """ for jail in JAILS: api_res = self.api_request("system.config.models.write_conf", [ "/zroot/{}{}".format(jail, MANAGEMENT_IP_PATH), self.management_ip, "root:wheel", "644" ]) """ Returns the last api request """ return api_res def addresses(self, nic=None): """ Return the list of network addresses on the current node, or node/nic :param nic: Optional NIC :return: list """ addresses = list() if not nic: for nic in NetworkInterfaceCard.objects.filter(node=self): addresses.extend(list(NetworkAddress.objects.filter(nic=nic))) else: addresses.extend(list(NetworkAddress.objects.filter(nic=nic))) return addresses @property def network_interfaces(self): return list(NetworkInterfaceCard.objects.filter(node=self)) @property def get_listeners_enabled(self): """ Return all listeners used by an enabled Frontend """ from services.frontend.models import Listener """ Retrieve network addresses on the current node """ addresses = self.addresses() """ Retrieve all Listeners that uses those network addresses """ listeners = Listener.objects.filter(network_address__in=addresses, frontend__enabled=True) listeners_enabled = list() for l in listeners: """ Protocol is tcp in any case, except for frontend mode=log => proto will be tcp or udp or tcp,udp listening_mode attribute """ proto = "tcp" if l.frontend.mode == "log" and "udp" in l.frontend.listening_mode: proto = l.frontend.listening_mode """ Return (ip, port, interface.family) """ listeners_enabled.append( (l.whitelist_ips, l.network_address.ip, l.port, l.rsyslog_port, proto, l.network_address.family, l.max_src, l.max_rate)) return listeners_enabled @property def get_forwarders_enabled(self): """ Return all tuples (family, proto, ip, port) for each LogForwarders """ from applications.logfwd.models import LogOMRELP, LogOMHIREDIS, LogOMFWD, LogOMElasticSearch, LogOMMongoDB # !!! REQUIRED BY listener_set ! from services.frontend.models import Listener result = list() """ Retrieve LogForwarders used in enabled Frontends """ addresses = self.addresses() """ For each address, retrieve the associated listeners having frontend enabled """ listener_addr = dict() for a in addresses: try: listener_addr[a].append([ l.id for l in a.listener_set.filter(frontend__enabled=True) ]) except KeyError: listener_addr[a] = [ l.id for l in a.listener_set.filter(frontend__enabled=True) ] """ Loop on each NetworkAddress to retrieve LogForwarder used by the frontend using listeners """ for network_address, listener_ids in listener_addr.items(): # Log Forwarder RELP for logfwd in LogOMRELP.objects.filter( enabled=True, frontend_set__listener__in=listener_ids): if (network_address.family, "tcp", logfwd.target, logfwd.port) not in result: result.append((network_address.family, "tcp", logfwd.target, logfwd.port)) # TCP # Log Forwarder REDIS for logfwd in LogOMHIREDIS.objects.filter( enabled=True, frontend_set__listener__in=listener_ids): if (network_address.family, "tcp", logfwd.target, logfwd.port) not in result: result.append((network_address.family, "tcp", logfwd.target, logfwd.port)) # TCP # Log Forwarder Syslog for logfwd in LogOMFWD.objects.filter( enabled=True, frontend_set__listener__in=listener_ids): if (network_address.family, logfwd.protocol, logfwd.target, logfwd.port) not in result: result.append((network_address.family, logfwd.protocol, logfwd.target, logfwd.port)) # proto # Log Forwarder ElasticSearch for logfwd in LogOMElasticSearch.objects.filter( enabled=True, frontend_set__listener__in=listener_ids): """ For elasticsearch, we need to parse the servers """ for ip, port in re_findall("https?://([^:]+):(\d+)", logfwd.server): if ("inet6" if ':' in ip else "inet", "tcp", ip, port) not in result: result.append(("inet6" if ':' in ip else "inet", "tcp", ip, port)) # Log Forwarder MongoDB for logfwd in LogOMMongoDB.objects.filter( enabled=True, frontend_set__listener__in=listener_ids): """ For OMMongoDB - parse uristr """ for ip, port in parse_uristr(logfwd.uristr): result.append((network_address.family, 'tcp', ip, port)) return result @property def get_backends_enabled(self): """ Return all tuples (family, proto, ip, port) for each enabled Backends on this node """ from applications.backend.models import Backend # !!! REQUIRED BY listener_set ! from services.frontend.models import Listener result = list() """ Retrieve Backends used in enabled Frontends """ addresses = self.addresses() """ For each address, retrieve the associated listeners having frontend enabled """ listener_addr = dict() for a in addresses: try: listener_addr[a].append([ l.id for l in a.listener_set.filter(frontend__enabled=True) ]) except KeyError: listener_addr[a] = [ l.id for l in a.listener_set.filter(frontend__enabled=True) ] """ Loop on each NetworkAddress to retrieve LogForwarder used by the frontend using listeners """ for network_address, listener_ids in listener_addr.items(): for backend in Backend.objects.filter( enabled=True, frontend__listener__in=listener_ids): for server in backend.server_set.all(): result.append((network_address.family, "tcp", server.target, server.port)) # TCP return result def process_messages(self): """ Function called from Cluster daemon: read queue and process asked functions :return: """ logger_daemon = logging.getLogger('daemon') messages = MessageQueue.objects.filter( node=self, status='new').order_by('modified') for message in messages: logger.debug("Cluster::process_messages: {},{},{}".format( message.action, message.config, message.node)) message.status = 'running' message.save() try: """ Big try in case of import or execution error """ # Call the function my_function = import_string(message.action) args = [logger_daemon] if message.config: args.append(message.config) message.result = my_function(*args) message.status = 'done' except ServiceExit: """ Service stop asked """ raise except KeyError as e: logger.exception(e) message.result = "KeyError {}".format(str(e)) except Exception as e: logger.exception(e) logger.error("Cluster::process_messages: {}".format(str(e))) message.status = 'failure' message.result = str(e) message.save()