class GenericModel(models.Model):
big_int = models.BigIntegerField()
bool = models.BooleanField()
char = models.CharField(max_length=20)
comma_int = models.CommaSeparatedIntegerField()
date = models.DateField()
date_time = models.DateTimeField()
decimal = models.DecimalField(max_digits=10, decimal_places=5)
email = models.EmailField()
float = models.FloatField()
integer = models.IntegerField()
null_bool = models.NullBooleanField()
pos_int = models.PositiveIntegerField()
pos_small_int = models.PositiveSmallIntegerField()
slug = models.SlugField()
small_int = models.SmallIntegerField()
text = models.TextField()
time = models.TimeField()
url = models.URLField()
ip = models.GenericIPAddressField()
uuid = models.UUIDField()
# TODO: add these
# basic_file = models.FileField()
# image = models.ImageField()
objects = models.DjongoManager()
class Log(models.Model):
path_info = models.CharField(max_length=200)
event_name = models.CharField(max_length=200)
page_title = models.CharField(max_length=200)
visited_by = models.CharField(max_length=100)
ip_address = models.GenericIPAddressField()
datetime = models.DateTimeField()
referrer = models.CharField(max_length=500)
browser_family = models.CharField(max_length=100)
browser_version = models.CharField(max_length=20)
os_family = models.CharField(max_length=100)
os_version = models.CharField(max_length=20)
device_family = models.CharField(max_length=100)
device_type = models.CharField(max_length=50)
latitude = models.FloatField()
longitude = models.FloatField()
country = models.CharField(max_length=100)
region = models.CharField(max_length=5)
city = models.CharField(max_length=100)
def __str__(self):
return "Website Log Object"
class Meta:
db_table = 'website_logs'
objects = models.DjongoManager()
class VisitorSpot(models.Model):
datetime = models.DateTimeField()
geom = PointField()
ip_address = models.GenericIPAddressField()
@property
def popupContent(self):
return '<span/>{}</span>'.format(self.ip_address)
class Contact(models.Model):
first_name = models.CharField(max_length=50)
last_name = models.CharField(max_length=50)
email = models.EmailField()
ip_address = models.GenericIPAddressField(null=True)
message = models.TextField()
def __str__(self):
return f"{self.first_name}.{self.last_name}"
class SourceRemote(djongo_models.Model):
systemIP = djongo_models.GenericIPAddressField()
username = djongo_models.CharField(max_length=100)
password = djongo_models.CharField(max_length=500)
file_path = djongo_models.CharField(max_length=200)
objects = djongo_models.DjongoManager()
def __str__(self):
return "remote_source" + '_' + str(self.id) + '_' + self.username
class VisitorPath(models.Model):
datetime = models.DateTimeField()
city = models.CharField(max_length=100)
region = models.CharField(max_length=100)
country = models.CharField(max_length=100)
ip_address = models.GenericIPAddressField()
visit_num = models.IntegerField()
browser = models.CharField(max_length=100)
os = models.CharField(max_length=100)
device = models.CharField(max_length=100)
path = models.ArrayField(model_container=Path, )
def __str__(self):
return "Visitor Path Model"
objects = models.DjongoManager()
class PageViewActivity(models.Model):
datetime = models.DateTimeField()
browser = models.CharField(max_length=100)
os = models.CharField(max_length=100)
city = models.CharField(max_length=100)
device = models.CharField(max_length=100)
region = models.CharField(max_length=100)
country = models.CharField(max_length=100)
ip_address = models.GenericIPAddressField()
page_name = models.CharField(max_length=300)
page_url = models.CharField(max_length=300)
referrer = models.CharField(max_length=300)
def __str__(self):
return "Page View Activity Model"
objects = models.DjongoManager()
class VisitorActivity(models.Model):
datetime = models.DateTimeField()
page_views = models.IntegerField()
total_visits = models.IntegerField()
latest_page_view = models.DateTimeField()
city = models.CharField(max_length=100)
region = models.CharField(max_length=100)
country = models.CharField(max_length=100)
visit_length_sec = models.IntegerField()
ip_address = models.GenericIPAddressField()
browser = models.CharField(max_length=100)
os = models.CharField(max_length=100)
device = models.CharField(max_length=100)
referrer = models.CharField(max_length=300)
entry_page = models.CharField(max_length=300)
latest_page = models.CharField(max_length=300)
def __str__(self):
return "Visitor Activity Model"
objects = models.DjongoManager()
class Server(models.Model):
""" Link class between NetworkAddress and Frontend """
""" NetworkAddress object to listen on """
target = models.TextField(default="1.2.3.4",
help_text=_("IP address/hostname of server"))
""" Port To listen on """
port = models.PositiveIntegerField(
default=80,
validators=[MinValueValidator(1),
MaxValueValidator(65535)])
""" Backend associated to this server """
backend = models.ForeignKey(to=Backend, on_delete=models.CASCADE)
""" TLSProfile to use when listening """
tls_profile = models.ForeignKey(TLSProfile,
null=True,
default="",
on_delete=models.SET_DEFAULT)
""" Weight of the server """
weight = models.PositiveIntegerField(
validators=[MinValueValidator(0),
MaxValueValidator(256)],
default=1,
help_text=_(""))
""" Source """
source = models.GenericIPAddressField(
default="", help_text=_("IP address to assign to sources"))
def to_template(self):
""" Create a JSON representation of the object
:return Dictionnary of configuration parameters
"""
""" Retrieve list/custom objects """
result = {
'id': str(self.id),
'target': self.target,
'port': self.port,
'backend': self.backend,
'weight': self.weight,
}
if self.source:
result['source'] = self.source
return result
def __str__(self):
return "{}:{}".format(self.target, self.port)
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['target', 'port']
def generate_conf(self):
""" Generate Listener HAProxy configuration
:return A string - Bind directive
"""
result = "server srv{} {}:{} weight {}".format(self.id, self.target,
self.port, self.weight)
if self.source:
if ":" in self.source:
result += " source [{}]".format(self.source)
else:
result += " source {}".format(self.source)
if self.tls_profile:
result += self.tls_profile.generate_conf(backend=True)
return result
class Backend(models.Model):
""" Model used to generated fontends configuration of HAProxy """
""" Is that section enabled or disabled """
enabled = models.BooleanField(
default=True,
help_text=_("Enable the backend"),
)
""" Name of the frontend, unique constraint """
name = models.TextField(
unique=True,
default="Backend",
help_text=_("Name of HAProxy backend"),
)
""" Listening mode of Frontend : tcp or http """
mode = models.TextField(
default=MODE_CHOICES[0][0],
choices=MODE_CHOICES,
help_text=_("Proxy mode"),
)
timeout_connect = models.PositiveIntegerField(
default=2000,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
timeout_server = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(3600)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Save of generated configuration """
configuration = models.TextField(default="{}")
""" Status of frontend for each nodes """
status = models.DictField(default={})
""" Headers """
headers = models.ArrayReferenceField(Header,
null=True,
blank=False,
on_delete=models.CASCADE,
help_text=_("Header rules"))
""" Custom HAProxy Backend directives """
custom_haproxy_conf = models.TextField(
default="", help_text=_("Custom HAProxy configuration directives."))
""" HTTP Options """
""" URI of backends """
http_backend_dir = models.TextField(
default="/",
help_text=_("Servers directory to prefix in front of requests uri"),
verbose_name=_("Servers directory"))
""" Enable or disable relaxing of HTTP response parsing """
accept_invalid_http_response = models.BooleanField(
default=False,
help_text=_("Enable relaxing of HTTP response parsing"),
verbose_name=_("Accept invalid HTTP response"))
""" Enable insertion of the X-Forwarded-For header to requests sent to servers """
http_forwardfor_header = models.TextField(
blank=True,
null=True,
help_text=_("Insertion of the X-Forwarded-For header"),
verbose_name=_("Send source ip in "))
""" Except the following IP"""
http_forwardfor_except = models.GenericIPAddressField(
protocol='IPv4',
blank=True,
null=True,
help_text=_("Except the specified IP address"),
verbose_name=_("Except for "))
""" Enable HTTP protocol to check on the servers health """
enable_http_health_check = models.BooleanField(
default=False,
help_text=_("Enable HTTP protocol health checker"),
verbose_name=_("HTTP health check"))
""" The optional HTTP method used with the requests """
http_health_check_method = models.TextField(
default=HEALTH_CHECK_METHOD_CHOICES[0][0],
choices=HEALTH_CHECK_METHOD_CHOICES,
help_text=_("HTTP method used"),
verbose_name=_("Method"))
""" The URI referenced in the HTTP requests """
http_health_check_uri = models.TextField(default='/',
blank=True,
null=True,
help_text=_("URI referenced"),
verbose_name=_("URI"))
""" The optional HTTP version string """
http_health_check_version = models.TextField(
default=HEALTH_CHECK_VERSION_CHOICES[0][0],
choices=HEALTH_CHECK_VERSION_CHOICES,
help_text=_("HTTP version"),
verbose_name=_("Version"))
""" """
http_health_check_headers = models.DictField(
default={},
help_text=_("HTTP Health Check Headers"),
verbose_name=_("HTTP Health Check Headers"))
""" Health check expect """
http_health_check_expect_match = models.TextField(
choices=HEALTH_CHECK_EXPECT_CHOICES,
default=HEALTH_CHECK_EXPECT_CHOICES[0][0],
null=True,
help_text=_("Type of match to expect"),
verbose_name=_("HTTP Health Check expected"))
http_health_check_expect_pattern = models.TextField(
default="200",
help_text=_("Type of pattern to match to expect"),
verbose_name=_("HTTP Health Check expected pattern"))
""" Enable or disable HTTP keep-alive from client to server """
enable_http_keep_alive = models.BooleanField(
default=True,
help_text=_("Enable HTTP keep-alive"),
verbose_name=_("HTTP Keep alive"))
""" Keep-alive Timeout """
http_keep_alive_timeout = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Balancing mode """
balancing_mode = models.TextField(
choices=BALANCING_CHOICES,
default=BALANCING_CHOICES[0][0],
help_text=_("Balancing mode between servers"),
verbose_name=_("Balancing mode"))
""" Balancing param """
balancing_param = models.TextField(
null=True,
default="",
help_text=_("Balancing param for balancing mode"),
verbose_name=_("Balancing parameter"))
""" Tags """
tags = models.ListField(
models.SlugField(default=""),
default=[],
help_text=_("Tags to set on this object for search"))
@property
def balancing(self):
if self.balancing_mode == "hdr":
result = "hdr({})".format(self.balancing_param)
elif self.balancing_mode == "url_param":
result = "url_param {}".format(self.balancing_param)
elif self.balancing_mode == "rdp-cookie":
result = "rdp-cookie({})".format(self.balancing_param)
else:
result = self.balancing_mode
return result
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name', 'mode']
def __str__(self):
return "{} Backend '{}'".format(self.mode.upper(), self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models like Listeners
:return A JSON object
"""
result = {
'id': self.id,
'enable': self.enabled,
'name': self.name,
'mode': self.mode,
'balancing_mode': self.balancing_mode,
'balancing_param': self.balancing_param,
'status': dict(self.status), # It is an OrderedDict
'servers': [],
'custom_haproxy_conf': self.custom_haproxy_conf,
'tags': self.tags
}
""" Add listeners """
for server in self.server_set.all():
s = server.to_template()
# Remove frontend to prevent infinite loop
del s['backend']
result['servers'].append(s)
""" Other attributes """
if self.mode == "http":
result['headers'] = []
for header in self.headers.all():
result['headers'].append(header.to_template())
return result
def to_html_template(self):
""" Dictionary used to render object as html
:return Dictionnary of configuration parameters
"""
""" Retrieve list/custom objects """
servers_list = [
str(s) for s in self.server_set.all().only(*Server.str_attrs())
]
mode = "UNKNOWN"
for m in MODE_CHOICES:
if self.mode == m[0]:
mode = m[1]
balancing_mode = "unknown"
for m in BALANCING_CHOICES:
if self.balancing_mode == m[0]:
balancing_mode = m[1]
additional_infos = "Balancing mode : {}".format(balancing_mode)
""" And returns the attributes of the class """
return {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'servers': servers_list,
'mode': mode,
'status': self.status,
'additional_infos': additional_infos,
'tags': self.tags
}
def to_template(self, server_list=None, header_list=None):
""" Dictionary used to create configuration file
:return Dictionnary of configuration parameters
"""
workflow_list = []
access_controls_list = []
for workflow in self.workflow_set.filter(enabled=True):
tmp = {
'id': str(workflow.pk),
'fqdn': workflow.fqdn,
'public_dir': workflow.public_dir,
'backend': workflow.backend,
'defender_policy': workflow.defender_policy
}
access_controls_deny = []
access_controls_301 = []
access_controls_302 = []
for acl in workflow.workflowacl_set.filter(before_policy=False):
rules, acl_name = acl.access_control.generate_rules()
access_controls_list.append(rules)
condition = acl.generate_condition(acl_name)
redirect_url = None
deny = False
for type_acl in ('action_satisfy', 'action_not_satisfy'):
action = getattr(acl, type_acl)
if action != "200":
if action in ("301", "302"):
redirect_url = getattr(
acl, type_acl.replace('action',
'redirect_url'))
elif action == "403":
deny = True
break
tmp_acl = {
'before_policy': acl.before_policy,
'redirect_url': redirect_url,
'conditions': condition,
'action': action,
'deny': deny
}
if action == "403":
access_controls_deny.append(tmp_acl)
elif action == "301":
access_controls_301.append(tmp_acl)
elif action == "302":
access_controls_302.append(tmp_acl)
tmp['access_controls_deny'] = access_controls_deny
tmp['access_controls_302'] = access_controls_302
tmp['access_controls_301'] = access_controls_301
workflow_list.append(tmp)
""" Simple attributes of the class """
result = {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'mode': self.mode,
'timeout_connect': self.timeout_connect,
'timeout_server': self.timeout_server,
'unix_socket': self.get_unix_socket(),
'custom_haproxy_conf': self.custom_haproxy_conf,
'JAIL_ADDRESSES': JAIL_ADDRESSES,
'accept_invalid_http_response': self.accept_invalid_http_response,
'http_forwardfor_header': self.http_forwardfor_header,
'http_forwardfor_except': self.http_forwardfor_except,
'enable_http_health_check': self.enable_http_health_check,
'http_health_check_method': self.http_health_check_method,
'http_health_check_uri': self.http_health_check_uri,
'http_health_check_version': self.http_health_check_version,
'http_health_check_headers': self.http_health_check_headers,
'enable_http_keep_alive': self.enable_http_keep_alive,
'http_keep_alive_timeout': self.http_keep_alive_timeout,
'access_controls_list': set(access_controls_list),
'http_backend_dir': self.http_backend_dir,
'balancing': self.balancing,
'workflows': workflow_list,
'tags': self.tags
}
""" Retrieve list/custom objects """
# If facultative arg listener_list is not given
if not server_list:
# Retrieve all the objects used by the current frontend
# No .only ! Used to generated conf, neither str cause we need the object
result['servers'] = self.server_set.all(
) # No .only() ! Used to generated conf
if self.mode == "http":
# Same for headers
result['headers'] = self.headers.all(
) if not header_list else header_list
if self.enable_http_health_check and self.http_health_check_expect_match:
result['http_health_check_expect'] = "{} {}".format(
self.http_health_check_expect_match,
self.http_health_check_expect_pattern)
return result
def generate_conf(self, server_list=None, header_list=None):
""" Render the conf with Jinja template and self.to_template() method
:return The generated configuration as string, or raise
"""
# The following var is only used by error, do not forget to adapt if needed
template_name = JINJA_PATH + JINJA_TEMPLATE
try:
jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH))
template = jinja2_env.get_template(JINJA_TEMPLATE)
return template.render({
'conf':
self.to_template(server_list=server_list,
header_list=header_list)
})
# In ALL exceptions, associate an error message
# The exception instantiation MUST be IN except statement, to retrieve traceback in __init__
except TemplateNotFound:
exception = ServiceJinjaError(
"The following file cannot be found : '{}'".format(
template_name), "haproxy")
except TemplatesNotFound:
exception = ServiceJinjaError(
"The following files cannot be found : '{}'".format(
template_name), "haproxy")
except (TemplateAssertionError, TemplateRuntimeError):
exception = ServiceJinjaError(
"Unknown error in template generation: {}".format(
template_name), "haproxy")
except UndefinedError:
exception = ServiceJinjaError(
"A variable is undefined while trying to render the following template: "
"{}".format(template_name), "haproxy")
except TemplateSyntaxError:
exception = ServiceJinjaError(
"Syntax error in the template: '{}'".format(template_name),
"haproxy")
# If there was an exception, raise a more general exception with the message and the traceback
raise exception
def test_conf(self):
""" Write the configuration attribute on disk, in test directory, as {id}.conf.new
And test the conf with 'haproxy -c'
:return True or raise
"""
""" No need to do API request cause Backends are not relative-to-node objects """
test_filename = self.get_test_filename()
conf = self.configuration
# NO Node-specific configuration, we can test-it on local node
# Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True
test_haproxy_conf(test_filename,
conf.replace(
"backend {}".format(self.name),
"backend test_{}".format(self.id or "test")),
disabled=True)
def get_test_filename(self):
""" Return test filename for test conf with haproxy
"""
""" If object is not already saved : no id so default=test """
return "backend_{}.conf".format(self.id or "test")
def get_base_filename(self):
""" Return the base filename (without path) """
return "backend_{}.cfg".format(self.id)
def get_filename(self):
""" Return filename depending on current frontend object
"""
return "{}/{}".format(HAPROXY_PATH, self.get_base_filename())
def get_unix_socket(self):
""" Return filename of unix socket on which HAProxy send data
and on which Rsyslog listen
"""
return "{}/backend_{}.sock".format(UNIX_SOCKET_PATH, self.id)
def save_conf(self):
""" Write configuration on disk
"""
if not self.configuration:
return
params = [
self.get_filename(), self.configuration, BACKEND_OWNER,
BACKEND_PERMS
]
try:
Cluster.api_request('system.config.models.write_conf',
config=params)
except Exception as e: # e used by VultureSystemConfigError
logger.error(e, exc_info=1)
raise VultureSystemConfigError(
"on cluster.\nRequest failure to write_conf()")
def reload_conf(self):
""" Generate conf and save it """
self.configuration = self.generate_conf()
self.save_conf()
self.save()
def enable(self):
""" Enable backend in HAProxy, by management socket """
""" Cannot enable a disabled frontend ('enable' field) """
if not self.enabled:
raise ServiceError(
"Cannot start a disabled backend.",
"haproxy",
"enable backend",
traceback="Please edit, enable and save a backend to start-it."
)
return hot_action_backend(self.name, "enable")
def disable(self):
""" Disable frontend in HAProxy, by management socket """
if not self.enabled:
return "This frontend is already disabled."
""" If it is an Rsyslog only conf, return error """
if self.mode == "log" and self.listening_mode == "udp":
raise ServiceError(
"Cannot hot disable an Rsyslog only frontend.",
"rsyslog",
"disable frontend",
traceback="Please edit, disable and save the frontend.")
return hot_action_backend(self.name, "disable")
class NetworkAddress(models.Model):
"""
This is a generic IPv[4|6] Address
"""
name = models.TextField(default=_('Friendly name'))
nic = models.ManyToManyField(NetworkInterfaceCard,
through='NetworkAddressNIC')
ip = models.GenericIPAddressField()
prefix_or_netmask = models.TextField()
is_system = models.BooleanField(default=False)
carp_vhid = models.SmallIntegerField(default=0)
# Needed to make alambiquate mongodb queries
objects = models.DjongoManager()
def to_dict(self):
return {
"id": str(self.id),
"name": self.name,
'ip': self.ip,
'is_system': self.is_system,
'prefix_or_netmask': self.prefix_or_netmask,
'carp_vhid': self.carp_vhid
}
def to_template(self):
""" Dictionary used to create configuration file related to the interface
:return Dictionnary of configuration parameters
"""
nic_list = list()
addresses_nic = NetworkAddressNIC.objects.filter(network_address=self)
for address_nic in addresses_nic:
nic = address_nic.nic
nic_list.append(str(nic))
conf = {
'id': str(self.id),
'name': self.name,
'nic': ', '.join(nic_list),
'ip': self.ip,
'is_system': self.is_system,
'prefix_or_netmask': self.prefix_or_netmask,
'carp_vhid': self.carp_vhid
}
return conf
@property
def is_carp(self):
if self.carp_vhid > 0:
return True
return False
@property
def version(self):
""" Return the version of the Ip address
:return: 4 or 6
"""
if ":" in self.ip:
return 6
else:
return 4
@property
def ip_cidr(self):
""" Return IP/CIDR notation of Listener
:return: String with ip/cidr notation
"""
ip = ipaddress.ip_address(self.ip)
if ip.version == 6:
prefix = self.prefix_or_netmask
else:
prefix = netmask2prefix(self.prefix_or_netmask)
if prefix == 0:
prefix = self.prefix_or_netmask.replace("/", "")
return "{}/{}".format(self.ip, prefix)
@property
def family(self):
"""
:return: inet or inet6, depending of the IP address
"""
ip = ipaddress.ip_address(self.ip)
if ip.version == 4:
return "inet"
else:
return "inet6"
def rc_config(self, force_dev=None, is_system=False):
"""
:param: force_dev: Optional NIC device. DO NOT USE except
you known what you are doing
:param: is_system: If True, "alias" keyword is not present
:return: The rc.conf configuration line for this Network Address
"""
dev = None
first = True
addresses = NetworkAddressNIC.objects.filter(network_address=self)
for address_nic in addresses:
if first and address_nic.nic.node == Cluster.get_current_node():
if force_dev:
dev = force_dev
else:
dev = address_nic.nic.dev
first = False
carp_priority = address_nic.carp_priority
carp_passwd = address_nic.carp_passwd
elif (not force_dev) and (address_nic.nic.node
== Cluster.get_current_node()):
logger.error(
"Node::NetworkAddress: Inconsistent configuration: SAME IP on MULTIPLE NIC !!! {}"
.format(self.ip))
if self.is_carp:
if is_system:
device = "{}".format(dev)
else:
device = "{}_alias".format(dev)
device += '{}'
inet = "{} {} vhid {} advskew {} pass {}".format(
self.family, self.ip_cidr, self.carp_vhid, carp_priority,
carp_passwd)
else:
if is_system:
device = "{}".format(dev)
else:
device = "{}_alias".format(dev)
device += '{}'
inet = "{} {}".format(self.family, self.ip_cidr)
if self.family == 'inet6':
device += "_ipv6"
return 'ifconfig_{}="{}"'.format(device, inet)
def __str__(self):
return "'{}' : {}/{}".format(self.name, self.ip,
self.prefix_or_netmask)
@staticmethod
def str_attrs():
""" Attributes required by __str__ method """
return ['name', 'ip', 'prefix_or_netmask']
class Node(models.Model):
"""
A vulture Node
"""
class Meta:
app_label = "system"
name = models.TextField(unique=True)
pf_limit_states = models.IntegerField(default=500000)
pf_limit_frags = models.IntegerField(default=25000)
pf_limit_src = models.IntegerField(default=50000)
pf_custom_config = models.TextField(blank=True, default="")
gateway = models.TextField(blank=True)
gateway_ipv6 = models.TextField(blank=True)
static_routes = models.TextField(
blank=True,
default='#static_routes=\"net1 net2\"\n'
'#route_net1=\"-net 192.168.0.0/24 192.168.0.1\"\n'
'#route_net2=\"-net 192.168.1.0/24 192.168.1.1\"\n')
management_ip = models.TextField(unique=True)
internet_ip = models.GenericIPAddressField(
help_text=_("IP used by jails to contact internet"))
scanner_ip = models.ForeignKey(to="NetworkAddress",
null=True,
on_delete=models.SET_NULL,
help_text=_("NAT IP used for scanner"),
verbose_name=_("Scanner IP"))
def __str__(self):
return self.name
def to_small_dict(self):
""" Return dictionnary of object attributes """
return {
"id": str(self.id),
"name": self.name,
"management_ip": self.management_ip,
"internet_ip": self.internet_ip,
"pf_limit_states": self.pf_limit_states,
"pf_limit_frags": self.pf_limit_frags,
"pf_limit_src": self.pf_limit_src,
"pf_custom_config": self.pf_custom_config,
"gateway": self.gateway,
"gateway_ipv6": self.gateway_ipv6,
"static_routes": self.static_routes
}
def to_dict(self):
excluded_intf = ("lo0", "lo1", "lo2", "lo3", "lo4", "lo5", "lo6",
"pflog0", "vm-public", "tap0", "tun0")
intfs = [
n.to_dict() for n in NetworkInterfaceCard.objects.filter(
node=self).exclude(dev__in=excluded_intf)
]
return {
"id": str(self.id),
"name": self.name,
'intfs': intfs,
"management_ip": self.management_ip,
"internet_ip": self.internet_ip,
"pf_limit_states": self.pf_limit_states,
"pf_limit_frags": self.pf_limit_frags,
"pf_limit_src": self.pf_limit_src,
"pf_custom_config": self.pf_custom_config,
"gateway": self.gateway,
"gateway_ipv6": self.gateway_ipv6,
"static_routes": self.static_routes,
"is_master_mongo": self.is_master_mongo,
"is_standalone": self.is_standalone,
'is_configured': self.is_configured,
"is_master_redis": self.is_master_redis,
'configured_forwarders':
len(self.get_forwarders_enabled
), # TODO : Implement less consuming function
'configured_backends':
len(self.get_backends_enabled
), # TODO : Implement less consuming function
'unix_timestamp': time.time()
}
def api_request(self, action, config=None, internal=False):
"""
:param action: The requested action
:param config: The associated config
:param internal: Is this request internal ? Means that it will not be shown to the admin
:return:
{ 'status': True, 'message': 'A meaningfull message' }
{ 'status': False, 'message': 'A meaningfull message' }
"""
return Cluster.api_request(action, config, self, internal=internal)
def synchronizeNICs(self):
"""
This call the net2mongo.py script in order to read
system's network configuration and put it in mongodb
- System wins for "Mains IPs": Mongodb will be overrided
:return: True / False
"""
proc = subprocess.Popen([
'/home/vlt-os/env/bin/python', '/home/vlt-os/scripts/net2mongo.py'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
success, error = proc.communicate()
# Logger are inside the script
if not error:
return True
logger.error("[synchronizeNICs] ERROR :: net2mongo : {}".format(error))
return False
def to_template(self):
""" Dictionary used to create configuration file related to the node
:return Dictionnary of configuration parameters
"""
addresses = list()
for nic in NetworkInterfaceCard.objects.filter(node=self):
for address in NetworkAddress.objects.filter(nic=nic):
data = str(address.ip) + " on " + str(nic.dev)
if address.is_carp:
data = " (CARP vhid = {})".format(address.carp_vhid)
addresses.append(data)
conf = {
'id': str(self.id),
'name': self.name,
'pf_limit_states': self.pf_limit_states,
'pf_limit_frags': self.pf_limit_frags,
'pf_limit_src': self.pf_limit_src,
'addresses': addresses,
'is_master_redis': self.is_master_redis,
'is_master_mongo': self.is_master_mongo,
'is_standalone': self.is_standalone
}
return conf
@property
def is_standalone(self):
"""
Check if the current Node is a member of mongoDB
:return: True / False, or None in case of a failure
"""
c = MongoBase()
ok = c.connect()
if ok:
c.connect_primary()
config = c.db.admin.command("replSetGetConfig")['config']
return len(config['members']) == 1
return True
@property
def is_master_mongo(self):
"""
Check if the current Node is master or not
:return: True / False, or None in case of a failure
"""
c = MongoBase()
ok = c.connect()
if ok:
primary_node = c.get_primary()
else:
return None
if ok and primary_node == self.name + ':9091':
return True
elif ok:
return False
return None
@property
def is_master_redis(self):
"""
Check if the current Node is master or not
:return: True / False, or None in case of a failure
"""
if self.management_ip:
c = RedisBase()
master_node = c.get_master(self.name)
if master_node == self.name:
return True
elif master_node:
return False
return None
@property
def is_configured(self):
if self.management_ip:
return True
return False
def write_management_ip(self):
""" Write self.management_ip in management.ip files (jails and host) """
""" First write on host's file """
api_res = self.api_request(
"system.config.models.write_conf",
[MANAGEMENT_IP_PATH, self.management_ip, "root:wheel", "644"])
""" Then, write into jails same path """
for jail in JAILS:
api_res = self.api_request("system.config.models.write_conf", [
"/zroot/{}{}".format(jail, MANAGEMENT_IP_PATH),
self.management_ip, "root:wheel", "644"
])
""" Returns the last api request """
return api_res
def addresses(self, nic=None):
"""
Return the list of network addresses on the current node, or node/nic
:param nic: Optional NIC
:return: list
"""
addresses = list()
if not nic:
for nic in NetworkInterfaceCard.objects.filter(node=self):
addresses.extend(list(NetworkAddress.objects.filter(nic=nic)))
else:
addresses.extend(list(NetworkAddress.objects.filter(nic=nic)))
return addresses
@property
def network_interfaces(self):
return list(NetworkInterfaceCard.objects.filter(node=self))
@property
def get_listeners_enabled(self):
""" Return all listeners used by an enabled Frontend """
from services.frontend.models import Listener
""" Retrieve network addresses on the current node """
addresses = self.addresses()
""" Retrieve all Listeners that uses those network addresses """
listeners = Listener.objects.filter(network_address__in=addresses,
frontend__enabled=True)
listeners_enabled = list()
for l in listeners:
""" Protocol is tcp in any case,
except for frontend mode=log => proto will be tcp or udp or tcp,udp
listening_mode attribute """
proto = "tcp"
if l.frontend.mode == "log" and "udp" in l.frontend.listening_mode:
proto = l.frontend.listening_mode
""" Return (ip, port, interface.family) """
listeners_enabled.append(
(l.whitelist_ips, l.network_address.ip, l.port, l.rsyslog_port,
proto, l.network_address.family, l.max_src, l.max_rate))
return listeners_enabled
@property
def get_forwarders_enabled(self):
""" Return all tuples (family, proto, ip, port) for each LogForwarders """
from applications.logfwd.models import LogOMRELP, LogOMHIREDIS, LogOMFWD, LogOMElasticSearch, LogOMMongoDB
# !!! REQUIRED BY listener_set !
from services.frontend.models import Listener
result = list()
""" Retrieve LogForwarders used in enabled Frontends """
addresses = self.addresses()
""" For each address, retrieve the associated listeners having frontend enabled """
listener_addr = dict()
for a in addresses:
try:
listener_addr[a].append([
l.id for l in a.listener_set.filter(frontend__enabled=True)
])
except KeyError:
listener_addr[a] = [
l.id for l in a.listener_set.filter(frontend__enabled=True)
]
""" Loop on each NetworkAddress to retrieve LogForwarder used by the frontend using listeners """
for network_address, listener_ids in listener_addr.items():
# Log Forwarder RELP
for logfwd in LogOMRELP.objects.filter(
enabled=True, frontend_set__listener__in=listener_ids):
if (network_address.family, "tcp", logfwd.target,
logfwd.port) not in result:
result.append((network_address.family, "tcp",
logfwd.target, logfwd.port)) # TCP
# Log Forwarder REDIS
for logfwd in LogOMHIREDIS.objects.filter(
enabled=True, frontend_set__listener__in=listener_ids):
if (network_address.family, "tcp", logfwd.target,
logfwd.port) not in result:
result.append((network_address.family, "tcp",
logfwd.target, logfwd.port)) # TCP
# Log Forwarder Syslog
for logfwd in LogOMFWD.objects.filter(
enabled=True, frontend_set__listener__in=listener_ids):
if (network_address.family, logfwd.protocol, logfwd.target,
logfwd.port) not in result:
result.append((network_address.family, logfwd.protocol,
logfwd.target, logfwd.port)) # proto
# Log Forwarder ElasticSearch
for logfwd in LogOMElasticSearch.objects.filter(
enabled=True, frontend_set__listener__in=listener_ids):
""" For elasticsearch, we need to parse the servers """
for ip, port in re_findall("https?://([^:]+):(\d+)",
logfwd.server):
if ("inet6" if ':' in ip else "inet", "tcp", ip,
port) not in result:
result.append(("inet6" if ':' in ip else "inet", "tcp",
ip, port))
# Log Forwarder MongoDB
for logfwd in LogOMMongoDB.objects.filter(
enabled=True, frontend_set__listener__in=listener_ids):
""" For OMMongoDB - parse uristr """
for ip, port in parse_uristr(logfwd.uristr):
result.append((network_address.family, 'tcp', ip, port))
return result
@property
def get_backends_enabled(self):
""" Return all tuples (family, proto, ip, port) for each enabled Backends on this node """
from applications.backend.models import Backend
# !!! REQUIRED BY listener_set !
from services.frontend.models import Listener
result = list()
""" Retrieve Backends used in enabled Frontends """
addresses = self.addresses()
""" For each address, retrieve the associated listeners having frontend enabled """
listener_addr = dict()
for a in addresses:
try:
listener_addr[a].append([
l.id for l in a.listener_set.filter(frontend__enabled=True)
])
except KeyError:
listener_addr[a] = [
l.id for l in a.listener_set.filter(frontend__enabled=True)
]
""" Loop on each NetworkAddress to retrieve LogForwarder used by the frontend using listeners """
for network_address, listener_ids in listener_addr.items():
for backend in Backend.objects.filter(
enabled=True, frontend__listener__in=listener_ids):
for server in backend.server_set.all():
result.append((network_address.family, "tcp",
server.target, server.port)) # TCP
return result
def process_messages(self):
"""
Function called from Cluster daemon: read
queue and process asked functions
:return:
"""
logger_daemon = logging.getLogger('daemon')
messages = MessageQueue.objects.filter(
node=self, status='new').order_by('modified')
for message in messages:
logger.debug("Cluster::process_messages: {},{},{}".format(
message.action, message.config, message.node))
message.status = 'running'
message.save()
try:
""" Big try in case of import or execution error """
# Call the function
my_function = import_string(message.action)
args = [logger_daemon]
if message.config:
args.append(message.config)
message.result = my_function(*args)
message.status = 'done'
except ServiceExit:
""" Service stop asked """
raise
except KeyError as e:
logger.exception(e)
message.result = "KeyError {}".format(str(e))
except Exception as e:
logger.exception(e)
logger.error("Cluster::process_messages: {}".format(str(e)))
message.status = 'failure'
message.result = str(e)
message.save()