class FilterPolicy(models.Model):
filter = models.ForeignKey(DarwinFilter, on_delete=models.CASCADE)
policy = models.ForeignKey(DarwinPolicy, on_delete=models.CASCADE)
enabled = models.BooleanField(default=False)
nb_thread = models.PositiveIntegerField(default=5)
log_level = models.TextField(default=DARWIN_LOGLEVEL_CHOICES[0][0],
choices=DARWIN_LOGLEVEL_CHOICES)
threshold = models.PositiveIntegerField(default=80)
mmdarwin_enabled = models.BooleanField(default=False)
mmdarwin_parameters = models.ListField(default=[])
""" Status of filter for each nodes """
status = models.DictField(default={})
cache_size = models.PositiveIntegerField(
default=1000,
help_text=_("The cache size to use for caching darwin requests."),
verbose_name=_("Cache size"))
"""Output format to send to next filter """
output = models.TextField(default=DARWIN_OUTPUT_CHOICES[0][0],
choices=DARWIN_OUTPUT_CHOICES)
""" Next filter in workflow """
next_filter = models.ForeignKey('self',
null=True,
default=None,
on_delete=models.SET_NULL)
conf_path = models.TextField()
config = models.DictField(default={})
@property
def name(self):
""" Method used in Darwin conf to define a filter """
return "{}_{}".format(self.filter.name, self.policy.id)
def __str__(self):
return "[{}] {}".format(self.policy.name, self.filter.name)
@staticmethod
def str_attrs():
return ['filter', 'conf_path', 'nb_thread', 'log_level', 'config']
@property
def socket_path(self):
return "{}/{}_{}.sock".format(SOCKETS_PATH, self.filter.name,
self.policy.id)
def mmdarwin_parameters_rsyslog_str(self):
return str(self.mmdarwin_parameters).replace("\'", "\"")
class Coupon(models.Model):
ptcs = models.DictField()
ssdkl = models.CharField(max_length=200)
origin = models.CharField(max_length=3)
destination = models.CharField(max_length=3)
departure = models.DateField()
campaign = models.CharField(max_length=20, default="base")
class DictEntry(models.Model):
_id = models.ObjectIdField()
headline = models.CharField(max_length=255)
blog = models.DictField()
def __str__(self):
return self.headline
class Client(models.Model):
_id = models.ObjectIdField(default=None)
username = models.CharField(max_length=64)
email = models.CharField(max_length=200, default="")
memberships = models.DictField(default={})
objects = models.DjongoManager()
class DatasetImage(models.Model):
_id = models.ObjectIdField()
name = models.CharField(max_length=128, null=False, blank=False)
url = models.URLField(blank=True, null=True)
created = models.DateTimeField(auto_now_add=True)
label = models.CharField(max_length=32, null=False, blank=False)
metadata = models.DictField(null=True)
class Node(models.Model):
_id = models.ObjectIdField(default=None)
node_name = models.CharField(max_length=64)
creators_name = models.CharField(max_length=128, default="")
members = models.DictField(default={})
current_ip_address = models.CharField(max_length=24, default='0.0.0.0')
objects = models.DjongoManager()
class Student(models.Model):
first_name = models.CharField(max_length=100)
last_name = models.CharField(max_length=100)
eid = models.CharField(max_length=20)
phone = models.CharField(max_length=12)
email = models.EmailField(max_length=100)
linkedIn = models.CharField(max_length=100, blank=True)
resume_link = models.URLField(max_length=100, blank=True)
intentions = models.DictField(default={})
interests = models.DictField(default={})
time_commitment = models.CharField(max_length=100)
international = models.BooleanField(default=False)
fin_aid = models.BooleanField(default=False)
transportation = models2.BooleanField(default=False)
flexible_hours = models2.BooleanField(default=False)
work_remotely = models2.BooleanField(default=False)
other_availability = models.TextField(max_length=500, blank=True)
school = models.DictField(default={})
program = models.DictField(default={})
experience = models.DictField(default={})
tech_skills = models.DictField(default={})
prof_skills = models.DictField(default={})
other_skills = models.TextField(max_length=500, blank=True)
cohort = models.CharField(max_length=100)
unique_id = models.CharField(max_length=100, unique=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
hear = models.CharField(max_length=100)
class CustomUser(AbstractUser):
role = models.CharField(blank=True, max_length=50, default='user')
dashboards = models.ListField(default=[])
# default_dashboard = models.CharField(blank=True, null=True, max_length=200)
default_dashboard = models.DictField(default={})
grants = models.ListField(default=[])
def __str__(self):
return self.username
class LogViewerSearches(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE)
type_logs = models.TextField()
name = models.TextField()
search = models.DictField()
def to_template(self):
return {
'pk': str(self.pk),
'type_logs': self.type_logs,
'name': self.name,
'search': self.search
}
class Project(models.Model):
contact_first_name = models.CharField(max_length=100)
contact_last_name = models.CharField(max_length=100)
contact_phone = models.CharField(max_length=12)
contact_email = models.EmailField(max_length=100)
organization_name = models.CharField(max_length=100)
organization_address = models.CharField(max_length=200)
organization_website = models.CharField(max_length=100)
project_name = models.CharField(max_length=100)
project_description = models.TextField(max_length=1000)
project_categories = models.DictField(default={})
time_commitment = models.CharField(max_length=100)
transportation = models.BooleanField(default=False)
flexible_hours = models.BooleanField(default=False)
work_remotely = models.BooleanField(default=False)
school = models.DictField(default={})
experience = models.DictField(default={})
tech_skills = models.DictField(default={})
prof_skills = models.DictField(default={})
other_skills = models.TextField(max_length=500, blank=True)
cohort = models.CharField(max_length=100)
unique_id = models.CharField(max_length=300)
created_at = models.DateTimeField(auto_now_add=True)
class Wishlist(models.Model):
mapping_id = models.IntegerField(default=0)
brand = models.CharField(max_length=50)
description = models.ListField(models.CharField(max_length=500))
user = models.ForeignKey(User, on_delete=models.CASCADE)
display_size = models.CharField(max_length=10)
graphics_memory = models.CharField(max_length=30)
img_link = models.URLField(max_length=10000)
product_title = models.CharField(max_length=1000)
ram = models.CharField(max_length=10)
ram_type = models.CharField(max_length=100)
storage = models.DictField()
websites = models.ListField(
models.EmbeddedModelField(model_container=Website))
class Startech(models.Model):
brand = models.CharField(max_length=50)
description = models.ListField(models.CharField(max_length=1000))
display_size = models.CharField(max_length=10)
graphics_memory = models.CharField(max_length=30)
img_link = models.URLField(max_length=10000)
price = models.IntegerField(default=0)
product_link = models.URLField(max_length=10000)
product_title = models.CharField(max_length=1000)
ram = models.CharField(max_length=10)
ram_type = models.CharField(max_length=100)
status = models.CharField(max_length=100)
processor = models.CharField(max_length=100)
storage = models.DictField()
website = models.CharField(max_length=100)
_id = models.CharField(primary_key=True, max_length=100)
class Mapping(models.Model):
brand = models.CharField(max_length=50)
description = models.ListField(models.CharField(max_length=500))
display_size = models.CharField(max_length=10)
graphics_memory = models.CharField(max_length=30)
img_link = models.URLField(max_length=10000)
product_title = models.CharField(max_length=1000)
ram = models.CharField(max_length=10)
ram_type = models.CharField(max_length=100)
storage = models.DictField()
_id = models.CharField(max_length=100)
websites = models.ListField(
models.EmbeddedModelField(
model_container=Website
)
)
id = models.IntegerField(primary_key=True)
class LogViewerConfiguration(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE)
type_logs = models.TextField()
displayed_columns = models.DictField()
nb_lines = models.IntegerField(default=25)
font_size = models.IntegerField(default=12)
class Meta:
unique_together = ('user', 'type_logs')
def to_template(self):
return {
'pk': str(self.pk),
'type_logs': self.type_logs,
'displayed_columns': self.displayed_columns,
'nb_lines': self.nb_lines,
'font_size': self.font_size
}
class DefenderProcessRule(models.Model):
job_id = models.TextField()
expiration_date = models.DateTimeField(auto_now_add=True)
rule_id = models.IntegerField()
rule_key = models.TextField(default="")
data = models.DictField(default={})
objects = models.DjongoManager()
def __init__(self, *args, ** kwargs):
DefenderProcessRule.objects._client.ensure_index('expiration_date', expireAfterSeconds=10 * 60)
super(DefenderProcessRule, self).__init__(*args, **kwargs)
def to_dict(self):
return {
"job_id": self.job_id,
"expiration_date": self.expiration_date,
"rule_id": self.rule_id,
"rule_key": self.rule_key,
"data": self.data
}
class Backend(models.Model):
""" Model used to generated fontends configuration of HAProxy """
""" Is that section enabled or disabled """
enabled = models.BooleanField(
default=True,
help_text=_("Enable the backend"),
)
""" Name of the frontend, unique constraint """
name = models.TextField(
unique=True,
default="Backend",
help_text=_("Name of HAProxy backend"),
)
""" Listening mode of Frontend : tcp or http """
mode = models.TextField(
default=MODE_CHOICES[0][0],
choices=MODE_CHOICES,
help_text=_("Proxy mode"),
)
timeout_connect = models.PositiveIntegerField(
default=2000,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
timeout_server = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(3600)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Save of generated configuration """
configuration = models.TextField(default="{}")
""" Status of frontend for each nodes """
status = models.DictField(default={})
""" Headers """
headers = models.ArrayReferenceField(Header,
null=True,
blank=False,
on_delete=models.CASCADE,
help_text=_("Header rules"))
""" Custom HAProxy Backend directives """
custom_haproxy_conf = models.TextField(
default="", help_text=_("Custom HAProxy configuration directives."))
""" HTTP Options """
""" URI of backends """
http_backend_dir = models.TextField(
default="/",
help_text=_("Servers directory to prefix in front of requests uri"),
verbose_name=_("Servers directory"))
""" Enable or disable relaxing of HTTP response parsing """
accept_invalid_http_response = models.BooleanField(
default=False,
help_text=_("Enable relaxing of HTTP response parsing"),
verbose_name=_("Accept invalid HTTP response"))
""" Enable insertion of the X-Forwarded-For header to requests sent to servers """
http_forwardfor_header = models.TextField(
blank=True,
null=True,
help_text=_("Insertion of the X-Forwarded-For header"),
verbose_name=_("Send source ip in "))
""" Except the following IP"""
http_forwardfor_except = models.GenericIPAddressField(
protocol='IPv4',
blank=True,
null=True,
help_text=_("Except the specified IP address"),
verbose_name=_("Except for "))
""" Enable HTTP protocol to check on the servers health """
enable_http_health_check = models.BooleanField(
default=False,
help_text=_("Enable HTTP protocol health checker"),
verbose_name=_("HTTP health check"))
""" The optional HTTP method used with the requests """
http_health_check_method = models.TextField(
default=HEALTH_CHECK_METHOD_CHOICES[0][0],
choices=HEALTH_CHECK_METHOD_CHOICES,
help_text=_("HTTP method used"),
verbose_name=_("Method"))
""" The URI referenced in the HTTP requests """
http_health_check_uri = models.TextField(default='/',
blank=True,
null=True,
help_text=_("URI referenced"),
verbose_name=_("URI"))
""" The optional HTTP version string """
http_health_check_version = models.TextField(
default=HEALTH_CHECK_VERSION_CHOICES[0][0],
choices=HEALTH_CHECK_VERSION_CHOICES,
help_text=_("HTTP version"),
verbose_name=_("Version"))
""" """
http_health_check_headers = models.DictField(
default={},
help_text=_("HTTP Health Check Headers"),
verbose_name=_("HTTP Health Check Headers"))
""" Health check expect """
http_health_check_expect_match = models.TextField(
choices=HEALTH_CHECK_EXPECT_CHOICES,
default=HEALTH_CHECK_EXPECT_CHOICES[0][0],
null=True,
help_text=_("Type of match to expect"),
verbose_name=_("HTTP Health Check expected"))
http_health_check_expect_pattern = models.TextField(
default="200",
help_text=_("Type of pattern to match to expect"),
verbose_name=_("HTTP Health Check expected pattern"))
""" Enable or disable HTTP keep-alive from client to server """
enable_http_keep_alive = models.BooleanField(
default=True,
help_text=_("Enable HTTP keep-alive"),
verbose_name=_("HTTP Keep alive"))
""" Keep-alive Timeout """
http_keep_alive_timeout = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Balancing mode """
balancing_mode = models.TextField(
choices=BALANCING_CHOICES,
default=BALANCING_CHOICES[0][0],
help_text=_("Balancing mode between servers"),
verbose_name=_("Balancing mode"))
""" Balancing param """
balancing_param = models.TextField(
null=True,
default="",
help_text=_("Balancing param for balancing mode"),
verbose_name=_("Balancing parameter"))
""" Tags """
tags = models.ListField(
models.SlugField(default=""),
default=[],
help_text=_("Tags to set on this object for search"))
@property
def balancing(self):
if self.balancing_mode == "hdr":
result = "hdr({})".format(self.balancing_param)
elif self.balancing_mode == "url_param":
result = "url_param {}".format(self.balancing_param)
elif self.balancing_mode == "rdp-cookie":
result = "rdp-cookie({})".format(self.balancing_param)
else:
result = self.balancing_mode
return result
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name', 'mode']
def __str__(self):
return "{} Backend '{}'".format(self.mode.upper(), self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models like Listeners
:return A JSON object
"""
result = {
'id': self.id,
'enable': self.enabled,
'name': self.name,
'mode': self.mode,
'balancing_mode': self.balancing_mode,
'balancing_param': self.balancing_param,
'status': dict(self.status), # It is an OrderedDict
'servers': [],
'custom_haproxy_conf': self.custom_haproxy_conf,
'tags': self.tags
}
""" Add listeners """
for server in self.server_set.all():
s = server.to_template()
# Remove frontend to prevent infinite loop
del s['backend']
result['servers'].append(s)
""" Other attributes """
if self.mode == "http":
result['headers'] = []
for header in self.headers.all():
result['headers'].append(header.to_template())
return result
def to_html_template(self):
""" Dictionary used to render object as html
:return Dictionnary of configuration parameters
"""
""" Retrieve list/custom objects """
servers_list = [
str(s) for s in self.server_set.all().only(*Server.str_attrs())
]
mode = "UNKNOWN"
for m in MODE_CHOICES:
if self.mode == m[0]:
mode = m[1]
balancing_mode = "unknown"
for m in BALANCING_CHOICES:
if self.balancing_mode == m[0]:
balancing_mode = m[1]
additional_infos = "Balancing mode : {}".format(balancing_mode)
""" And returns the attributes of the class """
return {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'servers': servers_list,
'mode': mode,
'status': self.status,
'additional_infos': additional_infos,
'tags': self.tags
}
def to_template(self, server_list=None, header_list=None):
""" Dictionary used to create configuration file
:return Dictionnary of configuration parameters
"""
workflow_list = []
access_controls_list = []
for workflow in self.workflow_set.filter(enabled=True):
tmp = {
'id': str(workflow.pk),
'fqdn': workflow.fqdn,
'public_dir': workflow.public_dir,
'backend': workflow.backend,
'defender_policy': workflow.defender_policy
}
access_controls_deny = []
access_controls_301 = []
access_controls_302 = []
for acl in workflow.workflowacl_set.filter(before_policy=False):
rules, acl_name = acl.access_control.generate_rules()
access_controls_list.append(rules)
condition = acl.generate_condition(acl_name)
redirect_url = None
deny = False
for type_acl in ('action_satisfy', 'action_not_satisfy'):
action = getattr(acl, type_acl)
if action != "200":
if action in ("301", "302"):
redirect_url = getattr(
acl, type_acl.replace('action',
'redirect_url'))
elif action == "403":
deny = True
break
tmp_acl = {
'before_policy': acl.before_policy,
'redirect_url': redirect_url,
'conditions': condition,
'action': action,
'deny': deny
}
if action == "403":
access_controls_deny.append(tmp_acl)
elif action == "301":
access_controls_301.append(tmp_acl)
elif action == "302":
access_controls_302.append(tmp_acl)
tmp['access_controls_deny'] = access_controls_deny
tmp['access_controls_302'] = access_controls_302
tmp['access_controls_301'] = access_controls_301
workflow_list.append(tmp)
""" Simple attributes of the class """
result = {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'mode': self.mode,
'timeout_connect': self.timeout_connect,
'timeout_server': self.timeout_server,
'unix_socket': self.get_unix_socket(),
'custom_haproxy_conf': self.custom_haproxy_conf,
'JAIL_ADDRESSES': JAIL_ADDRESSES,
'accept_invalid_http_response': self.accept_invalid_http_response,
'http_forwardfor_header': self.http_forwardfor_header,
'http_forwardfor_except': self.http_forwardfor_except,
'enable_http_health_check': self.enable_http_health_check,
'http_health_check_method': self.http_health_check_method,
'http_health_check_uri': self.http_health_check_uri,
'http_health_check_version': self.http_health_check_version,
'http_health_check_headers': self.http_health_check_headers,
'enable_http_keep_alive': self.enable_http_keep_alive,
'http_keep_alive_timeout': self.http_keep_alive_timeout,
'access_controls_list': set(access_controls_list),
'http_backend_dir': self.http_backend_dir,
'balancing': self.balancing,
'workflows': workflow_list,
'tags': self.tags
}
""" Retrieve list/custom objects """
# If facultative arg listener_list is not given
if not server_list:
# Retrieve all the objects used by the current frontend
# No .only ! Used to generated conf, neither str cause we need the object
result['servers'] = self.server_set.all(
) # No .only() ! Used to generated conf
if self.mode == "http":
# Same for headers
result['headers'] = self.headers.all(
) if not header_list else header_list
if self.enable_http_health_check and self.http_health_check_expect_match:
result['http_health_check_expect'] = "{} {}".format(
self.http_health_check_expect_match,
self.http_health_check_expect_pattern)
return result
def generate_conf(self, server_list=None, header_list=None):
""" Render the conf with Jinja template and self.to_template() method
:return The generated configuration as string, or raise
"""
# The following var is only used by error, do not forget to adapt if needed
template_name = JINJA_PATH + JINJA_TEMPLATE
try:
jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH))
template = jinja2_env.get_template(JINJA_TEMPLATE)
return template.render({
'conf':
self.to_template(server_list=server_list,
header_list=header_list)
})
# In ALL exceptions, associate an error message
# The exception instantiation MUST be IN except statement, to retrieve traceback in __init__
except TemplateNotFound:
exception = ServiceJinjaError(
"The following file cannot be found : '{}'".format(
template_name), "haproxy")
except TemplatesNotFound:
exception = ServiceJinjaError(
"The following files cannot be found : '{}'".format(
template_name), "haproxy")
except (TemplateAssertionError, TemplateRuntimeError):
exception = ServiceJinjaError(
"Unknown error in template generation: {}".format(
template_name), "haproxy")
except UndefinedError:
exception = ServiceJinjaError(
"A variable is undefined while trying to render the following template: "
"{}".format(template_name), "haproxy")
except TemplateSyntaxError:
exception = ServiceJinjaError(
"Syntax error in the template: '{}'".format(template_name),
"haproxy")
# If there was an exception, raise a more general exception with the message and the traceback
raise exception
def test_conf(self):
""" Write the configuration attribute on disk, in test directory, as {id}.conf.new
And test the conf with 'haproxy -c'
:return True or raise
"""
""" No need to do API request cause Backends are not relative-to-node objects """
test_filename = self.get_test_filename()
conf = self.configuration
# NO Node-specific configuration, we can test-it on local node
# Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True
test_haproxy_conf(test_filename,
conf.replace(
"backend {}".format(self.name),
"backend test_{}".format(self.id or "test")),
disabled=True)
def get_test_filename(self):
""" Return test filename for test conf with haproxy
"""
""" If object is not already saved : no id so default=test """
return "backend_{}.conf".format(self.id or "test")
def get_base_filename(self):
""" Return the base filename (without path) """
return "backend_{}.cfg".format(self.id)
def get_filename(self):
""" Return filename depending on current frontend object
"""
return "{}/{}".format(HAPROXY_PATH, self.get_base_filename())
def get_unix_socket(self):
""" Return filename of unix socket on which HAProxy send data
and on which Rsyslog listen
"""
return "{}/backend_{}.sock".format(UNIX_SOCKET_PATH, self.id)
def save_conf(self):
""" Write configuration on disk
"""
if not self.configuration:
return
params = [
self.get_filename(), self.configuration, BACKEND_OWNER,
BACKEND_PERMS
]
try:
Cluster.api_request('system.config.models.write_conf',
config=params)
except Exception as e: # e used by VultureSystemConfigError
logger.error(e, exc_info=1)
raise VultureSystemConfigError(
"on cluster.\nRequest failure to write_conf()")
def reload_conf(self):
""" Generate conf and save it """
self.configuration = self.generate_conf()
self.save_conf()
self.save()
def enable(self):
""" Enable backend in HAProxy, by management socket """
""" Cannot enable a disabled frontend ('enable' field) """
if not self.enabled:
raise ServiceError(
"Cannot start a disabled backend.",
"haproxy",
"enable backend",
traceback="Please edit, enable and save a backend to start-it."
)
return hot_action_backend(self.name, "enable")
def disable(self):
""" Disable frontend in HAProxy, by management socket """
if not self.enabled:
return "This frontend is already disabled."
""" If it is an Rsyslog only conf, return error """
if self.mode == "log" and self.listening_mode == "udp":
raise ServiceError(
"Cannot hot disable an Rsyslog only frontend.",
"rsyslog",
"disable frontend",
traceback="Please edit, disable and save the frontend.")
return hot_action_backend(self.name, "disable")
class Result(models.Model):
created_at = models.DateTimeField(auto_now_add=True)
cohort = models.CharField(max_length=100)
email = models.EmailField(null=True, blank=True)
data = models.DictField(default={})
class Strongswan(models.Model):
node = models.OneToOneField(Node, on_delete=models.CASCADE)
enabled = models.BooleanField(default=False)
ipsec_type = models.TextField(choices=TYPE, default="tunnel")
ipsec_keyexchange = models.TextField(choices=KEYEXCHANGE, default="ikev2")
ipsec_authby = models.TextField(choices=AUTHBY, default="secret")
ipsec_psk = models.TextField(default="V3ryStr0ngP@ssphr@se")
ipsec_fragmentation = models.BooleanField(default=True)
ipsec_forceencaps = models.BooleanField(default=False)
ipsec_ike = models.TextField(default="aes256-sha512-modp8192")
ipsec_esp = models.TextField(default="aes256-sha512-modp8192")
ipsec_dpdaction = models.TextField(choices=DPD, default="restart")
ipsec_dpddelay = models.TextField(default="35s")
ipsec_rekey = models.BooleanField(default=True)
ipsec_ikelifetime = models.TextField(default="3h")
ipsec_keylife = models.TextField(default="1h")
ipsec_right = models.TextField(default="")
ipsec_leftsubnet = models.TextField(default="")
ipsec_leftid = models.TextField(default="")
ipsec_rightsubnet = models.TextField(default="")
status = models.TextField(default="WAITING")
statusall = models.TextField(default="")
tunnels_status = models.DictField(default={})
tunnels_up = models.PositiveIntegerField(default=0)
tunnels_connecting = models.PositiveIntegerField(default=0)
def to_template(self):
""" Dictionary used to create configuration file.
:return: Dictionary of configuration parameters
"""
return {'conf': self.to_html_template()}
def to_dict(self):
""" Serialized version of object """
return {
'id': str(self.id),
'node': self.node.to_dict(),
'status': self.status,
'statusall': self.statusall,
'ipsec_type': self.ipsec_type,
'ipsec_keyexchange': self.ipsec_keyexchange,
'ipsec_authby': self.ipsec_authby,
'ipsec_psk': self.ipsec_psk,
'ipsec_fragmentation': self.ipsec_fragmentation,
'ipsec_forceencaps': self.ipsec_forceencaps,
'ipsec_ike': self.ipsec_ike,
'ipsec_esp': self.ipsec_esp,
'ipsec_dpdaction': self.ipsec_dpdaction,
'ipsec_dpddelay': self.ipsec_dpddelay,
'ipsec_rekey': self.ipsec_rekey,
'ipsec_ikelifetime': self.ipsec_ikelifetime,
'ipsec_keylife': self.ipsec_keylife,
'ipsec_right': self.ipsec_right,
'ipsec_leftsubnet': self.ipsec_leftsubnet,
'ipsec_leftid': self.ipsec_leftid,
'ipsec_rightsubnet': self.ipsec_rightsubnet
}
def get_status(self):
return {
'node': self.node.to_dict(),
'status': self.status,
'statusall': self.statusall,
'tunnels_status': self.tunnels_status,
'tunnels_up': self.tunnels_up,
'tunnels_connecting': self.tunnels_connecting
}
def to_html_template(self):
""" Dictionary used to render FORM
:return Dictionnary of configuration parameters
"""
""" And returns the attributes of the class """
return {
'id': str(self.id),
'node': self.node.name,
'status': self.status,
'statusall': self.statusall,
'ipsec_type': self.ipsec_type,
'ipsec_keyexchange': self.ipsec_keyexchange,
'ipsec_authby': self.ipsec_authby,
'ipsec_psk': self.ipsec_psk,
'ipsec_fragmentation': self.ipsec_fragmentation,
'ipsec_forceencaps': self.ipsec_forceencaps,
'ipsec_ike': self.ipsec_ike,
'ipsec_esp': self.ipsec_esp,
'ipsec_dpdaction': self.ipsec_dpdaction,
'ipsec_dpddelay': self.ipsec_dpddelay,
'ipsec_rekey': self.ipsec_rekey,
'ipsec_ikelifetime': self.ipsec_ikelifetime,
'ipsec_keylife': self.ipsec_keylife,
'ipsec_right': self.ipsec_right,
'ipsec_leftsubnet': self.ipsec_leftsubnet,
'ipsec_leftid': self.ipsec_leftid,
'ipsec_rightsubnet': self.ipsec_rightsubnet,
'tunnels_status': self.tunnels_status,
'tunnels_up': self.tunnels_up,
'tunnels_connecting': self.tunnels_connecting,
}
def generate_conf(self):
""" Render the conf with Jinja template and self.to_template() method
:return The generated configuration as string, or raise
"""
try:
jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH))
template_ipsec = jinja2_env.get_template(JINJA_TEMPLATE_IPSEC)
template_secrets = jinja2_env.get_template(JINJA_TEMPLATE_SECRETS)
template_strongswan = jinja2_env.get_template(
JINJA_TEMPLATE_STRONGSWAN)
conf = self.to_template()
return {
'template_ipsec': template_ipsec.render(conf),
'template_secrets': template_secrets.render(conf),
'template_strongswan': template_strongswan.render(conf)
}
# In ALL exceptions, associate an error message
except Exception as e:
# If there was an exception, raise a more general exception with the message and the traceback
raise ServiceJinjaError(
"Strongswan config generation error: '{}'".format(str(e)),
"strongswan")
def save_conf(self):
""" Write configuration on disk
"""
conf = self.generate_conf()
params_ipsec = [
'/usr/local/etc/ipsec.conf', conf['template_ipsec'],
STRONGSWAN_OWNER, "644"
]
params_secrets = [
'/usr/local/etc/ipsec.secrets', conf['template_secrets'],
STRONGSWAN_OWNER, "600"
]
params_strongswan = [
'/usr/local/etc/strongswan.conf', conf['template_strongswan'],
STRONGSWAN_OWNER, "644"
]
for params in (params_ipsec, params_secrets, params_strongswan):
try:
self.node.api_request('system.config.models.write_conf',
config=params)
except Exception as e:
raise VultureSystemConfigError(
"on node '{}'.\nRequest failure.".format(self.node.name))
class Tags(models.Model):
#No Mongo object ID is needed here, as the name of the tag will be able to serve as the primary key (which nicely prevents two people creating different tags for the same thing, though machine_learning, machinelearning, machine-learning, machineLearning and MachineLearing would all be able to )
# be stored unless we use some sanitization on tag creation, say enforce lowercase and no punctutation, or select from set (which uses a different backend method)
name = models.DictField(default={})
objects = models.DjongoManager()
class Seats(models.Model):
STATE = Choices((0, 'vacant', 'vacant'), (1, 'occupied', 'occupied'))
DEFAULT_LAYOUT = [
[
'1A',
'1B',
'1C',
'1D',
'1E',
'1F',
'1G',
],
[
'2A',
'2B',
'2C',
'2D',
'2E',
'2F',
'2G',
],
[
'3A',
'3B',
'3C',
'3D',
'3E',
'3F',
'3G',
],
[
'4A',
'4B',
'4C',
'4D',
'4E',
'4F',
'4G',
],
[
'5A',
'5B',
'5C',
'5D',
'5E',
'5F',
'5G',
],
[
'6A',
'6B',
'6C',
'6D',
'6E',
'6F',
'6G',
],
[
'7A',
'7B',
'7C',
'7D',
'7E',
'7F',
'7G',
],
[
'8A',
'8B',
'8C',
'8D',
'8E',
'8F',
'8G',
],
[
'9A',
'9B',
'9C',
'9D',
'9E',
'9F',
'9G',
],
]
layout = models.ListField(default=DEFAULT_LAYOUT)
states = models.DictField(default={})
def set_state(self, seat_id, state):
self.states[seat_id] = state
return self.states[seat_id]
def __str__(self):
return str(self.layout)
class Meta:
abstract = True
class ReputationContext(models.Model):
""" Model used to enrich logs in Rsyslog with mmdb database"""
name = models.TextField(
default="Reputation context",
verbose_name=_("Friendly name"),
help_text=_("Custom name of the current object"),
)
""" Database type """
db_type = models.TextField(
default=DBTYPE_CHOICES[0][0],
choices=DBTYPE_CHOICES,
verbose_name=_("Database type"),
help_text=_("Type of database"),
)
method = models.SlugField(
default=HTTP_METHOD_CHOICES[0][0],
choices=HTTP_METHOD_CHOICES,
verbose_name=_("HTTP method to use"),
help_text=_("HTTP method to use while retrieving url"))
url = models.URLField(help_text=_("URL to retrieve the database from"),
verbose_name=_("Database URL"))
verify_cert = models.BooleanField(
default=True,
help_text=
_("Verify certificate to prevent connexion to self-signed certificates."
),
verbose_name=_("Verify server certificate"))
post_data = models.TextField(default="",
null=True,
verbose_name=_("POST body"),
help_text=_("Body to send if method is POST"))
custom_headers = models.DictField(
default={},
verbose_name=_("Custom headers"),
help_text=_("Headers to send while retrieving url"))
auth_type = models.TextField(
default=HTTP_AUTH_TYPE_CHOICES[0][0],
choices=HTTP_AUTH_TYPE_CHOICES,
verbose_name=_("Authentication"),
help_text=_("Authentication type used to retrieve url"))
user = models.SlugField(default=None,
null=True,
verbose_name=_("Username"),
help_text=_("Username to use for authentication"))
password = models.TextField(
default=None,
null=True,
verbose_name=_("Password"),
help_text=_("Password to use for authentication"))
tags = models.ListField(
models.SlugField(default=""),
default=[],
help_text=_("Tags to set on this object for search"))
""" Field not stored in DB, it's just used as cache between fonction classes """
content = models.BinaryField(default="")
""" MMDB database attributes """
# There cannot be multiple files with the same filename
filename = models.FilePathField(path=DATABASES_PATH,
default="",
unique=True)
description = models.TextField(default="")
# When saving object, last_update will be automatically updated
last_update = models.DateTimeField(auto_now=True)
nb_netset = models.IntegerField(default=0)
nb_unique = models.IntegerField(default=0)
internal = models.BooleanField(default=False)
""" Use DjongoManager to use mongo_find() & Co """
objects = models.DjongoManager()
def save(self, *args, **kwargs):
""" Override mother fonction to prevent save of content attribute in MongoDB
"""
self.content = ""
super().save(*args, **kwargs)
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name']
def __str__(self):
return "ReputationContext '{}'".format(self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models
:return A JSON object
"""
result = {
'id': self.id,
'name': self.name,
'description': self.description,
'db_type': self.db_type,
'method': self.method,
'url': self.url,
'verify_cert': self.verify_cert,
'post_data': self.post_data,
'auth_type': self.auth_type,
'user': self.user,
'password': self.password,
'custom_headers': self.custom_headers,
'internal': self.internal,
'tags': self.tags
}
return result
def to_html_template(self):
""" Dictionary used to render object as html
:return Dictionnary of configuration parameters
"""
db_type = self.db_type
for d in DBTYPE_CHOICES:
if self.db_type == d[0]:
db_type = d[1]
uri = "{} {}".format(self.method, self.url)
if self.auth_type:
uri += " {}({}:xxx)".format(self.auth_type, self.user)
""" Retrieve list/custom objects """
return {
'id': str(self.id),
'name': self.name,
'db_type': db_type,
'uri': uri,
'internal': self.internal,
'tags': self.tags
}
def to_template(self):
""" Dictionary used to create configuration file
:return Dictionnary of configuration parameters
"""
result = {
'id': str(self.id),
'name': self.name,
'method': self.method,
'url': self.url,
'post_data': self.post_data,
'custom_headers': self.custom_headers,
'tags': self.tags
}
""" Download url """
#result['content'] = self.download_file()
""" And returns the attributes of the class """
return result
def download_file(self):
""" """
""" If we haven't already downloaded url """
if self.content:
return self.content
""" Retrieve url and content """
auth = None
if self.auth_type:
auth_type = AUTH_TYPE_CLASSES.get(self.auth_type)
if auth_type:
auth = auth_type(self.user, self.password)
logger.debug("Try to get URL {}".format(self.url))
try:
response = requests.request(
self.method,
self.url,
data=self.post_data if self.method == "POST" else None,
headers=self.custom_headers,
auth=auth,
allow_redirects=True,
proxies=get_proxy(),
timeout=(2.0, 2.0))
# logger.info("URL '{}' retrieved, status code = {}".format(self.url, response.status_code))
assert response.status_code == 200, "Response code is not 200 ({})".format(
response.status_code)
""" If its a .gz file, dezip-it """
if self.url[-3:] == ".gz":
self.filename = self.url.split('/')[-1][:-3]
return gzip_decompress(response.content)
if response.headers.get("Content-Disposition"):
match = REGEX_GZ.search(
response.headers.get("Content-Disposition"))
if match and match[1][-3:] == ".gz":
self.filename = match[1][:-3]
return gzip_decompress(response.content)
self.filename = self.url.split('/')[-1]
except Exception as e:
raise VultureSystemError(str(e), "download '{}'".format(self.url))
return response.content
def download_mmdb(self):
""" Always call this method first, to be sure the MMDB is OK """
content = self.download_file()
if self.db_type in ("ipv4", "ipv6", "GeoIP"):
try:
return open_mmdb_database(content)
except Exception as e:
logger.error("Downloaded content is not a valid MMDB database")
raise VultureSystemError(
"Downloaded content is not a valid MMDB database",
"download '{}'".format(self.url))
else:
return None
def download(self):
content = self.download_file()
if self.db_type in ("ipv4", "ipv6", "GeoIP"):
db_reader = open_mmdb_database(content)
db_metadata = db_reader.metadata()
db_reader.close()
# Do not erase nb_netset in internal db, its retrieved in index.json
if not self.internal:
self.nb_netset = db_metadata.node_count
else:
self.nb_unique = len(content.decode('utf8').split("\n"))
return content
@property
def absolute_filename(self):
""" Return filename depending on current frontend object
"""
# Escape quotes to prevent injections in config or in commands
return "{}/{}".format(DATABASES_PATH, self.filename.replace('"', '\"'))
def save_conf(self):
""" Write configuration on disk
"""
params = [
self.absolute_filename,
self.download_file(), DATABASES_OWNER, DATABASES_PERMS
]
try:
Cluster.api_request('system.config.models.write_conf',
config=params)
except Exception as e: # e used by VultureSystemConfigError
raise VultureSystemConfigError(
"on cluster.\n"
"Request failure to write conf of Reputation context '{}'".
format(self.name))
def get_nodes(self):
""" Return the list of nodes used by frontends using the current object """
# for listener in Listener.objects.filter()
return Node.objects.filter(
networkinterfacecard__frontend__reputation_ctxs=self.id)
def reload_frontend_conf(self):
""" Send API request on each Nodes using the frontends that uses the current CTX
:return The list of concerned nodes
"""
from services.frontend.models import Listener
res = []
# Loop on Nodes
for node in Node.objects.all():
frontends = []
# Get listeners enabled on this node, using the current reputation context
for listener in Listener.objects.filter(
frontend__enabled=True,
frontend__reputation_ctxs=self.id,
network_address__nic__node=node.id).distinct():
if listener.frontend.id not in frontends:
api_res = node.api_request(
"services.rsyslogd.rsyslog.build_conf",
listener.frontend.id)
if not api_res:
raise ServiceConfigError(
"on node '{}' \n API request error.".format(
node.name),
"rsyslog",
traceback=api_res.get('message'))
frontends.append(listener.frontend.id)
res.append(node)
return res
class FilterPolicy(models.Model):
""" Associated filter template """
filter = models.ForeignKey(
DarwinFilter,
on_delete=models.CASCADE
)
""" Associated policy """
policy = models.ForeignKey(
DarwinPolicy,
on_delete=models.CASCADE
)
""" Is the Filter activated? """
enabled = models.BooleanField(default=False)
""" Number of threads """
nb_thread = models.PositiveIntegerField(default=5)
""" Level of logging (not alerts) """
log_level = models.TextField(default=DARWIN_LOGLEVEL_CHOICES[1][0], choices=DARWIN_LOGLEVEL_CHOICES)
""" Alert detection thresold """
threshold = models.PositiveIntegerField(default=80)
""" Does the filter has a custom Rsyslog configuration? """
mmdarwin_enabled = models.BooleanField(default=False)
""" The custom rsyslog message's fields to get """
mmdarwin_parameters = models.ListField(default=[])
""" Status of filter for each nodes """
status = models.DictField(default={})
""" The number of cache entries (not memory size) """
cache_size = models.PositiveIntegerField(
default=0,
help_text=_("The cache size to use for caching darwin requests."),
verbose_name=_("Cache size")
)
"""Output format to send to next filter """
output = models.TextField(
default=DARWIN_OUTPUT_CHOICES[0][0],
choices=DARWIN_OUTPUT_CHOICES
)
""" Next filter in workflow """
next_filter = models.ForeignKey(
'self',
null=True,
default=None,
on_delete=models.SET_NULL
)
""" The fullpath to its configuration file """
conf_path = models.TextField()
""" A dict representing the filter configuration """
config = models.DictField(default={
"redis_socket_path": "/var/sockets/redis/redis.sock",
"alert_redis_list_name": DARWIN_REDIS_ALERT_LIST,
"alert_redis_channel_name": DARWIN_REDIS_ALERT_CHANNEL,
"log_file_path": "/var/log/darwin/alerts.log"
})
@property
def name(self):
""" Method used in Darwin conf to define a filter """
return "{}_{}".format(self.filter.name, self.policy.id)
def __str__(self):
return "[{}] {}".format(self.policy.name, self.filter.name)
@staticmethod
def str_attrs():
return ['filter', 'conf_path', 'nb_thread', 'log_level', 'config']
@property
def socket_path(self):
return "{}/{}_{}.sock".format(SOCKETS_PATH, self.filter.name, self.policy.id)
def mmdarwin_parameters_rsyslog_str(self):
return str(self.mmdarwin_parameters).replace("\'", "\"")
class Openvpn(models.Model):
node = models.OneToOneField(Node, on_delete=models.CASCADE)
enabled = models.BooleanField(default=False)
remote_server = models.TextField(default="")
remote_port = models.PositiveIntegerField(default=443)
tls_profile = models.ForeignKey(to=TLSProfile,
on_delete=models.CASCADE,
help_text=_("TLS Profile to use."))
proto = models.TextField(choices=PROTO, default="tcp")
status = models.TextField(default="WAITING")
tunnels_status = models.DictField(default={})
def to_template(self):
""" Dictionary used to create configuration file.
:return: Dictionary of configuration parameters
"""
return {'conf': self.to_html_template()}
def to_dict(self):
""" Serialized version of object """
return {
'id': str(self.id),
'node': self.node.to_dict(),
'status': self.status,
'remote_server': self.remote_server,
'remote_port': self.remote_port,
'tls_profile': self.tls_profile,
'proto': self.proto
}
def get_status(self):
return {
'node': self.node.to_dict(),
'status': self.status,
'tunnels_status': self.tunnels_status
}
def to_html_template(self):
""" Dictionary used to render FORM
:return Dictionnary of configuration parameters
"""
""" Retrieve optional proxy configuration """
tmp = get_proxy(True)
if tmp:
proxy_configuration = "http-proxy-retry" + '\n'
proxy_configuration += "http-proxy {} {}".format(tmp[0],
tmp[1]) + '\n'
else:
proxy_configuration = ""
""" And returns the attributes of the class """
return {
'id': str(self.id),
'node': self.node.name,
'remote_server': self.remote_server,
'remote_port': self.remote_port,
'proto': self.proto,
'status': self.status,
'tunnels_status': self.tunnels_status,
'proxy_configuration': proxy_configuration,
'ca':
self.tls_profile.x509_certificate.get_base_filename() + ".chain",
'cert':
self.tls_profile.x509_certificate.get_base_filename() + ".crt",
'key':
self.tls_profile.x509_certificate.get_base_filename() + ".key"
}
def generate_conf(self):
""" Render the conf with Jinja template and self.to_template() method
:return The generated configuration as string, or raise
"""
try:
jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH))
template_client = jinja2_env.get_template(JINJA_TEMPLATE_OPENVPN)
conf = self.to_template()
return {'template_client': template_client.render(conf)}
# In ALL exceptions, associate an error message
except Exception as e:
# If there was an exception, raise a more general exception with the message and the traceback
raise ServiceJinjaError(
"Openvpn config generation error: '{}'".format(str(e)),
"openvpn")
def save_conf(self):
""" Write configuration on disk
"""
conf = self.generate_conf()
params = [
'/usr/local/etc/openvpn/openvpn_client.conf',
conf['template_client'], OPENVPN_OWNER, "644"
]
try:
self.node.api_request('system.config.models.write_conf',
config=params)
except Exception as e:
raise VultureSystemConfigError(
"on node '{}'.\nRequest failure.".format(self.node.name))
class Issue(models.Model):
_id = models.ObjectIdField()
number = models.PositiveIntegerField(null=False)
repo = models.ForeignKey("project.Repository", on_delete=models.CASCADE)
project = models.ForeignKey("project.Project", on_delete=models.CASCADE)
title = models.CharField(max_length=100)
type = models.CharField(max_length=16, choices=ISSUE_TYPES, blank=True)
labels = models.CharField(max_length=255, blank=True)
invalid = models.BooleanField(default=False)
# author = models.CharField(max_length=64, blank=True)
service = models.CharField(max_length=16,
choices=CLASSES_OF_SERVICE,
blank=True)
timeline = models.ArrayField(model_container=Event, blank=True)
done = models.BooleanField(default=False, blank=True)
created_at = models.DateTimeField(blank=True)
updated_at = models.DateTimeField(blank=True)
# Kanban stuff
kanban_start_date = models.DateTimeField(blank=True)
kanban_end_date = models.DateTimeField(blank=True)
kanban_lead_time = models.PositiveIntegerField(blank=True)
kanban_total_hold_time = models.PositiveIntegerField(blank=True)
kanban_total_blocked_time = models.PositiveIntegerField(blank=True)
kanban_metrics = models.DictField(default=dict)
objects = models.DjongoManager()
class Meta:
# We can have the same issue on different projects
unique_together = (("repo", "number", "project"), )
@cached_property
def github(self):
return get_github_issue(self.repo.identifier, self.number)
@property
def github_url(self):
return format_html(
"<a href='https://github.com/{path}' target=_blank>{path}</a>",
path=
f"{self.repo.organization}/{self.repo.code}/issues/{self.number}",
)
@property
def github_labels(self):
return [l.name.lower() for l in self.github.labels]
@cached_property
def github_timeline(self):
events = []
project_id = self.project.github_id
for event in get_github_issue_timeline(self.github):
project_card = event.raw_data.get("project_card", {})
label = event.raw_data.get("label", {})
if event.event in EVENT_LABEL_MAPPING and label.get("name") in (
"hold",
"blocked",
):
events.append(
Event(
type=EVENT_LABEL_MAPPING[event.event][label["name"]],
date=as_utc(event.created_at),
))
elif project_card and project_card["project_id"] == project_id:
# Exclude "remove from project" for now
if event.event not in TIMELINE_CARD_EVENT_TYPES:
continue
events.append(
Event(
type="moved",
column=project_card["column_name"],
date=as_utc(event.created_at),
))
return events
def get_service_type(self):
if "critical" in self.github_labels:
return "expedite"
return "standard"
def __str__(self):
return f"{self.repo.code}/{self.number} - {self.title}"
def _col(self, name):
for column in self.project.columns:
if column.code == name:
return column
# This might break something, but oh well
return None
@cached_property
def _last_col_for_project(self):
for column in self.project.columns[::-1]:
if column.valid_wip:
return column
return None
def process_timeline_data(self):
# data = {column.code: {} for column in self.project.columns}
# data["blocked"] = 0
# data["on_hold"] = 0
blocked_since = None
on_hold_since = None
start_date = None
end_date = None
lead_time = None
blocked = 0
on_hold = 0
# current_column = None
for event in self.github_timeline:
if event.type == "on_hold":
on_hold_since = event.date
elif event.type == "blocked":
blocked_since = event.date
elif on_hold_since and event.type == "on_hold_removed":
# Ignore on hold for less than 2 hours
on_hold_time = on_hold_since - event.date
on_hold_since = None
if on_hold_time.seconds > 7200:
on_hold += max(on_hold_time.days, 1)
elif blocked_since and event.type == "blocked_removed":
# Ignore block if less than 2 hours
blocked_time = blocked_since - event.date
blocked_since = None
if blocked_time.seconds > 7200:
blocked += max(blocked_time.days, 1)
elif event.type == "moved":
column = self._col(event.column)
if start_date is None and column.valid_wip:
start_date = event.date
if column.code == self._last_col_for_project.code:
end_date = event.date
lead_time = max((end_date - start_date).days, 1)
# TODO: Handle time in each column
return {
"kanban_start_date": start_date,
"kanban_end_date": end_date,
"kanban_lead_time": lead_time,
"on_hold": on_hold,
"blocked": blocked,
}
def save(self, *args, **kwargs):
self.title = self.github.title
self.labels = " | ".join(self.github_labels)
# Find the first valid issue type
for label in self.github_labels:
if label in ISSUE_TYPE_LABEL_MAPPING:
self.type = ISSUE_TYPE_LABEL_MAPPING[label]
break
else:
self.type = "feature"
self.service = self.get_service_type()
self.created_at = as_utc(self.github.created_at)
self.updated_at = as_utc(self.github.updated_at)
self.timeline = self.github_timeline
self.invalid = bool(
set(self.github_labels) & set(INVALID_ISSUE_LABELS))
data = self.process_timeline_data()
self.done = data["kanban_lead_time"] is not None
self.kanban_start_date = data["kanban_start_date"]
self.kanban_end_date = data["kanban_end_date"]
self.kanban_lead_time = data["kanban_lead_time"]
self.kanban_total_hold_time = data["on_hold"]
self.kanban_total_blocked_time = data["blocked"]
super().save(*args, **kwargs)
