class GenericModel(models.Model):
big_int = models.BigIntegerField()
bool = models.BooleanField()
char = models.CharField(max_length=20)
comma_int = models.CommaSeparatedIntegerField()
date = models.DateField()
date_time = models.DateTimeField()
decimal = models.DecimalField(max_digits=10, decimal_places=5)
email = models.EmailField()
float = models.FloatField()
integer = models.IntegerField()
null_bool = models.NullBooleanField()
pos_int = models.PositiveIntegerField()
pos_small_int = models.PositiveSmallIntegerField()
slug = models.SlugField()
small_int = models.SmallIntegerField()
text = models.TextField()
time = models.TimeField()
url = models.URLField()
ip = models.GenericIPAddressField()
uuid = models.UUIDField()
# TODO: add these
# basic_file = models.FileField()
# image = models.ImageField()
objects = models.DjongoManager()
class Match(models.Model):
slug = models.SlugField(primary_key=True)
scheduled = models.DateTimeField()
event = models.CharField(max_length=255, null=True, blank=True)
group = models.CharField(max_length=2, null=True)
playoff = models.BooleanField(default=False)
winner = models.CharField(max_length=255, null=True)
timestamp = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
objects = models.DjongoManager()
class Meta:
ordering = ['event', 'group', 'scheduled']
unique_together = ['scheduled', 'event', 'group']
def __str__(self):
if self.playoff:
return "{}: playoff match {}".format(self.event,
self.matchmeta.__str__())
return "{}, {}: match {}".format(self.event, self.group,
self.matchmeta.__str__())
@property
def is_winner(self):
return True if self.winner else False
class Foul(models.Model):
code = models.SlugField(max_length=2, primary_key=True)
name = models.CharField(max_length=255)
description = models.TextField()
image = models.ImageField(blank=True, null=True)
objects = models.DjongoManager()
def __str__(self):
return self.name
class Archivo(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
nombre = models.CharField(max_length=120)
archivo = models.FileField(upload_to='archivos', storage=grid_fs_storage)
id_loc = models.IntegerField()
id_tipo_archivo = models.IntegerField()
slug = models.SlugField(blank=True, null=True)
def __str__(self):
return self.nombre
class Comentario(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)
nombre = models.CharField(max_length=120)
comentario = models.CharField(max_length=120)
id_usuario = models.IntegerField()
slug = models.SlugField(blank=True, null=True)
def __str__(self):
return self.nombre
class Group(models.Model):
slug = models.SlugField(primary_key=True)
name = models.CharField(max_length=1)
concluded = models.BooleanField(default=False)
objects = models.DjongoManager()
class Meta:
ordering = ['-concluded']
def __str__(self):
return self.name
class Announcement(models.Model):
topic = models.CharField(max_length=50)
slug = models.SlugField(unique=True)
author = models.ForeignKey(User, related_name='aut',
on_delete=models.CASCADE)
data = models.DateTimeField(auto_now_add=True)
text_of_announcement = models.TextField()
tags = models.ManyToManyField(Tag, blank=True,
related_name='announcement')
def __str__(self):
return self.topic
class Usuario(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)
username = models.CharField(max_length=120)
password = models.CharField(max_length=120)
correo = models.CharField(max_length=120)
Almdisp = models.IntegerField()
id_H_descargas = models.IntegerField()
id_publicacion = models.IntegerField()
id_PDR = models.IntegerField()
slug = models.SlugField(blank=True, null=True)
def __str__(self):
return self.username
class Team (models.Model):
slug = models.SlugField(primary_key=True)
name = models.CharField(max_length=255, verbose_name='Team Name', unique=True)
# athletes = models.ArrayReferenceField(
# to=Athlete,
# on_delete=models.CASCADE
# )
objects = TeamManager()
class Meta:
ordering = ['name']
def __str__(self):
return self.name
class Post(models.Model):
title = models.CharField(max_length=255)
slug = models.SlugField(max_length=255, unique=True)
summary = models.CharField(max_length=300)
content = models.TextField()
published = models.BooleanField(default=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ['-created']
def __unicode__(self):
return u'%s' % self.title
def get_absolute_url(self):
return reverse('blog.views.post', args=[self.slug])
class Book(models.Model):
_id = models.ObjectIdField()
title = models.CharField(max_length=255)
slug = models.SlugField()
body_text = models.TextField()
objects = models.DjongoManager()
genre = models.ArrayField(model_container=Genre, )
authors = models.ArrayField(model_container=Author, )
pub_date = models.DateField(auto_now_add=True)
rating = models.IntegerField(default=0, blank=True)
# n_comments = models.IntegerField()
comments = models.ArrayField(model_container=Comment, blank=True)
def __str__(self):
return self.title
class Post(models.Model):
title = models.CharField(max_length=100)
slug = models.SlugField(max_length=120)
author = models.ForeignKey(User, on_delete=models.CASCADE)
body = models.TextField()
likes = models.ManyToManyField(User, related_name='likes', blank=True)
is_liked = models.BooleanField(default=False)
total = models.IntegerField(blank=True, null=True)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
def __str__(self):
return self.title + "_" + self.author.username
def total_likes(self):
return self.likes.count()
class Event(models.Model):
"""
This uses embedded model field for meta, furthermore it also uses array reference
field for the teams that are competing in the event
"""
name = models.CharField(max_length=255, verbose_name='Name')
slug = models.SlugField(unique=True)
date = models.DateTimeField(verbose_name='Date')
closed = models.BooleanField(default=False)
objects = EventManager()
class Meta:
ordering = ['name']
unique_together = ('name', 'date')
def __str__(self):
return self.name
class IndeedJobs(models.Model):
title = models.CharField(max_length=200, null=True, blank=True)
slug = models.SlugField(max_length=100, unique=True, blank=True, null=True)
company = models.CharField(max_length=200, null=True, blank=True)
city = models.CharField(max_length=200, null=True, blank=True)
salary = models.CharField(max_length=200, null=True, blank=True)
# experience = models.CharField(max_length=100, null=True, blank=True)
job_description = HTMLField(null=True, blank=True)
employment_type = models.CharField(max_length=100, null=True, blank=True)
date_posted = models.CharField(max_length=200, null=True, blank=True)
job_url = models.URLField(max_length=1000,
unique=True,
null=True,
blank=True)
pub_date = models.DateTimeField(null=True,
auto_created=True,
auto_now_add=True)
timestamp = models.DateTimeField(auto_now_add=True, auto_created=True)
objects = models.Manager()
job_objects = IndeedJobsManager()
categories_objects = IndeedCategoriesManager()
class Meta:
ordering = ["-id"]
def __str__(self):
return str(self.title)
def _get_unique_slug(self):
slug = slugify(self.title)
unique_slug = slug
num = 1
while IndeedJobs.objects.filter(slug=unique_slug).exists():
unique_slug = '{}-{}'.format(slug, num)
num += 1
return unique_slug
def save(self, *args, **kwargs):
if not self.slug:
self.slug = self._get_unique_slug()
super().save(*args, **kwargs)
def get_absolute_url(self):
from django.urls import reverse
return reverse('detail', args=[str(self.slug, )])
class Product(models.Model):
product_name = models.CharField(max_length=100)
model_no = models.CharField(max_length=100)
product_category = models.CharField(max_length=100)
product_details = models.TextField()
slug = models.SlugField(verbose_name="URL link", null=True, unique=True)
updated_date = models.DateTimeField('Last Updated', auto_now=True)
main_product_image = models.ImageField(
upload_to='products')
addtional_image_1 = models.ImageField(null=True, blank=True,
upload_to='products')
addtional_image_2 = models.ImageField(null=True, blank=True,
upload_to='products')
addtional_image_3 = models.ImageField(null=True, blank=True,
upload_to='products')
addtional_image_4 = models.ImageField(null=True, blank=True,
upload_to='products')
addtional_image_5 = models.ImageField(null=True, blank=True,
upload_to='products')
def get_absolute_url(self):
return reverse('product_page', kwargs={'slug': self.slug})
def image_tag(self):
return mark_safe('<img src="%s" style="width: 45px; height:45px;" />' % self.main_product_image.url)
image_tag.short_description = 'Product Image'
def __str__(self):
return self.product_name
objects = models.DjongoManager()
class Post(models.Model):
# POST COLLECTION CONTAINS EMBEDDED METADATA & BLOG DATA,
# IT REFERENCES EXTERNAL USER MODEL AS WELL AS COMMENT MODEL
title = models.EmbeddedModelField(model_container=Blog,
model_form_class=BlogForm)
slug = models.SlugField(max_length=200, unique=True)
author = models.ForeignKey(User,
default=None,
related_name="blog_author",
on_delete=models.CASCADE)
meta_data = models.EmbeddedModelField(model_container=MetaData, )
headline = models.CharField(max_length=255)
content = models.TextField()
status = models.IntegerField(choices=STATUS, default=0)
object_id = models.PositiveIntegerField(default=True)
# CHALLENGES NOT OVERCOME:
# BEING ABLE TO REF EMBEDDED FIELDS WOULD BE AN ADVANTAGE FROM HERE
# IN THIS CASE TO ORDER DATA IN META_DATA > CREATED_ON
# class Meta:
# ordering = ['meta_data__updated_on']
def __str__(self):
return self.headline
class BookThroughFields(models.Model):
_id = models.ObjectIdField()
title = models.CharField(max_length=255)
slug = models.SlugField()
body_text = models.TextField()
objects = models.DjongoManager()
genre = models.ArrayReferenceField(
to=GenreField,
on_delete=models.CASCADE,
)
authors = models.ArrayReferenceField(
to=AuthorField,
on_delete=models.CASCADE,
)
pub_date = models.DateField(auto_now_add=True)
rating = models.IntegerField(default=0, blank=True)
# n_comments = models.IntegerField()
comments = models.ArrayReferenceField(
to=CommentField,
on_delete=models.CASCADE,
)
def __str__(self):
return self.title
class Comment(models.Model):
id = models.AutoField(primary_key=True)
post = models.ForeignKey(Post,
on_delete=models.CASCADE,
related_name='comments')
parent = models.ForeignKey('self',
on_delete=models.CASCADE,
blank=True,
related_name='children',
null=True)
reply_to = models.CharField(max_length=100, editable=False)
author_username = models.CharField(max_length=100, default="admin")
author = models.ForeignKey(get_user_model(),
on_delete=models.CASCADE,
related_name='comments',
default=1)
content = models.TextField()
created_at = models.DateTimeField(auto_now_add=True)
hidden = models.BooleanField(default=False)
flag = models.BooleanField(default=False)
level = models.SlugField(
default='0',
editable=False,
)
def save(self, *args, **kwargs):
if self.parent:
value = int(self.parent.level) + 1
else:
value = 0
self.level = slugify(value, allow_unicode=True)
super().save(*args, **kwargs)
def __str__(self):
return f'Post:{self.post}/Author:{self.author}'
class Product(models.Model):
name = models.CharField(max_length=100)
price = models.FloatField()
discount_price = models.FloatField(blank=True, null=True)
category = models.CharField(choices=CATEGORY_CHOICES, max_length=20)
availabily = models.CharField(choices=AVAILABILITY_PRODUCT,
max_length=1,
blank=True,
null=True)
description = models.TextField()
slug = models.SlugField()
image = models.ImageField()
def __str__(self):
return self.name
def get_url(self):
return reverse("details", kwargs={'slug': self.slug})
def get_cart_url(self):
return reverse("cart", kwargs={'slug': self.slug})
def get_remove_cart_url(self):
return reverse("remove-cart", kwargs={'slug': self.slug})
class Candidate(models.Model):
#Choices
source_choices = (('RF','Referral'),('JP','Job Portal'))
status_choices = (('pending','Pending'),('hired','Hired'),('reject','Reject'))
skill_choices = (('python','Python'),('java','Java'))
rating_choices = ((1,1),(2,2),(3,3),(4,4),(5,5))
sex_choices=(('m', 'Male'), ('f', 'Female'))
#fields
#user = models.ForeignKey(User, related_name='User', null=True,on_delete=models.CASCADE)
First_name = models.CharField(max_length=20,null = False,blank = False)
Last_name = models.CharField(max_length=20,null = False,blank = False)
Sex = models.CharField(max_length = 1,choices = sex_choices)
Phone = models.BigIntegerField(null = False,blank = False)
Experience = models.IntegerField(default = 2,blank = False,null =False)
Notice_period = models.IntegerField(default = 0,blank = False,null =False)
Skill = models.SlugField(default = 'Python',choices = skill_choices)
Source = models.CharField(max_length = 10,choices = source_choices)
#admin
Status = models.CharField(default = 'Pending',max_length = 10,choices = status_choices)
Rating = models.IntegerField(null = True,blank = True,choices = rating_choices)
feedback = models.TextField(null = True,blank = True)
Created_at = models.DateTimeField(auto_now=False, auto_now_add=True)
Updated_at = models.DateTimeField(auto_now=True, auto_now_add=False)
class Meta:
ordering = ['-Created_at','-Updated_at']
# def get_absolute_url(self):
# return f"/blog/{self.slug}"
# def __str__(self):
# Name = First_name + ' ' + Last_name
# return self.Name
class Signal(models.Model):
kenmark = models.SlugField(editable=False,
default='',
max_length=MAX_LENGTH)
text = models.CharField(max_length=1000)
# ? Location data
address = models.CharField(max_length=1000, null=True, blank=True)
coordinates = models.CharField(max_length=255, null=True, blank=True)
# ? Status data
state = models.CharField(max_length=20,
blank=True,
choices=workflow.STATUS_CHOICES,
default=workflow.GEMELD)
# ? Reporter data
email = models.CharField(max_length=255, null=True, blank=True)
phone = models.CharField(max_length=255, null=True, blank=True)
# ? Attachment
file = models.FileField(upload_to='attachments/%Y/%m/%d/',
null=True,
blank=True,
max_length=255)
file1 = models.FileField(upload_to='attachments/%Y/%m/%d/',
null=True,
blank=True)
file2 = models.FileField(upload_to='attachments/%Y/%m/%d/',
null=True,
blank=True)
file3 = models.FileField(upload_to='attachments/%Y/%m/%d/',
null=True,
blank=True)
# ? Priority data
priority = models.CharField(max_length=255,
choices=PRIORITY_CHOICES,
default=PRIORITY_NORMAL)
# ? Department data
code = models.CharField(max_length=255, null=True, blank=True)
name = models.CharField(max_length=255, null=True, blank=True)
created_at = models.DateTimeField(editable=False, auto_now_add=True)
updated_at = models.DateTimeField(editable=False, auto_now=True)
# ? Predefined Category
containerAdopt = models.CharField(max_length=20, null=True, blank=True)
containerType = models.CharField(max_length=200, null=True, blank=True)
fraction = models.CharField(max_length=200, null=True, blank=True)
def __str__(self):
return f"{self.id} - {self.text}"
def save(self, *args, **kwargs):
# s = (''.join(random.choice(string.ascii_uppercase) for x in range(3))).upper(
while True:
kenmark = ''.join(random.choice(string.digits)
for x in range(3)) + ''.join(
random.choice(string.ascii_uppercase)
for x in range(3))
if not Signal.objects.filter(kenmark=kenmark).exists():
break
self.kenmark = kenmark
super(Signal, self).save(*args, **kwargs)
class ReputationContext(models.Model):
""" Model used to enrich logs in Rsyslog with mmdb database"""
name = models.TextField(
default="Reputation context",
verbose_name=_("Friendly name"),
help_text=_("Custom name of the current object"),
)
""" Database type """
db_type = models.TextField(
default=DBTYPE_CHOICES[0][0],
choices=DBTYPE_CHOICES,
verbose_name=_("Database type"),
help_text=_("Type of database"),
)
method = models.SlugField(
default=HTTP_METHOD_CHOICES[0][0],
choices=HTTP_METHOD_CHOICES,
verbose_name=_("HTTP method to use"),
help_text=_("HTTP method to use while retrieving url"))
url = models.URLField(help_text=_("URL to retrieve the database from"),
verbose_name=_("Database URL"))
verify_cert = models.BooleanField(
default=True,
help_text=
_("Verify certificate to prevent connexion to self-signed certificates."
),
verbose_name=_("Verify server certificate"))
post_data = models.TextField(default="",
null=True,
verbose_name=_("POST body"),
help_text=_("Body to send if method is POST"))
custom_headers = models.DictField(
default={},
verbose_name=_("Custom headers"),
help_text=_("Headers to send while retrieving url"))
auth_type = models.TextField(
default=HTTP_AUTH_TYPE_CHOICES[0][0],
choices=HTTP_AUTH_TYPE_CHOICES,
verbose_name=_("Authentication"),
help_text=_("Authentication type used to retrieve url"))
user = models.SlugField(default=None,
null=True,
verbose_name=_("Username"),
help_text=_("Username to use for authentication"))
password = models.TextField(
default=None,
null=True,
verbose_name=_("Password"),
help_text=_("Password to use for authentication"))
tags = models.ListField(
models.SlugField(default=""),
default=[],
help_text=_("Tags to set on this object for search"))
""" Field not stored in DB, it's just used as cache between fonction classes """
content = models.BinaryField(default="")
""" MMDB database attributes """
# There cannot be multiple files with the same filename
filename = models.FilePathField(path=DATABASES_PATH,
default="",
unique=True)
description = models.TextField(default="")
# When saving object, last_update will be automatically updated
last_update = models.DateTimeField(auto_now=True)
nb_netset = models.IntegerField(default=0)
nb_unique = models.IntegerField(default=0)
internal = models.BooleanField(default=False)
""" Use DjongoManager to use mongo_find() & Co """
objects = models.DjongoManager()
def save(self, *args, **kwargs):
""" Override mother fonction to prevent save of content attribute in MongoDB
"""
self.content = ""
super().save(*args, **kwargs)
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name']
def __str__(self):
return "ReputationContext '{}'".format(self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models
:return A JSON object
"""
result = {
'id': self.id,
'name': self.name,
'description': self.description,
'db_type': self.db_type,
'method': self.method,
'url': self.url,
'verify_cert': self.verify_cert,
'post_data': self.post_data,
'auth_type': self.auth_type,
'user': self.user,
'password': self.password,
'custom_headers': self.custom_headers,
'internal': self.internal,
'tags': self.tags
}
return result
def to_html_template(self):
""" Dictionary used to render object as html
:return Dictionnary of configuration parameters
"""
db_type = self.db_type
for d in DBTYPE_CHOICES:
if self.db_type == d[0]:
db_type = d[1]
uri = "{} {}".format(self.method, self.url)
if self.auth_type:
uri += " {}({}:xxx)".format(self.auth_type, self.user)
""" Retrieve list/custom objects """
return {
'id': str(self.id),
'name': self.name,
'db_type': db_type,
'uri': uri,
'internal': self.internal,
'tags': self.tags
}
def to_template(self):
""" Dictionary used to create configuration file
:return Dictionnary of configuration parameters
"""
result = {
'id': str(self.id),
'name': self.name,
'method': self.method,
'url': self.url,
'post_data': self.post_data,
'custom_headers': self.custom_headers,
'tags': self.tags
}
""" Download url """
#result['content'] = self.download_file()
""" And returns the attributes of the class """
return result
def download_file(self):
""" """
""" If we haven't already downloaded url """
if self.content:
return self.content
""" Retrieve url and content """
auth = None
if self.auth_type:
auth_type = AUTH_TYPE_CLASSES.get(self.auth_type)
if auth_type:
auth = auth_type(self.user, self.password)
logger.debug("Try to get URL {}".format(self.url))
try:
response = requests.request(
self.method,
self.url,
data=self.post_data if self.method == "POST" else None,
headers=self.custom_headers,
auth=auth,
allow_redirects=True,
proxies=get_proxy(),
timeout=(2.0, 2.0))
# logger.info("URL '{}' retrieved, status code = {}".format(self.url, response.status_code))
assert response.status_code == 200, "Response code is not 200 ({})".format(
response.status_code)
""" If its a .gz file, dezip-it """
if self.url[-3:] == ".gz":
self.filename = self.url.split('/')[-1][:-3]
return gzip_decompress(response.content)
if response.headers.get("Content-Disposition"):
match = REGEX_GZ.search(
response.headers.get("Content-Disposition"))
if match and match[1][-3:] == ".gz":
self.filename = match[1][:-3]
return gzip_decompress(response.content)
self.filename = self.url.split('/')[-1]
except Exception as e:
raise VultureSystemError(str(e), "download '{}'".format(self.url))
return response.content
def download_mmdb(self):
""" Always call this method first, to be sure the MMDB is OK """
content = self.download_file()
if self.db_type in ("ipv4", "ipv6", "GeoIP"):
try:
return open_mmdb_database(content)
except Exception as e:
logger.error("Downloaded content is not a valid MMDB database")
raise VultureSystemError(
"Downloaded content is not a valid MMDB database",
"download '{}'".format(self.url))
else:
return None
def download(self):
content = self.download_file()
if self.db_type in ("ipv4", "ipv6", "GeoIP"):
db_reader = open_mmdb_database(content)
db_metadata = db_reader.metadata()
db_reader.close()
# Do not erase nb_netset in internal db, its retrieved in index.json
if not self.internal:
self.nb_netset = db_metadata.node_count
else:
self.nb_unique = len(content.decode('utf8').split("\n"))
return content
@property
def absolute_filename(self):
""" Return filename depending on current frontend object
"""
# Escape quotes to prevent injections in config or in commands
return "{}/{}".format(DATABASES_PATH, self.filename.replace('"', '\"'))
def save_conf(self):
""" Write configuration on disk
"""
params = [
self.absolute_filename,
self.download_file(), DATABASES_OWNER, DATABASES_PERMS
]
try:
Cluster.api_request('system.config.models.write_conf',
config=params)
except Exception as e: # e used by VultureSystemConfigError
raise VultureSystemConfigError(
"on cluster.\n"
"Request failure to write conf of Reputation context '{}'".
format(self.name))
def get_nodes(self):
""" Return the list of nodes used by frontends using the current object """
# for listener in Listener.objects.filter()
return Node.objects.filter(
networkinterfacecard__frontend__reputation_ctxs=self.id)
def reload_frontend_conf(self):
""" Send API request on each Nodes using the frontends that uses the current CTX
:return The list of concerned nodes
"""
from services.frontend.models import Listener
res = []
# Loop on Nodes
for node in Node.objects.all():
frontends = []
# Get listeners enabled on this node, using the current reputation context
for listener in Listener.objects.filter(
frontend__enabled=True,
frontend__reputation_ctxs=self.id,
network_address__nic__node=node.id).distinct():
if listener.frontend.id not in frontends:
api_res = node.api_request(
"services.rsyslogd.rsyslog.build_conf",
listener.frontend.id)
if not api_res:
raise ServiceConfigError(
"on node '{}' \n API request error.".format(
node.name),
"rsyslog",
traceback=api_res.get('message'))
frontends.append(listener.frontend.id)
res.append(node)
return res
class Tag(models.Model):
title = models.CharField(max_length=50)
slug = models.SlugField(unique=True)
def __str__(self):
return self.title
class Parser(models.Model):
""" Model used to enrich logs in Rsyslog with mmdb database"""
""" Name of the ReputationContext, unique constraint """
name = models.TextField(
unique=True,
default="Parser",
verbose_name=_("Friendly name"),
help_text=_("Custom name of the current object"),
)
""" Rulebase """
rulebase = models.TextField(
default="version=2\nrule=:%timestamp_app:float{\"format\": \"number\"}%",
verbose_name=_("Rulebase"),
help_text=_("Parser rules"),
)
to_test = models.TextField(
default="1553103776.045449327",
verbose_name=_("Lines to test"),
help_text=_("Lines to apply te parser on")
)
tags = models.ListField(
models.SlugField(default=""),
default=[],
blank=True,
help_text=_("Tags to set on this object for search")
)
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name']
def __str__(self):
return "Parser '{}'".format(self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models
:return A JSON object
"""
result = {
'id': str(self.id),
'name': self.name,
'rulebase': self.rulebase,
'to_test': self.to_test,
'tags': self.tags
}
return result
def get_base_filename(self):
return "parser_{}.rb".format(self.id or "tmp")
def get_filename(self):
return "{}/{}".format(RSYSLOG_PATH, self.get_base_filename())
def test_conf(self):
try:
if test_lognormalizer(self.get_base_filename(), self.rulebase, self.to_test):
return True
except json.JSONDecodeError:
return False
return False
def save_conf(self):
params = [self.get_filename(), self.rulebase, RSYSLOG_OWNER, RSYSLOG_PERMS]
try:
Cluster.api_request('system.config.models.write_conf', config=params)
except Exception as e: # e used by VultureSystemConfigError
logger.error(e, exc_info=1)
raise VultureSystemConfigError("on cluster.\nRequest failure to write_conf()")
def delete_conf(self):
Cluster.api_request("system.config.models.delete_conf", self.get_filename())
def to_template(self):
return {
"id": self.id,
"name": self.name,
"rulebase": self.rulebase,
"tags": json.dumps(self.tags)
}
class Backend(models.Model):
""" Model used to generated fontends configuration of HAProxy """
""" Is that section enabled or disabled """
enabled = models.BooleanField(
default=True,
help_text=_("Enable the backend"),
)
""" Name of the frontend, unique constraint """
name = models.TextField(
unique=True,
default="Backend",
help_text=_("Name of HAProxy backend"),
)
""" Listening mode of Frontend : tcp or http """
mode = models.TextField(
default=MODE_CHOICES[0][0],
choices=MODE_CHOICES,
help_text=_("Proxy mode"),
)
timeout_connect = models.PositiveIntegerField(
default=2000,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
timeout_server = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(3600)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Save of generated configuration """
configuration = models.TextField(default="{}")
""" Status of frontend for each nodes """
status = models.DictField(default={})
""" Headers """
headers = models.ArrayReferenceField(Header,
null=True,
blank=False,
on_delete=models.CASCADE,
help_text=_("Header rules"))
""" Custom HAProxy Backend directives """
custom_haproxy_conf = models.TextField(
default="", help_text=_("Custom HAProxy configuration directives."))
""" HTTP Options """
""" URI of backends """
http_backend_dir = models.TextField(
default="/",
help_text=_("Servers directory to prefix in front of requests uri"),
verbose_name=_("Servers directory"))
""" Enable or disable relaxing of HTTP response parsing """
accept_invalid_http_response = models.BooleanField(
default=False,
help_text=_("Enable relaxing of HTTP response parsing"),
verbose_name=_("Accept invalid HTTP response"))
""" Enable insertion of the X-Forwarded-For header to requests sent to servers """
http_forwardfor_header = models.TextField(
blank=True,
null=True,
help_text=_("Insertion of the X-Forwarded-For header"),
verbose_name=_("Send source ip in "))
""" Except the following IP"""
http_forwardfor_except = models.GenericIPAddressField(
protocol='IPv4',
blank=True,
null=True,
help_text=_("Except the specified IP address"),
verbose_name=_("Except for "))
""" Enable HTTP protocol to check on the servers health """
enable_http_health_check = models.BooleanField(
default=False,
help_text=_("Enable HTTP protocol health checker"),
verbose_name=_("HTTP health check"))
""" The optional HTTP method used with the requests """
http_health_check_method = models.TextField(
default=HEALTH_CHECK_METHOD_CHOICES[0][0],
choices=HEALTH_CHECK_METHOD_CHOICES,
help_text=_("HTTP method used"),
verbose_name=_("Method"))
""" The URI referenced in the HTTP requests """
http_health_check_uri = models.TextField(default='/',
blank=True,
null=True,
help_text=_("URI referenced"),
verbose_name=_("URI"))
""" The optional HTTP version string """
http_health_check_version = models.TextField(
default=HEALTH_CHECK_VERSION_CHOICES[0][0],
choices=HEALTH_CHECK_VERSION_CHOICES,
help_text=_("HTTP version"),
verbose_name=_("Version"))
""" """
http_health_check_headers = models.DictField(
default={},
help_text=_("HTTP Health Check Headers"),
verbose_name=_("HTTP Health Check Headers"))
""" Health check expect """
http_health_check_expect_match = models.TextField(
choices=HEALTH_CHECK_EXPECT_CHOICES,
default=HEALTH_CHECK_EXPECT_CHOICES[0][0],
null=True,
help_text=_("Type of match to expect"),
verbose_name=_("HTTP Health Check expected"))
http_health_check_expect_pattern = models.TextField(
default="200",
help_text=_("Type of pattern to match to expect"),
verbose_name=_("HTTP Health Check expected pattern"))
""" Enable or disable HTTP keep-alive from client to server """
enable_http_keep_alive = models.BooleanField(
default=True,
help_text=_("Enable HTTP keep-alive"),
verbose_name=_("HTTP Keep alive"))
""" Keep-alive Timeout """
http_keep_alive_timeout = models.PositiveIntegerField(
default=60,
validators=[MinValueValidator(1),
MaxValueValidator(20000)],
help_text=_("HTTP request Timeout"),
verbose_name=_("Timeout"))
""" Balancing mode """
balancing_mode = models.TextField(
choices=BALANCING_CHOICES,
default=BALANCING_CHOICES[0][0],
help_text=_("Balancing mode between servers"),
verbose_name=_("Balancing mode"))
""" Balancing param """
balancing_param = models.TextField(
null=True,
default="",
help_text=_("Balancing param for balancing mode"),
verbose_name=_("Balancing parameter"))
""" Tags """
tags = models.ListField(
models.SlugField(default=""),
default=[],
help_text=_("Tags to set on this object for search"))
@property
def balancing(self):
if self.balancing_mode == "hdr":
result = "hdr({})".format(self.balancing_param)
elif self.balancing_mode == "url_param":
result = "url_param {}".format(self.balancing_param)
elif self.balancing_mode == "rdp-cookie":
result = "rdp-cookie({})".format(self.balancing_param)
else:
result = self.balancing_mode
return result
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name', 'mode']
def __str__(self):
return "{} Backend '{}'".format(self.mode.upper(), self.name)
def to_dict(self):
""" This method MUST be used in API instead of to_template() method
to prevent no-serialization of sub-models like Listeners
:return A JSON object
"""
result = {
'id': self.id,
'enable': self.enabled,
'name': self.name,
'mode': self.mode,
'balancing_mode': self.balancing_mode,
'balancing_param': self.balancing_param,
'status': dict(self.status), # It is an OrderedDict
'servers': [],
'custom_haproxy_conf': self.custom_haproxy_conf,
'tags': self.tags
}
""" Add listeners """
for server in self.server_set.all():
s = server.to_template()
# Remove frontend to prevent infinite loop
del s['backend']
result['servers'].append(s)
""" Other attributes """
if self.mode == "http":
result['headers'] = []
for header in self.headers.all():
result['headers'].append(header.to_template())
return result
def to_html_template(self):
""" Dictionary used to render object as html
:return Dictionnary of configuration parameters
"""
""" Retrieve list/custom objects """
servers_list = [
str(s) for s in self.server_set.all().only(*Server.str_attrs())
]
mode = "UNKNOWN"
for m in MODE_CHOICES:
if self.mode == m[0]:
mode = m[1]
balancing_mode = "unknown"
for m in BALANCING_CHOICES:
if self.balancing_mode == m[0]:
balancing_mode = m[1]
additional_infos = "Balancing mode : {}".format(balancing_mode)
""" And returns the attributes of the class """
return {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'servers': servers_list,
'mode': mode,
'status': self.status,
'additional_infos': additional_infos,
'tags': self.tags
}
def to_template(self, server_list=None, header_list=None):
""" Dictionary used to create configuration file
:return Dictionnary of configuration parameters
"""
workflow_list = []
access_controls_list = []
for workflow in self.workflow_set.filter(enabled=True):
tmp = {
'id': str(workflow.pk),
'fqdn': workflow.fqdn,
'public_dir': workflow.public_dir,
'backend': workflow.backend,
'defender_policy': workflow.defender_policy
}
access_controls_deny = []
access_controls_301 = []
access_controls_302 = []
for acl in workflow.workflowacl_set.filter(before_policy=False):
rules, acl_name = acl.access_control.generate_rules()
access_controls_list.append(rules)
condition = acl.generate_condition(acl_name)
redirect_url = None
deny = False
for type_acl in ('action_satisfy', 'action_not_satisfy'):
action = getattr(acl, type_acl)
if action != "200":
if action in ("301", "302"):
redirect_url = getattr(
acl, type_acl.replace('action',
'redirect_url'))
elif action == "403":
deny = True
break
tmp_acl = {
'before_policy': acl.before_policy,
'redirect_url': redirect_url,
'conditions': condition,
'action': action,
'deny': deny
}
if action == "403":
access_controls_deny.append(tmp_acl)
elif action == "301":
access_controls_301.append(tmp_acl)
elif action == "302":
access_controls_302.append(tmp_acl)
tmp['access_controls_deny'] = access_controls_deny
tmp['access_controls_302'] = access_controls_302
tmp['access_controls_301'] = access_controls_301
workflow_list.append(tmp)
""" Simple attributes of the class """
result = {
'id': str(self.id),
'enabled': self.enabled,
'name': self.name,
'mode': self.mode,
'timeout_connect': self.timeout_connect,
'timeout_server': self.timeout_server,
'unix_socket': self.get_unix_socket(),
'custom_haproxy_conf': self.custom_haproxy_conf,
'JAIL_ADDRESSES': JAIL_ADDRESSES,
'accept_invalid_http_response': self.accept_invalid_http_response,
'http_forwardfor_header': self.http_forwardfor_header,
'http_forwardfor_except': self.http_forwardfor_except,
'enable_http_health_check': self.enable_http_health_check,
'http_health_check_method': self.http_health_check_method,
'http_health_check_uri': self.http_health_check_uri,
'http_health_check_version': self.http_health_check_version,
'http_health_check_headers': self.http_health_check_headers,
'enable_http_keep_alive': self.enable_http_keep_alive,
'http_keep_alive_timeout': self.http_keep_alive_timeout,
'access_controls_list': set(access_controls_list),
'http_backend_dir': self.http_backend_dir,
'balancing': self.balancing,
'workflows': workflow_list,
'tags': self.tags
}
""" Retrieve list/custom objects """
# If facultative arg listener_list is not given
if not server_list:
# Retrieve all the objects used by the current frontend
# No .only ! Used to generated conf, neither str cause we need the object
result['servers'] = self.server_set.all(
) # No .only() ! Used to generated conf
if self.mode == "http":
# Same for headers
result['headers'] = self.headers.all(
) if not header_list else header_list
if self.enable_http_health_check and self.http_health_check_expect_match:
result['http_health_check_expect'] = "{} {}".format(
self.http_health_check_expect_match,
self.http_health_check_expect_pattern)
return result
def generate_conf(self, server_list=None, header_list=None):
""" Render the conf with Jinja template and self.to_template() method
:return The generated configuration as string, or raise
"""
# The following var is only used by error, do not forget to adapt if needed
template_name = JINJA_PATH + JINJA_TEMPLATE
try:
jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH))
template = jinja2_env.get_template(JINJA_TEMPLATE)
return template.render({
'conf':
self.to_template(server_list=server_list,
header_list=header_list)
})
# In ALL exceptions, associate an error message
# The exception instantiation MUST be IN except statement, to retrieve traceback in __init__
except TemplateNotFound:
exception = ServiceJinjaError(
"The following file cannot be found : '{}'".format(
template_name), "haproxy")
except TemplatesNotFound:
exception = ServiceJinjaError(
"The following files cannot be found : '{}'".format(
template_name), "haproxy")
except (TemplateAssertionError, TemplateRuntimeError):
exception = ServiceJinjaError(
"Unknown error in template generation: {}".format(
template_name), "haproxy")
except UndefinedError:
exception = ServiceJinjaError(
"A variable is undefined while trying to render the following template: "
"{}".format(template_name), "haproxy")
except TemplateSyntaxError:
exception = ServiceJinjaError(
"Syntax error in the template: '{}'".format(template_name),
"haproxy")
# If there was an exception, raise a more general exception with the message and the traceback
raise exception
def test_conf(self):
""" Write the configuration attribute on disk, in test directory, as {id}.conf.new
And test the conf with 'haproxy -c'
:return True or raise
"""
""" No need to do API request cause Backends are not relative-to-node objects """
test_filename = self.get_test_filename()
conf = self.configuration
# NO Node-specific configuration, we can test-it on local node
# Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True
test_haproxy_conf(test_filename,
conf.replace(
"backend {}".format(self.name),
"backend test_{}".format(self.id or "test")),
disabled=True)
def get_test_filename(self):
""" Return test filename for test conf with haproxy
"""
""" If object is not already saved : no id so default=test """
return "backend_{}.conf".format(self.id or "test")
def get_base_filename(self):
""" Return the base filename (without path) """
return "backend_{}.cfg".format(self.id)
def get_filename(self):
""" Return filename depending on current frontend object
"""
return "{}/{}".format(HAPROXY_PATH, self.get_base_filename())
def get_unix_socket(self):
""" Return filename of unix socket on which HAProxy send data
and on which Rsyslog listen
"""
return "{}/backend_{}.sock".format(UNIX_SOCKET_PATH, self.id)
def save_conf(self):
""" Write configuration on disk
"""
if not self.configuration:
return
params = [
self.get_filename(), self.configuration, BACKEND_OWNER,
BACKEND_PERMS
]
try:
Cluster.api_request('system.config.models.write_conf',
config=params)
except Exception as e: # e used by VultureSystemConfigError
logger.error(e, exc_info=1)
raise VultureSystemConfigError(
"on cluster.\nRequest failure to write_conf()")
def reload_conf(self):
""" Generate conf and save it """
self.configuration = self.generate_conf()
self.save_conf()
self.save()
def enable(self):
""" Enable backend in HAProxy, by management socket """
""" Cannot enable a disabled frontend ('enable' field) """
if not self.enabled:
raise ServiceError(
"Cannot start a disabled backend.",
"haproxy",
"enable backend",
traceback="Please edit, enable and save a backend to start-it."
)
return hot_action_backend(self.name, "enable")
def disable(self):
""" Disable frontend in HAProxy, by management socket """
if not self.enabled:
return "This frontend is already disabled."
""" If it is an Rsyslog only conf, return error """
if self.mode == "log" and self.listening_mode == "udp":
raise ServiceError(
"Cannot hot disable an Rsyslog only frontend.",
"rsyslog",
"disable frontend",
traceback="Please edit, disable and save the frontend.")
return hot_action_backend(self.name, "disable")
class AccessControl(models.Model):
_id = models.ObjectIdField(default=ObjectId)
name = models.SlugField(
max_length=255,
verbose_name=_("Friendly name"),
help_text=_('Friendly name'),
unique=True
)
enabled = models.BooleanField(default=True)
acls = models.TextField(default="")
rules = models.ListField(default=[])
@staticmethod
def str_attrs():
""" List of attributes required by __str__ method """
return ['name']
def __str__(self):
return "ACL '{}'".format(self.name)
def to_html_template(self):
""" Returns needed attributes for html rendering """
return {
'id': str(self.pk),
'name': self.name,
'acls': self.acls,
'enabled': self.enabled,
'rules': json.dumps(self.rules)
}
def to_template(self):
return {
'id': str(self.pk),
'name': self.name,
'acls': self.acls,
'enabled': self.enabled,
'rules': json.dumps(self.rules)
}
def test_conf(self):
""" Write the configuration attribute on disk, in test directory, as {id}.conf.new
And test the conf with 'haproxy -c'
:return True or raise
"""
""" No need to do API request cause Backends are not relative-to-node objects """
test_filename = self.get_test_filename()
conf = self.generate_test_conf()
# NO Node-specific configuration, we can test-it on local node
# Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True
test_haproxy_conf(test_filename, conf, disabled=True)
# open("{}/{}".format(TEST_CONF_PATH, test_filename), "w+").write(conf)
def get_test_filename(self):
""" Return test filename for test conf with haproxy
"""
return "acl_{}.conf".format("test")
def generate_test_conf(self):
""" Generate ACL
"""
# Empty configuration case
if len(self.rules) <= 0:
return ""
rules, tmp_conditions = self.generate_rules()
conditions = []
for tmp in tmp_conditions:
conditions.extend(tmp)
conditions = " ".join(conditions)
template = render_to_string("{}/{}".format(JINJA_PATH, JINJA_TEST_TEMPLATE), {
'conditions': conditions,
'rules': rules
})
return template
def generate_rules(self):
def make_criterion(criterion, name):
"""
Generate a Criterion with a specific parameter
:param criterion:
:param name:
:return:
"""
if name != "":
choice = NAME_CHOICES.get(criterion, "")
if choice != "":
return "{}({})".format(criterion, name)
return "" + criterion
# Initialization
acls = []
acls_name = []
# For each OR block
for i, rule in enumerate(self.rules):
tmp_names = []
for j, line in enumerate(rule['lines']):
acl_name = str(uuid.uuid4())
tmp_names.append(acl_name)
acl = ""
if j > 0 or i > 0:
acl += " "
acl += "acl {}".format(acl_name)
acl += " {}".format(make_criterion(line['criterion'], line.get('criterion_name')))
tmp_lst = [line.get('converter', ''),
line.get('operator', ''),
line.get('pattern', '')]
# Add -m option if a converter is used
if line.get('converter', '') != "":
acl += " -m"
tmp_lst.insert(1, line.get('flags', ''))
acl += ''.join(" " + elem if (elem != "") else "" for elem in tmp_lst)
acls.append(acl)
acls_name.append(tmp_names)
return "\n".join(acls), acls_name
