class FilterPolicy(models.Model): filter = models.ForeignKey(DarwinFilter, on_delete=models.CASCADE) policy = models.ForeignKey(DarwinPolicy, on_delete=models.CASCADE) enabled = models.BooleanField(default=False) nb_thread = models.PositiveIntegerField(default=5) log_level = models.TextField(default=DARWIN_LOGLEVEL_CHOICES[0][0], choices=DARWIN_LOGLEVEL_CHOICES) threshold = models.PositiveIntegerField(default=80) mmdarwin_enabled = models.BooleanField(default=False) mmdarwin_parameters = models.ListField(default=[]) """ Status of filter for each nodes """ status = models.DictField(default={}) cache_size = models.PositiveIntegerField( default=1000, help_text=_("The cache size to use for caching darwin requests."), verbose_name=_("Cache size")) """Output format to send to next filter """ output = models.TextField(default=DARWIN_OUTPUT_CHOICES[0][0], choices=DARWIN_OUTPUT_CHOICES) """ Next filter in workflow """ next_filter = models.ForeignKey('self', null=True, default=None, on_delete=models.SET_NULL) conf_path = models.TextField() config = models.DictField(default={}) @property def name(self): """ Method used in Darwin conf to define a filter """ return "{}_{}".format(self.filter.name, self.policy.id) def __str__(self): return "[{}] {}".format(self.policy.name, self.filter.name) @staticmethod def str_attrs(): return ['filter', 'conf_path', 'nb_thread', 'log_level', 'config'] @property def socket_path(self): return "{}/{}_{}.sock".format(SOCKETS_PATH, self.filter.name, self.policy.id) def mmdarwin_parameters_rsyslog_str(self): return str(self.mmdarwin_parameters).replace("\'", "\"")
class Coupon(models.Model): ptcs = models.DictField() ssdkl = models.CharField(max_length=200) origin = models.CharField(max_length=3) destination = models.CharField(max_length=3) departure = models.DateField() campaign = models.CharField(max_length=20, default="base")
class DictEntry(models.Model): _id = models.ObjectIdField() headline = models.CharField(max_length=255) blog = models.DictField() def __str__(self): return self.headline
class Client(models.Model): _id = models.ObjectIdField(default=None) username = models.CharField(max_length=64) email = models.CharField(max_length=200, default="") memberships = models.DictField(default={}) objects = models.DjongoManager()
class DatasetImage(models.Model): _id = models.ObjectIdField() name = models.CharField(max_length=128, null=False, blank=False) url = models.URLField(blank=True, null=True) created = models.DateTimeField(auto_now_add=True) label = models.CharField(max_length=32, null=False, blank=False) metadata = models.DictField(null=True)
class Node(models.Model): _id = models.ObjectIdField(default=None) node_name = models.CharField(max_length=64) creators_name = models.CharField(max_length=128, default="") members = models.DictField(default={}) current_ip_address = models.CharField(max_length=24, default='0.0.0.0') objects = models.DjongoManager()
class Student(models.Model): first_name = models.CharField(max_length=100) last_name = models.CharField(max_length=100) eid = models.CharField(max_length=20) phone = models.CharField(max_length=12) email = models.EmailField(max_length=100) linkedIn = models.CharField(max_length=100, blank=True) resume_link = models.URLField(max_length=100, blank=True) intentions = models.DictField(default={}) interests = models.DictField(default={}) time_commitment = models.CharField(max_length=100) international = models.BooleanField(default=False) fin_aid = models.BooleanField(default=False) transportation = models2.BooleanField(default=False) flexible_hours = models2.BooleanField(default=False) work_remotely = models2.BooleanField(default=False) other_availability = models.TextField(max_length=500, blank=True) school = models.DictField(default={}) program = models.DictField(default={}) experience = models.DictField(default={}) tech_skills = models.DictField(default={}) prof_skills = models.DictField(default={}) other_skills = models.TextField(max_length=500, blank=True) cohort = models.CharField(max_length=100) unique_id = models.CharField(max_length=100, unique=True, null=True) created_at = models.DateTimeField(auto_now_add=True) hear = models.CharField(max_length=100)
class CustomUser(AbstractUser): role = models.CharField(blank=True, max_length=50, default='user') dashboards = models.ListField(default=[]) # default_dashboard = models.CharField(blank=True, null=True, max_length=200) default_dashboard = models.DictField(default={}) grants = models.ListField(default=[]) def __str__(self): return self.username
class LogViewerSearches(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) type_logs = models.TextField() name = models.TextField() search = models.DictField() def to_template(self): return { 'pk': str(self.pk), 'type_logs': self.type_logs, 'name': self.name, 'search': self.search }
class Project(models.Model): contact_first_name = models.CharField(max_length=100) contact_last_name = models.CharField(max_length=100) contact_phone = models.CharField(max_length=12) contact_email = models.EmailField(max_length=100) organization_name = models.CharField(max_length=100) organization_address = models.CharField(max_length=200) organization_website = models.CharField(max_length=100) project_name = models.CharField(max_length=100) project_description = models.TextField(max_length=1000) project_categories = models.DictField(default={}) time_commitment = models.CharField(max_length=100) transportation = models.BooleanField(default=False) flexible_hours = models.BooleanField(default=False) work_remotely = models.BooleanField(default=False) school = models.DictField(default={}) experience = models.DictField(default={}) tech_skills = models.DictField(default={}) prof_skills = models.DictField(default={}) other_skills = models.TextField(max_length=500, blank=True) cohort = models.CharField(max_length=100) unique_id = models.CharField(max_length=300) created_at = models.DateTimeField(auto_now_add=True)
class Wishlist(models.Model): mapping_id = models.IntegerField(default=0) brand = models.CharField(max_length=50) description = models.ListField(models.CharField(max_length=500)) user = models.ForeignKey(User, on_delete=models.CASCADE) display_size = models.CharField(max_length=10) graphics_memory = models.CharField(max_length=30) img_link = models.URLField(max_length=10000) product_title = models.CharField(max_length=1000) ram = models.CharField(max_length=10) ram_type = models.CharField(max_length=100) storage = models.DictField() websites = models.ListField( models.EmbeddedModelField(model_container=Website))
class Startech(models.Model): brand = models.CharField(max_length=50) description = models.ListField(models.CharField(max_length=1000)) display_size = models.CharField(max_length=10) graphics_memory = models.CharField(max_length=30) img_link = models.URLField(max_length=10000) price = models.IntegerField(default=0) product_link = models.URLField(max_length=10000) product_title = models.CharField(max_length=1000) ram = models.CharField(max_length=10) ram_type = models.CharField(max_length=100) status = models.CharField(max_length=100) processor = models.CharField(max_length=100) storage = models.DictField() website = models.CharField(max_length=100) _id = models.CharField(primary_key=True, max_length=100)
class Mapping(models.Model): brand = models.CharField(max_length=50) description = models.ListField(models.CharField(max_length=500)) display_size = models.CharField(max_length=10) graphics_memory = models.CharField(max_length=30) img_link = models.URLField(max_length=10000) product_title = models.CharField(max_length=1000) ram = models.CharField(max_length=10) ram_type = models.CharField(max_length=100) storage = models.DictField() _id = models.CharField(max_length=100) websites = models.ListField( models.EmbeddedModelField( model_container=Website ) ) id = models.IntegerField(primary_key=True)
class LogViewerConfiguration(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) type_logs = models.TextField() displayed_columns = models.DictField() nb_lines = models.IntegerField(default=25) font_size = models.IntegerField(default=12) class Meta: unique_together = ('user', 'type_logs') def to_template(self): return { 'pk': str(self.pk), 'type_logs': self.type_logs, 'displayed_columns': self.displayed_columns, 'nb_lines': self.nb_lines, 'font_size': self.font_size }
class DefenderProcessRule(models.Model): job_id = models.TextField() expiration_date = models.DateTimeField(auto_now_add=True) rule_id = models.IntegerField() rule_key = models.TextField(default="") data = models.DictField(default={}) objects = models.DjongoManager() def __init__(self, *args, ** kwargs): DefenderProcessRule.objects._client.ensure_index('expiration_date', expireAfterSeconds=10 * 60) super(DefenderProcessRule, self).__init__(*args, **kwargs) def to_dict(self): return { "job_id": self.job_id, "expiration_date": self.expiration_date, "rule_id": self.rule_id, "rule_key": self.rule_key, "data": self.data }
class Backend(models.Model): """ Model used to generated fontends configuration of HAProxy """ """ Is that section enabled or disabled """ enabled = models.BooleanField( default=True, help_text=_("Enable the backend"), ) """ Name of the frontend, unique constraint """ name = models.TextField( unique=True, default="Backend", help_text=_("Name of HAProxy backend"), ) """ Listening mode of Frontend : tcp or http """ mode = models.TextField( default=MODE_CHOICES[0][0], choices=MODE_CHOICES, help_text=_("Proxy mode"), ) timeout_connect = models.PositiveIntegerField( default=2000, validators=[MinValueValidator(1), MaxValueValidator(20000)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) timeout_server = models.PositiveIntegerField( default=60, validators=[MinValueValidator(1), MaxValueValidator(3600)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) """ Save of generated configuration """ configuration = models.TextField(default="{}") """ Status of frontend for each nodes """ status = models.DictField(default={}) """ Headers """ headers = models.ArrayReferenceField(Header, null=True, blank=False, on_delete=models.CASCADE, help_text=_("Header rules")) """ Custom HAProxy Backend directives """ custom_haproxy_conf = models.TextField( default="", help_text=_("Custom HAProxy configuration directives.")) """ HTTP Options """ """ URI of backends """ http_backend_dir = models.TextField( default="/", help_text=_("Servers directory to prefix in front of requests uri"), verbose_name=_("Servers directory")) """ Enable or disable relaxing of HTTP response parsing """ accept_invalid_http_response = models.BooleanField( default=False, help_text=_("Enable relaxing of HTTP response parsing"), verbose_name=_("Accept invalid HTTP response")) """ Enable insertion of the X-Forwarded-For header to requests sent to servers """ http_forwardfor_header = models.TextField( blank=True, null=True, help_text=_("Insertion of the X-Forwarded-For header"), verbose_name=_("Send source ip in ")) """ Except the following IP""" http_forwardfor_except = models.GenericIPAddressField( protocol='IPv4', blank=True, null=True, help_text=_("Except the specified IP address"), verbose_name=_("Except for ")) """ Enable HTTP protocol to check on the servers health """ enable_http_health_check = models.BooleanField( default=False, help_text=_("Enable HTTP protocol health checker"), verbose_name=_("HTTP health check")) """ The optional HTTP method used with the requests """ http_health_check_method = models.TextField( default=HEALTH_CHECK_METHOD_CHOICES[0][0], choices=HEALTH_CHECK_METHOD_CHOICES, help_text=_("HTTP method used"), verbose_name=_("Method")) """ The URI referenced in the HTTP requests """ http_health_check_uri = models.TextField(default='/', blank=True, null=True, help_text=_("URI referenced"), verbose_name=_("URI")) """ The optional HTTP version string """ http_health_check_version = models.TextField( default=HEALTH_CHECK_VERSION_CHOICES[0][0], choices=HEALTH_CHECK_VERSION_CHOICES, help_text=_("HTTP version"), verbose_name=_("Version")) """ """ http_health_check_headers = models.DictField( default={}, help_text=_("HTTP Health Check Headers"), verbose_name=_("HTTP Health Check Headers")) """ Health check expect """ http_health_check_expect_match = models.TextField( choices=HEALTH_CHECK_EXPECT_CHOICES, default=HEALTH_CHECK_EXPECT_CHOICES[0][0], null=True, help_text=_("Type of match to expect"), verbose_name=_("HTTP Health Check expected")) http_health_check_expect_pattern = models.TextField( default="200", help_text=_("Type of pattern to match to expect"), verbose_name=_("HTTP Health Check expected pattern")) """ Enable or disable HTTP keep-alive from client to server """ enable_http_keep_alive = models.BooleanField( default=True, help_text=_("Enable HTTP keep-alive"), verbose_name=_("HTTP Keep alive")) """ Keep-alive Timeout """ http_keep_alive_timeout = models.PositiveIntegerField( default=60, validators=[MinValueValidator(1), MaxValueValidator(20000)], help_text=_("HTTP request Timeout"), verbose_name=_("Timeout")) """ Balancing mode """ balancing_mode = models.TextField( choices=BALANCING_CHOICES, default=BALANCING_CHOICES[0][0], help_text=_("Balancing mode between servers"), verbose_name=_("Balancing mode")) """ Balancing param """ balancing_param = models.TextField( null=True, default="", help_text=_("Balancing param for balancing mode"), verbose_name=_("Balancing parameter")) """ Tags """ tags = models.ListField( models.SlugField(default=""), default=[], help_text=_("Tags to set on this object for search")) @property def balancing(self): if self.balancing_mode == "hdr": result = "hdr({})".format(self.balancing_param) elif self.balancing_mode == "url_param": result = "url_param {}".format(self.balancing_param) elif self.balancing_mode == "rdp-cookie": result = "rdp-cookie({})".format(self.balancing_param) else: result = self.balancing_mode return result @staticmethod def str_attrs(): """ List of attributes required by __str__ method """ return ['name', 'mode'] def __str__(self): return "{} Backend '{}'".format(self.mode.upper(), self.name) def to_dict(self): """ This method MUST be used in API instead of to_template() method to prevent no-serialization of sub-models like Listeners :return A JSON object """ result = { 'id': self.id, 'enable': self.enabled, 'name': self.name, 'mode': self.mode, 'balancing_mode': self.balancing_mode, 'balancing_param': self.balancing_param, 'status': dict(self.status), # It is an OrderedDict 'servers': [], 'custom_haproxy_conf': self.custom_haproxy_conf, 'tags': self.tags } """ Add listeners """ for server in self.server_set.all(): s = server.to_template() # Remove frontend to prevent infinite loop del s['backend'] result['servers'].append(s) """ Other attributes """ if self.mode == "http": result['headers'] = [] for header in self.headers.all(): result['headers'].append(header.to_template()) return result def to_html_template(self): """ Dictionary used to render object as html :return Dictionnary of configuration parameters """ """ Retrieve list/custom objects """ servers_list = [ str(s) for s in self.server_set.all().only(*Server.str_attrs()) ] mode = "UNKNOWN" for m in MODE_CHOICES: if self.mode == m[0]: mode = m[1] balancing_mode = "unknown" for m in BALANCING_CHOICES: if self.balancing_mode == m[0]: balancing_mode = m[1] additional_infos = "Balancing mode : {}".format(balancing_mode) """ And returns the attributes of the class """ return { 'id': str(self.id), 'enabled': self.enabled, 'name': self.name, 'servers': servers_list, 'mode': mode, 'status': self.status, 'additional_infos': additional_infos, 'tags': self.tags } def to_template(self, server_list=None, header_list=None): """ Dictionary used to create configuration file :return Dictionnary of configuration parameters """ workflow_list = [] access_controls_list = [] for workflow in self.workflow_set.filter(enabled=True): tmp = { 'id': str(workflow.pk), 'fqdn': workflow.fqdn, 'public_dir': workflow.public_dir, 'backend': workflow.backend, 'defender_policy': workflow.defender_policy } access_controls_deny = [] access_controls_301 = [] access_controls_302 = [] for acl in workflow.workflowacl_set.filter(before_policy=False): rules, acl_name = acl.access_control.generate_rules() access_controls_list.append(rules) condition = acl.generate_condition(acl_name) redirect_url = None deny = False for type_acl in ('action_satisfy', 'action_not_satisfy'): action = getattr(acl, type_acl) if action != "200": if action in ("301", "302"): redirect_url = getattr( acl, type_acl.replace('action', 'redirect_url')) elif action == "403": deny = True break tmp_acl = { 'before_policy': acl.before_policy, 'redirect_url': redirect_url, 'conditions': condition, 'action': action, 'deny': deny } if action == "403": access_controls_deny.append(tmp_acl) elif action == "301": access_controls_301.append(tmp_acl) elif action == "302": access_controls_302.append(tmp_acl) tmp['access_controls_deny'] = access_controls_deny tmp['access_controls_302'] = access_controls_302 tmp['access_controls_301'] = access_controls_301 workflow_list.append(tmp) """ Simple attributes of the class """ result = { 'id': str(self.id), 'enabled': self.enabled, 'name': self.name, 'mode': self.mode, 'timeout_connect': self.timeout_connect, 'timeout_server': self.timeout_server, 'unix_socket': self.get_unix_socket(), 'custom_haproxy_conf': self.custom_haproxy_conf, 'JAIL_ADDRESSES': JAIL_ADDRESSES, 'accept_invalid_http_response': self.accept_invalid_http_response, 'http_forwardfor_header': self.http_forwardfor_header, 'http_forwardfor_except': self.http_forwardfor_except, 'enable_http_health_check': self.enable_http_health_check, 'http_health_check_method': self.http_health_check_method, 'http_health_check_uri': self.http_health_check_uri, 'http_health_check_version': self.http_health_check_version, 'http_health_check_headers': self.http_health_check_headers, 'enable_http_keep_alive': self.enable_http_keep_alive, 'http_keep_alive_timeout': self.http_keep_alive_timeout, 'access_controls_list': set(access_controls_list), 'http_backend_dir': self.http_backend_dir, 'balancing': self.balancing, 'workflows': workflow_list, 'tags': self.tags } """ Retrieve list/custom objects """ # If facultative arg listener_list is not given if not server_list: # Retrieve all the objects used by the current frontend # No .only ! Used to generated conf, neither str cause we need the object result['servers'] = self.server_set.all( ) # No .only() ! Used to generated conf if self.mode == "http": # Same for headers result['headers'] = self.headers.all( ) if not header_list else header_list if self.enable_http_health_check and self.http_health_check_expect_match: result['http_health_check_expect'] = "{} {}".format( self.http_health_check_expect_match, self.http_health_check_expect_pattern) return result def generate_conf(self, server_list=None, header_list=None): """ Render the conf with Jinja template and self.to_template() method :return The generated configuration as string, or raise """ # The following var is only used by error, do not forget to adapt if needed template_name = JINJA_PATH + JINJA_TEMPLATE try: jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH)) template = jinja2_env.get_template(JINJA_TEMPLATE) return template.render({ 'conf': self.to_template(server_list=server_list, header_list=header_list) }) # In ALL exceptions, associate an error message # The exception instantiation MUST be IN except statement, to retrieve traceback in __init__ except TemplateNotFound: exception = ServiceJinjaError( "The following file cannot be found : '{}'".format( template_name), "haproxy") except TemplatesNotFound: exception = ServiceJinjaError( "The following files cannot be found : '{}'".format( template_name), "haproxy") except (TemplateAssertionError, TemplateRuntimeError): exception = ServiceJinjaError( "Unknown error in template generation: {}".format( template_name), "haproxy") except UndefinedError: exception = ServiceJinjaError( "A variable is undefined while trying to render the following template: " "{}".format(template_name), "haproxy") except TemplateSyntaxError: exception = ServiceJinjaError( "Syntax error in the template: '{}'".format(template_name), "haproxy") # If there was an exception, raise a more general exception with the message and the traceback raise exception def test_conf(self): """ Write the configuration attribute on disk, in test directory, as {id}.conf.new And test the conf with 'haproxy -c' :return True or raise """ """ No need to do API request cause Backends are not relative-to-node objects """ test_filename = self.get_test_filename() conf = self.configuration # NO Node-specific configuration, we can test-it on local node # Backends can not be used, so do not handle the HAProxy "not used" error by setting disabled=True test_haproxy_conf(test_filename, conf.replace( "backend {}".format(self.name), "backend test_{}".format(self.id or "test")), disabled=True) def get_test_filename(self): """ Return test filename for test conf with haproxy """ """ If object is not already saved : no id so default=test """ return "backend_{}.conf".format(self.id or "test") def get_base_filename(self): """ Return the base filename (without path) """ return "backend_{}.cfg".format(self.id) def get_filename(self): """ Return filename depending on current frontend object """ return "{}/{}".format(HAPROXY_PATH, self.get_base_filename()) def get_unix_socket(self): """ Return filename of unix socket on which HAProxy send data and on which Rsyslog listen """ return "{}/backend_{}.sock".format(UNIX_SOCKET_PATH, self.id) def save_conf(self): """ Write configuration on disk """ if not self.configuration: return params = [ self.get_filename(), self.configuration, BACKEND_OWNER, BACKEND_PERMS ] try: Cluster.api_request('system.config.models.write_conf', config=params) except Exception as e: # e used by VultureSystemConfigError logger.error(e, exc_info=1) raise VultureSystemConfigError( "on cluster.\nRequest failure to write_conf()") def reload_conf(self): """ Generate conf and save it """ self.configuration = self.generate_conf() self.save_conf() self.save() def enable(self): """ Enable backend in HAProxy, by management socket """ """ Cannot enable a disabled frontend ('enable' field) """ if not self.enabled: raise ServiceError( "Cannot start a disabled backend.", "haproxy", "enable backend", traceback="Please edit, enable and save a backend to start-it." ) return hot_action_backend(self.name, "enable") def disable(self): """ Disable frontend in HAProxy, by management socket """ if not self.enabled: return "This frontend is already disabled." """ If it is an Rsyslog only conf, return error """ if self.mode == "log" and self.listening_mode == "udp": raise ServiceError( "Cannot hot disable an Rsyslog only frontend.", "rsyslog", "disable frontend", traceback="Please edit, disable and save the frontend.") return hot_action_backend(self.name, "disable")
class Result(models.Model): created_at = models.DateTimeField(auto_now_add=True) cohort = models.CharField(max_length=100) email = models.EmailField(null=True, blank=True) data = models.DictField(default={})
class Strongswan(models.Model): node = models.OneToOneField(Node, on_delete=models.CASCADE) enabled = models.BooleanField(default=False) ipsec_type = models.TextField(choices=TYPE, default="tunnel") ipsec_keyexchange = models.TextField(choices=KEYEXCHANGE, default="ikev2") ipsec_authby = models.TextField(choices=AUTHBY, default="secret") ipsec_psk = models.TextField(default="V3ryStr0ngP@ssphr@se") ipsec_fragmentation = models.BooleanField(default=True) ipsec_forceencaps = models.BooleanField(default=False) ipsec_ike = models.TextField(default="aes256-sha512-modp8192") ipsec_esp = models.TextField(default="aes256-sha512-modp8192") ipsec_dpdaction = models.TextField(choices=DPD, default="restart") ipsec_dpddelay = models.TextField(default="35s") ipsec_rekey = models.BooleanField(default=True) ipsec_ikelifetime = models.TextField(default="3h") ipsec_keylife = models.TextField(default="1h") ipsec_right = models.TextField(default="") ipsec_leftsubnet = models.TextField(default="") ipsec_leftid = models.TextField(default="") ipsec_rightsubnet = models.TextField(default="") status = models.TextField(default="WAITING") statusall = models.TextField(default="") tunnels_status = models.DictField(default={}) tunnels_up = models.PositiveIntegerField(default=0) tunnels_connecting = models.PositiveIntegerField(default=0) def to_template(self): """ Dictionary used to create configuration file. :return: Dictionary of configuration parameters """ return {'conf': self.to_html_template()} def to_dict(self): """ Serialized version of object """ return { 'id': str(self.id), 'node': self.node.to_dict(), 'status': self.status, 'statusall': self.statusall, 'ipsec_type': self.ipsec_type, 'ipsec_keyexchange': self.ipsec_keyexchange, 'ipsec_authby': self.ipsec_authby, 'ipsec_psk': self.ipsec_psk, 'ipsec_fragmentation': self.ipsec_fragmentation, 'ipsec_forceencaps': self.ipsec_forceencaps, 'ipsec_ike': self.ipsec_ike, 'ipsec_esp': self.ipsec_esp, 'ipsec_dpdaction': self.ipsec_dpdaction, 'ipsec_dpddelay': self.ipsec_dpddelay, 'ipsec_rekey': self.ipsec_rekey, 'ipsec_ikelifetime': self.ipsec_ikelifetime, 'ipsec_keylife': self.ipsec_keylife, 'ipsec_right': self.ipsec_right, 'ipsec_leftsubnet': self.ipsec_leftsubnet, 'ipsec_leftid': self.ipsec_leftid, 'ipsec_rightsubnet': self.ipsec_rightsubnet } def get_status(self): return { 'node': self.node.to_dict(), 'status': self.status, 'statusall': self.statusall, 'tunnels_status': self.tunnels_status, 'tunnels_up': self.tunnels_up, 'tunnels_connecting': self.tunnels_connecting } def to_html_template(self): """ Dictionary used to render FORM :return Dictionnary of configuration parameters """ """ And returns the attributes of the class """ return { 'id': str(self.id), 'node': self.node.name, 'status': self.status, 'statusall': self.statusall, 'ipsec_type': self.ipsec_type, 'ipsec_keyexchange': self.ipsec_keyexchange, 'ipsec_authby': self.ipsec_authby, 'ipsec_psk': self.ipsec_psk, 'ipsec_fragmentation': self.ipsec_fragmentation, 'ipsec_forceencaps': self.ipsec_forceencaps, 'ipsec_ike': self.ipsec_ike, 'ipsec_esp': self.ipsec_esp, 'ipsec_dpdaction': self.ipsec_dpdaction, 'ipsec_dpddelay': self.ipsec_dpddelay, 'ipsec_rekey': self.ipsec_rekey, 'ipsec_ikelifetime': self.ipsec_ikelifetime, 'ipsec_keylife': self.ipsec_keylife, 'ipsec_right': self.ipsec_right, 'ipsec_leftsubnet': self.ipsec_leftsubnet, 'ipsec_leftid': self.ipsec_leftid, 'ipsec_rightsubnet': self.ipsec_rightsubnet, 'tunnels_status': self.tunnels_status, 'tunnels_up': self.tunnels_up, 'tunnels_connecting': self.tunnels_connecting, } def generate_conf(self): """ Render the conf with Jinja template and self.to_template() method :return The generated configuration as string, or raise """ try: jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH)) template_ipsec = jinja2_env.get_template(JINJA_TEMPLATE_IPSEC) template_secrets = jinja2_env.get_template(JINJA_TEMPLATE_SECRETS) template_strongswan = jinja2_env.get_template( JINJA_TEMPLATE_STRONGSWAN) conf = self.to_template() return { 'template_ipsec': template_ipsec.render(conf), 'template_secrets': template_secrets.render(conf), 'template_strongswan': template_strongswan.render(conf) } # In ALL exceptions, associate an error message except Exception as e: # If there was an exception, raise a more general exception with the message and the traceback raise ServiceJinjaError( "Strongswan config generation error: '{}'".format(str(e)), "strongswan") def save_conf(self): """ Write configuration on disk """ conf = self.generate_conf() params_ipsec = [ '/usr/local/etc/ipsec.conf', conf['template_ipsec'], STRONGSWAN_OWNER, "644" ] params_secrets = [ '/usr/local/etc/ipsec.secrets', conf['template_secrets'], STRONGSWAN_OWNER, "600" ] params_strongswan = [ '/usr/local/etc/strongswan.conf', conf['template_strongswan'], STRONGSWAN_OWNER, "644" ] for params in (params_ipsec, params_secrets, params_strongswan): try: self.node.api_request('system.config.models.write_conf', config=params) except Exception as e: raise VultureSystemConfigError( "on node '{}'.\nRequest failure.".format(self.node.name))
class Tags(models.Model): #No Mongo object ID is needed here, as the name of the tag will be able to serve as the primary key (which nicely prevents two people creating different tags for the same thing, though machine_learning, machinelearning, machine-learning, machineLearning and MachineLearing would all be able to ) # be stored unless we use some sanitization on tag creation, say enforce lowercase and no punctutation, or select from set (which uses a different backend method) name = models.DictField(default={}) objects = models.DjongoManager()
class Seats(models.Model): STATE = Choices((0, 'vacant', 'vacant'), (1, 'occupied', 'occupied')) DEFAULT_LAYOUT = [ [ '1A', '1B', '1C', '1D', '1E', '1F', '1G', ], [ '2A', '2B', '2C', '2D', '2E', '2F', '2G', ], [ '3A', '3B', '3C', '3D', '3E', '3F', '3G', ], [ '4A', '4B', '4C', '4D', '4E', '4F', '4G', ], [ '5A', '5B', '5C', '5D', '5E', '5F', '5G', ], [ '6A', '6B', '6C', '6D', '6E', '6F', '6G', ], [ '7A', '7B', '7C', '7D', '7E', '7F', '7G', ], [ '8A', '8B', '8C', '8D', '8E', '8F', '8G', ], [ '9A', '9B', '9C', '9D', '9E', '9F', '9G', ], ] layout = models.ListField(default=DEFAULT_LAYOUT) states = models.DictField(default={}) def set_state(self, seat_id, state): self.states[seat_id] = state return self.states[seat_id] def __str__(self): return str(self.layout) class Meta: abstract = True
class ReputationContext(models.Model): """ Model used to enrich logs in Rsyslog with mmdb database""" name = models.TextField( default="Reputation context", verbose_name=_("Friendly name"), help_text=_("Custom name of the current object"), ) """ Database type """ db_type = models.TextField( default=DBTYPE_CHOICES[0][0], choices=DBTYPE_CHOICES, verbose_name=_("Database type"), help_text=_("Type of database"), ) method = models.SlugField( default=HTTP_METHOD_CHOICES[0][0], choices=HTTP_METHOD_CHOICES, verbose_name=_("HTTP method to use"), help_text=_("HTTP method to use while retrieving url")) url = models.URLField(help_text=_("URL to retrieve the database from"), verbose_name=_("Database URL")) verify_cert = models.BooleanField( default=True, help_text= _("Verify certificate to prevent connexion to self-signed certificates." ), verbose_name=_("Verify server certificate")) post_data = models.TextField(default="", null=True, verbose_name=_("POST body"), help_text=_("Body to send if method is POST")) custom_headers = models.DictField( default={}, verbose_name=_("Custom headers"), help_text=_("Headers to send while retrieving url")) auth_type = models.TextField( default=HTTP_AUTH_TYPE_CHOICES[0][0], choices=HTTP_AUTH_TYPE_CHOICES, verbose_name=_("Authentication"), help_text=_("Authentication type used to retrieve url")) user = models.SlugField(default=None, null=True, verbose_name=_("Username"), help_text=_("Username to use for authentication")) password = models.TextField( default=None, null=True, verbose_name=_("Password"), help_text=_("Password to use for authentication")) tags = models.ListField( models.SlugField(default=""), default=[], help_text=_("Tags to set on this object for search")) """ Field not stored in DB, it's just used as cache between fonction classes """ content = models.BinaryField(default="") """ MMDB database attributes """ # There cannot be multiple files with the same filename filename = models.FilePathField(path=DATABASES_PATH, default="", unique=True) description = models.TextField(default="") # When saving object, last_update will be automatically updated last_update = models.DateTimeField(auto_now=True) nb_netset = models.IntegerField(default=0) nb_unique = models.IntegerField(default=0) internal = models.BooleanField(default=False) """ Use DjongoManager to use mongo_find() & Co """ objects = models.DjongoManager() def save(self, *args, **kwargs): """ Override mother fonction to prevent save of content attribute in MongoDB """ self.content = "" super().save(*args, **kwargs) @staticmethod def str_attrs(): """ List of attributes required by __str__ method """ return ['name'] def __str__(self): return "ReputationContext '{}'".format(self.name) def to_dict(self): """ This method MUST be used in API instead of to_template() method to prevent no-serialization of sub-models :return A JSON object """ result = { 'id': self.id, 'name': self.name, 'description': self.description, 'db_type': self.db_type, 'method': self.method, 'url': self.url, 'verify_cert': self.verify_cert, 'post_data': self.post_data, 'auth_type': self.auth_type, 'user': self.user, 'password': self.password, 'custom_headers': self.custom_headers, 'internal': self.internal, 'tags': self.tags } return result def to_html_template(self): """ Dictionary used to render object as html :return Dictionnary of configuration parameters """ db_type = self.db_type for d in DBTYPE_CHOICES: if self.db_type == d[0]: db_type = d[1] uri = "{} {}".format(self.method, self.url) if self.auth_type: uri += " {}({}:xxx)".format(self.auth_type, self.user) """ Retrieve list/custom objects """ return { 'id': str(self.id), 'name': self.name, 'db_type': db_type, 'uri': uri, 'internal': self.internal, 'tags': self.tags } def to_template(self): """ Dictionary used to create configuration file :return Dictionnary of configuration parameters """ result = { 'id': str(self.id), 'name': self.name, 'method': self.method, 'url': self.url, 'post_data': self.post_data, 'custom_headers': self.custom_headers, 'tags': self.tags } """ Download url """ #result['content'] = self.download_file() """ And returns the attributes of the class """ return result def download_file(self): """ """ """ If we haven't already downloaded url """ if self.content: return self.content """ Retrieve url and content """ auth = None if self.auth_type: auth_type = AUTH_TYPE_CLASSES.get(self.auth_type) if auth_type: auth = auth_type(self.user, self.password) logger.debug("Try to get URL {}".format(self.url)) try: response = requests.request( self.method, self.url, data=self.post_data if self.method == "POST" else None, headers=self.custom_headers, auth=auth, allow_redirects=True, proxies=get_proxy(), timeout=(2.0, 2.0)) # logger.info("URL '{}' retrieved, status code = {}".format(self.url, response.status_code)) assert response.status_code == 200, "Response code is not 200 ({})".format( response.status_code) """ If its a .gz file, dezip-it """ if self.url[-3:] == ".gz": self.filename = self.url.split('/')[-1][:-3] return gzip_decompress(response.content) if response.headers.get("Content-Disposition"): match = REGEX_GZ.search( response.headers.get("Content-Disposition")) if match and match[1][-3:] == ".gz": self.filename = match[1][:-3] return gzip_decompress(response.content) self.filename = self.url.split('/')[-1] except Exception as e: raise VultureSystemError(str(e), "download '{}'".format(self.url)) return response.content def download_mmdb(self): """ Always call this method first, to be sure the MMDB is OK """ content = self.download_file() if self.db_type in ("ipv4", "ipv6", "GeoIP"): try: return open_mmdb_database(content) except Exception as e: logger.error("Downloaded content is not a valid MMDB database") raise VultureSystemError( "Downloaded content is not a valid MMDB database", "download '{}'".format(self.url)) else: return None def download(self): content = self.download_file() if self.db_type in ("ipv4", "ipv6", "GeoIP"): db_reader = open_mmdb_database(content) db_metadata = db_reader.metadata() db_reader.close() # Do not erase nb_netset in internal db, its retrieved in index.json if not self.internal: self.nb_netset = db_metadata.node_count else: self.nb_unique = len(content.decode('utf8').split("\n")) return content @property def absolute_filename(self): """ Return filename depending on current frontend object """ # Escape quotes to prevent injections in config or in commands return "{}/{}".format(DATABASES_PATH, self.filename.replace('"', '\"')) def save_conf(self): """ Write configuration on disk """ params = [ self.absolute_filename, self.download_file(), DATABASES_OWNER, DATABASES_PERMS ] try: Cluster.api_request('system.config.models.write_conf', config=params) except Exception as e: # e used by VultureSystemConfigError raise VultureSystemConfigError( "on cluster.\n" "Request failure to write conf of Reputation context '{}'". format(self.name)) def get_nodes(self): """ Return the list of nodes used by frontends using the current object """ # for listener in Listener.objects.filter() return Node.objects.filter( networkinterfacecard__frontend__reputation_ctxs=self.id) def reload_frontend_conf(self): """ Send API request on each Nodes using the frontends that uses the current CTX :return The list of concerned nodes """ from services.frontend.models import Listener res = [] # Loop on Nodes for node in Node.objects.all(): frontends = [] # Get listeners enabled on this node, using the current reputation context for listener in Listener.objects.filter( frontend__enabled=True, frontend__reputation_ctxs=self.id, network_address__nic__node=node.id).distinct(): if listener.frontend.id not in frontends: api_res = node.api_request( "services.rsyslogd.rsyslog.build_conf", listener.frontend.id) if not api_res: raise ServiceConfigError( "on node '{}' \n API request error.".format( node.name), "rsyslog", traceback=api_res.get('message')) frontends.append(listener.frontend.id) res.append(node) return res
class FilterPolicy(models.Model): """ Associated filter template """ filter = models.ForeignKey( DarwinFilter, on_delete=models.CASCADE ) """ Associated policy """ policy = models.ForeignKey( DarwinPolicy, on_delete=models.CASCADE ) """ Is the Filter activated? """ enabled = models.BooleanField(default=False) """ Number of threads """ nb_thread = models.PositiveIntegerField(default=5) """ Level of logging (not alerts) """ log_level = models.TextField(default=DARWIN_LOGLEVEL_CHOICES[1][0], choices=DARWIN_LOGLEVEL_CHOICES) """ Alert detection thresold """ threshold = models.PositiveIntegerField(default=80) """ Does the filter has a custom Rsyslog configuration? """ mmdarwin_enabled = models.BooleanField(default=False) """ The custom rsyslog message's fields to get """ mmdarwin_parameters = models.ListField(default=[]) """ Status of filter for each nodes """ status = models.DictField(default={}) """ The number of cache entries (not memory size) """ cache_size = models.PositiveIntegerField( default=0, help_text=_("The cache size to use for caching darwin requests."), verbose_name=_("Cache size") ) """Output format to send to next filter """ output = models.TextField( default=DARWIN_OUTPUT_CHOICES[0][0], choices=DARWIN_OUTPUT_CHOICES ) """ Next filter in workflow """ next_filter = models.ForeignKey( 'self', null=True, default=None, on_delete=models.SET_NULL ) """ The fullpath to its configuration file """ conf_path = models.TextField() """ A dict representing the filter configuration """ config = models.DictField(default={ "redis_socket_path": "/var/sockets/redis/redis.sock", "alert_redis_list_name": DARWIN_REDIS_ALERT_LIST, "alert_redis_channel_name": DARWIN_REDIS_ALERT_CHANNEL, "log_file_path": "/var/log/darwin/alerts.log" }) @property def name(self): """ Method used in Darwin conf to define a filter """ return "{}_{}".format(self.filter.name, self.policy.id) def __str__(self): return "[{}] {}".format(self.policy.name, self.filter.name) @staticmethod def str_attrs(): return ['filter', 'conf_path', 'nb_thread', 'log_level', 'config'] @property def socket_path(self): return "{}/{}_{}.sock".format(SOCKETS_PATH, self.filter.name, self.policy.id) def mmdarwin_parameters_rsyslog_str(self): return str(self.mmdarwin_parameters).replace("\'", "\"")
class Openvpn(models.Model): node = models.OneToOneField(Node, on_delete=models.CASCADE) enabled = models.BooleanField(default=False) remote_server = models.TextField(default="") remote_port = models.PositiveIntegerField(default=443) tls_profile = models.ForeignKey(to=TLSProfile, on_delete=models.CASCADE, help_text=_("TLS Profile to use.")) proto = models.TextField(choices=PROTO, default="tcp") status = models.TextField(default="WAITING") tunnels_status = models.DictField(default={}) def to_template(self): """ Dictionary used to create configuration file. :return: Dictionary of configuration parameters """ return {'conf': self.to_html_template()} def to_dict(self): """ Serialized version of object """ return { 'id': str(self.id), 'node': self.node.to_dict(), 'status': self.status, 'remote_server': self.remote_server, 'remote_port': self.remote_port, 'tls_profile': self.tls_profile, 'proto': self.proto } def get_status(self): return { 'node': self.node.to_dict(), 'status': self.status, 'tunnels_status': self.tunnels_status } def to_html_template(self): """ Dictionary used to render FORM :return Dictionnary of configuration parameters """ """ Retrieve optional proxy configuration """ tmp = get_proxy(True) if tmp: proxy_configuration = "http-proxy-retry" + '\n' proxy_configuration += "http-proxy {} {}".format(tmp[0], tmp[1]) + '\n' else: proxy_configuration = "" """ And returns the attributes of the class """ return { 'id': str(self.id), 'node': self.node.name, 'remote_server': self.remote_server, 'remote_port': self.remote_port, 'proto': self.proto, 'status': self.status, 'tunnels_status': self.tunnels_status, 'proxy_configuration': proxy_configuration, 'ca': self.tls_profile.x509_certificate.get_base_filename() + ".chain", 'cert': self.tls_profile.x509_certificate.get_base_filename() + ".crt", 'key': self.tls_profile.x509_certificate.get_base_filename() + ".key" } def generate_conf(self): """ Render the conf with Jinja template and self.to_template() method :return The generated configuration as string, or raise """ try: jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH)) template_client = jinja2_env.get_template(JINJA_TEMPLATE_OPENVPN) conf = self.to_template() return {'template_client': template_client.render(conf)} # In ALL exceptions, associate an error message except Exception as e: # If there was an exception, raise a more general exception with the message and the traceback raise ServiceJinjaError( "Openvpn config generation error: '{}'".format(str(e)), "openvpn") def save_conf(self): """ Write configuration on disk """ conf = self.generate_conf() params = [ '/usr/local/etc/openvpn/openvpn_client.conf', conf['template_client'], OPENVPN_OWNER, "644" ] try: self.node.api_request('system.config.models.write_conf', config=params) except Exception as e: raise VultureSystemConfigError( "on node '{}'.\nRequest failure.".format(self.node.name))
class Issue(models.Model): _id = models.ObjectIdField() number = models.PositiveIntegerField(null=False) repo = models.ForeignKey("project.Repository", on_delete=models.CASCADE) project = models.ForeignKey("project.Project", on_delete=models.CASCADE) title = models.CharField(max_length=100) type = models.CharField(max_length=16, choices=ISSUE_TYPES, blank=True) labels = models.CharField(max_length=255, blank=True) invalid = models.BooleanField(default=False) # author = models.CharField(max_length=64, blank=True) service = models.CharField(max_length=16, choices=CLASSES_OF_SERVICE, blank=True) timeline = models.ArrayField(model_container=Event, blank=True) done = models.BooleanField(default=False, blank=True) created_at = models.DateTimeField(blank=True) updated_at = models.DateTimeField(blank=True) # Kanban stuff kanban_start_date = models.DateTimeField(blank=True) kanban_end_date = models.DateTimeField(blank=True) kanban_lead_time = models.PositiveIntegerField(blank=True) kanban_total_hold_time = models.PositiveIntegerField(blank=True) kanban_total_blocked_time = models.PositiveIntegerField(blank=True) kanban_metrics = models.DictField(default=dict) objects = models.DjongoManager() class Meta: # We can have the same issue on different projects unique_together = (("repo", "number", "project"), ) @cached_property def github(self): return get_github_issue(self.repo.identifier, self.number) @property def github_url(self): return format_html( "<a href='https://github.com/{path}' target=_blank>{path}</a>", path= f"{self.repo.organization}/{self.repo.code}/issues/{self.number}", ) @property def github_labels(self): return [l.name.lower() for l in self.github.labels] @cached_property def github_timeline(self): events = [] project_id = self.project.github_id for event in get_github_issue_timeline(self.github): project_card = event.raw_data.get("project_card", {}) label = event.raw_data.get("label", {}) if event.event in EVENT_LABEL_MAPPING and label.get("name") in ( "hold", "blocked", ): events.append( Event( type=EVENT_LABEL_MAPPING[event.event][label["name"]], date=as_utc(event.created_at), )) elif project_card and project_card["project_id"] == project_id: # Exclude "remove from project" for now if event.event not in TIMELINE_CARD_EVENT_TYPES: continue events.append( Event( type="moved", column=project_card["column_name"], date=as_utc(event.created_at), )) return events def get_service_type(self): if "critical" in self.github_labels: return "expedite" return "standard" def __str__(self): return f"{self.repo.code}/{self.number} - {self.title}" def _col(self, name): for column in self.project.columns: if column.code == name: return column # This might break something, but oh well return None @cached_property def _last_col_for_project(self): for column in self.project.columns[::-1]: if column.valid_wip: return column return None def process_timeline_data(self): # data = {column.code: {} for column in self.project.columns} # data["blocked"] = 0 # data["on_hold"] = 0 blocked_since = None on_hold_since = None start_date = None end_date = None lead_time = None blocked = 0 on_hold = 0 # current_column = None for event in self.github_timeline: if event.type == "on_hold": on_hold_since = event.date elif event.type == "blocked": blocked_since = event.date elif on_hold_since and event.type == "on_hold_removed": # Ignore on hold for less than 2 hours on_hold_time = on_hold_since - event.date on_hold_since = None if on_hold_time.seconds > 7200: on_hold += max(on_hold_time.days, 1) elif blocked_since and event.type == "blocked_removed": # Ignore block if less than 2 hours blocked_time = blocked_since - event.date blocked_since = None if blocked_time.seconds > 7200: blocked += max(blocked_time.days, 1) elif event.type == "moved": column = self._col(event.column) if start_date is None and column.valid_wip: start_date = event.date if column.code == self._last_col_for_project.code: end_date = event.date lead_time = max((end_date - start_date).days, 1) # TODO: Handle time in each column return { "kanban_start_date": start_date, "kanban_end_date": end_date, "kanban_lead_time": lead_time, "on_hold": on_hold, "blocked": blocked, } def save(self, *args, **kwargs): self.title = self.github.title self.labels = " | ".join(self.github_labels) # Find the first valid issue type for label in self.github_labels: if label in ISSUE_TYPE_LABEL_MAPPING: self.type = ISSUE_TYPE_LABEL_MAPPING[label] break else: self.type = "feature" self.service = self.get_service_type() self.created_at = as_utc(self.github.created_at) self.updated_at = as_utc(self.github.updated_at) self.timeline = self.github_timeline self.invalid = bool( set(self.github_labels) & set(INVALID_ISSUE_LABELS)) data = self.process_timeline_data() self.done = data["kanban_lead_time"] is not None self.kanban_start_date = data["kanban_start_date"] self.kanban_end_date = data["kanban_end_date"] self.kanban_lead_time = data["kanban_lead_time"] self.kanban_total_hold_time = data["on_hold"] self.kanban_total_blocked_time = data["blocked"] super().save(*args, **kwargs)