def upgrade(self, etc_dir): # import adblock path = os.path.join(etc_dir, "safety", "adblock.json") if not os.path.exists(path): path = os.path.join(etc_dir, "adblock.json") self.upgrade_adblock_impl(read_json_array(path)) # import privacy path = os.path.join(etc_dir, "safety", "privacy.json") if not os.path.exists(path): path = os.path.join(etc_dir, "privacy.json") self.upgrade_privacy_impl(read_json_array(path)) # now the new annoyances path = os.path.join(etc_dir, "safety", "subscriptions.json") if os.path.exists(path): self.upgrade_subscriptions(read_json_object(path))
def upgrade_ssl(self, folder): # upgrade advanced first if True: advanced_path = os.path.join(folder, 'ssl/exclude/advanced.conf') if os.path.isfile(advanced_path): advanced_data = load_payload(advanced_path) advanced_obj = ExcludeAdvanced.objects.first() advanced_obj.value_ssbump = "\n".join(advanced_data) advanced_obj.save() # upgrade NON bumped categories if True: path = os.path.join(os.path.dirname(folder), "safety", "non_bumped_categories.json") if os.path.exists(path): obj = ExcludeCategories.objects.first() entries = read_json_array(path) for entry in entries: attrname = "exclude_%s" % entry setattr(obj, attrname, True) obj.save() # upgrade ssl exclusions for name, s in { 'https_exclusions.conf': ExcludeDomainName.objects, 'ssl_exclude_domains.conf': ExcludeDomainName.objects, 'ssl/exclude/domains.conf': ExcludeDomainName.objects, 'ssl_exclude_ips.conf': ExcludeDomainIp.objects, 'ssl/exclude/ips.conf': ExcludeDomainIp.objects, 'ssl_exclude_subnets.conf': ExcludeDomainSubnet.objects, 'ssl/exclude/subnets.conf': ExcludeDomainSubnet.objects, }.iteritems(): # load file contents as array path1 = os.path.join(folder, name) if os.path.exists(path1): for (value, comment) in load_annotated_array(path1): try: v, c = s.get_or_create(value=value) v.bypass_sslbump = True v.comment = comment v.save() except Exception as e: # logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e))) pass # name to set map for name, s in { 'https_targets.conf': SslTargetDomain.objects, 'ssl_target_domains.conf': SslTargetDomain.objects, 'ssl/target/domains.conf': SslTargetDomain.objects, 'ssl_target_ips.conf': SslTargetIp.objects, 'ssl/target/ips.conf': SslTargetIp.objects, 'ssl_target_subnets.conf': SslTargetSubnet.objects, 'ssl/target/subnets.conf': SslTargetSubnet.objects, 'ssl_error_domains.conf': SslErrorDomain.objects, 'ssl/error/domains.conf': SslErrorDomain.objects, 'ssl_error_ips.conf': SslErrorIp.objects, 'ssl/error/ips.conf': SslErrorIp.objects, 'ssl_error_subnets.conf': SslErrorSubnet.objects, 'ssl/error/subnets.conf': SslErrorSubnet.objects }.iteritems(): # load file contents as array path1 = os.path.join(folder, name) if os.path.exists(path1): for (value, comment) in load_annotated_array(path1): try: v, c = s.get_or_create(value=value) v.comment = comment v.save() except Exception as e: # logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e))) pass
def upgrade_impl(self, folder): # adblock path = os.path.join(folder, "block_ads.json") if os.path.exists(path): data = read_json_object(path) obj = self.policy.ruleannoyances obj.block_ads = data.get('block_ads', False) obj.save() # privacy path = os.path.join(folder, "protect_privacy.json") if os.path.exists(path): data = read_json_object(path) obj = self.policy.ruleannoyances obj.protect_privacy = data.get('protect_privacy', False) obj.save() # new annoyances file path = os.path.join(folder, "block_annoyances.json") if os.path.exists(path): data = read_json_object(path) obj = self.policy.ruleannoyances obj.block_ads = data.get('block_ads', False) obj.protect_privacy = data.get('protect_privacy', False) obj.save() obj = self.policy.ruleapps obj.hide_yt_comments = data.get('hide_yt_comments', False) obj.hide_yt_suggestions = data.get('hide_yt_suggestions', False) obj.hide_yt_other = data.get('hide_yt_other', False) obj.save() # safe search may be in two places enable_safe_search = True # adult data = read_json_object(os.path.join(folder, "block_adult.json")) obj = self.policy.ruleadult if 'enable_safe_search' in data: enable_safe_search = data['enable_safe_search'] obj.enable_phrases = data['fulltext_detection_enable'] obj.phrases_maximum_weight = data['fulltext_detection_maximum_weight'] obj.phrases_maximum_size = data['fulltext_detection_maximum_size'] obj.fulltext_parse_links = data.get('fulltext_parse_links', False) obj.fulltext_scan_css = data.get('fulltext_scan_css', False) obj.fulltext_scan_javascript = data.get('fulltext_scan_javascript', False) obj.fulltext_scan_links = data.get('fulltext_scan_links', False) obj.enable_image_filtering = data.get('enable_image_filtering', False) obj.heuristics_maximum_weight = data.get('heuristics_maximum_weight', 40) obj.trust_allowed_categories = data.get('trust_allowed_categories', True) obj.enable_heuristics = data['enable_heuristics'] obj.save() # upgrade apps apps obj = self.policy.ruleapps path = os.path.join(folder, "apps.json") if os.path.exists(path): data = read_json_object(path) if 'enable_safe_search' in data: enable_safe_search = data['enable_safe_search'] obj.enable_google_apps = data.get('enable_google_apps', False) obj.hide_yt_comments = data.get('hide_yt_comments', False) obj.hide_yt_suggestions = data.get('hide_yt_suggestions', False) obj.hide_yt_other = data.get('hide_yt_other', False) obj.enable_safe_search = enable_safe_search obj.save() # normal items are simple for name, set in { 'block_urls.json': self.policy.ruleurl_set, 'block_file_name.json': self.policy.rulefilename_set, 'block_file_content.json': self.policy.rulefilecontent_set, 'block_domains.json': self.policy.ruledomain_set, 'block_content_type.json': self.policy.rulecontenttype_set, 'block_charset.json': self.policy.rulecharset_set, 'block_useragents.json': self.policy.ruleuseragent_set, }.iteritems(): # only import if file exists path = os.path.join(folder, name) if not os.path.exists(path): continue items = read_json_array(path) for item in items: if isinstance(item, basestring): value = item # in < 4.7 comment = "" else: value = item.get("value", "") comment = item.get("comment", "") if len(value) > 0: try: v, c = set.get_or_create(value=value) v.comment = comment v.save() except Exception as e: #logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e))) pass # categories are a bit different if True: data = read_json_object( os.path.join(folder, 'block_categories.json')) rule = self.policy.rulecategory if 'block_noncategorized_domains' in data: rule.block_noncategorized_domains = data[ 'block_noncategorized_domains'] items = None if 'blocked_categories' in data: items = data['blocked_categories'] else: items = data for item in items: setattr(rule, "block_%s" % name, True) rule.save() # dynamic categories are different too if True: data = read_json_object( os.path.join(folder, 'dynamic_categorization.json')) rule = self.policy.ruledynamiccategory rule.enabled = data.get('enabled', True) rule.analyze_request = data.get('analyze_request', True) rule.analyze_response = data.get('analyze_response', True) rule.classify_type = data.get( 'classify_type', RuleDynamicCategory.CLASSIFY_UNKNOWN_AND_UNTRUSTED) if 'classifiers' in data: items = data['classifiers'] for item in items: setattr(rule, "classify_%s" % name, True) rule.save() # custom categories are even more different for name, obj in { 'block_categories_custom.json': self.policy.rulecategorycustom_set }.iteritems(): for item in read_json_array(os.path.join(folder, name)): (category, created) = CustomCategory.objects.get_or_create(name=item) obj.get_or_create(category=category.name, enable=True) # block file size obj = self.policy.rulefilesize path = os.path.join(folder, "block_file_size.json") if os.path.exists(path): data = read_json_object(path) obj.enable = data.get('enable', False) obj.max_size = data.get('max_size', 1048576) obj.save()
def upgrade_impl(self, major, minor, folder): # normal items are simple for name, set in { 'user_ip.json': self.policy.memberip_set, 'user_ip6.json': self.policy.memberip_set, 'user_ip_range.json': self.policy.memberrange_set, 'user_ip6_range.json': self.policy.memberrange_set, 'user_ip_subnet.json': self.policy.membersubnet_set, 'user_ip6_subnet.json': self.policy.membersubnet_set, 'user_name.json': self.policy.membername_set, }.iteritems(): file = os.path.join(folder, name) if os.path.exists(file): items = read_json_array(file) for item in items: if isinstance(item, basestring): value = item # in < 4.7 comment = "" else: value = item.get("value", "") comment = item.get("comment", "") if len(value) > 0: try: v, c = set.get_or_create(value=value) v.comment = comment v.save() except Exception as e: # logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e))) pass # ldap needs to be parsed manually and differently in 4.6+ for item in read_json_array(os.path.join(folder, 'user_ldap.json')): if (("%d.%d" % (major, minor)) in ["4.6", "4.7", "4.8", "4.9" ]) or major >= 5: obj, created = self.policy.memberldap_set.get_or_create( dn=item["dn"]) obj.name = item["name"] obj.recursive = item["recursive"] if "comment" in item: obj.comment = item["comment"] obj.save() else: test_str0 = "memberOf=" test_str1 = "memberOf:1.2.840.113556.1.4.1941:=" # determine recursiveness recursive = False pos = item.rfind(test_str1) if pos != -1: recursive = True pos += len(test_str1) else: pos = item.rfind(test_str0) if pos == -1: continue else: pos += len(test_str0) # determine dn dn = item[pos:] while dn.strip(')') != dn: dn = dn.strip(')') # determine name matched = re.match(r'cn=(.*?),.*', dn, re.M | re.I) if matched: group_name = matched.group(1) else: group_name = "unknown" obj, created = self.policy.memberldap_set.get_or_create(dn=dn) obj.name = group_name obj.recursive = recursive obj.save()
def upgrade(self, etc_dir): path = os.path.join(etc_dir, "safety", "trusted_categories.json") if os.path.exists(path): self.upgrade_impl(read_json_array(path))
def upgrade(self, etc_dir): path = os.path.join(etc_dir, "safety", "recategorized_domains.json") if os.path.exists(path): self.upgrade_impl(read_json_array(path))
def upgrade(self, etc_dir): path = os.path.join(etc_dir, "safety", "bypass_tokens.json") if os.path.exists(path): self.upgrade_impl(read_json_array(path))