def upgrade(self, etc_dir):
# import adblock
path = os.path.join(etc_dir, "safety", "adblock.json")
if not os.path.exists(path):
path = os.path.join(etc_dir, "adblock.json")
self.upgrade_adblock_impl(read_json_array(path))
# import privacy
path = os.path.join(etc_dir, "safety", "privacy.json")
if not os.path.exists(path):
path = os.path.join(etc_dir, "privacy.json")
self.upgrade_privacy_impl(read_json_array(path))
# now the new annoyances
path = os.path.join(etc_dir, "safety", "subscriptions.json")
if os.path.exists(path):
self.upgrade_subscriptions(read_json_object(path))
def upgrade_ssl(self, folder):
# upgrade advanced first
if True:
advanced_path = os.path.join(folder, 'ssl/exclude/advanced.conf')
if os.path.isfile(advanced_path):
advanced_data = load_payload(advanced_path)
advanced_obj = ExcludeAdvanced.objects.first()
advanced_obj.value_ssbump = "\n".join(advanced_data)
advanced_obj.save()
# upgrade NON bumped categories
if True:
path = os.path.join(os.path.dirname(folder), "safety",
"non_bumped_categories.json")
if os.path.exists(path):
obj = ExcludeCategories.objects.first()
entries = read_json_array(path)
for entry in entries:
attrname = "exclude_%s" % entry
setattr(obj, attrname, True)
obj.save()
# upgrade ssl exclusions
for name, s in {
'https_exclusions.conf': ExcludeDomainName.objects,
'ssl_exclude_domains.conf': ExcludeDomainName.objects,
'ssl/exclude/domains.conf': ExcludeDomainName.objects,
'ssl_exclude_ips.conf': ExcludeDomainIp.objects,
'ssl/exclude/ips.conf': ExcludeDomainIp.objects,
'ssl_exclude_subnets.conf': ExcludeDomainSubnet.objects,
'ssl/exclude/subnets.conf': ExcludeDomainSubnet.objects,
}.iteritems():
# load file contents as array
path1 = os.path.join(folder, name)
if os.path.exists(path1):
for (value, comment) in load_annotated_array(path1):
try:
v, c = s.get_or_create(value=value)
v.bypass_sslbump = True
v.comment = comment
v.save()
except Exception as e:
# logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e)))
pass
# name to set map
for name, s in {
'https_targets.conf': SslTargetDomain.objects,
'ssl_target_domains.conf': SslTargetDomain.objects,
'ssl/target/domains.conf': SslTargetDomain.objects,
'ssl_target_ips.conf': SslTargetIp.objects,
'ssl/target/ips.conf': SslTargetIp.objects,
'ssl_target_subnets.conf': SslTargetSubnet.objects,
'ssl/target/subnets.conf': SslTargetSubnet.objects,
'ssl_error_domains.conf': SslErrorDomain.objects,
'ssl/error/domains.conf': SslErrorDomain.objects,
'ssl_error_ips.conf': SslErrorIp.objects,
'ssl/error/ips.conf': SslErrorIp.objects,
'ssl_error_subnets.conf': SslErrorSubnet.objects,
'ssl/error/subnets.conf': SslErrorSubnet.objects
}.iteritems():
# load file contents as array
path1 = os.path.join(folder, name)
if os.path.exists(path1):
for (value, comment) in load_annotated_array(path1):
try:
v, c = s.get_or_create(value=value)
v.comment = comment
v.save()
except Exception as e:
# logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e)))
pass
def upgrade_impl(self, folder):
# adblock
path = os.path.join(folder, "block_ads.json")
if os.path.exists(path):
data = read_json_object(path)
obj = self.policy.ruleannoyances
obj.block_ads = data.get('block_ads', False)
obj.save()
# privacy
path = os.path.join(folder, "protect_privacy.json")
if os.path.exists(path):
data = read_json_object(path)
obj = self.policy.ruleannoyances
obj.protect_privacy = data.get('protect_privacy', False)
obj.save()
# new annoyances file
path = os.path.join(folder, "block_annoyances.json")
if os.path.exists(path):
data = read_json_object(path)
obj = self.policy.ruleannoyances
obj.block_ads = data.get('block_ads', False)
obj.protect_privacy = data.get('protect_privacy', False)
obj.save()
obj = self.policy.ruleapps
obj.hide_yt_comments = data.get('hide_yt_comments', False)
obj.hide_yt_suggestions = data.get('hide_yt_suggestions', False)
obj.hide_yt_other = data.get('hide_yt_other', False)
obj.save()
# safe search may be in two places
enable_safe_search = True
# adult
data = read_json_object(os.path.join(folder, "block_adult.json"))
obj = self.policy.ruleadult
if 'enable_safe_search' in data:
enable_safe_search = data['enable_safe_search']
obj.enable_phrases = data['fulltext_detection_enable']
obj.phrases_maximum_weight = data['fulltext_detection_maximum_weight']
obj.phrases_maximum_size = data['fulltext_detection_maximum_size']
obj.fulltext_parse_links = data.get('fulltext_parse_links', False)
obj.fulltext_scan_css = data.get('fulltext_scan_css', False)
obj.fulltext_scan_javascript = data.get('fulltext_scan_javascript',
False)
obj.fulltext_scan_links = data.get('fulltext_scan_links', False)
obj.enable_image_filtering = data.get('enable_image_filtering', False)
obj.heuristics_maximum_weight = data.get('heuristics_maximum_weight',
40)
obj.trust_allowed_categories = data.get('trust_allowed_categories',
True)
obj.enable_heuristics = data['enable_heuristics']
obj.save()
# upgrade apps apps
obj = self.policy.ruleapps
path = os.path.join(folder, "apps.json")
if os.path.exists(path):
data = read_json_object(path)
if 'enable_safe_search' in data:
enable_safe_search = data['enable_safe_search']
obj.enable_google_apps = data.get('enable_google_apps', False)
obj.hide_yt_comments = data.get('hide_yt_comments', False)
obj.hide_yt_suggestions = data.get('hide_yt_suggestions', False)
obj.hide_yt_other = data.get('hide_yt_other', False)
obj.enable_safe_search = enable_safe_search
obj.save()
# normal items are simple
for name, set in {
'block_urls.json': self.policy.ruleurl_set,
'block_file_name.json': self.policy.rulefilename_set,
'block_file_content.json': self.policy.rulefilecontent_set,
'block_domains.json': self.policy.ruledomain_set,
'block_content_type.json': self.policy.rulecontenttype_set,
'block_charset.json': self.policy.rulecharset_set,
'block_useragents.json': self.policy.ruleuseragent_set,
}.iteritems():
# only import if file exists
path = os.path.join(folder, name)
if not os.path.exists(path):
continue
items = read_json_array(path)
for item in items:
if isinstance(item, basestring):
value = item # in < 4.7
comment = ""
else:
value = item.get("value", "")
comment = item.get("comment", "")
if len(value) > 0:
try:
v, c = set.get_or_create(value=value)
v.comment = comment
v.save()
except Exception as e:
#logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e)))
pass
# categories are a bit different
if True:
data = read_json_object(
os.path.join(folder, 'block_categories.json'))
rule = self.policy.rulecategory
if 'block_noncategorized_domains' in data:
rule.block_noncategorized_domains = data[
'block_noncategorized_domains']
items = None
if 'blocked_categories' in data:
items = data['blocked_categories']
else:
items = data
for item in items:
setattr(rule, "block_%s" % name, True)
rule.save()
# dynamic categories are different too
if True:
data = read_json_object(
os.path.join(folder, 'dynamic_categorization.json'))
rule = self.policy.ruledynamiccategory
rule.enabled = data.get('enabled', True)
rule.analyze_request = data.get('analyze_request', True)
rule.analyze_response = data.get('analyze_response', True)
rule.classify_type = data.get(
'classify_type',
RuleDynamicCategory.CLASSIFY_UNKNOWN_AND_UNTRUSTED)
if 'classifiers' in data:
items = data['classifiers']
for item in items:
setattr(rule, "classify_%s" % name, True)
rule.save()
# custom categories are even more different
for name, obj in {
'block_categories_custom.json':
self.policy.rulecategorycustom_set
}.iteritems():
for item in read_json_array(os.path.join(folder, name)):
(category,
created) = CustomCategory.objects.get_or_create(name=item)
obj.get_or_create(category=category.name, enable=True)
# block file size
obj = self.policy.rulefilesize
path = os.path.join(folder, "block_file_size.json")
if os.path.exists(path):
data = read_json_object(path)
obj.enable = data.get('enable', False)
obj.max_size = data.get('max_size', 1048576)
obj.save()
def upgrade_impl(self, major, minor, folder):
# normal items are simple
for name, set in {
'user_ip.json': self.policy.memberip_set,
'user_ip6.json': self.policy.memberip_set,
'user_ip_range.json': self.policy.memberrange_set,
'user_ip6_range.json': self.policy.memberrange_set,
'user_ip_subnet.json': self.policy.membersubnet_set,
'user_ip6_subnet.json': self.policy.membersubnet_set,
'user_name.json': self.policy.membername_set,
}.iteritems():
file = os.path.join(folder, name)
if os.path.exists(file):
items = read_json_array(file)
for item in items:
if isinstance(item, basestring):
value = item # in < 4.7
comment = ""
else:
value = item.get("value", "")
comment = item.get("comment", "")
if len(value) > 0:
try:
v, c = set.get_or_create(value=value)
v.comment = comment
v.save()
except Exception as e:
# logging.warning("Ignoring import of %s, error '%s' ..." % (value, str(e)))
pass
# ldap needs to be parsed manually and differently in 4.6+
for item in read_json_array(os.path.join(folder, 'user_ldap.json')):
if (("%d.%d" % (major, minor)) in ["4.6", "4.7", "4.8", "4.9"
]) or major >= 5:
obj, created = self.policy.memberldap_set.get_or_create(
dn=item["dn"])
obj.name = item["name"]
obj.recursive = item["recursive"]
if "comment" in item:
obj.comment = item["comment"]
obj.save()
else:
test_str0 = "memberOf="
test_str1 = "memberOf:1.2.840.113556.1.4.1941:="
# determine recursiveness
recursive = False
pos = item.rfind(test_str1)
if pos != -1:
recursive = True
pos += len(test_str1)
else:
pos = item.rfind(test_str0)
if pos == -1:
continue
else:
pos += len(test_str0)
# determine dn
dn = item[pos:]
while dn.strip(')') != dn:
dn = dn.strip(')')
# determine name
matched = re.match(r'cn=(.*?),.*', dn, re.M | re.I)
if matched:
group_name = matched.group(1)
else:
group_name = "unknown"
obj, created = self.policy.memberldap_set.get_or_create(dn=dn)
obj.name = group_name
obj.recursive = recursive
obj.save()
def upgrade(self, etc_dir):
path = os.path.join(etc_dir, "safety", "trusted_categories.json")
if os.path.exists(path):
self.upgrade_impl(read_json_array(path))
def upgrade(self, etc_dir):
path = os.path.join(etc_dir, "safety", "recategorized_domains.json")
if os.path.exists(path):
self.upgrade_impl(read_json_array(path))
def upgrade(self, etc_dir):
path = os.path.join(etc_dir, "safety", "bypass_tokens.json")
if os.path.exists(path):
self.upgrade_impl(read_json_array(path))