def generate_schedule(self, policy_dir, policy):
# generate schedule array
s = []
for schedule in policy.schedule_set.all():
s.append({
"days": {
"mon": schedule.on_mon,
"tue": schedule.on_tue,
"wed": schedule.on_wed,
"thu": schedule.on_thu,
"fri": schedule.on_fri,
"sat": schedule.on_sat,
"sun": schedule.on_sun
},
"from": {
"hours": schedule.from_hours,
"minutes": schedule.from_mins
},
"to": {
"hours": schedule.to_hours,
"minutes": schedule.to_mins
}
})
# create writer and dumper
w = FileWriter(policy_dir)
d = JsonDumper()
# write and dump
w.write('schedule.json', d.dumps(s))
def post(self, request, *args, **kwargs):
form = ClusterClientForm(request.POST)
if form.is_valid():
# ok form is valid, fill the data
data = {
"enabled" : form.cleaned_data['enabled'],
"server_port" : form.cleaned_data['server_port'],
"server_address": form.cleaned_data['server_address'],
"sync_interval" : form.cleaned_data['sync_interval']
}
# and write it
w = FileWriter(os.path.join(Paths.etc_dir(), "node"))
d = JsonDumper()
w.write('cluster_client.json', d.dumps(data))
# mark as needing relstart
messages.info(request, "need_squid_restart")
return HttpResponseRedirect(self.get_success_url())
return render(request, self.template_name, {'form': form })
def write(self, folder, file, text):
# construct data string
data_str = ""
data_str += "#\n"
data_str += "# PLEASE DO NOT EDIT THIS FILE MANUALLY - it is regenerated by Web Safety UI\n"
data_str += "#\n"
data_str += "#-----BEGIN GENERATED CONFIGURATION-----\n"
data_str += text + "\n"
data_str += "#-----END GENERATED CONFIGURATION-----\n"
# and now write
writer = FileWriter(folder)
writer.write(file, data_str)
def generate_config(self, target_dir):
# create writer and dumper
w = FileWriter(target_dir)
d = JsonDumper()
self.generate_safebrowsing(w, d)
def generate_ruleapps(self, rules_dir, rule):
o = rule
d = {
"enable_google_apps": o.enable_google_apps,
"enable_safe_search": o.enable_safe_search,
"youtube_restrictions": o.youtube_restrictions
}
FileWriter(rules_dir).write('apps.json', JsonDumper().dumps(d))
def update_comment(self, username, comment):
data = {}
name = self.json
if os.path.exists(name):
with open(name) as fin:
data = json.load(fin)
data[username] = {
"comment" : comment
}
# and save it
w = FileWriter(self.etc_dir)
d = JsonDumper()
w.write("users.json", d.dumps(data))
def write_auth_labels(self, new_dir):
# create writer and dumper
w = FileWriter(new_dir)
d = JsonDumper()
# write auth jsons
self.generate_auth_labels(w, d)
def generate_policy(self, policy_dir, policy):
# first generate the schedule
self.generate_schedule(policy_dir, policy)
# now generate the policy
o = policy
a = {
"enabled": o.advanced.enable,
"priority": o.priority,
"ignore_case_exclusions": o.advanced.ignore_case,
"ignore_case_members": o.advanced.ignore_case,
"ignore_case_rules": o.advanced.ignore_case,
"use_url_as_filename": True,
"description": o.advanced.comment,
"hide_history": o.advanced.hide_history,
"hide_result_info": o.advanced.hide_result_info,
"sslbump": o.advanced.sslbump,
"tunnel_block": o.advanced.tunnel_block,
# bypass settings
"bypass_allowed": o.bypass_allowed,
"bypass_strip_www": o.bypass_strip_www,
"bypass_children": o.bypass_children,
"bypass_referers": o.bypass_referers,
"bypass_duration": o.bypass_duration,
"bypass_token_required": o.bypass_token_required,
"bypass_adult": o.bypass_adult,
"bypass_categories": o.bypass_categories,
"bypass_file": o.bypass_file,
"bypass_adblock": o.bypass_adblock,
"bypass_privacy": o.bypass_privacy,
"bypass_http": o.bypass_http
}
# create writer and dumper
w = FileWriter(policy_dir)
d = JsonDumper()
# write and dump
w.write('policy.json', d.dumps(a))
# now generate policy components
self.generate_members(policy_dir, policy)
self.generate_exclusions(policy_dir, policy)
self.generate_rules(policy_dir, policy)
def generate_config(self, target_dir):
# create writer and dumper
w = FileWriter(target_dir)
d = JsonDumper()
# generate all global files
self.generate_sharing(w, d)
def generate_config(self, target_dir):
# create writer and dumper
w = FileWriter(target_dir)
d = JsonDumper()
# and generate
self.generate_reports(w, d)
self.generate_monitor(w, d)
self.generate_wsmgrd(w, d)
def generate_members(self, policy_dir, policy):
# construct the dir name
dir = os.path.join(policy_dir, "members")
# recreate it
self.recreate_dir(dir)
# create writer and dumper
w = FileWriter(dir)
j = JsonDumper()
# write and dump
for (n, v) in [('user_ip.json', policy.memberip_set),
('user_name.json', policy.membername_set),
('user_ip_range.json', policy.memberrange_set),
('user_ip_subnet.json', policy.membersubnet_set)]:
w.write(n, j.dumps(self.to_array(v.all())))
# construct ldap groups out of single ldap server and group dns
groups = []
for group in policy.memberldap_set.all():
groups.append({
"name": group.name,
"dn": group.dn,
"recursive": group.recursive,
"comment": group.comment
})
w.write('user_ldap.json', j.dumps(groups))
def write(self, folder, file, objects):
# based on each django object we create an array of (value => comments[])
data = []
for object in objects:
data.append({
"value": object.value,
"comments": object.comment.splitlines()
})
# for squid it is important that conf files containing lists of objects are sorted
data.sort(key=lambda x: len(x["value"]))
# construct data string
data_str = ""
data_str += "#\n"
data_str += "# PLEASE DO NOT EDIT THIS FILE MANUALLY - it is regenerated by Web Safety UI\n"
data_str += "#\n"
data_str += "#-----BEGIN GENERATED CONFIGURATION-----\n"
for item in data:
item_str = ""
for comment in item["comments"]:
stripped = comment.rstrip().rstrip('\n').rstrip()
if len(stripped) > 0:
item_str += "# %s\n" % stripped
else:
item_str += "#\n"
if len(item_str) > 0:
data_str += item_str
data_str += "%s\n" % item["value"]
data_str += "#-----END GENERATED CONFIGURATION-----\n"
# and now write
writer = FileWriter(folder)
writer.write(file, data_str)
def generate_ruleannoyances(self, rules_dir, rule1, rule2):
o1 = rule1 # annoyances
o2 = rule2 # apps
d = {
"block_ads": o1.block_ads,
"protect_privacy": o1.protect_privacy,
"hide_yt_comments": o2.hide_yt_comments,
"hide_yt_suggestions": o2.hide_yt_suggestions,
"hide_yt_other": o2.hide_yt_other
}
FileWriter(rules_dir).write('block_annoyances.json',
JsonDumper().dumps(d))
def generate_rules(self, policy_dir, policy):
# construct rules folder
rules_dir = os.path.join(policy_dir, "rules")
# recreate it
self.recreate_dir(rules_dir)
# create writer and dumper
w = FileWriter(rules_dir)
j = JsonDumper()
# call individual generators
self.generate_ruleannoyances(rules_dir, policy.ruleannoyances,
policy.ruleapps)
self.generate_ruleadult(rules_dir, policy.ruleadult)
self.generate_ruleapps(rules_dir, policy.ruleapps)
self.generate_rulefilesize(rules_dir, policy.rulefilesize)
self.generate_rule_dynamic_categorization(rules_dir, policy)
# write different file types
for (n, t) in [
('block_domains.json', policy.ruledomain_set),
('block_urls.json', policy.ruleurl_set),
('block_useragents.json', policy.ruleuseragent_set),
('block_content_type.json', policy.rulecontenttype_set),
('block_file_name.json', policy.rulefilename_set),
('block_file_content.json', policy.rulefilecontent_set),
('block_charset.json', policy.rulecharset_set),
]:
w.write(n, j.dumps(self.to_array(t.all())))
# write blocked categories
w.write(
'block_categories.json',
j.dumps({
"block_noncategorized_domains":
policy.rulecategory.block_noncategorized_domains,
"blocked_categories":
policy.rulecategory.categories_as_array()
}))
# write custom categories
array = []
for r in policy.rulecategorycustom_set.filter(enable=True):
array.append(r.category)
w.write('block_categories_custom.json', j.dumps(array))
def generate_rule_dynamic_categorization(self, rules_dir, policy):
o = policy.ruledynamiccategory
d = {
"enabled": o.enabled,
"analyze_request": o.analyze_request,
"analyze_response": o.analyze_response,
"classify_type": o.classify_type,
"classifiers": o.categories_as_array(),
"token_proximity": 4, # hardcoded and is NOT visible in UI
"check_delimiters": True, # hardcoded and is NOT visible in UI
"probability": 40 # hardcoded and is NOT visible in UI
}
FileWriter(rules_dir).write('dynamic_categorization.json',
JsonDumper().dumps(d))
def generate_ruleadult(self, rules_dir, rule):
o = rule
d = {
"trust_allowed_categories": o.trust_allowed_categories,
"enable_heuristics": o.enable_heuristics,
"heuristics_maximum_weight": o.heuristics_maximum_weight,
"fulltext_detection_enable": o.enable_phrases,
"fulltext_detection_maximum_weight": o.phrases_maximum_weight,
"fulltext_detection_maximum_size": o.phrases_maximum_size,
"fulltext_scan_links": o.phrases_scan_links,
"fulltext_scan_javascript": o.phrases_scan_javascript,
"fulltext_scan_css": o.phrases_scan_css,
"fulltext_parse_links": o.phrases_parse_links,
"enable_image_filtering": o.enable_image_filtering
}
FileWriter(rules_dir).write('block_adult.json', JsonDumper().dumps(d))
def test_ldap(self, object):
# here we have django model auth domain updated, but we also need to generate JSON files in /opt/websafety/etc
g = Generator(os.path.join(Paths.etc_dir(), "squid"))
w = FileWriter(os.path.join(Paths.etc_dir(), "squid"))
d = JsonDumper()
g.generate_auth(w, d)
# and run the tests
result = LdapTester().run();
if result['exit_code'] == 0:
return (True, "%s %s" % (result['stdout'], result['stderr']))
else:
return (False, "Exit code: %s\nSTDOUT: %s\nSTDERR: %s\n" % (str(result['exit_code']), result['stdout'], result['stderr']))
pass
def generate_config(self, target_dir):
# create writer and dumper
w = FileWriter(target_dir)
d = JsonDumper()
# generate all global files
self.generate_network(w, d)
self.generate_apps(w, d)
self.generate_subscriptions(w, d)
self.generate_trusted_categories(w, d)
self.generate_recategorized_domains(w, d)
self.generate_skip_sslbump(w, d)
self.generate_engine(w, d)
self.generate_bypass_tokens(w, d)
# now generate policies
self.generate_policies(target_dir)
def generate(self):
etc_dir = Paths.etc_dir()
# run all generators first
NodeGenerator(os.path.join(etc_dir, "node")).generate()
TrafficGenerator(os.path.join(etc_dir, "traffic")).generate()
SafetyGenerator(os.path.join(etc_dir, "safety")).generate()
SquidGenerator(os.path.join(etc_dir, "squid")).generate()
AntivirusGenerator(os.path.join(etc_dir, "antivirus")).generate()
# dump the copy of the database to /opt/websafety/var/db
self.dump_db()
# update config timestamp
d = {'version': self.version, 'timestamp': self.timestamp}
FileWriter(etc_dir).write('config.json', JsonDumper().dumps(d))
# and archive it
self.archive()
def generate_exclusions(self, policy_dir, policy):
# construct the dir name
dir = os.path.join(policy_dir, "exclusions")
# create it
self.recreate_dir(dir)
# create writer and dumper
w = FileWriter(dir)
j = JsonDumper()
# write and dump
for (n, v) in [('domain_name.json', policy.exclusiondomain_set),
('domain_ip.json', policy.exclusionip_set),
('content_type.json', policy.exclusioncontenttype_set),
('domain_ip_range.json', policy.exclusionrange_set),
('domain_ip_subnet.json', policy.exclusionsubnet_set),
('request_uri.json', policy.exclusionurl_set)]:
array = []
objects = v.all()
for object in objects:
array.append({
"value": object.value,
"comment": object.comment,
"scan": {
"adult": object.scan_adult,
"categories": object.scan_categories,
"file": object.scan_file,
"adblock": object.scan_adblock,
"privacy": object.scan_privacy,
"http": object.scan_http,
}
})
w.write(n, j.dumps(array))
# and also advanced
w.write(
'advanced.json',
j.dumps({"exclude_by_referer":
policy.advanced.exclude_by_referer}))
import django
import logging
sys.path.append(os.path.abspath(__file__))
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "console.settings")
django.setup()
from _domain.core import Paths
from _domain.utils import JsonDumper, FileWriter
from traffic.generator import Generator
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
logging.info(
"Syncing monitoring settings from /opt/websafety/var/db/config.sqlite to /opt/websafety/etc/traffic/monitor.json..."
)
f = os.path.join(Paths.etc_dir(), "traffic")
logging.info("Generating monitor.json in folder %s" % f)
g = Generator(f)
w = FileWriter(f)
d = JsonDumper()
g.generate_monitor(w, d)
logging.info("Success!!!")
def generate_rulefilesize(self, rules_dir, rule):
o = rule
d = {"enable": o.enable, "max_size": o.max_size}
FileWriter(rules_dir).write('block_file_size.json',
JsonDumper().dumps(d))