def update_hitcount(self, domain=None, protocol=None, whitelistobj=None):
"""Look for matching domains and update all matching entries regardless
of whether they are enabled or not.
If a queryset is specified, then just update the hitcount on that.
We are allowing general rules to co-exist with specific ones, e.g:
images.slashdot.org and *.slashdot.org could both be in the database.
Hitcount will updated on both entries for the images.slashdot.org domain."""
if whitelistobj:
whitelistobj.hitcount = F('hitcount') + 1
whitelistobj.save()
else:
lall = Whitelist.get_wildcard_choice("ALL")
lone = Whitelist.get_wildcard_choice("ONE")
return Whitelist.objects.filter(
Q(domain=domain, protocol=protocol)
| Q(domain=self.all_wildcard(domain),
protocol=protocol,
wildcard=lall)
| Q(domain=self.one_label_wildcard(domain),
protocol=protocol,
wildcard=lone)).update(hitcount=F('hitcount') + 1)
def testEnableDomain(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "",
"domain": "testing4.com",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertTemplateUsed(response, 'whitelist/whitelist_added.html')
self.assertContains(response,
"Whitetrash: Access Granted",
status_code=200)
self.assertTrue(
Whitelist.objects.filter(
domain="testing4.com",
protocol=Whitelist.get_protocol_choice("HTTP"),
enabled=True))
#Make sure we didn't create a new entry and leave the old one there.
self.assertFalse(
Whitelist.objects.filter(
domain="testing4.com",
protocol=Whitelist.get_protocol_choice("HTTP"),
enabled=False))
if settings.MEMCACHE:
result = settings.MEMCACHE.get(
"testing4.com|%s" % Whitelist.get_protocol_choice("HTTP"))
self.assertTrue(
result,
"Domain should be present because of the save operation when enabled"
)
(id, enabled) = result
self.assertTrue(enabled,
"Domain should be enabled in the memcache")
def check_captcha(form, request):
"""Check the CAPTCHA solution. If the solution is bad, return a render_to_response (True)
if the solution is correct, return False"""
if ((settings.CAPTCHA_HTTP
and protocol == Whitelist.get_protocol_choice('HTTP'))
or (settings.CAPTCHA_SSL
and protocol == Whitelist.get_protocol_choice('SSL'))):
captcha_passed = False
settings.LOG.debug("CAPTCHA response: %s" %
form.cleaned_data['captcha_response'])
for (sol, createtime) in request.session['captcha_solns']:
for thissol in sol:
if sha1(form.cleaned_data['captcha_response']).hexdigest(
) == thissol:
if ((datetime.datetime.now() - createtime) <
datetime.timedelta(
seconds=settings.CAPTCHA_WINDOW_SEC)):
request.session['captcha_solns'].remove(
(sol, createtime))
request.session.save()
captcha_passed = True
else:
settings.LOG.debug(
"CAPTCHA timediff: %s, window: %s " %
(datetime.datetime.now() - createtime,
settings.CAPTCHA_WINDOW_SEC))
form._errors["captcha_response"] = ErrorList(
["Captcha time window expired."])
return render_to_response(
'whitelist/whitelist_getform.html', {
'form': form,
'captcha': True
},
context_instance=RequestContext(request))
if not captcha_passed:
settings.LOG.debug("CAPTCHA response '%s' incorrect" %
form.cleaned_data['captcha_response'])
form._errors["captcha_response"] = ErrorList(
["Captcha test failed. Please try again."])
return render_to_response('whitelist/whitelist_getform.html', {
'form': form,
'captcha': True
},
context_instance=RequestContext(request))
return False
def testCheckDomainNotInList(self):
response = self.client.get(
"/whitelist/checkdomain/", {
"domain": "notinwhitelist.com",
"protocol": Whitelist.get_protocol_choice("HTTP")
})
self.assertContains(response, "0", status_code=200)
response = self.client.get(
"/whitelist/checkdomain/", {
"domain": "testing4.com",
"protocol": Whitelist.get_protocol_choice("HTTP")
})
self.assertContains(response, "0", status_code=200)
def testAddBadDomain(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//sldjflksjdf.com/",
"domain": "test1.invalidtoolong",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertTemplateUsed(response, 'whitelist/whitelist_getform.html')
self.assertFalse(
Whitelist.objects.filter(
domain="test1.invalidtoolong",
protocol=Whitelist.get_protocol_choice("HTTP")))
self.assertFormError(response, 'form', 'domain', 'Bad domain name.')
def is_whitelisted(self, domain, protocol):
"""Return true (the queryset) if the domain is whitelisted"""
lall = Whitelist.get_wildcard_choice("ALL")
lone = Whitelist.get_wildcard_choice("ONE")
return Whitelist.objects.filter(
Q(enabled=True, domain=domain, protocol=protocol)
| Q(enabled=True,
domain=self.all_wildcard(domain),
protocol=protocol,
wildcard=lall) | Q(enabled=True,
domain=self.one_label_wildcard(domain),
protocol=protocol,
wildcard=lone))
def get_or_create_disabled(self, domain, protocol, url, src_ip):
"""Return the disabled domain entry, creating one if necessary.
Note we don't guarantee the whitelist entry returned will be disabled.
We assume you checked is_whitelisted first. I'd rather have a race condition
where there is the chance we will return an enabled domain rather than create
a second entry for the domain, one enabled and one disabled.
Returning an enabled object is not a big problem since we only use this for updating
the hit count. If an entry is created, it is created disabled."""
user = User.objects.filter(username="******").get()
w, created = Whitelist.objects.get_or_create(
domain=domain,
protocol=protocol,
defaults={
'user': user,
'url': url,
'comment': "",
'enabled': False,
'client_ip': src_ip,
'wildcard': Whitelist.get_wildcard_choice("NONE"),
'hitcount': 0
})
return w
def testCheckDomainNotWildcarded(self):
"""If testing.com is in the whitelist, ONLY one level of subdomain should be allowed."""
response = self.client.get(
"/whitelist/checkdomain/", {
"domain": "www.test.testing1.com",
"protocol": Whitelist.get_protocol_choice("HTTP")
})
self.assertContains(response, "0", status_code=200)
def testAlreadyWhitelisted(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "http://anewurl",
"domain": "testing1.com",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "anewcomment"
})
self.assertTemplateUsed(response,
'whitelist/whitelist_already_listed.html')
self.assertContains(response, "http://anewurl", status_code=200)
#Check we didn't set a new url or comment - the original is intact.
self.assertTrue(
Whitelist.objects.filter(
domain="testing1.com",
protocol=Whitelist.get_protocol_choice("HTTP"),
url="http://testing1.com",
comment="to be deleted"))
def testAddSSL(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "",
"domain": "test1.com",
"protocol": Whitelist.get_protocol_choice("HTTPS"),
"comment": "testing"
})
self.assertTemplateUsed(response, 'whitelist/whitelist_added.html')
self.assertContains(response,
"Whitetrash: Access Granted",
status_code=200)
self.assertContains(response,
"Thank you whitetrashtestuser",
status_code=200)
self.assertTrue(
Whitelist.objects.filter(
domain="test1.com",
protocol=Whitelist.get_protocol_choice("HTTPS")))
def testAddBlacklistedDomain(self):
"""Adding a known bad blacklisted domain should fail if safebrowsing is enabled."""
if settings.SAFEBROWSING:
self.client.login(username='******', password='******')
response = self.client.post(
"/whitelist/addentry/", {
"url":
"http://malware.testing.google.test/testing/malware/",
"domain": "malware.testing.google.test",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertRedirects(
response,
"%s%s/whitelist/attackdomain=malware.testing.google.test" %
(settings.SERV_PREFIX, settings.DOMAIN),
status_code=302,
target_status_code=200)
self.assertFalse(
Whitelist.objects.filter(
domain="malware.testing.google.test",
protocol=Whitelist.get_protocol_choice("HTTP")))
#Same for SSL
response = self.client.post(
"/whitelist/addentry/", {
"url":
"https://malware.testing.google.test/testing/malware/",
"domain": "malware.testing.google.test",
"protocol": Whitelist.get_protocol_choice("HTTPS"),
"comment": "testing"
})
self.assertRedirects(
response,
"%s%s/whitelist/attackdomain=malware.testing.google.test" %
(settings.SERV_PREFIX, settings.DOMAIN),
status_code=302,
target_status_code=200)
self.assertFalse(
Whitelist.objects.filter(
domain="malware.testing.google.test",
protocol=Whitelist.get_protocol_choice("HTTPS")))
def testAddLongComment(self):
response = self.client.post(
"/whitelist/addentry/", {
"url":
"http%3A//sldjflksjdf.com/",
"domain":
"testlong1.com",
"protocol":
Whitelist.get_protocol_choice("HTTP"),
"comment":
"hundredandonehundredandonehundredandonehundredandonehundredandonehundredandonehundredandonehundredand"
})
self.assertTemplateUsed(response, 'whitelist/whitelist_getform.html')
self.assertFalse(
Whitelist.objects.filter(
domain="testlong1.com",
protocol=Whitelist.get_protocol_choice("HTTP")))
self.assertFormError(
response, 'form', 'comment',
'Ensure this value has at most 100 characters (it has 101).')
def testAddHTTP(self):
if settings.AUTO_WILDCARD == "ALL":
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//sldjflksjdf.com/",
"domain": "www.test1.com",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertContains(response,
"Whitetrash: Access Granted",
status_code=200)
self.assertContains(response,
"Thank you whitetrashtestuser",
status_code=200)
self.assertTrue(
Whitelist.objects.filter(
domain="%stest1.com" % settings.ALL_WILD_CHR,
protocol=Whitelist.get_protocol_choice("HTTP")))
elif settings.AUTO_WILDCARD == "ONE_LABEL":
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//sldjflksjdf.com/",
"domain": "www.test1.com",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertContains(response,
"Whitetrash: Access Granted",
status_code=200)
self.assertContains(response,
"Thank you whitetrashtestuser",
status_code=200)
self.assertTrue(
Whitelist.objects.filter(
domain="%stest1.com" % settings.ONE_WILD_CHR,
protocol=Whitelist.get_protocol_choice("HTTP")))
elif settings.AUTO_WILDCARD == "NONE":
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//sldjflksjdf.com/",
"domain": "www.test1.com",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertContains(response,
"Whitetrash: Access Granted",
status_code=200)
self.assertContains(response,
"Thank you whitetrashtestuser",
status_code=200)
self.assertTrue(
Whitelist.objects.filter(
domain="www.test1.com",
protocol=Whitelist.get_protocol_choice("HTTP")))
def testWhitelistPublicSuffix(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//sldjflksjdf.com.au/",
"domain": "com.au",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "testing"
})
self.assertTemplateUsed(response, 'whitelist/whitelist_getform.html')
self.assertContains(response, 'class="errorlist"', status_code=200)
self.assertFormError(response, 'form', 'domain',
"Public suffixes cannot be whitelisted.")
def testCheckError(self):
"""Attempt checks with bad domain and bad protocol"""
response = self.client.get(
"/whitelist/checkdomain/", {
"domain": "testing1.comsdfsds",
"protocol": Whitelist.get_protocol_choice("HTTP")
})
self.assertContains(response, "Error", status_code=200)
response = self.client.get("/whitelist/checkdomain/", {
"domain": "testing1.com",
"protocol": "'"
})
self.assertContains(response, "Error", status_code=200)
def testdelMultipleEntriesMemcache(self):
if settings.MEMCACHE:
self.assertTrue(Whitelist.objects.filter(pk=1))
self.assertTrue(Whitelist.objects.filter(pk=5))
id_5_key = "testwhitetrash.sf.net|%s" % Whitelist.get_protocol_choice(
"HTTP")
settings.MEMCACHE.set(id_5_key, (5, False))
response = self.client.post("/whitelist/delete/",
{"DeleteId": [1, 5]})
self.assertTemplateUsed(response,
'whitelist/whitelist_deleted.html')
self.assertFalse(Whitelist.objects.filter(pk=1))
self.assertFalse(Whitelist.objects.filter(pk=5))
result = settings.MEMCACHE.get(id_5_key)
self.assertFalse(result,
"Domain should have been removed from memcache")
def testAlreadyWhitelistedWithWildcard(self):
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//one.wildcardonelabel.com.au/",
"domain": "one.wildcardonelabel.com.au",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "anewcomment"
})
self.assertTemplateUsed(response,
'whitelist/whitelist_already_listed.html')
self.assertContains(response,
"http://one.wildcardonelabel.com.au",
status_code=200)
#Check we didn't add a new entry
self.assertFalse(
Whitelist.objects.filter(
domain="one.wildcardonelabel.com.au",
protocol=Whitelist.get_protocol_choice("HTTP")))
#Check we didn't set a new url or comment - the original is intact.
self.assertTrue(
Whitelist.objects.filter(
domain="wildcardonelabel.com.au",
protocol=Whitelist.get_protocol_choice("HTTP"),
url="http://wildcardonelabel.com.au",
comment="to be deleted"))
response = self.client.post(
"/whitelist/addentry/", {
"url": "http%3A//one.two.three.wildcardall.com.au/",
"domain": "one.two.three.wildcardall.com.au",
"protocol": Whitelist.get_protocol_choice("HTTP"),
"comment": "anewcomment"
})
self.assertTemplateUsed(response,
'whitelist/whitelist_already_listed.html')
self.assertContains(response,
"http://one.two.three.wildcardall.com.au",
status_code=200)
#Check we didn't add a new entry
self.assertFalse(
Whitelist.objects.filter(
domain="one.two.three.wildcardall.com.au",
protocol=Whitelist.get_protocol_choice("HTTP")))
#Check we didn't set a new url or comment - the original is intact.
self.assertTrue(
Whitelist.objects.filter(
domain="wildcardall.com.au",
protocol=Whitelist.get_protocol_choice("HTTP"),
url="http://wildcardall.com.au",
comment="to be deleted"))
def addentry(request):
"""Add an entry to the whitelist.
The domain will usually exist since it is created with enabled=False by the redirector the first
time it is requested.
"""
if request.method == 'POST':
#settings.LOG.debug("POST request: %s" % request)
form = WhiteListForm(request.POST)
if form.is_valid():
domain = form.cleaned_data['domain']
protocol = form.cleaned_data['protocol']
url = form.cleaned_data['url']
comment = form.cleaned_data['comment']
src_ip=whitetrash_filters.ip(request.META.get('REMOTE_ADDR'))
if settings.SAFEBROWSING:
sbcheck=check_safebrowse(domain,url)
if sbcheck:
settings.LOG.critical("****SAFEBROWSING BLACKLIST WHITELISTING ATTEMPT**** \
from IP:%s for url: %s, domain:%s using protocol:%s"
% (src_ip,url,domain,protocol))
return sbcheck
res = check_captcha(form,request)
if res:
return res
du = WTDomainUtils()
(template,dict) = du.add_domain(domain,protocol,url,comment,src_ip,request.user)
return render_to_response(template,dict,context_instance=RequestContext(request))
else:
#Pre-populate the form
#Do some quick checks on the pre-population data, if anything fails
#just present an empty form.
try:
url=request.GET["url"]
domain=whitetrash_filters.domain(request.GET["domain"])
except KeyError:
url=""
domain=""
#If this is SSL, it will come with proto set. Otherwise assume HTTP.
try:
proto=int(request.GET["protocol"])
if not (proto == Whitelist.get_protocol_choice('HTTP') or proto == Whitelist.get_protocol_choice('HTTPS')):
proto=Whitelist.get_protocol_choice('HTTP')
except KeyError:
proto = Whitelist.get_protocol_choice('HTTP')
form = WhiteListForm(initial={'url': url,
'protocol':proto,
'domain':domain})
if (proto==Whitelist.get_protocol_choice('HTTP') and settings.CAPTCHA_HTTP) or \
(proto==Whitelist.get_protocol_choice('HTTPS') and settings.CAPTCHA_SSL):
show_captcha=True
else:
show_captcha=False
return render_to_response('whitelist/whitelist_getform.html', {
'form': form, 'captcha': show_captcha},
context_instance=RequestContext(request))
#This is the last resort case if form is not valid.
#Make a best-guess at CAPTCHA requirement (if it is on for HTTP should probably display it)
return render_to_response('whitelist/whitelist_getform.html', {
'form': form, 'captcha': settings.CAPTCHA_HTTP},
context_instance=RequestContext(request))
def add_domain(self, domain, protocol, url, comment, src_ip, user):
#settings.LOG.debug("Checking dom:%s, proto:%s to see if it has been whitelisted by a wildcard" % (domain,protocol))
qs = self.is_whitelisted(domain, protocol)
if qs:
i = qs.get()
return ('whitelist/whitelist_already_listed.html', {
'url': url,
'domain': i.domain,
'client_ip': i.client_ip,
'prev_user': i.user,
'date_added': i.date_added
})
w, created = Whitelist.objects.get_or_create(
domain=domain,
protocol=protocol,
defaults={
'user': user,
'url': url,
'comment': comment,
'enabled': True,
'client_ip': src_ip,
'wildcard':
Whitelist.get_wildcard_choice(settings.AUTO_WILDCARD)
})
if not url:
#Handle SSL by refreshing to the domain added
if protocol == Whitelist.get_protocol_choice('SSL'):
url = "https://%s" % domain
else:
url = "http://%s" % domain
if not created and w.enabled:
#already in the db, so just redirect,
#show the info in the db but redirect to the new url
#This often happens if people open tabs with links to same domain.
return ('whitelist/whitelist_already_listed.html', {
'url': url,
'domain': w.domain,
'client_ip': w.client_ip,
'prev_user': w.user,
'date_added': w.date_added
})
elif not created and not w.enabled:
w.user = user
w.url = url
w.comment = comment
w.enabled = True
w.client_ip = src_ip
w.wildcard = Whitelist.get_wildcard_choice(settings.AUTO_WILDCARD)
w.save()
return ('whitelist/whitelist_added.html', {
'url': url,
'protocol': protocol,
'domain': domain,
'client_ip': src_ip,
'comment': comment
})
