class GeoIPTest(amo.tests.TestCase):
def setUp(self):
# Use GEOIP_NOOP to always return the default value.
# This *should* be properly tested against a call to a GeoIP server.
self.geoip = GeoIP(NOOP_Settings)
@mock.patch('socket.socket')
def test_lookup(self, mock_socket):
send_value = 'mozilla.com'
mock_socket.return_value.connect.return_value = True
rcv = list('{"success":{"country_code":"us"}}')
# 5 = prefixed "GET " and new line
mock_socket.return_value.send.return_value = len(send_value) + 5
mock_socket.return_value.recv.side_effect = rcv
result = self.geoip.lookup(send_value)
eq_(result, 'us')
@mock.patch('socket.socket')
def test_no_connect(self, mock_socket):
mock_socket.return_value.connect.side_effect = IOError
result = self.geoip.lookup('mozilla.com')
eq_(result, 'worldwide')
@mock.patch('socket.socket')
def test_timeout(self, mock_socket):
mock_socket.return_value.connect.return_value = True
mock_socket.return_value.send.side_effect = socket.timeout
result = self.geoip.lookup('mozilla.com')
eq_(result, 'worldwide')
def test_connection_error(self, mock_post):
url = "localhost"
geoip = GeoIP(generate_settings(url=url))
mock_post.side_effect = requests.ConnectionError
ip = "3.3.3.3"
result = geoip.lookup(ip)
mock_post.assert_called_with("{0}/country.json".format(url), timeout=0.2, data={"ip": ip})
eq_(result, "worldwide")
def test_bad_request(self, mock_post):
url = "localhost"
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(status_code=404, json=None)
ip = "3.3.3.3"
result = geoip.lookup(ip)
mock_post.assert_called_with("{0}/country.json".format(url), timeout=0.2, data={"ip": ip})
eq_(result, "worldwide")
def test_connection_error(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.side_effect = requests.ConnectionError
ip = '3.3.3.3'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2, data={'ip': ip})
eq_(result, 'restofworld')
def test_bad_request(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(status_code=404, json=lambda: None)
ip = '3.3.3.3'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2, data={'ip': ip})
eq_(result, 'restofworld')
def test_bad_request(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(status_code=404, json=lambda: None)
ip = '3.3.3.3'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2,
data={'ip': ip})
eq_(result, 'restofworld')
def test_lookup(self, mock_post):
url = "localhost"
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(
status_code=200, json={"country_code": "US", "country_name": "United States"}
)
ip = "1.1.1.1"
result = geoip.lookup(ip)
mock_post.assert_called_with("{0}/country.json".format(url), timeout=0.2, data={"ip": ip})
eq_(result, "us")
def test_connection_error(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.side_effect = requests.ConnectionError
ip = '3.3.3.3'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2,
data={'ip': ip})
eq_(result, 'restofworld')
def test_lookup(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(status_code=200, json=lambda: {
'country_code': 'US',
'country_name': 'United States'
})
ip = '1.1.1.1'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2, data={'ip': ip})
eq_(result, 'us')
def test_lookup(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
mock_post.return_value = mock.Mock(status_code=200,
json=lambda: {
'country_code': 'US',
'country_name': 'United States'
})
ip = '1.1.1.1'
result = geoip.lookup(ip)
mock_post.assert_called_with('{0}/country.json'.format(url),
timeout=0.2,
data={'ip': ip})
eq_(result, 'us')
def test_private_ip(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
addrs = [
'127.0.0.1', '10.{0}.{1}.{2}'.format(randint(0, 255),
randint(0, 255),
randint(0, 255)),
'192.168.{0}.{1}'.format(randint(0, 255), randint(0, 255)),
'172.{0}.{1}.{2}'.format(randint(16, 31), randint(0, 255),
randint(0, 255))
]
for ip in addrs:
result = geoip.lookup(ip)
assert not mock_post.called
eq_(result, 'restofworld')
def test_private_ip(self, mock_post):
url = 'localhost'
geoip = GeoIP(generate_settings(url=url))
addrs = [
'127.0.0.1',
'10.{0}.{1}.{2}'.format(randint(0, 255), randint(0, 255),
randint(0, 255)),
'192.168.{0}.{1}'.format(randint(0, 255), randint(0, 255)),
'172.{0}.{1}.{2}'.format(randint(16, 31), randint(0, 255),
randint(0, 255))
]
for ip in addrs:
result = geoip.lookup(ip)
assert not mock_post.called
eq_(result, 'restofworld')
class RegionMiddleware(object):
"""Figure out the user's region and set request.REGION accordingly, storing
it on the request.user if there is one.
- Outside the API, we automatically set RESTOFWORLD.
- In the API, it tries to find a valid region in the query parameters,
additionnally falling back to GeoIP for API v1 (for later versions we
never do GeoIP automatically)."""
def __init__(self):
self.geoip = GeoIP(settings)
def store_region(self, request, user_region):
request.REGION = user_region
mkt.regions.set_region(user_region)
def region_from_request(self, request):
address = request.META.get('REMOTE_ADDR')
ip_reg = self.geoip.lookup(address)
log.info('Geodude lookup for {0} returned {1}'
.format(address, ip_reg))
return parse_region(ip_reg) or mkt.regions.RESTOFWORLD
def process_request(self, request):
regions = mkt.regions.REGION_LOOKUP
user_region = restofworld = mkt.regions.RESTOFWORLD
if not getattr(request, 'API', False):
request.REGION = restofworld
mkt.regions.set_region(restofworld)
return
# Try 'region' in POST/GET data first, if it's not there try geoip.
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
user_region = regions[url_region]
log.info('Region {0} specified in URL; region set as {1}'
.format(url_region, user_region.slug))
elif getattr(request, 'API_VERSION', None) == 1:
# Fallback to GeoIP, but only for API version 1.
statsd.incr('z.regions.middleware.source.geoip')
user_region = self.region_from_request(request)
log.info('Region not specified in URL; region set as {0}'
.format(user_region.slug))
# Update the region on the user object if it changed.
if (request.user.is_authenticated() and
request.user.region != user_region.slug):
request.user.region = user_region.slug
request.user.save()
# Persist the region on the request / local thread.
self.store_region(request, user_region)
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['search_form'] = SearchForm()
ip = self.kwargs['pk']
try:
context['geoip'] = GeoIP().lookup(ip)
except Exception as e:
logger.error(e)
try:
context['domain'] = socket.gethostbyaddr(ip)[0]
except Exception as e:
logger.error(e)
try:
vt = VT()
context['vt_ip'] = vt.getIPReport(ip)
except Exception as e:
logger.error(e)
try:
tm = ThreatMiner()
context['tm_url'] = tm.getURIFromIP(ip)
context['tm_sample'] = tm.getSamplesFromIP(ip)
context['tm_report'] = tm.getReportFromIP(ip)
except Exception as e:
logger.error(e)
context['bls'] = blacklist.objects.filter(Q(ip=ip)|Q(url__contains=ip))
count = context['bls'].count()
if count > 0:
context['bls_count'] = count
context['events'] = Event.objects.filter(Q(info__icontains=ip)).order_by('-publish_timestamp')
count = context['events'].count()
if count > 0:
context['events_count'] = count
context['attributes'] = Attribute.objects.filter(Q(value__icontains=ip)).order_by('-timestamp')
count = context['attributes'].count()
if count > 0:
context['attributes_count'] = count
context['tws'] = tweet.objects.filter(Q(text__icontains=ip)).order_by('-datetime')
count = context['tws'].count()
if count > 0:
context['tws_count'] = count
context['exs'] = Exploit.objects.filter(Q(text__icontains=ip)).order_by('-datetime')
count = context['exs'].count()
if count > 0:
context['exs_count'] = count
return context
class RegionMiddleware(object):
"""Figure out the user's region and set request.REGION accordingly, storing
it on the request.user if there is one."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
address = request.META.get('REMOTE_ADDR')
ip_reg = self.geoip.lookup(address)
log.info('Geodude lookup for {0} returned {1}'
.format(address, ip_reg))
return parse_region(ip_reg) or mkt.regions.RESTOFWORLD
def process_request(self, request):
regions = mkt.regions.REGION_LOOKUP
user_region = restofworld = mkt.regions.RESTOFWORLD
if not getattr(request, 'API', False):
request.REGION = restofworld
mkt.regions.set_region(restofworld)
return
# Try 'region' in POST/GET data first, if it's not there try geoip.
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
user_region = regions[url_region]
log.info('Region {0} specified in URL; region set as {1}'
.format(url_region, user_region.slug))
else:
statsd.incr('z.regions.middleware.source.geoip')
user_region = self.region_from_request(request)
log.info('Region not specified in URL; region set as {0}'
.format(user_region.slug))
# Update the region on the user object if it changed.
if (request.user.is_authenticated() and
request.user.region != user_region.slug):
request.user.region = user_region.slug
request.user.save()
# Persist the region on the request / local thread.
request.REGION = user_region
mkt.regions.set_region(user_region)
class RegionMiddleware(object):
"""Figure out the user's region and set request.REGION accordingly, storing
it on the request.amo_user if there is one."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
address = request.META.get('REMOTE_ADDR')
ip_reg = self.geoip.lookup(address)
log.info('Geodude lookup for {0} returned {1}'.format(address, ip_reg))
return mkt.regions.REGIONS_DICT.get(ip_reg, mkt.regions.RESTOFWORLD)
def process_request(self, request):
regions = mkt.regions.REGION_LOOKUP
user_region = restofworld = mkt.regions.RESTOFWORLD
if not getattr(request, 'API', False):
request.REGION = restofworld
mkt.regions.set_region(restofworld)
return
# Try 'region' in POST/GET data first, if it's not there try geoip.
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
user_region = regions[url_region]
log.info('Region {0} specified in URL; region set as {1}'.format(
url_region, user_region.slug))
else:
statsd.incr('z.regions.middleware.source.geoip')
user_region = self.region_from_request(request)
log.info('Region not specified in URL; region set as {0}'.format(
user_region.slug))
# Update the region on the user object if it changed.
amo_user = getattr(request, 'amo_user', None)
if amo_user and amo_user.region != user_region.slug:
amo_user.region = user_region.slug
amo_user.save()
# Persist the region on the request / local thread.
request.REGION = user_region
mkt.regions.set_region(user_region)
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['search_form'] = SearchForm()
domain = self.kwargs['pk']
try:
context['geoip'] = GeoIP().lookup(domain)
except Exception as e:
logger.error(e)
try:
context['ipaddress'] = socket.gethostbyname(domain)
except Exception as e:
logger.error(e)
try:
context['vt_domain'] = VT().getDomainReport(domain)
except Exception as e:
logger.error(e)
context['bls'] = blacklist.objects.filter(Q(domain=domain)|Q(url__contains=domain))
count = context['bls'].count()
if count > 0:
context['bls_count'] = count
context['events'] = Event.objects.filter(Q(info__icontains=domain)).order_by('-publish_timestamp')
count = context['events'].count()
if count > 0:
context['events_count'] = count
context['attributes'] = Attribute.objects.filter(Q(value__icontains=domain)).order_by('-timestamp')
count = context['attributes'].count()
if count > 0:
context['attributes_count'] = count
context['tws'] = tweet.objects.filter(Q(text__icontains=domain)).order_by('-datetime')
count = context['tws'].count()
if count > 0:
context['tws_count'] = count
context['exs'] = Exploit.objects.filter(Q(text__icontains=domain)).order_by('-datetime')
count = context['exs'].count()
if count > 0:
context['exs_count'] = count
return context
class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
for name, region in mkt.regions.REGIONS_CHOICES:
if ip_reg == name:
return region.slug
return mkt.regions.RESTOFWORLD.slug
def process_request(self, request):
regions = mkt.regions.REGIONS_DICT
reg = restofworld = mkt.regions.RESTOFWORLD.slug
stored_reg = ''
if not getattr(request, 'API', False):
request.REGION = regions[restofworld]
mkt.regions.set_region(restofworld)
return
# ?region= -> geoip -> lang
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
reg = url_region
else:
reg = self.region_from_request(request)
# If the above fails, let's try `Accept-Language`.
if reg == restofworld:
statsd.incr('z.regions.middleware.source.accept-lang')
if request.LANG == settings.LANGUAGE_CODE:
choices = mkt.regions.REGIONS_CHOICES[1:]
else:
choices = mkt.regions.REGIONS_CHOICES
if request.LANG:
for name, region in choices:
if name.lower() in request.LANG.lower():
reg = region.slug
break
# All else failed, try to match against our forced Language.
if reg == mkt.regions.RESTOFWORLD.slug:
# Try to find a suitable region.
for name, region in choices:
if region.default_language == request.LANG:
reg = region.slug
break
a_l = request.META.get('HTTP_ACCEPT_LANGUAGE')
if (reg == 'us' and a_l is not None
and not a_l.startswith('en')):
# Let us default to restofworld if it's not English.
reg = mkt.regions.RESTOFWORLD.slug
else:
statsd.incr('z.regions.middleware.source.geoip')
# Update cookie if value have changed.
if reg != stored_reg:
if (getattr(request, 'amo_user', None)
and request.amo_user.region != reg):
request.amo_user.region = reg
request.amo_user.save()
request.REGION = regions[reg]
mkt.regions.set_region(reg)
class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
for name, region in mkt.regions.REGIONS_CHOICES:
if ip_reg == name:
return region.slug
return mkt.regions.WORLDWIDE.slug
def process_request(self, request):
regions = mkt.regions.REGIONS_DICT
reg = worldwide = mkt.regions.WORLDWIDE.slug
stored_reg = ''
# ?region= -> cookie -> geoip -> lang
url_region = request.REQUEST.get('region')
cookie_region = request.COOKIES.get('region')
if url_region in regions:
reg = url_region
elif cookie_region in regions:
reg = stored_reg = cookie_region
else:
reg = self.region_from_request(request)
# If the above fails, let's try `Accept-Language`.
if reg == worldwide:
if request.LANG == settings.LANGUAGE_CODE:
choices = mkt.regions.REGIONS_CHOICES[1:]
else:
choices = mkt.regions.REGIONS_CHOICES
if request.LANG:
for name, region in choices:
if name.lower() in request.LANG.lower():
reg = region.slug
break
# All else failed, try to match against our forced Language.
if reg == mkt.regions.WORLDWIDE.slug:
# Try to find a suitable region.
for name, region in choices:
if region.default_language == request.LANG:
reg = region.slug
break
a_l = request.META.get('HTTP_ACCEPT_LANGUAGE')
if (reg == 'us' and a_l is not None
and not a_l.startswith('en')):
# Let us default to worldwide if it's not English.
reg = mkt.regions.WORLDWIDE.slug
# Update cookie if value have changed.
if reg != stored_reg:
if (getattr(request, 'amo_user', None)
and request.amo_user.region != reg):
request.amo_user.region = reg
request.amo_user.save()
if not getattr(request, 'API', False):
request.set_cookie('region', reg)
request.REGION = regions[reg]
mkt.regions.set_region(reg)
def process_response(self, request, response):
patch_vary_headers(response, ['Accept-Language', 'Cookie'])
return response
def test_no_url(self, mock_post):
geoip = GeoIP(generate_settings())
result = geoip.lookup("2.2.2.2")
assert not mock_post.called
eq_(result, "worldwide")
def test_no_url(self, mock_post):
geoip = GeoIP(generate_settings())
result = geoip.lookup('2.2.2.2')
assert not mock_post.called
eq_(result, 'restofworld')
class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
return mkt.regions.REGIONS_DICT.get(ip_reg, mkt.regions.RESTOFWORLD)
def process_request(self, request):
regions = mkt.regions.REGION_LOOKUP
user_region = restofworld = mkt.regions.RESTOFWORLD
if not getattr(request, 'API', False):
request.REGION = restofworld
mkt.regions.set_region(restofworld)
return
# ?region= -> geoip -> lang
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
user_region = regions[url_region]
else:
user_region = self.region_from_request(request)
# If the above fails, let's try `Accept-Language`.
if user_region == restofworld:
statsd.incr('z.regions.middleware.source.accept-lang')
if request.LANG == settings.LANGUAGE_CODE:
choices = mkt.regions.REGIONS_CHOICES[1:]
else:
choices = mkt.regions.REGIONS_CHOICES
if request.LANG:
for name, region in choices:
if name.lower() in request.LANG.lower():
user_region = region
break
# All else failed, try to match against our forced Language.
if user_region == mkt.regions.RESTOFWORLD:
# Try to find a suitable region.
for name, region in choices:
if region.default_language == request.LANG:
user_region = region
break
accept_language = request.META.get('HTTP_ACCEPT_LANGUAGE')
if (user_region == mkt.regions.US
and accept_language is not None
and not accept_language.startswith('en')):
# Let us default to restofworld if it's not English.
user_region = mkt.regions.RESTOFWORLD
else:
statsd.incr('z.regions.middleware.source.geoip')
# Only update the user's region if it changed.
amo_user = getattr(request, 'amo_user', None)
if amo_user and amo_user.region != user_region.slug:
amo_user.region = user_region.slug
amo_user.save()
request.REGION = user_region
mkt.regions.set_region(user_region)
def setUp(self):
# Use GEOIP_NOOP to always return the default value.
# This *should* be properly tested against a call to a GeoIP server.
self.geoip = GeoIP(NOOP_Settings)
class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
self.geoip = GeoIP(settings)
def process_request(self, request):
regions = mkt.regions.REGIONS_DICT
reg = mkt.regions.WORLDWIDE.slug
stored_reg = ''
# If I have a cookie use that region.
remembered = request.COOKIES.get('region')
if remembered in regions:
reg = stored_reg = remembered
# Re-detect my region only if my *Accept-Language* is different from
# that of my previous language.
lang_changed = (get_accept_language(request)
not in request.COOKIES.get('lang', '').split(','))
if not remembered or lang_changed:
# If our locale is `en-US`, then exclude the Worldwide region.
if request.LANG == settings.LANGUAGE_CODE:
choices = mkt.regions.REGIONS_CHOICES[1:]
else:
choices = mkt.regions.REGIONS_CHOICES
# if we faked the user's LANG, and we still don't have a
# valid region, try from the IP
if (request.LANG.lower() not in
request.META.get('HTTP_ACCEPT_LANGUAGE', '').lower()):
ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
for name, region in choices:
if ip_reg == name:
reg = region.slug
break
elif request.LANG:
for name, region in choices:
if name.lower() in request.LANG.lower():
reg = region.slug
break
# All else failed, try to match against our forced Language.
if reg == mkt.regions.WORLDWIDE.slug:
# Try to find a suitable region.
for name, region in choices:
if region.default_language == request.LANG:
reg = region.slug
break
choice = request.REQUEST.get('region')
if choice in regions:
reg = choice
a_l = request.META.get('HTTP_ACCEPT_LANGUAGE')
if reg == 'us' and a_l is not None and not a_l.startswith('en'):
# Let us default to worldwide if it's not English.
reg = mkt.regions.WORLDWIDE.slug
# Update cookie if value have changed.
if reg != stored_reg:
request.set_cookie('region', reg)
request.REGION = regions[reg]
def process_response(self, request, response):
patch_vary_headers(response, ['Accept-Language', 'Cookie'])
return response
class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
self.geoip = GeoIP(settings)
def region_from_request(self, request):
ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
return mkt.regions.REGIONS_DICT.get(ip_reg, mkt.regions.RESTOFWORLD)
def process_request(self, request):
regions = mkt.regions.REGION_LOOKUP
user_region = restofworld = mkt.regions.RESTOFWORLD
if not getattr(request, 'API', False):
request.REGION = restofworld
mkt.regions.set_region(restofworld)
return
# ?region= -> geoip -> lang
url_region = request.REQUEST.get('region')
if url_region in regions:
statsd.incr('z.regions.middleware.source.url')
user_region = regions[url_region]
else:
user_region = self.region_from_request(request)
# If the above fails, let's try `Accept-Language`.
if user_region == restofworld:
statsd.incr('z.regions.middleware.source.accept-lang')
if request.LANG == settings.LANGUAGE_CODE:
choices = mkt.regions.REGIONS_CHOICES[1:]
else:
choices = mkt.regions.REGIONS_CHOICES
if request.LANG:
for name, region in choices:
if name.lower() in request.LANG.lower():
user_region = region
break
# All else failed, try to match against our forced Language.
if user_region == mkt.regions.RESTOFWORLD:
# Try to find a suitable region.
for name, region in choices:
if region.default_language == request.LANG:
user_region = region
break
accept_language = request.META.get('HTTP_ACCEPT_LANGUAGE')
if (user_region == mkt.regions.US
and accept_language is not None
and not accept_language.startswith('en')):
# Let us default to restofworld if it's not English.
user_region = mkt.regions.RESTOFWORLD
else:
statsd.incr('z.regions.middleware.source.geoip')
# Only update the user's region if it changed.
amo_user = getattr(request, 'amo_user', None)
if amo_user and amo_user.region != user_region.slug:
amo_user.region = user_region.slug
amo_user.save()
request.REGION = user_region
mkt.regions.set_region(user_region)
def __init__(self):
self.geoip = GeoIP(settings)
def test_no_url(self, mock_post):
geoip = GeoIP(generate_settings())
result = geoip.lookup('2.2.2.2')
assert not mock_post.called
eq_(result, 'restofworld')