def process_request(self, request):
from django.conf import settings
from meegloo.international.models import Country
try:
from django.contrib.gis.utils import GeoIP
except ImportError:
request.country = Country.objects.get(
pk = getattr(settings, 'COUNTRY_ID')
)
return
g = GeoIP()
ip = request.META.get('HTTP_X_FORWARDED_FOR',
request.META.get('REMOTE_IP', request.META.get('REMOTE_ADDR'))
)
if ip != '127.0.0.1':
d = g.country(ip)
code = d['country_code']
request.country = Country.objects.get(code = code)
else:
pk = getattr(settings, 'COUNTRY_ID')
request.country = Country.objects.get(pk = pk)
def test04_city(self):
"Testing GeoIP city querying methods."
g = GeoIP(country='<foo>')
addr = '130.80.29.3'
fqdn = 'chron.com'
for query in (fqdn, addr):
# Country queries should still work.
for func in (g.country_code, g.country_code_by_addr,
g.country_code_by_name):
self.assertEqual('US', func(query))
for func in (g.country_name, g.country_name_by_addr,
g.country_name_by_name):
self.assertEqual('United States', func(query))
self.assertEqual(
{
'country_code': 'US',
'country_name': 'United States'
}, g.country(query))
# City information dictionary.
d = g.city(query)
self.assertEqual('USA', d['country_code3'])
self.assertEqual('Houston', d['city'])
self.assertEqual('TX', d['region'])
self.assertEqual(713, d['area_code'])
geom = g.geos(query)
self.failIf(not isinstance(geom, GEOSGeometry))
lon, lat = (-95.3670, 29.7523)
lat_lon = g.lat_lon(query)
lat_lon = (lat_lon[1], lat_lon[0])
for tup in (geom.tuple, g.coords(query), g.lon_lat(query),
lat_lon):
self.assertAlmostEqual(lon, tup[0], 4)
self.assertAlmostEqual(lat, tup[1], 4)
def test04_city(self):
"Testing GeoIP city querying methods."
g = GeoIP(country='<foo>')
addr = '130.80.29.3'
fqdn = 'chron.com'
for query in (fqdn, addr):
# Country queries should still work.
for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
self.assertEqual('US', func(query))
for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
self.assertEqual('United States', func(query))
self.assertEqual({'country_code' : 'US', 'country_name' : 'United States'},
g.country(query))
# City information dictionary.
d = g.city(query)
self.assertEqual('USA', d['country_code3'])
self.assertEqual('Houston', d['city'])
self.assertEqual('TX', d['region'])
self.assertEqual(713, d['area_code'])
geom = g.geos(query)
self.failIf(not isinstance(geom, GEOSGeometry))
lon, lat = (-95.3670, 29.7523)
lat_lon = g.lat_lon(query)
lat_lon = (lat_lon[1], lat_lon[0])
for tup in (geom.tuple, g.coords(query), g.lon_lat(query), lat_lon):
self.assertAlmostEqual(lon, tup[0], 4)
self.assertAlmostEqual(lat, tup[1], 4)
def get_country_code(request):
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR')
if not ip:
return ''
g = GeoIP()
return g.country(ip)['country_code'] or ''
def test03_country(self):
"Testing GeoIP country querying methods."
g = GeoIP(city='<foo>')
fqdn = 'www.google.com'
addr = '12.215.42.19'
for query in (fqdn, addr):
for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
self.assertEqual('US', func(query))
for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
self.assertEqual('United States', func(query))
self.assertEqual({'country_code' : 'US', 'country_name' : 'United States'},
g.country(query))
def test03_country(self):
"Testing GeoIP country querying methods."
g = GeoIP(city='<foo>')
fqdn = 'www.google.com'
addr = '12.215.42.19'
for query in (fqdn, addr):
for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
self.assertEqual('US', func(query))
for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
self.assertEqual('United States', func(query))
self.assertEqual({'country_code' : 'US', 'country_name' : 'United States'},
g.country(query))
def dashboard(request):
ip = request.META['REMOTE_ADDR']
geoip = GeoIP()
try:
country = geoip.country(ip)
except:
country = "Ghost"
else:
country = country['country_name']
try:
coords = geoip.coords(ip)
if not coords:
coords = "on Earth"
except:
coords = "on Earth"
location = {
'country': country,
'coords': coords
}
checkins = Checkin.objects.all()
obj_list = list()
for item in checkins:
item.geodata.transform(900913)
obj_list.append(json.loads(item.geodata.geojson))
obj_list = json.dumps(obj_list)
last_20 = Checkin.objects.all()[:19]
data = {
"checkins": last_20,
"count": Checkin.objects.all().count(),
"obj_list": obj_list,
"location": location
}
return render_to_response("index.html", data,
context_instance=RequestContext(request, {}))
def get_country_by_ip(ip):
from api.views import create_dump_file
g = GeoIP()
country = None
try:
name = g.country(ip)['country_name']
if name == 'Russian Federation':
name = 'Russia'
elif name == 'United States':
name = 'USA'
elif name == 'Moldova, Republic of':
name = 'Moldova'
elif name == 'United Kingdom':
name = 'UK'
elif name == 'Iran, Islamic Republic of':
name = 'Iran'
try:
country = Country.objects.get(name_en=name)
except Country.DoesNotExist:
pass
'''
if name:
with open('%s/dump_geoip_nof_country.xml' % settings.NOF_DUMP_PATH, 'r') as f:
xml_data = BeautifulSoup(f.read(), from_encoding="utf-8")
name_slug = low(del_separator(name))
countries_slugs = [i.get('slug') for i in xml_data.findAll('country')]
data_nof_country = ''
if name_slug not in countries_slugs:
data_nof_country = '<country name="%s" slug="%s"></country>' % (name, name_slug)
xml_data = str(xml_data).replace('<html><head></head><body><data>','').replace('</data></body></html>','')
xml_data = '<data>%s%s</data>' % (xml_data, data_nof_country)
create_dump_file('geoip_nof_country', settings.NOF_DUMP_PATH, xml_data)
'''
except TypeError:
pass
return country
def process_request(self, request):
from django.conf import settings
from meegloo.international.models import Country
try:
from django.contrib.gis.utils import GeoIP
except ImportError:
request.country = Country.objects.get(
pk=getattr(settings, 'COUNTRY_ID'))
return
g = GeoIP()
ip = request.META.get(
'HTTP_X_FORWARDED_FOR',
request.META.get('REMOTE_IP', request.META.get('REMOTE_ADDR')))
if ip != '127.0.0.1':
d = g.country(ip)
code = d['country_code']
request.country = Country.objects.get(code=code)
else:
pk = getattr(settings, 'COUNTRY_ID')
request.country = Country.objects.get(pk=pk)
class GeoIPRedirectMiddleware(object):
"""
GeoIP redirect middleware. If enabled, the middleware determines the
visitor's location by checking his REMOTE_ADDR with the GeoIP DB
(working GeoIP setup required, see: 'django.contrib.gis.utils.geoip.py').
The determined country code is checked against 'settings.SITE_GEOIP_REDIRECT',
which must be a list of 2-tuples with iso country code
(see: http://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) and corresponding site_id.
If a matching country_code setting is found, a redirect (302) is issued to the
site's domain.
Author: Maik LUSTENBERGER, Divio GmbH, 2009
"""
def __init__(self):
if getattr(settings, 'IS_DEV_SERVER', False
) or 'django.contrib.sites' not in settings.INSTALLED_APPS:
raise MiddlewareNotUsed()
try:
from django.contrib.gis.utils import GeoIP, GeoIPException
self.g = GeoIP()
self.redirect_sites = dict(settings.SITE_GEOIP_REDIRECT)
except (ImportError, GeoIPException, ValueError, TypeError,
AttributeError):
raise ImproperlyConfigured(
"The GeoIPRedirectMiddleware requires the"
" setting SITE_GEOIP_REDIRECT and GEOIP_PATH to be defined"
" along with a working GeoIP installation."
" Please see 'django.contrib.gis.utils.geoip.py'.")
self.srs_url_re = re.compile("^/siteinfo/srs/?$")
self.srs_getvarname = getattr(settings,
'SITE_GEOIP_REDIRECT_GETVARNAME', 'srs')
def process_request(self, request):
assert hasattr(request, 'session'), \
"The 'GeoIPRedirectMiddleware' requires the app" \
" 'django.contrib.sessions' to be in INSTALLED_APPS." \
" Also make sure that 'GeoIPRedirectMiddleware' comes" \
" after 'django.contrib.sessions.middleware.SessionMiddleware'" \
" in the setting 'MIDDLEWARE_CLASSES'."
site_settings = SiteSettings.objects.get_current()
if not site_settings.geoip_redirect:
return None
if self.srs_url_re.match(request.path):
return None
if request.GET.get(self.srs_getvarname, False):
# Var ?srs=1 is in GET.
# Set session key. ATTENTION: This does not work, if a subsequent request middleware does a redirect!
request.session['site_redirect_stop'] = True
return None
if request.session.get('site_redirect_stop', False):
# The redirect stop flag has been set by 'siteinfo.views.set_redirect_stop'
return None
if getattr(settings, 'SITE_GEOIP_REDIRECT_ONCE', True):
if request.session.get('site_no_redirect', False):
# Redirect info already lives in session. No need for redirect. So no further action.
return None
if request.COOKIES.get('site_no_redirect', False):
# Redirect has already occurred. Write cookie info to session for accurate lifetime.
request.session['site_no_redirect'] = True
return None
if getattr(settings, 'SITE_GEOIP_REDIRECT_ROOT_ONLY',
True) and request.get_full_path() != '/':
return None
# Start with the geo action.
ip = request.META.get('REMOTE_ADDR')
geo_info = self.g.country(ip)
try:
country_code = geo_info['country_code'].lower()
except (KeyError, AttributeError):
return None
if country_code in self.redirect_sites:
site_id = self.redirect_sites[country_code]
try:
site = Site.objects.get(pk=site_id)
except Site.DoesNotExist:
return None
host = request.get_host()
if site.domain in host:
return None
if getattr(settings, 'SITE_GEOIP_REDIRECT_TO_FULL_PATH', False):
path = request.get_full_path()
else:
path = ''
response = HttpResponseRedirect('%s://%s%s' % (
request.is_secure() and 'https' or 'http',
site.domain,
path,
))
if getattr(settings, 'SITE_GEOIP_REDIRECT_ONCE', True):
# Set a cookie for the lifetime of a (new) session, to let subsequent requests know of the redirect.
# Also, subsequent requests will write this cookie information to the session for accurate lifetime.
response.set_cookie(
'site_no_redirect',
'1',
max_age=(not settings.SESSION_EXPIRE_AT_BROWSER_CLOSE
or None) and settings.SESSION_COOKIE_AGE)
return response
def loadfeeds(self):
# for now I keep this utility command very basic, much more refactoring in future
# now we start sorting the feeds
feeds = Feed.objects.all().filter(enabled=True)
for feed in feeds:
self.stdout.write('\n***Parsing feed %s' % feed.name.encode('utf-8'))
feed_parsed = feedparser.parse(feed.url_xml)
for item_parsed in feed_parsed.entries:
biased_link = item_parsed.link
# link must be restored to original (in some case it is: http://news.google.com/news...&url=http//www.example.com
self.stdout.write('Biased link: %s' % biased_link)
link = get_original_url(biased_link)
# then we unquote the url (ex http://voria.gr/index.php?module%3Dnews%26func%3Ddisplay%26sid%3D56406 becomes
# http://voria.gr/index.php?module=news&func=display&sid=56406
link = urllib.unquote(link)
if len(link)>255:
self.stdout.write('\nItem link is more than 255 chars!\n%s\n' % link)
link = link[:255]
# must import only items not already in the database (check URL of item)
items = Item.objects.filter(link=link)
if len(items)==0: # new item, must import it!
try:
self.stdout.write('\nImporting item %s' % item_parsed.title.encode('utf-8'))
# store item
item = Item()
item.title = item_parsed.title[:255].encode('utf-8')
item.summary = item_parsed.summary.encode('utf-8')
item.link = link
item.feed = feed
item.updated = datetime.datetime.fromtimestamp(mktime(item_parsed.updated_parsed))
# 0. domain
# first let's check domain country
parsed = urlparse.urlsplit(link)
domain_name = parsed.netloc.encode('utf-8')
g = GeoIP()
country_code = g.country(domain_name)['country_code']
self.stdout.write('\nDomain: %s, country code: %s' % (domain_name, country_code))
countries = Country.objects.filter(iso2=country_code)
country = None
if countries.count() > 0:
country = countries[0]
# check if if we need to add domain in db
domains = Domain.objects.all().filter(name=domain_name)
if not domains:
self.stdout.write('\nAdding a new domain to the system: %s for this item.' % domain_name)
domain = Domain()
domain.name = domain_name
domain.country = country
domain.save()
else:
domain = domains[0]
# add the item to the place
item.domain = domain
# save item
item.save()
# define text to be parsed
text2parse = item.title + item.summary
# 1. keywords
keywords = Keyword.objects.all()
for keyword in keywords:
if re.search(keyword.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Keyword %s is in this item.' % keyword.name)
#import ipdb;ipdb.set_trace()
keyword.item.add(item)
# 2. people
people = Person.objects.all()
for person in people:
if re.search(person.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Person %s is in this item.' % person.name)
#import ipdb;ipdb.set_trace()
person.item.add(item)
# 3. images
#url = item.link
#soup = bs(urlopen(url))
#parsed = list(urlparse.urlparse(url))
soup = bs(item.summary)
parsed = list(item.summary)
for img in soup.findAll("img"):
print img
alt = ''
if img.has_key('src'):
if img.has_key('alt'):
alt = img["alt"]
if img["src"].lower().startswith("http"):
#import ipdb;ipdb.set_trace()
src = img["src"]
else:
# TODO src extraction from relative url
src = urlparse.urlunparse(parsed)
print src
image = Image()
image.src = src
image.alt = alt
image.item = item
image.save()
# 4. tags
tags = Tag.objects.all()
for tag in tags:
if re.search(tag.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Tag %s is in this item.' % tag.name)
item.tags.add(tag)
# 5. places
# let's check if there are places linked to this item
pm=placemaker('4BRX3JfV34E7uaK02MDR.5nn7EAw7DptfhbRTdrMQQjHbXVedgXfsQLaFWwp7fIm')
pm_places = pm.find_places(text2parse)
for pm_place in pm_places:
placename = pm_place.name.decode('utf-8')
# first check if we need to add new place
places = Place.objects.all().filter(name=placename)
if not places:
self.stdout.write('\nAdding a new place to the system: %s for this item.' % pm_place.name)
place = Place()
place.name = placename
place.slug = slugify(place.name)
place.geometry = 'POINT(%s %s)' % (pm_place.centroid.longitude, pm_place.centroid.latitude)
place.save()
else:
place = places[0]
# add the item to the place
#import ipdb;ipdb.set_trace()
place.items.add(item)
except Exception,e:
self.stdout.write('\nThere was an error importing this item. Skipping to the next one...')
def test04_city(self):
"Testing GeoIP city querying methods."
g = GeoIP(country="<foo>")
addr = "130.80.29.3"
fqdn = "chron.com"
for query in (fqdn, addr):
# Country queries should still work.
for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
self.assertEqual("US", func(query))
for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
self.assertEqual("United States", func(query))
self.assertEqual({"country_code": "US", "country_name": "United States"}, g.country(query))
# City information dictionary.
d = g.city(query)
self.assertEqual("USA", d["country_code3"])
self.assertEqual("Houston", d["city"])
self.assertEqual("TX", d["region"])
self.assertEqual(713, d["area_code"])
geom = g.geos(query)
self.failIf(not isinstance(geom, GEOSGeometry))
lon, lat = (-95.4152, 29.7755)
lat_lon = g.lat_lon(query)
lat_lon = (lat_lon[1], lat_lon[0])
for tup in (geom.tuple, g.coords(query), g.lon_lat(query), lat_lon):
self.assertAlmostEqual(lon, tup[0], 4)
self.assertAlmostEqual(lat, tup[1], 4)
def test03_country(self):
"Testing GeoIP country querying methods."
g = GeoIP(city="<foo>")
fqdn = "www.google.com"
addr = "12.215.42.19"
for query in (fqdn, addr):
for func in (g.country_code, g.country_code_by_addr, g.country_code_by_name):
self.assertEqual("US", func(query))
for func in (g.country_name, g.country_name_by_addr, g.country_name_by_name):
self.assertEqual("United States", func(query))
self.assertEqual({"country_code": "US", "country_name": "United States"}, g.country(query))
