def loadfeeds(self):
# for now I keep this utility command very basic, much more refactoring in future
# now we start sorting the feeds
feeds = Feed.objects.all().filter(enabled=True)
for feed in feeds:
self.stdout.write('\n***Parsing feed %s' % feed.name.encode('utf-8'))
feed_parsed = feedparser.parse(feed.url_xml)
for item_parsed in feed_parsed.entries:
biased_link = item_parsed.link
# link must be restored to original (in some case it is: http://news.google.com/news...&url=http//www.example.com
self.stdout.write('Biased link: %s' % biased_link)
link = get_original_url(biased_link)
# then we unquote the url (ex http://voria.gr/index.php?module%3Dnews%26func%3Ddisplay%26sid%3D56406 becomes
# http://voria.gr/index.php?module=news&func=display&sid=56406
link = urllib.unquote(link)
if len(link)>255:
self.stdout.write('\nItem link is more than 255 chars!\n%s\n' % link)
link = link[:255]
# must import only items not already in the database (check URL of item)
items = Item.objects.filter(link=link)
if len(items)==0: # new item, must import it!
try:
self.stdout.write('\nImporting item %s' % item_parsed.title.encode('utf-8'))
# store item
item = Item()
item.title = item_parsed.title[:255].encode('utf-8')
item.summary = item_parsed.summary.encode('utf-8')
item.link = link
item.feed = feed
item.updated = datetime.datetime.fromtimestamp(mktime(item_parsed.updated_parsed))
# 0. domain
# first let's check domain country
parsed = urlparse.urlsplit(link)
domain_name = parsed.netloc.encode('utf-8')
g = GeoIP()
country_code = g.country(domain_name)['country_code']
self.stdout.write('\nDomain: %s, country code: %s' % (domain_name, country_code))
countries = Country.objects.filter(iso2=country_code)
country = None
if countries.count() > 0:
country = countries[0]
# check if if we need to add domain in db
domains = Domain.objects.all().filter(name=domain_name)
if not domains:
self.stdout.write('\nAdding a new domain to the system: %s for this item.' % domain_name)
domain = Domain()
domain.name = domain_name
domain.country = country
domain.save()
else:
domain = domains[0]
# add the item to the place
item.domain = domain
# save item
item.save()
# define text to be parsed
text2parse = item.title + item.summary
# 1. keywords
keywords = Keyword.objects.all()
for keyword in keywords:
if re.search(keyword.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Keyword %s is in this item.' % keyword.name)
#import ipdb;ipdb.set_trace()
keyword.item.add(item)
# 2. people
people = Person.objects.all()
for person in people:
if re.search(person.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Person %s is in this item.' % person.name)
#import ipdb;ipdb.set_trace()
person.item.add(item)
# 3. images
#url = item.link
#soup = bs(urlopen(url))
#parsed = list(urlparse.urlparse(url))
soup = bs(item.summary)
parsed = list(item.summary)
for img in soup.findAll("img"):
print img
alt = ''
if img.has_key('src'):
if img.has_key('alt'):
alt = img["alt"]
if img["src"].lower().startswith("http"):
#import ipdb;ipdb.set_trace()
src = img["src"]
else:
# TODO src extraction from relative url
src = urlparse.urlunparse(parsed)
print src
image = Image()
image.src = src
image.alt = alt
image.item = item
image.save()
# 4. tags
tags = Tag.objects.all()
for tag in tags:
if re.search(tag.name, text2parse, re.IGNORECASE):
self.stdout.write('\n***Tag %s is in this item.' % tag.name)
item.tags.add(tag)
# 5. places
# let's check if there are places linked to this item
pm=placemaker('4BRX3JfV34E7uaK02MDR.5nn7EAw7DptfhbRTdrMQQjHbXVedgXfsQLaFWwp7fIm')
pm_places = pm.find_places(text2parse)
for pm_place in pm_places:
placename = pm_place.name.decode('utf-8')
# first check if we need to add new place
places = Place.objects.all().filter(name=placename)
if not places:
self.stdout.write('\nAdding a new place to the system: %s for this item.' % pm_place.name)
place = Place()
place.name = placename
place.slug = slugify(place.name)
place.geometry = 'POINT(%s %s)' % (pm_place.centroid.longitude, pm_place.centroid.latitude)
place.save()
else:
place = places[0]
# add the item to the place
#import ipdb;ipdb.set_trace()
place.items.add(item)
except Exception,e:
self.stdout.write('\nThere was an error importing this item. Skipping to the next one...')
import csv
from feeds.models import Place
from django.template.defaultfilters import slugify
from django.contrib.gis.geos import Point
with open('/home/capooti/temp/allCountries.txt', 'rb') as csvfile:
Place.objects.all().delete()
csvreader = csv.reader(csvfile, delimiter='\t')
for row in csvreader:
name = row[1]
lat = float(row[4])
lon = float(row[5])
print name, lat, lon
p = Place()
p.name = name
p.slug = slugify(name)
p.geometry = Point(lon, lat)
p.save()