def parse_pnr_file(filename, realm):
""" """
i = new = updated = old = 0
start_time = last_time = time.time()
with open(filename, 'rt') as f:
for line in f.xreadlines():
data = parse_line(line)
#print data
id = int(data['placeid'])
if Spot.objects.filter(id=id).count() == 0:
s = Spot(realm=realm, id=id, name=data['name'], description=data['typetext'])
s.geography = data['point']
s.save()
new += 1
elif data['langcode'] == u'1':
# Spot exists, but this one with langcode 1 (Finnish) overrides it
s = Spot.objects.get(id=id)
s.name = data['name']
s.description = data['typetext']
s.geography = data['point']
s.save()
updated += 1
else:
# Spot exists, but this one won't override it because of langcode != 1
old += 1
so = Spot.objects.filter(id=id)[0]
print so.id, so.name, id, data['name']
i += 1
if i % 1000 == 0: # Commit once in a while
django.db.transaction.commit()
print "Commit %d (%d/%d/%d) %.6f" % (i, new, updated, old, time.time() - last_time)
last_time = time.time()
django.db.reset_queries() # Needed to empty django.db.connection.queries list
return i
def parse_geonames_file(geonames_file):
""" """
i = new = old = 0
start_time = last_time = time.time()
f = open(geonames_file, "rt")
for line in f.xreadlines():
t = [unicode(x, "utf8") for x in line.split("\t")]
id = int(t[0])
if Spot.objects.filter(id=id).count() == 0:
s = Spot(realm=realm, id=id, name=t[1])
s.geography = Point(float(t[5]), float(t[4]))
s.save()
new += 1
else:
old += 1
i += 1
if i % 1000 == 0:
django.db.transaction.commit()
print "Commit %d (%d/%d) %.6f" % (i, new, old, time.time() - last_time)
last_time = time.time()
django.db.reset_queries() # Needed to empty django.db.connection.queries list
f.close()
return i
