Пример #1
0
def parse_pnr_file(filename, realm):
    """    """
    i = new = updated = old = 0
    start_time = last_time = time.time()
    with open(filename, 'rt') as f:
        for line in f.xreadlines():
            data = parse_line(line)
            #print data
            id = int(data['placeid'])
            if Spot.objects.filter(id=id).count() == 0:
                s = Spot(realm=realm, id=id, name=data['name'], description=data['typetext'])
                s.geography = data['point']
                s.save()
                new += 1
            elif data['langcode'] == u'1':
                # Spot exists, but this one with langcode 1 (Finnish) overrides it
                s = Spot.objects.get(id=id)
                s.name = data['name']
                s.description = data['typetext']
                s.geography = data['point']
                s.save()
                updated += 1
            else:
                # Spot exists, but this one won't override it because of langcode != 1
                old += 1
                so = Spot.objects.filter(id=id)[0]
                print so.id, so.name, id, data['name']
            i += 1
            if i % 1000 == 0: # Commit once in a while
                django.db.transaction.commit()
                print "Commit %d (%d/%d/%d) %.6f" % (i, new, updated, old, time.time() - last_time)
                last_time = time.time()
                django.db.reset_queries() # Needed to empty django.db.connection.queries list
        return i
Пример #2
0
def parse_geonames_file(geonames_file):
    """    """
    i = new = old = 0
    start_time = last_time = time.time()
    f = open(geonames_file, "rt")
    for line in f.xreadlines():
        t = [unicode(x, "utf8") for x in line.split("\t")]
        id = int(t[0])
        if Spot.objects.filter(id=id).count() == 0:
            s = Spot(realm=realm, id=id, name=t[1])
            s.geography = Point(float(t[5]), float(t[4]))
            s.save()
            new += 1
        else:
            old += 1
        i += 1
        if i % 1000 == 0:
            django.db.transaction.commit()
            print "Commit %d (%d/%d) %.6f" % (i, new, old, time.time() - last_time)
            last_time = time.time()
            django.db.reset_queries()  # Needed to empty django.db.connection.queries list
    f.close()
    return i