Beispiel #1
0
 def get(self, key_name):
     p = OSMPOI.get_by_key_name(key_name)
     p.update_importance(p.importance + 1,
                         max_z=config.max_z_osm,
                         gt_key_name="osm")
     taskqueue.add(url='/gt/admin/update_tiles', method='GET')
     self.response.out.write('increased importance')
Beispiel #2
0
 def get(self, key_name):
     p = OSMPOI.get_by_key_name(key_name)
     if p:
         self.response.out.write("attr: ")
         attrs = json.loads(p.attr)
         for a in attrs:
             self.response.out.write("<br/>&nbsp;&nbsp;")
             self.response.out.write(a)
             self.response.out.write(": ")
             self.response.out.write(attrs[a])
         self.response.out.write("<br/>tags: ")
         tags = json.loads(p.tags)
         for t in tags:
             self.response.out.write("<br/>&nbsp;&nbsp;")
             self.response.out.write(t)
             self.response.out.write(": ")
             self.response.out.write(tags[t])
     else:
         self.response.out.write("no such place")
     self.response.out.write("<br/>importance: " + str(p.importance))
Beispiel #3
0
 def get(self,key_name):
     p = OSMPOI.get_by_key_name(key_name)
     if p:
         self.response.out.write("attr: ")
         attrs = json.loads(p.attr)
         for a in attrs:
             self.response.out.write("<br/>&nbsp;&nbsp;")
             self.response.out.write(a)
             self.response.out.write(": ")
             self.response.out.write(attrs[a])
         self.response.out.write("<br/>tags: ")
         tags = json.loads(p.tags)
         for t in tags:
             self.response.out.write("<br/>&nbsp;&nbsp;")
             self.response.out.write(t)
             self.response.out.write(": ")
             self.response.out.write(tags[t])
     else:
         self.response.out.write("no such place");
     self.response.out.write("<br/>importance: "+str(p.importance))
Beispiel #4
0
    def get(self, action=None):
        if action:
            if action == 'turn_download_on':
                turn_download_on()
            if action == 'turn_download_off':
                turn_download_off()

        self.response.out.write('Admin page<br/><br/>')
        self.response.out.write('<a href="/gt/">Home</a><br/><br/>')
        self.response.out.write(
            '<a href="/gt/admin/create_geo_trees">Create GeoTrees</a><br/><br/>'
        )
        self.response.out.write(
            '<a href="/_ah/admin">App Engine localhost admin</a><br/><br/>')
        self.response.out.write(
            '<a href="/gt/admin/add_points">Add OSM points to GeoTree</a><br/>'
        )
        self.response.out.write(
            '<a href="/gt/admin/add_cities">Add cities to GeoTree</a><br/><br/>'
        )
        self.response.out.write(
            '<a href="/gt/admin/update_tiles">Update OSM GeoTree tiles</a><br/>'
        )
        self.response.out.write(
            '<a href="/gt/admin/update_cities_tiles">Update Cities GeoTree tiles</a><br/><br/>'
        )
        if is_download_on():
            self.response.out.write(
                '<a href="/gt/admin/turn_download_off">Turn OSM Download OFF</a><br/><br/>'
            )
        else:
            self.response.out.write(
                '<a href="/gt/admin/turn_download_on">Turn OSM Download ON</a><br/><br/>'
            )

        if action:
            if action == 'create_geo_trees':
                gt = GeoTree.get(gt_key_name='osm')
                if not gt:
                    gt = GeoTree(key_name='osm',
                                 max_z=config.max_z_osm,
                                 min_z=config.min_z_osm)
                    gt.put()
                    self.response.out.write('\n\nInfo: Created osm GeoTree.')
                else:
                    gt.max_z = config.max_z_osm
                    gt.min_z = config.min_z_osm
                    gt.put()
                    self.response.out.write('\n\nInfo: OSM GeoTree exists.')
                gt = GeoTree.get(gt_key_name='cities')
                if not gt:
                    gt = GeoTree(key_name='cities',
                                 max_z=config.max_z_cities,
                                 min_z=config.min_z_cities)
                    gt.put()
                    self.response.out.write('\nInfo: Created cities GeoTree.')
                else:
                    gt.max_z = config.max_z_cities
                    gt.min_z = config.min_z_cities
                    gt.put()
                    self.response.out.write('\nInfo: Cities GeoTree exists.')
            if action == 'add_points':
                batch = OSMPOI.all().filter('is_in_tree =',
                                            False).fetch(self._BATCH_ADD_SIZE)
                if batch:
                    GeoTree.insert_points_list(batch,
                                               max_z=17,
                                               gt_key_name="osm")
                    self.response.out.write('\n\nInfo: added %d points' %
                                            len(batch))
                    for p in batch:
                        p.is_in_tree = True
                    db.put(batch)
                    taskqueue.add(url='/gt/admin/add_points', method='GET')
                else:
                    if GeoTree.exists(gt_key_name="osm"):
                        self.response.out.write('\n\nInfo: no POIs to add.')
                        taskqueue.add(url='/gt/admin/update_tiles',
                                      method='GET')
                    else:
                        self.response.out.write(
                            '\n\nInfo: GeoTree does not exist.')
            if action == 'add_cities':
                batch = City.all().filter('is_in_tree =',
                                          False).fetch(self._BATCH_ADD_SIZE)
                if batch:
                    GeoTree.insert_points_list(batch, gt_key_name="cities")
                    self.response.out.write('\n\nInfo: added %d cities' %
                                            len(batch))
                    for p in batch:
                        p.is_in_tree = True
                    db.put(batch)
                else:
                    if GeoTree.exists(gt_key_name="cities"):
                        self.response.out.write(
                            '\n\nInfo: no cities left out of tree')
                    else:
                        self.response.out.write(
                            '\n\nInfo: GeoTree does not exist')
            if action == 'update_tiles':
                message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE,
                                               gt_key_name="osm")
                if message:
                    if 'nothing to update' in message:
                        self.response.out.write('<br/>' + message)
                else:
                    taskqueue.add(url='/gt/admin/update_tiles', method='GET')
            if action == 'update_cities_tiles':
                message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE,
                                               gt_key_name="cities")
                if message:
                    self.response.out.write('<br/>' + message)
                else:
                    self.response.out.write('\n\nInfo: updated tiles')
            # memcaching is not used at the moment
            if action == 'clear_cache':
                memcache.flush_all()
                self.response.out.write(
                    '<br/>All memcache entries are deleted.')
Beispiel #5
0
class DoUpdateOSM(webapp.RequestHandler):
    "update 3x3 tiles around tile with zoom=17 where a point specified in ll is located"

    def MarkOSMTileUpdated(self, x, y, z):
        key_name = '%d,%d,%d' % (x, y, z)
        t = OSMTile.get_by_key_name(key_name)
        if not t:
            t = OSMTile(key_name=key_name)
        t.put()

    def isOSMTileUpdated(self, x, y, z):
        key_name = '%d,%d,%d' % (x, y, z)
        t = OSMTile.get_by_key_name(key_name)
        now = datetime.datetime.utcnow()
        if t:
            self.response.out.write("updated %d seconds ago\n" %
                                    timedelta_to_seconds(now - t.time_fetched))
            return True
        else:
            return False

    def post(self):
        ll = self.request.get('ll')
        lat, lon = map(float, ll.split(','))
        (x, y, z) = lat_lon_to_tile(lat, lon, config.max_z_osm)
        if self.isOSMTileUpdated(x, y, z):
            self.response.out.write("this tile is updated")
            return

        # OSM BBOX convention - left,bottom,right,top
        # http://wiki.openstreetmap.org/wiki/Bounding_Box

        top, left = tile_to_lat_lon(x - 1, y - 1, z)
        bottom, right = tile_to_lat_lon(x + 2, y + 2, z)

        url = "http://open.mapquestapi.com/xapi/api/0.6/map?bbox=%f,%f,%f,%f" % (
            left, bottom, right, top)
        # url = "http://api.openstreetmap.org/api/0.6/map?bbox=%f,%f,%f,%f" % (left, bottom, right, top)
        self.response.out.write("updating tiles %s" % (url))

        try:
            data = urllib2.urlopen(url)
        except urllib2.URLError, e:
            self.response.out.write("error fetching url")
            return

        count_ways = 0
        count_ways_closed = 0
        count_ways_closed_named = 0
        names_closed_ways = []

        count_nodes_tagged = 0
        count_nodes_named = 0
        names_nodes = []

        time_format = "%Y-%m-%dT%H:%M:%SZ"

        nodes_dict = {}

        for event, elem in etree.iterparse(data):

            if elem.tag == 'node':
                tags = elem.findall('tag')
                id = elem.attrib['id']
                nodes_dict[id] = elem
                if tags:
                    if len(tags) == 1 and tags[0].attrib['k'] == 'created_by':
                        continue
                    name = ''
                    importance = 1.
                    value_str = elem.attrib['timestamp']
                    time_edited_in_osm = datetime.datetime.fromtimestamp(
                        time.mktime(time.strptime(value_str, time_format)))
                    attr = json.dumps(elem.attrib)
                    lat = elem.attrib['lat']
                    lon = elem.attrib['lon']
                    coord = "%s,%s" % (lat, lon)

                    tags_dict = {}
                    for t in tags:
                        tags_dict[t.attrib['k']] = t.attrib['v']
                        if t.attrib['k'] == 'name':
                            count_nodes_named += 1
                            names_nodes.append(t.attrib['v'])
                            name = t.attrib['v']
                            importance = 3.

                    p = OSMPOI.get_by_key_name(id)
                    if p:
                        p = p
                    else:
                        p = OSMPOI(key_name=id,
                                   coord=coord,
                                   attr=attr,
                                   tags=json.dumps(tags_dict),
                                   time_edited_in_osm=time_edited_in_osm,
                                   name=name,
                                   importance=importance)
                    p.put()

                    self.response.out.write('\n%s' % elem.attrib['id'])
                    self.response.out.write('\n%s' % str(time_edited_in_osm))
                    count_nodes_tagged += 1

            if elem.tag == 'way':
                count_ways += 1
                nds = elem.findall('nd')
                # if closed way
                if nds[0].attrib['ref'] == nds[-1].attrib['ref']:

                    count_ways_closed += 1
                    tags = elem.findall('tag')
                    if tags:
                        if len(tags
                               ) == 1 and tags[0].attrib['k'] == 'created_by':
                            continue
                        id = 'w' + elem.attrib['id']
                        name = ''
                        importance = 2.
                        value_str = elem.attrib['timestamp']
                        time_edited_in_osm = datetime.datetime.fromtimestamp(
                            time.mktime(time.strptime(value_str, time_format)))
                        attr = json.dumps(elem.attrib)
                        tags_dict = {}
                        for t in tags:
                            tags_dict[t.attrib['k']] = t.attrib['v']
                            if t.attrib['k'] == 'name':
                                count_ways_closed_named += 1
                                names_closed_ways.append(t.attrib['v'])
                                name = t.attrib['v']
                                importance = 4.
                        # find coordinates
                        lat = 0.
                        lon = 0.
                        # do not count last node because it repeats the first
                        for n in nds[:-1]:
                            node_id = n.attrib['ref']
                            node = nodes_dict[node_id]
                            lat += float(node.attrib['lat'])
                            lon += float(node.attrib['lon'])
                        l = len(nds) - 1.
                        lat = lat / l
                        lon = lon / l
                        coord = "%f,%f" % (lat, lon)

                        p = OSMPOI.get_by_key_name(id)
                        if p:
                            p = p
                        else:
                            p = OSMPOI(key_name=id,
                                       coord=coord,
                                       attr=attr,
                                       tags=json.dumps(tags_dict),
                                       time_edited_in_osm=time_edited_in_osm,
                                       name=name,
                                       importance=importance)
                        p.put()

        # mark OSMTiles as updated
        for xx in range(x - 1, x + 2):
            for yy in range(y - 1, y + 2):
                self.MarkOSMTileUpdated(xx, yy, z)

        taskqueue.add(url='/gt/admin/add_points', method='GET')

        self.response.out.write('\nways: %d' % count_ways)
        self.response.out.write('\nclosed ways: %d' % count_ways_closed)
        self.response.out.write('\nnamed closed ways: %d' %
                                count_ways_closed_named)
        self.response.out.write('\ntagged nodes: %d' % count_nodes_tagged)
        self.response.out.write('\nnamed nodes: %d' % count_nodes_named)
Beispiel #6
0
    def get(self, action=None):
        if action:
            if action == "turn_download_on":
                turn_download_on()
            if action == "turn_download_off":
                turn_download_off()

        self.response.out.write("Admin page<br/><br/>")
        self.response.out.write('<a href="/gt/">Home</a><br/><br/>')
        self.response.out.write('<a href="/gt/admin/create_geo_trees">Create GeoTrees</a><br/><br/>')
        self.response.out.write('<a href="/_ah/admin">App Engine localhost admin</a><br/><br/>')
        self.response.out.write('<a href="/gt/admin/add_points">Add OSM points to GeoTree</a><br/>')
        self.response.out.write('<a href="/gt/admin/add_cities">Add cities to GeoTree</a><br/><br/>')
        self.response.out.write('<a href="/gt/admin/update_tiles">Update OSM GeoTree tiles</a><br/>')
        self.response.out.write('<a href="/gt/admin/update_cities_tiles">Update Cities GeoTree tiles</a><br/><br/>')
        if is_download_on():
            self.response.out.write('<a href="/gt/admin/turn_download_off">Turn OSM Download OFF</a><br/><br/>')
        else:
            self.response.out.write('<a href="/gt/admin/turn_download_on">Turn OSM Download ON</a><br/><br/>')

        if action:
            if action == "create_geo_trees":
                gt = GeoTree.get(gt_key_name="osm")
                if not gt:
                    gt = GeoTree(key_name="osm", max_z=config.max_z_osm, min_z=config.min_z_osm)
                    gt.put()
                    self.response.out.write("\n\nInfo: Created osm GeoTree.")
                else:
                    gt.max_z = config.max_z_osm
                    gt.min_z = config.min_z_osm
                    gt.put()
                    self.response.out.write("\n\nInfo: OSM GeoTree exists.")
                gt = GeoTree.get(gt_key_name="cities")
                if not gt:
                    gt = GeoTree(key_name="cities", max_z=config.max_z_cities, min_z=config.min_z_cities)
                    gt.put()
                    self.response.out.write("\nInfo: Created cities GeoTree.")
                else:
                    gt.max_z = config.max_z_cities
                    gt.min_z = config.min_z_cities
                    gt.put()
                    self.response.out.write("\nInfo: Cities GeoTree exists.")
            if action == "add_points":
                batch = OSMPOI.all().filter("is_in_tree =", False).fetch(self._BATCH_ADD_SIZE)
                if batch:
                    GeoTree.insert_points_list(batch, max_z=17, gt_key_name="osm")
                    self.response.out.write("\n\nInfo: added %d points" % len(batch))
                    for p in batch:
                        p.is_in_tree = True
                    db.put(batch)
                    taskqueue.add(url="/gt/admin/add_points", method="GET")
                else:
                    if GeoTree.exists(gt_key_name="osm"):
                        self.response.out.write("\n\nInfo: no POIs to add.")
                        taskqueue.add(url="/gt/admin/update_tiles", method="GET")
                    else:
                        self.response.out.write("\n\nInfo: GeoTree does not exist.")
            if action == "add_cities":
                batch = City.all().filter("is_in_tree =", False).fetch(self._BATCH_ADD_SIZE)
                if batch:
                    GeoTree.insert_points_list(batch, gt_key_name="cities")
                    self.response.out.write("\n\nInfo: added %d cities" % len(batch))
                    for p in batch:
                        p.is_in_tree = True
                    db.put(batch)
                else:
                    if GeoTree.exists(gt_key_name="cities"):
                        self.response.out.write("\n\nInfo: no cities left out of tree")
                    else:
                        self.response.out.write("\n\nInfo: GeoTree does not exist")
            if action == "update_tiles":
                message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE, gt_key_name="osm")
                if message:
                    if "nothing to update" in message:
                        self.response.out.write("<br/>" + message)
                else:
                    taskqueue.add(url="/gt/admin/update_tiles", method="GET")
            if action == "update_cities_tiles":
                message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE, gt_key_name="cities")
                if message:
                    self.response.out.write("<br/>" + message)
                else:
                    self.response.out.write("\n\nInfo: updated tiles")
            # memcaching is not used at the moment
            if action == "clear_cache":
                memcache.flush_all()
                self.response.out.write("<br/>All memcache entries are deleted.")
Beispiel #7
0
 def get(self,key_name):
     p = OSMPOI.get_by_key_name(key_name)
     if p.importance > 0:
         p.update_importance(p.importance-1, max_z=config.max_z_osm, gt_key_name="osm")
         taskqueue.add(url='/gt/admin/update_tiles', method='GET')
     self.response.out.write('decreased importance');