Example #1
0
def taskWriteBackTile(key, headers, data):
    GEVENT_MONKEY_PATCH = settings.TILEJET_GEVENT_MONKEY_PATCH
    now = datetime.datetime.now()
    tilecache, tile = getTileFromCache(
        settings.CACHES['tiles']['LOCATION'],
        settings.CACHES['tiles'],
        'tiles',
        key,
        True,
        GEVENT_MONKEY_PATCH=GEVENT_MONKEY_PATCH)
    if not tilecache:
        #log_root = settings.LOG_ERRORS_ROOT
        #if log_root:
        #    if not os.path.exists(log_root):
        #        os.makedirs(log_root)
        error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
        with open(error_file,'a') as f:
            line = "Error: Could not connect to cache (tiles)."
            f.write(line+"\n")
        return
    # Double check that another thread didn't writeback already
    if not tile:
        from json import loads
        from base64 import b64decode
        tile = {
            'headers': loads(headers),
            'data': b64decode(data)
        }
        tilecache.set(key, tile)
Example #2
0
def taskRequestTile(ts, iz, ix, iy, ext, verbose=True):

    now = datetime.datetime.now()
    # Load Logging Info
    #log_root = settings.LOG_REQUEST_ROOT
    log_format = settings.LOG_REQUEST_FORMAT
    #if log_root and log_format:
    #    if not os.path.exists(log_root):
    #        os.makedirs(log_root)

    #if settings.LOG_ERRORS_ROOT
    #    if not os.path.exists(log_root):
    #        os.makedirs(log_root)

    indirect_file = settings.LOG_INDIRECT_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_indirect.tsv"
    # Find TileSource
    tilesource = None
    tilesources = getTileSources(proxy=True)
    for candidate in tilesources:
        if candidate['id'] == ts:
            tilesource = candidate
            break

    if not tilesource:
        error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
        with open(error_file,'a') as f:
            line = "Error: Could not find tilesource for primary key "+str(ts)+"."
            f.write(line+"\n")
        return

    #Y is always in regualar TMS before being added to task queue
    iyf = flip_y(ix,iy,iz)
    #iy, iyf = getYValues(None,tilesource,ix,iy,iz)

    tile_bbox = tms_to_bbox(ix,iy,iz)

    #Check if requested tile is within source's extents
    returnBlankTile = False
    returnErrorTile = False
    intersects = True
    if tilesource['extents']:
        intersects = bbox_intersects_source(tilesource,ix,iyf,iz)
        if not intersects:
           returnBlankTile = True

    validZoom = 0
    #Check if inside source zoom levels
    if tilesource['minZoom'] or tilesource['maxZoom']:
        if (tilesource['minZoom'] and iz < tilesource['minZoom']):
            validZoom = -1
        elif (tilesource['maxZoom'] and iz > tilesource['maxZoom']):
           validZoom = 1

        if validZoom != 0:
            #returnBlank = True
            returnErrorTile = True

    if returnBlankTile or returnErrorTile:
        return

    tile = None
    if iz >= settings.TILEJET['cache']['memory']['minZoom'] and iz <= settings.TILEJET['cache']['memory']['maxZoom']:
        #key = "{layer},{z},{x},{y},{ext}".format(layer=tilesource.name,x=ix,y=iy,z=iz,ext=ext)
        key = ",".join([tilesource['name'],str(iz),str(ix),str(iy),ext])
        tilecache, tile = getTileFromCache(
            settings.CACHES['tiles']['LOCATION'],
            settings.CACHES['tiles'],
            'tiles',
            key,
            True,
            GEVENT_MONKEY_PATCH=True)


        if not tilecache:
            error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
            with open(error_file,'a') as f:
                line = "Error: Could not connect to cache (tiles)."
                f.write(line+"\n")
            return

        if tile:
            if verbose:
                print "task / cache hit for "+key
        else:
            if verbose:
                print "task / cache miss for "+key

            with open(indirect_file,'a') as f:
                line = log_format.format(
                    status='indirect',
                    tileorigin=tilesource['origin'],
                    tilesource=tilesource['name'],
                    z=iz,x=ix,y=iy,
                    ip='-',
                    datetime=now.isoformat())
                f.write(line+"\n")

            from urllib2 import HTTPError
            try:
                if tilesource['type'] == TYPE_TMS:
                    tile = requestTileFromSource(tilesource,ix,iy,iz,ext,True)
                elif tilesource['type'] == TYPE_TMS_FLIPPED:
                    tile = requestTileFromSource(tilesource,ix,iyf,iz,ext,True)
            except HTTPError, err:
                error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
                with open(error_file,'a') as f:
                    line = "Error: HTTPError.  Could not get tile ("+key+") from source."
                    f.write(line+"\n")
                return
            except:
Example #3
0
def _requestTile(request, tileservice=None, tilesource=None, tileorigin=None, z=None, x=None, y=None, u=None, ext=None):

    print "_requestTile"
    now = datetime.datetime.now()
    ip = getIPAddress(request)
    #==#
    if not tileorigin:
        tileorigin = tilesource['origin']
    #==#
    verbose = True
    ix = None
    iy = None
    iyf = None
    iz = None
    indirectTiles = None
    nearbyTiles = None
    parentTiles = None
    childrenTiles = None
    gw_client, gw_logs, gw_requests = None, None, None
    if settings.GEOWATCH_ENABLED:
        gw_client, gw_logs, gw_requests = provision_client_logs_requests()

    if u:
        iz, ix, iy = quadkey_to_tms(u)

    elif x and y and z:
        ix = int(x)
        iy = int(y)
        iz = int(z)

        if tilesource['type'] == TYPE_BING:
            u = tms_to_quadkey(ix, iy, iz)

    iy, iyf = getYValues(tileservice,tilesource,ix,iy,iz)

    tile_bbox = tms_to_bbox(ix,iy,iz)

    if tilesource['cacheable']:

        indirectTiles = []

        if settings.TILEJET['heuristic']['nearby']['enabled']:
            ir = settings.TILEJET['heuristic']['nearby']['radius']
            nearbyTiles = getNearbyTiles(ix, iy, iz, ir)
            indirectTiles.extend(nearbyTiles)
            #print "Nearby Tiles", nearbyTiles
            #print "Indirect Tiles", indirectTiles

        if settings.TILEJET['heuristic']['up']['enabled']:
            iDepth = getValue(settings.TILEJET['heuristic']['up'],'depth')
            if iDepth:
                parentTiles = getParentTiles(ix, iy, iz, depth=iDepth)
            else:
                parentTiles = getParentTiles(ix, iy, iz)
            indirectTiles.extend(parentTiles)
            #print "Parent Tiles"
            #print parentTiles

        heuristic_down = settings.TILEJET['heuristic']['down']
        if heuristic_down['enabled']:
            depth = heuristic_down['depth']
            minZoom = heuristic_down['minZoom']
            maxZoom = heuristic_down['maxZoom']
            childrenTiles = getChildrenTiles(ix, iy, iz, depth, minZoom, maxZoom)
            indirectTiles.extend(childrenTiles)
            #print "Children Tiles: "+str(len(childrenTiles))
            #print childrenTiles

        #print "indirectTiles: ", indirectTiles
        if gw_requests and indirectTiles:
            start = time.time()
            gw_requests.send_tile_requests(
                str(tilesource['id']),
                indirectTiles,
                extension=ext,
                now=now)
            print "Duration Q: ", (time.time() - start)

    #Check if requested tile is within source's extents
    returnBlankTile = False
    returnErrorTile = False
    intersects = True
    if tilesource['extents']:
        intersects = bbox_intersects_source(tilesource,ix,iyf,iz)
        if not intersects:
           returnBlankTile = True

    validZoom = 0
    #Check if inside source zoom levels
    if tilesource['minZoom'] or tilesource['maxZoom']:
        if (tilesource['minZoom'] and iz < tilesource['minZoom']):
            validZoom = -1
        elif (tilesource['maxZoom'] and iz > tilesource['maxZoom']):
           validZoom = 1

        if validZoom != 0:
            #returnBlank = True
            returnErrorTile = True 

    if returnBlankTile:
        print "responding with blank image"
        image = blankTile(width=256, height=256)
        response = HttpResponse(content_type="image/png")
        image.save(response, "PNG")
        return response

    if returnErrorTile:
        print "responding with a red image"
        image = redTile(width=256, height=256)
        response = HttpResponse(content_type="image/png")
        image.save(response, "PNG")
        return response

    tile = None
    if tilesource['cacheable'] and iz >= settings.TILEJET['cache']['memory']['minZoom'] and iz <= settings.TILEJET['cache']['memory']['maxZoom']:
        #key = "{layer},{z},{x},{y},{ext}".format(layer=tilesource.name,x=ix,y=iy,z=iz,ext=ext)
        key = ",".join([tilesource['name'],str(iz),str(ix),str(iy),ext])
        tilecache, tile = getTileFromCache(
            settings.CACHES['tiles']['LOCATION'],
            settings.CACHES['tiles'],
            'tiles',
            key,
            True,
            GEVENT_MONKEY_PATCH=True)

        if not tilecache:
            print "Error: Could not connect to cache (tiles)."
            line = "Error: Could not connect to cache (tiles)."
            logTileRequestError(line, now)

        if tile:
            if verbose:
                print "cache hit for "+key
            logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'hit', now, ip, gw_logs=gw_logs)
        else:
            if tilecache and verbose:
                print "cache miss for "+key
            logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'miss', now, ip, gw_logs=gw_logs)

            if tilesource['type'] == TYPE_TMS:
                tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iy,z=iz,ext=ext,verbose=True)
            elif tilesource['type'] == TYPE_TMS_FLIPPED:
                tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iyf,z=iz,ext=ext,verbose=True)
            elif tilesource['type'] == TYPE_BING:
                tile = requestTileFromSource(tilesource=tilesource,u=u,ext=ext,verbose=True)

            if settings.ASYNC_WRITEBACK:
                from base64 import b64encode
                try:
                    taskWriteBackTile.apply_async(
                        args=[key, json.dumps(tile['headers']), b64encode(tile['data'])],
                        kwargs=None,
                        queue="writeback")
                except:
                    print "Error: Could not connect to writeback queue."
                    line = "Error: Could not connect to writeback queue."
                    logTileRequestError(line, now)
            else:
                try:
                    tilecache.set(key, tile)
                except:
                    print "Error: Could not write back tile synchronously."
                    line = "Error: Could not write back tile synchronously."
                    logTileRequestError(line, now)

    else:
        if verbose:
            print "cache bypass for "+tilesource['name']+"/"+str(iz)+"/"+str(ix)+"/"+str(iy)
        logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'bypass', now, ip, gw_logs=gw_logs)

        if tilesource['type'] == TYPE_TMS:
            tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iy,z=iz,ext=ext,verbose=True)
        elif tilesource['type'] == TYPE_TMS_FLIPPED:
            tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iyf,z=iz,ext=ext,verbose=True)
        elif tilesource['type'] == TYPE_BING:
            tile = requestTileFromSource(tilesource=tilesource,u=u,ext=ext,verbose=True)



    if not tile:
        print "responding with a red image"
        image = redTile(width=256, height=256)
        response = HttpResponse(content_type="image/png")
        image.save(response, "PNG")
        return response

    #print "Headers:"
    #print tile['headers']
    image = Image.open(StringIO.StringIO(tile['data']))
    #Is Tile blank.  then band.getextrema should return 0,0 for band 4
    #Tile Cache watermarking is messing up bands
    #bands = image.split()
    #for band in bands:
    #    print band.getextrema()
    response = HttpResponse(content_type="image/png")
    image.save(response, "PNG")
    return response