def taskWriteBackTile(key, headers, data):
GEVENT_MONKEY_PATCH = settings.TILEJET_GEVENT_MONKEY_PATCH
now = datetime.datetime.now()
tilecache, tile = getTileFromCache(
settings.CACHES['tiles']['LOCATION'],
settings.CACHES['tiles'],
'tiles',
key,
True,
GEVENT_MONKEY_PATCH=GEVENT_MONKEY_PATCH)
if not tilecache:
#log_root = settings.LOG_ERRORS_ROOT
#if log_root:
# if not os.path.exists(log_root):
# os.makedirs(log_root)
error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
with open(error_file,'a') as f:
line = "Error: Could not connect to cache (tiles)."
f.write(line+"\n")
return
# Double check that another thread didn't writeback already
if not tile:
from json import loads
from base64 import b64decode
tile = {
'headers': loads(headers),
'data': b64decode(data)
}
tilecache.set(key, tile)
def taskRequestTile(ts, iz, ix, iy, ext, verbose=True):
now = datetime.datetime.now()
# Load Logging Info
#log_root = settings.LOG_REQUEST_ROOT
log_format = settings.LOG_REQUEST_FORMAT
#if log_root and log_format:
# if not os.path.exists(log_root):
# os.makedirs(log_root)
#if settings.LOG_ERRORS_ROOT
# if not os.path.exists(log_root):
# os.makedirs(log_root)
indirect_file = settings.LOG_INDIRECT_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_indirect.tsv"
# Find TileSource
tilesource = None
tilesources = getTileSources(proxy=True)
for candidate in tilesources:
if candidate['id'] == ts:
tilesource = candidate
break
if not tilesource:
error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
with open(error_file,'a') as f:
line = "Error: Could not find tilesource for primary key "+str(ts)+"."
f.write(line+"\n")
return
#Y is always in regualar TMS before being added to task queue
iyf = flip_y(ix,iy,iz)
#iy, iyf = getYValues(None,tilesource,ix,iy,iz)
tile_bbox = tms_to_bbox(ix,iy,iz)
#Check if requested tile is within source's extents
returnBlankTile = False
returnErrorTile = False
intersects = True
if tilesource['extents']:
intersects = bbox_intersects_source(tilesource,ix,iyf,iz)
if not intersects:
returnBlankTile = True
validZoom = 0
#Check if inside source zoom levels
if tilesource['minZoom'] or tilesource['maxZoom']:
if (tilesource['minZoom'] and iz < tilesource['minZoom']):
validZoom = -1
elif (tilesource['maxZoom'] and iz > tilesource['maxZoom']):
validZoom = 1
if validZoom != 0:
#returnBlank = True
returnErrorTile = True
if returnBlankTile or returnErrorTile:
return
tile = None
if iz >= settings.TILEJET['cache']['memory']['minZoom'] and iz <= settings.TILEJET['cache']['memory']['maxZoom']:
#key = "{layer},{z},{x},{y},{ext}".format(layer=tilesource.name,x=ix,y=iy,z=iz,ext=ext)
key = ",".join([tilesource['name'],str(iz),str(ix),str(iy),ext])
tilecache, tile = getTileFromCache(
settings.CACHES['tiles']['LOCATION'],
settings.CACHES['tiles'],
'tiles',
key,
True,
GEVENT_MONKEY_PATCH=True)
if not tilecache:
error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
with open(error_file,'a') as f:
line = "Error: Could not connect to cache (tiles)."
f.write(line+"\n")
return
if tile:
if verbose:
print "task / cache hit for "+key
else:
if verbose:
print "task / cache miss for "+key
with open(indirect_file,'a') as f:
line = log_format.format(
status='indirect',
tileorigin=tilesource['origin'],
tilesource=tilesource['name'],
z=iz,x=ix,y=iy,
ip='-',
datetime=now.isoformat())
f.write(line+"\n")
from urllib2 import HTTPError
try:
if tilesource['type'] == TYPE_TMS:
tile = requestTileFromSource(tilesource,ix,iy,iz,ext,True)
elif tilesource['type'] == TYPE_TMS_FLIPPED:
tile = requestTileFromSource(tilesource,ix,iyf,iz,ext,True)
except HTTPError, err:
error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
with open(error_file,'a') as f:
line = "Error: HTTPError. Could not get tile ("+key+") from source."
f.write(line+"\n")
return
except:
def _requestTile(request, tileservice=None, tilesource=None, tileorigin=None, z=None, x=None, y=None, u=None, ext=None):
print "_requestTile"
now = datetime.datetime.now()
ip = getIPAddress(request)
#==#
if not tileorigin:
tileorigin = tilesource['origin']
#==#
verbose = True
ix = None
iy = None
iyf = None
iz = None
indirectTiles = None
nearbyTiles = None
parentTiles = None
childrenTiles = None
gw_client, gw_logs, gw_requests = None, None, None
if settings.GEOWATCH_ENABLED:
gw_client, gw_logs, gw_requests = provision_client_logs_requests()
if u:
iz, ix, iy = quadkey_to_tms(u)
elif x and y and z:
ix = int(x)
iy = int(y)
iz = int(z)
if tilesource['type'] == TYPE_BING:
u = tms_to_quadkey(ix, iy, iz)
iy, iyf = getYValues(tileservice,tilesource,ix,iy,iz)
tile_bbox = tms_to_bbox(ix,iy,iz)
if tilesource['cacheable']:
indirectTiles = []
if settings.TILEJET['heuristic']['nearby']['enabled']:
ir = settings.TILEJET['heuristic']['nearby']['radius']
nearbyTiles = getNearbyTiles(ix, iy, iz, ir)
indirectTiles.extend(nearbyTiles)
#print "Nearby Tiles", nearbyTiles
#print "Indirect Tiles", indirectTiles
if settings.TILEJET['heuristic']['up']['enabled']:
iDepth = getValue(settings.TILEJET['heuristic']['up'],'depth')
if iDepth:
parentTiles = getParentTiles(ix, iy, iz, depth=iDepth)
else:
parentTiles = getParentTiles(ix, iy, iz)
indirectTiles.extend(parentTiles)
#print "Parent Tiles"
#print parentTiles
heuristic_down = settings.TILEJET['heuristic']['down']
if heuristic_down['enabled']:
depth = heuristic_down['depth']
minZoom = heuristic_down['minZoom']
maxZoom = heuristic_down['maxZoom']
childrenTiles = getChildrenTiles(ix, iy, iz, depth, minZoom, maxZoom)
indirectTiles.extend(childrenTiles)
#print "Children Tiles: "+str(len(childrenTiles))
#print childrenTiles
#print "indirectTiles: ", indirectTiles
if gw_requests and indirectTiles:
start = time.time()
gw_requests.send_tile_requests(
str(tilesource['id']),
indirectTiles,
extension=ext,
now=now)
print "Duration Q: ", (time.time() - start)
#Check if requested tile is within source's extents
returnBlankTile = False
returnErrorTile = False
intersects = True
if tilesource['extents']:
intersects = bbox_intersects_source(tilesource,ix,iyf,iz)
if not intersects:
returnBlankTile = True
validZoom = 0
#Check if inside source zoom levels
if tilesource['minZoom'] or tilesource['maxZoom']:
if (tilesource['minZoom'] and iz < tilesource['minZoom']):
validZoom = -1
elif (tilesource['maxZoom'] and iz > tilesource['maxZoom']):
validZoom = 1
if validZoom != 0:
#returnBlank = True
returnErrorTile = True
if returnBlankTile:
print "responding with blank image"
image = blankTile(width=256, height=256)
response = HttpResponse(content_type="image/png")
image.save(response, "PNG")
return response
if returnErrorTile:
print "responding with a red image"
image = redTile(width=256, height=256)
response = HttpResponse(content_type="image/png")
image.save(response, "PNG")
return response
tile = None
if tilesource['cacheable'] and iz >= settings.TILEJET['cache']['memory']['minZoom'] and iz <= settings.TILEJET['cache']['memory']['maxZoom']:
#key = "{layer},{z},{x},{y},{ext}".format(layer=tilesource.name,x=ix,y=iy,z=iz,ext=ext)
key = ",".join([tilesource['name'],str(iz),str(ix),str(iy),ext])
tilecache, tile = getTileFromCache(
settings.CACHES['tiles']['LOCATION'],
settings.CACHES['tiles'],
'tiles',
key,
True,
GEVENT_MONKEY_PATCH=True)
if not tilecache:
print "Error: Could not connect to cache (tiles)."
line = "Error: Could not connect to cache (tiles)."
logTileRequestError(line, now)
if tile:
if verbose:
print "cache hit for "+key
logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'hit', now, ip, gw_logs=gw_logs)
else:
if tilecache and verbose:
print "cache miss for "+key
logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'miss', now, ip, gw_logs=gw_logs)
if tilesource['type'] == TYPE_TMS:
tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iy,z=iz,ext=ext,verbose=True)
elif tilesource['type'] == TYPE_TMS_FLIPPED:
tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iyf,z=iz,ext=ext,verbose=True)
elif tilesource['type'] == TYPE_BING:
tile = requestTileFromSource(tilesource=tilesource,u=u,ext=ext,verbose=True)
if settings.ASYNC_WRITEBACK:
from base64 import b64encode
try:
taskWriteBackTile.apply_async(
args=[key, json.dumps(tile['headers']), b64encode(tile['data'])],
kwargs=None,
queue="writeback")
except:
print "Error: Could not connect to writeback queue."
line = "Error: Could not connect to writeback queue."
logTileRequestError(line, now)
else:
try:
tilecache.set(key, tile)
except:
print "Error: Could not write back tile synchronously."
line = "Error: Could not write back tile synchronously."
logTileRequestError(line, now)
else:
if verbose:
print "cache bypass for "+tilesource['name']+"/"+str(iz)+"/"+str(ix)+"/"+str(iy)
logTileRequest(tileorigin, tilesource['name'], x, y, z, ext, 'bypass', now, ip, gw_logs=gw_logs)
if tilesource['type'] == TYPE_TMS:
tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iy,z=iz,ext=ext,verbose=True)
elif tilesource['type'] == TYPE_TMS_FLIPPED:
tile = requestTileFromSource(tilesource=tilesource,x=ix,y=iyf,z=iz,ext=ext,verbose=True)
elif tilesource['type'] == TYPE_BING:
tile = requestTileFromSource(tilesource=tilesource,u=u,ext=ext,verbose=True)
if not tile:
print "responding with a red image"
image = redTile(width=256, height=256)
response = HttpResponse(content_type="image/png")
image.save(response, "PNG")
return response
#print "Headers:"
#print tile['headers']
image = Image.open(StringIO.StringIO(tile['data']))
#Is Tile blank. then band.getextrema should return 0,0 for band 4
#Tile Cache watermarking is messing up bands
#bands = image.split()
#for band in bands:
# print band.getextrema()
response = HttpResponse(content_type="image/png")
image.save(response, "PNG")
return response