def _get_acls(self, user_id, data_url):
# parts = urlsplit(data_url)
# check_url = urlunsplit((parts.scheme, parts.netloc, parts.path, None, None))
# # TODO: ... should we do just a HEAD requset instead of API call?
# if not check_url.endswith('/'):
# check_url = "{0}/".format(check_url)
# check_url = urljoin(check_url, 'API/site/v1/can_access')
# TODO: ensure that request Session passes cookie on to correct domain
s = requests.Session()
for name in ('__ac', ): # 'serverid'):
# pass all interesting cookies for our domain on
# we copy cookies, so that we can set domain etc...
cookie = self.request.cookies.get(name)
if not cookie:
continue
if name == '__ac':
# append tokens if we set __ac cookie
# get my tokens
tokens = ','.join([
token.strip()
for token in self.request.registry.settings.get(
'authtkt.tokens', '').split('\n') if token.strip()
])
if cookie and tokens:
cookie = update_auth_cookie(cookie, tokens, self.request)
if cookie:
secure = self.request.registry.get(
'authtkt.secure',
'False').lower() not in ('false', '0', 'no')
s.cookies.set(name,
cookie,
secure=secure,
domain=self.request.host,
path='/')
# TODO: use with or whatever to close session
from pyramid.settings import asbool # FIXME: avoid circular import?
verify = asbool(
self.request.registry.settings.get('bccvl.ssl.verify', True))
r = s.head(data_url, verify=verify, allow_redirects=True)
s.close()
# TODO: do we have to handle redirects specially?
if r.status_code == 200:
permission = security.Allow
else:
permission = security.Deny
# LOG.info('Access %s for user %s to %s', permission, user_id, data_url)
# LOG.info(' -> %s', r.status_code)
return (permission, user_id, 'view')
def _get_acls(self, user_id, data_url):
# parts = urlsplit(data_url)
# check_url = urlunsplit((parts.scheme, parts.netloc, parts.path, None, None))
# # TODO: ... should we do just a HEAD requset instead of API call?
# if not check_url.endswith('/'):
# check_url = "{0}/".format(check_url)
# check_url = urljoin(check_url, 'API/site/v1/can_access')
# TODO: ensure that request Session passes cookie on to correct domain
s = requests.Session()
for name in ('__ac',): # 'serverid'):
# pass all interesting cookies for our domain on
# we copy cookies, so that we can set domain etc...
cookie = self.request.cookies.get(name)
if not cookie:
continue
if name == '__ac':
# append tokens if we set __ac cookie
# get my tokens
tokens = ','.join([token.strip() for token in
self.request.registry.settings.get('authtkt.tokens', '').split('\n') if token.strip()])
if cookie and tokens:
cookie = update_auth_cookie(cookie, tokens, self.request)
if cookie:
secure = self.request.registry.get('authtkt.secure', 'False').lower() not in ('false', '0', 'no')
s.cookies.set(name, cookie, secure=secure, domain=self.request.host, path='/')
# TODO: use with or whatever to close session
from pyramid.settings import asbool # FIXME: avoid circular import?
verify = asbool(self.request.registry.settings.get('bccvl.ssl.verify', True))
r = s.head(data_url, verify=verify, allow_redirects=True)
s.close()
# TODO: do we have to handle redirects specially?
if r.status_code == 200:
permission = security.Allow
else:
permission = security.Deny
# LOG.info('Access %s for user %s to %s', permission, user_id, data_url)
# LOG.info(' -> %s', r.status_code)
return (permission, user_id, 'view')
def fetch_file(request, url):
"""Dowload the file from url and place it on the local file system.
If file is a zip file it will be extracted to the local file system.
The method returns the filename of the requested file on the
local file system.
"""
# TODO: optimize data files for mapserver?
# reproject/warp source? to avoid mapserver doing warp on the fly
# otheroptions:
# convert to tiled raster (makes access to tiles faster)
# gdal_translate -co TILED=YES original.tif tiled.tif
# use Erdas Imagine (HFA) format ... always tiled and supports>4GB files
# gdal_translate -of HFA original.tif tiled.img
# add overview image to raster (after possible translate)
# gdaladdo [-r average] tiled.tif 2 4 8 16 32 64 128
# for rs point data maybe convert to shapefile?
if not (url.startswith('http://') or url.startswith('https://')):
# TODO: probably allow more than just http and https
# and use better exception
raise Exception('unsupported url scheme: %s', url)
# Check if a local data file is already exist
datadir = data_dir(request, url)
url, fragment = urlparse.urldefrag(url)
# FIXME: have to import here due to circular import
from pyramid.settings import asbool
with LockFile(datadir + '.lock'):
if not os.path.exists(datadir):
# the folder doesn't exist so we'll have to fetch the file
# TODO: make sure there is no '..' in datadir
os.makedirs(datadir)
# not available yet so fetch it
try:
settings = request.registry.settings
destfile = os.path.join(datadir, os.path.basename(url))
try:
src = {
'url': url,
'verify': asbool(settings.get('bccvl.ssl.verify', True))
}
# do we have an __ac cookie?
cookie = request.cookies.get('__ac')
# get my tokens
tokens = ','.join([
token.strip()
for token in settings.get(
'authtkt.tokens', '').split('\n') if token.strip()
])
if cookie:
src['cookies'] = {
'name': '__ac',
'value': update_auth_cookie(cookie, tokens, request),
'secure': True,
'domain': request.host,
'path': '/'
}
dst = {'url': u'file://{0}'.format(destfile)}
movelib.move(src, dst)
except Exception as e:
# direct download failed what now?
LOG.exception('Failed to download data %s: %s', url, e)
raise
# if it is a zip we should unpack it
# FIXME: do some more robust zip detection
if 'application/zip' in mimetypes.guess_type(destfile):
with zipfile.ZipFile(destfile, 'r') as zipf:
zipf.extractall(datadir)
# remove zipfile
os.remove(destfile)
# search all tifs and try to generate overviews
for root, dirnames, filenames in os.walk(datadir):
for filename in fnmatch.filter(filenames, '*.tif'):
rasterfile = os.path.join(root, filename)
ds = gdal.Open(rasterfile)
if ds:
maxlevel = min(ds.RasterXSize, ds.RasterYSize) / 512
ovrclear = ['gdaladdo', '-clean', rasterfile]
ovradd = ['gdaladdo', '-ro',
#'--config', 'COMPRESS_OVERVIEW', 'LZW',
rasterfile,
]
level = 2
while level < maxlevel:
ovradd.append(str(level))
level = level * 2
if maxlevel > 2:
subprocess.check_call(ovrclear)
subprocess.check_call(ovradd)
except Exception as e:
LOG.error('Could not download %s to %s : %s', url, datadir, e)
shutil.rmtree(datadir)
raise e
# we have the data now construct the filepath
filename = fragment if fragment else os.path.basename(url)
# FIXME: make sure path.join works correctly (trailing/leading slash?)
filename = os.path.join(datadir, filename)
# make sure filename is within datadir
filename = os.path.normpath(filename)
if not os.path.normpath(filename).startswith(datadir):
# FIXME: should probably check if filename exists and is supported
# and use better exception here
raise Exception("Data file path not valid: '%s'", filename)
return filename
def fetch_file(request, url):
"""Dowload the file from url and place it on the local file system.
If file is a zip file it will be extracted to the local file system.
The method returns the filename of the requested file on the
local file system.
"""
# TODO: optimize data files for mapserver?
# reproject/warp source? to avoid mapserver doing warp on the fly
# otheroptions:
# convert to tiled raster (makes access to tiles faster)
# gdal_translate -co TILED=YES original.tif tiled.tif
# use Erdas Imagine (HFA) format ... always tiled and supports>4GB files
# gdal_translate -of HFA original.tif tiled.img
# add overview image to raster (after possible translate)
# gdaladdo [-r average] tiled.tif 2 4 8 16 32 64 128
# for rs point data maybe convert to shapefile?
if not (url.startswith('http://') or url.startswith('https://')):
# TODO: probably allow more than just http and https
# and use better exception
raise Exception('unsupported url scheme: %s', url)
# Check if a local data file is already exist
datadir = data_dir(request, url)
url, fragment = urlparse.urldefrag(url)
# FIXME: have to import here due to circular import
from pyramid.settings import asbool
with LockFile(datadir + '.lock'):
if not os.path.exists(datadir):
# the folder doesn't exist so we'll have to fetch the file
# TODO: make sure there is no '..' in datadir
os.makedirs(datadir)
# not available yet so fetch it
try:
settings = request.registry.settings
destfile = os.path.join(datadir, os.path.basename(url))
try:
src = {
'url': url,
'verify': asbool(settings.get('bccvl.ssl.verify',
True))
}
# do we have an __ac cookie?
cookie = request.cookies.get('__ac')
# get my tokens
tokens = ','.join([
token.strip() for token in settings.get(
'authtkt.tokens', '').split('\n') if token.strip()
])
if cookie:
src['cookies'] = {
'name': '__ac',
'value':
update_auth_cookie(cookie, tokens, request),
'secure': True,
'domain': request.host,
'path': '/'
}
dst = {'url': u'file://{0}'.format(destfile)}
movelib.move(src, dst)
except Exception as e:
# direct download failed what now?
LOG.exception('Failed to download data %s: %s', url, e)
raise
# if it is a zip we should unpack it
# FIXME: do some more robust zip detection
if 'application/zip' in mimetypes.guess_type(destfile):
with zipfile.ZipFile(destfile, 'r') as zipf:
zipf.extractall(datadir)
# remove zipfile
os.remove(destfile)
# search all tifs and try to generate overviews
for root, dirnames, filenames in os.walk(datadir):
for filename in fnmatch.filter(filenames, '*.tif'):
rasterfile = os.path.join(root, filename)
ds = gdal.Open(rasterfile)
if ds:
maxlevel = min(ds.RasterXSize,
ds.RasterYSize) / 512
ovrclear = ['gdaladdo', '-clean', rasterfile]
ovradd = [
'gdaladdo',
'-ro',
#'--config', 'COMPRESS_OVERVIEW', 'LZW',
rasterfile,
]
level = 2
while level < maxlevel:
ovradd.append(str(level))
level = level * 2
if maxlevel > 2:
subprocess.check_call(ovrclear)
subprocess.check_call(ovradd)
except Exception as e:
LOG.error('Could not download %s to %s : %s', url, datadir, e)
shutil.rmtree(datadir)
raise e
# we have the data now construct the filepath
filename = fragment if fragment else os.path.basename(url)
# FIXME: make sure path.join works correctly (trailing/leading slash?)
filename = os.path.join(datadir, filename)
# make sure filename is within datadir
filename = os.path.normpath(filename)
if not os.path.normpath(filename).startswith(datadir):
# FIXME: should probably check if filename exists and is supported
# and use better exception here
raise Exception("Data file path not valid: '%s'", filename)
return filename
