def getTimelineFromWmsLayer(current_timeline, layerIdFunc):
basetimeLayer = bottle.request.query.get("basetimelayer")
timelineSize = bottle.request.query.get("timelinesize")
layerTimespan = bottle.request.query.get("layertimespan") # in seconds
if not basetimeLayer or not timelineSize or not layerTimespan:
return None
timelineSize = int(timelineSize)
layerTimespan = int(layerTimespan)
# import ipdb;ipdb.set_trace()
localfile = None
try:
# import ipdb;ipdb.set_trace()
localfile = tempfile.NamedTemporaryFile(mode='w+b', delete=False, prefix=basetimeLayer.replace(":", "_"), suffix=".gif").name
subprocess.check_call(["curl", "-G", "--cookie", settings.get_session_cookie(template="{0}={1}"), basetime_url.format(basetimeLayer), "--output", localfile])
md5 = settings.get_file_md5(localfile)
if current_timeline and current_timeline["md5"] == md5:
return current_timeline
else:
img = Image.open(localfile)
img.load()
basetimestr = pytesseract.image_to_string(img, lang="bom")
m = basetime_re.search(basetimestr, re.I)
if not m:
raise bottle.HTTPError(status=500, body="Can't extract the base time from base time layer.")
basetime = datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), 0, 0, tzinfo=pytz.timezone(m.group(6)))
now = datetime.datetime.now(pytz.timezone('UTC'))
if basetime > now:
raise bottle.HTTPError(status=500, body="Extract the wrong base time from base time layer.")
if (now - basetime).seconds > 86400:
raise bottle.HTTPError(status=500, body="Extract the wrong base time from base time layer.")
if basetime.year != int(m.group(1)) or basetime.month != int(m.group(2)) or basetime.day != int(m.group(3)) or basetime.hour != int(m.group(4)) or basetime.minute != int(m.group(5)):
raise bottle.HTTPError(status=500, body="Extract the wrong base time from base time layer.")
basetime = basetime.astimezone(pytz.timezone("Australia/Perth"))
layers = []
layertime = None
layerId = None
for i in xrange(0, timelineSize):
layertime = basetime + datetime.timedelta(seconds=layerTimespan * i)
layerId = layerIdFunc(i, layerTimespan)
layers.append([layertime.strftime("%a %b %d %Y %H:%M:%S AWST"), layerId, None])
return {"refreshtime": datetime.datetime.now().strftime("%a %b %d %Y %H:%M:%S"), "layers": layers, "md5": md5, "updatetime": basetime.strftime("%a %b %d %Y %H:%M:%S AWST")}
except:
traceback.print_exc()
if localfile:
#keep the file in tmp folder if failed
print("Can't extract base time from file '{}'".format(localfile))
localfile = None
raise
finally:
if localfile:
os.remove(localfile)
def spatial():
# needs gdal 1.10+
try:
features = json.loads(bottle.request.forms.get("features"))
options = bottle.request.forms.get("options")
if options:
options = json.loads(options)
else:
options = {}
cookies = settings.get_session_cookie()
results = []
features = features["features"] or []
index = 0
while index < len(features):
feature = features[index]
index += 1
feature_result = {}
results.append(feature_result)
for key,val in options.iteritems():
if "action" not in val:
val["action"] = key
if val["action"] == "getArea":
feature_result[key] = calculateArea(feature,cookies,val)
else:
feature_result[key] = getFeature(feature,cookies,val)
bottle.response.set_header("Content-Type", "application/json")
#print("{}:return response to client.{}".format(datetime.now(),results))
return {"total_features":len(results),"features":results}
except:
if bottle.response.status < 400 :
bottle.response.status = 400
bottle.response.set_header("Content-Type","text/plain")
traceback.print_exc()
return traceback.format_exception_only(sys.exc_type,sys.exc_value)
def spatial():
# needs gdal 1.10+
try:
features = json.loads(bottle.request.forms.get("features"))
options = bottle.request.forms.get("options")
if options:
options = json.loads(options)
else:
options = {}
cookies = settings.get_session_cookie()
results = []
features = features["features"] or []
index = 0
while index < len(features):
feature = features[index]
index += 1
feature_result = {}
results.append(feature_result)
for key, val in options.iteritems():
if "action" not in val:
val["action"] = key
if val["action"] == "getArea":
feature_result[key] = calculateArea(feature, cookies, val)
else:
feature_result[key] = getFeature(feature, cookies, val)
bottle.response.set_header("Content-Type", "application/json")
#print("{}:return response to client.{}".format(datetime.now(),results))
return {"total_features": len(results), "features": results}
except:
if bottle.response.status < 400:
bottle.response.status = 400
bottle.response.set_header("Content-Type", "text/plain")
traceback.print_exc()
return traceback.format_exception_only(sys.exc_type, sys.exc_value)
def get_layermetadata(layerids, kmiserver=settings.KMI_SERVER, results={}):
multiple_layers = True
if isinstance(layerids, basestring):
layerids = [layerids]
multiple_layers = False
#group layers against layer workspace
layers = {}
for layerid in layerids:
layerid = layerid.strip()
#check whether it is cached or not
key = layermetadatakey(layerid)
if uwsgi.cache_exists(key):
try:
metadata = uwsgi.cache_get(key)
if metadata:
if layerid in results:
results[layerid].update(json.loads(metadata))
else:
results[layerid] = json.loads(metadata)
#print("Retrieve the metadata from cache for layer ({})".format(layerid))
continue
except:
pass
layer = layerid.split(":")
if len(layer) == 1:
#no workspace
layer_ws = ""
layer = layer[0]
else:
layer_ws = layer[0]
layer = layer[1]
if layer_ws not in layers:
layers[layer_ws] = [layer]
else:
layers[layer_ws].append(layer)
if layers:
session_cookie = settings.get_session_cookie()
kmiserver = get_kmiserver(kmiserver)
#find the layer's metadata
url = None
for layer_ws, layers in layers.iteritems():
if layer_ws:
url = "{}/{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format(
kmiserver, layer_ws)
else:
url = "{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format(
kmiserver)
res = requests.get(url, verify=False, cookies=session_cookie)
res.raise_for_status()
tree = ET.fromstring(res.content)
capability = tree.find('Capability')
if not len(capability):
raise Exception("getCapability failed")
kmi_layers = capability.findall("Layer")
while kmi_layers:
kmi_layer = kmi_layers.pop()
name = get_child_value(kmi_layer, "Name")
if name:
try:
index = layers.index(name)
except:
index = -1
if index >= 0:
#this layer's metadata is requsted by the user
if layer_ws:
layerid = "{}:{}".format(layer_ws, name)
else:
layerid = name
if layerid in results:
result = results[layerid]
else:
result = {"id": layerid}
results[layerid] = result
del layers[index]
result["title"] = get_child_value(kmi_layer, "Title")
result["abstract"] = get_child_value(
kmi_layer, "Abstract")
result["srs"] = get_child_value(kmi_layer, "SRS")
bbox = kmi_layer.find("LatLonBoundingBox")
if bbox is not None:
result["latlonBoundingBox"] = [
float(bbox.attrib["miny"]),
float(bbox.attrib["minx"]),
float(bbox.attrib["maxy"]),
float(bbox.attrib["maxx"])
]
else:
result["latlonBoundingBox"] = None
for bbox in kmi_layer.findall("BoundingBox"):
result["latlonBoundingBox_{}".format(
bbox.attrib["SRS"].upper())] = [
float(bbox.attrib["miny"]),
float(bbox.attrib["minx"]),
float(bbox.attrib["maxy"]),
float(bbox.attrib["maxx"])
]
#cache it for 6 hours
key = layermetadatakey(result["id"])
try:
if uwsgi.cache_exists(key):
uwsgi.cache_update(key, json.dumps(result),
6 * 3600)
else:
uwsgi.cache_set(key, json.dumps(result),
6 * 3600)
except:
pass
#print("Retrieve the metadata from kmi for layer ({})".format(result["id"]))
if len(layers):
continue
else:
#already find metadata for all required layers
break
sub_layers = kmi_layer.findall("Layer")
if sub_layers:
kmi_layers += sub_layers
if len(layers) == 1:
if layer_ws:
raise Exception("The layer({}:{}) Not Found".format(
layer_ws, layers[0]))
else:
raise Exception("The layer({}) Not Found".format(
layers[0]))
elif len(layers) > 1:
if layer_ws:
raise Exception("The layers({}) Not Found".format(",".join(
["{}:{}".format(layer_ws, l) for l in layers])))
else:
raise Exception("The layers({}) Not Found".format(
",".join(layers)))
if multiple_layers:
return results
else:
return results[layerids[0]]
def get_layerdefinition(layerids, kmiserver=settings.KMI_SERVER, results={}):
kmiserver = get_kmiserver(kmiserver)
multiple_layers = True
if isinstance(layerids, basestring):
layerids = [layerids]
multiple_layers = False
#group layers against layer workspace
layers = {}
for layerid in layerids:
layerid = layerid.strip()
#check whether it is cached or not
key = layerdefinitionkey(layerid)
if uwsgi.cache_exists(key):
try:
definitiondata = uwsgi.cache_get(key)
if definitiondata:
if layerid in results:
results[layerid].update(json.loads(definitiondata))
else:
results[layerid] = json.loads(definitiondata)
continue
except:
pass
layer = layerid.split(":")
if len(layer) == 1:
#no workspace
layer_ws = ""
layer = layer[0]
else:
layer_ws = layer[0]
layer = layer[1]
if layer_ws not in layers:
layers[layer_ws] = [layerid]
else:
layers[layer_ws].append(layerid)
if layers:
kmiserver = get_kmiserver(kmiserver)
session_cookie = settings.get_session_cookie()
url = None
for layer_ws, layers in layers.iteritems():
if layer_ws:
url = "{}/{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format(
kmiserver, layer_ws, ",".join(layers))
else:
url = "{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format(
kmiserver, ",".join(layers))
res = requests.get(url, verify=False, cookies=session_cookie)
res.raise_for_status()
layersdata = res.json()
for layer in layersdata.get("featureTypes") or []:
if layer_ws:
layerid = "{}:{}".format(layer_ws, layer["typeName"])
else:
layerid = layer["typeName"]
try:
index = layers.index(layerid)
except:
index = -1
if index >= 0:
#this layer's metadata is requsted by the user
if layerid in results:
result = results[layerid]
else:
result = {"id": layerid}
results[layerid] = result
result["properties"] = layer["properties"]
result["geometry_property"] = None
result["geometry_properties"] = []
result["geometry_type"] = None
result["geometry_property_msg"] = None
del layers[index]
#find spatial columns
for prop in layer["properties"]:
if prop["type"].startswith("gml:"):
#spatial column
result["geometry_properties"].append(prop)
if len(result["geometry_properties"]) == 1:
result["geometry_property"] = result[
"geometry_properties"][0]
result["geometry_type"] = result[
"geometry_properties"][0]["localType"].lower()
elif len(result["geometry_properties"]) > 1:
#have more than one geometry properties, try to find the right one
if layer_ws:
url = "{}/{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format(
kmiserver, layer_ws, layerid)
else:
url = "{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format(
kmiserver, layerid)
res = requests.get(url,
verify=False,
cookies=session_cookie)
res.raise_for_status()
featuresdata = res.json()
if len(featuresdata["features"]) > 0:
feat = featuresdata["features"][0]
for prop in result["geometry_properties"]:
if prop["name"] == feat["geometry_name"]:
result["geometry_property"] = prop
result["geometry_type"] = prop[
"localType"].lower()
break
if not result["geometry_property"]:
result[
"geometry_property_msg"] = "Layer '{}' has more than one geometry columns, can't identity which column is used as the geometry column.".format(
layerid)
else:
result[
"geometry_property_msg"] = "Layer '{}' is not a spatial layer".format(
layerid)
if result["geometry_property"]:
#found the geometry property, remove it from properties
index = len(result["properties"]) - 1
while index >= 0:
if result["properties"][index] == result[
"geometry_property"]:
#this is the geometry property,remove it from properties
del result["properties"][index]
break
index -= 1
#cache it for 1 day
key = layerdefinitionkey(layerid)
try:
if uwsgi.cache_exists(key):
uwsgi.cache_update(key, json.dumps(result),
24 * 3600)
else:
uwsgi.cache_set(key, json.dumps(result), 24 * 3600)
except:
pass
if len(layers) == 1:
if layer_ws:
raise Exception("The layer({}:{}) Not Found".format(
layer_ws, layers[0]))
else:
raise Exception("The layer({}) Not Found".format(layers[0]))
elif len(layers) > 1:
if layer_ws:
raise Exception("The layers({}) Not Found".format(",".join(
["{}:{}".format(layer_ws, l) for l in layers])))
else:
raise Exception("The layers({}) Not Found".format(
",".join(layers)))
if multiple_layers:
return results
else:
return results[layerids[0]]
def get_layermetadata(layerids,kmiserver="https://kmi.dbca.wa.gov.au/geoserver",results={}):
multiple_layers = True
if isinstance(layerids,basestring):
layerids = [layerids]
multiple_layers = False
#group layers against layer workspace
layers = {}
for layerid in layerids:
layerid = layerid.strip()
#check whether it is cached or not
key = layermetadatakey(layerid)
if uwsgi.cache_exists(key):
try:
metadata = uwsgi.cache_get(key)
if metadata:
if layerid in results:
results[layerid].update(json.loads(metadata))
else:
results[layerid] = json.loads(metadata)
#print("Retrieve the metadata from cache for layer ({})".format(layerid))
continue
except:
pass
layer = layerid.split(":")
if len(layer) == 1:
#no workspace
layer_ws = ""
layer = layer[0]
else:
layer_ws = layer[0]
layer = layer[1]
if layer_ws not in layers:
layers[layer_ws] = [layer]
else:
layers[layer_ws].append(layer)
if layers:
session_cookie = settings.get_session_cookie()
kmiserver = get_kmiserver(kmiserver)
#find the layer's metadata
url = None
for layer_ws,layers in layers.iteritems():
if layer_ws:
url = "{}/{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format(kmiserver,layer_ws)
else:
url = "{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format(kmiserver)
res = requests.get(
url,
verify=False,
cookies=session_cookie
)
res.raise_for_status()
tree = ET.fromstring(res.content)
capability = tree.find('Capability')
if not len(capability):
raise Exception("getCapability failed")
kmi_layers = capability.findall("Layer")
while kmi_layers:
kmi_layer = kmi_layers.pop()
name = get_child_value(kmi_layer,"Name")
if name:
try:
index = layers.index(name)
except:
index = -1
if index >= 0:
#this layer's metadata is requsted by the user
if layer_ws:
layerid = "{}:{}".format(layer_ws,name)
else:
layerid = name
if layerid in results:
result = results[layerid]
else:
result = {"id":layerid}
results[layerid] = result
del layers[index]
result["title"] = get_child_value(kmi_layer,"Title")
result["abstract"] = get_child_value(kmi_layer,"Abstract")
result["srs"] = get_child_value(kmi_layer,"SRS")
bbox = kmi_layer.find("LatLonBoundingBox")
if bbox is not None:
result["latlonBoundingBox"] = [float(bbox.attrib["miny"]),float(bbox.attrib["minx"]),float(bbox.attrib["maxy"]),float(bbox.attrib["maxx"])]
else:
result["latlonBoundingBox"] = None
for bbox in kmi_layer.findall("BoundingBox"):
result["latlonBoundingBox_{}".format(bbox.attrib["SRS"].upper())] = [float(bbox.attrib["miny"]),float(bbox.attrib["minx"]),float(bbox.attrib["maxy"]),float(bbox.attrib["maxx"])]
#cache it for 6 hours
key = layermetadatakey(result["id"])
try:
if uwsgi.cache_exists(key):
uwsgi.cache_update(key, json.dumps(result),6 * 3600)
else:
uwsgi.cache_set(key, json.dumps(result),6 * 3600)
except:
pass
#print("Retrieve the metadata from kmi for layer ({})".format(result["id"]))
if len(layers):
continue
else:
#already find metadata for all required layers
break
sub_layers = kmi_layer.findall("Layer")
if sub_layers:
kmi_layers += sub_layers
if len(layers) == 1:
if layer_ws:
raise Exception("The layer({}:{}) Not Found".format(layer_ws,layers[0]))
else:
raise Exception("The layer({}) Not Found".format(layers[0]))
elif len(layers) > 1:
if layer_ws:
raise Exception("The layers({}) Not Found".format(",".join(["{}:{}".format(layer_ws,l) for l in layers])))
else:
raise Exception("The layers({}) Not Found".format(",".join(layers)))
if multiple_layers:
return results
else:
return results[layerids[0]]
def get_layerdefinition(layerids,kmiserver="https://kmi.dbca.wa.gov.au/geoserver",results={}):
kmiserver = get_kmiserver(kmiserver)
multiple_layers = True
if isinstance(layerids,basestring):
layerids = [layerids]
multiple_layers = False
#group layers against layer workspace
layers = {}
for layerid in layerids:
layerid = layerid.strip()
#check whether it is cached or not
key = layerdefinitionkey(layerid)
if uwsgi.cache_exists(key):
try:
definitiondata = uwsgi.cache_get(key)
if definitiondata:
if layerid in results:
results[layerid].update(json.loads(definitiondata))
else:
results[layerid] = json.loads(definitiondata)
continue
except:
pass
layer = layerid.split(":")
if len(layer) == 1:
#no workspace
layer_ws = ""
layer = layer[0]
else:
layer_ws = layer[0]
layer = layer[1]
if layer_ws not in layers:
layers[layer_ws] = [layerid]
else:
layers[layer_ws].append(layerid)
if layers:
kmiserver = get_kmiserver(kmiserver)
session_cookie = settings.get_session_cookie()
url = None
for layer_ws,layers in layers.iteritems():
if layer_ws:
url = "{}/{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format(kmiserver,layer_ws,",".join(layers))
else:
url = "{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format(kmiserver,",".join(layers))
res = requests.get(
url,
verify=False,
cookies=session_cookie
)
res.raise_for_status()
layersdata = res.json()
for layer in layersdata.get("featureTypes") or []:
if layer_ws:
layerid = "{}:{}".format(layer_ws,layer["typeName"])
else:
layerid = layer["typeName"]
try:
index = layers.index(layerid)
except:
index = -1
if index >= 0:
#this layer's metadata is requsted by the user
if layerid in results:
result = results[layerid]
else:
result = {"id":layerid}
results[layerid] = result
result["properties"] = layer["properties"]
result["geometry_property"] = None
result["geometry_properties"] = []
result["geometry_type"] = None
result["geometry_property_msg"] = None
del layers[index]
#find spatial columns
for prop in layer["properties"]:
if prop["type"].startswith("gml:"):
#spatial column
result["geometry_properties"].append(prop)
if len(result["geometry_properties"]) == 1:
result["geometry_property"] = result["geometry_properties"][0]
result["geometry_type"] = result["geometry_properties"][0]["localType"].lower()
elif len(result["geometry_properties"]) > 1:
#have more than one geometry properties, try to find the right one
if layer_ws:
url = "{}/{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format(kmiserver,layer_ws,layerid)
else:
url = "{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format(kmiserver,layerid)
res = requests.get(
url,
verify=False,
cookies=session_cookie
)
res.raise_for_status()
featuresdata = res.json()
if len(featuresdata["features"]) > 0:
feat = featuresdata["features"][0]
for prop in result["geometry_properties"]:
if prop["name"] == feat["geometry_name"]:
result["geometry_property"] = prop
result["geometry_type"] = prop["localType"].lower()
break
if not result["geometry_property"]:
result["geometry_property_msg"] = "Layer '{}' has more than one geometry columns, can't identity which column is used as the geometry column.".format(layerid)
else:
result["geometry_property_msg"] = "Layer '{}' is not a spatial layer".format(layerid)
if result["geometry_property"]:
#found the geometry property, remove it from properties
index = len(result["properties"]) - 1
while index >= 0:
if result["properties"][index] == result["geometry_property"]:
#this is the geometry property,remove it from properties
del result["properties"][index]
break
index -= 1
#cache it for 1 day
key = layerdefinitionkey(layerid)
try:
if uwsgi.cache_exists(key):
uwsgi.cache_update(key, json.dumps(result),24 * 3600)
else:
uwsgi.cache_set(key, json.dumps(result),24 * 3600)
except:
pass
if len(layers) == 1:
if layer_ws:
raise Exception("The layer({}:{}) Not Found".format(layer_ws,layers[0]))
else:
raise Exception("The layer({}) Not Found".format(layers[0]))
elif len(layers) > 1:
if layer_ws:
raise Exception("The layers({}) Not Found".format(",".join(["{}:{}".format(layer_ws,l) for l in layers])))
else:
raise Exception("The layers({}) Not Found".format(",".join(layers)))
if multiple_layers:
return results
else:
return results[layerids[0]]
def getTimelineFromWmsLayer(current_timeline, layerIdFunc):
basetimeLayer = bottle.request.query.get("basetimelayer")
timelineSize = bottle.request.query.get("timelinesize")
layerTimespan = bottle.request.query.get("layertimespan") # in seconds
if not basetimeLayer or not timelineSize or not layerTimespan:
return None
timelineSize = int(timelineSize)
layerTimespan = int(layerTimespan)
# import ipdb;ipdb.set_trace()
localfile = None
try:
# import ipdb;ipdb.set_trace()
localfile = tempfile.NamedTemporaryFile(mode='w+b',
delete=False,
prefix=basetimeLayer.replace(
":", "_"),
suffix=".gif").name
subprocess.check_call([
"curl", "-G", "--cookie",
settings.get_session_cookie(template="{0}={1}"),
basetime_url.format(basetimeLayer), "--output", localfile
])
md5 = settings.get_file_md5(localfile)
if current_timeline and current_timeline["md5"] == md5:
return current_timeline
else:
img = Image.open(localfile)
img.load()
basetimestr = pytesseract.image_to_string(img, lang="bom")
m = basetime_re.search(basetimestr, re.I)
if not m:
raise bottle.HTTPError(
status=500,
body="Can't extract the base time from base time layer.")
basetime = datetime.datetime(int(m.group(1)),
int(m.group(2)),
int(m.group(3)),
int(m.group(4)),
int(m.group(5)),
0,
0,
tzinfo=pytz.timezone(m.group(6)))
now = datetime.datetime.now(pytz.timezone('UTC'))
if basetime > now:
raise bottle.HTTPError(
status=500,
body="Extract the wrong base time from base time layer.")
if (now - basetime).seconds > 86400:
raise bottle.HTTPError(
status=500,
body="Extract the wrong base time from base time layer.")
if basetime.year != int(m.group(1)) or basetime.month != int(
m.group(2)) or basetime.day != int(
m.group(3)) or basetime.hour != int(
m.group(4)) or basetime.minute != int(m.group(5)):
raise bottle.HTTPError(
status=500,
body="Extract the wrong base time from base time layer.")
basetime = basetime.astimezone(pytz.timezone("Australia/Perth"))
layers = []
layertime = None
layerId = None
for i in xrange(0, timelineSize):
layertime = basetime + datetime.timedelta(
seconds=layerTimespan * i)
layerId = layerIdFunc(i, layerTimespan)
layers.append([
layertime.strftime("%a %b %d %Y %H:%M:%S AWST"), layerId,
None
])
return {
"refreshtime":
datetime.datetime.now().strftime("%a %b %d %Y %H:%M:%S"),
"layers":
layers,
"md5":
md5,
"updatetime":
basetime.strftime("%a %b %d %Y %H:%M:%S AWST")
}
except:
traceback.print_exc()
if localfile:
#keep the file in tmp folder if failed
print("Can't extract base time from file '{}'".format(localfile))
localfile = None
raise
finally:
if localfile:
os.remove(localfile)
def download(fmt):
"""
form data:
layers: a layer or a list of layer
{
sourcename: output datasource name; if missing, derived from layer name or datasource
layer: output layer name,if missing, using the datasource layer name
default_geometry_type: The geometry type of the empty geometry
type_mapping: the mapping between geometry type and business name used in the datasource or layer name
srs: srs optional, default is first's source layer's srs
ignore_if_empty: empty layer will not be returned if true; default is false
sourcelayers: a source layer or a list of source layers
{
type: "WFS" or "UPLOAD" or "FORM". deduced from other properties
WFS: download the data from wfs server;
UPLOAD: download the data from http request
FORM: get the data form http form
url: wfs url if sourcetype is "WFS",
parameter: http request parameter if sourcetype is"UPLOAD" or "FORM"
srs: srs optional
defautl_srs:default srs; optional
datasource: used if sourcetype is "UPLOAD" and uploaded file contains multiple datasources
layer: layer name,required if datasource include multiple layers
where: filter the features
},
geometry_column: dictionary
name: geometry column name
type: geometry column type
field_strategy: default is Intersection
fields: optional
},
datasources:a datasource or a list of datasource
{
type: "WFS" or "UPLOAD" or "FORM", if missing, try to deduced from other data source properties.
WFS: download the data from wfs server;
UPLOAD: download the data from http request
FORM: get the data form http form
url: wfs url if sourcetype is "WFS",
parameter: http request parameter if sourcetype is"UPLOAD" or "FORM"
srs: srs optional
default_geometry_type: The geometry type of the empty geometry
defautl_srs:default srs; optional
datasource: used if sourcetype is "UPLOAD" and uploaded file contains multiple datasources
ignore_if_empty: empty layer will not be returned if true; default is false
field_strategy: default is Intersection
fields: optional
}
filename:optional, used when multiple output datasources are downloaded
srs: optional, output srs
"""
# needs gdal 1.10+
layers = bottle.request.forms.get("layers")
output = bottle.request.forms.get("output")
datasources = bottle.request.forms.get("datasources")
filename = bottle.request.forms.get("filename")
outputSrs = bottle.request.forms.get("srs")
try:
if layers:
layers = json.loads(layers)
if layers and not isinstance(layers,list):
#convert a layer to a list of layer
layers = [layers]
if datasources:
datasources = json.loads(datasources)
if datasources and not isinstance(datasources,list):
#convert a datasource to a list of datasource
datasources = [datasources]
if not layers and not datasources:
raise Exception("Both layers parameter and datasources parameter are missing.")
if layers is None:
layers = []
#if output format is not set, set to the default format "geojson"
fmt = SPATIAL_FORMATS.get(fmt.lower())
if not fmt:
raise Exception("Unsupported spatial format({})".format(fmt))
#If a source layer is not a union layer, changed it to a union layer which only have one sub layer
if layers:
for layer in layers:
if layer.get("fields"):
index = 0
while index < len(layer["fields"]):
if isinstance(layer["fields"][index],basestring):
layer["fields"][index] = {"name":layer["fields"][index],"src":layer["fields"][index]}
index += 1
if not layer.get("sourcelayers"):
raise Exception("Missing 'sourcelayers' in layer ({})".format(json.dumps(layer)))
elif not isinstance(layer["sourcelayers"],list):
layer["sourcelayers"] = [layer["sourcelayers"]]
for slayer in layer["sourcelayers"]:
if slayer.get("fields"):
index = 0
while index < len(slayer["fields"]):
if isinstance(slayer["fields"][index],basestring):
slayer["fields"][index] = {"name":slayer["fields"][index],"src":slayer["fields"][index]}
index += 1
elif layer.get("fields"):
slayer["fields"] = layer["fields"]
#set field strategy and ignore_if_empty for layers
if layers:
for layer in layers:
if not layer.get("field_strategy"):
layer["field_strategy"] = "Intersection"
layer["ignore_if_empty"] = layer.get("ignore_if_empty") or False
#if geometry column is a string, change it to a dict
if layer.get("geometry_column") and isinstance(layer["geometry_column"],basestring):
layer["geometry_column"] = {name:layer["geometry_column"]}
if layer.get("geometry_column",{}).get("type"):
if layer["geometry_column"]["type"].upper() in GEOMETRY_TYPE_MAP:
layer["geometry_column"]["type"] = GEOMETRY_TYPE_MAP[layer["geometry_column"]["type"].upper()]
else:
del layer["geometry_column"]["type"]
#set datasource's ignore_if_empty
if datasources:
for datasource in datasources:
datasource["ignore_if_empty"] = datasource.get("ignore_if_empty") or False
if datasource.get("fields"):
while index < len(datasource["fields"]):
if isinstance(datasource["fields"][index],basestring):
datasource["fields"][index] = {"name":datasource["fields"][index],"src":datasource["fields"][index]}
index += 1
if not datasource.get("field_strategy"):
datasource["field_strategy"] = "Intersection"
def setDatasourceType(ds):
if "type" in ds:
return
if "url" in ds:
ds["type"] = "WFS"
elif "parameter" in ds:
if bottle.request.forms.get(ds["parameter"]):
ds["type"] = "FORM"
elif bottle.request.files.get(ds["parameter"]):
ds["type"] = "UPLOAD"
else:
raise Exception("Can't locate the http request data ({})".format(ds["parameter"]))
else:
raise Exception("Can't deduce the type of the datasource ({})".format(json.dumps(ds)))
def getDatasourceName(ds,unique=False):
if "sourcename" in ds:
name = ds["sourcename"]
if unique and "where" in ds:
name = "{}-{}".format(name,settings.get_md5(ds["where"]))
elif ds["type"] == "WFS":
name = settings.typename(ds["url"])
if not name:
name = settings.get_md5(ds["url"])
if unique and "where" in ds:
name = "{}-{}".format(name,settings.get_md5(ds["where"]))
else:
name = name.replace(":","_")
if unique:
if "where" in ds:
name = "{}-{}-{}".format(name,settings.get_md5(ds["url"]),settings.get_md5(ds["where"]))
else:
name = "{}-{}".format(name,settings.get_md5(ds["url"]))
elif ds["type"] == "FORM":
name = ds["parameter"]
if unique and "where" in ds:
name = "{}-{}".format(name,settings.get_md5(ds["where"]))
elif ds["type"] == "UPLOAD":
filename = bottle.request.files.get(ds["parameter"]).filename
filename = os.path.split(filename)[1]
name = None
for fileext in COMPRESS_FILE_SETTINGS.iterkeys():
if filename.lower().endswith(fileext):
name = filename[:len(filename) - len(fileext)]
break
if not name:
name = os.path.splitext(filename)[0]
if unique and "where" in ds:
name = "{}-{}".format(name,settings.get_md5(ds["where"]))
return name
#set datasource type if not set
#import ipdb;ipdb.set_trace()
if layers:
for layer in layers:
for sourcelayer in layer["sourcelayers"]:
setDatasourceType(sourcelayer)
if datasources:
for ds in datasources:
setDatasourceType(ds)
#set the datasource name for all source layers or source datasources, if not set.
names = {}
if layers:
for layer in layers:
for sourcelayer in layer["sourcelayers"]:
name = getDatasourceName(sourcelayer)
names[name] = names.get(name,0) + 1
sourcelayer["sourcename"] = name
if datasources:
for ds in datasources:
name = getDatasourceName(ds)
names[name] = names.get(name,0) + 1
ds["sourcename"] = name
if layers:
for layer in layers:
for sourcelayer in layer["sourcelayers"]:
if names.get(sourcelayer["sourcename"],1) > 1:
sourcelayer["sourcename"] = getDatasourceName(sourcelayer,True)
if datasources:
for ds in datasources:
if names.get(ds["sourcename"],1) > 1:
ds["sourcename"] = getDatasourceName(ds,True)
del names
#import ipdb;ipdb.set_trace()
#load data sources
workdir = tempfile.mkdtemp()
cookies = settings.get_session_cookie()
loaddir = os.path.join(workdir,"load")
os.mkdir(loaddir)
loadedDatasources = {}
if layers:
for layer in layers:
for sourcelayer in layer["sourcelayers"]:
loadDatasource(cookies,loaddir,loadedDatasources,sourcelayer)
#import ipdb;ipdb.set_trace()
#load data sources and add all layers in datasources to layers
if datasources:
for datasource in datasources:
datasource["datasource"] = datasource.get("datasource") or "*"
loadDatasource(cookies,loaddir,loadedDatasources,datasource)
for dsfile in datasource["datasources"]:
if datasource.get("datasource") != "*" and datasource.get("datasource") != dsfile[0]:
continue
for metadata in getLayers(dsfile[0]):
sourcelayer = dict(datasource)
if "default_geometry_type" in sourcelayer:
del sourcelayer["default_geometry_type"]
sourcelayer["datasource"] = dsfile[1]
sourcelayer["meta"] = metadata
sourcelayer["layer"] = metadata["layer"]
loadDatasource(cookies,loaddir,loadedDatasources,sourcelayer)
layer = {
"sourcename":getBaseDatafileName(dsfile[1],True),
"sourcelayers":[sourcelayer],
"ignore_if_empty":datasource["ignore_if_empty"]
}
if "field_strategy" in sourcelayer:
layer["field_strategy"] = sourcelayer["field_strategy"]
del sourcelayer["field_strategy"]
if sourcelayer["format"]["name"] in ["geojson","json"]:
layer["layer"] = getBaseDatafileName(sourcelayer["datasource"][1],False)
else:
layer["layer"] = metadata["layer"]
if "default_geometry_type" in datasource:
layer["default_geometry_type"] = datasource["default_geometry_type"]
layers.append(layer)
del loadedDatasources
#remove layers which do not include spatial data
if layers:
index1 = len(layers) - 1
while index1 >= 0:
index2 = len(layers[index1]["sourcelayers"]) - 1
while index2 >= 0:
if not layers[index1]["sourcelayers"][index2].get("meta"):
#no spatial data
del layers[index1]["sourcelayers"][index2]
elif layers[index1]["ignore_if_empty"] and layers[index1]["sourcelayers"][index2]["meta"].get("features",0) == 0:
#no features
del layers[index1]["sourcelayers"][index2]
index2 -= 1
if len(layers[index1]["sourcelayers"]) == 0:
#no spatial data
del layers[index1]
index1 -= 1
if not layers:
raise Exception("No spatial data found.")
#import ipdb;ipdb.set_trace()
#determine the output layer name
if fmt["multilayer"]:
names = {}
#first get the shortest name and check whether it is unique
for layer in layers:
if layer.get("layer"):
name = layer["layer"]
else:
name = None
for srcLayer in layer["sourcelayers"]:
if srcLayer["meta"]["format"]["name"] not in ["geojson","json"]:
name = srcLayer["layer"]
break
elif not name:
#use the file name as the layer name if format is geojson or json
name = getBaseDatafileName(srcLayer["datasource"][1],False)
names[name] = names.get(name,0) + 1
layer["layer"] = name
#append the file path to the name if shortest name is duplicate.
for layer in layers:
name = layer["layer"]
if names.get(name,1) > 1:
sourcename = getBaseDatafileName(layer["sourcelayers"][0]["datasource"][1],True)
if sourcename.endswith(name):
sourcename = os.path.dirname(sourcename)
if sourcename and sourcename != "/":
sourcename = sourcename.replace("/","_")
layer["layer"] = "{}_from_{}".format(name,sourcename)
del names
else:
for layer in layers:
if not layer.get("layer"):
name = None
for srcLayer in layer["sourcelayers"]:
if srcLayer["meta"]["format"]["name"] not in ["geojson","json"]:
name = srcLayer["layer"]
break
elif not name:
#use the file name as the layer name if format is geojson or json
name = getBaseDatafileName(srcLayer["datasource"][1],False)
layer["layer"] = name
#import ipdb;ipdb.set_trace()
#determine the output datasource name
if fmt["multilayer"]:
for layer in layers:
#target format support multiple layer, use the output file name or the first source layer's file name as the source name
if filename:
layer["sourcename"] = filename
else:
layer["sourcename"] = getBaseDatafileName(layer["sourcelayers"][0]["file"])
else:
names = {}
#first get the prefered file name and check whether it is unique
for layer in layers:
if not layer.get("sourcename"):
layer["sourcename"] = getBaseDatafileName(layer["sourcelayers"][0]["datasource"][1],True)
names[layer["sourcename"]] = names.get(layer["sourcename"],0) + 1
#if all the sourcename are same, then no need to use separate folder, set the sourcename to None
if len(names) <= 1:
for layer in layers:
layer["sourcename"] = None
#append the layer name to the file name if the prefered name is duplicate.
#for layer in layers:
# if names[layer["sourcename"]] > 1:
# layer["sourcename"] = os.path.join(layer["sourcename"],layer["layer"])
# pass
del names
#determine the output srs
srss = {}
for layer in layers:
if srss.get(layer["sourcename"]):
layer["srs"] = srss[layer["sourcename"]]
elif layer.get("srs"):
srss[layer["sourcename"]] = layer["srs"]
elif outputSrs:
layer["srs"] = outputSrs
srss[layer["sourcename"]] = layer["srs"]
else:
layer["srs"] = None
for l in layer["sourcelayers"]:
if l["meta"]["srs"]:
layer["srs"] = l["meta"]["srs"]
srss[layer["sourcename"]] = layer["srs"]
break
del srss
#convert and union the layers
outputdir = os.path.join(workdir,"output")
os.mkdir(outputdir)
unsupported_layers = []
#print "{}".format(layers)
for layer in layers:
for sourcelayer in layer["sourcelayers"]:
if sourcelayer["meta"]["geometry"] not in SUPPORTED_GEOMETRY_TYPES and sourcelayer["meta"]["geometry"].find("UNKNOWN") < 0:
unsupported_layers.append("{}({})".format(sourcelayer["layer"],sourcelayer["meta"]["geometry"]))
if len(unsupported_layers) > 0:
raise Exception("Unsupported geometry type ({}).".format(str(unsupported_layers)))
outputFiles = []
vrtdir = os.path.join(workdir,"vrt")
os.mkdir(vrtdir)
#import ipdb;ipdb.set_trace()
for layer in layers:
#populate the vrt file
if layer.get("sourcename",None):
vrtFile = os.path.join(vrtdir,layer["sourcename"],"{}.vrt".format(layer["layer"]))
else:
vrtFile = os.path.join(vrtdir,"{}.vrt".format(layer["layer"]))
if not os.path.exists(os.path.dirname(vrtFile)):
os.makedirs(os.path.dirname(vrtFile))
if layer.get("geometry_column"):
layer["geometry_field"] = {"name":layer["geometry_column"]["name"]}
for slayer in layer["sourcelayers"]:
#populate fields
if slayer.get("fields"):
index = len(slayer["fields"]) - 1
while index >= 0:
try:
slayer["fields"][index] = [slayer["fields"][index]["name"]] + next(o for o in slayer["meta"]["fields"] if o[0].lower() == slayer["fields"][index]["src"].lower())
except StopIteration:
#field does not exist, remove it
del slayer["fields"][index]
index -= 1
if len(slayer["fields"]) == 0:
del slayer["fields"]
#populate geometry column
if layer.get("geometry_column"):
slayer["geometry_field"] = {"name":layer["geometry_column"]["name"]}
if slayer["meta"].get("geometry_column"):
slayer["geometry_field"]["src"] = slayer["meta"]["geometry_column"]
vrt = UNIONLAYERS_TEMPLATE.render(layer)
with open(vrtFile,"wb") as f:
f.write(vrt)
vrt = None
if fmt["multitype"]:
outputDatasource = getOutputDatasource(outputdir,fmt,layer)
if outputDatasource in outputFiles:
cmd = ["ogr2ogr","-skipfailures","-update" ,"-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
else:
cmd = ["ogr2ogr","-skipfailures" ,"-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
outputFiles.append(outputDatasource)
if "ogr2ogr_arguments" in fmt:
index = 1
for arg in fmt["ogr2ogr_arguments"]:
cmd.insert(index,arg)
index += 1
#print " ".join(cmd)
subprocess.check_call(cmd)
else:
#get all geometry types in the source layer list
srcTypes = []
index = len(layer["sourcelayers"]) - 1
while index >= 0:
srcLayer = layer["sourcelayers"][index]
if srcLayer["meta"]["geometry"] in SUPPORTED_GEOMETRY_TYPES:
if srcLayer["meta"]["geometry"] not in srcTypes:
srcTypes.append(srcLayer["meta"]["geometry"])
else:
for t in SUPPORTED_GEOMETRY_TYPES:
if getFeatureCount(srcLayer["datasource"][0],srcLayer["meta"]["layer"],t) and t not in srcTypes:
srcTypes.append(t)
index -= 1
if len(srcTypes) == 1:
if "default_geometry_type" in layer:
del layer["default_geometry_type"]
else:
hasEmptyFeatures = False
index = len(layer["sourcelayers"]) - 1
while index >= 0:
srcLayer = layer["sourcelayers"][index]
index -= 1
if getFeatureCount(srcLayer["datasource"][0],srcLayer["meta"]["layer"],"EMPTY") > 0:
hasEmptyFeatures = True
break
if hasEmptyFeatures:
if layer.get("default_geometry_type") == "auto":
for t in SUPPORTED_GEOMETRY_TYPES:
if t in srcTypes:
layer["default_geometry_type"] = t
break
elif layer.get("default_geometry_type"):
if layer.get("default_geometry_type") not in srcTypes:
srcTypes.append("default_geometry_type")
else:
srcTypes.append("EMPTY")
elif "default_geometry_type" in layer:
del layer["default_geometry_type"]
if len(srcTypes) == 1:
#has only one geometry type
outputDatasource = getOutputDatasource(outputdir,fmt,layer)
if outputDatasource in outputFiles:
cmd = ["ogr2ogr","-skipfailures","-update" ,"-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
else:
cmd = ["ogr2ogr","-skipfailures","-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
outputFiles.append(outputDatasource)
#print " ".join(cmd)
if "ogr2ogr_arguments" in fmt:
index = 1
for arg in fmt["ogr2ogr_arguments"]:
cmd.insert(index,arg)
index += 1
subprocess.check_call(cmd)
else:
for t in srcTypes:
where = ("OGR_GEOMETRY IS NULL" if t == "EMPTY" else ("OGR_GEOMETRY='{}' OR OGR_GEOMETRY IS NULL" if layer.get("default_geometry_type") == t else "OGR_GEOMETRY='{}'")).format(t)
outputDatasource = getOutputDatasource(outputdir,fmt,layer,t)
if outputDatasource in outputFiles:
cmd = ["ogr2ogr","-skipfailures","-update","-where", where,"-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
else:
cmd = ["ogr2ogr","-skipfailures","-where", where,"-t_srs",(layer.get("srs") or "EPSG:4326"), "-f", fmt["format"],outputDatasource, vrtFile]
outputFiles.append(outputDatasource)
#print " ".join(cmd)
if "ogr2ogr_arguments" in fmt:
index = 1
for arg in fmt["ogr2ogr_arguments"]:
cmd.insert(index,arg)
index += 1
subprocess.check_call(cmd)
#import ipdb;ipdb.set_trace()
#check whether outputfile is a single file or multiple files.
#if a single file is returned, outputfile will be the file ; otherwise outputfile will be none
outputfile = None
checkfiles = [outputdir]
while len(checkfiles) > 0:
checkfile = checkfiles.pop()
if os.path.isdir(checkfile):
dirlist = os.listdir(checkfile)
checkfiles.extend([os.path.join(checkfile,f) for f in dirlist])
elif outputfile:
outputfile = None
break
else:
outputfile = checkfile
del checkfiles
#import ipdb;ipdb.set_trace()
if outputfile:
#only one file is returned.
filemime = fmt["mime"]
if filename:
outputfilename = os.path.basename("{}{}".format(filename,fmt["fileext"]))
else:
outputfilename = os.path.basename(outputfile)
else:
#multiple files are returned, use zip to compress all files into one file.
ct = "application/zip"
if filename:
zipfile = os.path.join(workdir,filename)
else:
zipfile = os.path.join(workdir,getBaseDatafileName(layer["sourcelayers"][0]["file"]))
zipfile = zipfile + fmt["fileext"]
shutil.make_archive(zipfile, 'zip', outputdir)
outputfile = "{}.zip".format(zipfile)
filemime = "application/zip"
outputfilename = os.path.basename(outputfile)
output = open(outputfile)
bottle.response.set_header("Content-Type", filemime)
bottle.response.set_header("Content-Disposition", "attachment;filename='{}'".format(os.path.basename(outputfilename)))
return output
except:
bottle.response.status = 400
bottle.response.set_header("Content-Type","text/plain")
traceback.print_exc()
return traceback.format_exception_only(sys.exc_type,sys.exc_value)
finally:
try:
#print "workdir = {}".format(workdir)
shutil.rmtree(workdir)
pass
except:
pass