def process_esri_services(esri_services):
services_created = []
for esri_service in esri_services:
# for now we process only MapServer and ImageServer
if '/MapServer/' in esri_service.url or '/ImageServer/' in esri_service.url:
if '/ImageServer/' in esri_service.url:
service = create_service_from_endpoint(
esri_service.url, 'ESRI_ImageServer', '',
esri_service.serviceDescription)
if '/MapServer/' in esri_service.url:
# we import only MapServer with at least one layer
if hasattr(esri_service, 'layers'):
service = create_service_from_endpoint(
esri_service.url, 'ESRI_MapServer',
esri_service.mapName, esri_service.description)
services_created.append(service)
return services_created
def process_esri_services(esri_services):
services_created = []
for esri_service in esri_services:
# for now we process only MapServer and ImageServer
if '/MapServer/' in esri_service.url or '/ImageServer/' in esri_service.url:
if '/ImageServer/' in esri_service.url:
service = create_service_from_endpoint(
esri_service.url,
'ESRI_ImageServer',
'',
esri_service.serviceDescription
)
if '/MapServer/' in esri_service.url:
# we import only MapServer with at least one layer
if hasattr(esri_service, 'layers'):
service = create_service_from_endpoint(
esri_service.url,
'ESRI_MapServer',
esri_service.mapName,
esri_service.description
)
services_created.append(service)
return services_created
def update_layers_esri_mapserver(service):
"""
Update layers for an ESRI REST MapServer.
"""
esri_service = ArcMapService(service.url)
# check if it has a WMS interface
if 'supportedExtensions' in esri_service._json_struct:
if 'WMSServer' in esri_service._json_struct['supportedExtensions']:
# we need to change the url
# http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson
# http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS
wms_url = service.url.replace('/rest/services/', '/services/')
if '?f=pjson' in wms_url:
wms_url = wms_url.replace('?f=pjson', 'WMSServer?')
if '?f=json' in wms_url:
wms_url = wms_url.replace('?f=json', 'WMSServer?')
print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url
# import here as otherwise is circular (TODO refactor)
from utils import create_service_from_endpoint
create_service_from_endpoint(wms_url, 'OGC_WMS')
# now process the REST interface
for esri_layer in esri_service.layers:
# in some case the json is invalid
# esri_layer._json_struct
# {u'currentVersion': 10.01,
# u'error':
# {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}}
if 'error' not in esri_layer._json_struct:
print 'Updating layer %s' % esri_layer.name
layer, created = Layer.objects.get_or_create(name=esri_layer.id, service=service)
if layer.active:
layer.title = esri_layer.name
layer.abstract = esri_service.serviceDescription
layer.url = service.url
layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id})
# set a default srs
srs = 4326
try:
layer.bbox_x0 = esri_layer.extent.xmin
layer.bbox_y0 = esri_layer.extent.ymin
layer.bbox_x1 = esri_layer.extent.xmax
layer.bbox_y1 = esri_layer.extent.ymax
# crsOptions
srs = esri_layer.extent.spatialReference.wkid
# this is needed as esri_layer.extent can fail because of custom wkid in json
except KeyError:
pass
try:
layer.bbox_x0 = esri_layer._json_struct['extent']['xmin']
layer.bbox_y0 = esri_layer._json_struct['extent']['ymin']
layer.bbox_x1 = esri_layer._json_struct['extent']['xmax']
layer.bbox_y1 = esri_layer._json_struct['extent']['ymax']
wkt_text = esri_layer._json_struct['extent']['spatialReference']['wkt']
if wkt_text:
params = {'exact': 'True', 'error': 'True', 'mode': 'wkt', 'terms': wkt_text}
req = requests.get('http://prj2epsg.org/search.json', params=params)
object = json.loads(req.content)
srs = int(object['codes'][0]['code'])
except Exception:
pass
layer.save()
srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs)
layer.srs.add(srs)
# dates
add_mined_dates(layer)
def create_services_from_endpoint(url):
"""
Generate service/services from an endpoint.
WMS, WMTS, TMS endpoints correspond to a single service.
ESRI, CWS endpoints corrispond to many services.
"""
num_created = 0
endpoint = get_sanitized_endpoint(url)
try:
urllib2.urlopen(endpoint, timeout=10)
except Exception as e:
print 'ERROR! Cannot open this endpoint: %s' % endpoint
message = traceback.format_exception(*sys.exc_info())
return False, message
detected = False
# test if it is WMS, TMS, WMTS or Esri
# WMS
try:
service = WebMapService(endpoint, timeout=10)
service_type = 'OGC:WMS'
detected = True
service = create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract
)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OGC:TMS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract
)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# WMTS
if not detected:
try:
service = WebMapTileService(endpoint, timeout=10)
service_type = 'OGC:WMTS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract
)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# Esri
# a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services
if not detected:
try:
esri = ArcFolder(endpoint)
services = esri.services
service_type = 'ESRI'
detected = True
# root
root_services = process_esri_services(services)
num_created = num_created + len(root_services)
# folders
for folder in esri.folders:
folder_services = process_esri_services(folder.services)
num_created = num_created + len(folder_services)
except Exception as e:
print str(e)
if detected:
return True, '%s service/s created' % num_created
else:
return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def create_services_from_endpoint(url):
"""
Generate service/services from an endpoint.
WMS, WMTS, TMS endpoints correspond to a single service.
ESRI, CWS endpoints corrispond to many services.
"""
num_created = 0
endpoint = get_sanitized_endpoint(url)
try:
urllib2.urlopen(endpoint, timeout=10)
except Exception as e:
print 'ERROR! Cannot open this endpoint: %s' % endpoint
message = traceback.format_exception(*sys.exc_info())
return False, message
detected = False
# test if it is WMS, TMS, WMTS or Esri
# WMS
try:
service = WebMapService(endpoint, timeout=10)
service_type = 'OGC:WMS'
detected = True
service = create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OGC:TMS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# WMTS
if not detected:
try:
service = WebMapTileService(endpoint, timeout=10)
service_type = 'OGC:WMTS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# Esri
# a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services
if not detected:
try:
esri = ArcFolder(endpoint)
services = esri.services
service_type = 'ESRI'
detected = True
# root
root_services = process_esri_services(services)
num_created = num_created + len(root_services)
# folders
for folder in esri.folders:
folder_services = process_esri_services(folder.services)
num_created = num_created + len(folder_services)
except Exception as e:
print str(e)
if detected:
return True, '%s service/s created' % num_created
else:
return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def update_layers_esri_mapserver(service):
"""
Update layers for an ESRI REST MapServer.
"""
esri_service = ArcMapService(service.url)
# check if it has a WMS interface
if 'supportedExtensions' in esri_service._json_struct:
if 'WMSServer' in esri_service._json_struct['supportedExtensions']:
# we need to change the url
# http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson
# http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS
wms_url = service.url.replace('/rest/services/', '/services/')
if '?f=pjson' in wms_url:
wms_url = wms_url.replace('?f=pjson', 'WMSServer?')
if '?f=json' in wms_url:
wms_url = wms_url.replace('?f=json', 'WMSServer?')
print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url
# import here as otherwise is circular (TODO refactor)
from utils import create_service_from_endpoint
create_service_from_endpoint(wms_url, 'OGC_WMS')
# now process the REST interface
for esri_layer in esri_service.layers:
# in some case the json is invalid
# esri_layer._json_struct
# {u'currentVersion': 10.01,
# u'error':
# {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}}
if 'error' not in esri_layer._json_struct:
print 'Updating layer %s' % esri_layer.name
layer, created = Layer.objects.get_or_create(name=esri_layer.id,
service=service)
if layer.active:
layer.title = esri_layer.name
layer.abstract = esri_service.serviceDescription
layer.url = service.url
layer.page_url = reverse('layer_detail',
kwargs={'layer_id': layer.id})
# set a default srs
srs = 4326
try:
layer.bbox_x0 = esri_layer.extent.xmin
layer.bbox_y0 = esri_layer.extent.ymin
layer.bbox_x1 = esri_layer.extent.xmax
layer.bbox_y1 = esri_layer.extent.ymax
# crsOptions
srs = esri_layer.extent.spatialReference.wkid
# this is needed as esri_layer.extent can fail because of custom wkid in json
except KeyError:
pass
try:
layer.bbox_x0 = esri_layer._json_struct['extent']['xmin']
layer.bbox_y0 = esri_layer._json_struct['extent']['ymin']
layer.bbox_x1 = esri_layer._json_struct['extent']['xmax']
layer.bbox_y1 = esri_layer._json_struct['extent']['ymax']
wkt_text = esri_layer._json_struct['extent'][
'spatialReference']['wkt']
if wkt_text:
params = {
'exact': 'True',
'error': 'True',
'mode': 'wkt',
'terms': wkt_text
}
req = requests.get('http://prj2epsg.org/search.json',
params=params)
object = json.loads(req.content)
srs = int(object['codes'][0]['code'])
except Exception:
pass
layer.save()
srs, created = SpatialReferenceSystem.objects.get_or_create(
code=srs)
layer.srs.add(srs)
# dates
add_mined_dates(layer)
