def _register_arcgis_url(url, username, password, owner=None, parent=None):
"""
Register an ArcGIS REST service URL
"""
# http://maps1.arcgisonline.com/ArcGIS/rest/services
baseurl = _clean_url(url)
if re.search("\/MapServer\/*(f=json)*", baseurl):
# This is a MapService
arcserver = ArcMapService(baseurl)
if isinstance(arcserver,
ArcMapService) and arcserver.spatialReference.wkid in [
102100, 3857, 900913
]:
return_json = [
_process_arcgis_service(arcserver, owner=owner, parent=parent)
]
else:
return_json = [{
'msg':
_("Could not find any layers in a compatible projection.")
}]
else:
# This is a Folder
arcserver = ArcFolder(baseurl)
return_json = _process_arcgis_folder(arcserver,
services=[],
owner=owner,
parent=parent)
return HttpResponse(json.dumps(return_json),
mimetype='application/json',
status=200)
def __init__(self, url):
base.ServiceHandlerBase.__init__(self, url)
self.proxy_base = None
self.url = url
self.parsed_service = ArcMapService(self.url)
extent, srs = utils.get_esri_extent(self.parsed_service)
try:
_sname = utils.get_esri_service_name(self.url)
_title_safe = safe(os.path.basename(os.path.normpath(_sname)))
_title = _title_safe.replace('_', ' ').strip()
except Exception:
traceback.print_exc()
_title = self.parsed_service.mapName
if len(_title) == 0:
_title = utils.get_esri_service_name(self.url)
# wkt_geometry = utils.bbox2wktpolygon([
# extent['xmin'],
# extent['ymin'],
# extent['xmax'],
# extent['ymax']
# ])
self.indexing_method = INDEXED
self.name = slugify(self.url)[:255]
self.title = _title
def _register_arcgis_url(url,username, password, owner=None, parent=None):
"""
Register an ArcGIS REST service URL
"""
#http://maps1.arcgisonline.com/ArcGIS/rest/services
baseurl = _clean_url(url)
if re.search("\/MapServer\/*(f=json)*", baseurl):
#This is a MapService
arcserver = ArcMapService(baseurl)
return_json = [_process_arcgis_service(arcserver, owner=owner, parent=parent)]
else:
#This is a Folder
arcserver = ArcFolder(baseurl)
return_json = _process_arcgis_folder(arcserver, services=[], owner=owner, parent=parent)
return HttpResponse(json.dumps(return_json),
mimetype='application/json',
status=200)
def check(self):
"""
Check for availability of a service and provide run metrics.
"""
from utils import get_esri_service_name
success = True
start_time = datetime.datetime.utcnow()
message = ''
print 'Checking service id %s' % self.id
try:
title = None
if self.type == 'OGC_WMS':
ows = WebMapService(self.url)
title = ows.identification.title
if self.type == 'OGC_WMTS':
ows = WebMapTileService(self.url)
title = ows.identification.title
if self.type == 'ESRI_MapServer':
esri = ArcMapService(self.url)
title = esri.mapName
if len(title) == 0:
title = get_esri_service_name(self.url)
if self.type == 'ESRI_ImageServer':
esri = ArcImageService(self.url)
title = esri._json_struct['name']
if len(title) == 0:
title = get_esri_service_name(self.url)
if self.type == 'WM':
urllib2.urlopen(self.url)
title = 'Harvard WorldMap'
if self.type == 'WARPER':
urllib2.urlopen(self.url)
# update title without raising a signal and recursion
if title:
Service.objects.filter(id=self.id).update(title=title)
except Exception, err:
message = str(err)
success = False
def _register_arcgis_layers(service, arc=None):
"""
Register layers from an ArcGIS REST service
"""
arc = arc or ArcMapService(service.base_url)
for layer in arc.layers:
valid_name = slugify(layer.name)
count = 0
layer_uuid = str(uuid.uuid1())
bbox = [
layer.extent.xmin, layer.extent.ymin, layer.extent.xmax,
layer.extent.ymax
]
typename = layer.id
existing_layer = None
try:
existing_layer = Layer.objects.get(typename=typename,
service=service)
except Layer.DoesNotExist:
pass
llbbox = mercator_to_llbbox(bbox)
if existing_layer is None:
# Need to check if layer already exists??
saved_layer, created = Layer.objects.get_or_create(
typename=typename,
service=service,
defaults=dict(
name=valid_name,
store=service.name, # ??
storeType="remoteStore",
workspace="remoteWorkspace",
title=layer.name,
abstract=layer._json_struct['description']
or _("Not provided"),
uuid=layer_uuid,
owner=None,
srid="EPSG:%s" % layer.extent.spatialReference.wkid,
bbox_x0=llbbox[0],
bbox_x1=llbbox[2],
bbox_y0=llbbox[1],
bbox_y1=llbbox[3],
))
saved_layer.set_default_permissions()
saved_layer.save()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service, typename=layer.id)
service_layer.layer = saved_layer
service_layer.title = layer.name,
service_layer.description = saved_layer.abstract,
service_layer.styles = None
service_layer.save()
create_arcgis_links(saved_layer)
count += 1
message = "%d Layers Registered" % count
return_dict = {'status': 'ok', 'msg': message}
return return_dict
def parsed_service(self):
return ArcMapService(self.url)
def update_layers_esri_mapserver(service):
"""
Update layers for an ESRI REST MapServer.
"""
esri_service = ArcMapService(service.url)
# check if it has a WMS interface
if 'supportedExtensions' in esri_service._json_struct:
if 'WMSServer' in esri_service._json_struct['supportedExtensions']:
# we need to change the url
# http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson
# http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS
wms_url = service.url.replace('/rest/services/', '/services/')
if '?f=pjson' in wms_url:
wms_url = wms_url.replace('?f=pjson', 'WMSServer?')
if '?f=json' in wms_url:
wms_url = wms_url.replace('?f=json', 'WMSServer?')
print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url
# import here as otherwise is circular (TODO refactor)
from utils import create_service_from_endpoint
create_service_from_endpoint(wms_url, 'OGC_WMS')
# now process the REST interface
for esri_layer in esri_service.layers:
# in some case the json is invalid
# esri_layer._json_struct
# {u'currentVersion': 10.01,
# u'error':
# {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}}
if 'error' not in esri_layer._json_struct:
print 'Updating layer %s' % esri_layer.name
layer, created = Layer.objects.get_or_create(name=esri_layer.id,
service=service)
if layer.active:
layer.title = esri_layer.name
layer.abstract = esri_service.serviceDescription
layer.url = service.url
layer.page_url = reverse('layer_detail',
kwargs={'layer_id': layer.id})
# set a default srs
srs = 4326
try:
layer.bbox_x0 = esri_layer.extent.xmin
layer.bbox_y0 = esri_layer.extent.ymin
layer.bbox_x1 = esri_layer.extent.xmax
layer.bbox_y1 = esri_layer.extent.ymax
# crsOptions
srs = esri_layer.extent.spatialReference.wkid
# this is needed as esri_layer.extent can fail because of custom wkid in json
except KeyError:
pass
try:
layer.bbox_x0 = esri_layer._json_struct['extent']['xmin']
layer.bbox_y0 = esri_layer._json_struct['extent']['ymin']
layer.bbox_x1 = esri_layer._json_struct['extent']['xmax']
layer.bbox_y1 = esri_layer._json_struct['extent']['ymax']
wkt_text = esri_layer._json_struct['extent'][
'spatialReference']['wkt']
if wkt_text:
params = {
'exact': 'True',
'error': 'True',
'mode': 'wkt',
'terms': wkt_text
}
req = requests.get('http://prj2epsg.org/search.json',
params=params)
object = json.loads(req.content)
srs = int(object['codes'][0]['code'])
except Exception:
pass
layer.save()
srs, created = SpatialReferenceSystem.objects.get_or_create(
code=srs)
layer.srs.add(srs)
# dates
add_mined_dates(layer)
def update_thumbnail(self):
print 'Generating thumbnail for layer id %s' % self.id
if not self.has_valid_bbox():
raise ValueError('Extent for this layer is invalid, cannot generate thumbnail')
return None
format_error_message = 'This layer does not expose valid formats (png, jpeg) to generate the thumbnail'
img = None
if self.type == 'OGC:WMS':
ows = WebMapService(self.service.url)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'OGC:WMTS':
ows = WebMapTileService(self.service.url)
ows_layer = ows.contents[self.name]
image_format = 'image/png'
if image_format not in ows_layer.formats:
if 'image/jpeg' in ows_layer.formats:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.gettile(
layer=self.name,
tilematrixset=ows_layer.tilematrixsets[0],
tilematrix='0',
row='0',
column='0',
format=image_format
)
elif self.type == 'Hypermap:WorldMap':
ows = WebMapService(self.url, username=settings.WM_USERNAME, password=settings.WM_PASSWORD)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'Hypermap:WARPER':
ows = WebMapService(self.url)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'ESRI:ArcGIS:MapServer':
try:
image = None
arcserver = ArcMapService(self.service.url)
bbox = '%s, %s, %s, %s' % (
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
)
image = arcserver.ExportMap(
bbox=bbox,
layers='show:' + self.name,
transparent='true',
dpi='96',
format='jpg'
)
except Exception, e:
print e
name = re.sub('[^\w\-_\. ]', '_', self.name)
thumbnail_file_name = '%s%s.jpg' % ('/tmp/', name)
image.save(thumbnail_file_name)
img = open(thumbnail_file_name)
os.remove(thumbnail_file_name)
def check_available(self):
"""
Check for availability of a service and provide run metrics.
"""
success = True
start_time = datetime.datetime.utcnow()
message = ''
print 'Checking service id %s' % self.id
try:
title = None
abstract = None
keywords = []
wkt_geometry = None
srs = '4326'
if self.type == 'OGC:WMS':
ows = WebMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
for c in ows.contents:
if ows.contents[c].parent is None:
wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84)
break
if self.type == 'OGC:WMTS':
ows = WebMapTileService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'OSGeo:TMS':
ows = TileMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'ESRI:ArcGIS:MapServer':
esri = ArcMapService(self.url)
extent, srs = get_esri_extent(esri)
title = esri.mapName
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'ESRI:ArcGIS:ImageServer':
esri = ArcImageService(self.url)
extent, srs = get_esri_extent(esri)
title = esri._json_struct['name']
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'Hypermap:WorldMap':
urllib2.urlopen(self.url)
title = 'Harvard WorldMap'
if self.type == 'Hypermap:WARPER':
urllib2.urlopen(self.url)
# update title without raising a signal and recursion
if title:
self.title = title
Service.objects.filter(id=self.id).update(title=title)
if abstract:
self.abstract = abstract
Service.objects.filter(id=self.id).update(abstract=abstract)
if keywords:
for kw in keywords:
# FIXME: persist keywords to Django model
self.keywords.add(kw)
if wkt_geometry:
self.wkt_geometry = wkt_geometry
Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry)
xml = create_metadata_record(
identifier=self.id_string,
source=self.url,
links=[[self.type, self.url]],
format=self.type,
type='service',
title=title,
abstract=abstract,
keywords=keywords,
wkt_geometry=self.wkt_geometry,
srs=srs
)
anytexts = gen_anytext(title, abstract, keywords)
Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service')
except Exception as err:
print(err)
message = str(err)
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
check = Check(
resource=self,
success=success,
response_time=response_time,
message=message
)
check.save()
print 'Service checked in %s seconds, status is %s' % (response_time, success)
