def __init__(self, url):
ArcMapServiceHandler.__init__(self, url)
self.proxy_base = None
self.url = url
self.parsed_service = ArcImageService(self.url)
extent, srs = utils.get_esri_extent(self.parsed_service)
try:
_sname = utils.get_esri_service_name(self.url)
_title_safe = safe(os.path.basename(os.path.normpath(_sname)))
_title = _title_safe.replace('_', ' ').strip()
except Exception:
traceback.print_exc()
_title = self.parsed_service.mapName
if len(_title) == 0:
_title = utils.get_esri_service_name(self.url)
# wkt_geometry = utils.bbox2wktpolygon([
# extent['xmin'],
# extent['ymin'],
# extent['xmax'],
# extent['ymax']
# ])
self.indexing_method = INDEXED
self.name = slugify(self.url)[:255]
self.title = _title
def update_layers_esri_imageserver(service):
"""
Update layers for an ESRI REST ImageServer.
"""
esri_service = ArcImageService(service.url)
obj = json.loads(esri_service._contents)
layer, created = Layer.objects.get_or_create(name=obj['name'], service=service)
if layer.active:
layer.type = 'ESRI:ArcGIS:ImageServer'
links = [[layer.type, service.url]]
layer.title = obj['name']
layer.abstract = esri_service.serviceDescription
layer.url = service.url
layer.bbox_x0 = str(obj['extent']['xmin'])
layer.bbox_y0 = str(obj['extent']['ymin'])
layer.bbox_x1 = str(obj['extent']['xmax'])
layer.bbox_y1 = str(obj['extent']['ymax'])
layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id})
srs = obj['extent']['spatialReference']['wkid']
links.append([
'WWW:LINK',
settings.SITE_URL.rstrip('/') + layer.page_url
])
layer.wkt_geometry = bbox2wktpolygon([layer.bbox_x0, layer.bbox_y0, layer.bbox_x1, layer.bbox_y1])
layer.xml = create_metadata_record(
identifier=layer.id_string,
source=service.url,
links=links,
format='ESRI:ArcGIS:ImageServer',
type=layer.csw_type,
relation=service.id_string,
title=layer.title,
alternative=layer.title,
abstract=layer.abstract,
wkt_geometry=layer.wkt_geometry,
srs=srs
)
layer.anytext = gen_anytext(layer.title, layer.abstract)
layer.save()
# crsOptions
srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs)
layer.srs.add(srs)
# dates
add_mined_dates(layer)
def check(self):
"""
Check for availability of a service and provide run metrics.
"""
from utils import get_esri_service_name
success = True
start_time = datetime.datetime.utcnow()
message = ''
print 'Checking service id %s' % self.id
try:
title = None
if self.type == 'OGC_WMS':
ows = WebMapService(self.url)
title = ows.identification.title
if self.type == 'OGC_WMTS':
ows = WebMapTileService(self.url)
title = ows.identification.title
if self.type == 'ESRI_MapServer':
esri = ArcMapService(self.url)
title = esri.mapName
if len(title) == 0:
title = get_esri_service_name(self.url)
if self.type == 'ESRI_ImageServer':
esri = ArcImageService(self.url)
title = esri._json_struct['name']
if len(title) == 0:
title = get_esri_service_name(self.url)
if self.type == 'WM':
urllib2.urlopen(self.url)
title = 'Harvard WorldMap'
if self.type == 'WARPER':
urllib2.urlopen(self.url)
# update title without raising a signal and recursion
if title:
Service.objects.filter(id=self.id).update(title=title)
except Exception, err:
message = str(err)
success = False
def update_layers_esri_imageserver(service):
"""
Update layers for an ESRI REST ImageServer.
"""
esri_service = ArcImageService(service.url)
obj = json.loads(esri_service._contents)
layer, created = Layer.objects.get_or_create(name=obj['name'],
service=service)
if layer.active:
layer.title = obj['name']
layer.abstract = esri_service.serviceDescription
layer.url = service.url
layer.bbox_x0 = str(obj['extent']['xmin'])
layer.bbox_y0 = str(obj['extent']['ymin'])
layer.bbox_x1 = str(obj['extent']['xmax'])
layer.bbox_y1 = str(obj['extent']['ymax'])
layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id})
layer.save()
# crsOptions
srs = obj['spatialReference']['wkid']
srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs)
layer.srs.add(srs)
# dates
add_mined_dates(layer)
def parsed_service(self):
return ArcImageService(self.url)
class Layer(Resource):
"""
Service represents a remote layer.
"""
name = models.CharField(max_length=255, null=True, blank=True)
# bbox should be in WGS84
bbox_x0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
bbox_x1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
bbox_y0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
bbox_y1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True)
thumbnail = models.ImageField(upload_to='layers', blank=True, null=True)
page_url = models.URLField(max_length=255)
srs = models.ManyToManyField(SpatialReferenceSystem)
service = models.ForeignKey(Service)
catalogs = models.ManyToManyField(Catalog)
def __unicode__(self):
return '%s - %s' % (self.id, self.name)
def get_url_endpoint(self):
"""
Returns the Hypermap endpoint for a layer.
This endpoint will be the WMTS MapProxy endpoint, only for WM and Esri we use original endpoints.
"""
endpoint = self.url
if self.type not in ('Hypermap:WorldMap', 'ESRI:ArcGIS:MapServer', 'ESRI:ArcGIS:ImageServer'):
endpoint = '%s/layer/%s/map/wmts/1.0.0/WMTSCapabilities.xml' % (settings.SITE_URL, self.id)
return endpoint
def get_tile_url(self):
"""
Returns the tile url MapProxy endpoint for the layer.
"""
if self.type not in ('Hypermap:WorldMap', 'ESRI:ArcGIS:MapServer', 'ESRI:ArcGIS:ImageServer'):
return '/layers/%s/map/wmts/nypl_map/default_grid/{z}/{y}/{x}.png' % self.id
else:
return None
def has_valid_bbox(self):
if self.bbox_x0 is None or self.bbox_y0 is None or self.bbox_x1 is None or self.bbox_y1 is None:
return False
else:
if self.bbox_x0 > self.bbox_x1 or self.bbox_y0 > self.bbox_y1:
return False
else:
return True
def get_layer_dates(self):
dates = []
if hasattr(self, 'layerwm'):
if self.layerwm.temporal_extent_start:
pydate = get_parsed_date(self.layerwm.temporal_extent_start)
if pydate:
start_date = []
start_date.append(pydate)
start_date.append(1)
dates.append(start_date)
if self.layerwm.temporal_extent_start:
pydate = get_parsed_date(self.layerwm.temporal_extent_end)
if pydate:
end_date = []
end_date.append(pydate)
end_date.append(1)
dates.append(end_date)
for layerdate in self.layerdate_set.all().order_by('date'):
sdate = layerdate.date
if 'TO' not in sdate:
pydate = get_parsed_date(sdate)
if pydate:
date = []
date.append(pydate)
date.append(layerdate.type)
dates.append(date)
return dates
def update_thumbnail(self):
print 'Generating thumbnail for layer id %s' % self.id
if not self.has_valid_bbox():
raise ValueError('Extent for this layer is invalid, cannot generate thumbnail')
return None
format_error_message = 'This layer does not expose valid formats (png, jpeg) to generate the thumbnail'
img = None
if self.type == 'OGC:WMS':
ows = WebMapService(self.service.url)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'OGC:WMTS':
ows = WebMapTileService(self.service.url)
ows_layer = ows.contents[self.name]
image_format = 'image/png'
if image_format not in ows_layer.formats:
if 'image/jpeg' in ows_layer.formats:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.gettile(
layer=self.name,
tilematrixset=ows_layer.tilematrixsets[0],
tilematrix='0',
row='0',
column='0',
format=image_format
)
elif self.type == 'Hypermap:WorldMap':
ows = WebMapService(self.url, username=settings.WM_USERNAME, password=settings.WM_PASSWORD)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'Hypermap:WARPER':
ows = WebMapService(self.url)
op_getmap = ows.getOperationByName('GetMap')
image_format = 'image/png'
if image_format not in op_getmap.formatOptions:
if 'image/jpeg' in op_getmap.formatOptions:
image_format = 'image/jpeg'
else:
raise NotImplementedError(format_error_message)
img = ows.getmap(
layers=[self.name],
srs='EPSG:4326',
bbox=(
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
),
size=(50, 50),
format=image_format,
transparent=True
)
if 'ogc.se_xml' in img.info()['Content-Type']:
raise ValueError(img.read())
img = None
elif self.type == 'ESRI:ArcGIS:MapServer':
try:
image = None
arcserver = ArcMapService(self.service.url)
bbox = '%s, %s, %s, %s' % (
float(self.bbox_x0),
float(self.bbox_y0),
float(self.bbox_x1),
float(self.bbox_y1)
)
image = arcserver.ExportMap(
bbox=bbox,
layers='show:' + self.name,
transparent='true',
dpi='96',
format='jpg'
)
except Exception, e:
print e
name = re.sub('[^\w\-_\. ]', '_', self.name)
thumbnail_file_name = '%s%s.jpg' % ('/tmp/', name)
image.save(thumbnail_file_name)
img = open(thumbnail_file_name)
os.remove(thumbnail_file_name)
elif self.type == 'ESRI:ArcGIS:ImageServer':
image = None
try:
arcserver = ArcImageService(self.service.url)
bbox = (
str(self.bbox_x0) + ',' +
str(self.bbox_y0) + ',' +
str(self.bbox_x1) + ',' +
str(self.bbox_y1)
)
image = arcserver.ExportImage(bbox=bbox)
except Exception, e:
print e
def check_available(self):
"""
Check for availability of a service and provide run metrics.
"""
success = True
start_time = datetime.datetime.utcnow()
message = ''
print 'Checking service id %s' % self.id
try:
title = None
abstract = None
keywords = []
wkt_geometry = None
srs = '4326'
if self.type == 'OGC:WMS':
ows = WebMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
for c in ows.contents:
if ows.contents[c].parent is None:
wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84)
break
if self.type == 'OGC:WMTS':
ows = WebMapTileService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'OSGeo:TMS':
ows = TileMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'ESRI:ArcGIS:MapServer':
esri = ArcMapService(self.url)
extent, srs = get_esri_extent(esri)
title = esri.mapName
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'ESRI:ArcGIS:ImageServer':
esri = ArcImageService(self.url)
extent, srs = get_esri_extent(esri)
title = esri._json_struct['name']
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'Hypermap:WorldMap':
urllib2.urlopen(self.url)
title = 'Harvard WorldMap'
if self.type == 'Hypermap:WARPER':
urllib2.urlopen(self.url)
# update title without raising a signal and recursion
if title:
self.title = title
Service.objects.filter(id=self.id).update(title=title)
if abstract:
self.abstract = abstract
Service.objects.filter(id=self.id).update(abstract=abstract)
if keywords:
for kw in keywords:
# FIXME: persist keywords to Django model
self.keywords.add(kw)
if wkt_geometry:
self.wkt_geometry = wkt_geometry
Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry)
xml = create_metadata_record(
identifier=self.id_string,
source=self.url,
links=[[self.type, self.url]],
format=self.type,
type='service',
title=title,
abstract=abstract,
keywords=keywords,
wkt_geometry=self.wkt_geometry,
srs=srs
)
anytexts = gen_anytext(title, abstract, keywords)
Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service')
except Exception as err:
print(err)
message = str(err)
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
check = Check(
resource=self,
success=success,
response_time=response_time,
message=message
)
check.save()
print 'Service checked in %s seconds, status is %s' % (response_time, success)
