def __init__(self, url): ArcMapServiceHandler.__init__(self, url) self.proxy_base = None self.url = url self.parsed_service = ArcImageService(self.url) extent, srs = utils.get_esri_extent(self.parsed_service) try: _sname = utils.get_esri_service_name(self.url) _title_safe = safe(os.path.basename(os.path.normpath(_sname))) _title = _title_safe.replace('_', ' ').strip() except Exception: traceback.print_exc() _title = self.parsed_service.mapName if len(_title) == 0: _title = utils.get_esri_service_name(self.url) # wkt_geometry = utils.bbox2wktpolygon([ # extent['xmin'], # extent['ymin'], # extent['xmax'], # extent['ymax'] # ]) self.indexing_method = INDEXED self.name = slugify(self.url)[:255] self.title = _title
def update_layers_esri_imageserver(service): """ Update layers for an ESRI REST ImageServer. """ esri_service = ArcImageService(service.url) obj = json.loads(esri_service._contents) layer, created = Layer.objects.get_or_create(name=obj['name'], service=service) if layer.active: layer.type = 'ESRI:ArcGIS:ImageServer' links = [[layer.type, service.url]] layer.title = obj['name'] layer.abstract = esri_service.serviceDescription layer.url = service.url layer.bbox_x0 = str(obj['extent']['xmin']) layer.bbox_y0 = str(obj['extent']['ymin']) layer.bbox_x1 = str(obj['extent']['xmax']) layer.bbox_y1 = str(obj['extent']['ymax']) layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id}) srs = obj['extent']['spatialReference']['wkid'] links.append([ 'WWW:LINK', settings.SITE_URL.rstrip('/') + layer.page_url ]) layer.wkt_geometry = bbox2wktpolygon([layer.bbox_x0, layer.bbox_y0, layer.bbox_x1, layer.bbox_y1]) layer.xml = create_metadata_record( identifier=layer.id_string, source=service.url, links=links, format='ESRI:ArcGIS:ImageServer', type=layer.csw_type, relation=service.id_string, title=layer.title, alternative=layer.title, abstract=layer.abstract, wkt_geometry=layer.wkt_geometry, srs=srs ) layer.anytext = gen_anytext(layer.title, layer.abstract) layer.save() # crsOptions srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs) layer.srs.add(srs) # dates add_mined_dates(layer)
def check(self): """ Check for availability of a service and provide run metrics. """ from utils import get_esri_service_name success = True start_time = datetime.datetime.utcnow() message = '' print 'Checking service id %s' % self.id try: title = None if self.type == 'OGC_WMS': ows = WebMapService(self.url) title = ows.identification.title if self.type == 'OGC_WMTS': ows = WebMapTileService(self.url) title = ows.identification.title if self.type == 'ESRI_MapServer': esri = ArcMapService(self.url) title = esri.mapName if len(title) == 0: title = get_esri_service_name(self.url) if self.type == 'ESRI_ImageServer': esri = ArcImageService(self.url) title = esri._json_struct['name'] if len(title) == 0: title = get_esri_service_name(self.url) if self.type == 'WM': urllib2.urlopen(self.url) title = 'Harvard WorldMap' if self.type == 'WARPER': urllib2.urlopen(self.url) # update title without raising a signal and recursion if title: Service.objects.filter(id=self.id).update(title=title) except Exception, err: message = str(err) success = False
def update_layers_esri_imageserver(service): """ Update layers for an ESRI REST ImageServer. """ esri_service = ArcImageService(service.url) obj = json.loads(esri_service._contents) layer, created = Layer.objects.get_or_create(name=obj['name'], service=service) if layer.active: layer.title = obj['name'] layer.abstract = esri_service.serviceDescription layer.url = service.url layer.bbox_x0 = str(obj['extent']['xmin']) layer.bbox_y0 = str(obj['extent']['ymin']) layer.bbox_x1 = str(obj['extent']['xmax']) layer.bbox_y1 = str(obj['extent']['ymax']) layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id}) layer.save() # crsOptions srs = obj['spatialReference']['wkid'] srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs) layer.srs.add(srs) # dates add_mined_dates(layer)
def parsed_service(self): return ArcImageService(self.url)
class Layer(Resource): """ Service represents a remote layer. """ name = models.CharField(max_length=255, null=True, blank=True) # bbox should be in WGS84 bbox_x0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) bbox_x1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) bbox_y0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) bbox_y1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) thumbnail = models.ImageField(upload_to='layers', blank=True, null=True) page_url = models.URLField(max_length=255) srs = models.ManyToManyField(SpatialReferenceSystem) service = models.ForeignKey(Service) catalogs = models.ManyToManyField(Catalog) def __unicode__(self): return '%s - %s' % (self.id, self.name) def get_url_endpoint(self): """ Returns the Hypermap endpoint for a layer. This endpoint will be the WMTS MapProxy endpoint, only for WM and Esri we use original endpoints. """ endpoint = self.url if self.type not in ('Hypermap:WorldMap', 'ESRI:ArcGIS:MapServer', 'ESRI:ArcGIS:ImageServer'): endpoint = '%s/layer/%s/map/wmts/1.0.0/WMTSCapabilities.xml' % (settings.SITE_URL, self.id) return endpoint def get_tile_url(self): """ Returns the tile url MapProxy endpoint for the layer. """ if self.type not in ('Hypermap:WorldMap', 'ESRI:ArcGIS:MapServer', 'ESRI:ArcGIS:ImageServer'): return '/layers/%s/map/wmts/nypl_map/default_grid/{z}/{y}/{x}.png' % self.id else: return None def has_valid_bbox(self): if self.bbox_x0 is None or self.bbox_y0 is None or self.bbox_x1 is None or self.bbox_y1 is None: return False else: if self.bbox_x0 > self.bbox_x1 or self.bbox_y0 > self.bbox_y1: return False else: return True def get_layer_dates(self): dates = [] if hasattr(self, 'layerwm'): if self.layerwm.temporal_extent_start: pydate = get_parsed_date(self.layerwm.temporal_extent_start) if pydate: start_date = [] start_date.append(pydate) start_date.append(1) dates.append(start_date) if self.layerwm.temporal_extent_start: pydate = get_parsed_date(self.layerwm.temporal_extent_end) if pydate: end_date = [] end_date.append(pydate) end_date.append(1) dates.append(end_date) for layerdate in self.layerdate_set.all().order_by('date'): sdate = layerdate.date if 'TO' not in sdate: pydate = get_parsed_date(sdate) if pydate: date = [] date.append(pydate) date.append(layerdate.type) dates.append(date) return dates def update_thumbnail(self): print 'Generating thumbnail for layer id %s' % self.id if not self.has_valid_bbox(): raise ValueError('Extent for this layer is invalid, cannot generate thumbnail') return None format_error_message = 'This layer does not expose valid formats (png, jpeg) to generate the thumbnail' img = None if self.type == 'OGC:WMS': ows = WebMapService(self.service.url) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'OGC:WMTS': ows = WebMapTileService(self.service.url) ows_layer = ows.contents[self.name] image_format = 'image/png' if image_format not in ows_layer.formats: if 'image/jpeg' in ows_layer.formats: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.gettile( layer=self.name, tilematrixset=ows_layer.tilematrixsets[0], tilematrix='0', row='0', column='0', format=image_format ) elif self.type == 'Hypermap:WorldMap': ows = WebMapService(self.url, username=settings.WM_USERNAME, password=settings.WM_PASSWORD) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'Hypermap:WARPER': ows = WebMapService(self.url) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'ESRI:ArcGIS:MapServer': try: image = None arcserver = ArcMapService(self.service.url) bbox = '%s, %s, %s, %s' % ( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ) image = arcserver.ExportMap( bbox=bbox, layers='show:' + self.name, transparent='true', dpi='96', format='jpg' ) except Exception, e: print e name = re.sub('[^\w\-_\. ]', '_', self.name) thumbnail_file_name = '%s%s.jpg' % ('/tmp/', name) image.save(thumbnail_file_name) img = open(thumbnail_file_name) os.remove(thumbnail_file_name) elif self.type == 'ESRI:ArcGIS:ImageServer': image = None try: arcserver = ArcImageService(self.service.url) bbox = ( str(self.bbox_x0) + ',' + str(self.bbox_y0) + ',' + str(self.bbox_x1) + ',' + str(self.bbox_y1) ) image = arcserver.ExportImage(bbox=bbox) except Exception, e: print e
def check_available(self): """ Check for availability of a service and provide run metrics. """ success = True start_time = datetime.datetime.utcnow() message = '' print 'Checking service id %s' % self.id try: title = None abstract = None keywords = [] wkt_geometry = None srs = '4326' if self.type == 'OGC:WMS': ows = WebMapService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords for c in ows.contents: if ows.contents[c].parent is None: wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84) break if self.type == 'OGC:WMTS': ows = WebMapTileService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords if self.type == 'OSGeo:TMS': ows = TileMapService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords if self.type == 'ESRI:ArcGIS:MapServer': esri = ArcMapService(self.url) extent, srs = get_esri_extent(esri) title = esri.mapName if len(title) == 0: title = get_esri_service_name(self.url) wkt_geometry = bbox2wktpolygon([ extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax'] ]) if self.type == 'ESRI:ArcGIS:ImageServer': esri = ArcImageService(self.url) extent, srs = get_esri_extent(esri) title = esri._json_struct['name'] if len(title) == 0: title = get_esri_service_name(self.url) wkt_geometry = bbox2wktpolygon([ extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax'] ]) if self.type == 'Hypermap:WorldMap': urllib2.urlopen(self.url) title = 'Harvard WorldMap' if self.type == 'Hypermap:WARPER': urllib2.urlopen(self.url) # update title without raising a signal and recursion if title: self.title = title Service.objects.filter(id=self.id).update(title=title) if abstract: self.abstract = abstract Service.objects.filter(id=self.id).update(abstract=abstract) if keywords: for kw in keywords: # FIXME: persist keywords to Django model self.keywords.add(kw) if wkt_geometry: self.wkt_geometry = wkt_geometry Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry) xml = create_metadata_record( identifier=self.id_string, source=self.url, links=[[self.type, self.url]], format=self.type, type='service', title=title, abstract=abstract, keywords=keywords, wkt_geometry=self.wkt_geometry, srs=srs ) anytexts = gen_anytext(title, abstract, keywords) Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service') except Exception as err: print(err) message = str(err) success = False end_time = datetime.datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) check = Check( resource=self, success=success, response_time=response_time, message=message ) check.save() print 'Service checked in %s seconds, status is %s' % (response_time, success)