def _register_arcgis_url(url, username, password, owner=None, parent=None): """ Register an ArcGIS REST service URL """ # http://maps1.arcgisonline.com/ArcGIS/rest/services baseurl = _clean_url(url) if re.search("\/MapServer\/*(f=json)*", baseurl): # This is a MapService arcserver = ArcMapService(baseurl) if isinstance(arcserver, ArcMapService) and arcserver.spatialReference.wkid in [ 102100, 3857, 900913 ]: return_json = [ _process_arcgis_service(arcserver, owner=owner, parent=parent) ] else: return_json = [{ 'msg': _("Could not find any layers in a compatible projection.") }] else: # This is a Folder arcserver = ArcFolder(baseurl) return_json = _process_arcgis_folder(arcserver, services=[], owner=owner, parent=parent) return HttpResponse(json.dumps(return_json), mimetype='application/json', status=200)
def __init__(self, url): base.ServiceHandlerBase.__init__(self, url) self.proxy_base = None self.url = url self.parsed_service = ArcMapService(self.url) extent, srs = utils.get_esri_extent(self.parsed_service) try: _sname = utils.get_esri_service_name(self.url) _title_safe = safe(os.path.basename(os.path.normpath(_sname))) _title = _title_safe.replace('_', ' ').strip() except Exception: traceback.print_exc() _title = self.parsed_service.mapName if len(_title) == 0: _title = utils.get_esri_service_name(self.url) # wkt_geometry = utils.bbox2wktpolygon([ # extent['xmin'], # extent['ymin'], # extent['xmax'], # extent['ymax'] # ]) self.indexing_method = INDEXED self.name = slugify(self.url)[:255] self.title = _title
def _register_arcgis_url(url,username, password, owner=None, parent=None): """ Register an ArcGIS REST service URL """ #http://maps1.arcgisonline.com/ArcGIS/rest/services baseurl = _clean_url(url) if re.search("\/MapServer\/*(f=json)*", baseurl): #This is a MapService arcserver = ArcMapService(baseurl) return_json = [_process_arcgis_service(arcserver, owner=owner, parent=parent)] else: #This is a Folder arcserver = ArcFolder(baseurl) return_json = _process_arcgis_folder(arcserver, services=[], owner=owner, parent=parent) return HttpResponse(json.dumps(return_json), mimetype='application/json', status=200)
def check(self): """ Check for availability of a service and provide run metrics. """ from utils import get_esri_service_name success = True start_time = datetime.datetime.utcnow() message = '' print 'Checking service id %s' % self.id try: title = None if self.type == 'OGC_WMS': ows = WebMapService(self.url) title = ows.identification.title if self.type == 'OGC_WMTS': ows = WebMapTileService(self.url) title = ows.identification.title if self.type == 'ESRI_MapServer': esri = ArcMapService(self.url) title = esri.mapName if len(title) == 0: title = get_esri_service_name(self.url) if self.type == 'ESRI_ImageServer': esri = ArcImageService(self.url) title = esri._json_struct['name'] if len(title) == 0: title = get_esri_service_name(self.url) if self.type == 'WM': urllib2.urlopen(self.url) title = 'Harvard WorldMap' if self.type == 'WARPER': urllib2.urlopen(self.url) # update title without raising a signal and recursion if title: Service.objects.filter(id=self.id).update(title=title) except Exception, err: message = str(err) success = False
def _register_arcgis_layers(service, arc=None): """ Register layers from an ArcGIS REST service """ arc = arc or ArcMapService(service.base_url) for layer in arc.layers: valid_name = slugify(layer.name) count = 0 layer_uuid = str(uuid.uuid1()) bbox = [ layer.extent.xmin, layer.extent.ymin, layer.extent.xmax, layer.extent.ymax ] typename = layer.id existing_layer = None try: existing_layer = Layer.objects.get(typename=typename, service=service) except Layer.DoesNotExist: pass llbbox = mercator_to_llbbox(bbox) if existing_layer is None: # Need to check if layer already exists?? saved_layer, created = Layer.objects.get_or_create( typename=typename, service=service, defaults=dict( name=valid_name, store=service.name, # ?? storeType="remoteStore", workspace="remoteWorkspace", title=layer.name, abstract=layer._json_struct['description'] or _("Not provided"), uuid=layer_uuid, owner=None, srid="EPSG:%s" % layer.extent.spatialReference.wkid, bbox_x0=llbbox[0], bbox_x1=llbbox[2], bbox_y0=llbbox[1], bbox_y1=llbbox[3], )) saved_layer.set_default_permissions() saved_layer.save() service_layer, created = ServiceLayer.objects.get_or_create( service=service, typename=layer.id) service_layer.layer = saved_layer service_layer.title = layer.name, service_layer.description = saved_layer.abstract, service_layer.styles = None service_layer.save() create_arcgis_links(saved_layer) count += 1 message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message} return return_dict
def parsed_service(self): return ArcMapService(self.url)
def update_layers_esri_mapserver(service): """ Update layers for an ESRI REST MapServer. """ esri_service = ArcMapService(service.url) # check if it has a WMS interface if 'supportedExtensions' in esri_service._json_struct: if 'WMSServer' in esri_service._json_struct['supportedExtensions']: # we need to change the url # http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson # http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS wms_url = service.url.replace('/rest/services/', '/services/') if '?f=pjson' in wms_url: wms_url = wms_url.replace('?f=pjson', 'WMSServer?') if '?f=json' in wms_url: wms_url = wms_url.replace('?f=json', 'WMSServer?') print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url # import here as otherwise is circular (TODO refactor) from utils import create_service_from_endpoint create_service_from_endpoint(wms_url, 'OGC_WMS') # now process the REST interface for esri_layer in esri_service.layers: # in some case the json is invalid # esri_layer._json_struct # {u'currentVersion': 10.01, # u'error': # {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}} if 'error' not in esri_layer._json_struct: print 'Updating layer %s' % esri_layer.name layer, created = Layer.objects.get_or_create(name=esri_layer.id, service=service) if layer.active: layer.title = esri_layer.name layer.abstract = esri_service.serviceDescription layer.url = service.url layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id}) # set a default srs srs = 4326 try: layer.bbox_x0 = esri_layer.extent.xmin layer.bbox_y0 = esri_layer.extent.ymin layer.bbox_x1 = esri_layer.extent.xmax layer.bbox_y1 = esri_layer.extent.ymax # crsOptions srs = esri_layer.extent.spatialReference.wkid # this is needed as esri_layer.extent can fail because of custom wkid in json except KeyError: pass try: layer.bbox_x0 = esri_layer._json_struct['extent']['xmin'] layer.bbox_y0 = esri_layer._json_struct['extent']['ymin'] layer.bbox_x1 = esri_layer._json_struct['extent']['xmax'] layer.bbox_y1 = esri_layer._json_struct['extent']['ymax'] wkt_text = esri_layer._json_struct['extent'][ 'spatialReference']['wkt'] if wkt_text: params = { 'exact': 'True', 'error': 'True', 'mode': 'wkt', 'terms': wkt_text } req = requests.get('http://prj2epsg.org/search.json', params=params) object = json.loads(req.content) srs = int(object['codes'][0]['code']) except Exception: pass layer.save() srs, created = SpatialReferenceSystem.objects.get_or_create( code=srs) layer.srs.add(srs) # dates add_mined_dates(layer)
def update_thumbnail(self): print 'Generating thumbnail for layer id %s' % self.id if not self.has_valid_bbox(): raise ValueError('Extent for this layer is invalid, cannot generate thumbnail') return None format_error_message = 'This layer does not expose valid formats (png, jpeg) to generate the thumbnail' img = None if self.type == 'OGC:WMS': ows = WebMapService(self.service.url) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'OGC:WMTS': ows = WebMapTileService(self.service.url) ows_layer = ows.contents[self.name] image_format = 'image/png' if image_format not in ows_layer.formats: if 'image/jpeg' in ows_layer.formats: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.gettile( layer=self.name, tilematrixset=ows_layer.tilematrixsets[0], tilematrix='0', row='0', column='0', format=image_format ) elif self.type == 'Hypermap:WorldMap': ows = WebMapService(self.url, username=settings.WM_USERNAME, password=settings.WM_PASSWORD) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'Hypermap:WARPER': ows = WebMapService(self.url) op_getmap = ows.getOperationByName('GetMap') image_format = 'image/png' if image_format not in op_getmap.formatOptions: if 'image/jpeg' in op_getmap.formatOptions: image_format = 'image/jpeg' else: raise NotImplementedError(format_error_message) img = ows.getmap( layers=[self.name], srs='EPSG:4326', bbox=( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ), size=(50, 50), format=image_format, transparent=True ) if 'ogc.se_xml' in img.info()['Content-Type']: raise ValueError(img.read()) img = None elif self.type == 'ESRI:ArcGIS:MapServer': try: image = None arcserver = ArcMapService(self.service.url) bbox = '%s, %s, %s, %s' % ( float(self.bbox_x0), float(self.bbox_y0), float(self.bbox_x1), float(self.bbox_y1) ) image = arcserver.ExportMap( bbox=bbox, layers='show:' + self.name, transparent='true', dpi='96', format='jpg' ) except Exception, e: print e name = re.sub('[^\w\-_\. ]', '_', self.name) thumbnail_file_name = '%s%s.jpg' % ('/tmp/', name) image.save(thumbnail_file_name) img = open(thumbnail_file_name) os.remove(thumbnail_file_name)
def check_available(self): """ Check for availability of a service and provide run metrics. """ success = True start_time = datetime.datetime.utcnow() message = '' print 'Checking service id %s' % self.id try: title = None abstract = None keywords = [] wkt_geometry = None srs = '4326' if self.type == 'OGC:WMS': ows = WebMapService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords for c in ows.contents: if ows.contents[c].parent is None: wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84) break if self.type == 'OGC:WMTS': ows = WebMapTileService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords if self.type == 'OSGeo:TMS': ows = TileMapService(self.url) title = ows.identification.title abstract = ows.identification.abstract keywords = ows.identification.keywords if self.type == 'ESRI:ArcGIS:MapServer': esri = ArcMapService(self.url) extent, srs = get_esri_extent(esri) title = esri.mapName if len(title) == 0: title = get_esri_service_name(self.url) wkt_geometry = bbox2wktpolygon([ extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax'] ]) if self.type == 'ESRI:ArcGIS:ImageServer': esri = ArcImageService(self.url) extent, srs = get_esri_extent(esri) title = esri._json_struct['name'] if len(title) == 0: title = get_esri_service_name(self.url) wkt_geometry = bbox2wktpolygon([ extent['xmin'], extent['ymin'], extent['xmax'], extent['ymax'] ]) if self.type == 'Hypermap:WorldMap': urllib2.urlopen(self.url) title = 'Harvard WorldMap' if self.type == 'Hypermap:WARPER': urllib2.urlopen(self.url) # update title without raising a signal and recursion if title: self.title = title Service.objects.filter(id=self.id).update(title=title) if abstract: self.abstract = abstract Service.objects.filter(id=self.id).update(abstract=abstract) if keywords: for kw in keywords: # FIXME: persist keywords to Django model self.keywords.add(kw) if wkt_geometry: self.wkt_geometry = wkt_geometry Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry) xml = create_metadata_record( identifier=self.id_string, source=self.url, links=[[self.type, self.url]], format=self.type, type='service', title=title, abstract=abstract, keywords=keywords, wkt_geometry=self.wkt_geometry, srs=srs ) anytexts = gen_anytext(title, abstract, keywords) Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service') except Exception as err: print(err) message = str(err) success = False end_time = datetime.datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) check = Check( resource=self, success=success, response_time=response_time, message=message ) check.save() print 'Service checked in %s seconds, status is %s' % (response_time, success)