def process_esri_services(esri_services): services_created = [] for esri_service in esri_services: # for now we process only MapServer and ImageServer if '/MapServer/' in esri_service.url or '/ImageServer/' in esri_service.url: if '/ImageServer/' in esri_service.url: service = create_service_from_endpoint( esri_service.url, 'ESRI_ImageServer', '', esri_service.serviceDescription) if '/MapServer/' in esri_service.url: # we import only MapServer with at least one layer if hasattr(esri_service, 'layers'): service = create_service_from_endpoint( esri_service.url, 'ESRI_MapServer', esri_service.mapName, esri_service.description) services_created.append(service) return services_created
def process_esri_services(esri_services): services_created = [] for esri_service in esri_services: # for now we process only MapServer and ImageServer if '/MapServer/' in esri_service.url or '/ImageServer/' in esri_service.url: if '/ImageServer/' in esri_service.url: service = create_service_from_endpoint( esri_service.url, 'ESRI_ImageServer', '', esri_service.serviceDescription ) if '/MapServer/' in esri_service.url: # we import only MapServer with at least one layer if hasattr(esri_service, 'layers'): service = create_service_from_endpoint( esri_service.url, 'ESRI_MapServer', esri_service.mapName, esri_service.description ) services_created.append(service) return services_created
def update_layers_esri_mapserver(service): """ Update layers for an ESRI REST MapServer. """ esri_service = ArcMapService(service.url) # check if it has a WMS interface if 'supportedExtensions' in esri_service._json_struct: if 'WMSServer' in esri_service._json_struct['supportedExtensions']: # we need to change the url # http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson # http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS wms_url = service.url.replace('/rest/services/', '/services/') if '?f=pjson' in wms_url: wms_url = wms_url.replace('?f=pjson', 'WMSServer?') if '?f=json' in wms_url: wms_url = wms_url.replace('?f=json', 'WMSServer?') print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url # import here as otherwise is circular (TODO refactor) from utils import create_service_from_endpoint create_service_from_endpoint(wms_url, 'OGC_WMS') # now process the REST interface for esri_layer in esri_service.layers: # in some case the json is invalid # esri_layer._json_struct # {u'currentVersion': 10.01, # u'error': # {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}} if 'error' not in esri_layer._json_struct: print 'Updating layer %s' % esri_layer.name layer, created = Layer.objects.get_or_create(name=esri_layer.id, service=service) if layer.active: layer.title = esri_layer.name layer.abstract = esri_service.serviceDescription layer.url = service.url layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id}) # set a default srs srs = 4326 try: layer.bbox_x0 = esri_layer.extent.xmin layer.bbox_y0 = esri_layer.extent.ymin layer.bbox_x1 = esri_layer.extent.xmax layer.bbox_y1 = esri_layer.extent.ymax # crsOptions srs = esri_layer.extent.spatialReference.wkid # this is needed as esri_layer.extent can fail because of custom wkid in json except KeyError: pass try: layer.bbox_x0 = esri_layer._json_struct['extent']['xmin'] layer.bbox_y0 = esri_layer._json_struct['extent']['ymin'] layer.bbox_x1 = esri_layer._json_struct['extent']['xmax'] layer.bbox_y1 = esri_layer._json_struct['extent']['ymax'] wkt_text = esri_layer._json_struct['extent']['spatialReference']['wkt'] if wkt_text: params = {'exact': 'True', 'error': 'True', 'mode': 'wkt', 'terms': wkt_text} req = requests.get('http://prj2epsg.org/search.json', params=params) object = json.loads(req.content) srs = int(object['codes'][0]['code']) except Exception: pass layer.save() srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs) layer.srs.add(srs) # dates add_mined_dates(layer)
def create_services_from_endpoint(url): """ Generate service/services from an endpoint. WMS, WMTS, TMS endpoints correspond to a single service. ESRI, CWS endpoints corrispond to many services. """ num_created = 0 endpoint = get_sanitized_endpoint(url) try: urllib2.urlopen(endpoint, timeout=10) except Exception as e: print 'ERROR! Cannot open this endpoint: %s' % endpoint message = traceback.format_exception(*sys.exc_info()) return False, message detected = False # test if it is WMS, TMS, WMTS or Esri # WMS try: service = WebMapService(endpoint, timeout=10) service_type = 'OGC:WMS' detected = True service = create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract ) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # TMS if not detected: try: service = TileMapService(endpoint, timeout=10) service_type = 'OGC:TMS' detected = True create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract ) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # WMTS if not detected: try: service = WebMapTileService(endpoint, timeout=10) service_type = 'OGC:WMTS' detected = True create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract ) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # Esri # a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services if not detected: try: esri = ArcFolder(endpoint) services = esri.services service_type = 'ESRI' detected = True # root root_services = process_esri_services(services) num_created = num_created + len(root_services) # folders for folder in esri.folders: folder_services = process_esri_services(folder.services) num_created = num_created + len(folder_services) except Exception as e: print str(e) if detected: return True, '%s service/s created' % num_created else: return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def create_services_from_endpoint(url): """ Generate service/services from an endpoint. WMS, WMTS, TMS endpoints correspond to a single service. ESRI, CWS endpoints corrispond to many services. """ num_created = 0 endpoint = get_sanitized_endpoint(url) try: urllib2.urlopen(endpoint, timeout=10) except Exception as e: print 'ERROR! Cannot open this endpoint: %s' % endpoint message = traceback.format_exception(*sys.exc_info()) return False, message detected = False # test if it is WMS, TMS, WMTS or Esri # WMS try: service = WebMapService(endpoint, timeout=10) service_type = 'OGC:WMS' detected = True service = create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # TMS if not detected: try: service = TileMapService(endpoint, timeout=10) service_type = 'OGC:TMS' detected = True create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # WMTS if not detected: try: service = WebMapTileService(endpoint, timeout=10) service_type = 'OGC:WMTS' detected = True create_service_from_endpoint( endpoint, service_type, title=service.identification.title, abstract=service.identification.abstract) if service is not None: num_created = num_created + 1 except Exception as e: print str(e) # Esri # a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services if not detected: try: esri = ArcFolder(endpoint) services = esri.services service_type = 'ESRI' detected = True # root root_services = process_esri_services(services) num_created = num_created + len(root_services) # folders for folder in esri.folders: folder_services = process_esri_services(folder.services) num_created = num_created + len(folder_services) except Exception as e: print str(e) if detected: return True, '%s service/s created' % num_created else: return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def update_layers_esri_mapserver(service): """ Update layers for an ESRI REST MapServer. """ esri_service = ArcMapService(service.url) # check if it has a WMS interface if 'supportedExtensions' in esri_service._json_struct: if 'WMSServer' in esri_service._json_struct['supportedExtensions']: # we need to change the url # http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson # http://cga1.cga.harvard.edu/arcgis/services/ecuador/ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS wms_url = service.url.replace('/rest/services/', '/services/') if '?f=pjson' in wms_url: wms_url = wms_url.replace('?f=pjson', 'WMSServer?') if '?f=json' in wms_url: wms_url = wms_url.replace('?f=json', 'WMSServer?') print 'This ESRI REST endpoint has an WMS interface to process: %s' % wms_url # import here as otherwise is circular (TODO refactor) from utils import create_service_from_endpoint create_service_from_endpoint(wms_url, 'OGC_WMS') # now process the REST interface for esri_layer in esri_service.layers: # in some case the json is invalid # esri_layer._json_struct # {u'currentVersion': 10.01, # u'error': # {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}} if 'error' not in esri_layer._json_struct: print 'Updating layer %s' % esri_layer.name layer, created = Layer.objects.get_or_create(name=esri_layer.id, service=service) if layer.active: layer.title = esri_layer.name layer.abstract = esri_service.serviceDescription layer.url = service.url layer.page_url = reverse('layer_detail', kwargs={'layer_id': layer.id}) # set a default srs srs = 4326 try: layer.bbox_x0 = esri_layer.extent.xmin layer.bbox_y0 = esri_layer.extent.ymin layer.bbox_x1 = esri_layer.extent.xmax layer.bbox_y1 = esri_layer.extent.ymax # crsOptions srs = esri_layer.extent.spatialReference.wkid # this is needed as esri_layer.extent can fail because of custom wkid in json except KeyError: pass try: layer.bbox_x0 = esri_layer._json_struct['extent']['xmin'] layer.bbox_y0 = esri_layer._json_struct['extent']['ymin'] layer.bbox_x1 = esri_layer._json_struct['extent']['xmax'] layer.bbox_y1 = esri_layer._json_struct['extent']['ymax'] wkt_text = esri_layer._json_struct['extent'][ 'spatialReference']['wkt'] if wkt_text: params = { 'exact': 'True', 'error': 'True', 'mode': 'wkt', 'terms': wkt_text } req = requests.get('http://prj2epsg.org/search.json', params=params) object = json.loads(req.content) srs = int(object['codes'][0]['code']) except Exception: pass layer.save() srs, created = SpatialReferenceSystem.objects.get_or_create( code=srs) layer.srs.add(srs) # dates add_mined_dates(layer)