def test_tms():
# Find out what a TMS has to offer. Service metadata:
from owslib.tms import TileMapService
tms = TileMapService(SERVICE_URL)
# Fetch a tile (using some defaults):
tile = tms.gettile(7, 7, 4, title='brtachtergrondkaart', srs='EPSG:28992', mimetype='image/png')
out = open(scratch_file('brtachtergrondkaart.png'), 'wb')
out.write(tile.read())
out.close()
def get_metadata(self, resource, version='1.0.0'):
"""
Get metadata, specific per Resource type.
:param resource:
:param version:
:return: Metadata object
"""
return TileMapService(resource.url, version=version)
def before_request(self):
""" Before request to service, overridden from base class"""
# Get capabilities doc to get all layers
try:
self.tms = TileMapService(self._resource.url, version='1.0.0')
self.layers = self.tms.contents
except Exception as err:
self.result.set(False, str(err))
def _verify_service_type(base_url, service_type=None):
"""
Try to determine service type by process of elimination
"""
exceptions = []
if service_type in ['WMS', 'OWS', None]:
try:
service = WebMapService(base_url)
except:
pass
else:
return ['WMS', service]
if service_type in ['WFS', 'OWS', None]:
try:
servicewfs = WebFeatureService(base_url)
except:
pass
else:
return ['WFS', servicewfs]
if service_type in ['TMS', None]:
try:
service = TileMapService(base_url)
except:
pass
else:
return ['TMS', service]
if service_type in ['REST', None]:
try:
service = ArcFolder(base_url)
except:
pass
else:
service.services
return ['REST', service]
if service_type in ['CSW', None]:
try:
service = CatalogueServiceWeb(base_url)
except:
raise
else:
return ['CSW', service]
if service_type in ['OGP', None]:
#Just use a specific OGP URL for now
if base_url == settings.OGP_URL:
return ["OGP", None]
return [None, None]
def expand_params(self, resource):
# Use WMS Capabilities doc to get metadata for
# PARAM_DEFS ranges/defaults
try:
tms = TileMapService(resource.url, version='1.0.0')
layers = tms.contents
self.layer_count = len(layers)
# Determine Layers and Extensions
layer_list = [layer_name.split('1.0.0/')[-1]
for layer_name in layers]
self.PARAM_DEFS['layer']['range'] = layer_list
# Make a set of all extensions
extensions = set([layer.extension
for layer_name, layer in layers.items()])
self.PARAM_DEFS['extension']['range'] = list(extensions)
except Exception as err:
raise err
def create_services_from_endpoint(url, catalog, greedy_opt=True):
"""
Generate service/services from an endpoint.
WMS, WMTS, TMS endpoints correspond to a single service.
ESRI, CSW endpoints corrispond to many services.
:return: imported, message
"""
# this variable will collect any exception message during the routine.
# will be used in the last step to send a message if "detected" var is False.
messages = []
num_created = 0
endpoint = get_sanitized_endpoint(url)
try:
urllib.request.urlopen(endpoint, timeout=10)
except Exception as e:
message = traceback.format_exception(*sys.exc_info())
LOGGER.error('Cannot open this endpoint: %s' % endpoint)
LOGGER.error('ERROR MESSAGE: %s' % message)
LOGGER.error(e, exc_info=True)
return False, message
detected = False
# handle specific service types for some domains (WorldMap, Wrapper...)
parsed_uri = urlparse(endpoint)
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
if domain == 'http://worldmap.harvard.edu/':
service_type = 'Hypermap:WorldMap'
title = 'Harvard WorldMap'
abstract = 'Harvard WorldMap'
endpoint = domain
detected = True
if domain in [
'http://maps.nypl.org/',
'http://mapwarper.net/',
'http://warp.worldmap.harvard.edu/',
]:
service_type = 'Hypermap:WARPER'
title = 'Warper at %s' % domain
abstract = 'Warper at %s' % domain
detected = True
# test if it is CSW, WMS, TMS, WMTS or Esri
# CSW
try:
csw = CatalogueServiceWeb(endpoint)
service_type = 'OGC:CSW'
service_links = {}
detected = True
typenames = 'csw:Record'
outputschema = 'http://www.opengis.net/cat/csw/2.0.2'
if 'csw_harvest_pagesize' in settings.REGISTRY_PYCSW['manager']:
pagesize = int(
settings.REGISTRY_PYCSW['manager']['csw_harvest_pagesize'])
else:
pagesize = 10
LOGGER.debug('Harvesting CSW %s' % endpoint)
# now get all records
# get total number of records to loop against
try:
csw.getrecords2(typenames=typenames,
resulttype='hits',
outputschema=outputschema)
matches = csw.results['matches']
except: # this is a CSW, but server rejects query
raise RuntimeError(csw.response)
if pagesize > matches:
pagesize = matches
LOGGER.info('Harvesting %d CSW records' % matches)
# loop over all catalogue records incrementally
for r in range(1, matches + 1, pagesize):
LOGGER.info('Parsing %s from %s' % (r, matches))
try:
csw.getrecords2(typenames=typenames,
startposition=r,
maxrecords=pagesize,
outputschema=outputschema,
esn='full')
except Exception as err: # this is a CSW, but server rejects query
raise RuntimeError(csw.response)
for k, v in list(csw.records.items()):
# try to parse metadata
try:
LOGGER.info('Looking for service links')
LOGGER.debug(
'Looking for service links via dct:references')
if v.references:
for ref in v.references:
scheme = None
if ref['scheme'] in [
st[0] for st in SERVICE_TYPES
]:
if ref['url'] not in service_links:
scheme = ref['scheme']
service_links[ref['url']] = scheme
else: # loose detection
scheme = detect_metadata_url_scheme(ref['url'])
if scheme is not None:
if ref['url'] not in service_links:
service_links[ref['url']] = scheme
if scheme is None:
continue
try:
service = create_service_from_endpoint(
ref['url'], scheme, catalog=catalog)
if service is not None:
num_created = num_created + 1
LOGGER.info(
'Found %s services on endpoint' %
num_created)
except Exception as e:
LOGGER.error(
'Could not create service for %s : %s' %
(scheme, ref['url']))
LOGGER.error(e, exc_info=True)
LOGGER.debug(
'Looking for service links via the GeoNetwork-ish dc:URI'
)
if v.uris:
for u in v.uris: # loose detection
scheme = detect_metadata_url_scheme(u['url'])
if scheme is not None:
if u['url'] not in service_links:
service_links[u['url']] = scheme
else:
continue
try:
service = create_service_from_endpoint(
u['url'], scheme, catalog=catalog)
if service is not None:
num_created = num_created + 1
LOGGER.info(
'Found %s services on endpoint' %
num_created)
except Exception as e:
LOGGER.error(
'Could not create service for %s : %s' %
(scheme, u['url']))
LOGGER.error(e, exc_info=True)
except Exception as err: # parsing failed for some reason
LOGGER.warning('Metadata parsing failed %s', err)
LOGGER.error(err, exc_info=True)
except XMLSyntaxError as e:
# This is not XML, so likely not a CSW. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# WMS
if not detected:
try:
service = get_wms_version_negotiate(endpoint, timeout=10)
service_type = 'OGC:WMS'
title = service.identification.title,
abstract = service.identification.abstract
detected = True
except XMLSyntaxError as e:
# This is not XML, so likely not a WMS. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OSGeo:TMS'
title = service.identification.title,
abstract = service.identification.abstract
detected = True
except XMLSyntaxError as e:
# This is not XML, so likely not a TsMS. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# WMTS
if not detected:
try:
# @tomkralidis timeout is not implemented for WebMapTileService?
service = WebMapTileService(endpoint)
service_type = 'OGC:WMTS'
title = service.identification.title,
abstract = service.identification.abstract
detected = True
except XMLSyntaxError as e:
# This is not XML, so likely not a WMTS. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# if detected, let's create the service
if detected and service_type != 'OGC:CSW':
try:
service = create_service_from_endpoint(endpoint,
service_type,
title,
abstract=abstract,
catalog=catalog)
if service is not None:
num_created = num_created + 1
except XMLSyntaxError as e:
# This is not XML, so likely not a OGC:CSW. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# Esri
# a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services
# we can safely assume the following condition (at least it is true for 1170 services)
# we need to test this as arcrest.Folder can freeze with not esri url such as this one:
# http://hh.worldmap.harvard.edu/admin/aggregator/service/?q=%2Frest%2Fservices
if '/rest/services' in endpoint:
if not detected:
try:
esri = arcrest.Folder(endpoint)
service_type = 'ESRI'
detected = True
service_to_process, folder_to_process = esri.services, esri.folders
if not greedy_opt:
folder_to_process = []
sections = service_url_parse(url)
service_to_process = get_single_service(esri, sections)
processed_services = process_esri_services(
service_to_process, catalog)
num_created = num_created + len(processed_services)
for folder in folder_to_process:
folder_services = process_esri_services(
folder.services, catalog)
num_created = num_created + len(folder_services)
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
if detected:
return True, '%s service/s created' % num_created
else:
m = '|'.join(messages)
return False, 'ERROR! Could not detect service type for ' \
'endpoint %s or already existing. messages=(%s)' % (endpoint, m)
def sniff_test_resource(config, resource_type, url):
"""tests a service and provides run metrics"""
if resource_type not in RESOURCE_TYPES.keys():
msg = gettext('Invalid resource type')
msg2 = '%s: %s' % (msg, resource_type)
LOGGER.error(msg2)
raise RuntimeError(msg2)
title = None
start_time = datetime.datetime.utcnow()
message = None
try:
if resource_type == 'OGC:WMS':
ows = WebMapService(url)
elif resource_type == 'OGC:WMTS':
ows = WebMapTileService(url)
elif resource_type == 'OSGeo:TMS':
ows = TileMapService(url)
elif resource_type == 'OGC:WFS':
ows = WebFeatureService(url)
elif resource_type == 'OGC:WCS':
ows = WebCoverageService(url, version='1.0.0')
elif resource_type == 'OGC:WPS':
ows = WebProcessingService(url)
elif resource_type == 'OGC:CSW':
ows = CatalogueServiceWeb(url)
elif resource_type == 'OGC:SOS':
ows = SensorObservationService(url)
elif resource_type == 'OGC:STA':
ows = urlopen(url)
elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
ows = urlopen(url)
if resource_type == 'WWW:LINK':
content_type = ows.info().getheader('Content-Type')
# Check content if the response is not an image
if 'image/' not in content_type:
content = ows.read()
import re
try:
title_re = re.compile("<title>(.+?)</title>")
title = title_re.search(content).group(1)
except:
title = url
# Optional check for any OGC-Exceptions in Response
if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
exception_text = None
try:
except_re = re.compile(
"ServiceException>|ExceptionReport>")
exception_text = except_re.search(content).group(0)
except:
# No Exception in Response text
pass
if exception_text:
# Found OGC-Exception in Response text
raise Exception("Exception in response: %s" %
exception_text)
del content
elif resource_type == 'urn:geoss:waf':
title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
elif resource_type == 'FTP':
ows = urlopen(url)
title = urlparse(url).hostname
success = True
if resource_type.startswith(('OGC:', 'OSGeo')):
if resource_type == 'OGC:STA':
title = 'OGC STA'
else:
title = ows.identification.title
if title is None:
title = '%s %s %s' % (resource_type, gettext('for'), url)
except Exception as err:
msg = str(err)
LOGGER.exception(msg)
message = msg
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
return [title, success, response_time, message, start_time]
service = get_wms_version_negotiate(endpoint, timeout=10)
service_type = 'OGC:WMS'
title = service.identification.title,
abstract = service.identification.abstract
detected = True
except XMLSyntaxError as e:
# This is not XML, so likely not a WMS. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OSGeo:TMS'
title = service.identification.title,
abstract = service.identification.abstract
detected = True
except XMLSyntaxError as e:
# This is not XML, so likely not a TsMS. Moving on.
pass
except Exception as e:
LOGGER.error(e, exc_info=True)
messages.append(str(e))
# WMTS
if not detected:
try:
# @tomkralidis timeout is not implemented for WebMapTileService?
def create_services_from_endpoint(url):
"""
Generate service/services from an endpoint.
WMS, WMTS, TMS endpoints correspond to a single service.
ESRI, CWS endpoints corrispond to many services.
"""
num_created = 0
endpoint = get_sanitized_endpoint(url)
try:
urllib2.urlopen(endpoint, timeout=10)
except Exception as e:
print 'ERROR! Cannot open this endpoint: %s' % endpoint
message = traceback.format_exception(*sys.exc_info())
return False, message
detected = False
# test if it is WMS, TMS, WMTS or Esri
# WMS
try:
service = WebMapService(endpoint, timeout=10)
service_type = 'OGC:WMS'
detected = True
service = create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OGC:TMS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# WMTS
if not detected:
try:
service = WebMapTileService(endpoint, timeout=10)
service_type = 'OGC:WMTS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# Esri
# a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services
if not detected:
try:
esri = ArcFolder(endpoint)
services = esri.services
service_type = 'ESRI'
detected = True
# root
root_services = process_esri_services(services)
num_created = num_created + len(root_services)
# folders
for folder in esri.folders:
folder_services = process_esri_services(folder.services)
num_created = num_created + len(folder_services)
except Exception as e:
print str(e)
if detected:
return True, '%s service/s created' % num_created
else:
return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def create_services_from_endpoint(url):
"""
Generate service/services from an endpoint.
WMS, WMTS, TMS endpoints correspond to a single service.
ESRI, CWS endpoints corrispond to many services.
"""
num_created = 0
endpoint = get_sanitized_endpoint(url)
try:
urllib2.urlopen(endpoint, timeout=10)
except Exception as e:
print 'ERROR! Cannot open this endpoint: %s' % endpoint
message = traceback.format_exception(*sys.exc_info())
return False, message
detected = False
# test if it is WMS, TMS, WMTS or Esri
# WMS
try:
service = WebMapService(endpoint, timeout=10)
service_type = 'OGC:WMS'
detected = True
service = create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# TMS
if not detected:
try:
service = TileMapService(endpoint, timeout=10)
service_type = 'OSGeo:TMS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# WMTS
if not detected:
try:
service = WebMapTileService(endpoint, timeout=10)
service_type = 'OGC:WMTS'
detected = True
create_service_from_endpoint(
endpoint,
service_type,
title=service.identification.title,
abstract=service.identification.abstract)
if service is not None:
num_created = num_created + 1
except Exception as e:
print str(e)
# Esri
# a good sample is here: https://gis.ngdc.noaa.gov/arcgis/rest/services
# we can safely assume the following condition (at least it is true for 1170 services)
# we need to test this as ArcFolder can freeze with not esri url such as this one:
# http://hh.worldmap.harvard.edu/admin/aggregator/service/?q=%2Frest%2Fservices
if '/rest/services' in endpoint:
if not detected:
try:
esri = ArcFolder(endpoint)
services = esri.services
service_type = 'ESRI'
detected = True
# root
root_services = process_esri_services(services)
num_created = num_created + len(root_services)
# folders
for folder in esri.folders:
folder_services = process_esri_services(folder.services)
num_created = num_created + len(folder_services)
except Exception as e:
print str(e)
if detected:
return True, '%s service/s created' % num_created
else:
return False, 'ERROR! Could not detect service type for endpoint %s or already existing' % endpoint
def check_available(self):
"""
Check for availability of a service and provide run metrics.
"""
success = True
start_time = datetime.datetime.utcnow()
message = ''
print 'Checking service id %s' % self.id
try:
title = None
abstract = None
keywords = []
wkt_geometry = None
srs = '4326'
if self.type == 'OGC:WMS':
ows = WebMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
for c in ows.contents:
if ows.contents[c].parent is None:
wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84)
break
if self.type == 'OGC:WMTS':
ows = WebMapTileService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'OSGeo:TMS':
ows = TileMapService(self.url)
title = ows.identification.title
abstract = ows.identification.abstract
keywords = ows.identification.keywords
if self.type == 'ESRI:ArcGIS:MapServer':
esri = ArcMapService(self.url)
extent, srs = get_esri_extent(esri)
title = esri.mapName
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'ESRI:ArcGIS:ImageServer':
esri = ArcImageService(self.url)
extent, srs = get_esri_extent(esri)
title = esri._json_struct['name']
if len(title) == 0:
title = get_esri_service_name(self.url)
wkt_geometry = bbox2wktpolygon([
extent['xmin'],
extent['ymin'],
extent['xmax'],
extent['ymax']
])
if self.type == 'Hypermap:WorldMap':
urllib2.urlopen(self.url)
title = 'Harvard WorldMap'
if self.type == 'Hypermap:WARPER':
urllib2.urlopen(self.url)
# update title without raising a signal and recursion
if title:
self.title = title
Service.objects.filter(id=self.id).update(title=title)
if abstract:
self.abstract = abstract
Service.objects.filter(id=self.id).update(abstract=abstract)
if keywords:
for kw in keywords:
# FIXME: persist keywords to Django model
self.keywords.add(kw)
if wkt_geometry:
self.wkt_geometry = wkt_geometry
Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry)
xml = create_metadata_record(
identifier=self.id_string,
source=self.url,
links=[[self.type, self.url]],
format=self.type,
type='service',
title=title,
abstract=abstract,
keywords=keywords,
wkt_geometry=self.wkt_geometry,
srs=srs
)
anytexts = gen_anytext(title, abstract, keywords)
Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service')
except Exception as err:
print(err)
message = str(err)
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
check = Check(
resource=self,
success=success,
response_time=response_time,
message=message
)
check.save()
print 'Service checked in %s seconds, status is %s' % (response_time, success)