def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
def href(self):
query = {'name': self.name}
path_parts = ['layergroups']
if self.workspace is not None:
workspace_name = getattr(self.workspace, 'name', self.workspace)
path_parts = ["workspaces", workspace_name] + path_parts
return build_url(self.catalog.service_url, path_parts, query)
def mosaic_coverage_schema(self, coverage, store, workspace):
'''Returns the schema of a coverage in a coverage store'''
params = dict()
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"coveragestores",
store,
"coverages",
coverage,
"index.json"
],
params
)
# GET /workspaces/<ws>/coveragestores/<name>/coverages/<coverage>/index.json
headers = {
"Content-type": "application/json",
"Accept": "application/json"
}
resp = self.http_request(url, headers=headers)
if resp.status_code != 200:
FailedRequestError('Failed to get mosaic schema {} : {}, {}'.format(store, resp.status_code, resp.text))
self._cache.clear()
return resp.json()
def href(self):
path = [
"workspaces",
self.workspace.name,
"datastores"
]
query = dict(name=self.name)
return build_url(self.catalog.service_url, path, query)
def _build_href(self, extension, create=False):
path_parts = ["styles"]
query = {}
if not create:
path_parts.append(self.name + extension)
else:
query['name'] = self.name
if self.workspace is not None:
path_parts = ["workspaces", getattr(self.workspace, 'name', self.workspace)] + path_parts
return build_url(self.catalog.service_url, path_parts, query)
def href(self):
url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace.name,
"coveragestores",
"{}.xml".format(self.name)
]
)
return url
def href(self):
url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace,
"coveragestores"
],
dict(name=self.name)
)
return url
def href(self):
url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace.name,
"datastores",
self.name + ".xml"
]
)
return url
def resource_url(self):
url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace.name,
"datastores",
self.name,
"featuretypes.xml"
]
)
return url
def add_granule(self, data, store, workspace=None):
'''Harvest/add a granule into an existing imagemosaic'''
ext = os.path.splitext(data)[-1]
if ext == ".zip":
type = "file.imagemosaic"
upload_data = open(data, 'rb')
headers = {
"Content-type": "application/zip",
"Accept": "application/xml"
}
else:
type = "external.imagemosaic"
upload_data = data if data.startswith("file:") else "file:{data}".format(data=data)
headers = {
"Content-type": "text/plain",
"Accept": "application/xml"
}
params = dict()
workspace_name = workspace
if isinstance(store, basestring):
store_name = store
else:
store_name = store.name
workspace_name = store.workspace.name
if workspace_name is None:
raise ValueError("Must specify workspace")
url = build_url(
self.service_url,
[
"workspaces",
workspace_name,
"coveragestores",
store_name,
type
],
params
)
try:
resp = self.http_request(url, method='post', data=upload_data, headers=headers)
if resp.status_code != 202:
FailedRequestError('Failed to add granule to mosaic {} : {}, {}'.format(store, resp.status_code, resp.text))
self._cache.clear()
finally:
if hasattr(upload_data, "close"):
upload_data.close()
# maybe return a list of all granules?
return None
def publish_featuretype(self, name, store, native_crs, srs=None, jdbc_virtual_table=None, native_name=None):
'''Publish a featuretype from data in an existing store'''
# @todo native_srs doesn't seem to get detected, even when in the DB
# metadata (at least for postgis in geometry_columns) and then there
# will be a misconfigured layer
if native_crs is None:
raise ValueError("must specify native_crs")
if jdbc_virtual_table is None and native_name is None:
raise ValueError("must specify native_name")
srs = srs or native_crs
feature_type = FeatureType(self, store.workspace, store, name)
# because name is the in FeatureType base class, work around that
# and hack in these others that don't have xml properties
feature_type.dirty['name'] = name
feature_type.dirty['srs'] = srs
feature_type.dirty['nativeCRS'] = native_crs
feature_type.enabled = True
feature_type.advertised = True
feature_type.title = name
if native_name is not None:
feature_type.native_name = native_name
headers = {
"Content-type": "application/xml",
"Accept": "application/xml"
}
resource_url = store.resource_url
if jdbc_virtual_table is not None:
feature_type.metadata = ({'JDBC_VIRTUAL_TABLE': jdbc_virtual_table})
params = dict()
resource_url = build_url(
self.service_url,
[
"workspaces",
store.workspace.name,
"datastores", store.name,
"featuretypes.xml"
],
params
)
resp = self.http_request(resource_url, method='post', data=feature_type.message(), headers=headers)
if resp.status_code not in (200, 201, 202):
FailedRequestError('Failed to publish feature type {} : {}, {}'.format(name, resp.status_code, resp.text))
self._cache.clear()
feature_type.fetch()
return feature_type
def href(self):
url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace.name,
self.url_part_stores,
self.store.name,
self.url_part_types,
self.name + ".xml"
]
)
return self._href or url
def add_data_to_store(self, store, name, data, workspace=None, overwrite = False, charset = None):
if isinstance(store, basestring):
store = self.get_stores(names=store, workspaces=workspace)[0]
if workspace is not None:
workspace = _name(workspace)
assert store.workspace.name == workspace, "Specified store (%s) is not in specified workspace (%s)!" % (store, workspace)
else:
workspace = store.workspace.name
store = store.name
if isinstance(data, dict):
bundle = prepare_upload_bundle(name, data)
else:
bundle = data
params = dict()
if overwrite:
params["update"] = "overwrite"
if charset is not None:
params["charset"] = charset
params["filename"] = "{}.zip".format(name)
params["target"] = "shp"
# params["configure"] = "all"
headers = {'Content-Type': 'application/zip', 'Accept': 'application/xml'}
upload_url = build_url(
self.service_url,
[
"workspaces",
workspace,
"datastores",
store,
"file.shp"
],
params
)
try:
with open(bundle, "rb") as f:
data = f.read()
resp = self.http_request(upload_url, method='put', data=data, headers=headers)
if resp.status_code != 201:
FailedRequestError('Failed to add data to store {} : {}, {}'.format(store, resp.status_code, resp.text))
self._cache.clear()
finally:
# os.unlink(bundle)
pass
def list_granules(self, coverage, store, workspace=None, filter=None, limit=None, offset=None):
'''List granules of an imagemosaic'''
params = dict()
if filter is not None:
params['filter'] = filter
if limit is not None:
params['limit'] = limit
if offset is not None:
params['offset'] = offset
workspace_name = workspace
if isinstance(store, basestring):
store_name = store
else:
store_name = store.name
workspace_name = store.workspace.name
if workspace_name is None:
raise ValueError("Must specify workspace")
url = build_url(
self.service_url,
[
"workspaces",
workspace_name,
"coveragestores",
store_name,
"coverages",
coverage,
"index/granules.json"
],
params
)
# GET /workspaces/<ws>/coveragestores/<name>/coverages/<coverage>/index/granules.json
headers = {
"Content-type": "application/json",
"Accept": "application/json"
}
resp = self.http_request(url, headers=headers)
if resp.status_code != 200:
FailedRequestError('Failed to list granules in mosaic {} : {}, {}'.format(store, resp.status_code, resp.text))
self._cache.clear()
return resp.json()
def create_featurestore(self, name, data, workspace=None, overwrite=False, charset=None):
if workspace is None:
workspace = self.get_default_workspace()
workspace = _name(workspace)
if not overwrite:
stores = self.get_stores(names=name, workspaces=workspace)
if len(stores) > 0:
msg = "There is already a store named {} in workspace {}".format(name, workspace)
raise ConflictingDataError(msg)
params = dict()
if charset is not None:
params['charset'] = charset
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"datastores",
name,
"file.shp"
],
params
)
# PUT /workspaces/<ws>/datastores/<ds>/file.shp
headers = {
"Content-type": "application/zip",
"Accept": "application/xml"
}
if isinstance(data, dict):
logger.debug('Data is NOT a zipfile')
archive = prepare_upload_bundle(name, data)
else:
logger.debug('Data is a zipfile')
archive = data
file_obj = open(archive, 'rb')
try:
resp = self.http_request(url, method='put', data=file_obj, headers=headers)
if resp.status_code != 201:
FailedRequestError('Failed to create FeatureStore {} : {}, {}'.format(name, resp.status_code, resp.text))
self._cache.clear()
finally:
file_obj.close()
os.unlink(archive)
def delete_granule(self, coverage, store, granule_id, workspace=None):
'''Deletes a granule of an existing imagemosaic'''
params = dict()
workspace_name = workspace
if isinstance(store, basestring):
store_name = store
else:
store_name = store.name
workspace_name = store.workspace.name
if workspace_name is None:
raise ValueError("Must specify workspace")
url = build_url(
self.service_url,
[
"workspaces",
workspace_name,
"coveragestores",
store_name,
"coverages",
coverage,
"index/granules",
granule_id,
".json"
],
params
)
# DELETE /workspaces/<ws>/coveragestores/<name>/coverages/<coverage>/index/granules/<granule_id>.json
headers = {
"Content-type": "application/json",
"Accept": "application/json"
}
resp = self.http_request(url, method='delete', headers=headers)
if resp.status_code != 200:
FailedRequestError('Failed to delete granule from mosaic {} : {}, {}'.format(store, resp.status_code, resp.text))
self._cache.clear()
# maybe return a list of all granules?
return None
def settings(self):
settings = {}
settings_url = build_url(self.catalog.service_url, ['settings.xml'])
resp = self.catalog.http_request(settings_url, 'GET')
if resp.status_code != 200:
raise Exception('Settings listing failed: ' + resp.text)
dom = XML(resp.text)
sections = ['settings', 'jai','coverageAccess']
for section in sections:
params = []
node = dom.find(section)
if node is not None: #it will be none if the catalog does not support this operation
for entry in node:
if len(entry) == 0:
params.append((entry.tag, entry.text))
else:
for subentry in entry:
params.append((entry.tag + '/' + subentry.tag, subentry.text))
settings[section] = params
return settings
def update(self, settings):
root = ET.Element('global')
for section in settings:
params = settings[section]
element = ET.SubElement(root, section)
for name, value in params:
if '/' in name:
name, subname = name.split('/')
subelement = element.find(name)
if subelement is None:
subelement = ET.SubElement(element, name)
subsubelement = ET.SubElement(subelement, subname)
subsubelement.text = str(value)
else:
subelement = ET.SubElement(element, name)
subelement.text = str(value)
xml = ET.tostring(root)
settings_url = build_url(self.catalog.service_url, ['settings.xml'])
headers = {'Content-type': 'text/xml'}
resp = self.catalog.http_request(settings_url, xml, 'PUT', headers = headers)
if resp.status_code != 200:
raise Exception('Settings update failed: ' + resp.text)
def get_resources(self, name=None):
res_url = build_url(
self.catalog.service_url,
[
"workspaces",
self.workspace.name,
"coveragestores",
self.name,
"coverages.xml"
]
)
xml = self.catalog.get_xml(res_url)
def cov_from_node(node):
return coverage_from_index(self.catalog, self.workspace, self, node)
# if name passed, return only one Coverage, otherwise return all Coverages in store:
if name is not None:
for node in xml.findall("coverage"):
if node.findtext("name") == name:
return cov_from_node(node)
return None
return [cov_from_node(node) for node in xml.findall("coverage")]
def put_layer(name, data, source_type):
workspace = "adalitix"
params = {
"configure": "first",
"coverageName": name
} if source_type == "tiff" else dict()
url = build_url(
adalitix_catalog.service_url,
[
"workspaces",
workspace,
"coveragestores" if source_type == "tiff" else "datastores",
name,
# "file" is the upload method not the file name
"file.geotiff" if source_type == "tiff" else "file.shp"
],
params)
headers = {
"Content-type":
"image/tiff" if source_type == "tiff" else "application/zip"
}
resp = adalitix_catalog.http_request(url,
method='put',
data=data,
headers=headers)
data.close()
if resp.status_code != 201:
raise Exception(resp.text)
# rename shapefile layers so that their name matches the datastore name
if source_type != "tiff":
# load the feature type description from geoserver
url = build_url(
adalitix_catalog.service_url,
["workspaces", workspace, "datastores", name, "featuretypes.json"],
dict())
resp = adalitix_catalog.http_request(
url, method='get', headers={"Accept": "application/json"})
if resp.status_code != 200:
raise Exception(resp.text)
data = json.loads(resp.text)
layer_name = data["featureTypes"]["featureType"][0]["name"]
# update the feature type to change the name
url = build_url(adalitix_catalog.service_url, [
"workspaces", workspace, "datastores", name, "featuretypes",
layer_name
], dict())
payload = json.dumps({"featureType": {"name": name}})
resp = adalitix_catalog.http_request(url,
method='put',
data=payload,
headers={
"Content-type":
"application/json",
"Accept": "application/json"
})
if resp.status_code != 200:
raise Exception(resp.text)
return "Successful upload"
def contact_url(self):
return build_url(self.catalog.service_url,
["settings", "contact.json"])
def href(self):
url = build_url(self.catalog.service_url, [
"workspaces", self.workspace.name, "datastores", self.name + ".xml"
])
return url
def service_url(self, service):
assert service in ('wms', 'wfs', 'wcs'), "Unknow service type"
return build_url(self.catalog.service_url,
['services', service, "settings.json"])
def href(self):
url = build_url(self.catalog.service_url, [
"workspaces", self.workspace.name, "coveragestores",
"{}.xml".format(self.name)
])
return url
def href(self):
url = build_url(self.catalog.service_url,
["workspaces", self.workspace, "coveragestores"],
dict(name=self.name))
return url
def href(self):
return build_url(self.catalog.service_url,
["workspaces", self.name + ".xml"])
def href(self):
path = ["workspaces", self.workspace.name, "datastores"]
query = dict(name=self.name)
return build_url(self.catalog.service_url, path, query)
def create_coveragestore(self, name, workspace=None, path=None, type='GeoTIFF',
create_layer=True, layer_name=None, source_name=None, upload_data=False, contet_type="image/tiff",
overwrite=False):
"""
Create a coveragestore for locally hosted rasters.
If create_layer is set to true, will create a coverage/layer.
layer_name and source_name are only used if create_layer ia enabled. If not specified, the raster name will be used for both.
"""
if path is None:
raise Exception('You must provide a full path to the raster')
if layer_name is not None and ":" in layer_name:
ws_name, layer_name = layer_name.split(':')
allowed_types = [
'ImageMosaic',
'GeoTIFF',
'Gtopo30',
'WorldImage',
'AIG',
'ArcGrid',
'DTED',
'EHdr',
'ERDASImg',
'ENVIHdr',
'GeoPackage (mosaic)',
'NITF',
'RPFTOC',
'RST',
'VRT'
]
if type is None:
raise Exception('Type must be declared')
elif type not in allowed_types:
raise Exception('Type must be one of {}'.format(", ".join(allowed_types)))
if workspace is None:
workspace = self.get_default_workspace()
workspace = _name(workspace)
if not overwrite:
stores = self.get_stores(names=name, workspaces=[workspace])
if len(stores) > 0:
msg = "There is already a store named {} in workspace {}".format(name, workspace)
raise ConflictingDataError(msg)
if upload_data is False:
cs = UnsavedCoverageStore(self, name, workspace)
cs.type = type
cs.url = path if path.startswith("file:") else "file:{}".format(path)
self.save(cs)
if create_layer:
if layer_name is None:
layer_name = os.path.splitext(os.path.basename(path))[0]
if source_name is None:
source_name = os.path.splitext(os.path.basename(path))[0]
data = "<coverage><name>{}</name><nativeName>{}</nativeName></coverage>".format(layer_name, source_name)
url = "{}/workspaces/{}/coveragestores/{}/coverages.xml".format(self.service_url, workspace, name)
headers = {"Content-type": "application/xml"}
resp = self.http_request(url, method='post', data=data, headers=headers)
if resp.status_code != 201:
raise FailedRequestError('Failed to create coverage/layer {} for : {}, {}'.format(layer_name, name,
resp.status_code, resp.text))
self._cache.clear()
return self.get_resources(names=layer_name, workspaces=[workspace])[0]
else:
data = open(path, 'rb')
params = {"configure": "first", "coverageName": name}
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"coveragestores",
name,
"file.{}".format(type.lower())
],
params
)
headers = {"Content-type": contet_type}
resp = self.http_request(url, method='put', data=data, headers=headers)
if hasattr(data, "close"):
data.close()
if resp.status_code != 201:
raise FailedRequestError('Failed to create coverage/layer {} for : {}, {}'.format(layer_name, name, resp.status_code, resp.text))
return self.get_stores(names=name, workspaces=[workspace])[0]
def resource_url(self):
url = build_url(self.catalog.service_url, [
"workspaces", self.workspace.name, "datastores", self.name,
"featuretypes.xml"
])
return url
def href(self):
url = build_url(self.catalog.service_url, [
"workspaces", self.workspace.name, self.url_part_stores,
self.store.name, self.url_part_types, self.name + ".xml"
])
return url or self._href
def coveragestore_url(self):
return build_url(self.catalog.service_url,
["workspaces", self.name, "coveragestores.xml"])
def create_imagemosaic(self, name, data, configure='first', workspace=None, overwrite=False, charset=None):
if workspace is None:
workspace = self.get_default_workspace()
workspace = _name(workspace)
if not overwrite:
store = self.get_stores(names=name, workspaces=workspace)
if store:
raise ConflictingDataError("There is already a store named {}".format(name))
params = dict()
if charset is not None:
params['charset'] = charset
if configure.lower() not in ('first', 'none', 'all'):
raise ValueError("configure most be one of: first, none, all")
params['configure'] = configure.lower()
store_type = "file.imagemosaic"
contet_type = "application/zip"
if hasattr(data, 'read'):
# Adding this check only to pass tests. We should drop support for passing a file object
upload_data = data
elif isinstance(data, basestring):
if os.path.splitext(data)[-1] == ".zip":
upload_data = open(data, 'rb')
else:
store_type = "external.imagemosaic"
contet_type = "text/plain"
upload_data = data if data.startswith("file:") else "file:{data}".format(data=data)
else:
raise ValueError("ImageMosaic Dataset or directory: {data} is incorrect".format(data=data))
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"coveragestores",
name,
store_type
],
params
)
# PUT /workspaces/<ws>/coveragestores/<name>/file.imagemosaic?configure=none
headers = {
"Content-type": contet_type,
"Accept": "application/xml"
}
try:
resp = self.http_request(url, method='put', data=upload_data, headers=headers)
if resp.status_code != 201:
FailedRequestError('Failed to create ImageMosaic {} : {}, {}'.format(name, resp.status_code, resp.text))
self._cache.clear()
finally:
if hasattr(upload_data, "close"):
upload_data.close()
return self.get_stores(names=name, workspaces=workspace)[0]
def href(self):
return build_url(self.catalog.service_url, ["workspaces", self.name + ".xml"])
def datastore_url(self):
return build_url(self.catalog.service_url, ["workspaces", self.name, "datastores.xml"])
def create_imagemosaic(self, name, data, configure='first', workspace=None, overwrite=False, charset=None):
if workspace is None:
workspace = self.get_default_workspace()
workspace = _name(workspace)
if not overwrite:
store = self.get_stores(names=name, workspaces=[workspace])
if store:
raise ConflictingDataError("There is already a store named {}".format(name))
params = dict()
if charset is not None:
params['charset'] = charset
if configure.lower() not in ('first', 'none', 'all'):
raise ValueError("configure most be one of: first, none, all")
params['configure'] = configure.lower()
store_type = "file.imagemosaic"
contet_type = "application/zip"
if hasattr(data, 'read'):
# Adding this check only to pass tests. We should drop support for passing a file object
upload_data = data
elif isinstance(data, basestring):
if os.path.splitext(data)[-1] == ".zip":
upload_data = open(data, 'rb')
else:
store_type = "external.imagemosaic"
contet_type = "text/plain"
upload_data = data if data.startswith("file:") else "file:{data}".format(data=data)
else:
raise ValueError("ImageMosaic Dataset or directory: {data} is incorrect".format(data=data))
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"coveragestores",
name,
store_type
],
params
)
# PUT /workspaces/<ws>/coveragestores/<name>/file.imagemosaic?configure=none
headers = {
"Content-type": contet_type,
"Accept": "application/xml"
}
try:
resp = self.http_request(url, method='put', data=upload_data, headers=headers)
if resp.status_code != 201:
raise FailedRequestError('Failed to create ImageMosaic {} : {}, {}'.format(name, resp.status_code, resp.text))
self._cache.clear()
finally:
if hasattr(upload_data, "close"):
upload_data.close()
return self.get_stores(names=name, workspaces=[workspace])[0]
def create_coveragestore(self, name, workspace=None, path=None, type='GeoTIFF',
create_layer=True, layer_name=None, source_name=None, upload_data=False, contet_type="image/tiff"):
"""
Create a coveragestore for locally hosted rasters.
If create_layer is set to true, will create a coverage/layer.
layer_name and source_name are only used if create_layer ia enabled. If not specified, the raster name will be used for both.
"""
if path is None:
raise Exception('You must provide a full path to the raster')
if layer_name is not None and ":" in layer_name:
ws_name, layer_name = layer_name.split(':')
allowed_types = [
'ImageMosaic',
'GeoTIFF',
'Gtopo30',
'WorldImage',
'AIG',
'ArcGrid',
'DTED',
'EHdr',
'ERDASImg',
'ENVIHdr',
'GeoPackage (mosaic)',
'NITF',
'RPFTOC',
'RST',
'VRT'
]
if type is None:
raise Exception('Type must be declared')
elif type not in allowed_types:
raise Exception('Type must be one of {}'.format(", ".join(allowed_types)))
if workspace is None:
workspace = self.get_default_workspace()
workspace = _name(workspace)
if upload_data is False:
cs = UnsavedCoverageStore(self, name, workspace)
cs.type = type
cs.url = path if path.startswith("file:") else "file:{}".format(path)
self.save(cs)
if create_layer:
if layer_name is None:
layer_name = os.path.splitext(os.path.basename(path))[0]
if source_name is None:
source_name = os.path.splitext(os.path.basename(path))[0]
data = "<coverage><name>{}</name><nativeName>{}</nativeName></coverage>".format(layer_name, source_name)
url = "{}/workspaces/{}/coveragestores/{}/coverages.xml".format(self.service_url, workspace, name)
headers = {"Content-type": "application/xml"}
resp = self.http_request(url, method='post', data=data, headers=headers)
if resp.status_code != 201:
FailedRequestError('Failed to create coverage/layer {} for : {}, {}'.format(layer_name, name,
resp.status_code, resp.text))
self._cache.clear()
return self.get_resources(names=layer_name, workspaces=workspace)[0]
else:
data = open(path, 'rb')
params = {"configure": "first", "coverageName": name}
url = build_url(
self.service_url,
[
"workspaces",
workspace,
"coveragestores",
name,
"file.{}".format(type.lower())
],
params
)
headers = {"Content-type": contet_type}
resp = self.http_request(url, method='put', data=data, headers=headers)
if hasattr(data, "close"):
data.close()
if resp.status_code != 201:
FailedRequestError('Failed to create coverage/layer {} for : {}, {}'.format(layer_name, name, resp.status_code, resp.text))
return self.get_stores(names=name, workspaces=workspace)[0]
