def _create_geoserver_geonode_layer(new_table, sld_type, title, the_user, permissions): # Create the Layer in GeoServer from the table try: cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", settings.OGC_SERVER['default']['USER'], settings.OGC_SERVER['default']['PASSWORD']) ds = cat.get_store("uploaded") # name of store in WFP-Geonode cat.publish_featuretype(new_table, ds, "EPSG:4326", srs="EPSG:4326") # changing the layer's title in the title set by the user resource = cat.get_resource(new_table, workspace="geonode") resource.title = title cat.save(resource) except Exception as e: msg = "Error creating GeoServer layer for %s: %s" % (new_table, str(e)) print msg return None, msg # Create the Layer in GeoNode from the GeoServer Layer try: link_to_sld = "{location}styles/{sld_type}".format( **{ 'location': settings.OGC_SERVER['default']['LOCATION'], 'sld_type': sld_type }) r = requests.get(link_to_sld) sld = r.text sld = sld.replace( "name_of_layer", new_table ) # "name_of_layer" is set in the predefined sld in geoserver (polygon_style, line_style, point_style) cat.create_style(new_table, sld, overwrite=True) style = cat.get_style(new_table) layer = cat.get_layer(new_table) layer.default_style = style cat.save(layer) gs_slurp(owner=the_user, filter=new_table) from geonode.base.models import ResourceBase layer = ResourceBase.objects.get(title=title) geoserver_post_save(layer, ResourceBase) # assign permissions for this layer permissions_dict = json.loads(permissions) # needs to be dictionary if permissions_dict is not None and len(permissions_dict.keys()) > 0: layer.set_permissions(permissions_dict) except Exception as e: msg = "Error creating GeoNode layer for %s: %s" % (new_table, str(e)) return None, msg
def addLayersToGeoserver(self, options): cat = Catalog(self.geoserver_rest_url, options["geoserveradmin"], options["gpw"]) try: ds = cat.get_store(options["alias"]) except Exception as e: raise Exception("Erreur de récupération du workspace") layers = [] try: # connect to tables and create layers and correct urban styles for table in self.urb: try: style = self.urb[table] ft = cat.publish_featuretype(table, ds, "EPSG:31370", srs="EPSG:31370") gs_style = cat.get_style(style) cat.save(ft) res_name = ft.dirty["name"] res_title = options["alias"] + "_" + table cat.save(ft) layer_name = ds.workspace.name + ":" + res_name new_layer = cat.get_layer(layer_name) new_layer.default_style = gs_style cat.save(new_layer) layers.append({"res_name": res_name, "res_title": res_title}) except Exception as e: # a verifier une fois un possesion des styles print(str(e)) except Exception as e: print(str(e)) raise Exception("Erreur lors de la récupération des couches depuis Geoserver") return layers
def addLayersToGeoserver(self, options): cat = Catalog(self.geoserver_rest_url, options['geoserveradmin'], options['gpw']) try: ds = cat.get_store(options['alias']) except Exception as e: raise Exception('Erreur de récupération du workspace') layers = [] try: #connect to tables and create layers and correct urban styles for table in self.urb: try: style = self.urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') gs_style = cat.get_style(style) cat.save(ft) res_name = ft.dirty['name'] res_title = options['alias']+"_"+table cat.save(ft) layer_name = ds.workspace.name + ':' + res_name new_layer = cat.get_layer(layer_name) new_layer.default_style = gs_style cat.save(new_layer) layers.append({ 'res_name' : res_name, 'res_title' : res_title }) except Exception as e: # a verifier une fois un possesion des styles print(str(e)) except Exception as e: print(str(e)) raise Exception('Erreur lors de la récupération des couches depuis Geoserver') return layers
def main(options): #connect to geoserver cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw) #create datrastore for URB schema ws = cat.create_workspace(options.alias,'imio.be') ds = cat.create_datastore(options.alias, ws) ds.connection_parameters.update( host=options.urbanUrl, port="5432", database=options.database, user="******", passwd=options.ropw, dbtype="postgis") cat.save(ds) ds = cat.get_store(options.alias) #config object urb = { "capa":"Parcelles", "toli":"cadastre_ln_toponymiques", "canu":"cadastre_pt_num", "cabu":"Batiments", "gept":"cadastre_points_generaux", "gepn":"cadastre_pol_gen", "inpt":"point", "geli":"cadastre_ln_generales", "inli":"cadastre_ln_informations", "topt":"point", } #connect to tables and create layers and correct urban styles for table in urb: style = urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') ft.default_style = style cat.save(ft) resource = ft.resource resource.title = options.alias+"_"+table resource.save() layer, created = Layer.objects.get_or_create(name=layerName, defaults={ "workspace": ws.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', #"owner": owner, "uuid": str(uuid.uuid4()), "bbox_x0": Decimal(resource.latlon_bbox[0]), "bbox_x1": Decimal(resource.latlon_bbox[1]), "bbox_y0": Decimal(resource.latlon_bbox[2]), "bbox_y1": Decimal(resource.latlon_bbox[3]) }) set_attributes(layer, overwrite=True) if created: layer.set_default_permissions()
def handle_geoserver_layers(view_names, store_name, db_params, schema_name, logger=print): """Publish database views as GeoServer layers Parameters ---------- view_pairs: list An iterable of two-element tuples with the names of the ``coarse`` and ``detail`` database views that are to be published as GeoServer layers store_name: str Name of the GeoServer store to use. This store will be created in case it does not exist db_params: dict A mapping with parameters used for connecting to the database schema_name: str Name of the database schema where the views to publish reside logger: function, optional A function that is used to output information. Returns ------- list An iterable with the geoserver layers that have been published """ gs_catalog = Catalog( service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest", username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"]) logger("Retrieving geoserver workspace...") workspace = get_geoserver_workspace(gs_catalog) store = gs_catalog.get_store(store_name, workspace=workspace) if store is None: logger("Creating geoserver store...") store = get_postgis_store(gs_catalog, store_name, workspace, db_params, schema_name) layers = [] for view_name in view_names: logger("Adding {!r} as a geoserver layer...".format(view_name)) featuretype = gs_catalog.publish_featuretype(view_name, store, "EPSG:4326", srs="EPSG:4326") gs_catalog.save(featuretype) layers.append(featuretype) return layers
def handle_geoserver_layer(table_name, store_name, db_params, schema_name, default_style_name="", logger=print): """Publish a database table/view as a GeoServer layer Parameters ---------- store_name: str Name of the GeoServer store to use. This store will be created in case it does not exist db_params: dict A mapping with parameters used for connecting to the database schema_name: str Name of the database schema where the views to publish reside logger: function, optional A function that is used to output information. Returns ------- list An iterable with the geoserver layers that have been published """ logger("inside handle_geoserver_layer: {}".format(locals())) gs_catalog = Catalog( service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest", username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"] ) logger("Retrieving geoserver workspace...") workspace = get_geoserver_workspace(gs_catalog) store = gs_catalog.get_store(store_name, workspace=workspace) if store is None: logger("Creating geoserver store...") store = get_postgis_store(gs_catalog, store_name, workspace, db_params, schema_name) logger("Adding {!r} as a geoserver layer...".format(table_name)) featuretype = gs_catalog.publish_featuretype( table_name, store, "EPSG:4326", srs="EPSG:4326") logger("dir(featuretype): {}".format(dir(featuretype))) layer = gs_catalog.get_layer(featuretype.name) if default_style_name != "": logger("Setting default style for layer...") layer._set_default_style(default_style_name) gs_catalog.save(featuretype) gs_catalog.save(layer) return featuretype
def createLayer(): cat = Catalog('http://localhost:8081/geoserver/rest/', 'admin', 'geoserver') store = cat.get_store('test') geom = JDBCVirtualTableGeometry('newgeom', 'LineString', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = 'select ST_MakeLine(wkb_geometry ORDER BY waypoint) As newgeom, assetid, runtime from waypoints group by assetid,runtime' keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt)
class UploaderTests(MapStoryTestMixin): """ Basic checks to make sure pages load, etc. """ def create_datastore(self, connection, catalog): settings = connection.settings_dict params = {'database': settings['NAME'], 'passwd': settings['PASSWORD'], 'namespace': 'http://www.geonode.org/', 'type': 'PostGIS', 'dbtype': 'postgis', 'host': settings['HOST'], 'user': settings['USER'], 'port': settings['PORT'], 'enabled': "True"} store = catalog.create_datastore(settings['NAME'], workspace=self.workspace) store.connection_parameters.update(params) try: catalog.save(store) except FailedRequestError: # assuming this is because it already exists pass return catalog.get_store(settings['NAME']) def setUp(self): if not os.path.exists(os.path.join(os.path.split(__file__)[0], 'test_ogr')): self.skipTest('Skipping test due to missing test data.') # These tests require geonode to be running on :80! self.postgis = db.connections['datastore'] self.postgis_settings = self.postgis.settings_dict self.username, self.password = self.create_user('admin', 'admin', is_superuser=True) self.non_admin_username, self.non_admin_password = self.create_user('non_admin', 'non_admin') self.cat = Catalog(ogc_server_settings.internal_rest, *ogc_server_settings.credentials) self.workspace = 'geonode' self.datastore = self.create_datastore(self.postgis, self.cat) def tearDown(self): """ Clean up geoserver. """ self.cat.delete(self.datastore, recurse=True) def generic_import(self, file, configuration_options=[{'index': 0}]): f = file filename = os.path.join(os.path.dirname(__file__), 'test_ogr', f) res = self.import_file(filename, configuration_options=configuration_options) res = res[0] layer = res['layers'][0] self.assertEqual(layer['status'], 'updated') layer = Layer.objects.get(name=layer['name']) self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') if not filename.endswith('zip'): self.assertTrue(layer.attributes.count() >= DataSource(filename)[0].num_fields) # make sure we have at least one dateTime attribute self.assertTrue('xsd:dateTime' or 'xsd:date' in [n.attribute_type for n in layer.attributes.all()]) return layer def test_box_with_year_field(self): """ Tests the import of test_box_with_year_field. """ layer = self.generic_import('boxes_with_year_field.shp', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date(self): """ Tests the import of test_boxes_with_date. """ layer = self.generic_import('boxes_with_date.shp', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date_csv(self): """ Tests a CSV with WKT polygon. """ layer = self.generic_import('boxes_with_date.csv', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_iso_date(self): """ Tests the import of test_boxes_with_iso_date. """ layer = self.generic_import('boxes_with_date_iso_date.shp', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date_iso_date_zip(self): """ Tests the import of test_boxes_with_iso_date. """ layer = self.generic_import('boxes_with_date_iso_date.zip', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_dates_bc(self): """ Tests the import of test_boxes_with_dates_bc. """ layer = self.generic_import('boxes_with_dates_bc.shp', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_point_with_date(self): """ Tests the import of test_boxes_with_dates_bc. """ layer = self.generic_import('point_with_date.geojson', configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) # OGR will name geojson layers 'ogrgeojson' we rename to the path basename self.assertTrue(layer.name.startswith('point_with_date')) date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_end_date(self): """ Tests the import of test_boxes_with_end_date. This layer has a date and an end date field that are typed correctly. """ layer = self.generic_import('boxes_with_end_date.shp') date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0] end_date_attr = filter(lambda attr: attr.attribute == 'enddate', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, 'xsd:date') self.assertEqual(end_date_attr.attribute_type, 'xsd:date') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute) self.generic_time_check(layer, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute) def test_us_states_kml(self): """ Tests the import of us_states_kml. This layer has a date and an end date field that are typed correctly. """ # TODO: Support time in kmls. layer = self.generic_import('us_states.kml') def test_mojstrovka_gpx(self): """ Tests the import of mojstrovka.gpx. This layer has a date and an end date field that are typed correctly. """ layer = self.generic_import('mojstrovka.gpx') date_attr = filter(lambda attr: attr.attribute == 'time', layer.attributes)[0] self.assertEqual(date_attr.attribute_type, u'xsd:dateTime') configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute) self.generic_time_check(layer, attribute=date_attr.attribute) def generic_time_check(self, layer, attribute=None, end_attribute=None): """ Convenience method to run generic tests on time layers. """ self.cat._cache.clear() resource = self.cat.get_resource(layer.name, store=layer.store, workspace=self.workspace) timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual(attribute, timeInfo.attribute) self.assertEqual(end_attribute, timeInfo.end_attribute) def test_us_shootings_csv(self): """ Tests the import of US_shootings.csv. """ filename = 'US_shootings.csv' f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename) layer = self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['Date']}]) self.assertEqual(layer.name, 'us_shootings') date_field = 'date_as_date' configure_time(self.cat.get_layer(layer.name).resource, attribute=date_field) self.generic_time_check(layer, attribute=date_field) def test_sitins(self): """ Tests the import of US_shootings.csv. """ filename = 'US_Civil_Rights_Sitins0.csv' f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename) layer = self.generic_import(f, configuration_options=[{'index': 0, 'convert_to_date': ['Date']}]) def get_layer_names(self, in_file): """ Gets layer names from a data source. """ ds = DataSource(in_file) return map(lambda layer: layer.name, ds) def test_gdal_import(self): filename = 'point_with_date.geojson' f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename) self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['date']}]) def import_file(self, in_file, configuration_options=[]): """ Imports the file. """ self.assertTrue(os.path.exists(in_file)) # run ogr2ogr gi = GDALImport(in_file) layers = gi.handle(configuration_options=configuration_options) d = db.connections['datastore'].settings_dict connection_string = "PG:dbname='%s' user='******' password='******'" % (d['NAME'], d['USER'], d['PASSWORD']) output = [] for layer, layer_config in layers: # expose the featureType in geoserver self.cat.publish_featuretype(layer, self.datastore, 'EPSG:4326') output.append(gs_slurp(workspace='geonode', store=self.datastore.name, filter=layer)) return output @staticmethod def createFeatureType(catalog, datastore, name): """ Exposes a PostGIS feature type in geoserver. """ headers = {"Content-type": "application/xml"} data = "<featureType><name>{name}</name></featureType>".format(name=name) url = datastore.href.replace(".xml", '/featuretypes.xml'.format(name=name)) headers, response = catalog.http.request(url, "POST ", data, headers) return response def test_create_vrt(self): """ Tests the create_vrt function. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv') vrt = create_vrt(f) vrt.seek(0) output = vrt.read() self.assertTrue('name="US_shootings"' in output) self.assertTrue('<SrcDataSource>{0}</SrcDataSource>'.format(f) in output) self.assertTrue('<GeometryField encoding="PointFromColumns" x="Longitude" y="Latitude" />'.format(f) in output) self.assertEqual(os.path.splitext(vrt.name)[1], '.vrt') def test_file_add_view(self): """ Tests the file_add_view. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() # test login required for this view request = c.get(reverse('uploads-new')) self.assertEqual(request.status_code, 302) c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) self.assertEqual(response.status_code, 200) self.assertEqual(response.context['request'].path, reverse('uploads-list')) self.assertTrue(len(response.context['object_list']) == 1) upload = response.context['object_list'][0] self.assertEqual(upload.user.username, 'non_admin') self.assertEqual(upload.file_type, 'GeoJSON') self.assertTrue(upload.uploadlayer_set.all()) self.assertEqual(upload.state, 'UPLOADED') self.assertIsNotNone(upload.name) uploaded_file = upload.uploadfile_set.first() self.assertTrue(os.path.exists(uploaded_file.file.path)) f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'empty_file.geojson') with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) self.assertEqual(response.status_code, 200) self.assertIn('file', response.context_data['form'].errors) def test_describe_fields(self): """ Tests the describe fields functionality. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_shootings.csv') fields = None with GDALInspector(f) as f: layers = f.describe_fields() self.assertTrue(layers[0]['name'], 'us_shootings') self.assertEqual([n['name'] for n in layers[0]['fields']], ['Date', 'Shooter', 'Killed', 'Wounded', 'Location', 'City', 'Longitude', 'Latitude']) self.assertEqual(layers[0]['feature_count'], 203) def test_gdal_file_type(self): """ Tests the describe fields functionality. """ files = ((os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_shootings.csv'), 'CSV'), (os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson'), 'GeoJSON'), (os.path.join(os.path.dirname(__file__), 'test_ogr', 'mojstrovka.gpx'), 'GPX'), (os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_states.kml'), 'KML'), (os.path.join(os.path.dirname(__file__), 'test_ogr', 'boxes_with_year_field.shp'), 'ESRI Shapefile'), (os.path.join(os.path.dirname(__file__), 'test_ogr', 'boxes_with_date_iso_date.zip'), 'ESRI Shapefile'), ) for path, file_type in files: with GDALInspector(path) as f: self.assertEqual(f.file_type(), file_type) def test_configure_view(self): """ Tests the configuration view. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) upload = response.context['object_list'][0] payload = [{'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'editable': True}] response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload), content_type='application/json') self.assertTrue(response.status_code, 200) layer = Layer.objects.all()[0] self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime') self.assertEqual(Layer.objects.all()[0].owner.username, self.non_admin_username) lyr = self.cat.get_layer(layer.name) self.assertTrue('time' in lyr.resource.metadata) self.assertEqual(UploadLayer.objects.first().layer, layer) def test_configure_view_convert_date(self): """ Tests the configure view with a dataset that needs to be converted to a date. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv') c = AdminClient() c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) upload = response.context['object_list'][0] payload = [{'index': 0, 'convert_to_date': ['Date'], 'start_date': 'Date', 'configureTime': True, 'editable': True}] response = c.get('/importer-api/data-layers/{0}/configure/'.format(upload.id)) self.assertEqual(response.status_code, 405) response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id)) self.assertEqual(response.status_code, 400) response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload), content_type='application/json') self.assertTrue(response.status_code, 200) layer = Layer.objects.all()[0] self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime') self.assertTrue(layer.attributes.filter(attribute='date_as_date'), 'xsd:dateTime') lyr = self.cat.get_layer(layer.name) self.assertTrue('time' in lyr.resource.metadata) # ensure a user who does not own the upload cannot configure an import from it. c.logout() c.login_as_admin() response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id)) self.assertEqual(response.status_code, 404) def test_list_api(self): c = AdminClient() response = c.get('/importer-api/data/') self.assertEqual(response.status_code, 401) c.login_as_non_admin() response = c.get('/importer-api/data/') self.assertEqual(response.status_code, 200) admin = User.objects.get(username=self.username) non_admin = User.objects.get(username=self.non_admin_username) from .models import UploadFile f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv') with open(f, 'rb') as f: uploaded_file = SimpleUploadedFile('test_data', f.read()) admin_upload = UploadedData.objects.create(state='Admin test', user=admin) admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file)) non_admin_upload = UploadedData.objects.create(state='Non admin test', user=non_admin) non_admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file)) c.login_as_admin() response = c.get('/importer-api/data/') self.assertEqual(response.status_code, 200) body = json.loads(response.content) self.assertEqual(len(body['objects']), 2) response = c.get('/importer-api/data/?user__username=admin') self.assertEqual(response.status_code, 200) body = json.loads(response.content) self.assertEqual(len(body['objects']), 1) def test_layer_list_api(self): c = AdminClient() response = c.get('/importer-api/data-layers/') self.assertEqual(response.status_code, 401) c.login_as_non_admin() response = c.get('/importer-api/data-layers/') self.assertEqual(response.status_code, 200) def test_delete_from_non_admin_api(self): """ Ensure users can delete their data. """ c = AdminClient() f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) self.assertEqual(UploadedData.objects.all().count(), 1) id = UploadedData.objects.first().id response = c.delete('/importer-api/data/{0}/'.format(id)) self.assertEqual(response.status_code, 204) self.assertEqual(UploadedData.objects.all().count(), 0) def test_delete_from_admin_api(self): """ Ensure that administrators can delete data that isn't theirs. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) self.assertEqual(UploadedData.objects.all().count(), 1) c.logout() c.login_as_admin() id = UploadedData.objects.first().id response = c.delete('/importer-api/data/{0}/'.format(id)) self.assertEqual(response.status_code, 204) self.assertEqual(UploadedData.objects.all().count(), 0) def naming_an_import(self): """ Tests providing a name in the configuration options. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() c.login_as_non_admin() name = 'point-with-a-date' with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) payload = {'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'name': name, 'editable': True} response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps(payload), content_type='application/json') layer = Layer.objects.all()[0] self.assertEqual(layer.title, name.replace('-', '_')) def test_api_import(self): """ Tests the import api. """ f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson') c = AdminClient() c.login_as_non_admin() with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) payload = {'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'editable': True} self.assertTrue(isinstance(UploadLayer.objects.first().configuration_options, dict)) response = c.get('/importer-api/data-layers/1/') self.assertEqual(response.status_code, 200) response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps([payload]), content_type='application/json') self.assertEqual(response.status_code, 200) self.assertTrue('task' in response.content) layer = Layer.objects.all()[0] self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime') lyr = self.cat.get_layer(layer.name) self.assertTrue('time' in lyr.resource.metadata) self.assertEqual(UploadLayer.objects.first().layer, layer) self.assertTrue(UploadLayer.objects.first().task_id) def test_valid_file_extensions(self): """ Test the file extension validator. """ for extension in IMPORTER_VALID_EXTENSIONS: self.assertIsNone(validate_file_extension(SimpleUploadedFile('test.{0}'.format(extension), ''))) with self.assertRaises(ValidationError): validate_file_extension(SimpleUploadedFile('test.txt', '')) def test_no_geom(self): """ Test the file extension validator. """ with self.assertRaises(ValidationError): validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)')) # This should pass (geom type is unknown) validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)')) def test_numeric_overflow(self): """ Regression test for numeric field overflows in shapefiles. # https://trac.osgeo.org/gdal/ticket/5241 """ self.generic_import('Walmart.zip', configuration_options=[{'index': 0, 'convert_to_date': []}])
class Extractor: def __init__(self, input_dir): logging.basicConfig(filename='extractor.log', level=logging.INFO) self.table_count = 0 self.working_dir = input_dir self.schemas = [] self.tables = {} # pg connexion self.conn = psycopg2.connect\ (dbname=config.PGDATABASE, user=config.PGUSER, password=config.PGPASSWORD, host=config.PGHOST) self.cur = self.conn.cursor() self.sql_commands = [] #gs config self.cat = None def get_files_by_format(self, format): logging.info('Started') for subdir, dirs, files in os.walk(self.working_dir): # lower and remove spaces schema = ''.join(s.lower() for s in os.path.basename(subdir) if os.path.basename(subdir) and not s.isspace()) if schema != '': self.schemas.append(schema) for file in files: if file.endswith(".{}".format(format)): table_name = ''.join(s.lower() for s in file if not s.isspace()) self.tables[Path(table_name).resolve().stem] = { 'schema': schema, 'path': '{}/{}'.format(subdir, file) } def bulk_create(self): for schema in self.schemas: if schema == '': continue self._push_to_database(self._build_create_schema_query(schema)) self.commit_to_database() def gs_connect_geoserver(self): self.cat = Catalog(config.GSREST, username=config.GSUSER, password=config.GSPASSWORD) def gs_create_workspace(self, ws): if self.cat.get_workspace(ws): logging.info('workspace {} exists skipping...'.format(ws)) else: self.cat.create_workspace(ws) def gs_get_data_store(self, store): return self.cat.get_store(store) def gs_create_store(self, schema, workspace_name): try: self.cat.get_store(schema) except: ds = self.cat.create_datastore(schema, workspace_name) ds.connection_parameters.update({ 'host': config.PGHOST, 'port': config.PGPORT, 'database': config.PGDATABASE, 'user': config.PGUSER, 'passwd': config.PGPASSWORD, 'dbtype': 'postgis', 'schema': schema, 'Expose primary keys': 'true' }) self.cat.save(ds) def gs_publish_feature_type(self, table, store_name): try: data_store = self.gs_get_data_store(store_name) except: self.cat.publish_featuretype(table, data_store, 'EPSG:2154', srs='EPSG:2154') @staticmethod def insert_data_from_shapefile(table, **kwargs): schema = kwargs['schema'] path = kwargs['path'] cmd = 'shp2pgsql -s {} {} "{}"."{}" | ' \ 'PGPASSWORD={} psql -h {} -d {} -U {}'.format( EPSG, path, schema, table, config.PGPASSWORD, config.PGHOST, config.PGDATABASE, config.PGUSER) subprocess.call(cmd, shell=True) @staticmethod def _build_create_table_query(dirname, shapefile): return 'CREATE TABLE {}.{};'.format(dirname, shapefile) @staticmethod def _build_create_schema_query(schema): return 'CREATE SCHEMA {};'.format(schema) def _push_to_database(self, query): self.cur.execute(query) def commit_to_database(self): self.conn.commit()
class wrap_geoserver: """ Geoserver (gsconfig) wrapper """ def __init__(self, geoserver_name, username=username, password=password, easy=False): if geoserver_name in list(REST.keys()): self.path = REST[geoserver_name] else: self.path = geoserver_name self.wms = self.path.replace("rest/", "wms") self.name = geoserver_name self.catalog = Catalog(self.path, username, password) if not easy: self.layers = [] self.layer_names = [] for layer in self.catalog.get_layers(): self.layers.append(layer) self.layer_names.append(layer.name) self.stores = [store for store in self.catalog.get_stores()] self.store_names = [store.name for store in self.stores] styles = [] self.workspaces = [] self.workspace_names = [] for workspace in self.catalog.get_workspaces(): styles = styles + self.catalog.get_styles(workspace) self.workspace_names.append(workspace._name) self.workspaces.append(workspace) self.styles = styles + [ style for style in self.catalog.get_styles() ] self.style_names = [style.name for style in self.styles] def unpack(self, workspace_name, store_type="datastore"): layers_and_styles = {} features = [] workspace = self.get_workspace(workspace_name) if store_type == "datastore": store_url = workspace.datastore_url elif store_type == "coveragestore": store_url = workspace.coveragestore_url else: print("No correct store given") for datastore in tqdm(get(store_url, "name")): url = "{}workspaces/{}/datastores/{}".format( self.path, workspace.name, datastore) features = features + get(url, between_quotes=True) for feature in features: layer_name = os.path.basename(feature).split(".")[0] self.get_layer(self.get_slug(workspace.name, layer_name)) layers_and_styles[layer_name] = self.layer.default_style setattr(self, workspace_name + "_data", layers_and_styles) return layers_and_styles def get_layer(self, layer, easy=False): self.layer = self.catalog.get_layer(layer) if not easy: self.resource = self.layer.resource self.layer_name = self.layer.resource.name self.sld_name = self.layer.default_style.name self.sld_body = self.layer.default_style.sld_body self.layer_latlon_bbox = self.layer.resource.latlon_bbox self.layer_title = self.layer.resource.title self.layer_abstract = self.layer.resource.abstract def get_store(self, layer): self.store = self.layer.resource._store def get_resource(self): self.resource = self.catalog.get_resource(self.layer.name, self.store) def get_workspace(self, workspace_name): self.workspace = self.catalog.get_workspace(workspace_name) self.workspace_name = self.workspace._name return self.workspace def write_abstract(self, data, load_resource=True): if load_resource: self.get_resource() self.resource.abstract = data self.catalog.save(self.resource) def write_title(self, title): self.resource.title = title self.catalog.save(self.resource) def get_connection_parameters(self): self.get_resource() return self.resource.store.connection_parameters def create_workspace(self, workspace_name): workspace_exists = workspace_name in self.workspace_names if not workspace_exists: self.workspace = self.catalog.create_workspace(workspace_name) else: print("workspace already exists, using existing workspace") self.workspace = self.catalog.get_workspace(workspace_name) self.workspace_name = workspace_name def create_postgis_datastore(self, store_name, workspace_name, pg_data): try: self.store = self.catalog.get_store(store_name, self.workspace_name) print("store within workspace exists, using existing store") except Exception as e: print(e) ds = self.catalog.create_datastore(store_name, workspace_name) ds.connection_parameters.update( host=pg_data["host"], port=pg_data["port"], database=pg_data["database"], user=pg_data["username"], passwd=pg_data["password"], dbtype="postgis", schema="public", ) self.save(ds) self.store = self.catalog.get_store(store_name, self.workspace_name) self.store_name = store_name def publish_layer(self, layer_name, workspace_name, overwrite=False, epsg="3857", reload=False): layer_exists = layer_name in self.layer_names # if layer_name in self.workspace_layers[workspace_name]: slug = self.get_slug(workspace_name, layer_name) if overwrite and layer_exists: print("Layer exists, deleting layer") try: self.layer = self.catalog.get_layer(slug) self.delete(self.layer) self.reload() layer_exists = False except Exception as e: print(e) print("Layer does not exist in workspace") layer_exists = False if not layer_exists: feature_type = self.catalog.publish_featuretype( layer_name, self.store, "EPSG:{}".format(str(epsg)), srs="EPSG:{}".format(str(epsg)), ) self.save(feature_type) self.feature_type = feature_type else: print("layer already exists, using existing layer") if reload: self.get_layer(slug) self.layer_name = layer_name def publish_layergroup(self, name, layers, styles=(), bounds=None, workspace=None): layer_group = self.catalog.create_layergroup(name, layers, styles, bounds, workspace) self.save(layer_group) def save(self, save_object): return self.catalog.save(save_object) def close(self): self.catalog = None def delete(self, delete_object): self.catalog.delete(delete_object) def reload(self): self.catalog.reload() def upload_shapefile(self, layer_name, shapefile_path): path = shapefile_path.split(".shp")[0] shapefile = shapefile_and_friends(path) ft = self.catalog.create_featurestore(layer_name, shapefile, self.workspace) self.save(ft) def upload_sld(self, sld_name, workspace_name, sld, overwrite=True): style_exists = sld_name in self.style_names if overwrite and style_exists: print("Overwriting style") style = self.catalog.get_style(sld_name, workspace_name) self.delete(style) self.reload() style_exists = False if not style_exists: try: self.catalog.create_style(sld_name, sld, False, workspace_name, "sld11") except Exception as e: print(e) style = self.catalog.get_style(sld_name, workspace_name) self.delete(style) self.reload() self.catalog.create_style(sld_name, sld, False, workspace_name, "sld10") self.style_name = sld_name else: if style_exists: print("Style already exists, using current style") self.style_name = sld_name def set_sld_for_layer(self, workspace_name=None, style_name=None, use_custom=False): if not use_custom: workspace_name = self.workspace_name style_name = self.style_name self.style_slug = self.get_slug(workspace_name, style_name) else: if workspace_name is None: self.style_slug = style_name else: self.style_slug = self.get_slug(workspace_name, style_name) self.style = self.catalog.get_style(self.style_slug) print("Setting {} for {}".format(self.style.name, self.layer.name)) self.layer.default_style = self.style self.save(self.layer) def get_slug(self, workspace, name): return "{}:{}".format(workspace, name) def get_slug_data(self, slug): workspace_name = slug.split(":")[0] layer_name = slug.split(":")[1] return workspace_name, layer_name def get_sld(self, layer_slug=None): if layer_slug is None: self.style = self.catalog.get_style(self.layer_slug) else: self.style = self.catalog.get_style(layer_slug) self.sld_body = self.style.sld_body return self.sld_body def get_layer_workspace(self, layer_name): return self.catalog.get_layer(layer_name).resource.workspace.name
class GsConn(): def __init__(self, host, login, password, debug=False): """ Geoserver connection """ self.host = host self.login = login self.password = password self.debug = debug # Connect to server self.cat = Catalog("http://%s/geoserver/rest" % host, login, password) if self.debug is True: print "Connected to geoserver" def crate_workspace(self, name, uri, overwrite=False): """ Creates a workspace :param name: Workspace name. :param overwrite: If True, delete existing workspace. :return: None """ workspaces = [ workspace.name for workspace in self.cat.get_workspaces() ] if name in workspaces and overwrite is True: # ws2del = self.cat.get_workspace(name) # self.cat.delete(ws2del, purge=True, recurse=True) return None # NOTE: If we delete the workspace then all associated layers are lost. elif name in workspaces and overwrite is False: print "ERROR: Workspace %s already exists (use overwrite=True)." % name self.cat.create_workspace(name, uri) if self.debug is True: print "Workspace %s available." % name ws = self.cat.get_workspace(name) ws.enabled = True def create_pg_store(self, name, workspace, host, port, login, password, dbname, schema, overwrite=False): """ Creates datastore. :param name: Name of the datastore. :param workspace: Name of the workspace to use. :param overwrite: If True replace datastore. :return: None """ stores = [store.name for store in self.cat.get_stores()] if name in stores and overwrite is True: # st2del = self.cat.get_store(name) # self.cat.delete(st2del, purge=True, recurse=True) # self.cat.reload() return None # NOTE: If we delete store, every layers associated with are lost. elif name in stores and overwrite is False: print "ERROR: Store %s already exists (use overwrite=True)." % name ds = self.cat.create_datastore(name, workspace) ds.connection_parameters.update(host=host, port=port, user=login, passwd=password, dbtype='postgis', database=dbname, schema=schema) self.cat.save(ds) ds = self.cat.get_store(name) if ds.enabled is False: print "ERROR: Geoserver store %s not enabled" % name if self.debug is True: print "Datastore %s created." % name def publish_pg_layer(self, layer_table, layer_name, store, srid, overwrite=True): """ """ existing_lyr = self.cat.get_layer("participatubes:%s" % layer_table) if existing_lyr is not None: print "Layer participatubes:%s already exists, deleting it." % layer_table self.cat.delete(existing_lyr) self.cat.reload() ds = self.cat.get_store(store) ft = self.cat.publish_featuretype(layer_table, ds, 'EPSG:%s' % srid, srs='EPSG:4326') ft.projection_policy = "REPROJECT_TO_DECLARED" ft.title = layer_name self.cat.save(ft) if ft.enabled is False: print "ERROR: Layer %s %s %s is not enabled." (ft.workspace.name, ft.store.name, ft.title) if self.debug is True: print "Layer %s>%s>%s published." % (ft.workspace.name, ft.store.name, ft.title) def create_style_from_sld(self, style_name, sld_file, workspace, overwrite=True): """ """ if self.cat.get_style(style_name) is not None: print "Style %s already exists, deleting it." % style_name style2del = self.cat.get_style(style_name) self.cat.delete(style2del) self.cat.create_style( style_name, open(sld_file).read(), overwrite=overwrite ) # FIXME: if ", workspace=workspace" specified can't delete style if self.debug is True: print "Style %s created in Geoserver" % style_name def apply_style_to_layer(self, layer_name, style_name): """ Apply a geoserver styler to a layer """ gs_layer = self.cat.get_layer(layer_name) gs_style = self.cat.get_style(style_name) # FIXME: Which works better? # gs_layer.default_style = gs_style / gs_layer._set_default_style(gs_style) # FIXME: Maybe indicate workspace when saving style then name the style as "workspace:style" gs_layer._set_default_style(gs_style) self.cat.save(gs_layer) if self.debug is True: print "Style applied to %s" % layer_name
def create_point_col_from_lat_lon(new_table_owner, table_name, lat_column, lng_column): """ Using a new DataTable and specified lat/lng column names, map the points :param new_table_owner: :param table_name: :param lat_column: :param lng_column: :return: """ LOGGER.info('create_point_col_from_lat_lon') assert isinstance(new_table_owner, User), "new_table_owner must be a User object" assert table_name is not None, "table_name cannot be None" assert lat_column is not None, "lat_column cannot be None" assert lng_column is not None, "lng_column cannot be None" # ---------------------------------------------------- # Retrieve the DataTable and check for lat/lng columns # ---------------------------------------------------- try: dt = DataTable.objects.get(table_name=table_name) except DataTable.DoesNotExist: err_msg = "Could not find DataTable with name: %s" % (table_name) LOGGER.error(err_msg) return False, err_msg # ---------------------------------------------------- # Latitude attribute # ---------------------------------------------------- lat_col_attr = dt.get_attribute_by_name( standardize_column_name(lat_column)) if lat_col_attr is None: err_msg = 'DataTable "%s" does not have a latitude column named "%s" (formatted: %s)'\ % (table_name, lat_column, standardize_column_name(lat_column)) LOGGER.error(err_msg) return False, err_msg is_valid, err_msg = is_valid_lat_lng_attribute(lat_col_attr) if not is_valid: LOGGER.error(err_msg) return False, err_msg # ---------------------------------------------------- # Longitude attribute # ---------------------------------------------------- lng_col_attr = dt.get_attribute_by_name( standardize_column_name(lng_column)) if lng_col_attr is None: err_msg = 'DataTable "%s" does not have a longitude column named "%s" (formatted: %s)'\ % (table_name, lng_column, standardize_column_name(lng_column)) LOGGER.error(err_msg) return False, err_msg is_valid, err_msg = is_valid_lat_lng_attribute(lng_col_attr, lng_check=True) if not is_valid: LOGGER.error(err_msg) return False, err_msg # ---------------------------------------------------- # Start mapping record # ---------------------------------------------------- lat_lnt_map_record = LatLngTableMappingRecord(datatable=dt\ , lat_attribute=lat_col_attr\ , lng_attribute=lng_col_attr\ ) msg('create_point_col_from_lat_lon - 2') # ---------------------------------------------------- # Yank bad columns out of the DataTable # ---------------------------------------------------- # See https://github.com/IQSS/dataverse/issues/2949 (success, row_cnt_or_err_msg) = remove_bad_lat_lng_numbers(lat_lnt_map_record) if not success: if lat_lnt_map_record.id: lat_lnt_map_record.delete() return False, 'Failed to remove bad lat/lng values.' # The bad rows were not mapped lat_lnt_map_record.unmapped_record_count = row_cnt_or_err_msg # --------------------------------------------- # Format SQL to: # (a) Add the Geometry column to the Datatable # (b) Populate the column using the lat/lng attributes # (c) Create column index # --------------------------------------------- # (a) Add column SQL alter_table_sql = "ALTER TABLE %s ADD COLUMN geom geometry(POINT,4326);" % ( table_name) # postgi 2.x # (b) Populate column SQL update_table_sql = "UPDATE %s SET geom = ST_SetSRID(ST_MakePoint(%s,%s),4326);" \ % (table_name, lng_col_attr.attribute, lat_col_attr.attribute) #update_table_sql = "UPDATE %s SET geom = ST_SetSRID(ST_MakePoint(cast(%s AS float), cast(%s as float)),4326);" % (table_name, lng_column, lat_column) # (c) Index column SQL create_index_sql = "CREATE INDEX idx_%s_geom ON %s USING GIST(geom);" % ( table_name, table_name) # --------------------------------------------- # Run the SQL # --------------------------------------------- try: conn = psycopg2.connect( get_datastore_connection_string(is_dataverse_db=False)) cur = conn.cursor() LOGGER.debug('Run alter table SQL: %s', alter_table_sql) cur.execute(alter_table_sql) LOGGER.debug('Run update table SQL: %s', update_table_sql) cur.execute(update_table_sql) LOGGER.debug('Run create index SQL: %s', create_index_sql) cur.execute(create_index_sql) conn.commit() conn.close() except Exception as ex_obj: conn.close() err_msg = "Error Creating Point Column from Latitude and Longitude %s" % ( ex_obj) LOGGER.error(err_msg) return False, err_msg msg('create_point_col_from_lat_lon - 4') # ------------------------------------------------------ # Create the Layer in GeoServer from the table # ------------------------------------------------------ try: cat = Catalog(settings.GEOSERVER_BASE_URL + "rest",\ settings.GEOSERVER_CREDENTIALS[0],\ settings.GEOSERVER_CREDENTIALS[1]) # "admin", "geoserver") workspace = cat.get_workspace("geonode") ds_list = cat.get_xml(workspace.datastore_url) datastores = [ datastore_from_index(cat, workspace, n) for n in ds_list.findall("dataStore") ] #---------------------------- # Find the datastore #---------------------------- ds = None from geonode.maps.utils import get_db_store_name for datastore in datastores: if datastore.name == get_db_store_name(): ds = datastore if ds is None: err_msg = "Datastore not found: '%s' (lat/lng)" % ( settings.DB_DATASTORE_NAME) return False, err_msg ft = cat.publish_featuretype(table_name, ds, "EPSG:4326", srs="EPSG:4326") cat.save(ft) except Exception as e: lat_lnt_map_record.delete() traceback.print_exc(sys.exc_info()) err_msg = "Error creating GeoServer layer for %s: %s" % (table_name, str(e)) return False, err_msg msg('create_point_col_from_lat_lon - 5 - add style') # ------------------------------------------------------ # Set the Layer's default Style # ------------------------------------------------------ set_default_style_for_latlng_layer(cat, ft) # ------------------------------------------------------ # Create the Layer in GeoNode from the GeoServer Layer # ------------------------------------------------------ try: layer, created = Layer.objects.get_or_create( name=table_name, defaults={ "workspace": workspace.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), ft.name.encode('utf-8')), "title": dt.title or 'No title provided', #"name" : dt.title or 'No title provided', "abstract": dt.abstract or 'No abstract provided', "uuid": str(uuid.uuid4()), "owner": new_table_owner, #"bbox_x0": Decimal(ft.latlon_bbox[0]), #"bbox_x1": Decimal(ft.latlon_bbox[1]), #"bbox_y0": Decimal(ft.latlon_bbox[2]), #"bbox_y1": Decimal(ft.latlon_bbox[3]) }) #set_attributes(layer, overwrite=True) except Exception as e: traceback.print_exc(sys.exc_info()) err_msg = "Error creating GeoNode layer for %s: %s" % (table_name, str(e)) return False, err_msg # ---------------------------------- # Set default permissions (public) # ---------------------------------- layer.set_default_permissions() # ------------------------------------------------------------------ # Create LayerAttributes for the new Layer (not done in GeoNode 2.x) # ------------------------------------------------------------------ (attributes_created, err_msg) = create_layer_attributes_from_datatable(dt, layer) if not attributes_created: LOGGER.error(err_msg) layer.delete() # Delete the layer return False, "Sorry there was an error creating the Datatable. (s:ll)" # ---------------------------------- # Save a related LatLngTableMappingRecord # ---------------------------------- lat_lnt_map_record.layer = layer # ---------------------------------- # Retrieve matche/unmatched counts # ---------------------------------- # Get datatable feature count - total record to map (success_datatable, datatable_feature_count) = get_layer_feature_count(dt.table_name) # Get layer feature count - mapped records (success_layer, layer_feature_count) = get_layer_feature_count(layer.name) # Set Record counts if success_layer and success_datatable: lat_lnt_map_record.mapped_record_count = layer_feature_count new_misssed_records = datatable_feature_count - layer_feature_count if lat_lnt_map_record.unmapped_record_count and lat_lnt_map_record.unmapped_record_count > 0: lat_lnt_map_record.unmapped_record_count += new_misssed_records else: lat_lnt_map_record.unmapped_record_count = new_misssed_records else: LOGGER.error('Failed to calculate Lat/Lng record counts') lat_lnt_map_record.save() return True, lat_lnt_map_record
class GsConn: def __init__(self, host, login, password, debug=False): """ Geoserver connection """ self.host = host self.login = login self.password = password self.debug = debug # Connect to server self.cat = Catalog("http://%s/geoserver/rest" % host, login, password) if self.debug is True: print "Connected to geoserver" def crate_workspace(self, name, overwrite=False): """ Creates a workspace :param name: Workspace name. :param overwrite: If True, delete existing workspace. :return: None """ workspaces = [workspace.name for workspace in self.cat.get_workspaces()] if name in workspaces and overwrite is True: # ws2del = self.cat.get_workspace(name) # self.cat.delete(ws2del, purge=True, recurse=True) return None # NOTE: If we delete the workspace then all associated layers are lost. elif name in workspaces and overwrite is False: print "ERROR: Workspace %s already exists (use overwrite=True)." % name self.cat.create_workspace(name, "http://%s/%s" % (self.host, name)) if self.debug is True: print "Workspace %s available." % name ws = self.cat.get_workspace(name) ws.enabled = True def create_pg_store(self, name, workspace, host, port, login, password, dbname, schema, overwrite=False): """ Creates datastore. :param name: Name of the datastore. :param workspace: Name of the workspace to use. :param overwrite: If True replace datastore. :return: None """ stores = [store.name for store in self.cat.get_stores()] if name in stores and overwrite is True: # st2del = self.cat.get_store(name) # self.cat.delete(st2del, purge=True, recurse=True) # self.cat.reload() return None # NOTE: If we delete store, every layers associated with are lost. elif name in stores and overwrite is False: print "ERROR: Store %s already exists (use overwrite=True)." % name ds = self.cat.create_datastore(name, workspace) ds.connection_parameters.update( host=host, port=port, user=login, passwd=password, dbtype="postgis", database=dbname, schema=schema ) self.cat.save(ds) ds = self.cat.get_store(name) if ds.enabled is False: print "ERROR: Geoserver store %s not enabled" % name if self.debug is True: print "Datastore %s created." % name def publish_pg_layer(self, layer_table, layer_name, store, srid, overwrite=True): """ """ existing_lyr = self.cat.get_layer("ma_carte:%s" % layer_table) if existing_lyr is not None: print "Layer ma_carte:%s already exists, deleting it." % layer_table self.cat.delete(existing_lyr) self.cat.reload() ds = self.cat.get_store(store) ft = self.cat.publish_featuretype(layer_table, ds, "EPSG:%s" % srid, srs="EPSG:4326") ft.projection_policy = "REPROJECT_TO_DECLARED" ft.title = layer_name self.cat.save(ft) if ft.enabled is False: print "ERROR: Layer %s %s %s is not enabled."(ft.workspace.name, ft.store.name, ft.title) if self.debug is True: print "Layer %s>%s>%s published." % (ft.workspace.name, ft.store.name, ft.title) def create_style_from_sld(self, style_name, sld_file, workspace, overwrite=True): """ """ if self.cat.get_style(style_name) is not None: print "Style %s already exists, deleting it." % style_name style2del = self.cat.get_style(style_name) self.cat.delete(style2del) self.cat.create_style( style_name, open(sld_file).read(), overwrite=overwrite ) # FIXME: if ", workspace=workspace" specified can't delete style if self.debug is True: print "Style %s created in Geoserver" % style_name def apply_style_to_layer(self, layer_name, style_name): """ Apply a geoserver styler to a layer """ gs_layer = self.cat.get_layer(layer_name) gs_style = self.cat.get_style(style_name) # FIXME: Which works better? # gs_layer.default_style = gs_style / gs_layer._set_default_style(gs_style) # FIXME: Maybe indicate workspace when saving style then name the style as "workspace:style" gs_layer._set_default_style(gs_style) self.cat.save(gs_layer) if self.debug is True: print "Style applied to %s" % layer_name
def publishGeoserver(appdef, progress): viewCrs = appdef["Settings"]["App view CRS"] usesGeoServer = False for applayer in appdef["Layers"]: if applayer.method != METHOD_FILE: if applayer.layer.type() == applayer.layer.VectorLayer and applayer.layer.providerType().lower() != "wfs": usesGeoServer = True if not usesGeoServer: return progress.setText("Publishing to GeoServer") progress.setProgress(0) geoserverUrl = appdef["Deploy"]["GeoServer url"] + "/rest" geoserverPassword = appdef["Deploy"]["GeoServer password"] geoserverUsername = appdef["Deploy"]["GeoServer username"] workspaceName = appdef["Deploy"]["GeoServer workspace"] dsName = "ds_" + workspaceName host = appdef["Deploy"]["PostGIS host"] port = appdef["Deploy"]["PostGIS port"] postgisUsername = appdef["Deploy"]["PostGIS username"] postgisPassword = appdef["Deploy"]["PostGIS password"] database = appdef["Deploy"]["PostGIS database"] schema = appdef["Deploy"]["PostGIS schema"] catalog = Catalog(geoserverUrl, geoserverUsername, geoserverPassword) workspace = catalog.get_workspace(workspaceName) if workspace is None: workspace = catalog.create_workspace(workspaceName, workspaceName) try: store = catalog.get_store(dsName, workspace) resources = store.get_resources() for resource in resources: layers = catalog.get_layers(resource) for layer in layers: catalog.delete(layer) catalog.delete(resource) catalog.delete(store) except Exception: pass try: store = catalog.get_store(dsName, workspace) except FailedRequestError: store = None for i, applayer in enumerate(appdef["Layers"]): layer = applayer.layer if applayer.method != METHOD_FILE and applayer.method != METHOD_DIRECT: name = safeName(layer.name()) sld, icons = getGsCompatibleSld(layer) if sld is not None: catalog.create_style(name, sld, True) uploadIcons(icons, geoserverUsername, geoserverPassword, catalog.gs_base_url) if layer.type() == layer.VectorLayer: if applayer.method == METHOD_WFS_POSTGIS or applayer.method == METHOD_WMS_POSTGIS: if store is None: store = catalog.create_datastore(dsName, workspace) store.connection_parameters.update( host=host, port=str(port), database=database, user=postgisUsername, schema=schema, passwd=postgisPassword, dbtype="postgis") catalog.save(store) catalog.publish_featuretype(name, store, layer.crs().authid()) else: path = getDataFromLayer(layer, viewCrs) catalog.create_featurestore(name, path, workspace=workspace, overwrite=True) gslayer = catalog.get_layer(name) r = gslayer.resource r.dirty['srs'] = viewCrs catalog.save(r) elif layer.type() == layer.RasterLayer: path = getDataFromLayer(layer, viewCrs) catalog.create_coveragestore(name, path, workspace=workspace, overwrite=True) if sld is not None: publishing = catalog.get_layer(name) publishing.default_style = catalog.get_style(name) catalog.save(publishing) progress.setProgress(int((i+1)*100.0/len(appdef["Layers"])))
def processAlgorithm(self, parameters, context, model_feedback): """ Process the algorithm :param parameters: parameters of the process :param context: context of the process :param model_feedback: feedback instance for the process :return: """ # Use a multi-step feedback, so that individual child algorithm progress reports are adjusted for the # overall progress through the model self.xml_path = parameters["XML_PATH"] if not self.xml_path.lower().endswith(".xml"): feedback = QgsProcessingMultiStepFeedback(0, model_feedback) feedback.reportError( "XML Workspace Definition is not an XML file!", True) return {} db_name = parameters["DB_NAME"] db_params = self.get_db_params(db_name) store_name = parameters["GS_STORE"] wrk_name = parameters["GS_WORKSPACE"] dataset_list = [] datasets = self.getDatasets() for dataset in datasets: type = dataset.getElementsByTagName( TAG_DE_TYPE)[0].childNodes[0].data if type == "esriDTFeatureClass": ds_name = dataset.getElementsByTagName( TAG_DE_NAME)[0].childNodes[0].data dataset_list.append({ "name": ds_name.lower(), "srs": "EPSG:4326" }) feedback = QgsProcessingMultiStepFeedback(2 + len(dataset_list), model_feedback) feedback.pushInfo("Get GeoServer Catalog: " + parameters["GS_REST_URL"]) gs_catalogue = Catalog(parameters["GS_REST_URL"], parameters["GS_ADMIN"], parameters["GS_PASSWORD"]) # workspace if wrk_name == "" or wrk_name is None: wrk_name = db_name.lower() + "_ws" wrk_uri = "http://" + wrk_name feedback.pushInfo("GeoServer Workspace: " + wrk_name + " (" + wrk_uri + ")") workspace = gs_catalogue.get_workspace(wrk_name) if workspace is None: workspace = gs_catalogue.create_workspace(wrk_name, wrk_uri) feedback.setCurrentStep(1) # store if store_name == "" or store_name is None: store_name = db_name.lower() + "_ds" feedback.pushInfo("GeoServer Data Store: " + store_name) store = gs_catalogue.get_store(store_name, workspace) if store is None: store = gs_catalogue.create_datastore(store_name, workspace) store.connection_parameters.update(**db_params) gs_catalogue.save(store) feedback.setCurrentStep(2) step = 2 published_count = 0 layer_prefix = "v_" for ds_cur in dataset_list: layer_name = layer_prefix + ds_cur["name"] feedback.pushInfo("GeoServer Publish: " + layer_name + " (" + ds_cur["srs"] + ")") try: layer = gs_catalogue.get_layer(layer_name) if layer is not None: gs_catalogue.delete(layer) gs_catalogue.save() gs_catalogue.publish_featuretype(layer_name, store, ds_cur["srs"]) published_count += 1 except Exception as e: feedback.reportError("Error: " + str(e), False) step += 1 feedback.setCurrentStep(step) gs_catalogue.reload() layers = gs_catalogue.get_layers(store) feedback.pushInfo("-" * 80) feedback.pushInfo("Published layers: " + str(published_count)) for layer in layers: feedback.pushInfo(layer.name + " is published!") feedback.pushInfo("-" * 80) results = {} outputs = {} return results
def calccosts(cf, layer_name, json_matrix): new_layer = "classroads_{}".format(int(1000000 * time.time())) # calccost function calculates recalcutes the visualisation caterogies engine = create_engine( "postgresql+psycopg2://" + cf.get("PostGis", "user") + ":" + cf.get("PostGis", "pass") + "@" + cf.get("PostGis", "host") + ":" + str(cf.get("PostGis", "port")) + "/" + cf.get("PostGis", "db"), strategy="threadlocal", pool_pre_ping=True, ) # try: # TODO make the ra2ce schema name as a parameter (input to the service) strSql = """create table temp.{new_layer} as select geom, societal_class, repair_class, 0 as visclass from public.{layer_name}_priority;""".format( new_layer=new_layer, layer_name=layer_name ) ressql = engine.execute(strSql) print("create temp table", ressql) # except exc.DBAPIError as e: # an exception is raised, Connection is invalidated. # if e.connection_invalidated: # logging.info("Connection was invalidated!") data = json.loads(json_matrix) values = data["values"] for societalIndex in range(len(values)): for repairIndex in range(len(values[societalIndex])): strSql = """update temp.{new_layer} set visclass = {val} where societal_class = {s} and repair_class = {r};""".format( new_layer=new_layer, val=values[societalIndex][repairIndex], s=societalIndex + 1, r=repairIndex + 1, ) ressql = engine.execute(strSql) ressql.close() # publish the layer in geoserver cat = Catalog( "{}/rest/".format(cf.get("GeoServer", "host")), username=cf.get("GeoServer", "user"), password=cf.get("GeoServer", "pass"), ) print("cat", cat) datastore = cat.get_store("temp") ft = cat.publish_featuretype( new_layer, datastore, native_crs="EPSG:4326", jdbc_virtual_table="" ) cat.save(ft) layer = cat.get_layer(new_layer) print("layer", layer) # check if shapefile is line or point # TODO check styles with Gerrit style = "ra2ce" # set style and save layer # layer._set_default_style(style) layer.default_style = style cat.save(layer) res = writeOutput(cf=cf, wmslayer="ra2ce-dr:{}".format(new_layer), defstyle=style) # time.sleep(2) return res
def delete(uid): """A function to delete a deviation map, all its tables, files and GeoServer layers. GET request: Renders and returns a site showing delete options. POST request: Gets delete options chosen by user and uses them to delete the chosen parts of the deviation map. To delete GeoServer layers the GeoServer configuration client library is used. """ uid = uid.encode("ISO-8859-1") if request.method == "POST": dm = DevMap.query.filter_by(uid=uid).first() if ( current_user.is_authenticated() and dm.owner == current_user or dm.owner == User.query.filter_by(username="******").first() ): if ( dm.wmsposdevlines or dm.wmsmaxdevgrid or dm.wmsabsdevgrid or dm.wmsmatchingrategrid or dm.wmsunmatchedref or dm.wmsunmatchedosm or dm.wmsmatchedref or dm.wmsmatchedosm or dm.wmsminlevenshtein or dm.wmsmaxlevenshtein ): cat = Catalog(gs_url + "rest") cat.username = gs_user cat.password = gs_password ws = None try: ws = cat.get_workspace(gs_workspace) except socket.error, e: detail = "GeoServer is not available. Make sure that it is running and the connection is ok." return render_template("error.html", err=e, detail=detail) st = cat.get_store(gs_store, ws) if "deletemaxdevgrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_maxdevgrid" if dm.wmsmaxdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmaxdevgrid = False if "deleteposdevlines" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_posdevlines" if dm.wmsposdevlines: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsposdevlines = False if "deleteabsdevgrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_absdevgrid" if dm.wmsabsdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) dm.wmsabsdevgrid = False if "deletematchingrategrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchingrategrid" if dm.wmsmatchingrategrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchingrategrid = False if "deleteunmatchedref" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_unmatchedref" if dm.wmsunmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.deleteunmatchedref = False if "deleteunmatchedosm" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_unmatchedosm" if dm.wmsunmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsunmatchedosm = False if "deletematchedref" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchedref" if dm.wmsmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchedref = False if "deletematchedosm" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchedosm" if dm.wmsmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchedosm = False if "deleteminlevenshtein" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_minlevenshtein" if dm.wmsminlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsminlevenshtein = False if "deletemaxlevenshtein" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_maxlevenshtein" if dm.wmsmaxlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmaxlevenshtein = False if "deleteall" in request.form: folder = secure_filename(uid) folder = os.path.join(app.config["UPLOAD_FOLDER"], folder) shutil.rmtree(folder, True) db.engine.execute("drop table if exists odf_" + uid + "_ref") db.engine.execute("drop table if exists odf_" + uid + "_ref_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_ref_splitted") db.engine.execute("drop table if exists odf_" + uid + "_found") db.engine.execute("drop table if exists odf_" + uid + "_ref_junctions") db.engine.execute("drop table if exists odf_" + uid + "_ref_points") db.engine.execute("drop table if exists odf_" + uid + "_ref_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_osm_splitted") db.engine.execute("drop table if exists odf_" + uid + "_osm_junctions") db.engine.execute("drop table if exists odf_" + uid + "_osm_points") db.engine.execute("drop table if exists odf_" + uid + "_osm_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_unmatchedref;") db.engine.execute("drop table if exists odf_" + uid + "_unmatchedosm;") db.engine.execute("drop table if exists odf_" + uid + "_minlevenshtein;") db.engine.execute("drop table if exists odf_" + uid + "_maxlevenshtein;") db.engine.execute("drop table if exists odf_" + uid + "_grid;") db.engine.execute("drop table if exists odf_" + uid + "_maxdevgrid;") db.engine.execute("drop table if exists odf_" + uid + "_matchingrategrid;") db.engine.execute("drop table if exists odf_" + uid + "_deviationlines") db.engine.execute("drop table if exists odf_" + uid + "_junction_deviationlines") if DEBUG: db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_junctions") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_points") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junction_devvec") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junctions") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_points") db.engine.execute("drop table if exists odf_" + uid + "_result") if "deleteall" not in request.form: db.session.add(dm) db.session.commit() return render_template("delete.html", uid=uid, dm=dm, error=None) else: db.session.delete(dm) db.session.commit() return redirect(url_for("basic.index")) else: return render_template("error.html", err="You are not allowed to delete this map!")
def setup_join(new_table_owner, table_name, layer_typename, table_attribute_name, layer_attribute_name): LOGGER.info('setup_join') """ Setup the Table Join in GeoNode """ assert isinstance(new_table_owner, User), "new_table_owner must be a User object" assert table_name is not None, "table_name cannot be None" assert layer_typename is not None, "layer_typename cannot be None" assert table_attribute_name is not None, "table_attribute_name cannot be None" assert layer_attribute_name is not None, "layer_attribute_name cannot be None" LOGGER.info('setup_join. Step (1): Retrieve the DataTable object') try: dt = DataTable.objects.get(table_name=table_name) except DataTable.DoesNotExist: err_msg = 'No DataTable object found for table_name "%s"' % table_name LOGGER.error(err_msg) return None, err_msg LOGGER.info('setup_join. Step (2): Retrieve the Layer object') try: layer = Layer.objects.get(typename=layer_typename) except Layer.DoesNotExist: err_msg = 'No Layer object found for layer_typename "%s"' % layer_typename LOGGER.error(err_msg) return None, err_msg LOGGER.info('setup_join. Step (3): Retrieve the DataTableAttribute object') try: table_attribute = DataTableAttribute.objects.get(\ datatable=dt, attribute=table_attribute_name) except DataTableAttribute.DoesNotExist: err_msg = 'No DataTableAttribute object found for table/attribute (%s/%s)' \ % (dt, table_attribute_name) LOGGER.error(err_msg) return None, err_msg LOGGER.info('setup_join. Step (4): Retrieve the LayerAttribute object') try: layer_attribute = LayerAttribute.objects.get(\ layer=layer, attribute=layer_attribute_name) except LayerAttribute.DoesNotExist: err_msg = 'No LayerAttribute object found for layer/attribute (%s/%s)'\ % (layer, layer_attribute_name) LOGGER.error(err_msg) return None, err_msg LOGGER.info('setup_join. Step (5): Build SQL statement to create view') layer_name = layer.typename.split(':')[1] # ------------------------------------------------------------------ # (5) Check if the layer and the table are in the same store (database) # ------------------------------------------------------------------ if layer.store != dt.tablespace: err_msg = 'layer (%s) and tablespace (%s) must be in the same database.'\ % (layer.store, dt.tablespace) LOGGER.error(err_msg) return None, err_msg # ------------------------------------------------------------------ # (5a) Check if the join columns compatible # ------------------------------------------------------------------ column_checker = ColumnChecker(\ layer_name, layer_attribute.attribute, dt.table_name, table_attribute.attribute) (are_cols_compatible, err_msg) = column_checker.are_join_columns_compatible() if not are_cols_compatible: # Doesn't look good, return an error message return None, err_msg # ------------------------------------------------------------------ # (5b) Create SQL statement for the tablejoin # ------------------------------------------------------------------ view_name = get_unique_viewname(dt.table_name, layer_name) # SQL to create the view view_sql = ('CREATE VIEW {0}' ' AS SELECT {1}.{2}, {3}.*' ' FROM {1} INNER JOIN {3}' ' ON {1}."{4}" = {3}."{5}";'.format(\ view_name, # 0 layer_name, # 1 THE_GEOM_LAYER_COLUMN, # 2 dt.table_name, # 3 layer_attribute.attribute, # 4 table_attribute.attribute)) # 5 # Materialized view for next version of Postgres """view_sql = ('create materialized view {0} as' ' select {1}.the_geom, {2}.*' ' from {1} inner join {2}' ' on {1}."{3}" = {2}."{4}";').format(\ view_name, layer_name, dt.table_name, layer_attribute.attribute, table_attribute.attribute) """ LOGGER.info('setup_join. Step (6): Retrieve stats') # ------------------------------------------------------------------ # Retrieve stats # ------------------------------------------------------------------ matched_count_sql = ('select count({0}) from {1} where {1}.{0}' ' in (select "{2}" from {3});').format(\ table_attribute.attribute, dt.table_name, layer_attribute.attribute, layer_name) unmatched_count_sql = ('select count({0}) from {1} where {1}.{0}' ' not in (select "{2}" from {3});').format(\ table_attribute.attribute, dt.table_name, layer_attribute.attribute, layer_name) unmatched_list_sql = ('select {0} from {1} where {1}.{0}' ' not in (select "{2}" from {3}) limit 500;').format(\ table_attribute.attribute, dt.table_name, layer_attribute.attribute, layer_name) # ------------------------------------------------------------------ # Create a TableJoin object # ------------------------------------------------------------------ LOGGER.info('setup_join. Step (7): Create a TableJoin object') tj, created = TableJoin.objects.get_or_create(\ source_layer=layer, datatable=dt, table_attribute=table_attribute, layer_attribute=layer_attribute, view_name=view_name, view_sql=view_sql) tj.save() msgt('table join created! :) %s' % tj.id) # ------------------------------------------------------------------ # Create the View (and double view) # ------------------------------------------------------------------ LOGGER.info('setup_join. Step (8): Create the View (and double view)') # Convenience method to drop a view # drop_view_by_name(view_name) try: conn = psycopg2.connect(get_datastore_connection_string()) cur = conn.cursor() # Create the new view # msg('view_sql: %s' % view_sql) cur.execute(view_sql) #cur.execute(double_view_sql) # For later version of postgres # Record the counts for matched records and # add unmatched records to the TableJoin object # Unmatched count cur.execute(matched_count_sql) tj.matched_records_count = cur.fetchone()[0] # Matched count cur.execute(unmatched_count_sql) tj.unmatched_records_count = int(cur.fetchone()[0]) # Unmatched records list if tj.unmatched_records_count > 0: cur.execute(unmatched_list_sql) tj.unmatched_records_list = ",".join( ['%s' % r[0] for r in cur.fetchall()]) conn.commit() cur.close() # If no records match, then delete the TableJoin # if tj.matched_records_count == 0: # Delete the table join tj.delete() # Create an error message, log it, and send it back err_msg = ('No records matched. Make sure that you chose' ' the correct column and that the chosen layer' ' is in the same geographic area.') LOGGER.error(err_msg) return None, err_msg except Exception as ex_obj: tj.delete() # If needed for debugging, don't delete the table join traceback.print_exc(sys.exc_info()) err_msg = "Error Joining table %s to layer %s: %s" % (\ table_name, layer_typename, str(ex_obj[0])) LOGGER.error(err_msg) if err_msg.find('You might need to add explicit type casts.') > -1: user_msg = "The chosen column is a different data type than the one expected." else: user_msg = err_msg return None, user_msg finally: conn.close() #-------------------------------------------------- # Create the Layer in GeoServer from the view #-------------------------------------------------- LOGGER.info( 'setup_join. Step (9): Create the Layer in GeoServer from the view') try: LOGGER.info('setup_join. Step (9a): Find the datastore') #---------------------------- # Find the datastore #---------------------------- cat = Catalog( settings.GEOSERVER_BASE_URL + "rest", settings.GEOSERVER_CREDENTIALS[0], settings.GEOSERVER_CREDENTIALS[1]) # "admin", "geoserver") workspace = cat.get_workspace('geonode') ds_list = cat.get_xml(workspace.datastore_url) datastores = [ datastore_from_index(cat, workspace, n) for n in ds_list.findall("dataStore") ] ds = None # Iterate through datastores # for datastore in datastores: #msg ('datastore name:', datastore.name) if datastore.name == layer.store: ds = datastore if ds is None: tj.delete() err_msg = "Datastore name not found: '%s' (join)" % settings.DB_DATASTORE_NAME LOGGER.error(str(ds)) return None, err_msg # Publish the feature # LOGGER.info('setup_join. Step (9b): Publish the feature type') ft = cat.publish_featuretype(view_name, ds, layer.srs, srs=layer.srs) #ft = cat.publish_featuretype(double_view_name, ds, layer.srs, srs=layer.srs) LOGGER.info('setup_join. Step (9c): Save the feature type') cat.save(ft) except Exception as e: tj.delete() traceback.print_exc(sys.exc_info()) err_msg = "Error creating GeoServer layer for %s: %s" % (view_name, str(e)) LOGGER.error(err_msg) return None, err_msg # ------------------------------------------------------ # Set the Layer's default Style # ------------------------------------------------------ sld_success, err_msg = set_style_for_new_join_layer(cat, ft, layer) if not sld_success: return None, err_msg # ------------------------------------------------------------------ # Create the Layer in GeoNode from the GeoServer Layer # ------------------------------------------------------------------ LOGGER.info( 'setup_join. Step (10): Create the Layer in GeoNode from the GeoServer Layer' ) try: layer_params = { "workspace": workspace.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), ft.name.encode('utf-8')), "title": dt.title or 'No title provided', "abstract": dt.abstract or 'No abstract provided', "uuid": str(uuid.uuid4()), "owner": new_table_owner, #"bbox_x0": Decimal(ft.latlon_bbox[0]), #"bbox_x1": Decimal(ft.latlon_bbox[1]), #"bbox_y0": Decimal(ft.latlon_bbox[2]), #"bbox_y1": Decimal(ft.latlon_bbox[3]) } layer, created = Layer.objects.get_or_create(name=view_name, defaults=layer_params) # Set default permissions (public) layer.set_default_permissions() #set_attributes(layer, overwrite=True) tj.join_layer = layer tj.save() except Exception as e: tj.delete() traceback.print_exc(sys.exc_info()) err_msg = "Error creating GeoNode layer for %s: %s" % (view_name, str(e)) LOGGER.error(err_msg) return None, err_msg # ------------------------------------------------------------------ # Create LayerAttributes for the new Layer (not done in GeoNode 2.x) # ------------------------------------------------------------------ LOGGER.info( 'setup_join. Step (11): Create Layer Attributes from the Datatable') (attributes_created, err_msg) = create_layer_attributes_from_datatable(dt, layer) if not attributes_created: LOGGER.error(err_msg) tj.delete() # Delete the table join object return None, "Sorry there was an error creating the Datatable (s11)" return tj, ""
def export(uid): """This function is used to show and handle the export options and uses the GeoServer REST API to export results. This function uses the GeoServer configuration client library by boundlessgeo, see: https://github.com/boundlessgeo/gsconfig To export the results, the GeoServer application has to be running and should be correctly set up to have access to the database and the styles should already be imported, see geoserver styles folder. GET request: Renders and returns a site showing export options. POST request: Gets the export options from the user and exports the results if they are defined in the options. The export is made by using GeoServer configuration client library. After the export, the newly created layers should be visible in the GeoServer web interface and the WM(T)S links should also work. These links can then be used to display the WM(T)S layers in JOSM or a gis. Remarks: This function will be redesigned in future versions """ uid = uid.encode("ISO-8859-1") if request.method == "POST": dm = DevMap.query.filter_by(uid=uid).first() if dm.owner == current_user or dm.owner == User.query.filter_by(username="******").first(): title = request.form["title"] listedmap = False if "listed" in request.form: listedmap = True dm.title = title dm.listed = listedmap cat = Catalog(gs_url + "rest") cat.username = gs_user cat.password = gs_password ws = None try: ws = cat.get_workspace(gs_workspace) except socket.error, e: db.session.add(dm) db.session.commit() return render_template("export.html", uid=uid, error=e, dm=dm) st = cat.get_store(gs_store, ws) if "maxdevgrid" in request.form: feattype = "odf_" + uid + "_maxdevgrid" if dm.wmsmaxdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":maxdevgrid") cat.save(l) dm.wmsmaxdevgrid = True else: dm.wmsmaxdevgrid = True if "posdevlines" in request.form: feattype = "odf_" + uid + "_posdevlines" if dm.wmsposdevlines: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":posdevlines") cat.save(l) dm.wmsposdevlines = True else: dm.wmsposdevlines = False if "absdevgrid" in request.form: feattype = "odf_" + uid + "_absdevgrid" if dm.wmsabsdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":absdevgrid") cat.save(l) dm.wmsabsdevgrid = True else: dm.wmsabsdevgrid = False if "matchingrategrid" in request.form: feattype = "odf_" + uid + "_matchingrategrid" if dm.wmsmatchingrategrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":matchingrategrid") cat.save(l) dm.wmsmatchingrategrid = True else: dm.wmsmatchingrategrid = False if "unmatchedref" in request.form: feattype = "odf_" + uid + "_unmatchedref" if dm.wmsunmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.unmatchedref = True else: dm.unmatchedref = False if "unmatchedosm" in request.form: feattype = "odf_" + uid + "_unmatchedosm" if dm.unmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":OSMLines") cat.save(l) dm.wmsunmatchedosm = True else: dm.wmsunmatchedosm = False if "matchedref" in request.form: feattype = "odf_" + uid + "_matchedref" if dm.wmsmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsmatchedref = True else: dm.wmsmatchedref = False if "matchedosm" in request.form: feattype = "odf_" + uid + "_matchedosm" if dm.wmsmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":OSMLines") cat.save(l) dm.wmsmatchedosm = True else: dm.wmsmatchedosm = False if "minlevenshtein" in request.form: feattype = "odf_" + uid + "_minlevenshtein" if dm.wmsminlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsminlevenshtein = True else: dm.wmsminlevenshtein = False if "maxlevenshtein" in request.form: feattype = "odf_" + uid + "_maxlevenshtein" if dm.wmsmaxlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsmaxlevenshtein = True else: dm.wmsmaxlevenshtein = False db.session.add(dm) db.session.commit() return render_template("finished.html", uid=uid) else: return render_template("export.html", uid=uid, error="No User", dm=dm)
# Check if remote datastore exists on local system for dataStoreObj in dataStoresObj: if dataStoreObj.name == remoteDSName: localDSObj = dataStoreObj print "Datastore " + localDSObj.name + " already exists" if not localDSObj: localDSObj = cat.create_datastore(remoteDSName, localWSName) localDSObj.connection_parameters.update(host=remoteHost, port=remotePort, \ database=remoteDatabase, user=remoteUser, passwd=remotePassword, dbtype=remoteDBType) cat.save(localDSObj) print "Datastore " + localDSObj.name + " created" # Check if remote layer already exists on local system localLayersObj = cat.get_layers() for localLayerObj in localLayersObj: if localLayerObj.resource.name == remoteLayerName: localLayerName = remoteLayerName print "Layer " + remoteLayerName + " already published" if not localLayerName: # Publish remote layer featureType = cat.publish_featuretype(remoteLayerName, localDSObj, WGS84, srs=WGS84) print "Published layer " + remoteLayerName # Update layers in GeoSHAPE os.system("sudo geoshape-config updatelayers")
class GeoserverHelper: def __init__(self, geoserverUrl = "", geoserverUserName = "", geoserverPW = "", geoserverWorkSpace = "", postgreIP = "", postgreUserName = "", postgrePW = ""): """use the constructor given arguments if used""" self.geoserverUrl = geoserverUrl if geoserverUrl != "" else GEOSERVERURL self.geoserverUserName = geoserverUserName if geoserverUserName != "" else GEOSERVERUSERNAME self.geoserverPW = geoserverPW if geoserverPW != "" else GEOSERVERPW self.geoserverWorkSpace = geoserverWorkSpace if geoserverWorkSpace != "" else "crc" self.postgreIP = postgreIP if postgreIP != "" else POSTGREIP self.postgreUserName = postgreUserName if postgreUserName != "" else POSTGREUSERNAME self.postgrePW = postgrePW if postgrePW != "" else POSTGREPW if self.geoserverUrl[-1] != '/': raise Exception("GeoserverUrl must end with a slash ('/')") self.catalog = Catalog(self.geoserverUrl+"rest/") self.catalog.http.add_credentials(self.geoserverUserName,self.geoserverPW) try: workspaces = self.catalog.get_workspaces() except: e = sys.exc_info()[0] print e self.cWorkSpace = self.catalog.get_workspace(self.geoserverWorkSpace) def getLayers(self): return self.cWorkSpace.catalog.get_layers() def insertShapeIntoPostGis(self, shapeFile, databaseName, tableName, encoding=3857): '''returns the returnCode of the execution of the insert script, e.g: helper.insertShapeIntoPostGis('/home/c815/gsTest/test.shp','crc','testingHelper2')''' if not os.path.isfile(shapeFile): print "Shape file not found" return -1 cmds = "PGPASSWORD={pgPW} ./createWSFTFromSHP.sh -s {shapeFileF} -d {databaseNameF} -t {tableNameF} -u {postgreUsername} -i {postgreIP}".format(pgPW=self.postgrePW, shapeFileF=shapeFile,databaseNameF=databaseName, tableNameF=tableName, postgreUsername=self.postgreUserName, postgreIP=self.postgreIP) return subprocess.call(cmds, shell=True) def uploadShapeFile(self, shapeFile, storeName): shpPlusSidcars = geoserver.util.shapefile_and_friends(shapeFile[:-3]) shpPlusSidcars self.ft = self.catalog.create_featurestore(storeName, shpPlusSidcars, self.cWorkSpace) def getStyles(self): return self.catalog.get_styles() def uploadStyleFile(self, sldFile, styleName, overWrite, workSpace = None): f = open(sldFile,'r') styleSrc = f.read() uploadStyle(styleSrc, styleName, overWrite, workSpace) f.close() def uploadStyle(self, sldSrc, styleName, overWrite, workSpace = None): self.catalog.create_style(styleName,sldSrc, overWrite, workSpace) def publishPostGISLayer(self,postGISLayerName, storeName, crs='EPSG:3857'): '''cat.publish_featuretype('testingstuff',crcStore,native_crs='EPSG:3857')''' store = self.catalog.get_store(storeName) if store != None: self.catalog.publish_featuretype(postGISLayerName, store, crs) def setDefaultStyleForLayer(self, layerName, styleName): l = self.catalog.get_layer(layerName) sNames = [ i.name for i in self.getStyles() ] if styleName not in sNames: split = styleName.split(':') if len(split) == 2: workSpace = styleName.split(':')[0] newStyleName = styleName.split(':')[1] else: return -1 style = self.catalog.get_style(newStyleName, workSpace) if style is None: return -1 if l != None: l._set_default_style(styleName) self.catalog.save(l) return 0 def createPostGISDataStore(self, storeName, postGisPassword, postGisUser, postGisHost, postGisDatabase, workSpace = None): #check if connection parameter are valid try: conn = psycopg2.connect("dbname='{dbName}' user='******' host='{Host}' password='******'".format(dbName=postGisDatabase, dbUser=postGisUser, Host=postGisHost, password=postGisPassword)) except: return False w = self.catalog.create_datastore(storeName, workSpace) template = Template("""{'validate connections': 'true', 'port': '5432', 'Support on the fly geometry simplification': 'true', 'create database': 'false', 'dbtype': 'postgis', 'Connection timeout': '20', 'namespace': 'http://www.crcproject.com', 'Max connection idle time': '300', 'Expose primary keys': 'false', 'min connections': '1', 'Max open prepared statements':'50', 'passwd': '$passwd', 'encode functions': 'false', 'max connections': '10', 'Evictor tests per run': '3', 'Loose bbox': 'true', 'Evictor run periodicity': '300', 'Estimated extends': 'true', 'database': '$database', 'fetch size': '1000', 'Test while idle': 'true', 'host': '$host', 'preparedStatements': 'false', 'schema': 'public', 'user': '******'}""") dic = ast.literal_eval(template.substitute( passwd=postGisPassword, user=postGisUser, host=postGisHost, database=postGisDatabase)) #'passwd': 'crypt1:Bsaz2AUI8T+6Pj43krA7kg==', #'user': '******'} #'database': 'crc', #'host': 'localhost', w.connection_parameters = dic self.catalog.save(w) return True
#!/usr/bin/env python import sys from geoserver.catalog import Catalog workspace = sys.argv[1] datastore = sys.argv[1] shpname = sys.argv[2].lower() epsg = 'EPSG:' + sys.argv[3] catalog = Catalog("http://localhost:2012/geoserver/rest", username='******', password='******') ws = catalog.get_workspace(workspace) ds = catalog.get_store(datastore, ws) publish = catalog.publish_featuretype(shpname, ds, epsg) catalog.save(publish)
def logingeoserver(): form = LoginFormGeoserver() mytoken = session.get('mytoken') projectid = session.get('project') categoryid = session.get('category') segmentos = requests.get( 'https://raw.githubusercontent.com/guilhermeiablo/inventsys2infoambiente/master/dados/ERS_segmentos_rodoviarios.geojson' ) tabelagerada = session.get('tabelagerada') if form.validate_on_submit(): urlgeoserver = str(form.urlgeoserver.data) usrgeoserver = str(form.usrgeoserver.data) pwdgeoserver = str(form.pwdgeoserver.data) workspace = str(form.workspace.data) datastore = str(form.datastore.data) cat = Catalog(urlgeoserver, username=usrgeoserver, password=pwdgeoserver) cite = cat.get_workspace(workspace) ds = cat.get_store(datastore, workspace) ds.connection_parameters.update(host=session.get('hostinput'), port='5432', database=session.get('dbnameinput'), user=session.get('userinput'), passwd=session.get('pwdgeoserver'), dbtype='postgis', schema='public') cat.save(ds) if cat: flash(f'Dados enviados com sucesso para {form.workspace.data}!', 'success') for feature in segmentos.json()['features']: if projectid == '10762': nomedolayer = str( str(feature['properties']['nome']) + '_PMF_' + str(tabelagerada)) else: nomedolayer = str( str(feature['properties']['nome']) + '_' + str(tabelagerada)) while True: try: ft = cat.publish_featuretype(nomedolayer, ds, 'EPSG:4326', srs='EPSG:4326') except: # for handle unknown exception # define your parameters urldatastore = urlgeoserver + 'reset' headersdatastore = {'Content-Type': 'text/xml'} authdatastore = (usrgeoserver, pwdgeoserver) r = requests.post(urldatastore, headers=headersdatastore, auth=authdatastore) break return redirect( url_for('logininfoambiente', mytoken=session['mytoken'], project=session['project'], category=session['category'])) else: flash('Erro ao conectar ao servidor. Tente novamente.', 'danger') return render_template("logingeoserver.html", title='LoginGeoserver', form=form, mytoken=mytoken, project=session['project'], category=session['category'])
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) self.cat.delete(ds) def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) def testCoverageStoreCreate(self): ds = self.cat.create_coveragestore2("coverage_gsconfig") ds.data_url = "file:data/mytiff.tiff" self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: new_layer = self.cat.create_wmslayer(wmstest, wmsstore, layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf) ) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") # TODO: Uploading WorldImage file no longer works??? # ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) # self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) # self.assertRaises( # ConflictingDataError, # lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) # ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf) ) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff) ) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = "Not the original attribution" # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles)
def add_to_geoserver(self): # Inspired (copied :) ) from https://groups.google.com/forum/#!msg/geonode-users/R-u57r8aECw/AuEpydZayfIJ # TODO : check license # We connect to the catalog gsUrl = settings.OGC_SERVER['default']['LOCATION'] + "rest" gsUser = settings.OGC_SERVER['default']['USER'] gsPassword = settings.OGC_SERVER['default']['PASSWORD'] cat = Catalog(gsUrl, gsUser, gsPassword) if cat is None: raise Exception('unable to instantiate geoserver catalog') # We get the workspace ws = cat.get_workspace(settings.DEFAULT_WORKSPACE) if ws is None: raise Exception('workspace %s not found in geoserver' % settings.DEFAULT_WORKSPACE) # We get or create the datastore store = cat.get_store(self.datastore_name, ws) if store is None: store = cat.create_datastore(self.datastore_name, ws) store.connection_parameters.update(host="postgres", port="5432", database="postgres", user="******", passwd="postgres", schema='offline_osm', dbtype="postgis") cat.save(store) if store is None: raise Exception('datastore %s not found in geoserver' % self.datastore_name) # We get or create each layer then register it into geonode for layer in Command.layers: layername = layer["name"] self.stdout.write('Adding {} to geoserver...'.format(layername)) layer_exists = (not cat.get_layer(layername) is None) if not layer_exists or not self.options['no_overwrite']: self.stdout.write( ' layer does not exists or no_overwrite unset, we add...') ft = cat.publish_featuretype(layername, store, 'EPSG:4326', srs='EPSG:4326') if ft is None: raise Exception('unable to publish layer %s' % layername) ft.title = 'OpenStreetMap Offline - ' + layername.split( '_')[-1] ft.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.' cat.save(ft) self.stdout.write( ' adding the style for {}...'.format(layername)) # We get or create the workspace style_path = os.path.join(os.path.dirname(__file__), '..', '..', 'styles', layer["style_name"]) if not os.path.exists(style_path): self.stdout.write( ' The file {} does not exist. No style will be applied for {}.' .format(style_path, layername)) else: cat.create_style(layer["style_name"], open(style_path, 'r').read(), overwrite=True, workspace=settings.DEFAULT_WORKSPACE, raw=True) style = cat.get_style(layer["style_name"], ws) if style is None: raise Exception('style not found (%s)' % (layer["style_name"])) publishing = cat.get_layer(layername) if publishing is None: raise Exception('layer not found (%s)' % layerName) publishing.default_style = style cat.save(publishing) self.stdout.write( ' registering {} into geonode...'.format(layername)) resource = cat.get_resource(layername, store, ws) if resource is None: raise Exception('resource not found (%s)' % layername) layer, created = Layer.objects.get_or_create(name=layername) layer.workspace = ws.name layer.store = store.name layer.storeType = store.resource_type layer.typename = "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')) layer.title = resource.title layer.abstract = resource.abstract layer.temporal_extent_start = self.import_timestamp layer.temporal_extent_end = self.import_timestamp try: layer.save() except TypeError as e: # We ignore a specific error in Geonode resourcebase_post_save when settings.SITEURL has no host (relative URL) self.stdout.write( 'TODO : report Geonode exception in resourcebase_post_save when settings.SITEURL has no host (relative URL) : ' + str(e)) pass if created: layer.set_default_permissions() else: self.stdout.write(' layer already exists, we skip.') # We get or create the laygroup self.stdout.write('Adding layergroup to geoserver...') layername = 'offline_osm' layergroup_exists = (not cat.get_layergroup(layername) is None) if not layer_exists or not self.options['no_overwrite']: self.stdout.write( ' layer does not exists or no_overwrite unset, we add...') layergroup = cat.get_layergroup( layername, workspace=settings.DEFAULT_WORKSPACE) if layergroup is None: layergroup = cat.create_layergroup( layername, layers=layernames, workspace=settings.DEFAULT_WORKSPACE) if layergroup is None: raise Exception('unable to publish layer %s' % layername) layergroup.title = 'OpenStreetMap Offline' layergroup.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.' cat.save(layergroup) else: self.stdout.write(' layergroup already exists, we skip.') # TODO : can we add layergroups to Geonode ? self.stdout.write( "laygroup won't be added to geonode (not supported yet)")
# Criando um workspace ws = cat.create_workspace('gsc_test', 'http://localhost:8082/geoserver/testWs') # Criando um store ds = cat.create_datastore('bdg_gsc', 'gsc_test') ds.connection_parameters.update(host='localhost', port='5432', database='bdg_gsc', user='******', passwd='user', dbtype='postgis', schema='public') cat.save(ds) # Adicionando layers ft = cat.publish_featuretype('mun_pr', ds, 'EPSG:4674', srs='EPSG:4674') ft1 = cat.publish_featuretype('escolas_pr', ds, 'EPSG:4674', srs='EPSG:4674') # Adicionando styles - nao adiciona sld completo #with open("/home/user/gsc_style.sld") as f: # cat.create_style('gsc_style', f.read()) # Atribuindo o estilo a camada # SLD adicionado no geoserver style1 = cat.get_style("gsc_test:gsc_style_mun") layer1 = cat.get_layer("mun_pr") layer1.default_style = style1 cat.save(layer1) style2 = cat.get_style("gsc_test:gsc_style_esc") layer2 = cat.get_layer("escolas_pr")
def main(options): #connect to geoserver cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw) #create datrastore for URB schema ws = cat.create_workspace(options.alias, 'imio.be') ds = cat.create_datastore(options.alias, ws) ds.connection_parameters.update(host=options.urbanUrl, port="5432", database=options.database, user="******", passwd=options.ropw, dbtype="postgis") cat.save(ds) ds = cat.get_store(options.alias) #config object urb = { "capa": "Parcelles", "toli": "cadastre_ln_toponymiques", "canu": "cadastre_pt_num", "cabu": "Batiments", "gept": "cadastre_points_generaux", "gepn": "cadastre_pol_gen", "inpt": "point", "geli": "cadastre_ln_generales", "inli": "cadastre_ln_informations", "topt": "point", } #connect to tables and create layers and correct urban styles for table in urb: style = urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') ft.default_style = style cat.save(ft) resource = ft.resource resource.title = options.alias + "_" + table resource.save() layer, created = Layer.objects.get_or_create( name=layerName, defaults={ "workspace": ws.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', #"owner": owner, "uuid": str(uuid.uuid4()), "bbox_x0": Decimal(resource.latlon_bbox[0]), "bbox_x1": Decimal(resource.latlon_bbox[1]), "bbox_y0": Decimal(resource.latlon_bbox[2]), "bbox_y1": Decimal(resource.latlon_bbox[3]) }) set_attributes(layer, overwrite=True) if created: layer.set_default_permissions()
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assertIsNone(self.cat.get_layer('import')) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assertIsNotNone(self.cat.get_layer('import')) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except Exception: pass def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store( ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test2") self.cat.add_data_to_store( ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) store = self.cat.get_store("gsconfig_import_test2") geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set( ['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEqual(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEqual(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEqual(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501 ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501 wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(workspace=wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.items(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assertIsNotNone(self.cat.get_resource("states_test", workspace=sf)) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore( "states_test", shapefile_plus_sidecars, sf)) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore( "states_raster_test", shapefile_plus_sidecars, sf)) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assertIsNone(self.cat.get_layer("states_test")) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") self.cat.create_coveragestore("Pk50095", tiffdata, sf) self.assertIsNotNone(self.cat.get_resource("Pk50095", workspace=sf)) self.assertRaises( ConflictingDataError, lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)) self.assertRaises( UploadError, lambda: self.cat.create_featurestore( "Pk50095_vector", tiffdata, sf)) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)) self.cat.create_coveragestore_external_geotiff( "Pk50095_ext", 'file:test/data/Pk50095.tif', sf) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = next( (s for s in lyr.styles if s.name == "pophatch")) lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("fred", fred_sld.read()) fred = self.cat.get_style("fred") self.assertIsNotNone(fred) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes with open("test/ted.sld") as ted_sld: self.cat.create_style("fred", ted_sld.read(), overwrite=True) fred = self.cat.get_style("fred") self.assertIsNotNone(fred) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assertIsNone(self.cat.get_style("fred")) # attempt creating new style with open("test/fred.sld") as fred_sld: self.cat.create_style("fred", fred_sld.read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assertIsNotNone(f) self.assertEqual(f.name, fred.name) # compare count after upload self.assertEqual(count + 1, len(self.cat.get_styles())) # attempt creating a new style without "title" with open("test/notitle.sld") as notitle_sld: self.cat.create_style("notitle", notitle_sld.read()) notitle = self.cat.get_style("notitle") self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("jed", fred_sld.read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assertIsNone(jed) jed = self.cat.get_style("jed", workspace="topp") self.assertIsNotNone(jed) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assertIsNotNone(jed) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes with open("test/ted.sld") as ted_sld: self.cat.create_style("jed", ted_sld.read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertIsNotNone(jed) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assertIsNone(self.cat.get_style("jed", workspace="topp")) # attempt creating new style with open("test/fred.sld") as fred_sld: self.cat.create_style("jed", fred_sld.read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assertIsNotNone(f) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("ned", fred_sld.read(), overwrite=True, workspace="topp") with open("test/fred.sld") as ted_sld: self.cat.create_style("zed", ted_sld.read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assertIsNotNone(ned) self.assertIsNotNone(zed) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() if lyr.default_style is not None: self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assertIsNone(ws) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assertTrue(states.enabled) states.enabled = False self.assertFalse(states.enabled) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assertFalse(states.enabled) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assertTrue(states.enabled) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' with open('test/data/mosaic/cea.zip', 'rb') as data: self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assertIsNotNone(resource) # delete granule from mosaic coverage = name store = self.cat.get_store(name) granules = self.cat.list_granules(coverage, store) self.assertEqual(1, len(granules['features'])) granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id) granules = self.cat.list_granules(coverage, store) self.assertEqual(0, len(granules['features'])) """ testing external Image mosaic creation """ name = 'cea_mosaic_external' path = 'test/data/mosaic/external' self.cat.create_imagemosaic(name, path, workspace='topp') self.cat._cache.clear() resource = self.cat.get_layer("external").resource self.assertIsNotNone(resource) # add granule to mosaic granule_path = 'test/data/mosaic/granules/cea_20150102.tif' self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(2, len(granules['features'])) # add external granule to mosaic granule_path = os.path.join( os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(3, len(granules['features'])) # Delete store store = self.cat.get_store(name) self.cat.delete(store, purge=True, recurse=True) self.cat._cache.clear() def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends( os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer( 'boxes_with_end_date').resource # noqa: E501 # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
class GeoserverHelper: def __init__(self, geoserverUrl="", geoserverUserName="", geoserverPW="", geoserverWorkSpace="", postgreIP="", postgreUserName="", postgrePW=""): """use the constructor given arguments if used""" self.geoserverUrl = geoserverUrl if geoserverUrl != "" else GEOSERVERURL self.geoserverUserName = geoserverUserName if geoserverUserName != "" else GEOSERVERUSERNAME self.geoserverPW = geoserverPW if geoserverPW != "" else GEOSERVERPW self.geoserverWorkSpace = geoserverWorkSpace if geoserverWorkSpace != "" else "crc" self.postgreIP = postgreIP if postgreIP != "" else POSTGREIP self.postgreUserName = postgreUserName if postgreUserName != "" else POSTGREUSERNAME self.postgrePW = postgrePW if postgrePW != "" else POSTGREPW if self.geoserverUrl[-1] != '/': raise Exception("GeoserverUrl must end with a slash ('/')") self.catalog = Catalog(self.geoserverUrl + "rest/") self.catalog.http.add_credentials(self.geoserverUserName, self.geoserverPW) try: workspaces = self.catalog.get_workspaces() except: e = sys.exc_info()[0] print e self.cWorkSpace = self.catalog.get_workspace(self.geoserverWorkSpace) def getLayers(self): return self.cWorkSpace.catalog.get_layers() def insertShapeIntoPostGis(self, shapeFile, databaseName, tableName, encoding=3857): '''returns the returnCode of the execution of the insert script, e.g: helper.insertShapeIntoPostGis('/home/c815/gsTest/test.shp','crc','testingHelper2')''' if not os.path.isfile(shapeFile): print "Shape file not found" return -1 cmds = "PGPASSWORD={pgPW} ./createWSFTFromSHP.sh -s {shapeFileF} -d {databaseNameF} -t {tableNameF} -u {postgreUsername} -i {postgreIP}".format( pgPW=self.postgrePW, shapeFileF=shapeFile, databaseNameF=databaseName, tableNameF=tableName, postgreUsername=self.postgreUserName, postgreIP=self.postgreIP) return subprocess.call(cmds, shell=True) def uploadShapeFile(self, shapeFile, storeName): shpPlusSidcars = geoserver.util.shapefile_and_friends(shapeFile[:-3]) shpPlusSidcars self.ft = self.catalog.create_featurestore(storeName, shpPlusSidcars, self.cWorkSpace) def getStyles(self): return self.catalog.get_styles() def uploadStyleFile(self, sldFile, styleName, overWrite, workSpace=None): f = open(sldFile, 'r') styleSrc = f.read() uploadStyle(styleSrc, styleName, overWrite, workSpace) f.close() def uploadStyle(self, sldSrc, styleName, overWrite, workSpace=None): self.catalog.create_style(styleName, sldSrc, overWrite, workSpace) def publishPostGISLayer(self, postGISLayerName, storeName, crs='EPSG:3857'): '''cat.publish_featuretype('testingstuff',crcStore,native_crs='EPSG:3857')''' store = self.catalog.get_store(storeName) if store != None: self.catalog.publish_featuretype(postGISLayerName, store, crs) def setDefaultStyleForLayer(self, layerName, styleName): l = self.catalog.get_layer(layerName) sNames = [i.name for i in self.getStyles()] if styleName not in sNames: split = styleName.split(':') if len(split) == 2: workSpace = styleName.split(':')[0] newStyleName = styleName.split(':')[1] else: return -1 style = self.catalog.get_style(newStyleName, workSpace) if style is None: return -1 if l != None: l._set_default_style(styleName) self.catalog.save(l) return 0 def createPostGISDataStore(self, storeName, postGisPassword, postGisUser, postGisHost, postGisDatabase, workSpace=None): #check if connection parameter are valid try: conn = psycopg2.connect( "dbname='{dbName}' user='******' host='{Host}' password='******'" .format(dbName=postGisDatabase, dbUser=postGisUser, Host=postGisHost, password=postGisPassword)) except: return False w = self.catalog.create_datastore(storeName, workSpace) template = Template( """{'validate connections': 'true', 'port': '5432', 'Support on the fly geometry simplification': 'true', 'create database': 'false', 'dbtype': 'postgis', 'Connection timeout': '20', 'namespace': 'http://www.crcproject.com', 'Max connection idle time': '300', 'Expose primary keys': 'false', 'min connections': '1', 'Max open prepared statements':'50', 'passwd': '$passwd', 'encode functions': 'false', 'max connections': '10', 'Evictor tests per run': '3', 'Loose bbox': 'true', 'Evictor run periodicity': '300', 'Estimated extends': 'true', 'database': '$database', 'fetch size': '1000', 'Test while idle': 'true', 'host': '$host', 'preparedStatements': 'false', 'schema': 'public', 'user': '******'}""") dic = ast.literal_eval( template.substitute(passwd=postGisPassword, user=postGisUser, host=postGisHost, database=postGisDatabase)) #'passwd': 'crypt1:Bsaz2AUI8T+6Pj43krA7kg==', #'user': '******'} #'database': 'crc', #'host': 'localhost', w.connection_parameters = dic self.catalog.save(w) return True
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except: pass def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test2") self.cat.add_data_to_store(ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) store = self.cat.get_store("gsconfig_import_test2") geom = JDBCVirtualTableGeometry('the_geom','MultiPolygon','4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf) ) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf) ) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff) ) ft_ext = self.cat.create_coveragestore_external_geotiff("Pk50095_ext", 'file:test/data/Pk50095.tif', sf) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) # compare count after upload self.assertEqual(count +1, len(self.cat.get_styles())) # attempt creating a new style without "title" self.cat.create_style("notitle", open("test/notitle.sld").read()) notitle = self.cat.get_style("notitle") self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = self.cat.get_store(name) granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id) def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends(os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer('boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time':timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time' : timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time' : timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
workspace = "geonode" native_crs = 'EPSG:32717' for schema in schemas: print '*' * 50 ds = cat.create_datastore(schema, workspace) ds.connection_parameters.update(host="10.10.201.4", port="5432", database="gis_db", user="******", passwd="test", dbtype="postgis", schema=schema) try: cat.save(ds) except: print schema + ' datastore already exist' #ds = cat.get_store(schema) finally: all_layers = ds.get_resources(available=True) i = 0 for layer in all_layers: #print layer try: ft = cat.publish_featuretype(layer, ds, native_crs) i += 1 except: print 'Dont import:', layer next print 'Completed', i, 'of', len(all_layers), 'from', schema #cat.save(ft)
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) self.cat.delete(ds) def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store( ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) def testCoverageStoreCreate(self): ds = self.cat.create_coveragestore2("coverage_gsconfig") ds.data_url = "file:data/mytiff.tiff" self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set( ['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_( self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore( "states_test", shapefile_plus_sidecars, sf)) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore( "states_raster_test", shapefile_plus_sidecars, sf)) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") # TODO: Uploading WorldImage file no longer works??? # ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) # self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) # self.assertRaises( # ConflictingDataError, # lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) # ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore( "Pk50095_vector", tiffdata, sf)) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = "Not the original attribution" # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends( os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer( 'boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute) def testImageMosaic(self): # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = name granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id)
import requests from geoserver.catalog import Catalog # Set cat variable cat = Catalog('http://url/geoserver/rest/', username="******", password="******") # Create Workspace #ws = cat.create_workspace('ws_name','http://uri/ws_name') ws = cat.get_workspace('ws_name') # Create Store from PostGIS ds = cat.create_datastore('db_name', 'ws_name') ds.connection_parameters.update(host='host', port='5432', database='db_name', user='******', passwd='password', dbtype='postgis', schema='public') cat.save(ds) # Add Layers from Postgres ft = cat.publish_featuretype('table_name', ds, 'EPSG:3857', srs='EPSG:3857') cat.save(ft)