def set_style(layer_name, style): cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) layer = cat.get_layer(layer_name) style = cat.get_style(style) layer.default_style = style cat.save(layer)
def reload_config(instances): """Perform requests against GeoServer REST API to reload configurations on slaves Optional environment variables: GS_RELOAD_INTERVAL: time in seconds to delay between slave instance reloads GS_PROTOCOL: protocol prefix, should be set to 'HTTP' or 'HTTPS' GS_RELATIVE_URL: relative URL to GeoServer REST API :param instances: list of hostname:port requests :return: """ logging.info('Sending reload requests separated by %s second delay' % RELOAD_INTERVAL) for instance in instances: url = '%s://%s%s' % (PROTOCOL, instance, RELATIVE_URL) logging.info('Performing GeoServer configuration reload at: %s' % url) catalog = Catalog(url) result = catalog.reload() if result and result[0]: if 200 <= result[0].status < 300: logging.info('Successful configuration reload.') else: raise Exception('Failure processing reload with status %s and reason: %s' % (result[0].status, result[0].reason)) else: logging.error('No result received from reload request!') sleep(RELOAD_INTERVAL)
def addCatalog(self, explorer): dlg = DefineCatalogDialog(explorer) dlg.exec_() if dlg.ok: try: cat = Catalog(dlg.url, dlg.username, dlg.password) v = cat.gsversion() supported = v.startswith("2.3") or v.startswith("2.4") if not supported: ret = QtGui.QMessageBox.warning(explorer, "GeoServer catalog definition", "The specified catalog seems to be running an older version of GeoServer\n" "That might cause unexpected behaviour.\nDo you want to add the catalog anyway?", QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.No); if ret == QtGui.QMessageBox.No: return self.catalogs[dlg.name] = cat self.catalog = cat self.comboBox.addItem(dlg.name, cat) self.layersAction.setEnabled(True) self.workspacesAction.setEnabled(True) self.stylesAction.setEnabled(True) self.groupsAction.setEnabled(True) self.gwcAction.setEnabled(True) self.wpsAction.setEnabled(True) self.refreshButton.setEnabled(True) self.updateActionButtons() self.fillTree() self.layersAction.trigger() self.explorer.updateQgisContent() except Exception, e: explorer.setError(str(e)) return
def createDataStore(self, options): try: cat = Catalog(self.geoserver_rest_url, options["geoserveradmin"], options["gpw"]) # create datastore for URB schema try: ws = cat.create_workspace(options["alias"], options["uri"]) except Exception as e: raise Exception("Le nom du workspace ou l'alias est déja utiliser") try: ds = cat.create_datastore(options["alias"], ws) ds.connection_parameters.update( host=options["urbanUrl"], port=options["dbport"], database=options["database"], user=options["postuser"], passwd=options["ropw"], dbtype="postgis", ) cat.save(ds) except Exception as e: print(str(e)) raise Exception("Erreur de connexion au Geoserver lors de la création du DataStore") except Exception as e: raise Exception(str(e)) return ws.name, ds.name, ds.resource_type
def createGeoServerCatalog(service_url = "http://localhost:8080/geoserver/rest", username="******", password="******", authid=None, authtype=None, disable_ssl_certificate_validation=False): # if not authid use basic auth if not authid or not authtype: catalog = GSCatalog(service_url, username, password, disable_ssl_certificate_validation) elif QGis.QGIS_VERSION_INT < 21200: # if autcfg, then get certs and ca and create a PKICatalog certfile, keyfile, cafile = pem.getPemPkiPaths(authid, authtype) # set connection catalog = PKICatalog(service_url, keyfile, certfile, cafile) # set authid parameter used by uri.py functions to manage # uri creation with pki credential catalog.authid = authid else: # For QGIS > 2.12, use the new AuthCatalog and QgsNetworkAccessManager settings = QtCore.QSettings() cache_time = int(settings.value("/GeoServer/Settings/GeoServer/AuthCatalogXMLCacheTime", 180, int)) catalog = AuthCatalog(service_url, authid, cache_time) return CatalogWrapper(catalog)
def layer_group(self): """ Returns layer group name from local OWS for this map instance. """ cat = Catalog(ogc_server_settings.rest, _user, _password) lg_name = '%s_%d' % (slugify(self.title), self.id) return cat.get_layergroup(lg_name)
def export_layer(db_info, gs_info, data_info, layer_name): """\ This takes the parameters and performs the export. This can be run as a module or called from other code. """ geoValidate(layer_name) layer = explodeGeo(layer_name) (spaRef, native_crs) = get_srs(layer) cat = Catalog(*tuple(gs_info)) wspace = cat.create_workspace(data_info.workspace, data_info.namespace) datastore = create_datastore(cat, wspace, db_info, data_info) with closing(createConnObj(db_info._replace(database='postgres'))) as cxn: createNewDb(cxn, db_info.database) quick_export(layer, db_info) create_postgis_layer( cat, wspace, datastore, layer, spaRef, native_crs, db_info, ) try: arcpy.DeleteFeatures_management(layer) except: log(arcpy.GetMessages())
def publish_layer_group(self): """ Publishes local map layers as WMS layer group on local OWS. """ # temporary permission workaround: # only allow public maps to be published if not self.is_public: return 'Only public maps can be saved as layer group.' map_layers = MapLayer.objects.filter(map=self.id) # Local Group Layer layers and corresponding styles layers = [] lg_styles = [] for ml in map_layers: if ml.local: layer = Layer.objects.get(typename=ml.name) style = ml.styles or getattr(layer.default_style, 'name', '') layers.append(layer) lg_styles.append(style) lg_layers = [l.name for l in layers] # Group layer bounds and name lg_bounds = [str(coord) for coord in self.bbox] lg_name = '%s_%d' % (slugify(self.title), self.id) # Update existing or add new group layer cat = Catalog(ogc_server_settings.rest, _user, _password) lg = self.layer_group if lg is None: lg = GsUnsavedLayerGroup(cat, lg_name, lg_layers, lg_styles, lg_bounds) else: lg.layers, lg.styles, lg.bounds = lg_layers, lg_styles, lg_bounds cat.save(lg) return lg_name
def upload_impact_vector(impact_file_path): data = dict() try: cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) ws = cat.get_workspace(GS_IMPACT_WS) base = str(os.path.splitext(impact_file_path)[0]) name = str(os.path.splitext(os.path.basename(base))[0]) shx = base + '.shx' dbf = base + '.dbf' prj = base + '.prj' data = { 'shp' : impact_file_path, 'shx' : shx, 'dbf' : dbf, 'prj' : prj } cat.create_featurestore(name, data, ws, True) data = {'return': 'success', 'resource' : name } except: return {'return': 'fail'} else: return data
class TestCatalogBase(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): sql = """SELECT relname FROM pg_class WHERE relkind = 'r' AND relname !~ '^(pg_|sql_)' AND relname != 'spatial_ref_sys';""" conn = psycopg2.connect( database=_DATABASE, user=_USER, password=_PASSWORD, host=_HOST, port=_PORT ) cur = conn.cursor() cur.execute(sql) tables = [r[0] for r in cur.fetchall()] if len(tables) == 0: return cur.execute("DROP TABLE {};".format(','.join(tables))) conn.commit() def setUp(self): self.catalog = Catalog(settings.SERVICE_URL) self.workspace = self.catalog.create_workspace("gsconfig_test") def tearDown(self): self.catalog.delete(self.workspace, recurse=True)
def _get_layer_style(self,resource_id): geoserver_url=config['ckanext-vectorstorer.geoserver_url'] cat = Catalog(geoserver_url+"/rest") layer = cat.get_layer(c.layer_id) default_style=layer._get_default_style() xml = minidom.parseString(default_style.sld_body) return xml.toprettyxml()
class UploaderTests(MapStoryTestMixin): """ Basic checks to make sure pages load, etc. """ def create_datastore(self, connection, catalog): settings = connection.settings_dict params = {'database': settings['NAME'], 'passwd': settings['PASSWORD'], 'namespace': 'http://www.geonode.org/', 'type': 'PostGIS', 'dbtype': 'postgis', 'host': settings['HOST'], 'user': settings['USER'], 'port': settings['PORT'], 'enabled': "True"} store = catalog.create_datastore( settings['NAME'], workspace=self.workspace) store.connection_parameters.update(params) try: catalog.save(store) except FailedRequestError: # assuming this is because it already exists pass return catalog.get_store(settings['NAME']) def setUp(self): if not os.path.exists(os.path.join(os.path.split(__file__)[0], '..', 'importer-test-files')): self.skipTest('Skipping test due to missing test data.') # These tests require geonode to be running on :80! self.postgis = db.connections['datastore'] self.postgis_settings = self.postgis.settings_dict self.username, self.password = self.create_user( 'admin', 'admin', is_superuser=True) self.non_admin_username, self.non_admin_password = self.create_user( 'non_admin', 'non_admin') self.cat = Catalog( ogc_server_settings.internal_rest, *ogc_server_settings.credentials) if self.cat.get_workspace('geonode') is None: self.cat.create_workspace('geonode', 'http://geonode.org') self.workspace = 'geonode' self.datastore = self.create_datastore(self.postgis, self.cat) def tearDown(self): """ Clean up geoserver. """ self.cat.delete(self.datastore, recurse=True)
def geoserver_upload_shape(self, input_file): print "upload", input_file path, file_ext = os.path.split(input_file) cat = Catalog(self.GEOSERVER_HOST, self.GEOSERVER_USER, self.GEOSERVER_PASSWORD) shapeFile = shapefile_and_friends(input_file) print shapeFile cat.create_featurestore(name=file_ext, data=shapeFile, workspace=cat.get_workspace(self.GEOSERVER_WORKSPACE), overwrite=True, charset=None)
def layer_exists(layer_name, store, workspace): _user, _password = ogc_server_settings.credentials url = ogc_server_settings.rest gs_catalog = Catalog(url, _user, _password) try: layer = gs_catalog.get_resource(layer_name, store=store, workspace=workspace) return layer is not None except FailedRequestError: return False
def add_layer(shp): global ds_name, ws_name, appserver global instance cat = Catalog(appserver + '/rest') ws = cat.get_workspace(ws_name) ds = cat.get_store(ds_name, ws_name) print 'starting {0}...'.format(shp) components = dict((ext, shp + '.' + ext) for ext in ['shp', 'prj', 'shx', 'dbf']) cat.add_data_to_store(ds, shp, components, ws, True) print '{0} finished ...'.format(shp)
def raster_to_geoserver(geoserver_url, raster, workspace): cat = Catalog(geoserver_url, username, password) create_workspace(geoserver_url, username, password, workspace) file_name = os.path.splitext(os.path.basename(raster))[0] workspace = cat.get_workspace(workspace) if not file_name in [l.name for l in cat.get_layers()]: print 'Creating new layer: %s' %file_name cat.create_coveragestore(file_name, raster, workspace) else: print 'Layer %s already exists' %file_name
def main(options): #connect to geoserver cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw) #create datrastore for URB schema ws = cat.create_workspace(options.alias,'imio.be') ds = cat.create_datastore(options.alias, ws) ds.connection_parameters.update( host=options.urbanUrl, port="5432", database=options.database, user="******", passwd=options.ropw, dbtype="postgis") cat.save(ds) ds = cat.get_store(options.alias) #config object urb = { "capa":"Parcelles", "toli":"cadastre_ln_toponymiques", "canu":"cadastre_pt_num", "cabu":"Batiments", "gept":"cadastre_points_generaux", "gepn":"cadastre_pol_gen", "inpt":"point", "geli":"cadastre_ln_generales", "inli":"cadastre_ln_informations", "topt":"point", } #connect to tables and create layers and correct urban styles for table in urb: style = urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') ft.default_style = style cat.save(ft) resource = ft.resource resource.title = options.alias+"_"+table resource.save() layer, created = Layer.objects.get_or_create(name=layerName, defaults={ "workspace": ws.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', #"owner": owner, "uuid": str(uuid.uuid4()), "bbox_x0": Decimal(resource.latlon_bbox[0]), "bbox_x1": Decimal(resource.latlon_bbox[1]), "bbox_y0": Decimal(resource.latlon_bbox[2]), "bbox_y1": Decimal(resource.latlon_bbox[3]) }) set_attributes(layer, overwrite=True) if created: layer.set_default_permissions()
def shp_to_geoserver(geoserver_url, shapefile, workspace): cat = Catalog(geoserver_url, username, password) create_workspace(geoserver_url, username, password, workspace) shapefile_plus_sidecars = geoserver.util.shapefile_and_friends(shapefile) file_name = os.path.splitext(os.path.basename(shapefile))[0] workspace = cat.get_workspace(workspace) if not file_name in [l.name for l in cat.get_layers()]: print 'Creating new layer: %s' %file_name cat.create_featurestore(file_name, shapefile_plus_sidecars, workspace) else: print 'Layer %s already exists' %file_name
def upload_raster(file_path, workspace): data = dict() try: cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) base = str(os.path.splitext(file_path)[0]) name = str(os.path.splitext(os.path.basename(base))[0]) cat.create_coveragestore(name, file_path, workspace, True) except: raise
def upload_style(style_name): try: cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) style_filename = style_name + '.sld' style_file_path = str(os.path.join(DATA_PATH, 'styles', style_filename)) with open(style_file_path, 'r') as style: cat.create_style(style_name, style.read(), overwrite=True) style.close() except: raise
def get_stores(store_type = None): cat = Catalog(ogc_server_settings.rest, _user, _password) stores = cat.get_stores() store_list = [] for store in stores: store.fetch() stype = store.dom.find('type').text.lower() if store_type and store_type.lower() == stype: store_list.append({'name':store.name, 'type': stype}) elif store_type is None: store_list.append({'name':store.name, 'type': stype}) return store_list
def upload_raster(impact_file_path): data = dict() try: cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) ws = cat.get_workspace(GS_IMPACT_WS) base = str(os.path.splitext(impact_file_path)[0]) name = str(os.path.splitext(os.path.basename(base))[0]) cat.create_coveragestore(name, impact_file_path, ws, True) except: raise
def get_stores(store_type = None): url = "%srest" % settings.GEOSERVER_BASE_URL cat = Catalog(url, _user, _password) stores = cat.get_stores() store_list = [] for store in stores: store.fetch() stype = store.dom.find('type').text.lower() if store_type and store_type.lower() == stype: store_list.append({'name':store.name, 'type': stype}) elif store_type is None: store_list.append({'name':store.name, 'type': stype}) return store_list
def updateBbox(self): sql = u"with geom_union as (select st_envelope(st_union(the_geom)) as geom from uos_fast) " \ u"select st_xmin(geom), st_xmax(geom), st_ymin(geom), st_ymax(geom) from geom_union" self.cur.execute(sql) result = self.cur.fetchone() cat = Catalog("http://localhost:8081/geoserver/rest",username ="******",password="******") coverage = cat.get_resource_by_url("http://localhost:8081/geoserver/rest/workspaces/uos/coveragestores/" "uos_image/coverages/uos_fast.xml") coverage.native_bbox = (str(result[0]),str(result[1]),str(result[2]),str(result[3]),"EPSG:4326") cat.save(coverage)
def pre_save_maplayer(instance, sender, **kwargs): # If this object was saved via fixtures, # do not do post processing. if kwargs.get('raw', False): return try: c = Catalog(ogc_server_settings.rest, _user, _password) instance.local = isinstance(c.get_layer(instance.name),GsLayer) except EnvironmentError, e: if e.errno == errno.ECONNREFUSED: msg = 'Could not connect to catalog to verify if layer %s was local' % instance.name logger.warn(msg, e) else: raise e
def updateBbox(self): sql = u"with geom_union as (select st_envelope(st_union(the_geom)) as geom from " + self.DATABASE_TABLE_NAME + u") " \ u"select st_xmin(geom), st_xmax(geom), st_ymin(geom), st_ymax(geom) from geom_union" self.cur.execute(sql) result = self.cur.fetchone() #self.cur.close() cat = Catalog(self.HTTP + self.GEOSERVER_URL + "/geoserver/rest",username = self.GEOSERVER_USER, password=self.GEOSERVER_PASSWORD) print self.HTTP + self.GEOSERVER_URL + "/geoserver/rest/workspaces/uos/coveragestores/" + "uos_upload_image/coverages/" + self.DATABASE_TABLE_NAME + ".xml" coverage = cat.get_resource_by_url(self.HTTP + self.GEOSERVER_URL + "/geoserver/rest/workspaces/uos/coveragestores/" + "uos_upload_image/coverages/" + self.GEOSERVER_LAYER_NAME + ".xml") coverage.native_bbox = (str(result[0]),str(result[1]),str(result[2]),str(result[3]),"EPSG:4326") cat.save(coverage)
def __init__(self, gs_url, dico_gs, tipo, txt=''): u""" Uses OGR functions to extract basic informations about geographic Web Features Services. gs_url = url of a WFS service dico_wfs = dictionary for global informations dico_fields = dictionary for the fields' informations li_fieds = ordered list of fields tipo = format text = dictionary of text in the selected language """ cat = Catalog(gs_url, "####", "####") layers = cat.get_layers() for layer in layers: print(layer.name)
def make_style(style_name, style_info): cat = Catalog(GEOSERVER_REST_URL, GS_USERNAME, GS_PASSWORD) style_dir = path('data/styles') if not style_dir.exists(): style_dir.makedirs() style_filename = style_name + '.sld' style_file_path = str(os.path.join(DATA_PATH, 'styles', style_filename)) target_field = style_info['target_field'] sld_doc = StyledLayerDescriptor() nl = sld_doc.create_namedlayer(style_name) us = nl.create_userstyle() feature_style = us.create_featuretypestyle() for x in style_info['style_classes']: elem = feature_style._node.makeelement('{%s}Rule' % SLDNode._nsmap['sld'], nsmap=SLDNode._nsmap) feature_style._node.append(elem) r = Rule(feature_style, len(feature_style._node)-1) r.Title = x['label'] f1 = Filter(r) f1.PropertyIsGreaterThanOrEqualTo = PropertyCriterion(f1, 'PropertyIsGreaterThanOrEqualTo') f1.PropertyIsGreaterThanOrEqualTo.PropertyName = target_field f1.PropertyIsGreaterThanOrEqualTo.Literal = str(round(x['min'])) f2 = Filter(r) f2.PropertyIsLessThan = PropertyCriterion(f2, 'PropertyIsLessThan') f2.PropertyIsLessThan.PropertyName = target_field f2.PropertyIsLessThan.Literal = str(round(x['max'])) r.Filter = f1 + f2 sym = PolygonSymbolizer(r) fill = Fill(sym) fill.create_cssparameter('fill', x['colour']) fill.create_cssparameter('fill-opacity', '0.5') with open(style_file_path, 'w') as style: style.write(sld_doc.as_sld(True)) style.close() with open(style_file_path, 'r') as style: cat.create_style(style_name, style.read(), overwrite=True) style.close()
def pre_delete_service(instance, sender, **kwargs): for layer in instance.layer_set.all(): layer.delete() # if instance.method == 'H': # gn = Layer.objects.gn_catalog # gn.control_harvesting_task('stop', [instance.external_id]) # gn.control_harvesting_task('remove', [instance.external_id]) if instance.method == 'C': try: _user = settings.OGC_SERVER['default']['USER'] _password = settings.OGC_SERVER['default']['PASSWORD'] gs = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user , _password) cascade_store = gs.get_store(instance.name, settings.CASCADE_WORKSPACE) gs.delete(cascade_store, recurse=True) except FailedRequestError: logger.error("Could not delete cascading WMS Store for %s - maybe already gone" % instance.name)
def wrapper(self, *args, **kwargs): workspace_name = 'geonode' django_datastore = db.connections['datastore'] catalog = Catalog( ogc_server_settings.internal_rest, *ogc_server_settings.credentials ) # Set up workspace/datastore as appropriate ws = catalog.get_workspace(workspace_name) delete_ws = False if ws is None: ws = catalog.create_workspace(workspace_name, 'http://www.geonode.org/') delete_ws = True datastore = create_datastore(workspace_name, django_datastore, catalog) # test method called here try: ret = wrapped_func(self, *args, **kwargs) finally: # Tear down workspace/datastore as appropriate if delete_ws: catalog.delete(ws, recurse=True) else: catalog.delete(datastore, recurse=True) return ret
def synchronizeLayers(self): ''' Synchronizes layers/rules in the postgresql database, Geoserver, and Geofence. All three will match postgresql database. ''' to_return = True permanent_gs_layers = [ 'rapid_geoview_layers', 'rapid_feature', 'rapid_geoview' ] rapid_layers = { lyr.uid for lyr in DataLayer.objects.all() if lyr.feature_set.count() > 0 } cat = Catalog(GEOSERVER_REST_ENDPOINT) store = cat.get_store('RapidData') gs_layers = { lyr.name for lyr in store.get_resources() if lyr.name not in permanent_gs_layers } gf_rules_ids = { (rule.find('layer').text, int(rule.find('id').text)) for rule in ET.fromstring(getGeofenceRules('admin').content)[:-1] } gf_rules = {rule[0] for rule in gf_rules_ids} gs_layers_to_remove = gs_layers.difference(rapid_layers) gs_layers_to_add = rapid_layers.difference(gs_layers) gf_rules_to_remove = gf_rules.difference(rapid_layers) gf_rules_to_add = rapid_layers.difference(gf_rules) if len(gs_layers_to_add) == 0 and len(gs_layers_to_remove) == 0: if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0: return to_return elif len(gs_layers_to_remove) > 0: for lyr_uid in gs_layers_to_remove: print "Removing Geoserver layer {0}. . .".format(lyr_uid) response = self.removeFeatureType(lyr_uid) if response.status_code == 200: print "Success!" else: print "Failed to remove layer {0}".format(lyr_uid) to_return = False print response.content elif len(gs_layers_to_add) > 0: for uid in gs_layers_to_add: print "Adding FeatureType to Geoserver: {0}".format(uid) xml = self.createFeatureTypeFromUid(uid) response = self.sendFeatureType(xml) if response.status_code == 200: print "Success!" else: print "Failed to add FeatureType {0}".format(uid) to_return = False print response.content if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0: return to_return elif len(gf_rules_to_remove) > 0: gf_ids_to_remove = [ rule for rule in gf_rules_ids if rule[0] in gf_rules_to_remove ] for rule in gf_ids_to_remove: print "Removing Geofence rule for layer {0}".format(rule[0]) response = removeGeofenceRule(rule[1]) if response.status_code == 200: print "Success!" else: print "Failed to remove layer {0}".format(lyr_uid) to_return = false print response.content elif len(gf_rules_to_add) > 0: for rule in gf_rules_to_add: response = addGeofenceRule(layer=rule) if response.status_code == 200: print "Success!" else: print "Failed to add Geofence rule for layer {0}".format( rule) to_return = False print response.content return to_return
class ArchiveAndCatalogueI(geocloud.ArchiveAndCatalogue): def __init__(self, com): if not com: raise RuntimeError("Not communicator") self.com = com q = com.stringToProxy('IceGrid/Query') self.query = IceGrid.QueryPrx.checkedCast(q) if not self.query: raise RuntimeError("Invalid proxy") self.broker = geocloud.BrokerPrx.checkedCast( com.stringToProxy("broker")) self.sem = Ice.threading.RLock( ) #Lock for managing the data structures geoserver_path = "http://localhost:80/geoserver/rest" self.catalog = Catalog(geoserver_path) self.workspace = None def setBroker(self, broker, current=None): self.broker = broker def log(self, log): if self.broker: self.broker.begin_appendLog("<ArchiveAndCatalogue> " + str(log)) else: print log def createScenario(self, scenario, current=None): self.sem.acquire() try: if self.catalog == None: self.log("Catalog") raise geocloud.CreationScenarioException() try: self.workspace = self.catalog.create_workspace( scenario, scenario) self.workspace.enabled = True self.log("Created Scenario %s" % (scenario)) except Exception: print("workspace does not created") self.log("Workspace does not created") except FailedRequestError: self.log("Request failed") raise geocloud.CreationScenarioException() finally: self.sem.release() def catalogue(self, path, store, scenario, current=None): if self.workspace == None: raise geocloud.CataloguingException() else: try: if self.catalog == None: raise geocloud.CataloguingException() # store = path.split('/')[-1] self.sem.acquire() cv = self.catalog.create_coveragestore(store, path, self.workspace) self.log("%s Catalogued!" % (path)) except ConflictingDataError as e: self.log(e) except UploadError as e: self.log(e) except FailedRequestError as e: None except Exception as e: self.log(e) finally: self.sem.release() def deleteScenario(self, scenario, current=None): self.sem.acquire() try: for layer in self.catalog.get_layers(): self.catalog.delete(layer) for coverage in self.catalog.get_stores(): for resource in coverage.get_resources(): catalog.delete(resource) catalog.delete(coverage) for vk in catalog.get_workspaces(): catalog.delete(wk) except Exception: self.log("Exception while cleanning scenario")
def setUp(self): self.catalog = Catalog(settings.SERVICE_URL) self.workspace = self.catalog.create_workspace("gsconfig_test")
class GeoServer(MapServer): def __init__(self, process, sessId): MapServer.__init__(self, process, sessId) # initial myself # the catagory of this geoserver self.cat = Catalog( '/'.join((config.getConfigValue('geoserver', 'geoserveraddress'), 'rest')), config.getConfigValue('geoserver', 'admin'), config.getConfigValue('geoserver', 'adminpassword')) # get the workspace that datas will be saved and if workspace not set at configure file or # not exit then the default workspace will be used self.workspace = self.cat.get_workspace( config.getConfigValue('geoserver', 'workspace')) if not self.workspace: self.workspace = self.cat.get_default_workspace() def save(self, output): MapServer.save(self, output) name = "%s-%s-%s" % (output.identifier, self.process.identifier, self.sessId) service_url = '/'.join([ config.getConfigValue('geoserver', 'geoserveraddress'), self.workspace.name ]) if self.datatype == "raster": # save logging.info("upload raster file with name: {0} ".format(name)) self.cat.create_coveragestore(name, output.value, self.workspace, False) # construct the wcs url return urllib2.quote('/'.join((service_url, 'wcs')) + "?SERVICE=WCS" + "&REQUEST=GetCoverage" + "&VERSION=1.0.0" + "&COVERAGE=" + name + "&CRS=" + output.projection + "&BBOX=%s,%s,%s,%s" % (output.bbox[0], output.bbox[1], output.bbox[2], output.bbox[3]) + "&HEIGHT=%s" % (output.height) + "&WIDTH=%s" % (output.width) + "&FORMAT=%s" % output.format["mimetype"]) elif self.datatype == "vector": #save logging.info("upload vector file with name: {0} ".format(name)) zip_shpf, lyname = self.compressFeatureData(output.value) self.cat.create_featurestore(name, zip_shpf, self.workspace, False) # construct the wfs url return urllib2.quote('/'.join((service_url, 'wfs')) + "?SERVICE=WFS" + "&REQUEST=GetFeature" + "&VERSION=1.0.0" + "&TYPENAME=" + lyname) else: return None def compressFeatureData(self, s_ftfile): """ @return: 1. the zip file path; 2. layer name at geoserver """ from osgeo import ogr def zipFiles(zipname, files, arcnames): assert len(files) == len( arcnames), "size of file names and rename container not equal" zipf = zipfile.ZipFile(zipname, 'w') for i in range(len(files)): if os.path.exists(files[i]): zipf.write(files[i], arcnames[i]) zipf = None ft = ogr.Open(s_ftfile) ftDrv = ft.GetDriver() sft = os.path.splitext(s_ftfile)[0] archive_files = [ sft + '.shp', sft + '.shx', sft + '.prj', sft + '.dbf' ] filename = os.path.split(sft)[1] arcnames = [ filename + '.shp', filename + '.shx', filename + '.prj', filename + '.dbf' ] logging.info("the driver of vector data {0} is {1}".format( s_ftfile, ftDrv.name)) if (ftDrv.name != "ESRI Shapefile"): tempshpf = os.path.join( config.getConfigValue('server', 'tempPath'), str(uuid.uuid4())) shpDrv = ogr.GetDriverByName("ESRI Shapefile") shpft = shpDrv.CopyDataSource(ft, tempshpf + '.shp') if not shpft: raise exceptions.IOError( "{0} format vector data to shapefile format fault".format( s_ftfile)) # close the vector datasource ft = None shpft = None # zip shape files and delete them # create an defautl prj file for this shapefile if thereis not projection information if not os.path.exists(tempshpf + '.prj'): f = open(tempshpf + '.prj', 'w') f.write( 'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' ) f.close() archive_files = [ tempshpf + '.shp', tempshpf + '.shx', tempshpf + '.prj', tempshpf + '.dbf' ] zipFiles(tempshpf + '.zip', archive_files, arcnames) for f in archive_files: if os.path.exists(f): os.remove(f) return tempshpf + '.zip', filename else: zipFiles(sft + '.zip', archive_files, arcnames) return sft + '.zip', filename
def main(argv): # --------------------- Handler --------------------- # try: opts, args = getopt.getopt(argv, "hd:g:t:o:", [ "help", "test", "date=", "grid=", "target=", "only=", "overwrite" ]) except getopt.GetoptError: print('Invalid arguments. Add "-h" or "--help" for help.') sys.exit(1) date, grid, only, target, overwrite = None, None, None, None, False for opt, arg in opts: if opt in ("-h", "--help"): print('Usage: ./main.py [OPTIONS...]\n\nHelp options:\n\ -h, --help Show help\n\ --test Test that all packages and system variables work properly\n\ -d, --date Set date for all imagery capture (format: yyyy-mm-dd / yyyymmdd)\n\ -g, --grid Set center grid for sentinel imagery capture (format: east-decimal,north-decimal)\n\ -t, --target Set path to target directory (format: path/to/target)\n\ -o, --only Only create specified layer (format: layername)\n\ --overwrite Overwrite layers in geoserver') sys.exit() elif opt == ("--test"): print( "Success! All packages and sytem variables seems to work properly" ) sys.exit() elif opt in ("-d", "--date"): if '-' in arg: date = datetime.strptime(arg, '%Y-%m-%d').date() else: date = datetime.strptime(arg, '%Y%m%d').date() elif opt in ("-g", "--grid"): grid = arg elif opt in ("-t", "--target"): target = arg elif opt in ("-o", "--only"): only = arg elif opt in ("--overwrite"): overwrite = True if not opts: print('WARNING: No arguments provided. Using default values') # ------------------- Handler end ------------------- # d = DownloadManager(date=date, grid=grid, target=target) # Making download instance functions = { 's2c': d.getS2, 'terramos': d.getTerra, 's1c': d.getS1, 's1mos': d.getS1Mos, 'seaice': d.getSeaIce, 'icedrift': d.getIceDrift } # ------------------ Getting/processing data ------------------ # outfiles = [] if only: outfiles.append(functions[only](d.OUTDIR + only + '.tif')) else: for k, v in functions.iteritems(): try: funcs.printLayerStatus(str(k)) outfiles.append(v(d.OUTDIR + k + '.tif')) except Exception as e: print("Could not generate " + k + " layer") print(traceback.print_exc()) finally: print("") print('Created files: ' + str(outfiles) + '\n') if all(file == False for file in outfiles): print("No images found") exit(1) # ---------------- Getting/processing data end ---------------- # # --------------------- Upload to Geoserver -------------------- # cat = Catalog("http://localhost:8080/geoserver/rest/", "admin", os.environ["GEOSERVER_SECRET_KEY"]) print(cat) # BUG: ws is sometime cite instead of d.date (fixed?) ws = cat.get_workspace(str(d.DATE)) if ws is None: ws = cat.create_workspace(str(d.DATE), str(d.DATE) + 'uri') for layerdata in outfiles: if layerdata: layername = layerdata.split('/')[-1].split('.')[0] print('Adding ' + layername + ' to geoserver...') cat.create_coveragestore(name=layername, data=layerdata, workspace=ws, overwrite=overwrite) print('Sensor data for ' + str(d.DATE) + ' has been successfully uploaded!')
def main(options): #connect to geoserver cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw) #create datrastore for URB schema ws = cat.create_workspace(options.alias, 'imio.be') ds = cat.create_datastore(options.alias, ws) ds.connection_parameters.update(host=options.urbanUrl, port="5432", database=options.database, user="******", passwd=options.ropw, dbtype="postgis") cat.save(ds) ds = cat.get_store(options.alias) #config object urb = { "capa": "Parcelles", "toli": "cadastre_ln_toponymiques", "canu": "cadastre_pt_num", "cabu": "Batiments", "gept": "cadastre_points_generaux", "gepn": "cadastre_pol_gen", "inpt": "point", "geli": "cadastre_ln_generales", "inli": "cadastre_ln_informations", "topt": "point", } #connect to tables and create layers and correct urban styles for table in urb: style = urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') ft.default_style = style cat.save(ft) resource = ft.resource resource.title = options.alias + "_" + table resource.save() layer, created = Layer.objects.get_or_create( name=layerName, defaults={ "workspace": ws.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', #"owner": owner, "uuid": str(uuid.uuid4()), "bbox_x0": Decimal(resource.latlon_bbox[0]), "bbox_x1": Decimal(resource.latlon_bbox[1]), "bbox_y0": Decimal(resource.latlon_bbox[2]), "bbox_y1": Decimal(resource.latlon_bbox[3]) }) set_attributes(layer, overwrite=True) if created: layer.set_default_permissions()
def __init__(self): models.Manager.__init__(self) url = "%srest" % settings.GEOSERVER_BASE_URL self.gs_catalog = Catalog(url, _user, _password)
class GeoGigUploaderBase(ImportHelper): def __init__(self, *args, **kwargs): super(GeoGigUploaderBase, self).__init__(*args, **kwargs) setUpModule() # this isn't available when being used in other module def setUp(self): self.admin_user = self.create_user('admin', 'admin', is_superuser=True) self.non_admin_user = self.create_user('non_admin', 'non_admin') self.catalog = Catalog(ogc_server_settings.internal_rest, *ogc_server_settings.credentials) if self.catalog.get_workspace('geonode') is None: self.catalog.create_workspace('geonode', 'http://www.geonode.org/') self.workspace = 'geonode' self.datastoreNames = [] def tearDown(self): """Clean up geoserver/geogig catalog. """ # delete stores (will cascade to delete layers) for store_name in self.datastoreNames: self.catalog.delete(self.catalog.get_store(store_name), recurse=True) # delete repository reference in geoserver for store_name in self.datastoreNames: self.remove_geogig_repo(store_name) # geoserver can leave connections open - HACK HACK HACK self.free_geogig_connections() # HACK HACK HACK -- sometimes connections from geoserver to geogig are left open. This kills the postgresql backend! # this is a major hammer. Once geoserver/geogig are better at cleaning up, remove this. def free_geogig_connections(self): with db.connections["geogig"].cursor() as c: c.execute( "select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'PostgreSQL JDBC Driver' or application_name='geogig'" ) # aggressive delete of the repo (mostly cleans up after itself) # call the geogig rest API DELETE def remove_geogig_repo(self, ref_name): username = ogc_server_settings.credentials.username password = ogc_server_settings.credentials.password url = ogc_server_settings.rest http = httplib2.Http(disable_ssl_certificate_validation=False) http.add_credentials(username, password) netloc = urlparse(url).netloc http.authorizations.append( httplib2.BasicAuthentication((username, password), netloc, url, {}, None, None, http)) rest_url = ogc_server_settings.LOCATION + "geogig/repos/" + ref_name + "/delete.json" resp, content = http.request(rest_url, 'GET') response = json.loads(content) token = response["response"]["token"] rest_url = ogc_server_settings.LOCATION + "geogig/repos/" + ref_name + "?token=" + token resp, content = http.request(rest_url, 'DELETE') # convenience method to load in the test dataset # return a (geonode) layer # the layer will be in Geoserver and Geonode # self.catalog.get_layer(layer.name) -- to get the Geoserver Layer def fully_import_file(self, path, fname, start_time_column, end_time_column=None): # setup time if end_time_column is None: time_config = { 'convert_to_date': [start_time_column], 'start_date': start_time_column, 'configureTime': True } else: time_config = { 'convert_to_date': [start_time_column, end_time_column], 'start_date': start_time_column, 'end_date': end_time_column, 'configureTime': True } name = os.path.splitext(fname)[0] + "_" + str(uuid.uuid1())[:8] self.datastoreNames.append(name) # remember for future deletion full_fname = os.path.join(path, fname) configs = self.prepare_file_for_import(full_fname) configs[0].update({'name': name}) configs[0].update({'layer_name': name}) configs[0].update(time_config) # configure the datastore/repo configs[0]['geoserver_store'] = {} configs[0]['geoserver_store']['type'] = 'geogig' configs[0]['geoserver_store']['name'] = name configs[0]['geoserver_store']['create'] = 'true' configs[0]['geoserver_store']['branch'] = 'master' configs[0]['geoserver_store'][ 'geogig_repository'] = "geoserver://" + name result = self.generic_import(fname, path=path, configs=configs) return result def import_file(self, path, configs=None): """Imports the file. """ if configs is None: configs = [] self.assertTrue(os.path.exists(path), path) # run ogr2ogr ogr = OGRImport(path) layers = ogr.handle(configuration_options=configs) return layers def generic_import(self, filename, path, configs=None): if configs is None: configs = [{'index': 0}] path = os.path.join(path, filename) results = self.import_file(path, configs=configs) layer_results = [] for result in results: layer = Layer.objects.get(name=result[0]) self.assertEqual(layer.srid, 'EPSG:4326') self.assertTrue(layer.store in self.datastoreNames) self.assertEqual(layer.storeType, 'dataStore') if not path.endswith('zip'): self.assertGreaterEqual(layer.attributes.count(), DataSource(path)[0].num_fields) layer_results.append(layer) return layer_results[0] def prepare_file_for_import(self, filepath): """ Prepares the file path provided for import; performs some housekeeping, uploads & configures the file. Returns a list of dicts of the form {'index': <layer_index>, 'upload_layer_id': <upload_layer_id>} these may be used as configuration options for importing all of the layers in the file. """ # Make a copy of the test file, as it's removed in configure_upload() filename = os.path.basename(filepath) tmppath = os.path.join('/tmp', filename) shutil.copy(filepath, tmppath) # upload & configure_upload expect closed file objects # This is heritage from originally being closely tied to a view passing request.Files of = open(tmppath, 'rb') of.close() files = [of] uploaded_data = self.upload(files, self.admin_user) self.configure_upload(uploaded_data, files) configs = [{ 'index': l.index, 'upload_layer_id': l.id } for l in uploaded_data.uploadlayer_set.all()] return configs def create_user(self, username, password, **kwargs): """Convenience method for creating users. """ user, created = User.objects.get_or_create(username=username, **kwargs) if created: user.set_password(password) user.save() return user
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ url = ogc_server_settings.rest try: gs_catalog = Catalog(url, _user, _password) gs_resource = gs_catalog.get_resource(instance.name) except (FailedRequestError, EnvironmentError) as e: msg = ('Could not connect to geoserver at "%s"' 'to save information for layer "%s"' % (ogc_server_settings.LOCATION, instance.name.encode('utf-8'))) logger.warn(msg, e) # If geoserver is not online, there is no need to continue return # If there is no resource returned it could mean one of two things: # a) There is a synchronization problem in geoserver # b) The unit tests are running and another geoserver is running in the # background. # For both cases it is sensible to stop processing the layer if gs_resource is None: logger.warn('Could not get geoserver resource for %s' % instance) return gs_resource.keywords = instance.keyword_list() #gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wms_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='image', )) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', )) elif instance.storeType == 'coverageStore': #FIXME(Ariel): This works for public layers, does it work for restricted too? # would those end up with no geotiff links, like, forever? permissions = {} permissions['anonymous'] = instance.get_gen_level(ANONYMOUS_USERS) permissions['authenticated'] = instance.get_gen_level( AUTHENTICATED_USERS) instance.set_gen_level(ANONYMOUS_USERS, 'layer_readonly') links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8'), bbox=instance.bbox[:-1], crs=instance.bbox[-1], height=height, width=width) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', )) instance.set_gen_level(ANONYMOUS_USERS, permissions['anonymous']) instance.set_gen_level(AUTHENTICATED_USERS, permissions['authenticated']) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode( { 'layers': instance.typename.encode('utf-8'), 'mode': "download" }) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name=_("KML"), mime='text/xml', link_type='data', )) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode( { 'layers': instance.typename.encode('utf-8'), 'mode': "refresh" }) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name=_("View in Google Earth"), mime='text/xml', link_type='data', )) tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8') Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=tile_url, defaults=dict( extension='tiles', name=_("Tiles"), mime='image/png', link_type='image', )) html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', )) #remove links that belong to and old address for link in instance.link_set.all(): if not urlparse(ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() #Save layer attributes set_attributes(instance) #Save layer styles set_styles(instance, gs_catalog)
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog style_to_check = "point" cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") def has_the_style(l): return (l.default_style.name == style_to_check or any(s.name == style_to_check for s in l.styles)) print[l.name for l in cat.get_layers() if has_the_style(l)]
def connectCatalog(ip, user, password): ''' Return a catalog from geoserver REST API''' # example Catalog("http://192.168.1.41:8080/geoserver/rest",'user', 'password') cat = Catalog(ip, user, password) return cat
gsconfig_url = open(os.path.join(secure_path, "private_gsconfig_local_url.txt")).read() gsconfig_username = open(os.path.join(secure_path, "private_gsconfig_username.txt")).read() gsconfig_password = open(os.path.join(secure_path, "private_gsconfig_key.txt")).read() except FileNotFoundError as e: log.error("Missing default credentials") log.error(str(e)) raise e log.info("Connecting to PostgreSQL") conn = psycopg2.connect(pyscopg2_connection_string) conn.autocommit = True cur = conn.cursor() log.info("Connecting to GeoServer") cat = Catalog(gsconfig_url, gsconfig_username, gsconfig_password) if args.droptable: log.info("Dropping table %s if it exists" % table_name) dropTable(cur, table_name) log.info("Pushing data to PostgreSQL") insertVector(cur, in_path, table_name) log.info("Pushing data to GeoServer") publishVector(cur, table_name, pyscopg2_connection_string, cat) log.debug("Commiting changes to database and closing connection") cur.close() conn.commit()
import csv import getpass c = csv.writer(open( "TLayers " + strftime("%Y-%m-%d %H%M%S", gmtime()) + ".csv", "wb"), delimiter=';', quotechar='|') print "Usuario y contrasenia de Geoserver" user = raw_input("User: "******"http://geo.gob.bo/geoserver/rest" print "Conectandose a " + geoserver cat = Catalog(geoserver, user, password) # Recuperar los datos all_layers = cat.get_layers() # Recuperar estadisticas sobre las capas sin_metadata = 0 nb_capas = len(all_layers) print "Capas sin metadatos:" print "" #that_layer = cat.get_layer("PassifloraInsignis") # that_layer.enabled = True # print that_layer.resource.workspace,that_layer.name,that_layer.resource.metadata_links,that_layer.resource.native_bbox,that_layer.default_style
class UploaderTests(TestCase): """Basic checks to make sure pages load, etc. """ def create_datastore(self, connection, catalog): """Convenience method for creating a datastore. """ settings = connection.settings_dict ds_name = settings['NAME'] params = { 'database': ds_name, 'passwd': settings['PASSWORD'], 'namespace': 'http://www.geonode.org/', 'type': 'PostGIS', 'dbtype': 'postgis', 'host': settings['HOST'], 'user': settings['USER'], 'port': settings['PORT'], 'enabled': 'True' } store = catalog.create_datastore(ds_name, workspace=self.workspace) store.connection_parameters.update(params) try: catalog.save(store) except FailedRequestError: # assuming this is because it already exists pass return catalog.get_store(ds_name) def create_user(self, username, password, **kwargs): """Convenience method for creating users. """ user, created = User.objects.get_or_create(username=username, **kwargs) if created: user.set_password(password) user.save() return user def setUp(self): self.assertTrue( os.path.exists(_TEST_FILES_DIR), 'Test could not run due to missing test data at {0!r}' .format(_TEST_FILES_DIR) ) # These tests require geonode to be running on :80! self.postgis = db.connections['datastore'] self.postgis_settings = self.postgis.settings_dict self.admin_user = self.create_user('admin', 'admin', is_superuser=True) self.non_admin_user = self.create_user('non_admin', 'non_admin') self.catalog = Catalog( ogc_server_settings.internal_rest, *ogc_server_settings.credentials ) if self.catalog.get_workspace('geonode') is None: self.catalog.create_workspace('geonode', 'http://www.geonode.org/') self.workspace = 'geonode' self.datastore = self.create_datastore(self.postgis, self.catalog) def tearDown(self): """Clean up geoserver. """ self.catalog.delete(self.datastore, recurse=True) def import_file(self, path, configs=None): """Imports the file. """ if configs is None: configs = [] self.assertTrue(os.path.exists(path)) # run ogr2ogr ogr = OGRImport(path) layers = ogr.handle(configuration_options=configs) return layers def generic_import(self, filename, configs=None): if configs is None: configs = [{'index': 0}] path = test_file(filename) results = self.import_file(path, configs=configs) layer_results = [] for result in results: if result[1].get('raster'): layer_path = result[0] layer_name = os.path.splitext(os.path.basename(layer_path))[0] layer = Layer.objects.get(name=layer_name) self.assertTrue(layer_path.endswith('.tif')) self.assertTrue(os.path.exists(layer_path)) gdal_layer = gdal.OpenEx(layer_path) self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff') layer_results.append(layer) else: layer = Layer.objects.get(name=result[0]) self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') if not path.endswith('zip'): self.assertGreaterEqual( layer.attributes.count(), DataSource(path)[0].num_fields ) layer_results.append(layer) return layer_results[0] def generic_api_upload(self, filenames, configs=None): """Tests the import api. """ client = AdminClient() client.login_as_non_admin() # Don't accidentally iterate over given 'foo' as ['f', 'o', 'o']. self.assertNotIsInstance(filenames, str) # Upload Files outfiles = [] for filename in filenames: path = test_file(filename) with open(path) as stream: data = stream.read() upload = SimpleUploadedFile(filename, data) outfiles.append(upload) response = client.post( reverse('uploads-new-json'), {'file': outfiles, 'json': json.dumps(configs)}, follow=True) content = json.loads(response.content) self.assertEqual(response.status_code, 200) self.assertEqual(content['id'], 1) # Configure Uploaded Files upload_id = content['id'] upload_layers = UploadLayer.objects.filter(upload_id=upload_id) for upload_layer in upload_layers: for config in configs: if config['upload_file_name'] == upload_layer.name: payload = config['config'] url = '/importer-api/data-layers/{0}/configure/'.format(upload_layer.id) response = client.post( url, data=json.dumps(payload), content_type='application/json' ) self.assertTrue(response.status_code, 200) url = '/importer-api/data-layers/{0}/'.format(upload_layer.id) response = client.get(url, content_type='application/json') self.assertTrue(response.status_code, 200) return content def generic_raster_import(self, filename, configs=None): if configs is None: configs = [{'index': 0}] path = test_file(filename) results = self.import_file(path, configs=configs) layer_path = results[0][0] layer_name = os.path.splitext(os.path.basename(layer_path))[0] layer = Layer.objects.get(name=layer_name) self.assertTrue(layer_path.endswith('.tif')) self.assertTrue(os.path.exists(layer_path)) gdal_layer = gdal.OpenEx(layer_path) self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff') return layer def test_multi_upload(self): """Tests Uploading Multiple Files """ upload = self.generic_api_upload( filenames=[ 'boxes_with_year_field.zip', 'boxes_with_date.zip', 'point_with_date.geojson' ], configs=[ { 'upload_file_name': 'boxes_with_year_field.shp', 'config': [{'index': 0}] }, { 'upload_file_name': 'boxes_with_date.shp', 'config': [{'index': 0}] }, { 'upload_file_name': 'point_with_date.geojson', 'config': [{'index': 0}] } ] ) self.assertEqual(9, upload['count']) def test_upload_with_slds(self): """Tests Uploading sld """ upload = self.generic_api_upload( filenames=[ 'boxes_with_date.zip', 'boxes.sld', 'boxes1.sld' ], configs=[ { 'upload_file_name': 'boxes_with_date.shp', 'config': [ { 'index': 0, 'default_style': 'boxes.sld', 'styles': ['boxes.sld', 'boxes1.sld'] } ] } ] ) self.assertEqual(6, upload['count']) upload_id = upload['id'] upload_obj = UploadedData.objects.get(pk=upload_id) uplayers = UploadLayer.objects.filter(upload=upload_id) layer_id = uplayers[0].pk upfiles_count = UploadFile.objects.filter(upload=upload_id).count() self.assertEqual(6, upfiles_count) # Warning: this assumes that Layer pks equal UploadLayer pks layer = Layer.objects.get(pk=layer_id) gslayer = self.catalog.get_layer(layer.name) default_style = gslayer.default_style # TODO: can we use public API or omit this? self.catalog._cache.clear() self.assertEqual('boxes.sld', default_style.filename) def test_upload_with_metadata(self): """Tests Uploading metadata """ upload = self.generic_api_upload( filenames=[ 'boxes_with_date.zip', 'samplemetadata.xml', ], configs=[ { 'upload_file_name': 'boxes_with_date.shp', 'config': [ { 'index': 0, 'metadata': 'samplemetadata.xml' } ] } ] ) self.assertEqual(5, upload['count']) upload_id = upload['id'] upload_obj = UploadedData.objects.get(pk=upload_id) # TODO: why did we get upload_obj? uplayers = UploadLayer.objects.filter(upload=upload_id) layer_id = uplayers[0].pk upfiles_count = UploadFile.objects.filter(upload=upload_id).count() self.assertEqual(5, upfiles_count) layer = Layer.objects.get(pk=layer_id) self.assertEqual(layer.language, 'eng') self.assertEqual(layer.title, 'Old_Americas_LSIB_Polygons_Detailed_2013Mar') def test_raster(self): """Exercise raster import. """ layer = self.generic_raster_import( 'test_grid.tif', configs=[ { 'index': 0 } ] ) self.assertTrue(layer.name.startswith('test_grid')) def test_box_with_year_field(self): """Tests the import of test_box_with_year_field. """ layer = self.generic_import( 'boxes_with_year_field.shp', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date(self): """Tests the import of test_boxes_with_date. """ layer = self.generic_import( 'boxes_with_date.shp', configs=[ { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date_gpkg(self): """Tests the import of test_boxes_with_date.gpkg. """ layer = self.generic_import( 'boxes_with_date.gpkg', configs=[ { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_plus_raster_gpkg_by_index(self): """Tests the import of multilayer vector + raster geopackage using index """ layer = self.generic_import( 'boxes_plus_raster.gpkg', configs=[ { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True }, {'index': 1}, {'index': 2}, {'index': 3}, {'index': 4}, {'index': 5}, {'index': 6}, {'index': 7}, ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute,) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_date_csv(self): """Tests a CSV with WKT polygon. """ layer = self.generic_import( 'boxes_with_date.csv', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_csv_missing_features(self): """Test csv that has some rows without geoms and uses ISO-8859 (Latin 4) encoding. """ self.generic_import( 'missing-features.csv', configs=[ {'index': 0} ] ) def test_boxes_with_iso_date(self): """Tests the import of test_boxes_with_iso_date. """ layer = self.generic_import( 'boxes_with_date_iso_date.shp', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_duplicate_imports(self): """Import the same layer twice to ensure file names increment properly. """ path = test_file('boxes_with_date_iso_date.zip') ogr = OGRImport(path) layers1 = ogr.handle({'index': 0, 'name': 'test'}) layers2 = ogr.handle({'index': 0, 'name': 'test'}) self.assertEqual(layers1[0][0], 'test') self.assertEqual(layers2[0][0], 'test0') def test_launder(self): """Ensure the launder function works as expected. """ self.assertEqual(launder('tm_world_borders_simpl_0.3'), 'tm_world_borders_simpl_0_3') self.assertEqual(launder('Testing#'), 'testing_') self.assertEqual(launder(' '), '_') def test_boxes_with_date_iso_date_zip(self): """Tests the import of test_boxes_with_iso_date. """ layer = self.generic_import( 'boxes_with_date_iso_date.zip', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_dates_bc(self): """Tests the import of test_boxes_with_dates_bc. """ layer = self.generic_import( 'boxes_with_dates_bc.shp', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_point_with_date(self): """Tests the import of point_with_date.geojson """ layer = self.generic_import( 'point_with_date.geojson', configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) # OGR will name geojson layers 'ogrgeojson' we rename to the path basename self.assertTrue(layer.name.startswith('point_with_date')) date_attr = get_layer_attr(layer, 'date_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, ) self.generic_time_check(layer, attribute=date_attr.attribute) def test_boxes_with_end_date(self): """Tests the import of test_boxes_with_end_date. This layer has a date and an end date field that are typed correctly. """ layer = self.generic_import( 'boxes_with_end_date.shp', configs=[ { 'index': 0, 'convert_to_date': ['date', 'enddate'], 'start_date': 'date', 'end_date': 'enddate', 'configureTime': True } ] ) date_attr = get_layer_attr(layer, 'date_as_date') end_date_attr = get_layer_attr(layer, 'enddate_as_date') self.assertEqual(date_attr.attribute_type, 'xsd:dateTime') self.assertEqual(end_date_attr.attribute_type, 'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute ) self.generic_time_check( layer, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute ) def test_us_states_kml(self): """Tests the import of us_states_kml. This layer has a date and an end date field that are typed correctly. """ # TODO: Support time in kmls. layer = self.generic_import( 'us_states.kml', configs=[ { 'index': 0 } ] ) self.assertEqual(layer.name.lower(), "us_states") def test_mojstrovka_gpx(self): """Tests the import of mojstrovka.gpx. This layer has a date and an end date field that are typed correctly. """ layer = self.generic_import( 'mojstrovka.gpx', configs=[ { 'index': 0, 'convert_to_date': ['time'], 'configureTime': True } ] ) date_attr = get_layer_attr(layer, 'time_as_date') self.assertEqual(date_attr.attribute_type, u'xsd:dateTime') configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_attr.attribute ) self.generic_time_check(layer, attribute=date_attr.attribute) def generic_time_check(self, layer, attribute=None, end_attribute=None): """Convenience method to run generic tests on time layers. """ # TODO: can we use public API or omit this? self.catalog._cache.clear() resource = self.catalog.get_resource( layer.name, store=layer.store, workspace=self.workspace ) time_info = resource.metadata['time'] self.assertEqual('LIST', time_info.presentation) self.assertEqual(True, time_info.enabled) self.assertEqual(attribute, time_info.attribute) self.assertEqual(end_attribute, time_info.end_attribute) def test_us_shootings_csv(self): """Tests the import of US_Shootings.csv. """ if osgeo.gdal.__version__ < '2.0.0': self.skipTest('GDAL Version does not support open options') path = test_file('US_Shootings.csv') layer = self.generic_import( path, configs=[ { 'index': 0, 'convert_to_date': ['Date'] } ] ) self.assertTrue(layer.name.startswith('us_shootings')) date_field = 'date' configure_time( self.catalog.get_layer(layer.name).resource, attribute=date_field ) self.generic_time_check(layer, attribute=date_field) def test_sitins(self): """Tests the import of US_Civil_Rights_Sitins0.csv """ if osgeo.gdal.__version__ < '2.0.0': self.skipTest('GDAL Version does not support open options') layer = self.generic_import( 'US_Civil_Rights_Sitins0.csv', configs=[ { 'index': 0, 'convert_to_date': ['Date'] } ] ) self.assertEqual(layer.name.lower(), 'us_civil_rights_sitins0') def get_layer_names(self, path): """Gets layer names from a data source. """ data_source = DataSource(path) return [layer.name for layer in data_source] def test_gdal_import(self): path = test_file('point_with_date.geojson') self.generic_import( path, configs=[ { 'index': 0, 'convert_to_date': ['date'] } ] ) def test_wfs(self): """Tests the import from a WFS Endpoint """ wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs' ogr = OGRImport(wfs) configs = [ {'layer_name': 'og:bugsites'}, {'layer_name': 'topp:states'} ] layers = ogr.handle(configuration_options=configs) for result in layers: layer = Layer.objects.get(name=result[0]) self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') def test_arcgisjson(self): """Tests the import from a WFS Endpoint """ endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json' ogr = OGRImport(endpoint) configs = [{'index':0}] layers = ogr.handle(configuration_options=configs) for result in layers: layer = Layer.objects.get(name=result[0]) self.assertEqual(layer.srid, 'EPSG:4326') self.assertEqual(layer.store, self.datastore.name) self.assertEqual(layer.storeType, 'dataStore') def test_file_add_view(self): """Tests the file_add_view. """ client = AdminClient() # test login required for this view request = client.get(reverse('uploads-new')) self.assertEqual(request.status_code, 302) client.login_as_non_admin() with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) self.assertEqual(response.status_code, 200) self.assertEqual(response.context['request'].path, reverse('uploads-list')) self.assertEqual(len(response.context['object_list']), 1) upload = response.context['object_list'][0] self.assertEqual(upload.user.username, 'non_admin') self.assertEqual(upload.file_type, 'GeoJSON') self.assertTrue(upload.uploadlayer_set.all()) self.assertEqual(upload.state, 'UPLOADED') self.assertIsNotNone(upload.name) uploaded_file = upload.uploadfile_set.first() self.assertTrue(os.path.exists(uploaded_file.file.path)) with open(test_file('empty_file.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) self.assertEqual(response.status_code, 200) self.assertIn('file', response.context_data['form'].errors) def test_file_add_view_as_json(self): """Tests the file_add_view. """ client = AdminClient() client.login_as_non_admin() with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new-json'), {'file': stream}, follow=True ) self.assertEqual(response.status_code, 200) self.assertIn('application/json', response.get('Content-Type', '')) content = json.loads(response.content) self.assertIn('state', content) self.assertIn('id', content) def test_describe_fields(self): """Tests the describe fields functionality. """ path = test_file('US_Shootings.csv') with GDALInspector(path) as inspector: layers = inspector.describe_fields() self.assertTrue(layers[0]['layer_name'], 'us_shootings') self.assertEqual([n['name'] for n in layers[0]['fields']], ['Date', 'Shooter', 'Killed', 'Wounded', 'Location', 'City', 'Longitude', 'Latitude']) self.assertEqual(layers[0]['feature_count'], 203) def test_gdal_file_type(self): """Tests the describe fields functionality. """ filenames = { 'US_Shootings.csv': 'CSV', 'point_with_date.geojson': 'GeoJSON', 'mojstrovka.gpx': 'GPX', 'us_states.kml': 'KML', 'boxes_with_year_field.shp': 'ESRI Shapefile', 'boxes_with_date_iso_date.zip': 'ESRI Shapefile' } from .models import NoDataSourceFound try: for filename, file_type in sorted(filenames.items()): path = test_file(filename) with GDALInspector(path) as inspector: self.assertEqual(inspector.file_type(), file_type) except NoDataSourceFound: logging.exception('No data source found in: {0}'.format(path)) raise def test_configure_view(self): """Tests the configuration view. """ path = test_file('point_with_date.geojson') new_user = User.objects.create(username='******') new_user_perms = ['change_resourcebase_permissions'] client = AdminClient() client.login_as_non_admin() with open(path) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) upload = response.context['object_list'][0] payload = [ { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'editable': True, 'permissions': { 'users': { 'test': new_user_perms, 'AnonymousUser': [ 'change_layer_data', 'download_resourcebase', 'view_resourcebase' ] } } } ] response = client.post( '/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload), content_type='application/json' ) self.assertTrue(response.status_code, 200) first_layer = Layer.objects.all()[0] self.assertEqual(first_layer.srid, 'EPSG:4326') self.assertEqual(first_layer.store, self.datastore.name) self.assertEqual(first_layer.storeType, 'dataStore') self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime') self.assertEqual( Layer.objects.all()[0].owner.username, self.non_admin_user.username ) perms = first_layer.get_all_level_info() user = User.objects.get(username=self.non_admin_user.username) # check user permissions expected_perms = [ u'publish_resourcebase', u'change_resourcebase_permissions', u'delete_resourcebase', u'change_resourcebase', u'change_resourcebase_metadata', u'download_resourcebase', u'view_resourcebase', u'change_layer_style', u'change_layer_data' ] for perm in expected_perms: self.assertIn(perm, perms['users'][user]) self.assertTrue(perms['users'][new_user]) self.assertIn( 'change_resourcebase_permissions', perms['users'][new_user] ) self.assertIn( 'change_layer_data', perms['users'][User.objects.get(username='******')] ) catalog_layer = self.catalog.get_layer(first_layer.name) self.assertIn('time', catalog_layer.resource.metadata) self.assertEqual(UploadLayer.objects.first().layer, first_layer) def test_configure_view_convert_date(self): """Tests the configure view with a dataset that needs to be converted to a date. """ client = AdminClient() client.login_as_non_admin() with open(test_file('US_Shootings.csv')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) upload = response.context['object_list'][0] payload = [ { 'index': 0, 'convert_to_date': ['Date'], 'start_date': 'Date', 'configureTime': True, 'editable': True } ] response = client.get( '/importer-api/data-layers/{0}/configure/'.format(upload.id) ) self.assertEqual(response.status_code, 405) response = client.post( '/importer-api/data-layers/{0}/configure/'.format(upload.id) ) self.assertEqual(response.status_code, 400) response = client.post( '/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload), content_type='application/json' ) self.assertTrue(response.status_code, 200) first_layer = Layer.objects.all()[0] self.assertEqual(first_layer.srid, 'EPSG:4326') self.assertEqual(first_layer.store, self.datastore.name) self.assertEqual(first_layer.storeType, 'dataStore') self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime') self.assertTrue(first_layer.attributes.filter(attribute='date'), 'xsd:dateTime') catalog_layer = self.catalog.get_layer(first_layer.name) self.assertIn('time', catalog_layer.resource.metadata) # ensure a user who does not own the upload cannot configure an import from it. client.logout() client.login_as_admin() response = client.post( '/importer-api/data-layers/{0}/configure/'.format(upload.id) ) self.assertEqual(response.status_code, 404) def test_list_api(self): client = AdminClient() response = client.get('/importer-api/data/') self.assertEqual(response.status_code, 401) client.login_as_non_admin() response = client.get('/importer-api/data/') self.assertEqual(response.status_code, 200) admin = User.objects.get(username=self.admin_user.username) non_admin = User.objects.get(username=self.non_admin_user.username) path = test_file('US_Shootings.csv') with open(path, 'rb') as stream: uploaded_file = SimpleUploadedFile('test_data', stream.read()) admin_upload = UploadedData.objects.create(state='Admin test', user=admin) admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file)) non_admin_upload = UploadedData.objects.create(state='Non admin test', user=non_admin) non_admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file)) client.login_as_admin() response = client.get('/importer-api/data/') self.assertEqual(response.status_code, 200) body = json.loads(response.content) self.assertEqual(len(body['objects']), 2) response = client.get('/importer-api/data/?user__username=admin') self.assertEqual(response.status_code, 200) body = json.loads(response.content) self.assertEqual(len(body['objects']), 1) def test_layer_list_api(self): client = AdminClient() response = client.get('/importer-api/data-layers/') self.assertEqual(response.status_code, 401) client.login_as_non_admin() response = client.get('/importer-api/data-layers/') self.assertEqual(response.status_code, 200) def test_delete_from_non_admin_api(self): """Ensure users can delete their data. """ client = AdminClient() client = AdminClient() client.login_as_non_admin() with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) self.assertEqual(UploadedData.objects.all().count(), 1) upload_id = UploadedData.objects.first().id response = client.delete('/importer-api/data/{0}/'.format(upload_id)) self.assertEqual(response.status_code, 204) self.assertEqual(UploadedData.objects.all().count(), 0) def test_delete_from_admin_api(self): """Ensure that administrators can delete data that isn't theirs. """ client = AdminClient() client.login_as_non_admin() with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) self.assertEqual(UploadedData.objects.all().count(), 1) client.logout() client.login_as_admin() upload_id = UploadedData.objects.first().id response = client.delete('/importer-api/data/{0}/'.format(upload_id)) self.assertEqual(response.status_code, 204) self.assertEqual(UploadedData.objects.all().count(), 0) def naming_an_import(self): """Tests providing a name in the configuration options. """ client = AdminClient() client.login_as_non_admin() name = 'point-with-a-date' with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) payload = { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'name': name, 'editable': True } response = client.post( '/importer-api/data-layers/1/configure/', data=json.dumps(payload), content_type='application/json' ) self.assertEqual(response.status_code, 200) first_layer = Layer.objects.all()[0] self.assertEqual(first_layer.title, name.replace('-', '_')) def test_api_import(self): """Tests the import api. """ client = AdminClient() client.login_as_non_admin() with open(test_file('point_with_date.geojson')) as stream: response = client.post( reverse('uploads-new'), {'file': stream}, follow=True ) payload = { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True, 'editable': True } self.assertIsInstance( UploadLayer.objects.first().configuration_options, dict ) response = client.get('/importer-api/data-layers/1/') self.assertEqual(response.status_code, 200) response = client.post( '/importer-api/data-layers/1/configure/', data=json.dumps([payload]), content_type='application/json' ) self.assertEqual(response.status_code, 200) self.assertIn('task', response.content) first_layer = Layer.objects.all()[0] self.assertEqual(first_layer.srid, 'EPSG:4326') self.assertEqual(first_layer.store, self.datastore.name) self.assertEqual(first_layer.storeType, 'dataStore') self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime') catalog_layer = self.catalog.get_layer(first_layer.name) self.assertIn('time', catalog_layer.resource.metadata) self.assertEqual(UploadLayer.objects.first().layer, first_layer) self.assertTrue(UploadLayer.objects.first().task_id) def test_valid_file_extensions(self): """Test the file extension validator. """ for extension in load_handler(OSGEO_IMPORTER, 'test.txt').valid_extensions: filename = 'test.{0}'.format(extension) upload = SimpleUploadedFile(filename, '') self.assertIsNone(validate_file_extension(upload)) with self.assertRaises(ValidationError): validate_file_extension(SimpleUploadedFile('test.txt', '')) def test_no_geom(self): """Test the file extension validator. """ with self.assertRaises(ValidationError): validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)')) # This should pass (geom type is unknown) validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)')) def test_numeric_overflow(self): """Regression test for numeric field overflows in shapefiles. # https://trac.osgeo.org/gdal/ticket/5241 """ self.generic_import( 'Walmart.zip', configs=[ { 'configureTime':False, 'convert_to_date': ['W1_OPENDAT'], 'editable': True, 'index':0, 'name': 'Walmart', 'start_date': 'W1_OPENDAT' } ] ) def test_multipolygon_shapefile(self): """Tests shapefile with multipart polygons. """ self.generic_import('PhoenixFirstDues.zip', configs=[{'index': 0}]) def test_istanbul(self): """Tests shapefile with multipart polygons and non-WGS84 SR. """ result = self.generic_import( 'Istanbul.zip', configs=[ {'index': 0} ] ) feature_type = self.catalog.get_resource(result.name) self.assertEqual(feature_type.projection, 'EPSG:32635') def test_gwc_handler(self): """Tests the GeoWebCache handler """ layer = self.generic_import( 'boxes_with_date.shp', configs=[ { 'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', 'configureTime': True } ] ) gwc = GeoWebCacheHandler(None) gs_layer = self.catalog.get_layer(layer.name) self.assertTrue(gwc.time_enabled(gs_layer)) gs_layer.fetch() payload = self.catalog.http.request(gwc.gwc_url(gs_layer)) self.assertIn('regexParameterFilter', payload[1]) self.assertEqual(int(payload[0]['status']), 200) # Don't configure time, ensure everything still works layer = self.generic_import( 'boxes_with_date_iso_date.shp', configs=[ { 'index': 0 } ] ) gs_layer = self.catalog.get_layer(layer.name) self.catalog._cache.clear() gs_layer.fetch() payload = self.catalog.http.request(gwc.gwc_url(gs_layer)) self.assertNotIn('regexParameterFilter', payload[1]) self.assertEqual(int(payload[0]['status']), 200) def test_utf8(self): """Tests utf8 characters in attributes """ path = test_file('china_provinces.shp') layer = self.generic_import(path) ogr = OGRImport(path) datastore, _ = ogr.open_target_datastore(ogr.target_store) sql = ( "select NAME_CH from {0} where NAME_PY = 'An Zhou'" .format(layer.name) ) result = datastore.ExecuteSQL(sql) feature = result.GetFeature(0) self.assertEqual(feature.GetField('name_ch'), '安州') def test_non_converted_date(self): """Test converting a field as date. """ results = self.generic_import( 'TM_WORLD_BORDERS_2005.zip', configs=[ { 'index': 0, 'start_date': 'Year', 'configureTime': True } ] ) layer = self.catalog.get_layer(results.typename) self.assertIn('time', layer.resource.metadata) self.assertEqual('year', layer.resource.metadata['time'].attribute) def test_fid_field(self): """ Regression test for preserving an FID field when target layer supports it but source does not. """ self.generic_import( 'noaa_paleoclimate.zip', configs=[ { 'index': 0 } ] ) def test_csv_with_wkb_geometry(self): """Exercise import of CSV files with multiple geometries. """ filenames = [ 'police_csv.csv', 'police_csv_nOGC.csv', 'police_csv_noLatLon.csv', 'police_csv_WKR.csv', 'police_csv_nFID.csv', 'police_csv_nOGFID.csv', 'police_csv_noWKB.csv' ] for filename in filenames: self.generic_import( filename, { 'configureTime': True, 'convert_to_date': ['date_time'], 'editable': True, 'index': 0, 'name': filename.lower(), 'permissions': { 'users':{ 'AnonymousUser': [ 'change_layer_data', 'download_resourcebase', 'view_resourcebase' ] } }, 'start_date':'date_time', } )
def geoserver_pre_save(instance, sender, **kwargs): """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ url = ogc_server_settings.rest try: gs_catalog = Catalog(url, _user, _password) gs_resource = gs_catalog.get_resource(instance.name) except (EnvironmentError, FailedRequestError) as e: gs_resource = None msg = ('Could not connect to geoserver at "%s"' 'to save information for layer "%s"' % (ogc_server_settings.LOCATION, instance.name.encode('utf-8'))) logger.warn(msg, e) # If geoserver is not online, there is no need to continue return # If there is no resource returned it could mean one of two things: # a) There is a synchronization problem in geoserver # b) The unit tests are running and another geoserver is running in the # background. # For both cases it is sensible to stop processing the layer if gs_resource is None: logger.warn('Could not get geoserver resource for %s' % instance) return gs_resource.title = instance.title gs_resource.abstract = instance.abstract gs_resource.name = instance.name # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.name, link.mime, link.url)) gs_resource.metadata_links = metadata_links #gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) gs_layer = gs_catalog.get_layer(instance.name) if instance.poc and instance.poc.user: gs_layer.attribution = str(instance.poc.user) profile = Profile.objects.get(user=instance.poc.user) gs_layer.attribution_link = settings.SITEURL[: -1] + profile.get_absolute_url( ) #gs_layer should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_layer) """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ gs_resource = gs_catalog.get_resource(instance.name) bbox = gs_resource.latlon_bbox #FIXME(Ariel): Correct srid setting below #self.srid = gs_resource.src # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] instance.update_thumbnail(save=False)
def calculate_yield(db, schema): try: conn = psycopg2.connect( "dbname={0} user={1} host={2} password={3}".format( db, cfg.connection['user'], cfg.connection['host'], cfg.connection['password'])) cur = conn.cursor() storename = db + '_' + schema + '_agareas' cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'], disable_ssl_certificate_validation=True) try: print('Check if the layer exists') something = cat.get_store(storename, cfg.geoserver['workspace']) if not something: print("No store") raise Exception else: print("Store exists") except Exception as e: print('Entering geoserver code') temp_dir = tempfile.mkdtemp() pg_sql = """SELECT * FROM {0}.agareas""".format(schema) export_pg_table(temp_dir, storename, cfg.connection['host'], cfg.connection['user'], cfg.connection['password'], db, pg_sql) target_zip = os.path.join( os.path.join(temp_dir, storename + '.zip')) with zipfile.ZipFile(target_zip, 'w') as pg_zip: for f in os.listdir(os.path.join(temp_dir)): if '.zip' not in f: f = os.path.join(temp_dir, f) pg_zip.write(f, basename(f)) rest_url = cfg.geoserver['rest_url'] if rest_url[-1] != "/": rest_url = rest_url + '/' headers = { 'Content-type': 'application/zip', } request_url = '{0}workspaces/{1}/datastores/{2}/file.shp'.format( rest_url, cfg.geoserver['workspace'], storename) # Creating the rest url user = cfg.geoserver['user'] password = cfg.geoserver['password'] requests.put( request_url, verify=False, headers=headers, data=open(target_zip, 'rb'), auth=(user, password)) # Creating the resource on the geoserver if temp_dir is not None: if os.path.exists(temp_dir): print('whooo') #shutil.rmtree(temp_dir) sql = """SELECT gid,avg(max) as max FROM(SELECT gid,ensemble,max(gwad) FROM {0}.dssat GROUP BY gid,ensemble ORDER BY gid,ensemble) as foo GROUP BY gid""".format( schema) cur.execute(sql) data = cur.fetchall() data.sort() conn.close() return data, storename except Exception as e: print(e) return e
class UploaderBase(TestCase): settings_overrides = [] @classmethod def setUpClass(cls): super(UploaderBase, cls).setUpClass() # don't accidentally delete anyone's layers if Layer.objects.all().count(): if 'DELETE_LAYERS' not in os.environ: print print 'FAIL: There are layers in the test database' print 'Will not run integration tests unless `DELETE_LAYERS`' print 'Is specified as an environment variable' print raise Exception('FAIL, SEE ABOVE') # make a test_settings module that will apply our overrides test_settings = ['from geonode.settings import *'] if os.path.exists('geonode/upload/tests/test_settings.py'): test_settings.append( 'from geonode.upload.tests.test_settings import *') for so in cls.settings_overrides: test_settings.append('%s=%s' % so) with open('integration_settings.py', 'w') as fp: fp.write('\n'.join(test_settings)) # runserver with settings args = [ 'python', 'manage.py', 'runserver', '--settings=integration_settings', '--verbosity=0' ] # see http://www.doughellmann.com/PyMOTW/subprocess/#process-groups-sessions cls._runserver = subprocess.Popen(args, stderr=open('test.log', 'w'), preexec_fn=os.setsid) # await startup cl = Client(GEONODE_URL, GEONODE_USER, GEONODE_PASSWD) for i in range(5): try: cl.get_html('/') break except: time.sleep(.5) if cls._runserver.poll() is not None: raise Exception("Error starting server, check test.log") @classmethod def tearDownClass(cls): super(UploaderBase, cls).tearDownClass() # kill server process group os.killpg(cls._runserver.pid, signal.SIGKILL) if os.path.exists('integration_settings.py'): os.unlink('integration_settings.py') def setUp(self): super(UploaderBase, self).setUp() self._tempfiles = [] self.client = Client(GEONODE_URL, GEONODE_USER, GEONODE_PASSWD) self.catalog = Catalog(GEOSERVER_URL + 'rest', GEOSERVER_USER, GEOSERVER_PASSWD) # @todo - this is obviously the brute force approach - ideally, # these cases would be more declarative and delete only the things # they mess with for l in Layer.objects.all(): try: l.delete() except: print 'unable to delete layer', l # and destroy anything left dangling on geoserver cat = Layer.objects.gs_catalog map(lambda name: cascading_delete(cat, name), [l.name for l in cat.get_layers()]) def tearDown(self): super(UploaderBase, self).tearDown() map(os.unlink, self._tempfiles) def check_layer_geonode_page(self, path): """ Check that the final layer page render's correctly after an layer is uploaded """ # the final url for uploader process. This does a redirect to # the final layer page in geonode resp, _ = self.client.get_html(path) self.assertTrue('content-type' in resp.headers) # if we don't get a content type of html back, thats how we # know there was an error. self.assertTrue(resp.headers['content-type'].startswith('text/html')) def check_layer_geoserver_caps(self, original_name): """ Check that a layer shows up in GeoServer's get capabilities document """ # using owslib wms = get_wms(layer_name=original_name) self.assertTrue(original_name in wms.contents, '%s is not in %s' % (original_name, wms.contents)) def check_layer_geoserver_rest(self, original_name): """ Check that a layer shows up in GeoServer rest api after the uploader is done""" # using gsconfig to test the geoserver rest api. layer = self.catalog.get_layer(original_name) self.assertIsNotNone(layer is not None) def check_and_pass_through_timestep(self): raise Exception('not implemented') redirect_to = data['redirect_to'] self.assertEquals(redirect_to, upload_step('time')) resp = self.client.make_request(upload_step('time')) self.assertEquals(resp.code, 200) data = {'csrfmiddlewaretoken': self.client.get_crsf_token()} resp = self.client.make_request(upload_step('time'), data) data = json.loads(resp.read()) return resp, data def complete_raster_upload(self, file_path, resp, data): return self.complete_upload(file_path, resp, data, is_raster=True) def check_save_step(self, resp, data): """Verify the initial save step""" self.assertEquals(resp.code, 200) self.assertTrue(isinstance(data, dict)) # make that the upload returns a success True key self.assertTrue(data['success'], 'expected success but got %s' % data) self.assertTrue('redirect_to' in data) def complete_upload(self, file_path, resp, data, is_raster=False): """Method to check if a layer was correctly uploaded to the GeoNode. arguments: file path, the django http response Checks to see if a layer is configured in Django Checks to see if a layer is configured in GeoServer checks the Rest API checks the get cap document """ layer_name, ext = os.path.splitext(os.path.basename(file_path)) self.check_save_step(resp, data) layer_page = self.finish_upload(data['redirect_to'], layer_name, is_raster) self.check_layer_complete(layer_page, layer_name) def finish_upload(self, current_step, layer_name, is_raster=False, skip_srs=False): if (not is_raster and _ALLOW_TIME_STEP): resp, data = self.check_and_pass_through_timestep() self.assertEquals(resp.code, 200) self.assertTrue(data['success'], 'expected success but got %s' % data) self.assertTrue('redirect_to' in data) current_step = data['redirect_to'] self.wait_for_progress(data.get('progress')) if not is_raster and not skip_srs: self.assertEquals(current_step, upload_step('srs')) # if all is good, the srs step will redirect to the final page resp = self.client.get(current_step) else: self.assertEquals(current_step, upload_step('final')) resp = self.client.get(current_step) self.assertEquals(resp.code, 200) url = json.loads(resp.read())['url'] # and the final page should redirect to tha layer page # @todo - make the check match completely (endswith at least) # currently working around potential 'orphaned' db tables self.assertTrue(layer_name in url, 'expected %s in URL, got %s' % (layer_name, url)) return url def check_upload_model(self, original_name): # we can only test this if we're using the same DB as the test instance if not settings.OGC_SERVER['default']['OPTIONS']['DATASTORE']: return try: upload = Upload.objects.get(layer__name=original_name) except Upload.DoesNotExist: self.fail('expected to find Upload object for %s' % original_name) self.assertTrue(upload.complete) def check_layer_complete(self, layer_page, original_name): '''check everything to verify the layer is complete''' self.check_layer_geonode_page(layer_page) # @todo use the original_name # currently working around potential 'orphaned' db tables # this grabs the name from the url (it might contain a 0) original_name = os.path.basename(layer_page).split(':')[1] self.check_layer_geoserver_caps(original_name) self.check_layer_geoserver_rest(original_name) self.check_upload_model(original_name) def check_invalid_projection(self, layer_name, resp, data): """ Makes sure that we got the correct response from an layer that can't be uploaded""" if _ALLOW_TIME_STEP: resp, data = self.check_and_pass_through_timestep() self.assertTrue(resp.code, 200) self.assertTrue(data['success']) self.assertEquals(upload_step("srs"), data['redirect_to']) resp, soup = self.client.get_html(data['redirect_to']) # grab an h2 and find the name there as part of a message saying it's bad h2 = soup.find_all(['h2'])[0] self.assertTrue(str(h2).find(layer_name)) def upload_folder_of_files(self, folder, final_check): mains = ('.tif', '.shp', '.zip') def is_main(_file): _, ext = os.path.splitext(_file) return (ext.lower() in mains) self.client.login() main_files = filter(is_main, os.listdir(folder)) for main in main_files: # get the abs path to the file _file = os.path.join(folder, main) base, _ = os.path.splitext(_file) resp, data = self.client.upload_file(_file) self.wait_for_progress(data.get('progress')) final_check(base, resp, data) def upload_file(self, fname, final_check, check_name=None): self.client.login() if not check_name: check_name, _ = os.path.splitext(fname) resp, data = self.client.upload_file(fname) self.wait_for_progress(data.get('progress')) final_check(check_name, resp, data) def wait_for_progress(self, progress_url): if progress_url: resp = self.client.get(progress_url) assert resp.getcode() == 200, 'Invalid progress status code' raw_data = resp.read() json_data = json.loads(raw_data) # "COMPLETE" state means done if json_data.get('state', '') == 'RUNNING': time.sleep(0.1) self.wait_for_progress(progress_url) def temp_file(self, ext): fd, abspath = tempfile.mkstemp(ext) self._tempfiles.append(abspath) return fd, abspath def make_csv(self, *rows): fd, abspath = self.temp_file('.csv') fp = os.fdopen(fd, 'wb') out = csv.writer(fp) for r in rows: out.writerow(r) fp.close() return abspath
class wrap_geoserver: """ Geoserver (gsconfig) wrapper """ def __init__( self, geoserver_name, username=username, password=password, easy=False ): if geoserver_name in list(REST.keys()): self.path = REST[geoserver_name] else: self.path = geoserver_name self.wms = self.path.replace("rest/", "wms") self.name = geoserver_name self.catalog = Catalog(self.path, username, password) if not easy: self.layers = [] self.layer_names = [] for layer in self.catalog.get_layers(): self.layers.append(layer) self.layer_names.append(layer.name) self.stores = [store for store in self.catalog.get_stores()] self.store_names = [store.name for store in self.stores] styles = [] self.workspaces = [] self.workspace_names = [] for workspace in self.catalog.get_workspaces(): styles = styles + self.catalog.get_styles(workspace) self.workspace_names.append(workspace._name) self.workspaces.append(workspace) self.styles = styles + [style for style in self.catalog.get_styles()] self.style_names = [style.name for style in self.styles] def unpack(self, workspace_name, store_type="datastore"): layers_and_styles = {} features = [] workspace = self.get_workspace(workspace_name) if store_type == "datastore": store_url = workspace.datastore_url elif store_type == "coveragestore": store_url = workspace.coveragestore_url else: print("No correct store given") for datastore in tqdm(get(store_url, "name")): url = "{}workspaces/{}/datastores/{}".format( self.path, workspace.name, datastore ) features = features + get(url, between_quotes=True) for feature in features: layer_name = os.path.basename(feature).split(".")[0] self.get_layer(self.get_slug(workspace.name, layer_name)) layers_and_styles[layer_name] = self.layer.default_style setattr(self, workspace_name + "_data", layers_and_styles) return layers_and_styles def get_layer(self, layer, easy=False): self.layer = self.catalog.get_layer(layer) if not easy: self.resource = self.layer.resource self.layer_name = self.layer.resource.name self.sld_name = self.layer.default_style.name self.sld_body = self.layer.default_style.sld_body self.layer_latlon_bbox = self.layer.resource.latlon_bbox self.layer_title = self.layer.resource.title self.layer_abstract = self.layer.resource.abstract def get_store(self, layer): self.store = self.layer.resource._store def get_resource(self): self.resource = self.catalog.get_resource(self.layer.name, self.store) def get_workspace(self, workspace_name): self.workspace = self.catalog.get_workspace(workspace_name) self.workspace_name = self.workspace._name return self.workspace def write_abstract(self, data, load_resource=True): if load_resource: self.get_resource() self.resource.abstract = data self.catalog.save(self.resource) def write_title(self, title): self.resource.title = title self.catalog.save(self.resource) def get_connection_parameters(self): self.get_resource() return self.resource.store.connection_parameters def create_workspace(self, workspace_name): workspace_exists = workspace_name in self.workspace_names if not workspace_exists: self.workspace = self.catalog.create_workspace(workspace_name) else: print("workspace already exists, using existing workspace") self.workspace = self.catalog.get_workspace(workspace_name) self.workspace_name = workspace_name def create_postgis_datastore(self, store_name, workspace_name, pg_data): try: self.store = self.catalog.get_store(store_name, self.workspace_name) print("store within workspace exists, using existing store") except Exception as e: print(e) ds = self.catalog.create_datastore(store_name, workspace_name) ds.connection_parameters.update( host=pg_data["host"], port=pg_data["port"], database=pg_data["database"], user=pg_data["username"], passwd=pg_data["password"], dbtype="postgis", schema="public", ) self.save(ds) self.store = self.catalog.get_store(store_name, self.workspace_name) self.store_name = store_name def publish_layer( self, layer_name, workspace_name, overwrite=False, epsg="3857", reload=False ): layer_exists = layer_name in self.layer_names # if layer_name in self.workspace_layers[workspace_name]: slug = self.get_slug(workspace_name, layer_name) if overwrite and layer_exists: print("Layer exists, deleting layer") try: self.layer = self.catalog.get_layer(slug) self.delete(self.layer) self.reload() layer_exists = False except Exception as e: print(e) print("Layer does not exist in workspace") layer_exists = False if not layer_exists: feature_type = self.catalog.publish_featuretype( layer_name, self.store, "EPSG:{}".format(str(epsg)), srs="EPSG:{}".format(str(epsg)), ) self.save(feature_type) self.feature_type = feature_type else: print("layer already exists, using existing layer") if reload: self.get_layer(slug) self.layer_name = layer_name def publish_layergroup(self, name, layers, styles=(), bounds=None, workspace=None): layer_group = self.catalog.create_layergroup( name, layers, styles, bounds, workspace ) self.save(layer_group) def save(self, save_object): return self.catalog.save(save_object) def close(self): self.catalog = None def delete(self, delete_object): self.catalog.delete(delete_object) def reload(self): self.catalog.reload() def upload_shapefile(self, layer_name, shapefile_path): path = shapefile_path.split(".shp")[0] shapefile = shapefile_and_friends(path) ft = self.catalog.create_featurestore(layer_name, shapefile, self.workspace) self.save(ft) def upload_sld(self, sld_name, workspace_name, sld, overwrite=True): style_exists = sld_name in self.style_names if overwrite and style_exists: print("Overwriting style") style = self.catalog.get_style(sld_name, workspace_name) self.delete(style) self.reload() style_exists = False if not style_exists: try: self.catalog.create_style(sld_name, sld, False, workspace_name, "sld11") except Exception as e: print(e) style = self.catalog.get_style(sld_name, workspace_name) self.delete(style) self.reload() self.catalog.create_style(sld_name, sld, False, workspace_name, "sld10") self.style_name = sld_name else: if style_exists: print("Style already exists, using current style") self.style_name = sld_name def set_sld_for_layer(self, workspace_name=None, style_name=None, use_custom=False): if not use_custom: workspace_name = self.workspace_name style_name = self.style_name self.style_slug = self.get_slug(workspace_name, style_name) else: if workspace_name is None: self.style_slug = style_name else: self.style_slug = self.get_slug(workspace_name, style_name) self.style = self.catalog.get_style(self.style_slug) print("Setting {} for {}".format(self.style.name, self.layer.name)) self.layer.default_style = self.style self.save(self.layer) def get_slug(self, workspace, name): return "{}:{}".format(workspace, name) def get_slug_data(self, slug): workspace_name = slug.split(":")[0] layer_name = slug.split(":")[1] return workspace_name, layer_name def get_sld(self, layer_slug=None): if layer_slug is None: self.style = self.catalog.get_style(self.layer_slug) else: self.style = self.catalog.get_style(layer_slug) self.sld_body = self.style.sld_body return self.sld_body def get_layer_workspace(self, layer_name): return self.catalog.get_layer(layer_name).resource.workspace.name
def _geoserver_info_provision(url): from geoserver.catalog import Catalog from geosk.mdtools.geoserver_extra import Settings cat = Catalog(url, username="******", password="******" ) gs_settings = Settings(cat) print "GeoServer service url is {0}".format(cat.service_url) contact_country = os.getenv("SERVICEPROVIDER_COUNTRY", "") print "contact_country is {0}".format(contact_country) provider_url = os.getenv("SERVICEPROVIDER_SITE", "") print "provider_url is {0}".format(provider_url) contact_role = os.getenv("SERVICEPROVIDER_INDIVIDUALNAME", "") print "contact_role is {0}".format(contact_role) contact_city = os.getenv("SERVICEPROVIDER_CITY", "") print "contact_city is {0}".format(contact_city) contact_instructions = os.getenv("SERVICEPROVIDER_INSTRUCTIONS", "") print "contact_instructions is {0}".format(contact_instructions) contact_position = os.getenv("SERVICEPROVIDER_POSITIONNAME", "") print "contact_position is {0}".format(contact_position) contact_fax = os.getenv("SERVICEPROVIDER_FAX", "") print "contact_fax is {0}".format(contact_fax) node_title = os.getenv("SERVICEPROVIDER_NODETITLE", "") print "node_title is {0}".format(node_title) contact_hours = os.getenv("SERVICEPROVIDER_HOURS", "") print "contact_hours is {0}".format(contact_hours) node_name = os.getenv("SERVICEPROVIDER_NODENAME", "") print "node_name is {0}".format(node_name) node_abstract = os.getenv("SERVICEPROVIDER_NODEABSTRACT", "") print "node_abstract is {0}".format(node_abstract) contact_address = os.getenv("SERVICEPROVIDER_ADDRESS", "") print "contact_address is {0}".format(contact_address) contact_email = os.getenv("SERVICEPROVIDER_EMAIL", "") print "contact_email is {0}".format(contact_email) contact_url = os.getenv("SERVICEPROVIDER_SITE", "") print "contact_url is {0}".format(contact_url) contact_stateprovince = os.getenv("SERVICEPROVIDER_STATEPROVINCE", "") print "contact_stateprovince is {0}".format(contact_stateprovince) provider_name = os.getenv("SERVICEPROVIDER_NAME", "") print "provider_name is {0}".format(provider_name) contact_postalcode = os.getenv("SERVICEPROVIDER_POSTALCODE", "") print "contact_postalcode is {0}".format(contact_postalcode) contact_phone = os.getenv("SERVICEPROVIDER_PHONE", "") print "contact_phone is {0}".format(contact_phone) contact_name = os.getenv("SERVICEPROVIDER_NAME", "") print "contact_name is {0}".format(contact_name) node_keywords = os.getenv("SERVICEPROVIDER_NODEKEYWORDS", "") print "node_keywords is {0}".format(node_keywords) contact = { "contact": { "address": contact_address, "addressCity": contact_city, "addressCountry": contact_country, "addressPostalCode": contact_postalcode, "addressState": contact_stateprovince, "addressType": None, "contactEmail": contact_email, "contactFacsimile": contact_fax, "contactOrganization": provider_name, "contactPerson": contact_name, "contactPosition": contact_position, "contactVoice": contact_phone } } def get_service_json(service): # get current values service_base = gs_settings.get_service_config(service) service_base[service]['name'] = "{0} - {1}".format( node_name, service ) service_base[service]['title'] = "{0} - {1}".format( node_title, service ) service_base[service]['maintainer'] = provider_url if provider_url is not None and provider_url else os.environ['SITEURL'] service_base[service]['abstrct'] = node_abstract if node_abstract is not None else '-' service_base[service]['onlineResource'] = os.environ['SITEURL'] return service_base gs_settings.update_contact(contact) gs_settings.update_service('wms', get_service_json('wms')) gs_settings.update_service('wfs', get_service_json('wfs')) gs_settings.update_service('wcs', get_service_json('wcs'))
def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest")
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." url = "%srest" % settings.GEOSERVER_BASE_URL cat = Catalog(url, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = [] for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create( name=name, defaults={ "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name, resource.name), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception('Failed to process %s' % resource.name, e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' else: status = 'updated' msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) info = {'name': name, 'status': status} if status == 'failed': info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output.append(info) if verbosity > 0: print >> console, msg
class UploaderBase(GeoNodeLiveTestSupport): type = 'layer' @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): if os.path.exists('integration_settings.py'): os.unlink('integration_settings.py') def setUp(self): # await startup cl = Client(GEONODE_URL, GEONODE_USER, GEONODE_PASSWD) for i in range(10): time.sleep(.2) try: cl.get_html('/', debug=False) break except BaseException: pass self.client = Client(GEONODE_URL, GEONODE_USER, GEONODE_PASSWD) self.catalog = Catalog( GEOSERVER_URL + 'rest', GEOSERVER_USER, GEOSERVER_PASSWD, retries=ogc_server_settings.MAX_RETRIES, backoff_factor=ogc_server_settings.BACKOFF_FACTOR) settings.DATABASES['default']['NAME'] = DB_NAME connections['default'].settings_dict['ATOMIC_REQUESTS'] = False connections['default'].connect() self._tempfiles = [] def _post_teardown(self): pass def tearDown(self): connections.databases['default']['ATOMIC_REQUESTS'] = False for temp_file in self._tempfiles: os.unlink(temp_file) # Cleanup Upload.objects.all().delete() Layer.objects.all().delete() Map.objects.all().delete() Document.objects.all().delete() if settings.OGC_SERVER['default'].get("GEOFENCE_SECURITY_ENABLED", False): from geonode.security.utils import purge_geofence_all purge_geofence_all() def check_layer_geonode_page(self, path): """ Check that the final layer page render's correctly after an layer is uploaded """ # the final url for uploader process. This does a redirect to # the final layer page in geonode resp, _ = self.client.get_html(path) self.assertEqual(resp.status_code, 200) self.assertTrue('content-type' in resp.headers) def check_layer_geoserver_caps(self, type_name): """ Check that a layer shows up in GeoServer's get capabilities document """ # using owslib wms = get_wms(type_name=type_name, username=GEOSERVER_USER, password=GEOSERVER_PASSWD) ws, layer_name = type_name.split(':') self.assertTrue(layer_name in wms.contents, '%s is not in %s' % (layer_name, wms.contents)) def check_layer_geoserver_rest(self, layer_name): """ Check that a layer shows up in GeoServer rest api after the uploader is done""" # using gsconfig to test the geoserver rest api. layer = self.catalog.get_layer(layer_name) self.assertIsNotNone(layer is not None) def check_and_pass_through_timestep(self, redirect_to): time_step = upload_step('time') srs_step = upload_step('srs') if srs_step in redirect_to: resp = self.client.make_request(redirect_to) else: self.assertTrue(time_step in redirect_to) resp = self.client.make_request(redirect_to) token = self.client.get_csrf_token(True) self.assertEqual(resp.status_code, 200) resp = self.client.make_request(redirect_to, {'csrfmiddlewaretoken': token}, ajax=True) return resp, resp.json() def complete_raster_upload(self, file_path, resp, data): return self.complete_upload(file_path, resp, data, is_raster=True) def check_save_step(self, resp, data): """Verify the initial save step""" self.assertEqual(resp.status_code, 200) self.assertTrue(isinstance(data, dict)) # make that the upload returns a success True key self.assertTrue(data['success'], 'expected success but got %s' % data) self.assertTrue('redirect_to' in data) def complete_upload(self, file_path, resp, data, is_raster=False): """Method to check if a layer was correctly uploaded to the GeoNode. arguments: file path, the django http response Checks to see if a layer is configured in Django Checks to see if a layer is configured in GeoServer checks the Rest API checks the get cap document """ layer_name, ext = os.path.splitext(os.path.basename(file_path)) if not isinstance(data, string_types): self.check_save_step(resp, data) layer_page = self.finish_upload(data['redirect_to'], layer_name, is_raster) self.check_layer_complete(layer_page, layer_name) def finish_upload(self, current_step, layer_name, is_raster=False, skip_srs=False): if not is_raster and _ALLOW_TIME_STEP: resp, data = self.check_and_pass_through_timestep(current_step) self.assertEqual(resp.status_code, 200) if not isinstance(data, string_types): if data['success']: self.assertTrue(data['success'], 'expected success but got %s' % data) self.assertTrue('redirect_to' in data) current_step = data['redirect_to'] self.wait_for_progress(data.get('progress')) if not is_raster and not skip_srs: self.assertTrue(upload_step('srs') in current_step) # if all is good, the srs step will redirect to the final page final_step = current_step.replace('srs', 'final') resp = self.client.make_request(final_step) else: self.assertTrue(upload_step('final') in current_step) resp = self.client.get(current_step) self.assertEqual(resp.status_code, 200) try: c = resp.json() url = c['url'] url = unquote(url) # and the final page should redirect to the layer page # @todo - make the check match completely (endswith at least) # currently working around potential 'orphaned' db tables self.assertTrue(layer_name in url, 'expected %s in URL, got %s' % (layer_name, url)) return url except BaseException: return current_step def check_upload_model(self, original_name): # we can only test this if we're using the same DB as the test instance if not settings.OGC_SERVER['default']['DATASTORE']: return upload = None try: upload = Upload.objects.filter( name__icontains=str(original_name)).last() # Making sure the Upload object is present on the DB and # the import session is COMPLETE if upload and not upload.complete: logger.warning("Upload not complete for Layer %s" % original_name) except Upload.DoesNotExist: self.fail('expected to find Upload object for %s' % original_name) def check_layer_complete(self, layer_page, original_name): '''check everything to verify the layer is complete''' self.check_layer_geonode_page(layer_page) # @todo use the original_name # currently working around potential 'orphaned' db tables # this grabs the name from the url (it might contain a 0) type_name = os.path.basename(layer_page) layer_name = original_name try: layer_name = type_name.split(':')[1] except BaseException: pass # work around acl caching on geoserver side of things caps_found = False for i in range(10): time.sleep(.5) try: self.check_layer_geoserver_caps(type_name) caps_found = True except BaseException: pass if not caps_found: logger.warning( "Could not recognize Layer %s on GeoServer WMS Capa" % original_name) self.check_layer_geoserver_rest(layer_name) self.check_upload_model(layer_name) def check_invalid_projection(self, layer_name, resp, data): """ Makes sure that we got the correct response from an layer that can't be uploaded""" self.assertTrue(resp.status_code, 200) if not isinstance(data, string_types): self.assertTrue(data['success']) srs_step = upload_step("srs") if "srs" in data['redirect_to']: self.assertTrue(srs_step in data['redirect_to']) resp, soup = self.client.get_html(data['redirect_to']) # grab an h2 and find the name there as part of a message saying it's # bad h2 = soup.find_all(['h2'])[0] self.assertTrue(str(h2).find(layer_name)) def check_upload_complete(self, layer_name, resp, data): """ Makes sure that we got the correct response from an layer that can't be uploaded""" self.assertTrue(resp.status_code, 200) if not isinstance(data, string_types): self.assertTrue(data['success']) final_step = upload_step("final") if "final" in data['redirect_to']: self.assertTrue(final_step in data['redirect_to']) def upload_folder_of_files(self, folder, final_check, session_ids=None): mains = ('.tif', '.shp', '.zip', '.asc') def is_main(_file): _, ext = os.path.splitext(_file) return (ext.lower() in mains) for main in filter(is_main, os.listdir(folder)): # get the abs path to the file _file = os.path.join(folder, main) base, _ = os.path.splitext(_file) resp, data = self.client.upload_file(_file) if session_ids is not None: if not isinstance(data, string_types) and data.get('url'): session_id = re.search(r'.*id=(\d+)', data.get('url')).group(1) if session_id: session_ids += [session_id] if not isinstance(data, string_types): self.wait_for_progress(data.get('progress')) final_check(base, resp, data) def upload_file(self, fname, final_check, check_name=None, session_ids=None): if not check_name: check_name, _ = os.path.splitext(fname) resp, data = self.client.upload_file(fname) if session_ids is not None: if not isinstance(data, string_types): if data.get('url'): session_id = re.search(r'.*id=(\d+)', data.get('url')).group(1) if session_id: session_ids += [session_id] if not isinstance(data, string_types): self.wait_for_progress(data.get('progress')) final_check(check_name, resp, data) def wait_for_progress(self, progress_url): if progress_url: resp = self.client.get(progress_url) assert resp.getcode() == 200, 'Invalid progress status code' json_data = resp.json() # "COMPLETE" state means done if json_data.get('state', '') == 'RUNNING': time.sleep(0.1) self.wait_for_progress(progress_url) def temp_file(self, ext): fd, abspath = tempfile.mkstemp(ext) self._tempfiles.append(abspath) return fd, abspath def make_csv(self, *rows): fd, abspath = self.temp_file('.csv') fp = os.fdopen(fd, 'wb') out = csv.writer(fp) for r in rows: out.writerow(r) fp.close() return abspath
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testAbout(self): about_html = self.cat.about() self.assertTrue( '<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assert_(self.cat.get_style('non-existing-style') is None) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") try: self.cat.get_store(workspace="best workspace ever", name="best store ever") self.fail('expected exception') except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message) try: self.cat.get_resource(workspace="best workspace ever", store="best store ever", name="best resource ever") except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message)
def createDataStore(name, filename, format="shapefile"): cat = Catalog("{0}/rest/".format(geoserver_connection), geoserver_username, geoserver_password) ws = cat.get_workspace(workspace) msg = "" if format == "shapefile": shapefile = shapefile_and_friends(filename) try: cat.create_featurestore(name, shapefile, workspace) except ConflictingDataError as inst: msg = str(inst) except: raise resource = cat.get_resource(name, workspace=ws) resource.projection = 'EPSG:4326' cat.save(resource) resource.projection_policy = 'REPROJECT_TO_DECLARED' cat.save(resource) resource.refresh() bbox = resource.latlon_bbox[:4] solr_geom = 'ENVELOPE({0},{1},{2},{3})'.format(bbox[0], bbox[1], bbox[3], bbox[2]) return { "solr_geom": solr_geom, "msg": msg, "resource_type": resource.resource_type } elif format == "image": if cat.get_store(name): newcs = cat.get_store(name, workspace=ws) msg = "Geoserver datastore already existed. Update existing datastore." else: newcs = cat.create_coveragestore2(name, ws) newcs.type = "GeoTIFF" newcs.url = filename cat.save(newcs) # add coverage url = "{0}/rest/workspaces/{1}/coveragestores/{2}/coverages.json" url = url.format(geoserver_connection, ws.name, name) headers = {"Content-Type": "application/json"} coverageName = os.path.splitext(os.path.basename(filename))[0] postdata = { "coverage": { "nativeCoverageName": coverageName, "name": coverageName, 'projectionPolicy': 'REPROJECT_TO_DECLARED', 'srs': 'EPSG:4326' } } requests.post(url, json.dumps(postdata), headers=headers, auth=(geoserver_username, geoserver_password)) # Reproject resource = cat.get_resource(name, workspace=ws) url = "{0}/rest/workspaces/{1}/coveragestores/{2}/coverages/{2}?{3}" parameters = "recalculate=nativebbox,latlonbbox" url = url.format(geoserver_connection, ws.name, name, parameters) headers = {"Content-Type": "application/x-www-form-urlencoded"} requests.post(url, headers=headers, auth=(geoserver_username, geoserver_password)) # resource.refresh() bbox = resource.latlon_bbox[:4] solr_geom = 'ENVELOPE({0},{1},{2},{3})'.format(bbox[0], bbox[1], bbox[3], bbox[2]) return { "solr_geom": solr_geom, "msg": msg, "resource_type": resource.resource_type } return True
def _register_cascaded_layers(service, owner=None): """ Register layers for a cascading WMS """ if service.type == 'WMS' or service.type == "OWS": cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) # Can we always assume that it is geonode? # Should cascading layers have a separate workspace? cascade_ws = cat.get_workspace(service.name) if cascade_ws is None: cascade_ws = cat.create_workspace(service.name, 'cascade') try: store = cat.get_store(service.name, cascade_ws) except Exception: store = cat.create_wmsstore(service.name, cascade_ws) cat.save(store) wms = WebMapService(service.base_url) layers = list(wms.contents) count = 0 for layer in layers: lyr = cat.get_resource(layer, store, cascade_ws) if lyr is None: if service.type in ["WMS", "OWS"]: resource = cat.create_wmslayer(cascade_ws, store, layer) elif service.type == "WFS": resource = cat.create_wfslayer(cascade_ws, store, layer) if resource: bbox = resource.latlon_bbox cascaded_layer, created = Layer.objects.get_or_create( typename="%s:%s" % (cascade_ws.name, resource.name), service=service, defaults={ "name": resource.name, "workspace": cascade_ws.name, "store": store.name, "storeType": store.resource_type, "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": None, "uuid": str(uuid.uuid4()), "bbox_x0": bbox[0], "bbox_x1": bbox[1], "bbox_y0": bbox[2], "bbox_y1": bbox[3], }) if created: cascaded_layer.save() if cascaded_layer is not None and cascaded_layer.bbox is None: cascaded_layer._populate_from_gs( gs_resource=resource) cascaded_layer.set_default_permissions() service_layer, created = ServiceLayer.objects.get_or_create( service=service, typename=cascaded_layer.name) service_layer.layer = cascaded_layer service_layer.title = cascaded_layer.title, service_layer.description = cascaded_layer.abstract, service_layer.styles = cascaded_layer.styles service_layer.save() count += 1 else: logger.error( "Resource %s from store %s could not be saved as layer" % (layer, store.name)) message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message} return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200) elif service.type == 'WCS': return HttpResponse('Not Implemented (Yet)', status=501) else: return HttpResponse('Invalid Service Type', status=400)
def __init__(self): models.Manager.__init__(self) url = ogc_server_settings.rest self.gs_catalog = Catalog(url, _user, _password)
def get_selected_raster(db, region, variable, date): # logging.info(str(region)+','+str(variable)+','+str(date)) try: # logging.info('Connecting to the database') conn = psycopg2.connect( "dbname={0} user={1} host={2} password={3}".format( db, cfg.connection['user'], cfg.connection['host'], cfg.connection['password'])) cur = conn.cursor() storename = db + '_' + region + '_' + variable + '_' + date cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'], disable_ssl_certificate_validation=True) try: # logging.info('Check if the layer exists') something = cat.get_store(storename, cfg.geoserver['workspace']) if not something: # logging.info('Layer doesnt exist') print("No store") raise Exception else: mean, stddev, min, max = get_vic_summary( db, region, variable, date) # logging.info(str(mean)+str(stddev)+str(min)+str(max)) return storename, mean, stddev, min, max except Exception as e: # logging.info('Entering geoserver code') # logging.error('Error at failed request ' + str(e)) try: # logging.info('Starting the geoserver stuff') sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format( region, variable, date) cur.execute(sql) data = cur.fetchall() # logging.info(str(data)) mean, stddev, min, max = get_vic_summary( db, region, variable, date) # logging.info('Work you piece ...') rest_url = cfg.geoserver['rest_url'] # logging.info(str(rest_url)) if rest_url[-1] != "/": rest_url = rest_url + '/' headers = { 'Content-type': 'image/tiff', } request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format( rest_url, cfg.geoserver['workspace'], storename) # Creating the rest url # logging.info('Get the username and password') user = cfg.geoserver['user'] password = cfg.geoserver['password'] # logging.info('Right before the put command') requests.put( request_url, verify=False, headers=headers, data=data[0][0], auth=(user, password)) # Creating the resource on the geoserver # logging.info(request_url) return storename, mean, stddev, min, max except Exception as er: # logging.info('Error at uplaoding tiff '+ str(e)) return str(er) + ' This is while adding the raster layer.' except Exception as err: # logging.info(str(err) + ' This is generic catch all') return str(err) + ' This is the generic one'
from geoserver.catalog import Catalog cat = Catalog("http://172.18.77.15:8089/geoserver/rest", username="******", password="******") url = 'http://172.18.77.15:8089/geoserver/%s/wms?service=WMS&version=1.1.0&request=GetMap&layers=%s:%s&styles=&bbox=%s,%s,%s,%s&width=80&height=80&srs=%s&format=image/png' rr = cat.get_resources(workspace=cat.get_workspace('shenzhen')) for r in rr: print r.name,url%(r.workspace.name,r.workspace.name,r.name,r.native_bbox[0],r.native_bbox[2],\ r.native_bbox[1],r.native_bbox[3],r.native_bbox[4] if r.native_bbox[4] is not None \ and r.native_bbox[4].startswith('EPSG') else ('EPSG:4326' if float(r.native_bbox[0])<10000 else 'EPSG:2309' ) )
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") native_bbox = \ ['589434.856', '4914006.338', '609527.21', '4928063.398', 'EPSG:26713'] latlon_bbox = ['-103.877', '44.371', '-103.622', '44.5', 'EPSG:4326'] sf = cat.get_workspace('sf') for rs in cat.get_resources(workspace=sf): rs.native_bbox = native_bbox rs.latlon_bbox = latlon_bbox cat.save(rs)
def _register_cascaded_service(url, type, name, username, password, wms=None, owner=None, parent=None): """ Register a service as cascading WMS """ try: service = Service.objects.get(base_url=url) return_dict = {} return_dict['service_id'] = service.pk return_dict['msg'] = "This is an existing Service" return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200) except: # TODO: Handle this error properly pass if wms is None: wms = WebMapService(url) # TODO: Make sure we are parsing all service level metadata # TODO: Handle for setting ServiceProfiletRole service = Service.objects.create(base_url=url, type=type, method='C', name=name, version=wms.identification.version, title=wms.identification.title, abstract=wms.identification.abstract, online_resource=wms.provider.url, owner=owner, parent=parent) service.keywords = ','.join(wms.identification.keywords) service.save() service.set_default_permissions() if type in ['WMS', 'OWS']: # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) cascade_ws = cat.get_workspace(name) if cascade_ws is None: cascade_ws = cat.create_workspace(name, "http://geonode.org/cascade") # TODO: Make sure there isn't an existing store with that name, and # deal with it if there is try: cascade_store = cat.get_store(name, cascade_ws) except: cascade_store = cat.create_wmsstore(name, cascade_ws, username, password) cascade_store.capabilitiesURL = url cascade_store.type = "WMS" cat.save(cascade_store) available_resources = cascade_store.get_resources(available=True) elif type == 'WFS': # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) # Can we always assume that it is geonode? cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE) if cascade_ws is None: cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, "http://geonode.org/cascade") try: wfs_ds = cat.get_store(name, cascade_ws) except: wfs_ds = cat.create_datastore(name, cascade_ws) connection_params = { "WFSDataStoreFactory:MAXFEATURES": "0", "WFSDataStoreFactory:TRY_GZIP": "true", "WFSDataStoreFactory:PROTOCOL": "false", "WFSDataStoreFactory:LENIENT": "true", "WFSDataStoreFactory:TIMEOUT": "3000", "WFSDataStoreFactory:BUFFER_SIZE": "10", "WFSDataStoreFactory:ENCODING": "UTF-8", "WFSDataStoreFactory:WFS_STRATEGY": "nonstrict", "WFSDataStoreFactory:GET_CAPABILITIES_URL": url, } if username and password: connection_params["WFSDataStoreFactory:USERNAME"] = username connection_params["WFSDataStoreFactory:PASSWORD"] = password wfs_ds.connection_parameters = connection_params cat.save(wfs_ds) available_resources = wfs_ds.get_resources(available=True) # Save the Service record service, created = Service.objects.get_or_create(type=type, method='C', base_url=url, name=name, owner=owner) service.save() service.set_default_permissions() elif type == 'WCS': return HttpResponse('Not Implemented (Yet)', status=501) else: return HttpResponse('Invalid Method / Type combo: ' + 'Only Cascaded WMS, WFS and WCS supported', mimetype="text/plain", status=400) message = "Service %s registered" % service.name return_dict = [{ 'status': 'ok', 'msg': message, 'service_id': service.pk, 'service_name': service.name, 'service_title': service.title, 'available_layers': available_resources }] if settings.USE_QUEUE: # Create a layer import job WebServiceHarvestLayersJob.objects.get_or_create(service=service) else: _register_cascaded_layers(service) return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200)