def final_step(upload_session, user): import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = Layer.objects.gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py # @todo see above in save_step, regarding computed unique name name = import_session.tasks[0].items[0].layer.name import time time.sleep(4) _log('Creating style for [%s]', name) publishing = cat.get_layer(name) if publishing is None: raise Exception("Expected to find layer named '%s' in geoserver", name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = os.path.join( os.path.dirname(base_file), upload_session.import_sld_file ) f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError, e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) #FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing)
def final_step(upload_session, user): import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = Layer.objects.gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py # @todo see above in save_step, regarding computed unique name name = import_session.tasks[0].items[0].layer.name _log('Creating style for [%s]', name) publishing = cat.get_layer(name) if publishing is None: raise Exception("Expected to find layer named '%s' in geoserver", name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = os.path.join( os.path.dirname(base_file), upload_session.import_sld_file ) f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError, e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) #FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing)
def handle(self, layer, layer_config, *args, **kwargs): """ Replace the default layer style with one of the boilerplate styles. """ name = self.layer.name sld = get_sld_for(self.catalog, self.layer) style = None if sld is not None: try: self.catalog.create_style(name, sld, raw=True) except ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) logger.error(msg) try: self.catalog.create_style(name + '_layer', sld, raw=True) except ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg, ) if style is None: try: style = self.catalog.get_style(name) except BaseException: logger.warn( 'Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = self.catalog.get_style(name + '_layer') except BaseException: style = self.catalog.get_style('point') logger.warn(msg) e.args = (msg, ) if style: self.layer.default_style = style self.catalog.save(self.layer) save_style(style)
def handle(self, layer, layer_config, *args, **kwargs): """ Replace the default layer style with one of the boilerplate styles. """ name = self.layer.name sld = get_sld_for(self.catalog, self.layer) style = None if sld is not None: try: self.catalog.create_style(name, sld, raw=True) except ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % (name, str(e)) logger.error(msg) try: self.catalog.create_style(name + '_layer', sld, raw=True) except ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (name, str(e)) logger.error(msg) e.args = (msg,) if style is None: try: style = self.catalog.get_style(name) except BaseException: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = self.catalog.get_style(name + '_layer') except BaseException: style = self.catalog.get_style('point') logger.warn(msg) e.args = (msg,) if style: self.layer.default_style = style self.catalog.save(self.layer)
def test_ogc_server_defaults(self): """ Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. """ from django.urls import reverse, resolve from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities OGC_SERVER = {'default': dict()} defaults = self.OGC_DEFAULT_SETTINGS.get('default') ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest") self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows") # Make sure we get None vs a KeyError when the key does not exist self.assertIsNone(ogc_settings.SFDSDFDSF) # Testing REST endpoints route = resolve('/gs/rest/layers').route self.assertEqual(route, '^gs/rest/layers') route = resolve('/gs/rest/imports').route self.assertEqual(route, '^gs/rest/imports') route = resolve('/gs/rest/sldservice').route self.assertEqual(route, '^gs/rest/sldservice') store_resolver = resolve('/gs/rest/stores/geonode_data/') self.assertEqual(store_resolver.url_name, 'stores') self.assertEqual(store_resolver.kwargs['store_type'], 'geonode_data') self.assertEqual(store_resolver.route, '^gs/rest/stores/(?P<store_type>\\w+)/$') sld_resolver = resolve('/gs/rest/styles') self.assertIsNone(sld_resolver.url_name) self.assertTrue('workspace' not in sld_resolver.kwargs) self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/styles') self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/styles') self.assertEqual(sld_resolver.route, '^gs/rest/styles') sld_resolver = resolve('/gs/rest/workspaces/geonode/styles') self.assertIsNone(sld_resolver.url_name) self.assertEqual(sld_resolver.kwargs['workspace'], 'geonode') self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/workspaces') self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/workspaces') self.assertEqual(sld_resolver.route, '^gs/rest/workspaces/(?P<workspace>\\w+)') # Testing OWS endpoints wcs = _wcs_get_capabilities() logger.debug(wcs) self.assertIsNotNone(wcs) try: wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wcs.startswith(wcs_url)) self.assertIn("service=WCS", wcs) self.assertIn("request=GetCapabilities", wcs) self.assertIn("version=2.0.1", wcs) wfs = _wfs_get_capabilities() logger.debug(wfs) self.assertIsNotNone(wfs) try: wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wfs.startswith(wfs_url)) self.assertIn("service=WFS", wfs) self.assertIn("request=GetCapabilities", wfs) self.assertIn("version=1.1.0", wfs) wms = _wms_get_capabilities() logger.debug(wms) self.assertIsNotNone(wms) try: wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wms.startswith(wms_url)) self.assertIn("service=WMS", wms) self.assertIn("request=GetCapabilities", wms) self.assertIn("version=1.3.0", wms) # Test OWS Download Links from geonode.geoserver.ows import wcs_links, wfs_links, wms_links instance = create_single_dataset("san_andres_y_providencia_water") instance.name = 'san_andres_y_providencia_water' instance.save() bbox = instance.bbox srid = instance.srid height = 512 width = 512 # Default Style (expect exception since we are offline) style = get_sld_for(gs_catalog, instance) logger.error( f" style -------------------------------------------> {style}") if isinstance(style, str): style = gs_catalog.get_style(instance.name, workspace=instance.workspace) self.assertIsNotNone(style) self.assertFalse(isinstance(style, str)) instance.default_style, _ = Style.objects.get_or_create( name=style.name, defaults=dict(sld_title=style.sld_title, sld_body=style.sld_body)) self.assertIsNotNone(instance.default_style) self.assertIsNotNone(instance.default_style.name) # WMS Links wms_links = wms_links(f"{ogc_settings.public_url}wms?", instance.alternate, bbox, srid, height, width) self.assertIsNotNone(wms_links) self.assertEqual(len(wms_links), 3) wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms') identifier = urlencode({'layers': instance.alternate}) for _link in wms_links: logger.debug(f'{wms_url} --> {_link[3]}') self.assertTrue(wms_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WFS Links wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?", instance.alternate, bbox, srid) self.assertIsNotNone(wfs_links) self.assertEqual(len(wfs_links), 6) wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs') identifier = urlencode({'typename': instance.alternate}) for _link in wfs_links: logger.debug(f'{wfs_url} --> {_link[3]}') self.assertTrue(wfs_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WCS Links wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?", instance.alternate, bbox, srid) self.assertIsNotNone(wcs_links) self.assertEqual(len(wcs_links), 2) wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs') identifier = urlencode( {'coverageid': instance.alternate.replace(':', '__', 1)}) for _link in wcs_links: logger.debug(f'{wcs_url} --> {_link[3]}') self.assertTrue(wcs_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3])
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(cat, publishing) style = None if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style(name + '_layer', sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg,) if style is None: try: style = cat.get_style(name) except: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = cat.get_style(name + '_layer') except: style = cat.get_style('point') logger.warn(msg) e.args = (msg,) if style: publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime(upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex ) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create(name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log('There was an error updating the mosaic temporal extent: ' + str(e)) else: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def test_save_and_delete_signals(self): """Test that GeoServer Signals methods work as espected""" layers = Layer.objects.all()[:2].values_list('id', flat=True) test_perm_layer = Layer.objects.get(id=layers[0]) self.client.login(username='******', password='******') if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.signals import (geoserver_pre_delete, geoserver_post_save, geoserver_post_save_local) # Handle Layer Save and Upload Signals geoserver_post_save(test_perm_layer, sender=Layer) geoserver_post_save_local(test_perm_layer) # Check instance bbox and links self.assertIsNotNone(test_perm_layer.bbox) self.assertIsNotNone(test_perm_layer.srid) self.assertIsNotNone(test_perm_layer.link_set) self.assertEquals(len(test_perm_layer.link_set.all()), 7) # Layer Manipulation from geonode.geoserver.upload import geoserver_upload from geonode.geoserver.signals import gs_catalog from geonode.geoserver.helpers import ( check_geoserver_is_up, get_sld_for, fixup_style, set_layer_style, get_store, set_attributes_from_geoserver, set_styles, create_gs_thumbnail, cleanup) check_geoserver_is_up() admin_user = get_user_model().objects.get(username="******") saved_layer = geoserver_upload( test_perm_layer, os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp"), admin_user, test_perm_layer.name, overwrite=True) self.assertIsNotNone(saved_layer) _log(saved_layer) workspace, name = test_perm_layer.alternate.split(':') self.assertIsNotNone(workspace) self.assertIsNotNone(name) ws = gs_catalog.get_workspace(workspace) self.assertIsNotNone(ws) store = get_store(gs_catalog, name, workspace=ws) _log("1. ------------ %s " % store) self.assertIsNotNone(store) # Save layer attributes set_attributes_from_geoserver(test_perm_layer) # Save layer styles set_styles(test_perm_layer, gs_catalog) # set SLD sld = test_perm_layer.default_style.sld_body if test_perm_layer.default_style else None if sld: _log("2. ------------ %s " % sld) set_layer_style(test_perm_layer, test_perm_layer.alternate, sld) fixup_style(gs_catalog, test_perm_layer.alternate, None) self.assertIsNone(get_sld_for(gs_catalog, test_perm_layer)) _log("3. ------------ %s " % get_sld_for(gs_catalog, test_perm_layer)) create_gs_thumbnail(test_perm_layer, overwrite=True) self.assertIsNotNone(test_perm_layer.get_thumbnail_url()) self.assertTrue(test_perm_layer.has_thumbnail()) # Handle Layer Delete Signals geoserver_pre_delete(test_perm_layer, sender=Layer) # Check instance has been removed from GeoServer also from geonode.geoserver.views import get_layer_capabilities self.assertIsNone(get_layer_capabilities(test_perm_layer)) # Cleaning Up test_perm_layer.delete() cleanup(test_perm_layer.name, test_perm_layer.uuid)
def final_step(upload_session, user, charset="UTF-8"): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] task.set_charset(charset) # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'READY': import_session.commit() elif import_session.state == 'PENDING': if task.state == 'READY': # if not task.data.format or task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = None if os.path.isfile(sld_file): try: f = open(sld_file, 'r') except Exception: pass elif upload_session.tempdir and os.path.exists(upload_session.tempdir): tempdir = upload_session.tempdir if os.path.isfile(os.path.join(tempdir, sld_file)): try: f = open(os.path.join(tempdir, sld_file), 'r') except Exception: pass if f: sld = f.read() f.close() else: sld = get_sld_for(cat, publishing) else: sld = get_sld_for(cat, publishing) style = None if sld is not None: try: cat.create_style(name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) cat.reset() except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style(name + '_layer', sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) cat.reset() except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg, ) if style is None: try: style = cat.get_style(name, workspace=settings.DEFAULT_WORKSPACE ) or cat.get_style(name) except Exception: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = cat.get_style(name + '_layer', workspace=settings.DEFAULT_WORKSPACE) or \ cat.get_style(name + '_layer') except Exception as e: style = cat.get_style('point') logger.warn(str(e)) if style: publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target alternate = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime( upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict( store=target.name, storeType=target.store_type, alternate=alternate, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user, ), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create( name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter( name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter( name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log('There was an error updating the mosaic temporal extent: ' + str(e)) else: _has_time = (True if upload_session.time and upload_session.time_info and upload_session.time_transforms else False) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict( store=target.name, storeType=target.store_type, alternate=alternate, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user, ), has_time=_has_time) # Should we throw a clearer error here? assert saved_layer is not None saved_layer.save() # Create a new upload session geonode_upload_session = UploadSession.objects.create(resource=saved_layer, user=user) # Add them to the upload session (new file fields are created). assigned_name = None def _store_file(saved_layer, geonode_upload_session, base_file, assigned_name, base=False): with open(base_file, 'rb') as f: file_name, type_name = os.path.splitext( os.path.basename(base_file)) geonode_upload_session.layerfile_set.create( name=file_name, base=base, file=File(f, name='%s%s' % (assigned_name or saved_layer.name, type_name))) # save the system assigned name for the remaining files if not assigned_name: the_file = geonode_upload_session.layerfile_set.all( )[0].file.name assigned_name = os.path.splitext(os.path.basename(the_file))[0] return assigned_name if upload_session.base_file: uploaded_files = upload_session.base_file[0] base_file = uploaded_files.base_file aux_files = uploaded_files.auxillary_files sld_files = uploaded_files.sld_files xml_files = uploaded_files.xml_files assigned_name = _store_file(saved_layer, geonode_upload_session, base_file, assigned_name, base=True) for _f in aux_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) for _f in sld_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) for _f in xml_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() saved_layer.handle_moderated_uploads() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml defaults = {} xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r', allowZip64=True) zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata( open(xml_file[0]).read()) saved_layer.metadata_xml = xml_file[0] regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # Assign the regions (needs to be done after saving) regions_resolved = list(set(regions_resolved)) if regions_resolved: if len(regions_resolved) > 0: if not saved_layer.regions: saved_layer.regions = regions_resolved else: saved_layer.regions.clear() saved_layer.regions.add(*regions_resolved) # Assign the keywords (needs to be done after saving) keywords = list(set(keywords)) if keywords: if len(keywords) > 0: if not saved_layer.keywords: saved_layer.keywords = keywords else: saved_layer.keywords.add(*keywords) # set model properties for key, value in vals.items(): if key == 'spatial_representation_type': value = SpatialRepresentationType(identifier=value) elif key == 'topic_category': value, created = TopicCategory.objects.get_or_create( identifier=value.lower(), defaults={ 'description': '', 'gn_description': value }) key = 'category' defaults[key] = value else: defaults[key] = value # update with new information db_layer = Layer.objects.filter(id=saved_layer.id) db_layer.update(**defaults) saved_layer.refresh_from_db() # Pass the parameter overwrite to tell whether the # geoserver_post_save_signal should upload the new file or not saved_layer.overwrite = True saved_layer.save() # look for SLD sld_file = upload_session.base_file[0].sld_files if sld_file: # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r', allowZip64=True) zf.extract(sld_file[0], os.path.dirname(archive)) # Assign the absolute path to this file sld_file[0] = os.path.dirname(archive) + '/' + sld_file[0] sld = open(sld_file[0]).read() set_layer_style(saved_layer, saved_layer.alternate, sld, base_file=sld_file[0]) # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) if geonode_upload_session: geonode_upload_session.processed = True saved_layer.upload_session = geonode_upload_session signals.upload_complete.send(sender=final_step, layer=saved_layer) geonode_upload_session.save() saved_layer.save(notify=not created) cat._cache.clear() return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) style = None print " **************************************** " if sld is not None: try: cat.create_style(name, sld) style = cat.get_style(name) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style(name + '_layer', sld) style = cat.get_style(name + '_layer') except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg,) # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' style = cat.get_style('point') logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime(upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex ) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create(name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log('There was an error updating the mosaic temporal extent: ' + str(e)) else: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def test_ogc_server_defaults(self): """ Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. """ OGC_SERVER = {'default': dict()} defaults = self.OGC_DEFAULT_SETTINGS.get('default') ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest') self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows') # Make sure we get None vs a KeyError when the key does not exist self.assertIsNone(ogc_settings.SFDSDFDSF) # Testing OWS endpoints from django.urls import reverse from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities wcs = _wcs_get_capabilities() logger.debug(wcs) self.assertIsNotNone(wcs) try: wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except BaseException: wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wcs.startswith(wcs_url)) self.assertIn("service=WCS", wcs) self.assertIn("request=GetCapabilities", wcs) self.assertIn("version=2.0.1", wcs) wfs = _wfs_get_capabilities() logger.debug(wfs) self.assertIsNotNone(wfs) try: wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except BaseException: wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wfs.startswith(wfs_url)) self.assertIn("service=WFS", wfs) self.assertIn("request=GetCapabilities", wfs) self.assertIn("version=1.1.0", wfs) wms = _wms_get_capabilities() logger.debug(wms) self.assertIsNotNone(wms) try: wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except BaseException: wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wms.startswith(wms_url)) self.assertIn("service=WMS", wms) self.assertIn("request=GetCapabilities", wms) self.assertIn("version=1.3.0", wms) # Test OWS Download Links from geonode.geoserver.ows import wcs_links, wfs_links, wms_links instance = Layer.objects.all()[0] bbox = instance.bbox srid = instance.srid height = 512 width = 512 # Default Style (expect exception since we are offline) style = None with self.assertRaises(GeoNodeException): style = get_sld_for(gs_catalog, instance) self.assertIsNone(style) style = gs_catalog.get_style("line") self.assertIsNotNone(style) instance.default_style, _ = Style.objects.get_or_create( name=style.name, defaults=dict( sld_title=style.sld_title, sld_body=style.sld_body ) ) self.assertIsNotNone(instance.default_style) self.assertIsNotNone(instance.default_style.name) # WMS Links wms_links = wms_links(ogc_settings.public_url + 'wms?', instance.alternate, bbox, srid, height, width) self.assertIsNotNone(wms_links) self.assertEquals(len(wms_links), 3) wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms') identifier = urlencode({'layers': instance.alternate}) for _link in wms_links: logger.debug('%s --> %s' % (wms_url, _link[3])) self.assertTrue(wms_url in _link[3]) logger.debug('%s --> %s' % (identifier, _link[3])) self.assertTrue(identifier in _link[3]) # WFS Links wfs_links = wfs_links(ogc_settings.public_url + 'wfs?', instance.alternate, bbox, srid) self.assertIsNotNone(wfs_links) self.assertEquals(len(wfs_links), 6) wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs') identifier = urlencode({'typename': instance.alternate}) for _link in wfs_links: logger.debug('%s --> %s' % (wfs_url, _link[3])) self.assertTrue(wfs_url in _link[3]) logger.debug('%s --> %s' % (identifier, _link[3])) self.assertTrue(identifier in _link[3]) # WCS Links wcs_links = wcs_links(ogc_settings.public_url + 'wcs?', instance.alternate, bbox, srid) self.assertIsNotNone(wcs_links) self.assertEquals(len(wcs_links), 2) wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs') identifier = urlencode({'coverageid': instance.alternate}) for _link in wcs_links: logger.debug('%s --> %s' % (wcs_url, _link[3])) self.assertTrue(wcs_url in _link[3]) logger.debug('%s --> %s' % (identifier, _link[3])) self.assertTrue(identifier in _link[3]) # Thumbnails Generation Default create_gs_thumbnail(instance, overwrite=True) self.assertIsNotNone(instance.get_thumbnail_url()) # Thumbnails Generation Through "remote url" create_gs_thumbnail_geonode(instance, overwrite=True, check_bbox=True) # Thumbnails Generation Through "image" request_body = { 'width': width, 'height': height, 'layers': instance.alternate } if hasattr(instance, 'default_style'): if instance.default_style: request_body['styles'] = instance.default_style.name self.assertIsNotNone(request_body['styles']) try: image = _prepare_thumbnail_body_from_opts(request_body) except BaseException as e: logger.exception(e) image = None # We are offline here, the layer does not exists in GeoServer # - we expect the image is None self.assertIsNone(image)
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() defaults = dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) _log('record defaults: %s', defaults) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=defaults ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'READY': import_session.commit() elif import_session.state == 'PENDING': if task.state == 'READY': # if not task.data.format or task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady( "Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = None if os.path.isfile(sld_file): try: f = open(sld_file, 'r') except BaseException: pass elif upload_session.tempdir and os.path.exists(upload_session.tempdir): tempdir = upload_session.tempdir if os.path.isfile(os.path.join(tempdir, sld_file)): try: f = open(os.path.join(tempdir, sld_file), 'r') except BaseException: pass if f: sld = f.read() f.close() else: sld = get_sld_for(cat, publishing) else: sld = get_sld_for(cat, publishing) style = None if sld is not None: try: cat.create_style( name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style( name + '_layer', sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg,) if style is None: try: style = cat.get_style( name, workspace=settings.DEFAULT_WORKSPACE) or cat.get_style(name) except BaseException: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = cat.get_style(name + '_layer', workspace=settings.DEFAULT_WORKSPACE) or \ cat.get_style(name + '_layer') except BaseException: style = cat.get_style('point') logger.warn(msg) e.args = (msg,) if style: publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target alternate = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime(upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, alternate=alternate, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex ) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create( name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize( saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter( name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter( name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log( 'There was an error updating the mosaic temporal extent: ' + str(e)) else: _has_time = (True if upload_session.time and upload_session.time_info and upload_session.time_transforms else False) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, alternate=alternate, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), has_time=_has_time ) # Should we throw a clearer error here? assert saved_layer is not None # Create a new upload session geonode_upload_session = UploadSession.objects.create(resource=saved_layer, user=user) # Add them to the upload session (new file fields are created). assigned_name = None def _store_file(saved_layer, geonode_upload_session, base_file, assigned_name, base=False): with open(base_file, 'rb') as f: file_name, type_name = os.path.splitext(os.path.basename(base_file)) geonode_upload_session.layerfile_set.create( name=file_name, base=base, file=File( f, name='%s%s' % (assigned_name or saved_layer.name, type_name))) # save the system assigned name for the remaining files if not assigned_name: the_file = geonode_upload_session.layerfile_set.all()[0].file.name assigned_name = os.path.splitext(os.path.basename(the_file))[0] return assigned_name if upload_session.base_file: uploaded_files = upload_session.base_file[0] base_file = uploaded_files.base_file aux_files = uploaded_files.auxillary_files sld_files = uploaded_files.sld_files xml_files = uploaded_files.xml_files assigned_name = _store_file(saved_layer, geonode_upload_session, base_file, assigned_name, base=True) for _f in aux_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) for _f in sld_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) for _f in xml_files: _store_file(saved_layer, geonode_upload_session, _f, assigned_name) # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() saved_layer.handle_moderated_uploads() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml defaults = {} xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata( open(xml_file[0]).read()) saved_layer.metadata_xml = xml_file[0] regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) if getattr(settings, 'NLP_ENABLED', False): try: from geonode.contrib.nlp.utils import nlp_extract_metadata_dict nlp_metadata = nlp_extract_metadata_dict({ 'title': defaults.get('title', None), 'abstract': defaults.get('abstract', None), 'purpose': defaults.get('purpose', None)}) if nlp_metadata: regions_resolved.extend(nlp_metadata.get('regions', [])) keywords.extend(nlp_metadata.get('keywords', [])) except BaseException: print "NLP extraction failed." # Assign the regions (needs to be done after saving) regions_resolved = list(set(regions_resolved)) if regions_resolved: if len(regions_resolved) > 0: if not saved_layer.regions: saved_layer.regions = regions_resolved else: saved_layer.regions.clear() saved_layer.regions.add(*regions_resolved) # Assign the keywords (needs to be done after saving) keywords = list(set(keywords)) if keywords: if len(keywords) > 0: if not saved_layer.keywords: saved_layer.keywords = keywords else: saved_layer.keywords.add(*keywords) # set model properties for key, value in vals.items(): if key == 'spatial_representation_type': value = SpatialRepresentationType(identifier=value) elif key == 'topic_category': value, created = TopicCategory.objects.get_or_create( identifier=value.lower(), defaults={'description': '', 'gn_description': value}) key = 'category' defaults[key] = value else: defaults[key] = value # update with new information db_layer = Layer.objects.filter(id=saved_layer.id) db_layer.update(**defaults) saved_layer.refresh_from_db() # Pass the parameter overwrite to tell whether the # geoserver_post_save_signal should upload the new file or not saved_layer.overwrite = True saved_layer.save() # look for SLD sld_file = upload_session.base_file[0].sld_files if sld_file: # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(sld_file[0], os.path.dirname(archive)) # Assign the absolute path to this file sld_file[0] = os.path.dirname(archive) + '/' + sld_file[0] sld = open(sld_file[0]).read() set_layer_style( saved_layer, saved_layer.alternate, sld, base_file=sld_file[0]) # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) if geonode_upload_session: geonode_upload_session.processed = True saved_layer.upload_session = geonode_upload_session signals.upload_complete.send(sender=final_step, layer=saved_layer) geonode_upload_session.save() saved_layer.save() cat._cache.clear() cat.reload() return saved_layer
def create_geoserver_layer(name, user, srid, overwrite=False, title=None, abstract=None, charset='UTF-8'): if "geonode.geoserver" in settings.INSTALLED_APPS: _user, _password = ogc_server_settings.credentials # # Step 2. Check that it is uploading to the same resource type as # the existing resource logger.info( '>>> Step 2. Make sure we are not trying to overwrite a ' 'existing resource named [%s] with the wrong type', name) the_layer_type = "vector" # Get a short handle to the gsconfig geoserver catalog cat = Catalog(ogc_server_settings.internal_rest, _user, _password) workspace = cat.get_default_workspace() # Check if the store exists in geoserver try: store = get_store(cat, name, workspace=workspace) except FailedRequestError as e: # There is no store, ergo the road is clear pass else: # If we get a store, we do the following: resources = store.get_resources() # If the store is empty, we just delete it. if len(resources) == 0: cat.delete(store) else: # If our resource is already configured in the store it needs # to have the right resource type for resource in resources: if resource.name == name: msg = 'Name already in use and overwrite is False' assert overwrite, msg existing_type = resource.resource_type if existing_type != the_layer_type: msg = ('Type of uploaded file %s (%s) ' 'does not match type of existing ' 'resource type ' '%s' % (name, the_layer_type, existing_type)) logger.info(msg) raise GeoNodeException(msg) logger.debug('Creating vector layer: [%s]', name) ds = create_feature_store(cat, workspace) gs_resource = gs_catalog.publish_featuretype(name, ds, "EPSG:" + str(srid)) # # Step 7. Create the style and assign it to the created resource # # FIXME: Put this in gsconfig.py logger.info('>>> Step 7. Creating style for [%s]' % name) publishing = cat.get_layer(name) create_style() sld = get_sld_for(gs_catalog, publishing) style = None if sld is not None: try: cat.create_style(name, sld) style = cat.get_style(name) except geoserver.catalog.ConflictingDataError as e: msg = ('There was already a style named %s in GeoServer, ' 'try to use: "%s"' % (name + "_layer", str(e))) logger.warn(msg) e.args = (msg, ) try: cat.create_style(name + '_layer', sld) style = cat.get_style(name + "_layer") except geoserver.catalog.ConflictingDataError as e: style = cat.get_style('point') msg = ('There was already a style named %s in GeoServer, ' 'cannot overwrite: "%s"' % (name, str(e))) logger.error(msg) e.args = (msg, ) # FIXME: Should we use the fully qualified typename? publishing.default_style = style cat.save(publishing) return gs_resource
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = None for i in xrange(60): publishing = cat.get_layer(name) if publishing: break time.sleep(.5) if not publishing: raise Exception( "Expected to find layer named '%s' in geoserver, tried %s times" % (name, i)) _log('Had to try %s times to get layer from catalog' % (i + 1)) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() defaults = dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) _log('record defaults: %s', defaults) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=defaults ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML vals, keywords = set_metadata(open(xml_file[0]).read()) # set taggit keywords saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions _log('Setting default permissions for [%s]', name) if permissions is not None: saved_layer.set_permissions(permissions) _log('Verifying the layer [%s] was created correctly' % name) # Verify the object was saved to the Django database # @revisit - this should always work since we just created it above and the # message is confusing try: saved_layer = Layer.objects.get(name=name) except Layer.DoesNotExist as e: msg = ( 'There was a problem saving the layer %s to GeoNetwork/Django. Error is: %s' % (name, str(e))) logger.exception(msg) logger.debug( 'Attempting to clean up after failed save for layer [%s]', name) # Since the layer creation was not successful, we need to clean up # @todo implement/test cleanup # cleanup(name, layer_uuid) raise GeoNodeException(msg) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: saved_layer.set_time_info(**upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() defaults = dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) _log('record defaults: %s', defaults) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=defaults ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions _log('Setting default permissions for [%s]', name) if permissions is not None: saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
'the layer.') logger.info(msg, name) cascading_delete(cat, name) raise GeoNodeException(msg % name) # Step 7. Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py logger.info('>>> Step 7. Creating style for [%s]' % name) publishing = cat.get_layer(name) if 'sld' in files: f = open(files['sld'], 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError, e: msg = ('There was already a style named %s in GeoServer, ' 'cannot overwrite: "%s"' % (name, str(e))) logger.warn(msg) e.args = (msg, ) #FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) cat.save(publishing) # Step 10. Create the Django record for the layer
'the layer.') logger.info(msg, name) cascading_delete(cat, name) raise GeoNodeException(msg % name) # Step 7. Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py logger.info('>>> Step 7. Creating style for [%s]' % name) publishing = cat.get_layer(name) if 'sld' in files: f = open(files['sld'], 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError, e: msg = ('There was already a style named %s in GeoServer, ' 'cannot overwrite: "%s"' % (name, str(e))) logger.warn(msg) e.args = (msg,) #FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) cat.save(publishing) # Step 10. Create the Django record for the layer
def test_save_and_delete_signals(self): """Test that GeoServer Signals methods work as espected""" layers = Layer.objects.all()[:2].values_list('id', flat=True) test_perm_layer = Layer.objects.get(id=layers[0]) self.client.login(username='******', password='******') if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.signals import (geoserver_pre_delete, geoserver_post_save, geoserver_post_save_local) # Handle Layer Save and Upload Signals geoserver_post_save(test_perm_layer, sender=Layer) geoserver_post_save_local(test_perm_layer) # Check instance bbox and links self.assertIsNotNone(test_perm_layer.bbox) self.assertIsNotNone(test_perm_layer.srid) self.assertIsNotNone(test_perm_layer.link_set) self.assertEquals(len(test_perm_layer.link_set.all()), 9) # Layer Manipulation from geonode.geoserver.upload import geoserver_upload from geonode.geoserver.signals import gs_catalog from geonode.geoserver.helpers import (check_geoserver_is_up, get_sld_for, fixup_style, set_layer_style, get_store, set_attributes_from_geoserver, set_styles, create_gs_thumbnail, cleanup) check_geoserver_is_up() admin_user = get_user_model().objects.get(username="******") saved_layer = geoserver_upload( test_perm_layer, os.path.join( gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp"), admin_user, test_perm_layer.name, overwrite=True ) self.assertIsNotNone(saved_layer) _log(saved_layer) workspace, name = test_perm_layer.alternate.split(':') self.assertIsNotNone(workspace) self.assertIsNotNone(name) ws = gs_catalog.get_workspace(workspace) self.assertIsNotNone(ws) store = get_store(gs_catalog, name, workspace=ws) _log("1. ------------ %s " % store) self.assertIsNotNone(store) # Save layer attributes set_attributes_from_geoserver(test_perm_layer) # Save layer styles set_styles(test_perm_layer, gs_catalog) # set SLD sld = test_perm_layer.default_style.sld_body if test_perm_layer.default_style else None if sld: _log("2. ------------ %s " % sld) set_layer_style(test_perm_layer, test_perm_layer.alternate, sld) fixup_style(gs_catalog, test_perm_layer.alternate, None) self.assertIsNone(get_sld_for(gs_catalog, test_perm_layer)) _log("3. ------------ %s " % get_sld_for(gs_catalog, test_perm_layer)) create_gs_thumbnail(test_perm_layer, overwrite=True) self.assertIsNotNone(test_perm_layer.get_thumbnail_url()) self.assertTrue(test_perm_layer.has_thumbnail()) # Handle Layer Delete Signals geoserver_pre_delete(test_perm_layer, sender=Layer) # Check instance has been removed from GeoServer also from geonode.geoserver.views import get_layer_capabilities self.assertIsNone(get_layer_capabilities(test_perm_layer)) # Cleaning Up test_perm_layer.delete() cleanup(test_perm_layer.name, test_perm_layer.uuid)