def handle(self, *args, **options): ignore_errors = options.get('ignore_errors') filter = options.get('filter') if not options.get('username'): username = None else: username = options.get('username') all_layers = Layer.objects.all().order_by('name') if filter: all_layers = all_layers.filter(name__icontains=filter) if username: all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): print "[%s / %s] Updating Layer [%s] ..." % ((index + 1), len(all_layers), layer.name) try: # recalculate the layer statistics set_attributes(layer, overwrite=True) # refresh metadata links set_resource_default_links(layer, layer, prune=True) # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) except BaseException as e: # import traceback # traceback.print_exc() if ignore_errors: print "[ERROR] Layer [%s] couldn't be updated" % (layer.name) else: raise e
def handle(self, *args, **options): ignore_errors = options.get('ignore_errors') remove_duplicates = options.get('remove_duplicates') filter = options.get('filter') if not options.get('username'): username = None else: username = options.get('username') all_layers = Layer.objects.all().order_by('name') if filter: all_layers = all_layers.filter(name__icontains=filter) if username: all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): print "[%s / %s] Updating Layer [%s] ..." % ( (index + 1), len(all_layers), layer.name) try: # recalculate the layer statistics set_attributes(layer, overwrite=True) # refresh metadata links set_resource_default_links(layer, layer, prune=False) # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) if remove_duplicates: # remove duplicates for _n in _names: _links = Link.objects.filter(resource__id=layer.id, name=_n) while _links.count() > 1: _links.last().delete() print '.', # fixup Legend links legend_url_template = ogc_server_settings.PUBLIC_LOCATION + \ 'ows?service=WMS&request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ '{alternate}&STYLE={style_name}' + \ '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' if layer.default_style and not layer.get_legend_url( style_name=layer.default_style.name): Link.objects.update_or_create( resource=layer.resourcebase_ptr, name='Legend', extension='png', url=legend_url_template.format( alternate=layer.alternate, style_name=layer.default_style.name), mime='image/png', link_type='image') except BaseException as e: import traceback traceback.print_exc() if ignore_errors: print "[ERROR] Layer [%s] couldn't be updated" % ( layer.name) else: raise e
def harvest_resource(self, harvest_job_id): harvest_job = models.HarvestJob.objects.get(pk=harvest_job_id) harvest_job.update_status(status=enumerations.IN_PROCESS, details="Harvesting resource...") result = False details = "" try: handler = get_service_handler( base_url=harvest_job.service.base_url, proxy_base=harvest_job.service.proxy_base, service_type=harvest_job.service.type) with transaction.atomic(): logger.debug("harvesting resource...") handler.harvest_resource(harvest_job.resource_id, harvest_job.service) result = True logger.debug("Resource harvested successfully") logger.debug("Updating Layer Metadata ...") try: layer = Layer.objects.get(alternate=harvest_job.resource_id) catalogue_post_save(instance=layer, sender=layer.__class__) except BaseException: logger.error("Remote Layer [%s] couldn't be updated" % (harvest_job.resource_id)) except Exception as err: logger.exception(msg="An error has occurred while harvesting " "resource {!r}".format(harvest_job.resource_id)) details = str(err) # TODO: pass more context about the error finally: harvest_job.update_status( status=enumerations.PROCESSED if result else enumerations.FAILED, details=details)
def handle(self, *args, **options): ignore_errors = options.get('ignore_errors') filter = options.get('filter') if not options.get('username'): username = None else: username = options.get('username') all_layers = Layer.objects.all().order_by('name') if filter: all_layers = all_layers.filter(name__icontains=filter) if username: all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): print "[%s / %s] Updating Layer [%s] ..." % ( (index + 1), len(all_layers), layer.name) try: # recalculate the layer statistics set_attributes(layer, overwrite=True) # refresh metadata links set_resource_default_links(layer, layer, prune=False) # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) except BaseException as e: # import traceback # traceback.print_exc() if ignore_errors: print "[ERROR] Layer [%s] couldn't be updated" % ( layer.name) else: raise e
def harvest_resource(self, harvest_job_id): harvest_job = models.HarvestJob.objects.get(pk=harvest_job_id) harvest_job.update_status( status=enumerations.IN_PROCESS, details="Harvesting resource...") result = False details = "" try: handler = get_service_handler( base_url=harvest_job.service.base_url, proxy_base=harvest_job.service.proxy_base, service_type=harvest_job.service.type ) with transaction.atomic(): logger.debug("harvesting resource...") handler.harvest_resource( harvest_job.resource_id, harvest_job.service) result = True logger.debug("Resource harvested successfully") logger.debug("Updating Layer Metadata ...") try: layer = Layer.objects.get(alternate=harvest_job.resource_id) catalogue_post_save(instance=layer, sender=layer.__class__) except BaseException: logger.error("Remote Layer [%s] couldn't be updated" % (harvest_job.resource_id)) except Exception as err: logger.exception(msg="An error has occurred while harvesting " "resource {!r}".format(harvest_job.resource_id)) details = str(err) # TODO: pass more context about the error finally: harvest_job.update_status( status=enumerations.PROCESSED if result else enumerations.FAILED, details=details )
def handle(self, *args, **options): ignore_errors = options.get('ignore_errors') remove_duplicates = options.get('remove_duplicates') prune = options.get('prune') set_uuid = ast.literal_eval(options.get('set_uuid', 'False')) set_attrib = ast.literal_eval(options.get('set_attrib', 'True')) set_links = ast.literal_eval(options.get('set_links', 'True')) delete_orphaned_thumbnails = options.get('delete_orphaned_thumbnails') filter = options.get('filter') if not options.get('username'): username = None else: username = options.get('username') all_layers = Layer.objects.all().order_by('name') if filter: all_layers = all_layers.filter(name__icontains=filter) if username: all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): print(f"[{(index + 1)} / {len(all_layers)}] Updating Layer [{layer.name}] ...") try: # recalculate the layer statistics if set_attrib: set_attributes(layer, overwrite=True) if set_uuid and hasattr(settings, 'LAYER_UUID_HANDLER') and settings.LAYER_UUID_HANDLER != '': from geonode.layers.utils import get_uuid_handler uuid = get_uuid_handler()(layer).create_uuid() la = Layer.objects.filter(resourcebase_ptr=layer.resourcebase_ptr) la.update(uuid=uuid) layer.refresh_from_db() # refresh metadata links if set_links: set_resource_default_links(layer, layer, prune=prune) # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) # remove duplicates if remove_duplicates: remove_duplicate_links(layer) except Exception as e: import traceback traceback.print_exc() if ignore_errors: logger.error(f"[ERROR] Layer [{layer.name}] couldn't be updated") else: raise e # delete orphaned thumbs if delete_orphaned_thumbnails: delete_orphaned_thumbs()
def process_inasafe_metadata(sender, instance, created, **kwargs): """Extract and save uploaded InaSAFE metadata.""" # retrieve metadata_xml from layer original xml file try: xml_file = instance.upload_session.layerfile_set.get(name='xml') except LayerFile.DoesNotExist: # if no xml file, we do nothing return metadata_xml = xml_file.file.read() inasafe_el, inasafe_provenance_el = \ extract_inasafe_keywords_from_metadata(metadata_xml) # Update metadata instance metadata, created = Metadata.objects.get_or_create(layer=instance) inasafe_metadata_xml = '' if inasafe_el: inasafe_metadata_xml = etree.tostring(inasafe_el, pretty_print=True) if inasafe_provenance_el: inasafe_metadata_xml += '\n' inasafe_metadata_xml += etree.tostring(inasafe_provenance_el, pretty_print=True) metadata.keywords_xml = inasafe_metadata_xml instance.refresh_from_db() instance.inasafe_metadata = metadata # Trigger catalogue metadata xml update catalogue_post_save(instance, sender, **kwargs) # After this. The new corrected xml file (which contains InaSAFE metadata) # will exists on layer.metadata_xml # In turn, we overwrite xml file in both media folder and QGIS layer # folder, because QGIS will use the actual file. instance.refresh_from_db() # Update layer xml path xml_file_name = xml_file.file.name xml_file.file.delete(save=False) xml_file.file.save(xml_file_name, ContentFile(instance.metadata_xml), save=True) # Update QGIS server xml path with codecs.open(instance.qgis_layer.xml_path, mode='w', encoding='utf-8') as f: f.write(instance.metadata_xml) # Save metadata and trigger signal handlers # This will process layer purpose from InaSAFE get_keywords task metadata.save()
def handle(self, *args, **options): all_layers = Layer.objects.all() for index, layer in enumerate(all_layers): print "[%s / %s] Updating Layer [%s] ..." % ((index + 1), len(all_layers), layer.name) try: catalogue_post_save(instance=layer, sender=layer.__class__) except: # import traceback # traceback.print_exc() print "[ERROR] Layer [%s] couldn't be updated" % (layer.name)
def handle(self, *args, **options): all_layers = Layer.objects.all() for index, layer in enumerate(all_layers): print "[%s / %s] Updating Layer [%s] ..." % ( (index + 1), len(all_layers), layer.name) try: catalogue_post_save(instance=layer, sender=layer.__class__) except: # import traceback # traceback.print_exc() print "[ERROR] Layer [%s] couldn't be updated" % (layer.name)
def handle(self, *args, **options): ignore_errors = options.get('ignore_errors') remove_duplicates = options.get('remove_duplicates') prune = options.get('prune') delete_orphaned_thumbnails = options.get('delete_orphaned_thumbnails') filter = options.get('filter') if not options.get('username'): username = None else: username = options.get('username') all_layers = Layer.objects.all().order_by('name') if filter: all_layers = all_layers.filter(name__icontains=filter) if username: all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): print( f"[{(index + 1)} / {len(all_layers)}] Updating Layer [{layer.name}] ..." ) try: # recalculate the layer statistics set_attributes(layer, overwrite=True) # refresh metadata links set_resource_default_links(layer, layer, prune=prune) # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) # remove duplicates if remove_duplicates: remove_duplicate_links(layer) except Exception as e: import traceback traceback.print_exc() if ignore_errors: logger.error( f"[ERROR] Layer [{layer.name}] couldn't be updated") else: raise e # delete orphaned thumbs if delete_orphaned_thumbnails: delete_orphaned_thumbs()
def test_update_metadata_records(self): layer = Layer.objects.first() self.assertIsNotNone(layer) layer.abstract = "<p>Test HTML abstract</p>" layer.save() self.assertEqual(layer.abstract, "<p>Test HTML abstract</p>") self.assertEqual(layer.raw_abstract, "Test HTML abstract") # refresh catalogue metadata records catalogue_post_save(instance=layer, sender=layer.__class__) # get all records csw = get_catalogue() record = csw.get_record(layer.uuid) self.assertIsNotNone(record) self.assertEqual(record.identification.title, layer.title) self.assertEqual(record.identification.abstract, layer.raw_abstract) if len(record.identification.otherconstraints) > 0: self.assertEqual(record.identification.otherconstraints[0], layer.raw_constraints_other)
def set_resource_links(*args, **kwargs): from geonode.utils import set_resource_default_links from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer if settings.UPDATE_RESOURCE_LINKS_AT_MIGRATE: _all_layers = Layer.objects.all() for index, layer in enumerate(_all_layers, start=1): _lyr_name = layer.name message = f"[{index} / {len(_all_layers)}] Updating Layer [{_lyr_name}] ..." logger.debug(message) try: set_resource_default_links(layer, layer) catalogue_post_save(instance=layer, sender=layer.__class__) except Exception: logger.exception( f"[ERROR] Layer [{_lyr_name}] couldn't be updated")
def test_update_metadata_records(self): dataset = Dataset.objects.first() self.assertIsNotNone(dataset) dataset.abstract = "<p>Test HTML abstract</p>" dataset.save() self.assertEqual(dataset.abstract, "<p>Test HTML abstract</p>") self.assertEqual(dataset.raw_abstract, "Test HTML abstract") # refresh catalogue metadata records catalogue_post_save(instance=dataset, sender=dataset.__class__) # get all records csw = get_catalogue() record = csw.get_record(dataset.uuid) self.assertIsNotNone(record) self.assertEqual(record.identification.title, dataset.title) self.assertEqual(record.identification.abstract, dataset.raw_abstract) if len(record.identification.otherconstraints) > 0: self.assertEqual(record.identification.otherconstraints[0], dataset.raw_constraints_other)
def set_resource_links(*args, **kwargs): from geonode.utils import set_resource_default_links from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer _all_layers = Layer.objects.all() for index, layer in enumerate(_all_layers, start=1): _lyr_name = layer.name message = "[%s / %s] Updating Layer [%s] ..." % ( index, len(_all_layers), _lyr_name) print(message) logger.debug(message) try: set_resource_default_links(layer, layer) catalogue_post_save(instance=layer, sender=layer.__class__) except BaseException: logger.exception("[ERROR] Layer [%s] couldn't be updated" % _lyr_name)
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return if instance.storeType == "remoteStore": # Save layer attributes set_attributes(instance) return if not getattr(instance, 'gs_resource', None): try: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return else: gs_resource = instance.gs_resource if gs_resource is None: return if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', ) ) if instance.storeType == "dataStore": links = wfs_links( ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', ) ) if gs_resource.store.type and gs_resource.store.type.lower() == 'geogig' and \ gs_resource.store.connection_parameters.get('geogig_repository'): repo_url = '{url}geogig/{geogig_repository}'.format( url=ogc_server_settings.public_url, geogig_repository=gs_resource.store.connection_parameters.get('geogig_repository')) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html' ) ) def command_url(command): return "{repo_url}/{command}.json?{path}".format(repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html' ) ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html' ) ) elif instance.storeType == 'coverageStore': # FIXME(Ariel): This works for public layers, does it work for restricted too? # would those end up with no geotiff links, like, forever? permissions = instance.get_all_level_info() instance.set_permissions( {'users': {'AnonymousUser': ['view_resourcebase']}}) try: # Potentially 3 dimensions can be returned by the grid if there is a z # axis. Since we only want width/height, slice to the second # dimension covWidth, covHeight = get_coverage_grid_extent(instance)[:2] except GeoNodeException as e: msg = _('Could not create a download link for layer.') logger.warn(msg, e) else: links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8'), bbox=gs_resource.native_bbox[:-1], crs=gs_resource.native_bbox[-1], height=str(covHeight), width=str(covWidth)) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', ) ) instance.set_permissions(permissions) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', ) ) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', ) ) tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8' ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=tile_url, defaults=dict( extension='tiles', name="Tiles", mime='image/png', link_type='image', ) ) html_link_url = '%s%s' % ( settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', ) ) params = { 'layers': instance.typename.encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.PUBLIC_LOCATION + \ "wms/reflect?" + p thumbnail_create_url = ogc_server_settings.LOCATION + \ "wms/reflect?" + p create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url, ogc_client=http_client) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.typename + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', ) ) ogc_wms_path = '%s/wms' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', ) ) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', ) ) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', ) ) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse( settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Save layer attributes set_attributes(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer)
def geoserver_post_save_layers(self, instance_id, *args, **kwargs): """ Runs update layers. """ instance = None try: instance = Layer.objects.get(id=instance_id) except Layer.DoesNotExist: logger.debug(f"Layer id {instance_id} does not exist yet!") raise lock_id = f'{self.request.id}' with AcquireLock(lock_id) as lock: if lock.acquire() is True: # Don't run this signal if is a Layer from a remote service if getattr(instance, "remote_service", None) is not None: return if instance.storeType == "remoteStore": return # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr( instance, 'storeType') in ['tileStore', 'remoteStore']: return instance if isinstance(instance, ResourceBase): if hasattr(instance, 'layer'): instance = instance.layer else: return geonode_upload_sessions = UploadSession.objects.filter( resource=instance) geonode_upload_sessions.update(processed=False) gs_resource = None values = None _tries = 0 _max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 2) # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is no file. if base_file is None: return gs_name, workspace, values, gs_resource = geoserver_upload( instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, charset=instance.charset) values, gs_resource = fetch_gs_resource(instance, values, _tries) while not gs_resource and _tries < _max_tries: values, gs_resource = fetch_gs_resource( instance, values, _tries) _tries += 1 # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) if gs_resource: logger.debug("Found geoserver resource for this layer: %s" % instance.name) gs_resource.metadata_links = metadata_links instance.gs_resource = gs_resource # Update Attribution link if instance.poc: # gsconfig now utilizes an attribution dictionary gs_resource.attribution = { 'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None } profile = get_user_model().objects.get( username=instance.poc.username) site_url = settings.SITEURL.rstrip( '/') if settings.SITEURL.startswith( 'http') else settings.SITEURL gs_resource.attribution_link = site_url + profile.get_absolute_url( ) """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ try: # This is usually done in Layer.pre_save, however if the hooks # are bypassed by custom create/updates we need to ensure the # bbox is calculated properly. bbox = gs_resource.native_bbox instance.set_bbox_polygon( [bbox[0], bbox[2], bbox[1], bbox[3]], gs_resource.projection) except Exception as e: logger.exception(e) if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" elif instance.bbox_polygon is not None: # Guessing 'EPSG:4326' by default instance.srid = 'EPSG:4326' else: raise GeoNodeException( "Invalid Projection. Layer is missing CRS!") # Iterate over values from geoserver. for key in ['alternate', 'store', 'storeType']: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, values[key]) try: if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: gs_resource.advertised = 'true' if not settings.FREETEXT_KEYWORDS_READONLY: # AF: Warning - this won't allow people to have empty keywords on GeoNode if len(instance.keyword_list() ) == 0 and gs_resource.keywords: for keyword in gs_resource.keywords: if keyword not in instance.keyword_list(): instance.keywords.add(keyword) if any(instance.keyword_list()): keywords = instance.keyword_list() gs_resource.keywords = [ kw for kw in list(set(keywords)) ] # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) except Exception as e: msg = ( 'Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg, ) logger.exception(e) # store the resource to avoid another geoserver call in the post_save to_update = { 'title': instance.title or instance.name, 'abstract': instance.abstract or "", 'alternate': instance.alternate, 'bbox_polygon': instance.bbox_polygon, 'srid': 'EPSG:4326' } if is_monochromatic_image(instance.thumbnail_url): to_update['thumbnail_url'] = staticfiles.static( settings.MISSING_THUMBNAIL) # Save all the modified information in the instance without triggering signals. try: with transaction.atomic(): ResourceBase.objects.filter( id=instance.resourcebase_ptr.id).update( **to_update) # to_update['name'] = instance.name, to_update[ 'workspace'] = gs_resource.store.workspace.name to_update['store'] = gs_resource.store.name to_update['storeType'] = instance.storeType to_update['typename'] = instance.alternate Layer.objects.filter(id=instance.id).update( **to_update) # Refresh from DB instance.refresh_from_db() except Exception as e: logger.exception(e) # Refreshing CSW records logger.debug( f"... Updating the Catalogue entries for Layer {instance.title}" ) try: catalogue_post_save(instance=instance, sender=instance.__class__) except Exception as e: logger.exception(e) # Refreshing layer links logger.debug( f"... Creating Default Resource Links for Layer {instance.title}" ) try: set_resource_default_links(instance, instance, prune=True) except Exception as e: logger.exception(e) # Save layer attributes logger.debug( f"... Refresh GeoServer attributes list for Layer {instance.title}" ) try: set_attributes_from_geoserver(instance) except Exception as e: logger.exception(e) # Save layer styles logger.debug( f"... Refresh Legend links for Layer {instance.title}") try: set_styles(instance, gs_catalog) except Exception as e: logger.exception(e) # Invalidate GeoWebCache for the updated resource try: _stylefilterparams_geowebcache_layer(instance.alternate) _invalidate_geowebcache_layer(instance.alternate) except Exception: pass # Creating Layer Thumbnail by sending a signal from geonode.geoserver.signals import geoserver_post_save_complete geoserver_post_save_complete.send(sender=instance.__class__, instance=instance) try: geonode_upload_sessions = UploadSession.objects.filter( resource=instance) geonode_upload_sessions.update(processed=True) except Exception as e: logger.exception(e) # Updating HAYSTACK Indexes if needed if settings.HAYSTACK_SEARCH: call_command('update_index')
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ if type(instance) is ResourceBase: if hasattr(instance, "layer"): instance = instance.layer else: return if instance.storeType == "remoteStore": # Save layer attributes set_attributes(instance) return if not getattr(instance, "gs_resource", None): try: gs_resource = gs_catalog.get_resource(instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return else: gs_resource = instance.gs_resource if gs_resource is None: return if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links( ogc_server_settings.public_url + "wms?", instance.typename.encode("utf-8"), instance.bbox_string, instance.srid, height, width, ) for ext, name, mime, wms_url in links: Link.objects.get_or_create( resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict(extension=ext, url=wms_url, mime=mime, link_type="image"), ) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + "wfs?", instance.typename.encode("utf-8")) for ext, name, mime, wfs_url in links: if mime == "SHAPE-ZIP": name = "Zipped Shapefile" Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict(extension=ext, name=name, mime=mime, url=wfs_url, link_type="data"), ) if ( gs_resource.store.type and gs_resource.store.type.lower() == "geogig" and gs_resource.store.connection_parameters.get("geogig_repository") ): repo_url = "{url}geogig/{geogig_repository}".format( url=ogc_server_settings.public_url, geogig_repository=gs_resource.store.connection_parameters.get("geogig_repository"), ) path = gs_resource.dom.findall("nativeName") if path: path = "path={path}".format(path=path[0].text) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension="html", name="Clone in GeoGig", mime="text/xml", link_type="html"), ) def command_url(command): return "{repo_url}/{command}.json?{path}".format(repo_url=repo_url, path=path, command=command) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=command_url("log"), defaults=dict(extension="json", name="GeoGig log", mime="application/json", link_type="html"), ) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=command_url("statistics"), defaults=dict(extension="json", name="GeoGig statistics", mime="application/json", link_type="html"), ) elif instance.storeType == "coverageStore": # FIXME(Ariel): This works for public layers, does it work for restricted too? # would those end up with no geotiff links, like, forever? permissions = instance.get_all_level_info() instance.set_permissions({"users": {"AnonymousUser": ["view_resourcebase"]}}) try: # Potentially 3 dimensions can be returned by the grid if there is a z # axis. Since we only want width/height, slice to the second # dimension covWidth, covHeight = get_coverage_grid_extent(instance)[:2] except GeoNodeException as e: msg = _("Could not create a download link for layer.") try: # HACK: The logger on signals throws an exception logger.warn(msg, e) except: pass else: links = wcs_links( ogc_server_settings.public_url + "wcs?", instance.typename.encode("utf-8"), bbox=gs_resource.native_bbox[:-1], crs=gs_resource.native_bbox[-1], height=str(covHeight), width=str(covWidth), ) for ext, name, mime, wcs_url in links: Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict(extension=ext, name=name, mime=mime, link_type="data"), ) instance.set_permissions(permissions) kml_reflector_link_download = ( ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({"layers": instance.typename.encode("utf-8"), "mode": "download"}) ) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict(extension="kml", name="KML", mime="text/xml", link_type="data"), ) kml_reflector_link_view = ( ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({"layers": instance.typename.encode("utf-8"), "mode": "refresh"}) ) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict(extension="kml", name="View in Google Earth", mime="text/xml", link_type="data"), ) tile_url = ( "%sgwc/service/gmaps?" % ogc_server_settings.public_url + "layers=%s" % instance.typename.encode("utf-8") + "&zoom={z}&x={x}&y={y}" + "&format=image/png8" ) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=tile_url, defaults=dict(extension="tiles", name="Tiles", mime="image/png", link_type="image"), ) html_link_url = "%s%s" % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict(extension="html", name=instance.typename, mime="text/html", link_type="html"), ) params = {"layers": instance.typename.encode("utf-8"), "format": "image/png8", "width": 200, "height": 150} # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.PUBLIC_LOCATION + "wms/reflect?" + p thumbnail_create_url = ogc_server_settings.LOCATION + "wms/reflect?" + p create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url, ogc_client=http_client) legend_url = ( ogc_server_settings.PUBLIC_LOCATION + "wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=" + instance.typename + "&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on" ) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=legend_url, defaults=dict(extension="png", name="Legend", url=legend_url, mime="image/png", link_type="image"), ) ogc_wms_path = "%s/wms" % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = "OGC WMS: %s Service" % instance.workspace Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict(extension="html", name=ogc_wms_name, url=ogc_wms_url, mime="text/html", link_type="OGC:WMS"), ) if instance.storeType == "dataStore": ogc_wfs_path = "%s/wfs" % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = "OGC WFS: %s Service" % instance.workspace Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict(extension="html", name=ogc_wfs_name, url=ogc_wfs_url, mime="text/html", link_type="OGC:WFS"), ) if instance.storeType == "coverageStore": ogc_wcs_path = "%s/wcs" % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = "OGC WCS: %s Service" % instance.workspace Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict(extension="html", name=ogc_wcs_name, url=ogc_wcs_url, mime="text/html", link_type="OGC:WCS"), ) # remove links that belong to and old address for link in instance.link_set.all(): if ( not urlparse(settings.SITEURL).hostname == urlparse(link.url).hostname and not urlparse(ogc_server_settings.public_url).hostname == urlparse(link.url).hostname ): link.delete() # Save layer attributes set_attributes(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer)
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return if instance.storeType == "remoteStore": # Save layer attributes set_attributes(instance) return if not getattr(instance, 'gs_resource', None): try: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return else: gs_resource = instance.gs_resource if gs_resource is None: return if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', ) ) if instance.storeType == "dataStore": links = wfs_links( ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', ) ) if gs_resource.store.type and gs_resource.store.type.lower() == 'geogig' and \ gs_resource.store.connection_parameters.get('geogig_repository'): repo_url = '{url}geogig/{geogig_repository}'.format( url=ogc_server_settings.public_url, geogig_repository=gs_resource.store.connection_parameters.get('geogig_repository')) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html' ) ) def command_url(command): return "{repo_url}/{command}.json?{path}".format(repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html' ) ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html' ) ) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8')) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', ) ) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', ) ) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', ) ) tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8' ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=tile_url, defaults=dict( extension='tiles', name="Tiles", mime='image/png', link_type='image', ) ) html_link_url = '%s%s' % ( settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', ) ) params = { 'layers': instance.typename.encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, 'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z' } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.PUBLIC_LOCATION + \ "wms/reflect?" + p thumbnail_create_url = ogc_server_settings.LOCATION + \ "wms/reflect?" + p create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url, ogc_client=http_client) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.typename + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', ) ) ogc_wms_path = '%s/wms' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', ) ) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', ) ) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', ) ) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse( settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Save layer attributes set_attributes(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer)
def geoserver_post_save2(layer_id): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ from geonode.layers.models import Layer instance = Layer.objects.get(id=layer_id) # Don't run this signal if is a Layer from a remote service if getattr(instance, "service", None) is not None: return instance # Don't run this signal handler if it is a tile layer # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') == 'tileStore': return instance gs_resource = None # If the store in None then it's a new instance from an upload, # only in this case run the geonode_uplaod method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return instance gs_name, workspace, values, gs_resource = geoserver_upload( instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) # Set fields obtained via the geoserver upload. instance.name = gs_name instance.workspace = workspace instance.store = values['store'] # Iterate over values from geoserver. for key in ['typename', 'store', 'storeType']: setattr(instance, key, values[key]) instance.save() if not gs_resource: gs_resource = gs_catalog.get_resource(instance.name, store=instance.store, workspace=instance.workspace) if gs_resource: gs_resource.title = instance.title if instance.title else "" gs_resource.abstract = instance.abstract if instance.abstract else "" gs_resource.name = instance.name if instance.name else "" # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) if gs_resource: gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if gs_resource and getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) gs_layer = gs_catalog.get_layer(instance.name) if instance.poc and instance.poc: # gsconfig now utilizes an attribution dictionary gs_layer.attribution = { 'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None } profile = Profile.objects.get(username=instance.poc.username) gs_layer.attribution_link = settings.SITEURL[: -1] + profile.get_absolute_url( ) # gs_layer should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_layer) """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ if gs_resource: bbox = gs_resource.latlon_bbox # FIXME(Ariel): Correct srid setting below # self.srid = gs_resource.src instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource instance.save() if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return instance if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return instance if not getattr(instance, 'gs_resource', None): try: gs_resource = gs_catalog.get_resource(instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return instance else: gs_resource = instance.gs_resource if gs_resource is None: return instance if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', )) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', )) gs_store_type = gs_resource.store.type.lower( ) if gs_resource.store.type else None geogig_repository = gs_resource.store.connection_parameters.get( 'geogig_repository', '') geogig_repo_name = geogig_repository.replace('geoserver://', '') if gs_store_type == 'geogig' and geogig_repo_name: repo_url = '{url}geogig/repos/{repo_name}'.format( url=ogc_server_settings.public_url, repo_name=geogig_repo_name) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html')) def command_url(command): return "{repo_url}/{command}.json?{path}".format( repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html')) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html')) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8')) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', )) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', )) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', )) html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', )) create_gs_thumbnail(instance, overwrite=False) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.typename + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', )) ogc_wms_path = '%s/wms' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', )) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', )) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', )) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse(settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Define the link after the cleanup, we should use this more rather then remove # potential parasites tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8') link, created = Link.objects.get_or_create( resource=instance.resourcebase_ptr, extension='tiles', name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer) return instance
def geoserver_post_save_local(instance, *args, **kwargs): """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ # Don't run this signal if is a Layer from a remote service if getattr(instance, "remote_service", None) is not None: return # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') in ['tileStore', 'remoteStore']: return instance gs_resource = None values = None # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return gs_name, workspace, values, gs_resource = geoserver_upload(instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) if not gs_resource: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) if not gs_resource: gs_resource = gs_catalog.get_resource(instance.alternate) if gs_resource: gs_resource.title = instance.title or "" gs_resource.abstract = instance.abstract or "" gs_resource.name = instance.name or "" if not values: values = dict(store=gs_resource.store.name, storeType=gs_resource.store.resource_type, alternate=gs_resource.store.workspace.name + ':' + gs_resource.name, title=gs_resource.title or gs_resource.store.name, abstract=gs_resource.abstract or '', owner=instance.owner) else: msg = "There isn't a geoserver resource for this layer: %s" % instance.name logger.exception(msg) raise Exception(msg) # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) # Update Attribution link if instance.poc: # gsconfig now utilizes an attribution dictionary gs_resource.attribution = {'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None} profile = Profile.objects.get(username=instance.poc.username) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL gs_resource.attribution_link = site_url + profile.get_absolute_url() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save layer named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) if isinstance(instance, ResourceBase): if hasattr(instance, 'layer'): instance = instance.layer else: return # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # set SLD sld = instance.default_style.sld_body if instance.default_style else None if sld: set_layer_style(instance, instance.alternate, sld) # Invalidate GeoWebCache for the updated resource try: _stylefilterparams_geowebcache_layer(instance.alternate) _invalidate_geowebcache_layer(instance.alternate) except BaseException: pass if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ # instance.name = instance.name or gs_resource.name # instance.title = instance.title or gs_resource.title instance.abstract = gs_resource.abstract or '' instance.workspace = gs_resource.store.workspace.name instance.store = gs_resource.store.name try: logger.debug(" -------------------------------------------------- ") bbox = gs_resource.native_bbox logger.debug(bbox) logger.debug(" -------------------------------------------------- ") # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] instance.srid = bbox[4] except BaseException: pass if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" elif instance.bbox_x0 and instance.bbox_x1 and instance.bbox_y0 and instance.bbox_y1: # Guessing 'EPSG:4326' by default instance.srid = 'EPSG:4326' else: raise GeoNodeException("Invalid Projection. Layer is missing CRS!") # Iterate over values from geoserver. for key in ['alternate', 'store', 'storeType']: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, values[key]) if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = 'true' gs_catalog.save(gs_resource) if not settings.FREETEXT_KEYWORDS_READONLY: try: if len(instance.keyword_list()) == 0 and gs_resource.keywords: for keyword in gs_resource.keywords: if keyword not in instance.keyword_list(): instance.keywords.add(keyword) except BaseException: pass if any(instance.keyword_list()): keywords = instance.keyword_list() gs_resource.keywords = [kw for kw in list(set(keywords))] # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) to_update = { 'title': instance.title or instance.name, 'abstract': instance.abstract or "", 'alternate': instance.alternate, 'bbox_x0': instance.bbox_x0, 'bbox_x1': instance.bbox_x1, 'bbox_y0': instance.bbox_y0, 'bbox_y1': instance.bbox_y1, 'srid': instance.srid } # Update ResourceBase resources = ResourceBase.objects.filter(id=instance.resourcebase_ptr.id) resources.update(**to_update) # to_update['name'] = instance.name, to_update['workspace'] = instance.workspace to_update['store'] = instance.store to_update['storeType'] = instance.storeType to_update['typename'] = instance.alternate # Save all the modified information in the instance without triggering signals. Layer.objects.filter(id=instance.id).update(**to_update) # Refresh from DB instance.refresh_from_db() # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource # Refresh and create the instance default links layer = Layer.objects.get(id=instance.id) set_resource_default_links(instance, layer, prune=True) # some thumbnail generators will update thumbnail_url. If so, don't # immediately re-generate the thumbnail here. use layer#save(update_fields=['thumbnail_url']) if 'update_fields' in kwargs and kwargs['update_fields'] is not None and \ 'thumbnail_url' in kwargs['update_fields']: logger.info("... Creating Thumbnail for Layer [%s]" % (instance.alternate)) create_gs_thumbnail(instance, overwrite=True) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 catalogue_post_save(instance, Layer) # Updating HAYSTACK Indexes if needed if settings.HAYSTACK_SEARCH: from django.core.management import call_command call_command('update_index')
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ # Don't run this signal handler if it is a tile layer # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') == 'tileStore': return if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return if not getattr(instance, 'gs_resource', None): try: gs_resource = gs_catalog.get_resource(instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return else: gs_resource = instance.gs_resource if gs_resource is None: return if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', )) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', )) gs_store_type = gs_resource.store.type.lower( ) if gs_resource.store.type else None geogig_repository = gs_resource.store.connection_parameters.get( 'geogig_repository', '') geogig_repo_name = geogig_repository.replace('geoserver://', '') if gs_store_type == 'geogig' and geogig_repo_name: repo_url = '{url}geogig/repos/{repo_name}'.format( url=ogc_server_settings.public_url, repo_name=geogig_repo_name) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html')) def command_url(command): return "{repo_url}/{command}.json?{path}".format( repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html')) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html')) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8')) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', )) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', )) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', )) html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', )) create_gs_thumbnail(instance, overwrite=True) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.typename + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', )) ogc_wms_path = '%s/wms' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', )) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', )) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', )) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse(settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Define the link after the cleanup, we should use this more rather then remove # potential parasites tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8') link, created = Link.objects.get_or_create( resource=instance.resourcebase_ptr, extension='tiles', name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer)
def geoserver_post_save2(layer_id): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ from geonode.layers.models import Layer instance = Layer.objects.get(id=layer_id) # Don't run this signal if is a Layer from a remote service if getattr(instance, "service", None) is not None: return instance # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') in ['tileStore', 'remoteStore']: return instance gs_resource = None # If the store in None then it's a new instance from an upload, # only in this case run the geonode_uplaod method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return instance gs_name, workspace, values, gs_resource = geoserver_upload(instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) # Set fields obtained via the geoserver upload. instance.name = gs_name instance.workspace = workspace instance.store = values['store'] # Iterate over values from geoserver. for key in ['typename', 'store', 'storeType']: setattr(instance, key, values[key]) instance.save() if not gs_resource: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) if gs_resource: gs_resource.title = instance.title if instance.title else "" gs_resource.abstract = instance.abstract if instance.abstract else "" gs_resource.name = instance.name if instance.name else "" # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) if gs_resource: gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if gs_resource and getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) gs_layer = gs_catalog.get_layer(instance.name) if instance.poc and instance.poc: # gsconfig now utilizes an attribution dictionary gs_layer.attribution = {'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None} profile = Profile.objects.get(username=instance.poc.username) gs_layer.attribution_link = settings.SITEURL[ :-1] + profile.get_absolute_url() # gs_layer should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_layer) """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ if gs_resource: bbox = gs_resource.latlon_bbox # FIXME(Ariel): Correct srid setting below # self.srid = gs_resource.src instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource instance.save() if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return instance if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return instance if not getattr(instance, 'gs_resource', None): try: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) except socket_error as serr: if serr.errno != errno.ECONNREFUSED: # Not the error we are looking for, re-raise raise serr # If the connection is refused, take it easy. return instance else: gs_resource = instance.gs_resource if gs_resource is None: return instance if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = instance.is_published gs_catalog.save(gs_resource) if any(instance.keyword_list()): gs_resource.keywords = instance.keyword_list() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', ) ) if instance.storeType == "dataStore": links = wfs_links( ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', ) ) gs_store_type = gs_resource.store.type.lower() if gs_resource.store.type else None geogig_repository = gs_resource.store.connection_parameters.get('geogig_repository', '') geogig_repo_name = geogig_repository.replace('geoserver://', '') if gs_store_type == 'geogig' and geogig_repo_name: repo_url = '{url}geogig/repos/{repo_name}'.format( url=ogc_server_settings.public_url, repo_name=geogig_repo_name) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html' ) ) def command_url(command): return "{repo_url}/{command}.json?{path}".format(repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html' ) ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html' ) ) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8')) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', ) ) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', ) ) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.typename.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', ) ) html_link_url = '%s%s' % ( settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', ) ) create_gs_thumbnail(instance, overwrite=False) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.typename + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', ) ) ogc_wms_path = '%s/wms' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', ) ) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', ) ) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', ) ) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse( settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Define the link after the cleanup, we should use this more rather then remove # potential parasites tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8' ) link, created = Link.objects.get_or_create(resource=instance.resourcebase_ptr, extension='tiles', name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Layer catalogue_post_save(instance, Layer) return instance
def geoserver_post_save_local(instance, *args, **kwargs): """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ # Don't run this signal if is a Layer from a remote service if getattr(instance, "remote_service", None) is not None: return # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') in ['tileStore', 'remoteStore']: return instance gs_resource = None values = None # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return gs_name, workspace, values, gs_resource = geoserver_upload(instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) if not gs_resource: gs_resource = gs_catalog.get_resource( instance.name, store=instance.store, workspace=instance.workspace) if not gs_resource: gs_resource = gs_catalog.get_resource(instance.alternate) if gs_resource: gs_resource.title = instance.title or "" gs_resource.abstract = instance.abstract or "" gs_resource.name = instance.name or "" if not values: values = dict(store=gs_resource.store.name, storeType=gs_resource.store.resource_type, alternate=gs_resource.store.workspace.name + ':' + gs_resource.name, title=gs_resource.title or gs_resource.store.name, abstract=gs_resource.abstract or '', owner=instance.owner) else: msg = "There isn't a geoserver resource for this layer: %s" % instance.name logger.exception(msg) raise Exception(msg) # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) # Update Attribution link if instance.poc: # gsconfig now utilizes an attribution dictionary gs_resource.attribution = {'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None} profile = Profile.objects.get(username=instance.poc.username) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL gs_resource.attribution_link = site_url + profile.get_absolute_url() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save layer named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) if isinstance(instance, ResourceBase): if hasattr(instance, 'layer'): instance = instance.layer else: return # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # set SLD sld = instance.default_style.sld_body if instance.default_style else None if sld: set_layer_style(instance, instance.alternate, sld) # Invalidate GeoWebCache for the updated resource try: _stylefilterparams_geowebcache_layer(instance.alternate) _invalidate_geowebcache_layer(instance.alternate) except BaseException: pass if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ # instance.name = instance.name or gs_resource.name # instance.title = instance.title or gs_resource.title instance.abstract = gs_resource.abstract or '' instance.workspace = gs_resource.store.workspace.name instance.store = gs_resource.store.name try: bbox = gs_resource.native_bbox # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] instance.srid = bbox[4] except BaseException: pass if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" else: raise GeoNodeException("Invalid Projection. Layer is missing CRS!") # Iterate over values from geoserver. for key in ['alternate', 'store', 'storeType']: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, values[key]) if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = 'true' if instance.is_published else 'false' gs_catalog.save(gs_resource) if not settings.FREETEXT_KEYWORDS_READONLY: try: if len(instance.keyword_list()) == 0 and gs_resource.keywords: for keyword in gs_resource.keywords: if keyword not in instance.keyword_list(): instance.keywords.add(keyword) except BaseException: pass if any(instance.keyword_list()): keywords = instance.keyword_list() gs_resource.keywords = [kw.decode("utf-8", "replace") for kw in list(set(keywords))] # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) to_update = { 'title': instance.title or instance.name, 'abstract': instance.abstract or "", 'alternate': instance.alternate, 'bbox_x0': instance.bbox_x0, 'bbox_x1': instance.bbox_x1, 'bbox_y0': instance.bbox_y0, 'bbox_y1': instance.bbox_y1, 'srid': instance.srid } # Update ResourceBase resources = ResourceBase.objects.filter(id=instance.resourcebase_ptr.id) resources.update(**to_update) # to_update['name'] = instance.name, to_update['workspace'] = instance.workspace to_update['store'] = instance.store to_update['storeType'] = instance.storeType to_update['typename'] = instance.alternate # Save all the modified information in the instance without triggering signals. Layer.objects.filter(id=instance.id).update(**to_update) # Refresh from DB instance.refresh_from_db() # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource try: bbox = gs_resource.native_bbox except BaseException: bbox = instance.bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Parse Layer BBOX and SRID srid = instance.srid if instance.srid else getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:4326') if srid and instance.bbox_x0: bbox = ','.join(str(x) for x in [instance.bbox_x0, instance.bbox_y0, instance.bbox_x1, instance.bbox_y1]) # Create Raw Data download link try: path = gs_resource.dom.findall('nativeName') except BaseException: path = instance.alternate download_url = urljoin(settings.SITEURL, reverse('download', args=[instance.id])) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=download_url, defaults=dict(extension='zip', name='Original Dataset', mime='application/octet-stream', link_type='original', ) ) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.alternate.encode('utf-8'), bbox, srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', ) ) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.alternate.encode('utf-8'), bbox=None, # bbox filter should be set at runtime otherwise conflicting with CQL srid=srid) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', ) ) gs_store_type = gs_resource.store.type.lower() if gs_resource.store.type else None geogig_repository = gs_resource.store.connection_parameters.get('geogig_repository', '') geogig_repo_name = geogig_repository.replace('geoserver://', '') if gs_store_type == 'geogig' and geogig_repo_name: repo_url = '{url}geogig/repos/{repo_name}'.format( url=ogc_server_settings.public_url, repo_name=geogig_repo_name) try: path = gs_resource.dom.findall('nativeName') except BaseException: path = instance.alternate if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html' ) ) def command_url(command): return "{repo_url}/{command}.json?{path}".format(repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html' ) ) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html' ) ) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.alternate.encode('utf-8'), bbox, srid) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', ) ) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.alternate.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', ) ) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.alternate.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', ) ) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL html_link_url = '%s%s' % ( site_url, instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.alternate, mime='text/html', link_type='html', ) ) # some thumbnail generators will update thumbnail_url. If so, don't # immediately re-generate the thumbnail here. use layer#save(update_fields=['thumbnail_url']) if 'update_fields' in kwargs and kwargs['update_fields'] is not None and \ 'thumbnail_url' in kwargs['update_fields']: logger.info("... Creating Thumbnail for Layer [%s]" % (instance.alternate)) create_gs_thumbnail(instance, overwrite=True) try: Link.objects.filter(resource=instance.resourcebase_ptr, name='Legend').delete() except BaseException: pass for style in instance.styles.all(): legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.alternate + '&STYLE=' + style.name + \ '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', ) ) # ogc_wms_path = '%s/ows' % instance.workspace ogc_wms_path = 'ows' ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', ) ) if instance.storeType == "dataStore": # ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_path = 'wfs' ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', ) ) if instance.storeType == "coverageStore": # ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_path = 'wcs' ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', ) ) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse( settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Define the link after the cleanup, we should use this more rather then remove # potential parasites tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.alternate.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8' ) link, created = Link.objects.get_or_create(resource=instance.resourcebase_ptr, extension='tiles', name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 catalogue_post_save(instance, Layer) # Updating HAYSTACK Indexes if needed if settings.HAYSTACK_SEARCH: from django.core.management import call_command call_command('update_index')
def geoserver_post_save_local(instance, *args, **kwargs): """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ # Don't run this signal if is a Layer from a remote service if getattr(instance, "remote_service", None) is not None: return # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr( instance, 'storeType') in ['tileStore', 'remoteStore']: return instance gs_resource = None values = None # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return gs_name, workspace, values, gs_resource = geoserver_upload( instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) if not gs_resource: gs_resource = gs_catalog.get_resource(instance.name, store=instance.store, workspace=instance.workspace) if not gs_resource: gs_resource = gs_catalog.get_resource(instance.alternate) if gs_resource: gs_resource.title = instance.title or "" gs_resource.abstract = instance.abstract or "" gs_resource.name = instance.name or "" if not values: values = dict(store=gs_resource.store.name, storeType=gs_resource.store.resource_type, alternate=gs_resource.store.workspace.name + ':' + gs_resource.name, title=gs_resource.title or gs_resource.store.name, abstract=gs_resource.abstract or '', owner=instance.owner) else: msg = "There isn't a geoserver resource for this layer: %s" % instance.name logger.exception(msg) raise Exception(msg) # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ( 'Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg, ) logger.exception(e) gs_layer = gs_catalog.get_layer(instance.name) if not gs_layer: gs_layer = gs_catalog.get_layer(instance.alternate) if gs_layer and instance.poc: # gsconfig now utilizes an attribution dictionary gs_layer.attribution = { 'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None } profile = Profile.objects.get(username=instance.poc.username) gs_layer.attribution_link = settings.SITEURL[: -1] + profile.get_absolute_url( ) # gs_layer should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_layer) except geoserver.catalog.FailedRequestError as e: msg = ( 'Error while trying to save layer named %s in GeoServer, ' 'try to use: "%s"' % (gs_layer, str(e))) e.args = (msg, ) logger.exception(e) if type(instance) is ResourceBase: if hasattr(instance, 'layer'): instance = instance.layer else: return if instance.storeType == "remoteStore": # Save layer attributes set_attributes_from_geoserver(instance) return """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ # instance.name = instance.name or gs_layer.name # instance.title = instance.title or gs_resource.title instance.abstract = gs_resource.abstract or '' instance.workspace = gs_resource.store.workspace.name instance.store = gs_resource.store.name bbox = gs_resource.native_bbox # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] instance.srid = bbox[4] if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" else: raise GeoNodeException("Invalid Projection. Layer is missing CRS!") # Iterate over values from geoserver. for key in ['alternate', 'store', 'storeType']: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, values[key]) if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = 'true' if instance.is_published else 'false' gs_catalog.save(gs_resource) if not settings.FREETEXT_KEYWORDS_READONLY: if gs_resource.keywords: for keyword in gs_resource.keywords: if keyword not in instance.keyword_list(): instance.keywords.add(keyword) if any(instance.keyword_list()): keywords = instance.keyword_list() gs_resource.keywords = list(set(keywords)) # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ( 'Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg, ) logger.exception(e) to_update = { 'title': instance.title or instance.name, 'abstract': instance.abstract or "", 'alternate': instance.alternate, 'bbox_x0': instance.bbox_x0, 'bbox_x1': instance.bbox_x1, 'bbox_y0': instance.bbox_y0, 'bbox_y1': instance.bbox_y1, 'srid': instance.srid } # Update ResourceBase resources = ResourceBase.objects.filter(id=instance.resourcebase_ptr.id) resources.update(**to_update) # to_update['name'] = instance.name, to_update['workspace'] = instance.workspace to_update['store'] = instance.store to_update['storeType'] = instance.storeType to_update['typename'] = instance.alternate # Save all the modified information in the instance without triggering signals. Layer.objects.filter(id=instance.id).update(**to_update) # Refresh from DB instance.refresh_from_db() # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource bbox = gs_resource.native_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.alternate.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', )) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.alternate.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', )) gs_store_type = gs_resource.store.type.lower( ) if gs_resource.store.type else None geogig_repository = gs_resource.store.connection_parameters.get( 'geogig_repository', '') geogig_repo_name = geogig_repository.replace('geoserver://', '') if gs_store_type == 'geogig' and geogig_repo_name: repo_url = '{url}geogig/repos/{repo_name}'.format( url=ogc_server_settings.public_url, repo_name=geogig_repo_name) path = gs_resource.dom.findall('nativeName') if path: path = 'path={path}'.format(path=path[0].text) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=repo_url, defaults=dict(extension='html', name='Clone in GeoGig', mime='text/xml', link_type='html')) def command_url(command): return "{repo_url}/{command}.json?{path}".format( repo_url=repo_url, path=path, command=command) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('log'), defaults=dict(extension='json', name='GeoGig log', mime='application/json', link_type='html')) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=command_url('statistics'), defaults=dict(extension='json', name='GeoGig statistics', mime='application/json', link_type='html')) elif instance.storeType == 'coverageStore': links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.alternate.encode('utf-8'), ','.join(str(x) for x in instance.bbox[0:4]), instance.srid) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', )) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.alternate.encode('utf-8'), 'mode': "download"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name="KML", mime='text/xml', link_type='data', )) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + \ urllib.urlencode({'layers': instance.alternate.encode('utf-8'), 'mode': "refresh"}) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', )) html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.alternate, mime='text/html', link_type='html', )) # some thumbnail generators will update thumbnail_url. If so, don't # immediately re-generate the thumbnail here. use layer#save(update_fields=['thumbnail_url']) if not ('update_fields' in kwargs and kwargs['update_fields'] is not None and 'thumbnail_url' in kwargs['update_fields']): logger.info("Creating Thumbnail for Layer [%s]" % (instance.alternate)) create_gs_thumbnail(instance, overwrite=True) legend_url = ogc_server_settings.PUBLIC_LOCATION + \ 'wms?request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=' + \ instance.alternate + '&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=legend_url, defaults=dict( extension='png', name='Legend', url=legend_url, mime='image/png', link_type='image', )) ogc_wms_path = '%s/ows' % instance.workspace ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=ogc_wms_name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', )) if instance.storeType == "dataStore": ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=ogc_wfs_name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', )) if instance.storeType == "coverageStore": ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=ogc_wcs_name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', )) # remove links that belong to and old address for link in instance.link_set.all(): if not urlparse(settings.SITEURL).hostname == urlparse( link.url).hostname and not urlparse( ogc_server_settings.public_url).hostname == urlparse( link.url).hostname: link.delete() # Define the link after the cleanup, we should use this more rather then remove # potential parasites tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.alternate.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8') link, created = Link.objects.get_or_create( resource=instance.resourcebase_ptr, extension='tiles', name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 catalogue_post_save(instance, Layer)
def geoserver_post_save_local(instance, *args, **kwargs): """Send information to geoserver. The attributes sent include: * Title * Abstract * Name * Keywords * Metadata Links, * Point of Contact name and url """ instance.refresh_from_db() # Don't run this signal if is a Layer from a remote service if getattr(instance, "remote_service", None) is not None: return # Don't run this signal handler if it is a tile layer or a remote store (Service) # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. if hasattr(instance, 'storeType') and getattr(instance, 'storeType') in ['tileStore', 'remoteStore']: return instance gs_resource = None values = None _tries = 0 _max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 2) # If the store in None then it's a new instance from an upload, # only in this case run the geoserver_upload method if not instance.store or getattr(instance, 'overwrite', False): base_file, info = instance.get_base_file() # There is no need to process it if there is not file. if base_file is None: return gs_name, workspace, values, gs_resource = geoserver_upload(instance, base_file.file.path, instance.owner, instance.name, overwrite=True, title=instance.title, abstract=instance.abstract, # keywords=instance.keywords, charset=instance.charset) def fetch_gs_resource(values, tries): try: gs_resource = gs_catalog.get_resource( name=instance.name, store=instance.store, workspace=instance.workspace) except Exception: try: gs_resource = gs_catalog.get_resource( name=instance.alternate, store=instance.store, workspace=instance.workspace) except Exception: try: gs_resource = gs_catalog.get_resource( name=instance.alternate or instance.typename) except Exception: gs_resource = None if gs_resource: gs_resource.title = instance.title or "" gs_resource.abstract = instance.abstract or "" gs_resource.name = instance.name or "" if not values: values = dict(store=gs_resource.store.name, storeType=gs_resource.store.resource_type, alternate=gs_resource.store.workspace.name + ':' + gs_resource.name, title=gs_resource.title or gs_resource.store.name, abstract=gs_resource.abstract or '', owner=instance.owner) else: msg = "There isn't a geoserver resource for this layer: %s" % instance.name logger.exception(msg) if tries >= _max_tries: # raise GeoNodeException(msg) return (values, None) gs_resource = None sleep(3.00) return (values, gs_resource) while not gs_resource and _tries < _max_tries: values, gs_resource = fetch_gs_resource(values, _tries) _tries += 1 # Get metadata links metadata_links = [] for link in instance.link_set.metadata(): metadata_links.append((link.mime, link.name, link.url)) if gs_resource: logger.debug("Found geoserver resource for this layer: %s" % instance.name) gs_resource.metadata_links = metadata_links # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) # Update Attribution link if instance.poc: # gsconfig now utilizes an attribution dictionary gs_resource.attribution = {'title': str(instance.poc), 'width': None, 'height': None, 'href': None, 'url': None, 'type': None} profile = get_user_model().objects.get(username=instance.poc.username) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL gs_resource.attribution_link = site_url + profile.get_absolute_url() # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): try: gs_catalog.save(gs_resource) except geoserver.catalog.FailedRequestError as e: msg = ('Error while trying to save layer named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) else: msg = "There isn't a geoserver resource for this layer: %s" % instance.name logger.warn(msg) if isinstance(instance, ResourceBase): if hasattr(instance, 'layer'): instance = instance.layer else: return # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # Invalidate GeoWebCache for the updated resource try: _stylefilterparams_geowebcache_layer(instance.alternate) _invalidate_geowebcache_layer(instance.alternate) except Exception: pass if instance.storeType == "remoteStore": return if gs_resource: """Get information from geoserver. The attributes retrieved include: * Bounding Box * SRID * Download links (WMS, WCS or WFS and KML) * Styles (SLD) """ try: instance.abstract = gs_resource.abstract or '' except Exception as e: logger.exception(e) instance.abstract = '' instance.workspace = gs_resource.store.workspace.name instance.store = gs_resource.store.name try: bbox = gs_resource.native_bbox # Set bounding box values instance.bbox_x0 = bbox[0] instance.bbox_x1 = bbox[1] instance.bbox_y0 = bbox[2] instance.bbox_y1 = bbox[3] instance.srid = bbox[4] except Exception as e: logger.exception(e) if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ instance.srid.replace(':', '/').lower() + "/" elif instance.bbox_x0 and instance.bbox_x1 and instance.bbox_y0 and instance.bbox_y1: # Guessing 'EPSG:4326' by default instance.srid = 'EPSG:4326' else: raise GeoNodeException("Invalid Projection. Layer is missing CRS!") # Iterate over values from geoserver. if gs_resource: for key in ['alternate', 'store', 'storeType']: # attr_name = key if 'typename' not in key else 'alternate' # print attr_name setattr(instance, key, values[key]) if gs_resource: try: if settings.RESOURCE_PUBLISHING: if instance.is_published != gs_resource.advertised: if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_resource.advertised = 'true' gs_catalog.save(gs_resource) if not settings.FREETEXT_KEYWORDS_READONLY: # AF: Warning - this won't allow people to have empty keywords on GeoNode if len(instance.keyword_list()) == 0 and gs_resource.keywords: for keyword in gs_resource.keywords: if keyword not in instance.keyword_list(): instance.keywords.add(keyword) if any(instance.keyword_list()): keywords = instance.keyword_list() gs_resource.keywords = [kw for kw in list(set(keywords))] # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) except Exception as e: msg = ('Error while trying to save resource named %s in GeoServer, ' 'try to use: "%s"' % (gs_resource, str(e))) e.args = (msg,) logger.exception(e) to_update = { 'title': instance.title or instance.name, 'abstract': instance.abstract or "", 'alternate': instance.alternate, 'bbox_x0': instance.bbox_x0, 'bbox_x1': instance.bbox_x1, 'bbox_y0': instance.bbox_y0, 'bbox_y1': instance.bbox_y1, 'srid': instance.srid } # Update ResourceBase resources = ResourceBase.objects.filter(id=instance.resourcebase_ptr.id) resources.update(**to_update) # to_update['name'] = instance.name, to_update['workspace'] = instance.workspace to_update['store'] = instance.store to_update['storeType'] = instance.storeType to_update['typename'] = instance.alternate # Save all the modified information in the instance without triggering signals. Layer.objects.filter(id=instance.id).update(**to_update) # Refresh from DB instance.refresh_from_db() # Updating the Catalogue catalogue_post_save(instance=instance, sender=instance.__class__) # store the resource to avoid another geoserver call in the post_save if gs_resource: instance.gs_resource = gs_resource # some thumbnail generators will update thumbnail_url. If so, don't # immediately re-generate the thumbnail here. use layer#save(update_fields=['thumbnail_url']) if gs_resource: logger.debug("... Creating Default Resource Links for Layer [%s]" % (instance.alternate)) set_resource_default_links(instance, instance, prune=True) if 'update_fields' in kwargs and kwargs['update_fields'] is not None and \ 'thumbnail_url' in kwargs['update_fields']: logger.debug("... Creating Thumbnail for Layer [%s]" % (instance.alternate)) create_gs_thumbnail(instance, overwrite=True) # Updating HAYSTACK Indexes if needed if settings.HAYSTACK_SEARCH: from django.core.management import call_command call_command('update_index')
name="Tiles", mime='image/png', link_type='image', ) if created: Link.objects.filter(pk=link.pk).update(url=tile_url) # Save layer attributes set_attributes_from_geoserver(instance) # Save layer styles set_styles(instance, gs_catalog) # NOTTODO by simod: we should not do this! # need to be removed when fixing #2015 catalogue_post_save(instance, Layer) try: # update the elastic search index for the object after post_save triggers have fired. update_es_index(sender, sender.objects.get(id=instance.id)) update_es_index(MapStory, MapStory.objects.get(id=instance.story.id)) except: pass def geoserver_pre_save_maplayer(instance, sender, **kwargs): # If this object was saved via fixtures, # do not do post processing. if kwargs.get('raw', False): return