def test_update_xml(self):
"""Test inserting xml element."""
file_path = os.path.realpath(__file__)
test_dir = os.path.dirname(file_path)
xml_file_path = os.path.join(test_dir, 'data', 'test.xml')
# Creating temporary xml file
temp_xml_file_path = os.path.join(test_dir, 'data', 'temp_test.xml')
shutil.copy2(xml_file_path, temp_xml_file_path)
# Check if the temp file is created
self.assertTrue(os.path.exists(temp_xml_file_path))
new_values = {
'title': 'New Title',
}
# Make sure there is no new value in the xml
with open(temp_xml_file_path, 'r') as temp_xml_file:
temp_xml_content = temp_xml_file.read()
self.assertNotIn(new_values['title'], temp_xml_content)
# Updating the xml file
update_xml(temp_xml_file_path, new_values)
# Check if the new value has been added.
with open(temp_xml_file_path, 'r') as temp_xml_file:
temp_xml_content = temp_xml_file.read()
self.assertIn(new_values['title'], temp_xml_content)
# Remove temp file
os.remove(temp_xml_file_path)
self.assertFalse(os.path.exists(temp_xml_file_path))
def test_update_xml(self):
"""Test inserting xml element."""
file_path = os.path.realpath(__file__)
test_dir = os.path.dirname(file_path)
xml_file_path = os.path.join(test_dir, 'data', 'test.xml')
# Creating temporary xml file
temp_xml_file_path = os.path.join(test_dir, 'data', 'temp_test.xml')
shutil.copy2(xml_file_path, temp_xml_file_path)
# Check if the temp file is created
self.assertTrue(os.path.exists(temp_xml_file_path))
new_values = {
'title': 'New Title',
}
# Make sure there is no new value in the xml
with open(temp_xml_file_path, 'r') as temp_xml_file:
temp_xml_content = temp_xml_file.read()
self.assertNotIn(new_values['title'], temp_xml_content)
# Updating the xml file
update_xml(temp_xml_file_path, new_values)
# Check if the new value has been added.
with open(temp_xml_file_path, 'r') as temp_xml_file:
temp_xml_content = temp_xml_file.read()
self.assertIn(new_values['title'], temp_xml_content)
# Remove temp file
os.remove(temp_xml_file_path)
self.assertFalse(os.path.exists(temp_xml_file_path))
def qgis_server_post_save(instance, sender, **kwargs):
"""Save keywords to QGIS Server.
The way keywords are implemented requires the layer to be saved to the
database before accessing them.
This hook also creates QGIS Project. Which is essentials for QGIS Server.
There are also several Geonode Links generated, like thumbnail and legends
:param instance: geonode Layer
:type instance: Layer
:param sender: geonode Layer type
:type sender: type(Layer)
"""
if not sender == Layer:
return
# TODO
# 1. Create or update associated QGISServerLayer [Done]
# 2. Create Link for the tile and legend.
logger.debug('QGIS Server Post Save')
# copy layer to QGIS Layer Directory
try:
geonode_layer_path = instance.get_base_file()[0].file.path
except AttributeError:
logger.debug('Layer does not have base file')
return
qgis_layer, created = QGISServerLayer.objects.get_or_create(layer=instance)
logger.debug('Geonode Layer Path %s' % geonode_layer_path)
base_filename, original_ext = os.path.splitext(geonode_layer_path)
extensions = QGISServerLayer.accepted_format
is_shapefile = False
for ext in extensions:
if os.path.exists(base_filename + '.' + ext):
is_shapefile = is_shapefile or ext == 'shp'
try:
if created:
# Assuming different layer has different filename because
# geonode rename it
shutil.copy2(base_filename + '.' + ext,
QGIS_layer_directory)
logger.debug('Create new basefile')
else:
# If there is already a file, replace the old one
qgis_layer_base_filename = qgis_layer.qgis_layer_path_prefix
shutil.copy2(base_filename + '.' + ext,
qgis_layer_base_filename + '.' + ext)
logger.debug('Overwrite existing basefile')
logger.debug('Copying %s' % base_filename + '.' + ext +
' Success')
logger.debug('Into %s' % QGIS_layer_directory)
except IOError as e:
logger.debug('Copying %s' % base_filename + '.' + ext +
' FAILED ' + e)
if created:
# Only set when creating new QGISServerLayer Object
geonode_filename = os.path.basename(geonode_layer_path)
basename = os.path.splitext(geonode_filename)[0]
qgis_layer.base_layer_path = os.path.join(
QGIS_layer_directory,
basename + original_ext # Already with dot
)
qgis_layer.save()
# refresh to get QGIS Layer
instance.refresh_from_db()
# Set layer crs
try:
if is_shapefile:
dataset = ogr.Open(geonode_layer_path)
layer = dataset.GetLayer()
spatial_ref = layer.GetSpatialRef()
srid = spatial_ref.GetAuthorityCode(None) if spatial_ref else None
if srid:
instance.srid = srid
else:
dataset = gdal.Open(geonode_layer_path)
prj = dataset.GetProjection()
srs = osr.SpatialReference(wkt=prj)
srid = srs.GetAuthorityCode(None) if srs else None
if srid:
instance.srid = srid
except Exception as e:
logger.debug("Can't retrieve projection: {layer}".format(
layer=geonode_layer_path))
logger.exception(e)
# Refresh and create the instance default links
set_resource_default_links(instance,
qgis_layer,
is_shapefile=is_shapefile,
original_ext=original_ext)
# Create thumbnail
overwrite = getattr(instance, 'overwrite', False)
create_qgis_server_thumbnail.delay(get_model_path(instance),
instance.id,
overwrite=overwrite)
# Attributes
set_attributes(instance)
# Update xml file
# Read metadata from layer that InaSAFE use.
# Some are not found: organisation, email, url
try:
new_values = {
'date': instance.date.isoformat(),
'abstract': instance.abstract,
'title': instance.title,
'license': instance.license_verbose,
}
except (TypeError, AttributeError):
new_values = {}
pass
# Get the path of the metadata file
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
xml_file_path = basename + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Also update xml in QGIS Server
xml_file_path = qgis_layer.qgis_layer_path_prefix + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Remove existing tile caches if overwrite
if overwrite:
tiles_directory = settings.QGIS_SERVER_CONFIG['tiles_directory']
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
basename = os.path.basename(basename)
tiles_cache_path = os.path.join(tiles_directory, basename)
try:
shutil.rmtree(tiles_cache_path)
except OSError:
# The path doesn't exists yet
pass
def qgis_server_post_save(instance, sender, **kwargs):
"""Save keywords to QGIS Server.
The way keywords are implemented requires the layer to be saved to the
database before accessing them.
This hook also creates QGIS Project. Which is essentials for QGIS Server.
There are also several Geonode Links generated, like thumbnail and legends
:param instance: geonode Layer
:type instance: Layer
:param sender: geonode Layer type
:type sender: type(Layer)
"""
if not sender == Layer:
return
# TODO
# 1. Create or update associated QGISServerLayer [Done]
# 2. Create Link for the tile and legend.
logger.debug('QGIS Server Post Save')
# copy layer to QGIS Layer Directory
try:
geonode_layer_path = instance.get_base_file()[0].file.path
except AttributeError:
logger.debug('Layer does not have base file')
return
qgis_layer, created = QGISServerLayer.objects.get_or_create(
layer=instance)
logger.debug('Geonode Layer Path %s' % geonode_layer_path)
base_filename, original_ext = os.path.splitext(geonode_layer_path)
extensions = QGISServerLayer.accepted_format
is_shapefile = False
for ext in extensions:
if os.path.exists(base_filename + '.' + ext):
is_shapefile = is_shapefile or ext == 'shp'
try:
if created:
# Assuming different layer has different filename because
# geonode rename it
shutil.copy2(
base_filename + '.' + ext,
QGIS_layer_directory
)
logger.debug('Create new basefile')
else:
# If there is already a file, replace the old one
qgis_layer_base_filename = qgis_layer.qgis_layer_path_prefix
shutil.copy2(
base_filename + '.' + ext,
qgis_layer_base_filename + '.' + ext
)
logger.debug('Overwrite existing basefile')
logger.debug(
'Copying %s' % base_filename + '.' + ext + ' Success')
logger.debug('Into %s' % QGIS_layer_directory)
except IOError as e:
logger.debug(
'Copying %s' % base_filename + '.' + ext + ' FAILED ' + e)
if created:
# Only set when creating new QGISServerLayer Object
geonode_filename = os.path.basename(geonode_layer_path)
basename = os.path.splitext(geonode_filename)[0]
qgis_layer.base_layer_path = os.path.join(
QGIS_layer_directory,
basename + original_ext # Already with dot
)
qgis_layer.save()
# Set layer crs
try:
if is_shapefile:
dataset = ogr.Open(geonode_layer_path)
layer = dataset.GetLayer()
spatial_ref = layer.GetSpatialRef()
srid = spatial_ref.GetAuthorityCode(None)
if srid:
instance.srid = srid
else:
dataset = gdal.Open(geonode_layer_path)
prj = dataset.GetProjection()
srs = osr.SpatialReference(wkt=prj)
srid = srs.GetAuthorityCode(None)
if srid:
instance.srid = srid
except Exception as e:
logger.debug("Can't retrieve projection: {layer}".format(
layer=geonode_layer_path))
logger.exception(e)
# base url for geonode
base_url = settings.SITEURL
# Set Link for Download Raw in Zip File
zip_download_url = reverse(
'qgis_server:download-zip', kwargs={'layername': instance.name})
zip_download_url = urljoin(base_url, zip_download_url)
logger.debug('zip_download_url: %s' % zip_download_url)
if is_shapefile:
link_name = 'Zipped Shapefile'
link_mime = 'SHAPE-ZIP'
else:
link_name = 'Zipped All Files'
link_mime = 'ZIP'
# Zip file
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(
extension='zip',
mime=link_mime,
url=zip_download_url,
link_type='data'
)
)
# WMS link layer workspace
ogc_wms_url = urljoin(
settings.SITEURL,
reverse(
'qgis_server:layer-request', kwargs={'layername': instance.name}))
ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace
ogc_wms_link_type = 'OGC:WMS'
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name=ogc_wms_name,
link_type=ogc_wms_link_type,
defaults=dict(
extension='html',
url=ogc_wms_url,
mime='text/html',
link_type=ogc_wms_link_type
)
)
# QGS link layer workspace
ogc_qgs_url = urljoin(
base_url,
reverse(
'qgis_server:download-qgs',
kwargs={'layername': instance.name}))
logger.debug('qgs_download_url: %s' % ogc_qgs_url)
link_name = 'QGIS project file (.qgs)'
link_mime = 'application/xml'
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(
extension='qgs',
mime=link_mime,
url=ogc_qgs_url,
link_type='data'
)
)
if instance.is_vector():
# WFS link layer workspace
ogc_wfs_url = urljoin(
settings.SITEURL,
reverse(
'qgis_server:layer-request',
kwargs={'layername': instance.name}))
ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace
ogc_wfs_link_type = 'OGC:WFS'
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name=ogc_wfs_name,
link_type=ogc_wfs_link_type,
defaults=dict(
extension='html',
url=ogc_wfs_url,
mime='text/html',
link_type=ogc_wfs_link_type
)
)
# QLR link layer workspace
ogc_qlr_url = urljoin(
base_url,
reverse(
'qgis_server:download-qlr',
kwargs={'layername': instance.name}))
logger.debug('qlr_download_url: %s' % ogc_qlr_url)
link_name = 'QGIS layer file (.qlr)'
link_mime = 'application/xml'
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(
extension='qlr',
mime=link_mime,
url=ogc_qlr_url,
link_type='data'
)
)
# if layer has overwrite attribute, then it probably comes from
# importlayers management command and needs to be overwritten
overwrite = getattr(instance, 'overwrite', False)
# Create the QGIS Project
response = create_qgis_project(
instance, qgis_layer.qgis_project_path, overwrite=overwrite,
internal=True)
logger.debug('Creating the QGIS Project : %s' % response.url)
if response.content != 'OK':
logger.debug('Result : %s' % response.content)
# Generate style model cache
style_list(instance, internal=False)
# Remove QML file if necessary
try:
qml_file = instance.upload_session.layerfile_set.get(name='qml')
if not os.path.exists(qml_file.file.path):
qml_file.delete()
except LayerFile.DoesNotExist:
pass
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name="Tiles",
defaults=dict(
url=tile_url_format(instance.name),
extension='tiles',
mime='image/png',
link_type='image'
)
)
if original_ext.split('.')[-1] in QGISServerLayer.geotiff_format:
# geotiff link
geotiff_url = reverse(
'qgis_server:geotiff', kwargs={'layername': instance.name})
geotiff_url = urljoin(base_url, geotiff_url)
logger.debug('geotif_url: %s' % geotiff_url)
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name="GeoTIFF",
defaults=dict(
extension=original_ext.split('.')[-1],
url=geotiff_url,
mime='image/tiff',
link_type='image'
)
)
# Create legend link
legend_url = reverse(
'qgis_server:legend',
kwargs={'layername': instance.name}
)
legend_url = urljoin(base_url, legend_url)
Link.objects.update_or_create(
resource=instance.resourcebase_ptr,
name='Legend',
defaults=dict(
extension='png',
url=legend_url,
mime='image/png',
link_type='image',
)
)
# Create thumbnail
create_qgis_server_thumbnail.delay(
instance, overwrite=overwrite)
# Attributes
set_attributes(instance)
# Update xml file
# Read metadata from layer that InaSAFE use.
# Some are not found: organisation, email, url
try:
new_values = {
'date': instance.date.isoformat(),
'abstract': instance.abstract,
'title': instance.title,
'license': instance.license_verbose,
}
except (TypeError, AttributeError):
new_values = {}
pass
# Get the path of the metadata file
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
xml_file_path = basename + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Also update xml in QGIS Server
xml_file_path = qgis_layer.qgis_layer_path_prefix + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Remove existing tile caches if overwrite
if overwrite:
tiles_directory = settings.QGIS_SERVER_CONFIG['tiles_directory']
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
basename = os.path.basename(basename)
tiles_cache_path = os.path.join(tiles_directory, basename)
try:
shutil.rmtree(tiles_cache_path)
except OSError:
# The path doesn't exists yet
pass
def qgis_server_post_save(instance, sender, **kwargs):
"""Save keywords to QGIS Server.
The way keywords are implemented requires the layer to be saved to the
database before accessing them.
This hook also creates QGIS Project. Which is essentials for QGIS Server.
There are also several Geonode Links generated, like thumbnail and legends
:param instance: geonode Layer
:type instance: Layer
:param sender: geonode Layer type
:type sender: type(Layer)
"""
if not sender == Layer:
return
# TODO
# 1. Create or update associated QGISServerLayer [Done]
# 2. Create Link for the tile and legend.
logger.debug('QGIS Server Post Save')
# copy layer to QGIS Layer Directory
try:
geonode_layer_path = instance.get_base_file()[0].file.path
except AttributeError:
logger.debug('Layer does not have base file')
return
qgis_layer, created = QGISServerLayer.objects.get_or_create(
layer=instance)
logger.debug('Geonode Layer Path %s' % geonode_layer_path)
base_filename, original_ext = os.path.splitext(geonode_layer_path)
extensions = QGISServerLayer.accepted_format
is_shapefile = False
for ext in extensions:
if os.path.exists(base_filename + '.' + ext):
is_shapefile = is_shapefile or ext == 'shp'
try:
if created:
# Assuming different layer has different filename because
# geonode rename it
shutil.copy2(
base_filename + '.' + ext,
QGIS_layer_directory
)
logger.debug('Create new basefile')
else:
# If there is already a file, replace the old one
qgis_layer_base_filename = qgis_layer.qgis_layer_path_prefix
shutil.copy2(
base_filename + '.' + ext,
qgis_layer_base_filename + '.' + ext
)
logger.debug('Overwrite existing basefile')
logger.debug(
'Copying %s' % base_filename + '.' + ext + ' Success')
logger.debug('Into %s' % QGIS_layer_directory)
except IOError as e:
logger.debug(
'Copying %s' % base_filename + '.' + ext + ' FAILED ' + e)
if created:
# Only set when creating new QGISServerLayer Object
geonode_filename = os.path.basename(geonode_layer_path)
basename = os.path.splitext(geonode_filename)[0]
qgis_layer.base_layer_path = os.path.join(
QGIS_layer_directory,
basename + original_ext # Already with dot
)
qgis_layer.save()
# refresh to get QGIS Layer
instance.refresh_from_db()
# Set layer crs
try:
if is_shapefile:
dataset = ogr.Open(geonode_layer_path)
layer = dataset.GetLayer()
spatial_ref = layer.GetSpatialRef()
srid = spatial_ref.GetAuthorityCode(None) if spatial_ref else None
if srid:
instance.srid = srid
else:
dataset = gdal.Open(geonode_layer_path)
prj = dataset.GetProjection()
srs = osr.SpatialReference(wkt=prj)
srid = srs.GetAuthorityCode(None) if srs else None
if srid:
instance.srid = srid
except Exception as e:
logger.debug("Can't retrieve projection: {layer}".format(
layer=geonode_layer_path))
logger.exception(e)
# Refresh and create the instance default links
set_resource_default_links(instance, qgis_layer, is_shapefile=is_shapefile, original_ext=original_ext)
# Create thumbnail
overwrite = getattr(instance, 'overwrite', False)
create_qgis_server_thumbnail.delay(
instance, overwrite=overwrite)
# Attributes
set_attributes(instance)
# Update xml file
# Read metadata from layer that InaSAFE use.
# Some are not found: organisation, email, url
try:
new_values = {
'date': instance.date.isoformat(),
'abstract': instance.abstract,
'title': instance.title,
'license': instance.license_verbose,
}
except (TypeError, AttributeError):
new_values = {}
pass
# Get the path of the metadata file
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
xml_file_path = basename + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Also update xml in QGIS Server
xml_file_path = qgis_layer.qgis_layer_path_prefix + '.xml'
if os.path.exists(xml_file_path):
try:
update_xml(xml_file_path, new_values)
except (TypeError, AttributeError):
pass
# Remove existing tile caches if overwrite
if overwrite:
tiles_directory = settings.QGIS_SERVER_CONFIG['tiles_directory']
basename, _ = os.path.splitext(qgis_layer.base_layer_path)
basename = os.path.basename(basename)
tiles_cache_path = os.path.join(tiles_directory, basename)
try:
shutil.rmtree(tiles_cache_path)
except OSError:
# The path doesn't exists yet
pass