def final_step(upload_session, user, charset="UTF-8"):
from geonode.geoserver.helpers import get_sld_for
import_session = upload_session.import_session
_log('Reloading session %s to check validity', import_session.id)
import_session = import_session.reload()
upload_session.import_session = import_session
# the importer chooses an available featuretype name late in the game need
# to verify the resource.name otherwise things will fail. This happens
# when the same data is uploaded a second time and the default name is
# chosen
cat = gs_catalog
cat._cache.clear()
# Create the style and assign it to the created resource
# FIXME: Put this in gsconfig.py
task = import_session.tasks[0]
task.set_charset(charset)
# @todo see above in save_step, regarding computed unique name
name = task.layer.name
_log('Getting from catalog [%s]', name)
publishing = cat.get_layer(name)
if import_session.state == 'INCOMPLETE':
if task.state != 'ERROR':
raise Exception('unknown item state: %s' % task.state)
elif import_session.state == 'READY':
import_session.commit()
elif import_session.state == 'PENDING':
if task.state == 'READY':
# if not task.data.format or task.data.format != 'Shapefile':
import_session.commit()
if not publishing:
raise LayerNotReady("Expected to find layer named '%s' in geoserver" %
name)
_log('Creating style for [%s]', name)
# get_files will not find the sld if it doesn't match the base name
# so we've worked around that in the view - if provided, it will be here
if upload_session.import_sld_file:
_log('using provided sld file')
base_file = upload_session.base_file
sld_file = base_file[0].sld_files[0]
f = None
if os.path.isfile(sld_file):
try:
f = open(sld_file, 'r')
except BaseException:
pass
elif upload_session.tempdir and os.path.exists(upload_session.tempdir):
tempdir = upload_session.tempdir
if os.path.isfile(os.path.join(tempdir, sld_file)):
try:
f = open(os.path.join(tempdir, sld_file), 'r')
except BaseException:
pass
if f:
sld = f.read()
f.close()
else:
sld = get_sld_for(cat, publishing)
else:
sld = get_sld_for(cat, publishing)
style = None
if sld is not None:
try:
cat.create_style(name,
sld,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
except geoserver.catalog.ConflictingDataError as e:
msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % (
name, str(e))
try:
cat.create_style(name + '_layer',
sld,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
except geoserver.catalog.ConflictingDataError as e:
msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (
name, str(e))
logger.error(msg)
e.args = (msg, )
if style is None:
try:
style = cat.get_style(name,
workspace=settings.DEFAULT_WORKSPACE
) or cat.get_style(name)
except BaseException:
logger.warn('Could not retreive the Layer default Style name')
# what are we doing with this var?
msg = 'No style could be created for the layer, falling back to POINT default one'
try:
style = cat.get_style(name + '_layer', workspace=settings.DEFAULT_WORKSPACE) or \
cat.get_style(name + '_layer')
except BaseException:
style = cat.get_style('point')
logger.warn(msg)
e.args = (msg, )
if style:
publishing.default_style = style
_log('default style set to %s', name)
cat.save(publishing)
_log('Creating Django record for [%s]', name)
target = task.target
alternate = task.get_target_layer_name()
layer_uuid = str(uuid.uuid1())
title = upload_session.layer_title
abstract = upload_session.layer_abstract
# @todo hacking - any cached layers might cause problems (maybe
# delete hook on layer should fix this?)
cat._cache.clear()
# Is it a regular file or an ImageMosaic?
# if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
if upload_session.mosaic:
import pytz
import datetime
from geonode.layers.models import TIME_REGEX_FORMAT
# llbbox = publishing.resource.latlon_bbox
start = None
end = None
if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
has_time = True
start = datetime.datetime.strptime(
upload_session.mosaic_time_value,
TIME_REGEX_FORMAT[upload_session.mosaic_time_regex])
start = pytz.utc.localize(start, is_dst=False)
end = start
else:
has_time = False
if not upload_session.append_to_mosaic_opts:
saved_layer, created = Layer.objects.get_or_create(
name=task.layer.name,
defaults=dict(
store=target.name,
storeType=target.store_type,
alternate=alternate,
workspace=target.workspace_name,
title=title,
uuid=layer_uuid,
abstract=abstract or '',
owner=user,
),
temporal_extent_start=start,
temporal_extent_end=end,
is_mosaic=True,
has_time=has_time,
has_elevation=False,
time_regex=upload_session.mosaic_time_regex)
else:
# saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name)
# created = False
saved_layer, created = Layer.objects.get_or_create(
name=upload_session.append_to_mosaic_name)
try:
if saved_layer.temporal_extent_start and end:
if pytz.utc.localize(saved_layer.temporal_extent_start,
is_dst=False) < end:
saved_layer.temporal_extent_end = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_end=end)
else:
saved_layer.temporal_extent_start = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_start=end)
except Exception as e:
_log('There was an error updating the mosaic temporal extent: '
+ str(e))
else:
_has_time = (True if upload_session.time and upload_session.time_info
and upload_session.time_transforms else False)
saved_layer, created = Layer.objects.get_or_create(
name=task.layer.name,
defaults=dict(
store=target.name,
storeType=target.store_type,
alternate=alternate,
workspace=target.workspace_name,
title=title,
uuid=layer_uuid,
abstract=abstract or '',
owner=user,
),
has_time=_has_time)
# Should we throw a clearer error here?
assert saved_layer is not None
# Create a new upload session
geonode_upload_session = UploadSession.objects.create(resource=saved_layer,
user=user)
# Add them to the upload session (new file fields are created).
assigned_name = None
def _store_file(saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=False):
with open(base_file, 'rb') as f:
file_name, type_name = os.path.splitext(
os.path.basename(base_file))
geonode_upload_session.layerfile_set.create(
name=file_name,
base=base,
file=File(f,
name='%s%s' %
(assigned_name or saved_layer.name, type_name)))
# save the system assigned name for the remaining files
if not assigned_name:
the_file = geonode_upload_session.layerfile_set.all(
)[0].file.name
assigned_name = os.path.splitext(os.path.basename(the_file))[0]
return assigned_name
if upload_session.base_file:
uploaded_files = upload_session.base_file[0]
base_file = uploaded_files.base_file
aux_files = uploaded_files.auxillary_files
sld_files = uploaded_files.sld_files
xml_files = uploaded_files.xml_files
assigned_name = _store_file(saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=True)
for _f in aux_files:
_store_file(saved_layer, geonode_upload_session, _f, assigned_name)
for _f in sld_files:
_store_file(saved_layer, geonode_upload_session, _f, assigned_name)
for _f in xml_files:
_store_file(saved_layer, geonode_upload_session, _f, assigned_name)
# @todo if layer was not created, need to ensure upload target is
# same as existing target
_log('layer was created : %s', created)
if created:
saved_layer.set_default_permissions()
saved_layer.handle_moderated_uploads()
# Create the points of contact records for the layer
_log('Creating points of contact records for [%s]', name)
saved_layer.poc = user
saved_layer.metadata_author = user
# look for xml
defaults = {}
xml_file = upload_session.base_file[0].xml_files
if xml_file:
saved_layer.metadata_uploaded = True
# get model properties from XML
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r')
zf.extract(xml_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0]
identifier, vals, regions, keywords = set_metadata(
open(xml_file[0]).read())
saved_layer.metadata_xml = xml_file[0]
regions_resolved, regions_unresolved = resolve_regions(regions)
keywords.extend(regions_unresolved)
# Assign the regions (needs to be done after saving)
regions_resolved = list(set(regions_resolved))
if regions_resolved:
if len(regions_resolved) > 0:
if not saved_layer.regions:
saved_layer.regions = regions_resolved
else:
saved_layer.regions.clear()
saved_layer.regions.add(*regions_resolved)
# Assign the keywords (needs to be done after saving)
keywords = list(set(keywords))
if keywords:
if len(keywords) > 0:
if not saved_layer.keywords:
saved_layer.keywords = keywords
else:
saved_layer.keywords.add(*keywords)
# set model properties
for key, value in vals.items():
if key == 'spatial_representation_type':
value = SpatialRepresentationType(identifier=value)
elif key == 'topic_category':
value, created = TopicCategory.objects.get_or_create(
identifier=value.lower(),
defaults={
'description': '',
'gn_description': value
})
key = 'category'
defaults[key] = value
else:
defaults[key] = value
# update with new information
db_layer = Layer.objects.filter(id=saved_layer.id)
db_layer.update(**defaults)
saved_layer.refresh_from_db()
# Pass the parameter overwrite to tell whether the
# geoserver_post_save_signal should upload the new file or not
saved_layer.overwrite = True
saved_layer.save()
# look for SLD
sld_file = upload_session.base_file[0].sld_files
if sld_file:
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r')
zf.extract(sld_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
sld_file[0] = os.path.dirname(archive) + '/' + sld_file[0]
sld = open(sld_file[0]).read()
set_layer_style(saved_layer,
saved_layer.alternate,
sld,
base_file=sld_file[0])
# Set default permissions on the newly created layer
# FIXME: Do this as part of the post_save hook
permissions = upload_session.permissions
if created and permissions is not None:
_log('Setting default permissions for [%s]', name)
saved_layer.set_permissions(permissions)
if upload_session.tempdir and os.path.exists(upload_session.tempdir):
shutil.rmtree(upload_session.tempdir)
upload = Upload.objects.get(import_id=import_session.id)
upload.layer = saved_layer
upload.complete = True
upload.save()
if upload_session.time_info:
set_time_info(saved_layer, **upload_session.time_info)
if geonode_upload_session:
geonode_upload_session.processed = True
saved_layer.upload_session = geonode_upload_session
signals.upload_complete.send(sender=final_step, layer=saved_layer)
geonode_upload_session.save()
saved_layer.save()
cat._cache.clear()
return saved_layer
def final_step(upload_session, user, charset="UTF-8", layer_id=None):
import_session = upload_session.import_session
import_id = import_session.id
_log(f'Reloading session {import_id} to check validity')
try:
import_session = import_session.reload()
except gsimporter.api.NotFound as e:
Upload.objects.invalidate_from_session(upload_session)
raise UploadException.from_exc(
_("The GeoServer Import Session is no more available"), e)
if Upload.objects.filter(import_id=import_id).count():
Upload.objects.filter(import_id=import_id).update(complete=False)
upload = Upload.objects.filter(import_id=import_id).get()
if upload.state == Upload.STATE_RUNNING:
return
# WAITING state is set when lat and lng are not selected for a csv upload
# The the final_step everything is ok, the state value can return to PENDING
# During the final_step the state will change again to complete the operation
if upload.state == Upload.STATE_WAITING:
upload.set_processing_state(Upload.STATE_PENDING)
upload_session.import_session = import_session
Upload.objects.update_from_session(upload_session)
# Create the style and assign it to the created resource
# FIXME: Put this in gsconfig.py
task = import_session.tasks[0]
task.set_charset(charset)
# @todo see above in save_step, regarding computed unique name
name = task.layer.name
if layer_id:
name = Layer.objects.get(resourcebase_ptr_id=layer_id).name
_log(f'Getting from catalog [{name}]')
try:
# the importer chooses an available featuretype name late in the game need
# to verify the resource.name otherwise things will fail. This happens
# when the same data is uploaded a second time and the default name is
# chosen
gs_catalog.get_layer(name)
except Exception:
Upload.objects.invalidate_from_session(upload_session)
raise LayerNotReady(
_(f"Expected to find layer named '{name}' in geoserver"))
if import_session.state == 'READY' or (import_session.state == 'PENDING' and task.state == 'READY'):
import_session.commit()
elif import_session.state == 'INCOMPLETE' and task.state != 'ERROR':
Upload.objects.invalidate_from_session(upload_session)
raise Exception(f'unknown item state: {task.state}')
try:
import_session = import_session.reload()
except gsimporter.api.NotFound as e:
Upload.objects.invalidate_from_session(upload_session)
raise UploadException.from_exc(
_("The GeoServer Import Session is no more available"), e)
upload_session.import_session = import_session
Upload.objects.update_from_session(upload_session)
_log(f'Creating Django record for [{name}]')
target = task.target
alternate = task.get_target_layer_name()
layer_uuid = None
title = upload_session.layer_title
abstract = upload_session.layer_abstract
regions = []
keywords = []
vals = {}
custom = {}
# look for xml and finalize Layer metadata
metadata_uploaded = False
xml_file = upload_session.base_file[0].xml_files
if xml_file:
try:
# get model properties from XML
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(xml_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
xml_file = f"{os.path.dirname(archive)}/{xml_file[0]}"
# Sanity checks
if isinstance(xml_file, list):
if len(xml_file) > 0:
xml_file = xml_file[0]
else:
xml_file = None
elif not isinstance(xml_file, str):
xml_file = None
if xml_file and os.path.exists(xml_file) and os.access(xml_file, os.R_OK):
layer_uuid, vals, regions, keywords, custom = parse_metadata(
open(xml_file).read())
metadata_uploaded = True
except Exception as e:
Upload.objects.invalidate_from_session(upload_session)
logger.error(e)
raise GeoNodeException(
_("Exception occurred while parsing the provided Metadata file."), e)
# Make sure the layer does not exists already
if layer_uuid and Layer.objects.filter(uuid=layer_uuid).count():
Upload.objects.invalidate_from_session(upload_session)
logger.error("The UUID identifier from the XML Metadata is already in use in this system.")
raise GeoNodeException(
_("The UUID identifier from the XML Metadata is already in use in this system."))
# Is it a regular file or an ImageMosaic?
has_time = has_elevation = False
start = end = None
if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
has_time = True
start = datetime.datetime.strptime(upload_session.mosaic_time_value,
TIME_REGEX_FORMAT[upload_session.mosaic_time_regex])
start = pytz.utc.localize(start, is_dst=False)
end = start
if upload_session.time and upload_session.time_info and upload_session.time_transforms:
has_time = True
saved_layer = None
if upload_session.mosaic:
if not upload_session.append_to_mosaic_opts:
saved_dataset_filter = Layer.objects.filter(
store=target.name,
alternate=alternate,
workspace=target.workspace_name,
name=task.layer.name)
if not saved_dataset_filter.exists():
saved_layer = Layer.objects.create(
uuid=layer_uuid or str(uuid.uuid1()),
store=target.name,
storeType=target.store_type,
alternate=alternate,
workspace=target.workspace_name,
title=title,
name=task.layer.name,
abstract=abstract or '',
owner=user,
temporal_extent_start=start,
temporal_extent_end=end,
is_mosaic=True,
has_time=has_time,
has_elevation=has_elevation,
time_regex=upload_session.mosaic_time_regex)
created = True
else:
saved_layer = saved_dataset_filter.get()
created = False
else:
saved_layer, created = Layer.objects.get_or_create(
name=upload_session.append_to_mosaic_name)
try:
if saved_layer.temporal_extent_start and end:
if pytz.utc.localize(
saved_layer.temporal_extent_start,
is_dst=False) < end:
saved_layer.temporal_extent_end = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_end=end)
else:
saved_layer.temporal_extent_start = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_start=end)
except Exception as e:
_log(
f"There was an error updating the mosaic temporal extent: {str(e)}")
else:
saved_dataset_filter = Layer.objects.filter(
store=target.name,
alternate=alternate,
workspace=target.workspace_name,
name=task.layer.name)
if not saved_dataset_filter.exists():
saved_layer = Layer.objects.create(
uuid=layer_uuid or str(uuid.uuid1()),
store=target.name,
storeType=target.store_type,
alternate=alternate,
workspace=target.workspace_name,
title=title,
name=task.layer.name,
abstract=abstract or '',
owner=user,
temporal_extent_start=start,
temporal_extent_end=end,
is_mosaic=False,
has_time=has_time,
has_elevation=has_elevation,
time_regex=upload_session.mosaic_time_regex)
created = True
else:
saved_layer = saved_dataset_filter.get()
created = False
assert saved_layer
if not created:
return saved_layer
# Hide the resource until finished
saved_layer.set_dirty_state()
# Create a new upload session
geonode_upload_session, created = UploadSession.objects.get_or_create(
resource=saved_layer, user=user
)
geonode_upload_session.processed = False
geonode_upload_session.save()
Upload.objects.update_from_session(upload_session, layer=saved_layer)
# Add them to the upload session (new file fields are created).
assigned_name = None
# Update Layer with information coming from XML File if available
saved_layer = _update_layer_with_xml_info(saved_layer, xml_file, regions, keywords, vals)
def _store_file(saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=False):
if os.path.exists(base_file):
with open(base_file, 'rb') as f:
file_name, type_name = os.path.splitext(os.path.basename(base_file))
geonode_upload_session.layerfile_set.create(
name=file_name,
base=base,
file=File(
f, name=f'{assigned_name or saved_layer.name}{type_name}'))
# save the system assigned name for the remaining files
if not assigned_name:
the_file = geonode_upload_session.layerfile_set.all()[0].file.name
assigned_name = os.path.splitext(os.path.basename(the_file))[0]
return assigned_name
if upload_session.base_file:
uploaded_files = upload_session.base_file[0]
base_file = uploaded_files.base_file
aux_files = uploaded_files.auxillary_files
sld_files = uploaded_files.sld_files
xml_files = uploaded_files.xml_files
assigned_name = _store_file(
saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=True)
for _f in aux_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
for _f in sld_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
for _f in xml_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
# @todo if layer was not created, need to ensure upload target is
# same as existing target
# Create the points of contact records for the layer
_log(f'Creating points of contact records for {name}')
saved_layer.poc = user
saved_layer.metadata_author = user
saved_layer.metadata_uploaded = metadata_uploaded
_log('Creating style for [%s]', name)
# look for SLD
sld_file = upload_session.base_file[0].sld_files
sld_uploaded = False
if sld_file:
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
_log(f'using uploaded sld file from {archive}')
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(sld_file[0], os.path.dirname(archive), path=upload_session.tempdir)
# Assign the absolute path to this file
sld_file[0] = f"{os.path.dirname(archive)}/{sld_file[0]}"
else:
_sld_file = f"{os.path.dirname(upload_session.tempdir)}/{os.path.basename(sld_file[0])}"
_log(f"copying [{sld_file[0]}] to [{_sld_file}]")
try:
shutil.copyfile(sld_file[0], _sld_file)
sld_file = _sld_file
except (IsADirectoryError, shutil.SameFileError) as e:
logger.exception(e)
sld_file = sld_file[0]
except Exception as e:
raise UploadException.from_exc(_('Error uploading Dataset'), e)
sld_uploaded = True
else:
# get_files will not find the sld if it doesn't match the base name
# so we've worked around that in the view - if provided, it will be here
if upload_session.import_sld_file:
_log('using provided sld file')
base_file = upload_session.base_file
sld_file = base_file[0].sld_files[0]
sld_uploaded = False
_log(f'[sld_uploaded: {sld_uploaded}] sld_file: {sld_file}')
if upload_session.time_info:
set_time_info(saved_layer, **upload_session.time_info)
# Set default permissions on the newly created layer and send notifications
permissions = upload_session.permissions
geoserver_finalize_upload.apply_async(
(import_session.id, saved_layer.id, permissions, created,
xml_file, sld_file, sld_uploaded, upload_session.tempdir))
saved_layer = utils.metadata_storers(saved_layer, custom)
return saved_layer
def final_step(upload_session, user, charset="UTF-8", dataset_id=None):
import_session = upload_session.import_session
import_id = import_session.id
_log(f'Reloading session {import_id} to check validity')
try:
import_session = import_session.reload()
except gsimporter.api.NotFound as e:
Upload.objects.invalidate_from_session(upload_session)
raise UploadException.from_exc(
_("The GeoServer Import Session is no more available"), e)
if Upload.objects.filter(import_id=import_id).count():
Upload.objects.filter(import_id=import_id).update(complete=False)
upload = Upload.objects.filter(import_id=import_id).get()
if upload.state == enumerations.STATE_RUNNING:
return
upload_session.import_session = import_session
Upload.objects.update_from_session(upload_session)
# Create the style and assign it to the created resource
# FIXME: Put this in gsconfig.py
task = import_session.tasks[0]
task.set_charset(charset)
# @todo see above in save_step, regarding computed unique name
name = task.layer.name
if dataset_id:
name = Dataset.objects.get(resourcebase_ptr_id=dataset_id).name
_log(f'Getting from catalog [{name}]')
try:
# the importer chooses an available featuretype name late in the game need
# to verify the resource.name otherwise things will fail. This happens
# when the same data is uploaded a second time and the default name is
# chosen
gs_catalog.get_layer(name)
except Exception:
Upload.objects.invalidate_from_session(upload_session)
raise LayerNotReady(
_(f"Expected to find layer named '{name}' in geoserver"))
if import_session.state == 'READY' or (import_session.state == 'PENDING'
and task.state == 'READY'):
import_session.commit()
elif import_session.state == 'INCOMPLETE' and task.state != 'ERROR':
Upload.objects.invalidate_from_session(upload_session)
raise Exception(f'unknown item state: {task.state}')
try:
import_session = import_session.reload()
except gsimporter.api.NotFound as e:
Upload.objects.invalidate_from_session(upload_session)
raise UploadException.from_exc(
_("The GeoServer Import Session is no more available"), e)
upload_session.import_session = import_session
Upload.objects.update_from_session(upload_session)
_log(f'Creating Django record for [{name}]')
target = task.target
alternate = task.get_target_layer_name()
dataset_uuid = None
title = upload_session.dataset_title
abstract = upload_session.dataset_abstract
metadata_uploaded = False
xml_file = upload_session.base_file[0].xml_files
if xml_file:
try:
# get model properties from XML
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(xml_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
xml_file = f"{os.path.dirname(archive)}/{xml_file[0]}"
# Sanity checks
if isinstance(xml_file, list):
if len(xml_file) > 0:
xml_file = xml_file[0]
else:
xml_file = None
elif not isinstance(xml_file, str):
xml_file = None
if xml_file and os.path.exists(xml_file) and os.access(
xml_file, os.R_OK):
dataset_uuid, vals, regions, keywords, custom = parse_metadata(
open(xml_file).read())
metadata_uploaded = True
except Exception as e:
Upload.objects.invalidate_from_session(upload_session)
logger.error(e)
raise GeoNodeException(
_("Exception occurred while parsing the provided Metadata file."
), e)
# look for SLD
sld_file = upload_session.base_file[0].sld_files
sld_uploaded = False
if sld_file:
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
logger.debug(f'using uploaded sld file from {archive}')
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(sld_file[0],
os.path.dirname(archive),
path=upload_session.tempdir)
# Assign the absolute path to this file
sld_file[0] = f"{os.path.dirname(archive)}/{sld_file[0]}"
else:
_sld_file = f"{os.path.dirname(upload_session.tempdir)}/{os.path.basename(sld_file[0])}"
logger.debug(f"copying [{sld_file[0]}] to [{_sld_file}]")
try:
shutil.copyfile(sld_file[0], _sld_file)
sld_file = _sld_file
except (IsADirectoryError, shutil.SameFileError) as e:
logger.exception(e)
sld_file = sld_file[0]
except Exception as e:
raise UploadException.from_exc(_('Error uploading Dataset'), e)
sld_uploaded = True
else:
# get_files will not find the sld if it doesn't match the base name
# so we've worked around that in the view - if provided, it will be here
if upload_session.import_sld_file:
logger.debug('using provided sld file from importer')
base_file = upload_session.base_file
sld_file = base_file[0].sld_files[0]
sld_uploaded = False
logger.debug(f'[sld_uploaded: {sld_uploaded}] sld_file: {sld_file}')
# Make sure the layer does not exists already
if dataset_uuid and Dataset.objects.filter(uuid=dataset_uuid).count():
Upload.objects.invalidate_from_session(upload_session)
logger.error(
"The UUID identifier from the XML Metadata is already in use in this system."
)
raise GeoNodeException(
_("The UUID identifier from the XML Metadata is already in use in this system."
))
# Is it a regular file or an ImageMosaic?
saved_dataset = None
is_mosaic = False
has_time = has_elevation = False
start = end = None
if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
has_time = True
is_mosaic = True
start = datetime.datetime.strptime(
upload_session.mosaic_time_value,
TIME_REGEX_FORMAT[upload_session.mosaic_time_regex])
start = pytz.utc.localize(start, is_dst=False)
end = start
if upload_session.time and upload_session.time_info and upload_session.time_transforms:
has_time = True
if upload_session.append_to_mosaic_opts:
# Is it a mosaic or a granule that must be added to an Image Mosaic?
saved_dataset_filter = Dataset.objects.filter(
name=upload_session.append_to_mosaic_name)
if not saved_dataset_filter.exists():
saved_dataset = resource_manager.create(
name=upload_session.append_to_mosaic_name,
defaults=dict(dirty_state=True,
state=enumerations.STATE_READY))
created = True
else:
saved_dataset = saved_dataset_filter.get()
created = False
saved_dataset.set_dirty_state()
if saved_dataset.temporal_extent_start and end:
if pytz.utc.localize(saved_dataset.temporal_extent_start,
is_dst=False) < end:
saved_dataset.temporal_extent_end = end
Dataset.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_end=end)
else:
saved_dataset.temporal_extent_start = end
Dataset.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_start=end)
else:
# The dataset is a standard one, no mosaic options enabled...
saved_dataset_filter = Dataset.objects.filter(
store=target.name,
alternate=alternate,
workspace=target.workspace_name,
name=task.layer.name)
if not saved_dataset_filter.exists():
saved_dataset = resource_manager.create(
dataset_uuid,
resource_type=Dataset,
defaults=dict(store=target.name,
subtype=get_dataset_storetype(target.store_type),
alternate=alternate,
workspace=target.workspace_name,
title=title,
name=task.layer.name,
abstract=abstract or _('No abstract provided'),
owner=user,
dirty_state=True,
state=enumerations.STATE_READY,
temporal_extent_start=start,
temporal_extent_end=end,
is_mosaic=is_mosaic,
has_time=has_time,
has_elevation=has_elevation,
time_regex=upload_session.mosaic_time_regex))
created = True
else:
saved_dataset = saved_dataset_filter.get()
created = False
assert saved_dataset
if not created:
return saved_dataset
try:
# Update the state from session...
Upload.objects.update_from_session(upload_session,
resource=saved_dataset)
# Hide the dataset until the upload process finishes...
saved_dataset.set_dirty_state()
# Finalize the upload...
with transaction.atomic():
# Set default permissions on the newly created layer and send notifications
permissions = upload_session.permissions
# Finalize Upload
resource_manager.set_permissions(None,
instance=saved_dataset,
permissions=permissions,
created=created)
resource_manager.update(None,
instance=saved_dataset,
xml_file=xml_file,
metadata_uploaded=metadata_uploaded)
resource_manager.exec('set_style',
None,
instance=saved_dataset,
sld_uploaded=sld_uploaded,
sld_file=sld_file,
tempdir=upload_session.tempdir)
resource_manager.exec('set_time_info',
None,
instance=saved_dataset,
time_info=upload_session.time_info)
resource_manager.set_thumbnail(None, instance=saved_dataset)
Upload.objects.filter(
resource=saved_dataset.get_self_resource()).update(
complete=True)
Upload.objects.get(resource=saved_dataset.get_self_resource(
)).set_processing_state(enumerations.STATE_PROCESSED)
except Exception as e:
saved_dataset.set_processing_state(enumerations.STATE_INVALID)
raise GeoNodeException(e)
finally:
# Get rid if temporary files that have been uploaded via Upload form
try:
logger.debug(
f"... Cleaning up the temporary folders {upload_session.tempdir}"
)
if saved_dataset.processed and upload_session.tempdir and os.path.exists(
upload_session.tempdir):
shutil.rmtree(upload_session.tempdir)
except Exception as e:
logger.warning(e)
return saved_dataset
def final_step(upload_session, user, charset="UTF-8"):
import_session = upload_session.import_session
_log('Reloading session %s to check validity', import_session.id)
import_session = import_session.reload()
upload_session.import_session = import_session
# the importer chooses an available featuretype name late in the game need
# to verify the resource.name otherwise things will fail. This happens
# when the same data is uploaded a second time and the default name is
# chosen
cat = gs_catalog
# Create the style and assign it to the created resource
# FIXME: Put this in gsconfig.py
task = import_session.tasks[0]
task.set_charset(charset)
# @todo see above in save_step, regarding computed unique name
name = task.layer.name
_log('Getting from catalog [%s]', name)
publishing = cat.get_layer(name)
if import_session.state == 'INCOMPLETE':
if task.state != 'ERROR':
raise Exception(f'unknown item state: {task.state}')
elif import_session.state == 'READY':
import_session.commit()
elif import_session.state == 'PENDING':
if task.state == 'READY':
# if not task.data.format or task.data.format != 'Shapefile':
import_session.commit()
if not publishing:
raise LayerNotReady(
f"Expected to find layer named '{name}' in geoserver")
_log('Creating Django record for [%s]', name)
target = task.target
alternate = task.get_target_layer_name()
layer_uuid = str(uuid.uuid1())
title = upload_session.layer_title
abstract = upload_session.layer_abstract
# look for xml and finalize Layer metadata
metadata_uploaded = False
xml_file = upload_session.base_file[0].xml_files
if xml_file:
# get model properties from XML
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(xml_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
xml_file = os.path.dirname(archive) + '/' + xml_file[0]
# Sanity checks
if isinstance(xml_file, list):
if len(xml_file) > 0:
xml_file = xml_file[0]
else:
xml_file = None
elif not isinstance(xml_file, str):
xml_file = None
if xml_file and os.path.exists(xml_file) and os.access(xml_file, os.R_OK):
metadata_uploaded = True
layer_uuid, vals, regions, keywords = set_metadata(
open(xml_file).read())
# Make sure the layer does not exists already
if Layer.objects.filter(uuid=layer_uuid).count():
logger.error("The UUID identifier from the XML Metadata is already in use in this system.")
raise GeoNodeException(
_("The UUID identifier from the XML Metadata is already in use in this system."))
# Is it a regular file or an ImageMosaic?
# if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
if upload_session.mosaic:
import pytz
import datetime
from geonode.layers.models import TIME_REGEX_FORMAT
# llbbox = publishing.resource.latlon_bbox
start = None
end = None
if upload_session.mosaic_time_regex and upload_session.mosaic_time_value:
has_time = True
start = datetime.datetime.strptime(upload_session.mosaic_time_value,
TIME_REGEX_FORMAT[upload_session.mosaic_time_regex])
start = pytz.utc.localize(start, is_dst=False)
end = start
else:
has_time = False
if not upload_session.append_to_mosaic_opts:
saved_layer, created = Layer.objects.get_or_create(
uuid=layer_uuid,
defaults=dict(
store=target.name,
storeType=target.store_type,
alternate=alternate,
workspace=target.workspace_name,
title=title,
name=task.layer.name,
abstract=abstract or '',
owner=user),
temporal_extent_start=start,
temporal_extent_end=end,
is_mosaic=True,
has_time=has_time,
has_elevation=False,
time_regex=upload_session.mosaic_time_regex
)
assert saved_layer is not None
else:
# saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name)
# created = False
saved_layer, created = Layer.objects.get_or_create(
name=upload_session.append_to_mosaic_name)
assert saved_layer is not None
try:
if saved_layer.temporal_extent_start and end:
if pytz.utc.localize(
saved_layer.temporal_extent_start,
is_dst=False) < end:
saved_layer.temporal_extent_end = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_end=end)
else:
saved_layer.temporal_extent_start = end
Layer.objects.filter(
name=upload_session.append_to_mosaic_name).update(
temporal_extent_start=end)
except Exception as e:
_log(
'There was an error updating the mosaic temporal extent: ' +
str(e))
else:
_has_time = (True if upload_session.time and upload_session.time_info and
upload_session.time_transforms else False)
saved_layer = None
try:
with transaction.atomic():
saved_layer = Layer.objects.create(uuid=layer_uuid, owner=user)
assert saved_layer is not None
created = Layer.objects.filter(id=saved_layer.id).exists()
if created:
to_update = {
"name": task.layer.name,
"store": target.name,
"storeType": target.store_type,
"alternate": alternate,
"workspace": target.workspace_name,
"title": title,
"abstract": abstract or '',
"has_time": _has_time
}
Layer.objects.filter(id=saved_layer.id).update(**to_update)
# Refresh from DB
saved_layer.refresh_from_db()
except IntegrityError:
raise
# Create a new upload session
try:
with transaction.atomic():
geonode_upload_session, created = UploadSession.objects.get_or_create(
resource=saved_layer, user=user
)
geonode_upload_session.processed = False
geonode_upload_session.save()
except IntegrityError:
raise
# Add them to the upload session (new file fields are created).
assigned_name = None
def _store_file(saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=False):
with open(base_file, 'rb') as f:
file_name, type_name = os.path.splitext(os.path.basename(base_file))
geonode_upload_session.layerfile_set.create(
name=file_name,
base=base,
file=File(
f, name=f'{assigned_name or saved_layer.name}{type_name}'))
# save the system assigned name for the remaining files
if not assigned_name:
the_file = geonode_upload_session.layerfile_set.all()[0].file.name
assigned_name = os.path.splitext(os.path.basename(the_file))[0]
return assigned_name
if upload_session.base_file:
uploaded_files = upload_session.base_file[0]
base_file = uploaded_files.base_file
aux_files = uploaded_files.auxillary_files
sld_files = uploaded_files.sld_files
xml_files = uploaded_files.xml_files
assigned_name = _store_file(
saved_layer,
geonode_upload_session,
base_file,
assigned_name,
base=True)
for _f in aux_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
for _f in sld_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
for _f in xml_files:
_store_file(saved_layer,
geonode_upload_session,
_f,
assigned_name)
# @todo if layer was not created, need to ensure upload target is
# same as existing target
# Create the points of contact records for the layer
_log(f'Creating points of contact records for {name}')
saved_layer.poc = user
saved_layer.metadata_author = user
saved_layer.metadata_uploaded = metadata_uploaded
_log('Creating style for [%s]', name)
# look for SLD
sld_file = upload_session.base_file[0].sld_files
sld_uploaded = False
if sld_file:
# If it's contained within a zip, need to extract it
if upload_session.base_file.archive:
archive = upload_session.base_file.archive
zf = zipfile.ZipFile(archive, 'r', allowZip64=True)
zf.extract(sld_file[0], os.path.dirname(archive))
# Assign the absolute path to this file
sld_file[0] = os.path.dirname(archive) + '/' + sld_file[0]
sld_file = sld_file[0]
sld_uploaded = True
# geoserver_set_style.apply_async((saved_layer.id, sld_file))
else:
# get_files will not find the sld if it doesn't match the base name
# so we've worked around that in the view - if provided, it will be here
if upload_session.import_sld_file:
_log('using provided sld file')
base_file = upload_session.base_file
sld_file = base_file[0].sld_files[0]
sld_uploaded = False
# geoserver_create_style.apply_async((saved_layer.id, name, sld_file, upload_session.tempdir))
if upload_session.time_info:
set_time_info(saved_layer, **upload_session.time_info)
# Set default permissions on the newly created layer and send notifications
permissions = upload_session.permissions
geoserver_finalize_upload.apply_async(
(import_session.id, saved_layer.id, permissions, created,
xml_file, sld_file, sld_uploaded, upload_session.tempdir))
return saved_layer