def test_configure_time(self): # make sure it's not there (and configured) cascading_delete(gs_catalog, 'boxes_with_end_date') def get_wms_timepositions(): metadata = get_wms().contents['geonode:boxes_with_end_date'] self.assertTrue(metadata is not None) return metadata.timepositions thefile = os.path.join(GOOD_DATA, 'time', 'boxes_with_end_date.shp') uploaded = file_upload(thefile, overwrite=True) check_layer(uploaded) # initial state is no positions or info self.assertTrue(get_wms_timepositions() is None) self.assertTrue(get_time_info(uploaded) is None) # enable using interval and single attribute set_time_info(uploaded, 'date', None, 'DISCRETE_INTERVAL', 3, 'days') self.assertEquals( ['2000-03-01T00:00:00.000Z/2000-06-08T00:00:00.000Z/P3D'], get_wms_timepositions()) self.assertEquals( { 'end_attribute': None, 'presentation': 'DISCRETE_INTERVAL', 'attribute': 'date', 'enabled': True, 'precision_value': '3', 'precision_step': 'days' }, get_time_info(uploaded)) # disable but configure to use enddate attribute in list set_time_info(uploaded, 'date', 'enddate', 'LIST', None, None, enabled=False) # verify disabled self.assertTrue(get_wms_timepositions() is None) # test enabling now info = get_time_info(uploaded) info['enabled'] = True set_time_info(uploaded, **info) self.assertEquals(100, len(get_wms_timepositions()))
def test_configure_time(self): # make sure it's not there (and configured) cascading_delete(gs_catalog, 'boxes_with_end_date') def get_wms_timepositions(): metadata = get_wms().contents['geonode:boxes_with_end_date'] self.assertTrue(metadata is not None) return metadata.timepositions thefile = os.path.join( GOOD_DATA, 'time', 'boxes_with_end_date.shp' ) uploaded = file_upload(thefile, overwrite=True) check_layer(uploaded) # initial state is no positions or info self.assertTrue(get_wms_timepositions() is None) self.assertTrue(get_time_info(uploaded) is None) # enable using interval and single attribute set_time_info(uploaded, 'date', None, 'DISCRETE_INTERVAL', 3, 'days') self.assertEquals( ['2000-03-01T00:00:00.000Z/2000-06-08T00:00:00.000Z/P3D'], get_wms_timepositions() ) self.assertEquals( {'end_attribute': None, 'presentation': 'DISCRETE_INTERVAL', 'attribute': 'date', 'enabled': True, 'precision_value': '3', 'precision_step': 'days'}, get_time_info(uploaded) ) # disable but configure to use enddate attribute in list set_time_info(uploaded, 'date', 'enddate', 'LIST', None, None, enabled=False) # verify disabled self.assertTrue(get_wms_timepositions() is None) # test enabling now info = get_time_info(uploaded) info['enabled'] = True set_time_info(uploaded, **info) self.assertEquals(100, len(get_wms_timepositions()))
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() defaults = dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) _log('record defaults: %s', defaults) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=defaults ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions _log('Setting default permissions for [%s]', name) if permissions is not None: saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(cat, publishing) style = None if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style(name + '_layer', sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg,) if style is None: try: style = cat.get_style(name) except: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = cat.get_style(name + '_layer') except: style = cat.get_style('point') logger.warn(msg) e.args = (msg,) if style: publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime(upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex ) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create(name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log('There was an error updating the mosaic temporal extent: ' + str(e)) else: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) style = None print " **************************************** " if sld is not None: try: cat.create_style(name, sld) style = cat.get_style(name) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % ( name, str(e)) try: cat.create_style(name + '_layer', sld) style = cat.get_style(name + '_layer') except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) logger.error(msg) e.args = (msg,) # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' style = cat.get_style('point') logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = style _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() # Is it a regular file or an ImageMosaic? # if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: if upload_session.mosaic: import pytz import datetime from geonode.layers.models import TIME_REGEX_FORMAT # llbbox = publishing.resource.latlon_bbox start = None end = None if upload_session.mosaic_time_regex and upload_session.mosaic_time_value: has_time = True start = datetime.datetime.strptime(upload_session.mosaic_time_value, TIME_REGEX_FORMAT[upload_session.mosaic_time_regex]) start = pytz.utc.localize(start, is_dst=False) end = start else: has_time = False if not upload_session.append_to_mosaic_opts: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,), temporal_extent_start=start, temporal_extent_end=end, is_mosaic=True, has_time=has_time, has_elevation=False, time_regex=upload_session.mosaic_time_regex ) else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False saved_layer, created = Layer.objects.get_or_create(name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: saved_layer.temporal_extent_end = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: _log('There was an error updating the mosaic temporal extent: ' + str(e)) else: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML # If it's contained within a zip, need to extract it if upload_session.base_file.archive: archive = upload_session.base_file.archive zf = zipfile.ZipFile(archive, 'r') zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] identifier, vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer
def final_step(upload_session, user): from geonode.geoserver.helpers import get_sld_for import_session = upload_session.import_session _log('Reloading session %s to check validity', import_session.id) import_session = import_session.reload() upload_session.import_session = import_session # the importer chooses an available featuretype name late in the game need # to verify the resource.name otherwise things will fail. This happens # when the same data is uploaded a second time and the default name is # chosen cat = gs_catalog cat._cache.clear() # Create the style and assign it to the created resource # FIXME: Put this in gsconfig.py task = import_session.tasks[0] # @todo see above in save_step, regarding computed unique name name = task.layer.name _log('Getting from catalog [%s]', name) publishing = cat.get_layer(name) if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING': if task.state == 'READY' and task.data.format != 'Shapefile': import_session.commit() if not publishing: raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name # so we've worked around that in the view - if provided, it will be here if upload_session.import_sld_file: _log('using provided sld file') base_file = upload_session.base_file sld_file = base_file[0].sld_files[0] f = open(sld_file, 'r') sld = f.read() f.close() else: sld = get_sld_for(publishing) if sld is not None: try: cat.create_style(name, sld) except geoserver.catalog.ConflictingDataError as e: msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % ( name, str(e)) # what are we doing with this var? # style = cat.get_style(name) logger.warn(msg) e.args = (msg,) # FIXME: Should we use the fully qualified typename? publishing.default_style = cat.get_style(name) _log('default style set to %s', name) cat.save(publishing) _log('Creating Django record for [%s]', name) target = task.target typename = task.get_target_layer_name() layer_uuid = str(uuid.uuid1()) title = upload_session.layer_title abstract = upload_session.layer_abstract # @todo hacking - any cached layers might cause problems (maybe # delete hook on layer should fix this?) cat._cache.clear() defaults = dict(store=target.name, storeType=target.store_type, typename=typename, workspace=target.workspace_name, title=title, uuid=layer_uuid, abstract=abstract or '', owner=user,) _log('record defaults: %s', defaults) saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, defaults=defaults ) # Should we throw a clearer error here? assert saved_layer is not None # @todo if layer was not created, need to ensure upload target is # same as existing target _log('layer was created : %s', created) if created: saved_layer.set_default_permissions() # Create the points of contact records for the layer _log('Creating points of contact records for [%s]', name) saved_layer.poc = user saved_layer.metadata_author = user # look for xml xml_file = upload_session.base_file[0].xml_files if xml_file: saved_layer.metadata_uploaded = True # get model properties from XML vals, regions, keywords = set_metadata(open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) # set regions regions_resolved = list(set(regions_resolved)) if regions: if len(regions) > 0: saved_layer.regions.add(*regions_resolved) # set taggit keywords keywords = list(set(keywords)) saved_layer.keywords.add(*keywords) # set model properties for (key, value) in vals.items(): if key == "spatial_representation_type": # value = SpatialRepresentationType.objects.get(identifier=value) pass else: setattr(saved_layer, key, value) saved_layer.save() # Set default permissions on the newly created layer # FIXME: Do this as part of the post_save hook permissions = upload_session.permissions if created and permissions is not None: _log('Setting default permissions for [%s]', name) saved_layer.set_permissions(permissions) if upload_session.tempdir and os.path.exists(upload_session.tempdir): shutil.rmtree(upload_session.tempdir) upload = Upload.objects.get(import_id=import_session.id) upload.layer = saved_layer upload.complete = True upload.save() if upload_session.time_info: set_time_info(saved_layer, **upload_session.time_info) signals.upload_complete.send(sender=final_step, layer=saved_layer) return saved_layer