示例#1
0
    def test_get_or_create_datastore(self):
        # --- Check that it creates a missing datastore
        # No importer needed
        importer = None
        gph = GeoserverPublishHandler(importer)
        connection_string = gph.get_default_store()
        ds_name = connection_string['name']
        # FailedRequestError indicates the data store couldn't be found
        self.assertRaises(FailedRequestError, gs_catalog.get_store, ds_name)

        layer_config = {'geoserver_store': connection_string}
        gph.get_or_create_datastore(layer_config)
        ds2 = gs_catalog.get_store(ds_name)
        self.assertNotEqual(ds2, None)
    def test_get_or_create_datastore(self):
        # --- Check that it creates a missing datastore
        # No importer needed
        importer = None
        gph = GeoserverPublishHandler(importer)
        connection_string = gph.get_default_store()
        ds_name = connection_string['name']
        # FailedRequestError indicates the data store couldn't be found
        logging.disable(logging.ERROR)
        self.assertRaises(FailedRequestError, gs_catalog.get_store, ds_name)
        logging.disable(logging.NOTSET)

        layer_config = {'geoserver_store': connection_string}
        gph.get_or_create_datastore(layer_config, None)
        ds2 = gs_catalog.get_store(ds_name)
        self.assertNotEqual(ds2, None)
示例#3
0
def save_step(user, layer, spatial_files, overwrite=True):
    _log('Uploading layer: [%s], files [%s]', layer, spatial_files)

    if len(spatial_files) > 1:
        # we only support more than one file if they're rasters for mosaicing
        if not all([f.file_type.layer_type == 'coverage' for f in spatial_files]):
            raise UploadException("Please upload only one type of file at a time")

    name = get_valid_layer_name(layer, overwrite)
    _log('Name for layer: [%s]', name)

    if not spatial_files:
        raise UploadException("Unable to recognize the uploaded file(s)")

    the_layer_type = spatial_files[0].file_type.layer_type

    # Check if the store exists in geoserver
    try:
        store = gs_catalog.get_store(name)
    except geoserver.catalog.FailedRequestError, e:
        # There is no store, ergo the road is clear
        pass
示例#4
0
文件: upload.py 项目: rbs-pli/geonode
def save_step(user, layer, spatial_files, overwrite=True):
    _log('Uploading layer: [%s], files [%s]', layer, spatial_files)

    if len(spatial_files) > 1:
        # we only support more than one file if they're rasters for mosaicing
        if not all([f.file_type.layer_type == 'coverage' for f in spatial_files]):
            raise UploadException("Please upload only one type of file at a time")

    name = get_valid_layer_name(layer, overwrite)
    _log('Name for layer: [%s]', name)

    if not spatial_files:
        raise UploadException("Unable to recognize the uploaded file(s)")

    the_layer_type = spatial_files[0].file_type.layer_type

    # Check if the store exists in geoserver
    try:
        store = gs_catalog.get_store(name)
    except geoserver.catalog.FailedRequestError, e:
        # There is no store, ergo the road is clear
        pass
示例#5
0
def save_step(user, layer, spatial_files, overwrite=True):
    _log('Uploading layer: [%s], files [%s]', layer, spatial_files)

    if len(spatial_files) > 1:
        # we only support more than one file if they're rasters for mosaicing
        if not all(
                [f.file_type.layer_type == 'coverage' for f in spatial_files]):
            raise UploadException(
                "Please upload only one type of file at a time")

    name = get_valid_layer_name(layer, overwrite)
    _log('Name for layer: [%s]', name)

    if not spatial_files:
        raise UploadException("Unable to recognize the uploaded file(s)")

    the_layer_type = spatial_files[0].file_type.layer_type

    # Check if the store exists in geoserver
    try:
        store = gs_catalog.get_store(name)
    except geoserver.catalog.FailedRequestError as e:
        # There is no store, ergo the road is clear
        pass
    else:
        # If we get a store, we do the following:
        resources = store.get_resources()
        # Is it empty?
        if len(resources) == 0:
            # What should we do about that empty store?
            if overwrite:
                # We can just delete it and recreate it later.
                store.delete()
            else:
                msg = (
                    'The layer exists and the overwrite parameter is %s' %
                    overwrite)
                raise GeoNodeException(msg)
        else:

            # If our resource is already configured in the store it
            # needs to have the right resource type

            for resource in resources:
                if resource.name == name:

                    assert overwrite, "Name already in use and overwrite is False"

                    existing_type = resource.resource_type
                    if existing_type != the_layer_type:
                        msg = (
                            'Type of uploaded file %s (%s) does not match type '
                            'of existing resource type %s' %
                            (name, the_layer_type, existing_type))
                        _log(msg)
                        raise GeoNodeException(msg)

    if the_layer_type not in (
            FeatureType.resource_type,
            Coverage.resource_type):
        raise Exception(
            'Expected the layer type to be a FeatureType or Coverage, not %s' %
            the_layer_type)
    _log('Uploading %s', the_layer_type)

    error_msg = None
    try:
        # importer tracks ids by autoincrement but is prone to corruption
        # which potentially may reset the id - hopefully prevent this...
        next_id = Upload.objects.all().aggregate(Max('import_id')).values()[0]
        next_id = next_id + 1 if next_id else 1

        # save record of this whether valid or not - will help w/ debugging
        upload = Upload.objects.create(
            user=user,
            name=name,
            state=Upload.STATE_INVALID,
            upload_dir=spatial_files.dirname)

        # @todo settings for use_url or auto detection if geoserver is
        # on same host
        import_session = gs_uploader.upload_files(
            spatial_files.all_files(),
            use_url=False,
            import_id=next_id,
            mosaic=len(spatial_files) > 1)

        upload.import_id = import_session.id
        upload.save()

        # any unrecognized tasks/files must be deleted or we can't proceed
        import_session.delete_unrecognized_tasks()

        if not import_session.tasks:
            error_msg = 'No valid upload files could be found'
        elif import_session.tasks[0].state == 'NO_FORMAT':
            error_msg = 'There may be a problem with the data provided - ' \
                        'we could not identify it'

        if len(import_session.tasks) > 1:
            error_msg = "Only a single upload is supported at the moment"

        if not error_msg and import_session.tasks:
            task = import_session.tasks[0]
            # single file tasks will have just a file entry
            if hasattr(task, 'files'):
                # @todo gsimporter - test this
                if not all([hasattr(f, 'timestamp')
                            for f in task.source.files]):
                    error_msg = (
                        "Not all timestamps could be recognized."
                        "Please ensure your files contain the correct formats.")

        if error_msg:
            upload.state = upload.STATE_INVALID
            upload.save()

        # @todo once the random tmp9723481758915 type of name is not
        # around, need to track the name computed above, for now, the
        # target store name can be used
    except Exception as e:
        logger.exception('Error creating import session')
        raise e

    if error_msg:
        raise UploadException(error_msg)
    else:
        _log("Finished upload of [%s] to GeoServer without errors.", name)

    return import_session
示例#6
0
def save_step(user, layer, spatial_files, overwrite=True,
              mosaic=False,
              append_to_mosaic_opts=None, append_to_mosaic_name=None,
              mosaic_time_regex=None, mosaic_time_value=None,
              time_presentation=None, time_presentation_res=None,
              time_presentation_default_value=None, time_presentation_reference_value=None):
    _log('Uploading layer: [%s], files [%s]', layer, spatial_files)

    if len(spatial_files) > 1:
        # we only support more than one file if they're rasters for mosaicing
        if not all(
                [f.file_type.layer_type == 'coverage' for f in spatial_files]):
            raise UploadException(
                "Please upload only one type of file at a time")

    name = get_valid_layer_name(layer, overwrite)
    _log('Name for layer: [%s]', name)

    if not spatial_files:
        raise UploadException("Unable to recognize the uploaded file(s)")

    the_layer_type = spatial_files[0].file_type.layer_type

    # Check if the store exists in geoserver
    try:
        store = gs_catalog.get_store(name)
    except geoserver.catalog.FailedRequestError as e:
        # There is no store, ergo the road is clear
        pass
    else:
        # If we get a store, we do the following:
        resources = store.get_resources()
        # Is it empty?
        if len(resources) == 0:
            # What should we do about that empty store?
            if overwrite:
                # We can just delete it and recreate it later.
                store.delete()
            else:
                msg = (
                    'The layer exists and the overwrite parameter is %s' %
                    overwrite)
                raise GeoNodeException(msg)
        else:

            # If our resource is already configured in the store it
            # needs to have the right resource type

            for resource in resources:
                if resource.name == name:

                    assert overwrite, "Name already in use and overwrite is False"

                    existing_type = resource.resource_type
                    if existing_type != the_layer_type:
                        msg = (
                            'Type of uploaded file %s (%s) does not match type '
                            'of existing resource type %s' %
                            (name, the_layer_type, existing_type))
                        _log(msg)
                        raise GeoNodeException(msg)

    if the_layer_type not in (
            FeatureType.resource_type,
            Coverage.resource_type):
        raise Exception(
            'Expected the layer type to be a FeatureType or Coverage, not %s' %
            the_layer_type)
    _log('Uploading %s', the_layer_type)

    error_msg = None
    try:
        # importer tracks ids by autoincrement but is prone to corruption
        # which potentially may reset the id - hopefully prevent this...
        upload_next_id = Upload.objects.all().aggregate(Max('import_id')).values()[0]
        upload_next_id = upload_next_id if upload_next_id else 0
        # next_id = next_id + 1 if next_id else 1
        importer_sessions = gs_uploader.get_sessions()

        last_importer_session = importer_sessions[len(importer_sessions)-1].id if importer_sessions else 0
        next_id = max(int(last_importer_session), int(upload_next_id)) + 1
        next_id = max(int(last_importer_session), int(upload_next_id)) + 1

        # Truncate name to maximum length defined by the field.
        max_length = Upload._meta.get_field('name').max_length
        name = name[:max_length]

        # save record of this whether valid or not - will help w/ debugging
        upload = Upload.objects.create(
            user=user,
            name=name,
            state=Upload.STATE_INVALID,
            upload_dir=spatial_files.dirname)

        # @todo settings for use_url or auto detection if geoserver is
        # on same host

        # Is it a regular file or an ImageMosaic?
        # if mosaic_time_regex and mosaic_time_value:
        if mosaic:
            # we want to ingest as ImageMosaic
            target_store = import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name,
                                                       mosaic_time_regex, mosaic_time_value, time_presentation,
                                                       time_presentation_res, time_presentation_default_value,
                                                       time_presentation_reference_value)

            # moving forward with a regular Importer session
            import_session = gs_uploader.upload_files(
                spatial_files.all_files(),
                use_url=False,
                import_id=next_id,
                mosaic=len(spatial_files) > 1,
                target_store=target_store)

            upload.moasic = mosaic
            upload.append_to_mosaic_opts = append_to_mosaic_opts
            upload.append_to_mosaic_name = append_to_mosaic_name
            upload.mosaic_time_regex = mosaic_time_regex
            upload.mosaic_time_value = mosaic_time_value
        else:
            # moving forward with a regular Importer session
            import_session = gs_uploader.upload_files(
                spatial_files.all_files(),
                use_url=False,
                import_id=next_id,
                mosaic=len(spatial_files) > 1)

        upload.import_id = import_session.id
        upload.save()

        # any unrecognized tasks/files must be deleted or we can't proceed
        import_session.delete_unrecognized_tasks()

        if not import_session.tasks:
            error_msg = 'No valid upload files could be found'
        elif import_session.tasks[0].state == 'NO_FORMAT':
            error_msg = 'There may be a problem with the data provided - ' \
                        'we could not identify it'

        if len(import_session.tasks) > 1:
            error_msg = "Only a single upload is supported at the moment"

        if not error_msg and import_session.tasks:
            task = import_session.tasks[0]
            # single file tasks will have just a file entry
            if hasattr(task, 'files'):
                # @todo gsimporter - test this
                if not all([hasattr(f, 'timestamp')
                            for f in task.source.files]):
                    error_msg = (
                        "Not all timestamps could be recognized."
                        "Please ensure your files contain the correct formats.")

        if error_msg:
            upload.state = upload.STATE_INVALID
            upload.save()

        # @todo once the random tmp9723481758915 type of name is not
        # around, need to track the name computed above, for now, the
        # target store name can be used
    except Exception as e:
        logger.exception('Error creating import session')
        raise e

    if error_msg:
        raise UploadException(error_msg)
    else:
        _log("Finished upload of [%s] to GeoServer without errors.", name)

    return import_session