Exemplo n.º 1
0
    def test_layer_delete_from_geoserver(self):
        """Verify that layer is correctly deleted from GeoServer
        """
        # Layer.delete() calls the pre_delete hook which uses cascading_delete()
        # Should we explicitly test that the styles and store are
        # deleted as well as the resource itself?
        # There is already an explicit test for cascading delete

        gs_cat = gs_catalog

        # Test Uploading then Deleting a Shapefile from GeoServer
        shp_file = os.path.join(
            gisdata.VECTOR_DATA,
            'san_andres_y_providencia_poi.shp')
        shp_layer = file_upload(shp_file, overwrite=True)
        shp_store = gs_cat.get_store(shp_layer.name)
        shp_layer.delete()
        self.assertRaises(
            FailedRequestError,
            lambda: gs_cat.get_resource(
                shp_layer.name,
                store=shp_store))

        # Test Uploading then Deleting a TIFF file from GeoServer
        tif_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        tif_layer = file_upload(tif_file)
        tif_store = gs_cat.get_store(tif_layer.name)
        tif_layer.delete()
        self.assertRaises(
            FailedRequestError,
            lambda: gs_cat.get_resource(
                shp_layer.name,
                store=tif_store))
Exemplo n.º 2
0
    def test_unpublished(self):
        """Test permissions on an unpublished layer
        """

        thefile = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp")
        layer = file_upload(thefile, overwrite=True)
        layer.set_default_permissions()
        check_layer(layer)

        # we need some time to have the service up and running
        time.sleep(20)

        # request getCapabilities: layer must be there as it is published and
        # advertised: we need to check if in response there is
        # <Name>geonode:san_andres_y_providencia_water</Name>
        url = "http://localhost:8080/geoserver/ows?" "service=wms&version=1.3.0&request=GetCapabilities"
        str_to_check = "<Name>geonode:san_andres_y_providencia_poi</Name>"
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        self.assertTrue(any(str_to_check in s for s in response.readlines()))

        # by default the uploaded layer is
        self.assertTrue(layer.is_published, True)

        # Clean up and completely delete the layer
        layer.delete()

        # with settings disabled
        with self.settings(RESOURCE_PUBLISHING=True):

            thefile = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_administrative.shp")
            layer = file_upload(thefile, overwrite=True)
            layer.set_default_permissions()
            check_layer(layer)

            # we need some time to have the service up and running
            time.sleep(20)

            str_to_check = "<Name>san_andres_y_providencia_administrative</Name>"

            # by default the uploaded layer must be unpublished
            self.assertEqual(layer.is_published, False)

            # check the layer is not in GetCapabilities
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            self.assertFalse(any(str_to_check in s for s in response.readlines()))

            # now test with published layer
            resource = layer.get_self_resource()
            resource.is_published = True
            resource.save()

            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            self.assertTrue(any(str_to_check in s for s in response.readlines()))

            # Clean up and completely delete the layer
            layer.delete()
Exemplo n.º 3
0
 def test_extension_not_implemented(self):
     """Verify a GeoNodeException is returned for not compatible extensions
     """
     sampletxt = os.path.join(gisdata.VECTOR_DATA, "points_epsg2249_no_prj.dbf")
     try:
         file_upload(sampletxt)
     except GeoNodeException, e:
         pass
Exemplo n.º 4
0
    def save_shape(self, shape_in_memory=None, user=None):

        if shape_in_memory is None:
            shape_in_memory = self.shp

        shape_in_memory.save(self.path_file)
        self.__create_projection()

        file = self.path_file + ".shp"
        file_upload(file, user=user)
        command_remove_shape = "rm -f " + self.path_file + ".*"
        call(command_remove_shape, shell=True)
Exemplo n.º 5
0
    def test_bad_shapefile(self):
        """Verifying GeoNode complains about a shapefile without .prj
        """

        thefile = os.path.join(gisdata.BAD_DATA, 'points_epsg2249_no_prj.shp')
        try:
            file_upload(thefile, overwrite=True)
        except UploadError:
            pass
        except GeoNodeException:
            pass
        except Exception:
            raise
Exemplo n.º 6
0
    def test_map_json(self):
        # 2 layers to be added to the map
        filename = os.path.join(
            gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
        layer1 = file_upload(filename)

        filename = os.path.join(
            gisdata.GOOD_DATA,
            'vector/san_andres_y_providencia_administrative.shp')
        layer2 = file_upload(filename)

        json_payload = InitialSetup.generate_initial_map(layer1, layer2)
        # First, create a map with two layers
        # Need to log in for saving a map
        self.client.login(username='******', password='******')

        result_new_map = self.client.post(
            reverse('new_map_json'),
            json.dumps(json_payload),
            content_type='application/json')
        # the new map is successfully saved
        self.assertEqual(result_new_map.status_code, 200)

        map_id = json.loads(result_new_map.content).get('id')
        # try to remove one layer
        layers = json_payload['map']['layers']
        before_remove = len(layers)
        after_remove = before_remove - 1
        layer = layers[0]
        layers.remove(layer)

        # check if the layer is eliminated from the map
        result_update_map = self.client.post(
            reverse('map_json', kwargs={'mapid': map_id}),
            data=json.dumps(json_payload),
            content_type='application/json')
        # successfully updated
        self.assertEqual(result_update_map.status_code, 200)
        # the number of layers on the map decrease by 1
        self.assertEqual(
            len(result_update_map.context_data['map'].layers),
            after_remove)

        # clean up
        map = Map.objects.get(id=map_id)
        map.delete()
        layer1.delete()
        layer2.delete()
Exemplo n.º 7
0
    def test_cascading_delete(self):
        """Verify that the helpers.cascading_delete() method is working properly
        """
        gs_cat = gs_catalog

        # Upload a Shapefile
        shp_file = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
        shp_layer = file_upload(shp_file)

        # Save the names of the Resource/Store/Styles
        resource_name = shp_layer.name
        ws = gs_cat.get_workspace(shp_layer.workspace)
        store = gs_cat.get_store(shp_layer.store, ws)
        store_name = store.name
        layer = gs_cat.get_layer(resource_name)
        styles = layer.styles + [layer.default_style]

        # Delete the Layer using cascading_delete()
        cascading_delete(gs_cat, shp_layer.typename)

        # Verify that the styles were deleted
        for style in styles:
            s = gs_cat.get_style(style.name)
            assert s == None

        # Verify that the resource was deleted
        self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(resource_name, store=store))

        # Verify that the store was deleted
        self.assertRaises(FailedRequestError, lambda: gs_cat.get_store(store_name))

        # Clean up by deleting the layer from GeoNode's DB and GeoNetwork
        shp_layer.delete()
Exemplo n.º 8
0
    def test_list_style(self):
        """Test querying list of styles from QGIS Server."""
        filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
        layer = file_upload(filename)
        """:type: geonode.layers.models.Layer"""

        actual_list_style = style_list(layer, internal=False)
        expected_list_style = ['default']

        # There will be a default style
        self.assertEqual(
            set(expected_list_style),
            set([style.name for style in actual_list_style]))

        style_list_url = reverse(
            'qgis_server:download-qml',
            kwargs={
                'layername': layer.name
            })
        response = self.client.get(style_list_url)
        self.assertEqual(response.status_code, 200)
        actual_list_style = json.loads(response.content)

        # There will be a default style
        self.assertEqual(
            set(expected_list_style),
            set([style['name'] for style in actual_list_style]))

        layer.delete()
Exemplo n.º 9
0
    def test_map_thumbnail(self):
        """Test the map save method generates a thumbnail link
        """
        # TODO: Would be nice to ensure the name is available before
        # running the test...
        norman = get_user_model().objects.get(username="******")
        saved_layer = file_upload(
            os.path.join(
                gisdata.VECTOR_DATA,
                "san_andres_y_providencia_poi.shp"),
            name="san_andres_y_providencia_poi_by_norman",
            user=norman,
            overwrite=True,
        )
        try:
            self.client.login(username='******', password='******')

            saved_layer.set_default_permissions()
            map_obj = Map(owner=norman, zoom=0,
                          center_x=0, center_y=0)
            map_obj.create_from_layer_list(norman, [saved_layer], 'title', '')

            thumbnail_url = map_obj.get_thumbnail_url()

            self.assertNotEqual(thumbnail_url, staticfiles.static(settings.MISSING_THUMBNAIL))
        finally:
            # Cleanup
            saved_layer.delete()
Exemplo n.º 10
0
 def test_empty_bbox(self):
     """Regression-test for failures caused by zero-width bounding boxes"""
     thefile = os.path.join(gisdata.VECTOR_DATA, 'single_point.shp')
     uploaded = file_upload(thefile, overwrite=True)
     self.client.login(username='******', password='******')
     resp = self.client.get(uploaded.get_absolute_url())
     self.assertEquals(resp.status_code, 200)
Exemplo n.º 11
0
    def test_unpublished(self):
        """Test permissions on an unpublished layer
        """

        thefile = os.path.join(
            gisdata.VECTOR_DATA,
            'san_andres_y_providencia_poi.shp')
        layer = file_upload(thefile, overwrite=True)
        check_layer(layer)

        # we need some time to have the service up and running
        time.sleep(20)

        # request getCapabilities: layer must be there as it is published and
        # advertised: we need to check if in response there is
        # <Name>geonode:san_andres_y_providencia_water</Name>
        url = 'http://localhost:8080/geoserver/ows?' \
            'service=wms&version=1.3.0&request=GetCapabilities'
        str_to_check = '<Name>geonode:san_andres_y_providencia_poi</Name>'
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        self.assertTrue(any(str_to_check in s for s in response.readlines()))

        # now test with unpublished layer
        resource = layer.get_self_resource()
        resource.is_published = False
        resource.save()

        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        self.assertFalse(any(str_to_check in s for s in response.readlines()))

        # Clean up and completely delete the layer
        layer.delete()
Exemplo n.º 12
0
    def test_getcapabilities_filters_by_site(self):
        thefile1 = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp")
        thefile2 = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_administrative.shp")
        file_upload(thefile1, overwrite=True)
        file_upload(thefile2, overwrite=True)

        # remove one layer from the site resources
        SiteResources.objects.get(site=self.slave_site).resources.remove(Layer.objects.all()[0])

        self.assertEqual(SiteResources.objects.get(site=self.slave_site).resources.count(), 1)

        self.client.login(username="******", password="******")
        resp = self.client.get(self.getcaps_url)
        xml = minidom.parseString(resp.content)
        getcaps_layer = xml.getElementsByTagName("Layer")[0]
        self.assertEqual(len(getcaps_layer.getElementsByTagName("Layer")), 1)
Exemplo n.º 13
0
    def test_set_attributes(self):
        """Test attributes syncronization
        """

        # upload a shapefile
        shp_file = os.path.join(
            gisdata.VECTOR_DATA,
            'san_andres_y_providencia_poi.shp')
        layer = file_upload(shp_file)

        # set attributes for resource
        for attribute in layer.attribute_set.all():
            attribute.attribute_label = '%s_label' % attribute.attribute
            attribute.description = '%s_description' % attribute.attribute
            attribute.save()

        # sync the attributes with GeoServer
        set_attributes(layer)

        # tests if everything is synced properly
        for attribute in layer.attribute_set.all():
            self.assertEquals(
                attribute.attribute_label,
                '%s_label' % attribute.attribute
            )
            self.assertEquals(
                attribute.description,
                '%s_description' % attribute.attribute
            )
Exemplo n.º 14
0
def layer_replace(request, layername, template='layers/layer_replace.html'):
    layer = _resolve_layer(request, layername, 'base.change_resourcebase',_PERMISSION_MSG_MODIFY)

    if request.method == 'GET':
        return render_to_response(template,
                                  RequestContext(request, {'layer': layer,
                                                           'is_featuretype': layer.is_vector()}))
    elif request.method == 'POST':

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                saved_layer = file_upload(base_file, name=layer.name,
                                          user=request.user, overwrite=True)
            except Exception, e:
                out['success'] = False
                out['errors'] = str(e)
            else:
                out['success'] = True
                out['url'] = reverse('layer_detail', args=[saved_layer.service_typename])
            finally:
Exemplo n.º 15
0
def layer_replace(request, layername, template='layers/layer_replace.html'):
    layer = _resolve_layer(
        request,
        layername,
        'base.change_resourcebase',
        _PERMISSION_MSG_MODIFY)

    if request.method == 'GET':
        ctx = {
            'charsets': CHARSETS,
            'layer': layer,
            'is_featuretype': layer.is_vector(),
            'is_layer': True,
        }
        return render_to_response(template,
                                  RequestContext(request, ctx))
    elif request.method == 'POST':

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                saved_layer = file_upload(
                    base_file,
                    name=layer.name,
                    user=request.user,
                    overwrite=True,
                    charset=form.cleaned_data["charset"],
                )
            except Exception as e:
                out['success'] = False
                out['errors'] = str(e)
            else:
                out['success'] = True
                out['url'] = reverse(
                    'layer_detail', args=[
                        saved_layer.service_typename])
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            errormsgs = []
            for e in form.errors.values():
                errormsgs.append([escape(v) for v in e])

            out['errors'] = form.errors
            out['errormsgs'] = errormsgs

        if out['success']:
            status_code = 200
        else:
            status_code = 500
        return HttpResponse(
            json.dumps(out),
            mimetype='application/json',
            status=status_code)
Exemplo n.º 16
0
    def test_layer_delete_from_geoserver(self):
        """Verify that layer is correctly deleted from GeoServer
        """
        # Layer.delete() calls the pre_delete hook which uses cascading_delete()
        # Should we explicitly test that the styles and store are
        # deleted as well as the resource itself?
        # There is already an explicit test for cascading delete

        gs_cat = gs_catalog

        # Test Uploading then Deleting a Shapefile from GeoServer
        shp_file = os.path.join(
            gisdata.VECTOR_DATA,
            'san_andres_y_providencia_poi.shp')
        shp_layer = file_upload(shp_file, overwrite=True)

        # we need some time to have the service up and running
        time.sleep(20)

        ws = gs_cat.get_workspace(shp_layer.workspace)
        shp_store = gs_cat.get_store(shp_layer.store, ws)
        shp_store_name = shp_store.name
        shp_layer.delete()

        # Verify that it no longer exists in GeoServer
        self.assertRaises(
            FailedRequestError,
            lambda: gs_cat.get_resource(
                shp_layer.name,
                store=shp_store))

        # Verify that the store was deleted
        ds = gs_cat.get_store(shp_store_name)
        self.assertIsNone(ds)

        # Test Uploading then Deleting a TIFF file from GeoServer
        tif_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        tif_layer = file_upload(tif_file)
        ws = gs_cat.get_workspace(tif_layer.workspace)
        tif_store = gs_cat.get_store(tif_layer.store, ws)
        tif_layer.delete()
        self.assertRaises(
            FailedRequestError,
            lambda: gs_cat.get_resource(
                shp_layer.name,
                store=tif_store))
Exemplo n.º 17
0
def layer_replace(request, layername, template='layers/layer_replace.html'):
    layer = _resolve_layer(
        request,
        layername,
        'base.change_resourcebase_permissions',
        _PERMISSION_MSG_MODIFY)
    try:
        if "permission_denied" in layer:
            return HttpResponse('You are not allowed to replace this layer',
                mimetype="text/plain",
                status=401)
    except:
        pass


    if request.method == 'GET':
        return render_to_response(
            template, RequestContext(
                request, {
                    'layer': layer, 'is_featuretype': layer.is_vector()}))
    elif request.method == 'POST':

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                saved_layer = file_upload(base_file, name=layer.name,
                                          user=request.user, overwrite=True)
            except Exception as e:
                out['success'] = False
                out['errors'] = str(e)
            else:
                out['success'] = True
                out['url'] = reverse(
                    'layer_detail', args=[
                        saved_layer.service_typename])
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            errormsgs = []
            for e in form.errors.values():
                errormsgs.append([escape(v) for v in e])

            out['errors'] = form.errors
            out['errormsgs'] = errormsgs

        if out['success']:
            status_code = 200
        else:
            status_code = 500
        return HttpResponse(
            json.dumps(out),
            mimetype='application/json',
            status=status_code)
Exemplo n.º 18
0
    def test_download_map_qlr(self):
        """Test download QLR file for a map"""
        # 2 layers to be added to the map
        filename = os.path.join(
            gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
        layer1 = file_upload(filename)

        filename = os.path.join(
            gisdata.GOOD_DATA,
            'vector/san_andres_y_providencia_administrative.shp')
        layer2 = file_upload(filename)

        # construct json request for new map
        json_payload = InitialSetup.generate_initial_map(layer1, layer2)

        self.client.login(username='******', password='******')

        response = self.client.post(
            reverse('new_map_json'),
            json.dumps(json_payload),
            content_type='application/json')
        # map is successfull saved
        self.assertEqual(response.status_code, 200)

        map_id = json.loads(response.content).get('id')

        map = Map.objects.get(id=map_id)

        # check that QLR is added to the links
        links = map.link_set.download()
        map_qlr_link = links.get(name='Download QLR Layer file')
        self.assertIn('qlr', map_qlr_link.url)

        # QLR
        response = self.client.get(
            reverse('map_download_qlr', kwargs={'mapid': map_id}))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.get('Content-Type'),
            'application/x-qgis-layer-definition')

        # cleanup
        map.delete()
        layer1.delete()
        layer2.delete()
Exemplo n.º 19
0
    def test_tiff(self):
        """Uploading a good .tiff
        """
        thefile = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        uploaded = file_upload(thefile, overwrite=True)
        check_layer(uploaded)

        # Clean up and completely delete the layer
        uploaded.delete()
Exemplo n.º 20
0
    def setUp(self):
        super(LayerMetadataDetailTest, self).setUp()

        from geonode.layers.utils import file_upload
        self.layer = file_upload(
            os.path.join(TESTDIR, 'test_point.shp'),
            name='testlayer'
        )
        self.url = '/layers/geonode:testlayer/metadata_detail'
Exemplo n.º 21
0
 def test_raster_upload(self):
     """Test that the wcs links are correctly created for a raster"""
     filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
     uploaded = file_upload(filename)
     wcs_link = False
     for link in uploaded.link_set.all():
         if link.mime == 'GeoTIFF':
             wcs_link = True
     self.assertTrue(wcs_link)
Exemplo n.º 22
0
 def setUp(self):
     super(NewMapConfigTest, self).setUp()
     from geonode.layers.utils import file_upload
     self.layer = file_upload(
         os.path.join(TESTDIR, 'test_point.shp'),
         name='testlayer'
     )
     self.url = '/layers/geonode:testlayer'
     self.service_name = 'data-test'
Exemplo n.º 23
0
    def test_bad_shapefile(self):
        """Verifying GeoNode complains about a shapefile without .prj
        """

        thefile = os.path.join(gisdata.BAD_DATA, "points_epsg2249_no_prj.shp")
        try:
            uploaded = file_upload(thefile, overwrite=True)
        except GeoNodeException, e:
            pass
Exemplo n.º 24
0
 def test_zipped_files(self):
     """Test that the zipped files is created for raster."""
     filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
     uploaded = file_upload(filename)
     zip_link = False
     for link in uploaded.link_set.all():
         if link.mime == 'ZIP':
             zip_link = True
     self.assertTrue(zip_link)
Exemplo n.º 25
0
    def test_shapefile(self):
        """Test Uploading a good shapefile
        """
        thefile = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
        uploaded = file_upload(thefile, overwrite=True)
        check_layer(uploaded)

        # Clean up and completely delete the layer
        uploaded.delete()
Exemplo n.º 26
0
 def test_keywords_upload(self):
     """Check that keywords can be passed to file_upload
     """
     thefile = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
     uploaded = file_upload(thefile, keywords=['foo', 'bar'], overwrite=True)
     keywords = uploaded.keyword_list()
     msg='No keywords found in layer %s' % uploaded.name
     assert len(keywords)>0, msg
     assert 'foo' in uploaded.keyword_list(), 'Could not find "foo" in %s' % keywords
     assert 'bar' in uploaded.keyword_list(), 'Could not find "bar" in %s' % keywords
Exemplo n.º 27
0
    def setUp(self):
        super(LayerPublishTest, self).setUp()

        from geonode.layers.utils import file_upload
        self.layer = file_upload(
            os.path.join(TESTDIR, 'test_point.shp'),
            name='testlayer'
        )
        self.url = '/layers/geonode:testlayer/publish'
        # We expect to be redirected
        self.expected_status = 302
Exemplo n.º 28
0
def create_layer(number):
    """ Creates a new layer """
    file_list = glob.glob('%s*.shp' % shp_path)
    random_index = randint(0, len(file_list) -1)
    file_uri = file_list[random_index]
    layer = file_upload(file_uri)
    # keywords
    assign_keywords(layer)
    # other stuff
    resource = layer.get_self_resource()
    set_resource(resource)
Exemplo n.º 29
0
def layer_replace(request, layername, template="layers/layer_replace.html"):
    layer = _resolve_layer(request, layername, "base.change_resourcebase", _PERMISSION_MSG_MODIFY)

    if request.method == "GET":
        ctx = {"charsets": CHARSETS, "layer": layer, "is_featuretype": layer.is_vector(), "is_layer": True}
        return render_to_response(template, RequestContext(request, ctx))
    elif request.method == "POST":

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                if layer.is_vector() and is_raster(base_file):
                    out["success"] = False
                    out["errors"] = _("You are attempting to replace a vector layer with a raster.")
                elif (not layer.is_vector()) and is_vector(base_file):
                    out["success"] = False
                    out["errors"] = _("You are attempting to replace a raster layer with a vector.")
                else:
                    # delete geoserver's store before upload
                    cat = gs_catalog
                    cascading_delete(cat, layer.typename)
                    saved_layer = file_upload(
                        base_file,
                        name=layer.name,
                        user=request.user,
                        overwrite=True,
                        charset=form.cleaned_data["charset"],
                    )
                    out["success"] = True
                    out["url"] = reverse("layer_detail", args=[saved_layer.service_typename])
            except Exception as e:
                out["success"] = False
                out["errors"] = str(e)
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            errormsgs = []
            for e in form.errors.values():
                errormsgs.append([escape(v) for v in e])

            out["errors"] = form.errors
            out["errormsgs"] = errormsgs

        if out["success"]:
            status_code = 200
        else:
            status_code = 400
        return HttpResponse(json.dumps(out), mimetype="application/json", status=status_code)
Exemplo n.º 30
0
    def test_repeated_upload(self):
        """Upload the same file more than once
        """
        thefile = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        uploaded1 = file_upload(thefile, overwrite=True)
        check_layer(uploaded1)
        uploaded2 = file_upload(thefile, overwrite=True)
        check_layer(uploaded2)
        uploaded3 = file_upload(thefile, overwrite=False)
        check_layer(uploaded3)
        msg = ('Expected %s but got %s' % (uploaded1.name, uploaded2.name))
        assert uploaded1.name == uploaded2.name, msg
        msg = ('Expected a different name when uploading %s using '
               'overwrite=False but got %s' % (thefile, uploaded3.name))
        assert uploaded1.name != uploaded3.name, msg

        # Clean up and completely delete the layers

        # uploaded1 is overwritten by uploaded2 ... no need to delete it
        uploaded2.delete()
        uploaded3.delete()
Exemplo n.º 31
0
    def test_ogc_specific_layer(self):
        """Test we can use QGIS Server API for a layer.

        For now, we are just checking we can call these views without any
        exceptions. We should improve this test by checking the result.
        """
        filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
        uploaded = file_upload(filename)

        filename = os.path.join(
            gisdata.GOOD_DATA,
            'vector/san_andres_y_providencia_administrative.shp')
        vector_layer = file_upload(filename)

        params = {'layername': uploaded.name}

        # Zip
        response = self.client.get(
            reverse('qgis_server:download-zip', kwargs=params))
        self.assertEqual(response.status_code, 200)
        try:
            f = StringIO.StringIO(response.content)
            zipped_file = zipfile.ZipFile(f, 'r')

            for one_file in zipped_file.namelist():
                # We shoudn't get any QGIS project
                self.assertFalse(one_file.endswith('.qgs'))
            self.assertIsNone(zipped_file.testzip())
        finally:
            zipped_file.close()
            f.close()

        # Legend
        response = self.client.get(
            reverse('qgis_server:legend', kwargs=params))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/png')
        self.assertEqual(what('', h=response.content), 'png')

        # Tile
        coordinates = {'z': '11', 'x': '1576', 'y': '1054'}
        coordinates.update(params)
        response = self.client.get(
            reverse('qgis_server:tile', kwargs=coordinates))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/png')
        self.assertEqual(what('', h=response.content), 'png')

        # Tile 404
        response = self.client.get(
            reverse('qgis_server:tile', kwargs=params))
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.get('Content-Type'), 'text/html; charset=utf-8')

        # Geotiff
        response = self.client.get(
            reverse('qgis_server:geotiff', kwargs=params))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/tiff')
        self.assertEqual(what('', h=response.content), 'tiff')

        # Layer is already on the database
        # checking the Link
        links = uploaded.link_set.download().filter(
            name__in=settings.DOWNLOAD_FORMATS_RASTER)

        # checks signals.py for the hardcoded names in QLR and QGS
        qlr_link = links.get(name='QGIS layer file (.qlr)')
        self.assertIn("download-qlr", qlr_link.url)
        qgs_link = links.get(name='QGIS project file (.qgs)')
        self.assertIn("download-qgs", qgs_link.url)

        # QLR
        response = self.client.get(
            reverse('qgis_server:download-qlr', kwargs=params))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.get('Content-Type'),
            'application/x-qgis-layer-definition')
        # check file name's extension
        file_name = response.get('Content-Disposition').split('filename=')
        file_ext = file_name[1].split('.')
        self.assertEqual(file_ext[1], "qlr")

        # QGS
        response = self.client.get(
            reverse('qgis_server:download-qgs', kwargs=params))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.get('Content-Type'),
            'application/x-qgis-project')
        # check file name's extension
        file_name = response.get('Content-Disposition').split('filename=')
        file_ext = file_name[1].split('.')
        self.assertEqual(file_ext[1], "qgs")

        response = self.client.get(
            reverse('qgis_server:geotiff', kwargs={
                'layername': vector_layer.name
            }))
        self.assertEqual(response.status_code, 404)

        # QML Styles
        # Request list of styles
        response = self.client.get(
            reverse('qgis_server:download-qml', kwargs=params))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'application/json')
        # Should return a default style list
        actual_result = json.loads(response.content)
        actual_result = [s['name'] for s in actual_result]
        expected_result = ['default']
        self.assertEqual(set(expected_result), set(actual_result))

        # Get single styles
        response = self.client.get(
            reverse('qgis_server:download-qml', kwargs={
                'layername': params['layername'],
                'style_name': 'default'
            }))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'text/xml')

        # Set thumbnail from viewed bbox
        response = self.client.get(
            reverse('qgis_server:set-thumbnail', kwargs=params))
        self.assertEqual(response.status_code, 400)
        data = {
            'bbox': '-5.54025,96.9406,-5.2820,97.1250'
        }
        response = self.client.post(
            reverse('qgis_server:set-thumbnail', kwargs=params),
            data=data)
        # User dont have permission
        self.assertEqual(response.status_code, 403)
        # Should log in
        self.client.login(username='******', password='******')
        response = self.client.post(
            reverse('qgis_server:set-thumbnail', kwargs=params),
            data=data)
        self.assertEqual(response.status_code, 200)
        retval = json.loads(response.content)
        expected_retval = {
            'success': True
        }
        self.assertEqual(retval, expected_retval)

        # OGC Server specific for THE layer
        query_string = {
            'SERVICE': 'WMS',
            'VERSION': '1.3.0',
            'REQUEST': 'GetLegendGraphics',
            'FORMAT': 'image/png',
            'LAYERS': uploaded.name,
        }
        response = self.client.get(
            reverse('qgis_server:layer-request', kwargs=params), query_string)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/png')
        self.assertEqual(what('', h=response.content), 'png')

        # OGC Server for the Geonode instance
        # GetLegendGraphics is a shortcut when using the main OGC server.
        query_string = {
            'SERVICE': 'WMS',
            'VERSION': '1.3.0',
            'REQUEST': 'GetLegendGraphics',
            'FORMAT': 'image/png',
            'LAYERS': uploaded.name,
        }
        response = self.client.get(
            reverse('qgis_server:request'), query_string)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/png')
        self.assertEqual(what('', h=response.content), 'png')

        # WMS GetCapabilities
        query_string = {
            'SERVICE': 'WMS',
            'VERSION': '1.3.0',
            'REQUEST': 'GetCapabilities'
        }

        response = self.client.get(
            reverse('qgis_server:request'), query_string)
        self.assertEqual(response.status_code, 200, response.content)
        self.assertEqual(
            response.content, 'GetCapabilities is not supported yet.')

        query_string['LAYERS'] = uploaded.name

        response = self.client.get(
            reverse('qgis_server:request'), query_string)
        get_capabilities_content = response.content

        # Check xml content
        self.assertEqual(response.status_code, 200, response.content)
        root = etree.fromstring(response.content)
        layer_xml = root.xpath(
            'wms:Capability/wms:Layer/wms:Layer/wms:Name',
            namespaces={'wms': 'http://www.opengis.net/wms'})
        self.assertEqual(len(layer_xml), 1)
        self.assertEqual(layer_xml[0].text, uploaded.name)
        # GetLegendGraphic request returned must be valid
        layer_xml = root.xpath(
            'wms:Capability/wms:Layer/'
            'wms:Layer/wms:Style/wms:LegendURL/wms:OnlineResource',
            namespaces={
                'xlink': 'http://www.w3.org/1999/xlink',
                'wms': 'http://www.opengis.net/wms'
            })
        legend_url = layer_xml[0].attrib[
            '{http://www.w3.org/1999/xlink}href']

        response = self.client.get(legend_url)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.get('Content-Type'), 'image/png')
        self.assertEqual(what('', h=response.content), 'png')

        # Check get capabilities using helper returns the same thing
        response = requests.get(wms_get_capabilities_url(
            uploaded, internal=False))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(get_capabilities_content, response.content)

        # WMS GetMap
        query_string = {
            'SERVICE': 'WMS',
            'VERSION': '1.3.0',
            'REQUEST': 'GetMap',
            'FORMAT': 'image/png',
            'LAYERS': uploaded.name,
            'HEIGHT': 250,
            'WIDTH': 250,
            'SRS': 'EPSG:4326',
            'BBOX': '-5.54025,96.9406,-5.2820,97.1250',
        }
        response = self.client.get(
            reverse('qgis_server:request'), query_string)
        self.assertEqual(response.status_code, 200, response.content)
        self.assertEqual(
            response.get('Content-Type'), 'image/png', response.content)
        self.assertEqual(what('', h=response.content), 'png')

        # End of the test, we should remove every files related to the test.
        uploaded.delete()
        vector_layer.delete()
Exemplo n.º 32
0
    def test_layer_replace(self):
        """Test layer replace functionality
        """
        vector_file = os.path.join(
            gisdata.VECTOR_DATA, 'san_andres_y_providencia_administrative.shp')
        vector_layer = file_upload(vector_file, overwrite=True)

        raster_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        raster_layer = file_upload(raster_file, overwrite=True)

        # we need some time to have the service up and running
        time.sleep(20)
        new_vector_layer = None
        try:
            self.client.login(username='******', password='******')

            # test the program can determine the original layer in raster type
            raster_replace_url = reverse('layer_replace',
                                         args=[raster_layer.service_typename])
            response = self.client.get(raster_replace_url)
            self.assertEquals(response.status_code, 200)
            self.assertEquals(response.context['is_featuretype'], False)

            # test the program can determine the original layer in vector type
            vector_replace_url = reverse('layer_replace',
                                         args=[vector_layer.service_typename])
            response = self.client.get(vector_replace_url)
            self.assertEquals(response.status_code, 200)
            self.assertEquals(response.context['is_featuretype'], True)

            # test replace a vector with a raster
            response = self.client.post(vector_replace_url,
                                        {'base_file': open(raster_file, 'rb')})
            # TODO: This should really return a 400 series error with the json dict
            self.assertEquals(response.status_code, 400)
            response_dict = json.loads(response.content)
            self.assertEquals(response_dict['success'], False)

            # test replace a vector with a different vector
            new_vector_file = os.path.join(
                gisdata.VECTOR_DATA, 'san_andres_y_providencia_coastline.shp')
            layer_path, __ = os.path.splitext(new_vector_file)
            layer_base = open(layer_path + '.shp', 'rb')
            layer_dbf = open(layer_path + '.dbf', 'rb')
            layer_shx = open(layer_path + '.shx', 'rb')
            layer_prj = open(layer_path + '.prj', 'rb')

            response = self.client.post(
                vector_replace_url, {
                    'base_file': layer_base,
                    'dbf_file': layer_dbf,
                    'shx_file': layer_shx,
                    'prj_file': layer_prj,
                    'charset': 'UTF-8'
                })
            response_dict = json.loads(response.content)

            if not response_dict[
                    'success'] and 'unknown encoding' in response_dict[
                        'errors']:
                print(response_dict['errors'])
                pass
            else:
                self.assertEquals(response.status_code, 200)
                self.assertEquals(response_dict['success'], True)

                # Get a Layer object for the newly created layer.
                new_vector_layer = Layer.objects.get(pk=vector_layer.pk)
                # FIXME(Ariel): Check the typename does not change.

                # Test the replaced layer is indeed different from the original layer
                self.assertNotEqual(vector_layer.bbox_x0,
                                    new_vector_layer.bbox_x0)
                self.assertNotEqual(vector_layer.bbox_x1,
                                    new_vector_layer.bbox_x1)
                self.assertNotEqual(vector_layer.bbox_y0,
                                    new_vector_layer.bbox_y0)
                self.assertNotEqual(vector_layer.bbox_y1,
                                    new_vector_layer.bbox_y1)

                # test an invalid user without layer replace permission
                self.client.logout()
                self.client.login(username='******', password='******')

                response = self.client.post(
                    vector_replace_url, {
                        'base_file': layer_base,
                        'dbf_file': layer_dbf,
                        'shx_file': layer_shx,
                        'prj_file': layer_prj
                    })
                self.assertEquals(response.status_code, 401)
        finally:
            # Clean up and completely delete the layer
            try:
                if vector_layer:
                    vector_layer.delete()
                if raster_layer:
                    raster_layer.delete()
                if new_vector_layer:
                    new_vector_layer.delete()
            except:
                tb = traceback.format_exc()
                print(tb)
Exemplo n.º 33
0
def process_impact_result(self, impact_url, analysis_id):
    """Extract impact analysis after running it via InaSAFE-Headless celery

    :param self: Task instance
    :type self: celery.task.Task

    :param impact_url: impact url returned from analysis
    :type impact_url: str

    :param analysis_id: analysis id of the object
    :type analysis_id: int

    :return: True if success
    :rtype: bool
    """
    # Track the current task_id
    analysis = Analysis.objects.get(id=analysis_id)

    analysis.task_id = self.request.id
    analysis.save()

    # decide if we are using direct access or not
    impact_url = get_impact_path(impact_url)

    # download impact zip
    impact_path = download_file(impact_url)
    dir_name = os.path.dirname(impact_path)
    success = False
    with ZipFile(impact_path) as zf:
        zf.extractall(path=dir_name)
        for name in zf.namelist():
            basename, ext = os.path.splitext(name)
            if ext in ['.shp', '.tif']:
                # process this in the for loop to make sure it works only
                # when we found the layer
                saved_layer = file_upload(os.path.join(dir_name, name),
                                          overwrite=True)
                saved_layer.set_default_permissions()
                if analysis.user_title:
                    layer_name = analysis.user_title
                else:
                    layer_name = analysis.get_default_impact_title()
                saved_layer.title = layer_name
                saved_layer.save()
                current_impact = None
                if analysis.impact_layer:
                    current_impact = analysis.impact_layer
                analysis.impact_layer = saved_layer

                # check map report and table
                report_map_path = os.path.join(dir_name, '%s.pdf' % basename)

                if os.path.exists(report_map_path):
                    analysis.assign_report_map(report_map_path)

                report_table_path = os.path.join(dir_name,
                                                 '%s_table.pdf' % basename)

                if os.path.exists(report_table_path):
                    analysis.assign_report_table(report_table_path)

                analysis.task_id = process_impact_result.request.id
                analysis.task_state = 'SUCCESS'
                analysis.end_time = datetime.now().strftime('%Y-%m-%d %H:%M')
                analysis.save()

                if current_impact:
                    current_impact.delete()
                success = True
                break

        # cleanup
        for name in zf.namelist():
            filepath = os.path.join(dir_name, name)
            try:
                os.remove(filepath)
            except BaseException:
                pass

    # cleanup
    try:
        os.remove(impact_path)
    except BaseException:
        pass

    if not success:
        LOGGER.info('No impact layer found in %s' % impact_url)

    return success
Exemplo n.º 34
0
    def test_layer_upload_metadata(self):
        """Test uploading a layer with XML metadata"""

        thelayer = os.path.join(
            gisdata.PROJECT_ROOT,
            'both',
            'good',
            'sangis.org',
            'Airport',
            'Air_Runways.shp')

        self.assertTrue('%s.xml' % thelayer,
                        'Expected layer XML metadata to exist')

        uploaded = file_upload(thelayer, overwrite=True)

        self.assertEqual(
            uploaded.title,
            'Air_Runways',
            'Expected specific title from uploaded layer XML metadata')

        self.assertEqual(
            uploaded.abstract,
            'Airport Runways',
            'Expected specific abstract from uploaded layer XML metadata')

        self.assertEqual(
            uploaded.purpose,
            'To show the location of Public Airports '
            'and Runways within San Diego County',
            'Expected specific purpose from uploaded layer XML metadata')

        self.assertEqual(uploaded.supplemental_information,
                         'No information provided',
                         'Expected specific supplemental information '
                         'from uploaded layer XML metadata')

        self.assertEqual(len(uploaded.keyword_list(
        )), 5, 'Expected specific number of keywords from uploaded layer XML metadata')

        self.assertTrue(
            'Landing Strips' in uploaded.keyword_list(),
            'Expected specific keyword from uploaded layer XML metadata')

        self.assertEqual(
            uploaded.constraints_other,
            'None',
            'Expected specific constraint from uploaded layer XML metadata')

        self.assertEqual(
            uploaded.date,
            datetime.datetime(
                2010,
                8,
                3,
                0,
                0),
            'Expected specific date from uploaded layer XML metadata')

        # Clean up and completely delete the layer
        uploaded.delete()
Exemplo n.º 35
0
def data_request_view(request):
    profile_request_obj = request.session.get('profile_request_obj', None)
    if not profile_request_obj:
        pprint("no profile request object found")
    if not request.user.is_authenticated() and not profile_request_obj:
        return redirect(reverse('datarequests:profile_request_form'))

    request.session['data_request_session'] = True

    form = DataRequestForm()

    if request.method == 'POST' :
        pprint("detected data request post")
        post_data = request.POST.copy()
        post_data['permissions'] = '{"users":{"dataRegistrationUploader": ["view_resourcebase"] }}'
        data_classes = post_data.getlist('data_class_requested')
        #data_classes = post_data.get('data_class_requested')
        data_class_objs = []
        pprint(data_classes)
        pprint("len:"+str(len(data_classes)))

        if len(data_classes) == 1:
            post_data.setlist('data_class_requested',data_classes[0].replace('[','').replace(']','').replace('"','').split(','))
            pprint(post_data.getlist('data_class_requested'))

        details_form = DataRequestForm(post_data, request.FILES)
        data_request_obj = None

        errormsgs = []
        out = {}
        out['errors'] = {}
        out['success'] = False
        saved_layer = None
        if not details_form.is_valid():
            for e in details_form.errors.values():
                errormsgs.extend([escape(v) for v in e])

            out['errors'] =  dict(
                (k, map(unicode, v))
                for (k,v) in details_form.errors.iteritems())

            pprint(out['errors'])
        else:
            tempdir = None
            shapefile_form = DataRequestShapefileForm(post_data, request.FILES)
            if u'base_file' in request.FILES:
                if shapefile_form.is_valid():
                    title = shapefile_form.cleaned_data["layer_title"]

                    # Replace dots in filename - GeoServer REST API upload bug
                    # and avoid any other invalid characters.
                    # Use the title if possible, otherwise default to the filename
                    if title is not None and len(title) > 0:
                        name_base = title
                    else:
                        name_base, __ = os.path.splitext(
                            shapefile_form.cleaned_data["base_file"].name)

                    name = slugify(name_base.replace(".", "_"))

                    try:
                        # Moved this inside the try/except block because it can raise
                        # exceptions when unicode characters are present.
                        # This should be followed up in upstream Django.
                        tempdir, base_file = shapefile_form.write_files()
                        registration_uploader, created = Profile.objects.get_or_create(username='******')
                        pprint("saving jurisdiction")
                        saved_layer = file_upload(
                            base_file,
                            name=name,
                            user=registration_uploader,
                            overwrite=False,
                            charset=shapefile_form.cleaned_data["charset"],
                            abstract=shapefile_form.cleaned_data["abstract"],
                            title=shapefile_form.cleaned_data["layer_title"],
                        )

                    except Exception as e:
                        exception_type, error, tb = sys.exc_info()
                        print traceback.format_exc()
                        out['success'] = False
                        out['errors'] = "An unexpected error was encountered. Check the files you have uploaded, clear your selected files, and try again."
                        # Assign the error message to the latest UploadSession of the data request uploader user.
                        latest_uploads = UploadSession.objects.filter(
                            user=registration_uploader
                        ).order_by('-date')
                        if latest_uploads.count() > 0:
                            upload_session = latest_uploads[0]
                            upload_session.error = str(error)
                            upload_session.traceback = traceback.format_exc(tb)
                            upload_session.context = "Data requester's jurisdiction file upload"
                            upload_session.save()
                            out['traceback'] = upload_session.traceback
                            out['context'] = upload_session.context
                            out['upload_session'] = upload_session.id
                    else:
                        pprint("layer_upload is successful")
                        out['success'] = True
                        out['url'] = reverse(
                            'layer_detail', args=[
                                saved_layer.service_typename])

                        upload_session = saved_layer.upload_session
                        upload_session.processed = True
                        upload_session.save()
                        permissions = {
                            'users': {'dataRegistrationUploader': []},
                            'groups': {}
                        }
                        if request.user.is_authenticated():
                            permissions = {
                                'users': {request.user.username : ['view_resourcebase']},
                                'groups': {}
                            }

                        if permissions is not None and len(permissions.keys()) > 0:
                            saved_layer.set_permissions(permissions)

                        jurisdiction_style.delay(saved_layer)


                    finally:
                        if tempdir is not None:
                            shutil.rmtree(tempdir)

                else:
                    for e in shapefile_form.errors.values():
                        errormsgs.extend([escape(v) for v in e])
                    out['success'] = False
                    out['errors'].update(dict(
                            (k, map(unicode, v))
                            for (k,v) in shapefile_form.errors.iteritems()
                    ))
                    pprint(out['errors'])
                    out['errormsgs'] = out['errors']

        #if out['success']:
        if not out['errors']:
            out['success'] = True
            data_request_obj = details_form.save()
            if request.user.is_authenticated() and not request.user.username  == 'AnonymousUser':
                request_letter = create_letter_document(details_form.clean()['letter_file'], profile = request.user)
                data_request_obj.request_letter = request_letter
                data_request_obj.save()
                data_request_obj.profile = request.user
                data_request_obj.save()
                data_request_obj.set_status("pending")
            else:
                request_letter = create_letter_document(details_form.clean()['letter_file'], profile_request = profile_request_obj)
                data_request_obj.request_letter = request_letter
                data_request_obj.save()
                data_request_obj.profile_request = profile_request_obj
                data_request_obj.save()
                profile_request_obj.data_request= data_request_obj
                profile_request_obj.save()
            data_request_obj.save()
            if saved_layer:
                data_request_obj.jurisdiction_shapefile = saved_layer
                data_request_obj.save()
                place_name_update.delay([data_request_obj])
                compute_size_update.delay([data_request_obj])
                tag_request_suc.delay([data_request_obj])
            status_code = 200
            pprint("request has been succesfully submitted")
            if profile_request_obj and not request.user.is_authenticated():
            #    request_data.send_verification_email()

                out['success_url'] = reverse('datarequests:email_verification_send')

                out['redirect_to'] = reverse('datarequests:email_verification_send')

            elif request.user.is_authenticated():
                messages.info(request, "Your request has been submitted")
                out['success_url'] = reverse('home')

                out['redirect_to'] = reverse('home')

            del request.session['data_request_session']

            if 'profile_request_obj' in request.session:
                del request.session['profile_request_obj']

        else:
            status_code = 400
        pprint("sending this out")
        return HttpResponse(
            json.dumps(out),
            mimetype='application/json',
            status=status_code)
    pprint("i'm here")
    return render(
        request,
        'datarequests/registration/shapefile.html',
        {
            'charsets': CHARSETS,
            'is_layer': True,
            'form': form,
            'support_email': settings.LIPAD_SUPPORT_MAIL,
        })
Exemplo n.º 36
0
    def test_delete_orphan(self):
        """Test orphan deletions.

        This test only started in travis to avoid accidentally deleting owners
        data.
        """
        filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
        uploaded = file_upload(filename)

        # Clean up first
        call_command('delete_orphaned_qgis_server_layers')

        # make request to generate tile cache
        response = self.client.get(
            reverse('qgis_server:tile', kwargs={
                'z': '11',
                'x': '1576',
                'y': '1054',
                'layername': uploaded.name
            }))
        self.assertEqual(response.status_code, 200)

        # register file list
        layer_path = settings.QGIS_SERVER_CONFIG['layer_directory']
        tiles_path = settings.QGIS_SERVER_CONFIG['tiles_directory']
        geonode_layer_path = os.path.join(settings.MEDIA_ROOT, 'layers')

        qgis_layer_list = set(os.listdir(layer_path))
        tile_cache_list = set(os.listdir(tiles_path))
        geonode_layer_list = set(os.listdir(geonode_layer_path))

        # run management command. should not change anything
        call_command('delete_orphaned_qgis_server_layers')

        actual_qgis_layer_list = set(os.listdir(layer_path))
        actual_tile_cache_list = set(os.listdir(tiles_path))
        actual_geonode_layer_list = set(os.listdir(geonode_layer_path))

        self.assertEqual(qgis_layer_list, actual_qgis_layer_list)
        self.assertEqual(tile_cache_list, actual_tile_cache_list)
        self.assertEqual(geonode_layer_list, actual_geonode_layer_list)

        # now create random file without reference
        shutil.copy(
            os.path.join(layer_path, 'test_grid.tif'),
            os.path.join(layer_path, 'test_grid_copy.tif'))
        shutil.copytree(
            os.path.join(tiles_path, 'test_grid'),
            os.path.join(tiles_path, 'test_grid_copy'))
        shutil.copy(
            os.path.join(geonode_layer_path, 'test_grid.tif'),
            os.path.join(geonode_layer_path, 'test_grid_copy.tif'))

        actual_qgis_layer_list = set(os.listdir(layer_path))
        actual_tile_cache_list = set(os.listdir(tiles_path))
        actual_geonode_layer_list = set(os.listdir(geonode_layer_path))

        # run management command. This should clear the files. But preserve
        # registered files (the one that is saved in database)
        call_command('delete_orphaned_qgis_server_layers')

        self.assertEqual(
            {'test_grid_copy.tif'},
            actual_qgis_layer_list - qgis_layer_list)
        self.assertEqual(
            {'test_grid_copy'},
            actual_tile_cache_list - tile_cache_list)
        self.assertEqual(
            {'test_grid_copy.tif'},
            actual_geonode_layer_list - geonode_layer_list)

        # cleanup
        uploaded.delete()
Exemplo n.º 37
0
    def test_capabilities(self):
        """Test capabilities
        """

        # a category
        category = TopicCategory.objects.all()[0]

        # some users
        norman = get_user_model().objects.get(username="******")
        admin = get_user_model().objects.get(username="******")

        # create 3 layers, 2 with norman as an owner an 2 with category as a category
        layer1 = file_upload(
            os.path.join(gisdata.VECTOR_DATA,
                         "san_andres_y_providencia_poi.shp"),
            name='layer1',
            user=norman,
            category=category,
            overwrite=True,
        )
        layer2 = file_upload(
            os.path.join(gisdata.VECTOR_DATA, "single_point.shp"),
            name='layer2',
            user=norman,
            overwrite=True,
        )
        layer3 = file_upload(
            os.path.join(gisdata.VECTOR_DATA,
                         "san_andres_y_providencia_administrative.shp"),
            name='layer3',
            user=admin,
            category=category,
            overwrite=True,
        )
        try:
            namespaces = {
                'wms': 'http://www.opengis.net/wms',
                'xlink': 'http://www.w3.org/1999/xlink',
                'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
            }

            # 0. test capabilities_layer
            url = reverse('capabilities_layer', args=[layer1.id])
            resp = self.client.get(url)
            layercap = dlxml.fromstring(resp.content)
            rootdoc = etree.ElementTree(layercap)
            layernodes = rootdoc.findall('./[wms:Name]', namespaces)
            layernode = layernodes[0]

            self.assertEqual(1, len(layernodes))
            self.assertEqual(
                layernode.find('wms:Name', namespaces).text,
                "geonode:{}".format(layer1.name))

            # 1. test capabilities_user
            url = reverse('capabilities_user', args=[norman.username])
            resp = self.client.get(url)
            layercap = dlxml.fromstring(resp.content)
            rootdoc = etree.ElementTree(layercap)
            layernodes = rootdoc.findall('./[wms:Name]', namespaces)

            # norman has 2 layers
            self.assertEqual(1, len(layernodes))

            # the norman two layers are named layer1 and layer2
            count = 0
            for layernode in layernodes:
                if layernode.find(
                        'wms:Name',
                        namespaces).text == '%s:%s' % ('geonode', layer1.name):
                    count += 1
                elif layernode.find(
                        'wms:Name',
                        namespaces).text == '%s:%s' % ('geonode', layer2.name):
                    count += 1
            self.assertEqual(1, count)

            # 2. test capabilities_category
            url = reverse('capabilities_category', args=[category.identifier])
            resp = self.client.get(url)
            layercap = dlxml.fromstring(resp.content)
            rootdoc = etree.ElementTree(layercap)
            layernodes = rootdoc.findall('./[wms:Name]', namespaces)

            # category is in two layers
            self.assertEqual(1, len(layernodes))

            # the layers for category are named layer1 and layer3
            count = 0
            for layernode in layernodes:
                if layernode.find(
                        'wms:Name',
                        namespaces).text == '%s:%s' % ('geonode', layer1.name):
                    count += 1
                elif layernode.find(
                        'wms:Name',
                        namespaces).text == '%s:%s' % ('geonode', layer3.name):
                    count += 1
            self.assertEqual(1, count)

            # 3. test for a map
            # TODO
        finally:
            # Clean up and completely delete the layers
            layer1.delete()
            layer2.delete()
            layer3.delete()
Exemplo n.º 38
0
def layer_replace(request, layername, template='layers/layer_replace.html'):
    layer = _resolve_layer(
        request,
        layername,
        'base.change_resourcebase',
        _PERMISSION_MSG_MODIFY)

    if request.method == 'GET':
        ctx = {
            'charsets': CHARSETS,
            'layer': layer,
            'is_featuretype': layer.is_vector(),
            'is_layer': True,
        }
        return render_to_response(template,
                                  RequestContext(request, ctx))
    elif request.method == 'POST':

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                if layer.is_vector() and is_raster(base_file):
                    out['success'] = False
                    out['errors'] = _("You are attempting to replace a vector layer with a raster.")
                elif (not layer.is_vector()) and is_vector(base_file):
                    out['success'] = False
                    out['errors'] = _("You are attempting to replace a raster layer with a vector.")
                else:
                    saved_layer = file_upload(
                        base_file,
                        name=layer.name,
                        user=request.user,
                        overwrite=True,
                        charset=form.cleaned_data["charset"],
                    )
                    out['success'] = True
                    out['url'] = reverse(
                        'layer_detail', args=[
                            saved_layer.service_typename])
            except Exception as e:
                out['success'] = False
                out['errors'] = str(e)
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            errormsgs = []
            for e in form.errors.values():
                errormsgs.append([escape(v) for v in e])

            out['errors'] = form.errors
            out['errormsgs'] = errormsgs

        if out['success']:
            status_code = 200
        else:
            status_code = 400
        return HttpResponse(
            json.dumps(out),
            mimetype='application/json',
            status=status_code)
Exemplo n.º 39
0
    def test_layer_replace(self):
        """Test layer replace functionality
        """
        vector_file = os.path.join(
            gisdata.VECTOR_DATA, 'san_andres_y_providencia_administrative.shp')
        vector_layer = file_upload(vector_file, overwrite=True)

        raster_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
        raster_layer = file_upload(raster_file, overwrite=True)

        self.client.login(username='******', password='******')

        # test the program can determine the original layer in raster type
        raster_replace_url = reverse('layer_replace',
                                     args=[raster_layer.service_typename])
        response = self.client.get(raster_replace_url)
        self.assertEquals(response.status_code, 200)
        self.assertEquals(response.context['is_featuretype'], False)

        # test the program can determine the original layer in vector type
        vector_replace_url = reverse('layer_replace',
                                     args=[vector_layer.service_typename])
        response = self.client.get(vector_replace_url)
        self.assertEquals(response.status_code, 200)
        self.assertEquals(response.context['is_featuretype'], True)

        # test replace a vector with a raster
        response = self.client.post(vector_replace_url,
                                    {'base_file': open(raster_file, 'rb')})
        # TODO: This should really return a 400 series error with the json dict
        self.assertEquals(response.status_code, 400)
        response_dict = json.loads(response.content)
        self.assertEquals(response_dict['success'], False)

        # test replace a vector with a different vector
        new_vector_file = os.path.join(gisdata.VECTOR_DATA,
                                       'san_andres_y_providencia_poi.shp')
        layer_path, __ = os.path.splitext(new_vector_file)
        layer_base = open(layer_path + '.shp', 'rb')
        layer_dbf = open(layer_path + '.dbf', 'rb')
        layer_shx = open(layer_path + '.shx', 'rb')
        layer_prj = open(layer_path + '.prj', 'rb')

        response = self.client.post(
            vector_replace_url, {
                'base_file': layer_base,
                'dbf_file': layer_dbf,
                'shx_file': layer_shx,
                'prj_file': layer_prj
            })
        self.assertEquals(response.status_code, 200)
        response_dict = json.loads(response.content)
        self.assertEquals(response_dict['success'], True)

        # Get a Layer object for the newly created layer.
        new_vector_layer = Layer.objects.get(pk=vector_layer.pk)
        # FIXME(Ariel): Check the typename does not change.

        # Test the replaced layer is indeed different from the original layer
        self.assertNotEqual(vector_layer.bbox_x0, new_vector_layer.bbox_x0)
        self.assertNotEqual(vector_layer.bbox_x1, new_vector_layer.bbox_x1)
        self.assertNotEqual(vector_layer.bbox_y0, new_vector_layer.bbox_y0)
        self.assertNotEqual(vector_layer.bbox_y1, new_vector_layer.bbox_y1)

        # test an invalid user without layer replace permission
        self.client.logout()
        self.client.login(username='******', password='******')

        response = self.client.post(
            vector_replace_url, {
                'base_file': layer_base,
                'dbf_file': layer_dbf,
                'shx_file': layer_shx,
                'prj_file': layer_prj
            })
        self.assertEquals(response.status_code, 401)
Exemplo n.º 40
0
    def test_unpublished(self):
        """Test permissions on an unpublished layer
        """
        thefile = os.path.join(gisdata.VECTOR_DATA,
                               'san_andres_y_providencia_highway.shp')
        layer = file_upload(thefile, overwrite=True)
        layer.set_default_permissions()
        check_layer(layer)

        # we need some time to have the service up and running
        time.sleep(20)

        try:
            # request getCapabilities: layer must be there as it is published and
            # advertised: we need to check if in response there is
            # <Name>geonode:san_andres_y_providencia_water</Name>
            geoserver_base_url = settings.OGC_SERVER['default']['LOCATION']
            get_capabilities_url = 'ows?' \
                'service=wms&version=1.3.0&request=GetCapabilities'
            url = urljoin(geoserver_base_url, get_capabilities_url)
            str_to_check = '<Name>geonode:san_andres_y_providencia_highway</Name>'
            request = Request(url)
            response = urlopen(request)

            # by default the uploaded layer is published
            self.assertTrue(layer.is_published, True)
            self.assertTrue(
                any(str_to_check in ensure_string(s)
                    for s in response.readlines()))
        finally:
            # Clean up and completely delete the layer
            layer.delete()

        # with settings disabled
        with self.settings(RESOURCE_PUBLISHING=True):
            layer = file_upload(thefile,
                                overwrite=True,
                                is_approved=False,
                                is_published=False)
            layer.set_default_permissions()
            check_layer(layer)

            # we need some time to have the service up and running
            time.sleep(20)

            try:
                # by default the uploaded layer must be unpublished
                self.assertEqual(layer.is_published, False)

                # check the layer is not in GetCapabilities
                request = Request(url)
                response = urlopen(request)

                # now test with published layer
                layer = Layer.objects.get(pk=layer.pk)
                layer.is_published = True
                layer.save()

                # we need some time to have the service up and running
                time.sleep(20)

                request = Request(url)
                response = urlopen(request)
                self.assertTrue(
                    any(str_to_check in ensure_string(s)
                        for s in response.readlines()))
            finally:
                # Clean up and completely delete the layer
                layer.delete()
Exemplo n.º 41
0
    def test_permissions(self):
        """Test permissions on a layer
        """

        # grab norman
        norman = get_user_model().objects.get(username="******")

        thefile = os.path.join(gisdata.VECTOR_DATA,
                               'san_andres_y_providencia_poi.shp')
        layer = file_upload(thefile, overwrite=True)
        check_layer(layer)

        # we need some time to have the service up and running
        time.sleep(20)

        # Set the layer private for not authenticated users
        layer.set_permissions({'users': {'AnonymousUser': []}})

        url = 'http://localhost:8080/geoserver/geonode/wms?' \
            'LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES=' \
            '&FORMAT=image%2Fpng&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap' \
            '&SRS=EPSG%3A4326' \
            '&BBOX=-81.394599749999,13.316009005566,' \
            '-81.370560451855,13.372728455566' \
            '&WIDTH=217&HEIGHT=512'

        # test view_resourcebase permission on anonymous user
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        self.assertTrue(response.info().getheader('Content-Type'),
                        'application/vnd.ogc.se_xml;charset=UTF-8')

        # test WMS with authenticated user that has not view_resourcebase:
        # the layer must be not accessible (response is xml)
        request = urllib2.Request(url)
        base64string = base64.encodestring(
            '%s:%s' % ('norman', 'norman')).replace('\n', '')
        request.add_header("Authorization", "Basic %s" % base64string)
        response = urllib2.urlopen(request)
        self.assertTrue(response.info().getheader('Content-Type'),
                        'application/vnd.ogc.se_xml;charset=UTF-8')

        # test WMS with authenticated user that has view_resourcebase: the layer
        # must be accessible (response is image)
        assign_perm('view_resourcebase', norman, layer.get_self_resource())
        request = urllib2.Request(url)
        base64string = base64.encodestring(
            '%s:%s' % ('norman', 'norman')).replace('\n', '')
        request.add_header("Authorization", "Basic %s" % base64string)
        response = urllib2.urlopen(request)
        self.assertTrue(response.info().getheader('Content-Type'), 'image/png')

        # test change_layer_data
        # would be nice to make a WFS/T request and test results, but this
        # would work only on PostGIS layers

        # test change_layer_style
        url = 'http://localhost:8000/gs/rest/styles/san_andres_y_providencia_poi.xml'
        sld = """<?xml version="1.0" encoding="UTF-8"?>
<sld:StyledLayerDescriptor xmlns:sld="http://www.opengis.net/sld"
xmlns:gml="http://www.opengis.net/gml" xmlns:ogc="http://www.opengis.net/ogc"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0"
xsi:schemaLocation="http://www.opengis.net/sld http://schemas.opengis.net/sld/1.0.0/StyledLayerDescriptor.xsd">
   <sld:NamedLayer>
      <sld:Name>geonode:san_andres_y_providencia_poi</sld:Name>
      <sld:UserStyle>
         <sld:Name>san_andres_y_providencia_poi</sld:Name>
         <sld:Title>san_andres_y_providencia_poi</sld:Title>
         <sld:IsDefault>1</sld:IsDefault>
         <sld:FeatureTypeStyle>
            <sld:Rule>
               <sld:PointSymbolizer>
                  <sld:Graphic>
                     <sld:Mark>
                        <sld:Fill>
                           <sld:CssParameter name="fill">#8A7700
                           </sld:CssParameter>
                        </sld:Fill>
                        <sld:Stroke>
                           <sld:CssParameter name="stroke">#bbffff
                           </sld:CssParameter>
                        </sld:Stroke>
                     </sld:Mark>
                     <sld:Size>10</sld:Size>
                  </sld:Graphic>
               </sld:PointSymbolizer>
            </sld:Rule>
         </sld:FeatureTypeStyle>
      </sld:UserStyle>
   </sld:NamedLayer>
</sld:StyledLayerDescriptor>"""

        # user without change_layer_style cannot edit it
        self.client.login(username='******', password='******')
        response = self.client.put(url,
                                   sld,
                                   content_type='application/vnd.ogc.sld+xml')
        self.assertEquals(response.status_code, 401)

        # user with change_layer_style can edit it
        assign_perm('change_layer_style', norman, layer)
        response = self.client.put(url,
                                   sld,
                                   content_type='application/vnd.ogc.sld+xml')
        self.assertEquals(response.status_code, 200)

        # Clean up and completely delete the layer
        layer.delete()
Exemplo n.º 42
0
    def test_unpublished(self):
        """Test permissions on an unpublished layer
        """

        thefile = os.path.join(gisdata.VECTOR_DATA,
                               'san_andres_y_providencia_poi.shp')
        layer = file_upload(thefile, overwrite=True)
        layer.set_default_permissions()
        check_layer(layer)

        # we need some time to have the service up and running
        time.sleep(20)

        # request getCapabilities: layer must be there as it is published and
        # advertised: we need to check if in response there is
        # <Name>geonode:san_andres_y_providencia_water</Name>
        url = 'http://localhost:8080/geoserver/ows?' \
            'service=wms&version=1.3.0&request=GetCapabilities'
        str_to_check = '<Name>geonode:san_andres_y_providencia_poi</Name>'
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        self.assertTrue(any(str_to_check in s for s in response.readlines()))

        # by default the uploaded layer is
        self.assertTrue(layer.is_published, True)

        # Clean up and completely delete the layer
        layer.delete()

        # with settings disabled
        with self.settings(RESOURCE_PUBLISHING=True):

            thefile = os.path.join(
                gisdata.VECTOR_DATA,
                'san_andres_y_providencia_administrative.shp')
            layer = file_upload(thefile, overwrite=True)
            layer.set_default_permissions()
            check_layer(layer)

            # we need some time to have the service up and running
            time.sleep(20)

            str_to_check = '<Name>san_andres_y_providencia_administrative</Name>'

            # by default the uploaded layer must be unpublished
            self.assertEqual(layer.is_published, False)

            # check the layer is not in GetCapabilities
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            self.assertFalse(
                any(str_to_check in s for s in response.readlines()))

            # now test with published layer
            resource = layer.get_self_resource()
            resource.is_published = True
            resource.save()

            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            self.assertTrue(
                any(str_to_check in s for s in response.readlines()))

            # Clean up and completely delete the layer
            layer.delete()
Exemplo n.º 43
0
    def testPrintProxy(self):
        """ Test the PrintProxyMiddleware if activated.
            It should respect the permissions on private layers.
        """

        if 'geonode.middleware.PrintProxyMiddleware' in settings.MIDDLEWARE_CLASSES:
            # STEP 1: Import a layer
            from geonode.maps.models import Map

            self.client.login(username='******', password='******')

            # TODO: Would be nice to ensure the name is available before
            # running the test...
            norman = get_user_model().objects.get(username="******")
            saved_layer = file_upload(
                os.path.join(gisdata.VECTOR_DATA,
                             "san_andres_y_providencia_poi.shp"),
                name="san_andres_y_providencia_poi_by_norman",
                user=norman,
                overwrite=True,
            )
            # Set the layer private
            saved_layer.set_permissions(
                {'users': {
                    'AnonymousUser': ['view_resourcebase']
                }})

            url = reverse('layer_metadata',
                          args=[saved_layer.service_typename])

            # check is accessible while logged in
            resp = self.client.get(url)
            self.assertEquals(resp.status_code, 200)

            # check is inaccessible when not logged in
            self.client.logout()
            resp = self.client.get(url)
            self.assertEquals(resp.status_code, 302)

            # STEP 2: Create a Map with that layer

            map_obj = Map(owner=norman, zoom=0, center_x=0, center_y=0)
            map_obj.create_from_layer_list(norman, [saved_layer], 'title', '')

            # STEP 3: Print the map

            print_url = settings.OGC_SERVER['default'][
                'LOCATION'] + 'pdf/create.json'

            post_payload = {
                'dpi':
                75,
                'layers': [{
                    'baseURL':
                    settings.OGC_SERVER['default']['LOCATION'] +
                    'wms?SERVICE=WMS&',
                    'format':
                    "image/png",
                    'customParams': {
                        'TILED': True,
                        'TRANSPARENT': True
                    },
                    'layers': [saved_layer.service_typename],
                    'opacity':
                    1,
                    'singleTile':
                    False,
                    'type':
                    'WMS'
                }],
                'layout':
                'A4 portrait',
                'mapTitle':
                'test',
                'outputFilename':
                'print',
                'srs':
                'EPSG:900913',
                'units':
                'm'
            }

            self.client.post(print_url, post_payload)

            # Test the layer is still inaccessible as non authenticated
            resp = self.client.get(url)
            self.assertEquals(resp.status_code, 302)

        else:
            pass
Exemplo n.º 44
0
    def test_map_thumbnail(self):
        """Creating map will create thumbnail."""
        filename = os.path.join(
            gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
        layer1 = file_upload(filename)

        filename = os.path.join(
            gisdata.GOOD_DATA,
            'vector/san_andres_y_providencia_administrative.shp')
        layer2 = file_upload(filename)
        """:type: geonode.layers.models.Layer"""

        # construct json request for new map
        json_payload = InitialSetup.generate_initial_map(layer1, layer2)

        self.client.login(username='******', password='******')

        response = self.client.post(
            reverse('new_map_json'),
            json.dumps(json_payload),
            content_type='application/json')

        self.assertEqual(response.status_code, 200)

        map_id = json.loads(response.content).get('id')

        map = Map.objects.get(id=map_id)

        # check that we have remote thumbnail
        remote_thumbnail_link = map.link_set.filter(
            name__icontains='remote thumbnail').first()
        self.assertTrue(remote_thumbnail_link.url)

        # thumbnail won't generate because remote thumbnail uses public
        # address

        remote_thumbnail_url = remote_thumbnail_link.url

        # Replace url's basename, we want to access it using django client
        parse_result = urlparse.urlsplit(remote_thumbnail_url)
        remote_thumbnail_url = urlparse.urlunsplit(
            ('', '', parse_result.path, parse_result.query, ''))

        response = self.client.get(remote_thumbnail_url)

        thumbnail_dir = os.path.join(settings.MEDIA_ROOT, 'thumbs')
        thumbnail_path = os.path.join(thumbnail_dir, 'map-thumb.png')

        map.save_thumbnail(thumbnail_path, response.content)

        # Check thumbnail created
        self.assertTrue(os.path.exists(thumbnail_path))
        self.assertEqual(what(thumbnail_path), 'png')

        # Check that now we have thumbnail
        self.assertTrue(map.has_thumbnail())

        missing_thumbnail_url = staticfiles.static(settings.MISSING_THUMBNAIL)

        self.assertTrue(map.get_thumbnail_url() != missing_thumbnail_url)

        thumbnail_links = map.link_set.filter(name__icontains='thumbnail')
        self.assertTrue(len(thumbnail_links) > 0)

        link_names = ['remote thumbnail', 'thumbnail']
        for link in thumbnail_links:
            self.assertIn(link.name.lower(), link_names)

        # cleanup
        map.delete()
        layer1.delete()
        layer2.delete()
Exemplo n.º 45
0
    def test_inasafe_metadata_capture(self):
        """Test capturing InaSAFE metadata from layer's metadata."""
        data_helper = self.data_helper
        """Case 1. Uploading layer that have no InaSAFE metadata"""

        # Uploading base file only should generate basic metadata.
        ash_fall_only = data_helper.misc('ash_fall_base_only.tif')
        ash_fall_only_layer = file_upload(ash_fall_only)

        # There should be no metadata processing so we can just inspect the
        # layer
        metadata_link = ash_fall_only_layer.link_set.metadata().get(name='ISO')
        response = self.client.get(metadata_link.url)

        self.assertEqual(response.status_code, 200)

        metadata_xml = response.content

        # This should be a valid xml
        namespaces = ISO_METADATA_NAMESPACES
        root_metadata_xml = etree.fromstring(metadata_xml)
        identification_info = root_metadata_xml.xpath(
            '//gmd:identificationInfo', namespaces=namespaces)

        self.assertTrue(identification_info)
        inasafe_keywords = root_metadata_xml.xpath(
            ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)

        # Will not find inasafe keywords safely
        self.assertFalse(inasafe_keywords)

        self.assertTrue(metadata_xml)
        """Case 2. Uploading layer that have InaSAFE metadata"""

        # Upload layer with metadata
        ash_fall = data_helper.hazard('ash_fall.tif')
        ash_fall_layer = file_upload(ash_fall)

        # Wait for InaSAFE to process metadata
        wait_metadata(ash_fall_layer)

        # Validate internal model
        ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)

        self.assertTrue(ash_fall_layer.inasafe_metadata)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_xml)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_json)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords)
        self.assertEqual(ash_fall_layer.inasafe_metadata.layer_purpose,
                         'hazard')
        self.assertEqual(ash_fall_layer.inasafe_metadata.category,
                         'volcanic_ash')

        # Fetch metadata from URL
        metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
        response = self.client.get(metadata_link.url)

        self.assertEqual(response.status_code, 200)

        metadata_xml = response.content

        # This should be a valid xml
        namespaces = ISO_METADATA_NAMESPACES
        root_metadata_xml = etree.fromstring(metadata_xml)
        identification_info = root_metadata_xml.xpath(
            '//gmd:identificationInfo', namespaces=namespaces)

        self.assertTrue(identification_info)
        inasafe_keywords = root_metadata_xml.xpath(
            ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)

        # Now it should find InaSAFE metadata
        self.assertTrue(inasafe_keywords)

        # Xml in metadata page and model should be the same structurally
        pattern = r'<inasafe [^<]*>'
        replace = '<inasafe xmlns:gmd="http://www.isotc211.org/2005/gmd" ' \
                  'xmlns:gco="http://www.isotc211.org/2005/gco">'
        link_xml = etree.tostring(inasafe_keywords[0],
                                  pretty_print=True,
                                  xml_declaration=False)
        model_xml = etree.tostring(etree.fromstring(
            ash_fall_layer.inasafe_metadata.keywords_xml),
                                   pretty_print=True,
                                   xml_declaration=False)

        # We replace XML header so it will have the same header
        link_xml = re.sub(pattern, replace, link_xml).strip()
        model_xml = re.sub(pattern, replace, model_xml).strip()

        # Match XML representations
        self.assertXMLEqual(link_xml, model_xml)

        # QGIS XML file should have matched string representation
        with codecs.open(ash_fall_layer.qgis_layer.xml_path,
                         encoding='utf-8') as f:
            qgis_xml = f.read()

        self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)
        """Case 3. Test that uploading new metadata will also update
        InaSAFE metadata. It will be gone if there is no InaSAFE metadata
        """

        # Upload metadata with no InaSAFE keywords
        previous_inasafe_keywords = inasafe_keywords[0]
        inasafe_keywords[0].getparent().remove(inasafe_keywords[0])
        __, metadata_filename = tempfile.mkstemp('.xml')

        with codecs.open(metadata_filename, mode='w', encoding='utf-8') as f:
            f.write(etree.tostring(root_metadata_xml))

        ash_fall_layer = file_upload(metadata_filename,
                                     metadata_upload_form=True)

        # We do not wait for InaSAFE to process metadata
        # Supposedly, there's no metadata

        # Validate internal model
        ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)

        self.assertTrue(ash_fall_layer.inasafe_metadata)
        # Now there is no InaSAFE keywords
        self.assertFalse(ash_fall_layer.inasafe_metadata.keywords_xml)
        self.assertFalse(ash_fall_layer.inasafe_metadata.keywords_json)
        self.assertFalse(ash_fall_layer.inasafe_metadata.keywords)
        self.assertFalse(ash_fall_layer.inasafe_metadata.layer_purpose)
        self.assertFalse(ash_fall_layer.inasafe_metadata.category)

        # Metadata download will not show InaSAFE keywords
        metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
        response = self.client.get(metadata_link.url)

        self.assertEqual(response.status_code, 200)

        metadata_xml = response.content

        root_metadata_xml = etree.fromstring(metadata_xml)
        inasafe_keywords = root_metadata_xml.xpath(
            ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)

        self.assertFalse(inasafe_keywords)

        # QGIS XML file should have matched string representation
        with codecs.open(ash_fall_layer.qgis_layer.xml_path,
                         encoding='utf-8') as f:
            qgis_xml = f.read()

        self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)
        """Case 4. Test that uploading new metadata will also update
        InaSAFE metadata. Upload new metadata with InaSAFE keywords.
        """

        # Change some detail in the metadata
        inasafe_keywords = previous_inasafe_keywords
        keyword_version = inasafe_keywords.xpath(
            'keyword_version/gco:CharacterString', namespaces=namespaces)[0]

        self.assertEqual(keyword_version.text, '4.3')
        # Change keyword version
        keyword_version.text = '4.4'

        # Attach InaSAFE keywords to the new metadata
        supplemental_info = root_metadata_xml.xpath(
            '//gmd:supplementalInformation', namespaces=namespaces)[0]
        supplemental_info.append(inasafe_keywords)

        # Upload metadata with new InaSAFE keywords
        with codecs.open(metadata_filename, mode='w', encoding='utf-8') as f:
            f.write(etree.tostring(root_metadata_xml))

        ash_fall_layer = file_upload(metadata_filename,
                                     metadata_upload_form=True)

        # Wait for InaSAFE to process metadata
        wait_metadata(ash_fall_layer)

        # Validate internal model
        ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)

        self.assertTrue(ash_fall_layer.inasafe_metadata)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_xml)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_json)
        self.assertTrue(ash_fall_layer.inasafe_metadata.keywords)
        self.assertEqual(ash_fall_layer.inasafe_metadata.layer_purpose,
                         'hazard')
        self.assertEqual(ash_fall_layer.inasafe_metadata.category,
                         'volcanic_ash')

        # Fetch metadata from URL
        metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
        response = self.client.get(metadata_link.url)

        self.assertEqual(response.status_code, 200)

        metadata_xml = response.content

        # This should be a valid xml
        namespaces = ISO_METADATA_NAMESPACES
        root_metadata_xml = etree.fromstring(metadata_xml)
        inasafe_keywords = root_metadata_xml.xpath(
            ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)

        # Now it should find InaSAFE metadata
        self.assertTrue(inasafe_keywords)

        inasafe_keywords = inasafe_keywords[0]

        # Check new keywords version
        keyword_version = inasafe_keywords.xpath(
            '//keyword_version/gco:CharacterString', namespaces=namespaces)[0]

        self.assertEqual(keyword_version.text, '4.4')

        # QGIS XML file should have matched string representation
        with codecs.open(ash_fall_layer.qgis_layer.xml_path,
                         encoding='utf-8') as f:
            qgis_xml = f.read()

        self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)

        os.remove(metadata_filename)
        ash_fall_only_layer.delete()
        ash_fall_layer.delete()
Exemplo n.º 46
0
    def test_add_delete_style(self):
        """Test add new style using qgis_server views."""
        filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
        layer = file_upload(filename)
        """:type: geonode.layers.models.Layer"""

        self.client.login(username='******', password='******')

        qml_path = self.data_path('test_grid.qml')
        add_style_url = reverse(
            'qgis_server:upload-qml',
            kwargs={
                'layername': layer.name})

        with open(qml_path) as file_handle:
            form_data = {
                'name': 'new_style',
                'title': 'New Style',
                'qml': file_handle
            }
            response = self.client.post(
                add_style_url,
                data=form_data)

        self.assertEqual(response.status_code, 201)

        actual_list_style = style_list(layer, internal=False)
        if actual_list_style:
            expected_list_style = ['default', 'new_style']
            self.assertEqual(
                set(expected_list_style),
                set([style.name for style in actual_list_style]))

        # Test delete request
        delete_style_url = reverse(
            'qgis_server:remove-qml',
            kwargs={
                'layername': layer.name,
                'style_name': 'default'})

        response = self.client.delete(delete_style_url)
        self.assertEqual(response.status_code, 200)

        actual_list_style = style_list(layer, internal=False)
        if actual_list_style:
            expected_list_style = ['new_style']
            self.assertEqual(
                set(expected_list_style),
                set([style.name for style in actual_list_style]))

        # Check new default
        default_style_url = reverse(
            'qgis_server:default-qml',
            kwargs={
                'layername': layer.name})

        response = self.client.get(default_style_url)

        self.assertEqual(response.status_code, 200)
        expected_default_style_retval = {
            'name': 'new_style',
        }
        actual_default_style_retval = json.loads(response.content)

        for key, value in expected_default_style_retval.iteritems():
            self.assertEqual(actual_default_style_retval[key], value)

        layer.delete()
Exemplo n.º 47
0
    def test_layercount_strategy(self):
        """
        Test layercount SHARD_STRATEGY.
        """
        settings.SHARD_STRATEGY = 'layercount'

        owner = get_user_model().objects.get(username="******")
        layers_to_upload = ('layer_01', 'layer_02', 'layer_03', 'layer_04',
                            'layer_05')
        for layer in layers_to_upload:
            print 'Uploading layer %s...' % layer
            saved_layer = file_upload(
                os.path.join(gisdata.VECTOR_DATA,
                             'san_andres_y_providencia_poi.shp'),
                name=layer,
                user=owner,
                overwrite=True,
            )
            saved_layer.set_default_permissions()

        # check layers in geoserver if they have correct store type and datasource
        cat = gs_catalog
        i = 0
        for layer in layers_to_upload:
            i += 1
            gs_layer = cat.get_layer(layer)
            self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
            if i in (1, 2):
                self.assertEqual(gs_layer.resource.store.name,
                                 '%s00000' % SHARD_PREFIX)
            if i in (3, 4):
                self.assertEqual(gs_layer.resource.store.name,
                                 '%s00001' % SHARD_PREFIX)
            if i == 5:
                self.assertEqual(gs_layer.resource.store.name,
                                 '%s00002' % SHARD_PREFIX)

        # remove one layer and see what happens when adding a new one
        layer = Layer.objects.get(name='layer_05')
        layer.delete()

        saved_layer = file_upload(
            os.path.join(gisdata.VECTOR_DATA,
                         'san_andres_y_providencia_poi.shp'),
            name='layer_06',
            user=owner,
            overwrite=True,
        )
        saved_layer.set_default_permissions()

        gs_layer = cat.get_layer('layer_06')
        self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
        self.assertEqual(gs_layer.resource.store.name,
                         '%s00002' % SHARD_PREFIX)

        # check Database objects and layers_count
        self.assertEqual(
            Database.objects.filter(name__contains=('%s0000' %
                                                    SHARD_PREFIX)).count(), 3)
        self.assertEqual(
            Database.objects.get(name='%s00000' % SHARD_PREFIX).layers_count,
            2)
        self.assertEqual(
            Database.objects.get(name='%s00001' % SHARD_PREFIX).layers_count,
            2)
        self.assertEqual(
            Database.objects.get(name='%s00002' % SHARD_PREFIX).layers_count,
            1)

        # remove layers
        layers_to_delete = ('layer_01', 'layer_02', 'layer_03', 'layer_04',
                            'layer_06')
        for layer_name in layers_to_delete:
            layer = Layer.objects.get(name=layer_name)
            layer.delete()

        for i in (0, 1, 2):
            self.assertEqual(
                Database.objects.get(name='%s0000%s' %
                                     (SHARD_PREFIX, i)).layers_count, 0)
Exemplo n.º 48
0
def add_wfs_layer(
        endpoint,
        version,
        typename,
        metadata_string=None,
        title=None,
        bbox=None,
        user=None,
        password=None):
    endpoint_parsed = urlparse.urlparse(endpoint)
    q_dict = {
        'version': version,
        'typename': typename,
        'outputFormat': 'shape-zip',
        'request': 'GetFeature',
        'service': 'WFS',
        'srsName': 'EPSG:4326',
    }
    if bbox:
        q_dict['bbox'] = ','.join(bbox)
    parsed_url = urlparse.ParseResult(
        scheme=endpoint_parsed.scheme,
        netloc=endpoint_parsed.netloc,
        path=endpoint_parsed.path,
        params=None,
        query=urllib.urlencode(q_dict),
        fragment=None
    )
    tmpfile = download_file(parsed_url.geturl(), user=user, password=password)

    # args = [
    #     'ogr2ogr',
    #     '-nlt POLYGON',
    #     '-skipfailures',
    #     '%s.shp' % tmpfile,
    #     tmpfile,
    #     'OGRGeoJSON'
    # ]
    #
    # retval = subprocess.call(args)

    # # get metadata file
    # if metadata_string:
    #     if not isinstance(metadata_string, unicode):
    #         metadata_string = unicode(metadata_string, 'utf-8')
    #     metadata_file = '%s.xml' % tmpfile
    #     with io.open(metadata_file, mode='w', encoding='utf-8') as f:
    #         f.write(metadata_string)
    #
    # saved_layer = None
    # if retval == 0:
    #     saved_layer = file_upload(
    #         '%s.shp' % tmpfile,
    #         overwrite=True)
    #     saved_layer.set_default_permissions()
    #     saved_layer.title = title or typename
    #     saved_layer.save()
    #
    # # cleanup
    # dir_name = os.path.dirname(tmpfile)
    # for root, dirs, files in os.walk(dir_name):
    #     for f in files:
    #         if tmpfile in f:
    #             try:
    #                 os.remove(os.path.join(root, f))
    #             except:
    #                 pass

    dir_name = os.path.dirname(tmpfile)
    saved_layer = None
    metadata_file = None

    with ZipFile(tmpfile) as zf:
        zf.extractall(path=dir_name)
        for name in zf.namelist():
            basename, ext = os.path.splitext(name)
            if '.shp' in ext:
                # get metadata file
                if metadata_string:
                    if not isinstance(metadata_string, unicode):
                        metadata_string = unicode(metadata_string, 'utf-8')
                    metadata_file = '%s.xml' % basename
                    metadata_file = os.path.join(dir_name, metadata_file)
                    metadata_string = cleanup_metadata(metadata_string)
                    with io.open(metadata_file, mode='w',
                                 encoding='utf-8') as f:
                        f.write(metadata_string)

                # process shapefile layer
                saved_layer = file_upload(
                    os.path.join(dir_name, name),
                    overwrite=True)
                saved_layer.set_default_permissions()
                saved_layer.title = title or typename
                saved_layer.save()
                break

        # cleanup
        for name in zf.namelist():
            filepath = os.path.join(dir_name, name)
            try:
                os.remove(filepath)
            except:
                pass

        if metadata_file:
            try:
                os.remove(metadata_file)
            except:
                pass

    # cleanup
    try:
        os.remove(tmpfile)
    except:
        pass
    return saved_layer
Exemplo n.º 49
0
    def test_monthly_strategy(self, mock_obj):
        """
        Test monthly SHARD_STRATEGY.
        """
        settings.SHARD_STRATEGY = 'monthly'

        # testing if layer is created in 197102 shard
        global MONTH
        MONTH = 2

        owner = get_user_model().objects.get(username="******")
        saved_layer = file_upload(
            os.path.join(gisdata.VECTOR_DATA,
                         'san_andres_y_providencia_poi.shp'),
            name='layer_a',
            user=owner,
            overwrite=True,
        )
        saved_layer.set_default_permissions()

        cat = gs_catalog
        gs_layer = cat.get_layer('layer_a')
        self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
        self.assertEqual(gs_layer.resource.store.name,
                         '%s197102' % SHARD_PREFIX)

        # afer one month
        # testing if layer is created in 197103 shard
        global MONTH
        MONTH = 3
        saved_layer = file_upload(
            os.path.join(gisdata.VECTOR_DATA,
                         'san_andres_y_providencia_poi.shp'),
            name='layer_b',
            user=owner,
            overwrite=True,
        )
        saved_layer.set_default_permissions()

        gs_layer = cat.get_layer('layer_b')
        self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
        self.assertEqual(gs_layer.resource.store.name,
                         '%s197103' % SHARD_PREFIX)

        # check Database objects
        self.assertEqual(
            Database.objects.filter(name__contains=('%s1971' %
                                                    SHARD_PREFIX)).count(), 2)
        self.assertEqual(
            Database.objects.get(name='%s197102' % SHARD_PREFIX).layers_count,
            1)
        self.assertEqual(
            Database.objects.get(name='%s197103' % SHARD_PREFIX).layers_count,
            1)

        # remove the layers and see what happens
        for layer_name in ('layer_a', 'layer_b'):
            layer = Layer.objects.get(name=layer_name)
            layer.delete()

        self.assertEqual(
            Database.objects.get(name='%s197102' % SHARD_PREFIX).layers_count,
            0)
        self.assertEqual(
            Database.objects.get(name='%s197103' % SHARD_PREFIX).layers_count,
            0)
Exemplo n.º 50
0
    def test_style_management_url(self):
        """Test QGIS Server style management url construction."""
        filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
        uploaded = file_upload(filename)

        # Get default style
        # There will always be a default style when uploading a layer
        style_url = style_get_url(uploaded, 'default', internal=True)

        response = requests.get(style_url)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.headers.get('Content-Type'), 'text/xml')

        # it has to contains qgis tags
        style_xml = dlxml.fromstring(response.content)
        self.assertTrue('qgis' in style_xml.tag)

        # Add new style
        # change default style slightly
        self.assertTrue('WhiteToBlack' not in response.content)
        self.assertTrue('BlackToWhite' in response.content)
        new_style_xml = dlxml.fromstring(
            response.content.replace('BlackToWhite', 'WhiteToBlack'))
        new_xml_content = etree.tostring(new_style_xml, pretty_print=True)

        # save it to qml file, accessible by qgis server
        qgis_layer = QGISServerLayer.objects.get(layer=uploaded)
        with open(qgis_layer.qml_path, mode='w') as f:
            f.write(new_xml_content)

        style_url = style_add_url(uploaded, 'new_style', internal=True)

        response = requests.get(style_url)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, 'OK')

        # Get style list
        qml_styles = style_list(uploaded, internal=False)
        if qml_styles:
            expected_style_names = ['default', 'new_style']
            actual_style_names = [s.name for s in qml_styles]
            self.assertEqual(
                set(expected_style_names),
                set(actual_style_names))

        # Get new style
        style_url = style_get_url(uploaded, 'new_style', internal=True)

        response = requests.get(style_url)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.headers.get('Content-Type'), 'text/xml')
        self.assertTrue('WhiteToBlack' in response.content)

        # Set default style
        style_url = style_set_default_url(
            uploaded, 'new_style', internal=True)

        response = requests.get(style_url)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, 'OK')

        # Remove style
        style_url = style_remove_url(uploaded, 'new_style', internal=True)

        response = requests.get(style_url)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, 'OK')

        # Cleanup
        uploaded.delete()
Exemplo n.º 51
0
def layer_upload(request, template='upload/layer_upload.html'):
    if request.method == 'GET':
        ctx = {'charsets': CHARSETS}
        return render_to_response(template, RequestContext(request, ctx))
    elif request.method == 'POST':
        form = NewLayerUploadForm(request.POST, request.FILES)
        tempdir = None
        errormsgs = []
        out = {'success': False}

        if form.is_valid():
            title = form.cleaned_data["layer_title"]

            # Replace dots in filename - GeoServer REST API upload bug
            # and avoid any other invalid characters.
            # Use the title if possible, otherwise default to the filename
            if title is not None and len(title) > 0:
                name_base = title
            else:
                name_base, __ = os.path.splitext(
                    form.cleaned_data["base_file"].name)

            name = slugify(name_base.replace(".", "_"))

            try:
                # Moved this inside the try/except block because it can raise
                # exceptions when unicode characters are present.
                # This should be followed up in upstream Django.
                tempdir, base_file = form.write_files()
                saved_layer = file_upload(
                    base_file,
                    name=name,
                    user=request.user,
                    overwrite=False,
                    charset=form.cleaned_data["charset"],
                    abstract=form.cleaned_data["abstract"],
                    title=form.cleaned_data["layer_title"],
                )

            except Exception as e:
                logger.exception(e)
                out['success'] = False
                out['errors'] = str(e)
            else:
                out['success'] = True
                out['url'] = reverse('layer_detail',
                                     args=[saved_layer.service_typename])

                permissions = form.cleaned_data["permissions"]
                if permissions is not None and len(permissions.keys()) > 0:
                    saved_layer.set_permissions(permissions)

            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            for e in form.errors.values():
                errormsgs.extend([escape(v) for v in e])

            out['errors'] = form.errors
            out['errormsgs'] = errormsgs

        if out['success']:
            status_code = 200
        else:
            status_code = 500
        return HttpResponse(json.dumps(out),
                            mimetype='application/json',
                            status=status_code)
Exemplo n.º 52
0
def _save_layer(layer_dict,
                base_file,
                user,
                tempdir=None,
                overwrite=False,
                keywords=[]):
    """Thin wrapper around file_upload."""
    out = {}
    title = layer_dict.get("layer_title", "")

    # Replace dots in filename - GeoServer REST API upload bug
    # and avoid any other invalid characters.
    # Use the title if possible, otherwise default to the filename
    if title is not None and len(title) > 0:
        name_base = title
    else:
        name_base, __ = os.path.splitext(base_file)

    name = slugify(name_base.replace(".", "_"))

    try:
        # Moved this inside the try/except block because it can raise
        # exceptions when unicode characters are present.
        # This should be followed up in upstream Django.
        saved_layer = file_upload(base_file,
                                  name=name,
                                  user=user,
                                  style_name=layer_dict.get(
                                      "style_name", None),
                                  overwrite=overwrite,
                                  charset=layer_dict.get("charset", "UTF-8"),
                                  abstract=layer_dict.get("abstract", ""),
                                  title=title,
                                  keywords=layer_dict.get("keywords", []),
                                  date=layer_dict.get("date", None))

    except Exception as e:
        exception_type, error, tb = sys.exc_info()
        logger.exception(e)
        out['success'] = False
        out['errors'] = str(error)
        # Assign the error message to the latest UploadSession from that user.
        latest_uploads = UploadSession.objects.filter(
            user=user).order_by('-date')
        if latest_uploads.count() > 0:
            upload_session = latest_uploads[0]
            upload_session.error = str(error)
            upload_session.traceback = traceback.format_exc(tb)
            upload_session.context = log_snippet(CONTEXT_LOG_FILE)
            upload_session.save()
            out['traceback'] = upload_session.traceback
            out['context'] = upload_session.context
            out['upload_session'] = upload_session.id
    else:
        out['success'] = True
        if hasattr(saved_layer, 'info'):
            out['info'] = saved_layer.info
        out['url'] = reverse('layer_detail',
                             args=[saved_layer.service_typename])
        out["layer"] = saved_layer

        upload_session = saved_layer.upload_session
        upload_session.processed = True
        upload_session.save()
        permissions = layer_dict["permissions"]
        if permissions is not None and len(permissions.keys()) > 0:
            saved_layer.set_permissions(permissions)

    finally:
        if tempdir is not None:
            shutil.rmtree(tempdir)
    return out
Exemplo n.º 53
0
    def test_run_analysis_no_aggregation(self):
        """Test running analysis without aggregation."""
        data_helper = InaSAFETestData()
        flood = data_helper.hazard('flood_data.geojson')
        buildings = data_helper.exposure('buildings.geojson')

        flood_layer = file_upload(flood)
        # Wait until metadata is read
        wait_metadata(flood_layer)

        buildings_layer = file_upload(buildings)
        # Wait until metadata is read
        wait_metadata(buildings_layer)

        # Check layer uploaded
        self.assertEqual(Layer.objects.filter(id=flood_layer.id).count(), 1)
        self.assertEqual(
            Layer.objects.filter(id=buildings_layer.id).count(), 1)

        form_data = {
            'user': AnonymousUser(),
            'user_title': 'Flood on Buildings custom Analysis',
            'exposure_layer': buildings_layer.id,
            'hazard_layer': flood_layer.id,
            'keep': False,
            'extent_option': Analysis.HAZARD_EXPOSURE_CODE
        }

        form = AnalysisCreationForm(form_data, user=form_data['user'])

        if not form.is_valid():
            LOGGER.debug(form.errors)

        self.assertTrue(form.is_valid())
        analysis = form.save()
        """:type: geosafe.models.Analysis"""

        while analysis.get_task_state() == 'PENDING':
            analysis.refresh_from_db()
            time.sleep(1)

        if analysis.get_task_result().failed():
            LOGGER.debug(analysis.get_task_result().traceback)

        self.assertTrue(analysis.get_task_result().successful())
        self.assertEqual(analysis.get_task_state(), 'SUCCESS')

        while not analysis.impact_layer:
            analysis.refresh_from_db()
            time.sleep(1)

        impact_layer = analysis.impact_layer
        wait_metadata(impact_layer)

        LOGGER.debug('Layers: {0}'.format(Layer.objects.all()))

        self.assertEqual(impact_layer.inasafe_metadata.layer_purpose,
                         'impact_analysis')

        flood_layer.delete()
        buildings_layer.delete()
        impact_layer.delete()
Exemplo n.º 54
0
def layer_upload(request, template='upload/layer_upload.html'):
    if request.method == 'GET':
        mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
        ctx = {
            'mosaics': mosaics,
            'charsets': CHARSETS,
            'is_layer': True,
        }
        return render_to_response(template, RequestContext(request, ctx))
    elif request.method == 'POST':
        form = NewLayerUploadForm(request.POST, request.FILES)
        tempdir = None
        errormsgs = []
        out = {'success': False}
        if form.is_valid():
            title = form.cleaned_data["layer_title"]
            # Replace dots in filename - GeoServer REST API upload bug
            # and avoid any other invalid characters.
            # Use the title if possible, otherwise default to the filename
            if title is not None and len(title) > 0:
                name_base = title
            else:
                name_base, __ = os.path.splitext(
                    form.cleaned_data["base_file"].name)
            name = slugify(name_base.replace(".", "_"))
            try:
                # Moved this inside the try/except block because it can raise
                # exceptions when unicode characters are present.
                # This should be followed up in upstream Django.
                tempdir, base_file = form.write_files()
                saved_layer = file_upload(
                    base_file,
                    name=name,
                    user=request.user,
                    overwrite=False,
                    charset=form.cleaned_data["charset"],
                    #abstract=form.cleaned_data["abstract"],
                    abstract=" ",
                    title=form.cleaned_data["layer_title"],
                    metadata_uploaded_preserve=form.
                    cleaned_data["metadata_uploaded_preserve"])
            except Exception as e:
                exception_type, error, tb = sys.exc_info()
                logger.exception(e)
                out['success'] = False
                out['errors'] = str(error)
                # Assign the error message to the latest UploadSession from that user.
                latest_uploads = UploadSession.objects.filter(
                    user=request.user).order_by('-date')
                if latest_uploads.count() > 0:
                    upload_session = latest_uploads[0]
                    upload_session.error = str(error)
                    upload_session.traceback = traceback.format_exc(tb)
                    upload_session.context = log_snippet(CONTEXT_LOG_FILE)
                    upload_session.save()
                    out['traceback'] = upload_session.traceback
                    out['context'] = upload_session.context
                    out['upload_session'] = upload_session.id
            else:
                out['success'] = True
                if hasattr(saved_layer, 'info'):
                    out['info'] = saved_layer.info
                out['url'] = reverse('layer_detail',
                                     args=[saved_layer.service_typename])
                upload_session = saved_layer.upload_session
                upload_session.processed = True
                upload_session.save()
                permissions = form.cleaned_data["permissions"]
                if permissions is not None and len(permissions.keys()) > 0:
                    saved_layer.set_permissions(permissions)
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            for e in form.errors.values():
                errormsgs.extend([escape(v) for v in e])
            out['errors'] = form.errors
            out['errormsgs'] = errormsgs
        if out['success']:
            status_code = 200
        else:
            status_code = 400
        return HttpResponse(json.dumps(out),
                            content_type='application/json',
                            status=status_code)
Exemplo n.º 55
0
def layer_upload(request, template='upload/layer_upload.html'):
    if request.method == 'GET':
        mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
        ctx = {
            'mosaics': mosaics,
            'charsets': CHARSETS,
            'is_layer': True,
        }
        return render_to_response(template, RequestContext(request, ctx))
    elif request.method == 'POST':
        form = NewLayerUploadForm(request.POST, request.FILES)
        tempdir = None
        saved_layer = None
        errormsgs = []
        out = {'success': False}
        if form.is_valid():
            title = form.cleaned_data["layer_title"]

            # Replace dots in filename - GeoServer REST API upload bug
            # and avoid any other invalid characters.
            # Use the title if possible, otherwise default to the filename
            if title is not None and len(title) > 0:
                name_base = title
            else:
                name_base, __ = os.path.splitext(
                    form.cleaned_data["base_file"].name)
                title = slugify(name_base.replace(".", "_"))
            name = slugify(name_base.replace(".", "_"))

            if form.cleaned_data["abstract"] is not None and len(form.cleaned_data["abstract"]) > 0:
                abstract = form.cleaned_data["abstract"]
            else:
                abstract = "No abstract provided."

            try:
                # Moved this inside the try/except block because it can raise
                # exceptions when unicode characters are present.
                # This should be followed up in upstream Django.
                tempdir, base_file = form.write_files()
                if not form.cleaned_data["style_upload_form"]:
                    saved_layer = file_upload(
                        base_file,
                        name=name,
                        user=request.user,
                        overwrite=False,
                        charset=form.cleaned_data["charset"],
                        abstract=abstract,
                        title=title,
                        metadata_uploaded_preserve=form.cleaned_data[
                            "metadata_uploaded_preserve"],
                        metadata_upload_form=form.cleaned_data["metadata_upload_form"])
                else:
                    saved_layer = Layer.objects.get(alternate=title)
                    if not saved_layer:
                        msg = 'Failed to process.  Could not find matching layer.'
                        raise Exception(msg)
                    sld = open(base_file).read()
                    # Check SLD is valid
                    extract_name_from_sld(gs_catalog, sld, sld_file=base_file)

                    match = None
                    styles = list(saved_layer.styles.all()) + [
                        saved_layer.default_style]
                    for style in styles:
                        if style and style.name == saved_layer.name:
                            match = style
                            break
                    cat = gs_catalog
                    layer = cat.get_layer(title)
                    if match is None:
                        try:
                            cat.create_style(saved_layer.name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE)
                            style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
                                cat.get_style(saved_layer.name)
                            if layer and style:
                                layer.default_style = style
                                cat.save(layer)
                                saved_layer.default_style = save_style(style)
                        except Exception as e:
                            logger.exception(e)
                    else:
                        style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
                            cat.get_style(saved_layer.name)
                        # style.update_body(sld)
                        try:
                            cat.create_style(saved_layer.name, sld, overwrite=True, raw=True,
                                             workspace=settings.DEFAULT_WORKSPACE)
                            style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
                                cat.get_style(saved_layer.name)
                            if layer and style:
                                layer.default_style = style
                                cat.save(layer)
                                saved_layer.default_style = save_style(style)
                        except Exception as e:
                            logger.exception(e)

                    # Invalidate GeoWebCache for the updated resource
                    _invalidate_geowebcache_layer(saved_layer.alternate)

            except Exception as e:
                exception_type, error, tb = sys.exc_info()
                logger.exception(e)
                out['success'] = False
                out['errors'] = str(error)
                # Assign the error message to the latest UploadSession from
                # that user.
                latest_uploads = UploadSession.objects.filter(
                    user=request.user).order_by('-date')
                if latest_uploads.count() > 0:
                    upload_session = latest_uploads[0]
                    upload_session.error = str(error)
                    upload_session.traceback = traceback.format_exc(tb)
                    upload_session.context = log_snippet(CONTEXT_LOG_FILE)
                    upload_session.save()
                    out['traceback'] = upload_session.traceback
                    out['context'] = upload_session.context
                    out['upload_session'] = upload_session.id
            else:
                out['success'] = True
                if hasattr(saved_layer, 'info'):
                    out['info'] = saved_layer.info
                out['url'] = reverse(
                    'layer_detail', args=[
                        saved_layer.service_typename])
                if hasattr(saved_layer, 'bbox_string'):
                    out['bbox'] = saved_layer.bbox_string
                if hasattr(saved_layer, 'srid'):
                    out['crs'] = {
                        'type': 'name',
                        'properties': saved_layer.srid
                    }
                out['ogc_backend'] = settings.OGC_SERVER['default']['BACKEND']
                upload_session = saved_layer.upload_session
                if upload_session:
                    upload_session.processed = True
                    upload_session.save()
                permissions = form.cleaned_data["permissions"]
                if permissions is not None and len(permissions.keys()) > 0:
                    saved_layer.set_permissions(permissions)
                saved_layer.handle_moderated_uploads()
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            for e in form.errors.values():
                errormsgs.extend([escape(v) for v in e])
            out['errors'] = form.errors
            out['errormsgs'] = errormsgs
        if out['success']:
            status_code = 200
        else:
            status_code = 400
        if settings.MONITORING_ENABLED:
            request.add_resource('layer', saved_layer.alternate if saved_layer else name)
        return HttpResponse(
            json.dumps(out),
            content_type='application/json',
            status=status_code)
Exemplo n.º 56
0
def layer_replace(request, layername, template='layers/layer_replace.html'):
    layer = _resolve_layer(
        request,
        layername,
        'base.change_resourcebase',
        _PERMISSION_MSG_MODIFY)

    if request.method == 'GET':
        ctx = {
            'charsets': CHARSETS,
            'layer': layer,
            'is_featuretype': layer.is_vector(),
            'is_layer': True,
        }
        return render_to_response(template,
                                  RequestContext(request, ctx))
    elif request.method == 'POST':

        form = LayerUploadForm(request.POST, request.FILES)
        tempdir = None
        out = {}

        if form.is_valid():
            try:
                tempdir, base_file = form.write_files()
                if layer.is_vector() and is_raster(base_file):
                    out['success'] = False
                    out['errors'] = _(
                        "You are attempting to replace a vector layer with a raster.")
                elif (not layer.is_vector()) and is_vector(base_file):
                    out['success'] = False
                    out['errors'] = _(
                        "You are attempting to replace a raster layer with a vector.")
                else:
                    if check_ogc_backend(geoserver.BACKEND_PACKAGE):
                        # delete geoserver's store before upload
                        cat = gs_catalog
                        cascading_delete(cat, layer.typename)
                        out['ogc_backend'] = geoserver.BACKEND_PACKAGE
                    elif check_ogc_backend(qgis_server.BACKEND_PACKAGE):
                        try:
                            qgis_layer = QGISServerLayer.objects.get(
                                layer=layer)
                            qgis_layer.delete()
                        except QGISServerLayer.DoesNotExist:
                            pass
                        out['ogc_backend'] = qgis_server.BACKEND_PACKAGE

                    saved_layer = file_upload(
                        base_file,
                        name=layer.name,
                        user=request.user,
                        overwrite=True,
                        charset=form.cleaned_data["charset"],
                    )
                    out['success'] = True
                    out['url'] = reverse(
                        'layer_detail', args=[
                            saved_layer.service_typename])
            except Exception as e:
                logger.exception(e)
                tb = traceback.format_exc()
                out['success'] = False
                out['errors'] = str(tb)
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            errormsgs = []
            for e in form.errors.values():
                errormsgs.append([escape(v) for v in e])

            out['errors'] = form.errors
            out['errormsgs'] = errormsgs

        if out['success']:
            status_code = 200
        else:
            status_code = 400
        return HttpResponse(
            json.dumps(out),
            content_type='application/json',
            status=status_code)
Exemplo n.º 57
0
def layer_upload(request, template='upload/layer_upload.html'):
    if request.method == 'GET':
        mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
        ctx = {
            'mosaics':
            mosaics,
            'charsets':
            CHARSETS,
            'is_layer':
            True,
            'allowed_file_types': ['.cst', '.dbf', '.prj', '.shp', '.shx'],
            'categories':
            TopicCategory.objects.all(),
            'organizations':
            GroupProfile.objects.filter(groupmember__user=request.user),
            'user_organization':
            GroupProfile.objects.filter(
                groupmember__user=request.user).first()
        }
        return render_to_response(template, RequestContext(request, ctx))
    elif request.method == 'POST':
        form = NewLayerUploadForm(request.POST, request.FILES)
        tempdir = None
        errormsgs = []
        out = {'success': False}
        if form.is_valid():
            title = form.cleaned_data["layer_title"]
            category = form.cleaned_data["category"]
            organization_id = form.cleaned_data["organization"]
            admin_upload = form.cleaned_data["admin_upload"]
            group = GroupProfile.objects.get(id=organization_id)
            # Replace dots in filename - GeoServer REST API upload bug
            # and avoid any other invalid characters.
            # Use the title if possible, otherwise default to the filename
            if title is not None and len(title) > 0:
                name_base = title
                keywords = title.split()
            else:
                name_base, __ = os.path.splitext(
                    form.cleaned_data["base_file"].name)
                keywords = name_base.split()
            ignore_keys = KeywordIgnoreListModel.objects.values_list('key',
                                                                     flat=True)
            keyword_list = []
            for key in keywords:
                if key not in ignore_keys and not key.isdigit() and any(
                        c.isalpha() for c in key) and len(key) > 2:
                    keyword_list.append(key)

            keywords = keyword_list

            name = slugify(name_base.replace(".", "_"))
            try:
                # Moved this inside the try/except block because it can raise
                # exceptions when unicode characters are present.
                # This should be followed up in upstream Django.
                tempdir, base_file, file_type = form.write_files()
                saved_layer = file_upload(
                    base_file,
                    name=name,
                    user=request.user,
                    category=category,
                    group=group,
                    keywords=keywords,
                    overwrite=False,
                    charset=form.cleaned_data["charset"],
                    abstract=form.cleaned_data["abstract"],
                    title=form.cleaned_data["layer_title"],
                    metadata_uploaded_preserve=form.
                    cleaned_data["metadata_uploaded_preserve"])
                file = form.cleaned_data['base_file']
                saved_layer.file_size = file.size
                saved_layer.file_type = file_type
                saved_layer.save()
                if admin_upload:
                    saved_layer.status = 'ACTIVE'
                    saved_layer.save()

            except Exception as e:
                exception_type, error, tb = sys.exc_info()
                logger.exception(e)
                out['success'] = False
                out['errors'] = str(error)
                # Assign the error message to the latest UploadSession from that user.
                latest_uploads = UploadSession.objects.filter(
                    user=request.user).order_by('-date')
                if latest_uploads.count() > 0:
                    upload_session = latest_uploads[0]
                    upload_session.error = str(error)
                    upload_session.traceback = traceback.format_exc(tb)
                    upload_session.context = log_snippet(CONTEXT_LOG_FILE)
                    upload_session.save()
                    out['traceback'] = upload_session.traceback
                    out['context'] = upload_session.context
                    out['upload_session'] = upload_session.id
            else:
                out['success'] = True
                if hasattr(saved_layer, 'info'):
                    out['info'] = saved_layer.info
                out['url'] = reverse('layer_detail',
                                     args=[saved_layer.service_typename])
                upload_session = saved_layer.upload_session
                upload_session.processed = True
                upload_session.save()
                permissions = form.cleaned_data["permissions"]
                if permissions is not None and len(permissions.keys()) > 0:
                    permissions = saved_layer.resolvePermission(permissions)
                    saved_layer.set_permissions(permissions)
            finally:
                if tempdir is not None:
                    shutil.rmtree(tempdir)
        else:
            for e in form.errors.values():
                errormsgs.extend([escape(v) for v in e])
            out['errors'] = form.errors
            out['errormsgs'] = errormsgs
        if out['success']:
            status_code = 200
        else:
            status_code = 400
        return HttpResponse(json.dumps(out),
                            content_type='application/json',
                            status=status_code)