def test_duplicate_imports(self):
        """Import the same layer twice to ensure file names increment properly.
        """
        path = test_file('boxes_with_date_iso_date.zip')
        ogr = OGRImport(path)
        layers1 = ogr.handle({'index': 0, 'name': 'test'})
        layers2 = ogr.handle({'index': 0, 'name': 'test'})

        self.assertEqual(layers1[0][0], 'test')
        self.assertEqual(layers2[0][0], 'test0')
    def test_duplicate_imports(self):
        """Import the same layer twice to ensure file names increment properly.
        """
        path = test_file('boxes_with_date_iso_date.zip')
        ogr = OGRImport(path)
        layers1 = ogr.handle({'index': 0, 'name': 'test'})
        layers2 = ogr.handle({'index': 0, 'name': 'test'})

        self.assertEqual(layers1[0][0], 'test')
        self.assertEqual(layers2[0][0], 'test0')
Example #3
0
    def import_file(self, in_file, configuration_options=[]):
        """
        Imports the file.
        """
        self.assertTrue(os.path.exists(in_file))

        # run ogr2ogr
        gi = OGRImport(in_file)
        layers = gi.handle(configuration_options=configuration_options)

        return layers
Example #4
0
    def import_file(self, in_file, configuration_options=[]):
        """
        Imports the file.
        """
        self.assertTrue(os.path.exists(in_file))

        # run ogr2ogr
        gi = OGRImport(in_file)
        layers = gi.handle(configuration_options=configuration_options)

        return layers
Example #5
0
    def import_file(self, path, configs=None):
        """Imports the file.
        """
        if configs is None:
            configs = []
        self.assertTrue(os.path.exists(path), path)

        # run ogr2ogr
        ogr = OGRImport(path)
        layers = ogr.handle(configuration_options=configs)

        return layers
 def test_utf8(self):
     """Tests utf8 characters in attributes
     """
     path = test_file('china_provinces.shp')
     layer = self.generic_import(path)
     ogr = OGRImport(path)
     datastore, _ = ogr.open_target_datastore(ogr.target_store)
     sql = ("select NAME_CH from {0} where NAME_PY = 'An Zhou'".format(
         layer.name))
     result = datastore.ExecuteSQL(sql)
     feature = result.GetFeature(0)
     self.assertEqual(feature.GetField('name_ch'), '安州')
Example #7
0
 def test_utf8(self):
     """
     Tests utf8 characters in attributes
     """
     filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'china_provinces.shp')
     layer = self.generic_import('china_provinces.shp')
     gi = OGRImport(filename)
     ds, insp = gi.open_target_datastore(gi.target_store)
     sql = str("select NAME_CH from %s where NAME_PY = 'An Zhou'" % (layer.name))
     res = ds.ExecuteSQL(sql)
     feat = res.GetFeature(0)
     self.assertEqual(feat.GetField('name_ch'), "安州")
    def import_file(self, path, configs=None):
        """Imports the file.
        """
        if configs is None:
            configs = []
        self.assertTrue(os.path.exists(path), path)

        # run ogr2ogr
        ogr = OGRImport(path)
        layers = ogr.handle(configuration_options=configs)

        return layers
Example #9
0
 def test_arcgisjson(self):
     """
     Tests the import from a WFS Endpoint
     """
     endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json'
     gi = OGRImport(endpoint)
     layers = gi.handle(configuration_options=[{'index': 0}])
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
Example #10
0
 def test_arcgisjson(self):
     """
     Tests the import from a WFS Endpoint
     """
     endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json'
     gi = OGRImport(endpoint)
     layers = gi.handle(configuration_options=[{'index':0}])
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
Example #11
0
    def test_duplicate_imports(self):
        """
        Tests importing the same layer twice to ensure incrementing file names is properly handled.
        """
        filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'boxes_with_date_iso_date.zip')

        gi = OGRImport(filename)
        layers1 = gi.handle({'index': 0, 'name': 'test'})
        layers2 = gi.handle({'index': 0,  'name': 'test'})

        self.assertEqual(layers1[0][0], 'test')
        self.assertEqual(layers2[0][0], 'test0')
Example #12
0
 def test_wfs(self):
     """
     Tests the import from a WFS Endpoint
     """
     wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs'
     gi = OGRImport(wfs)
     layers = gi.handle(configuration_options=[{'layer_name':'og:bugsites'},
                                                {'layer_name':'topp:states'}])
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
 def test_arcgisjson(self):
     """Tests the import from a WFS Endpoint
     """
     endpoint = 'http://sampleserver6.arcgisonline.com/arcgis/rest/services/Water_Network/FeatureServer/16/query'\
         '?where=objectid=326&outfields=*&f=json'
     ogr = OGRImport(endpoint)
     configs = [{'index': 0}]
     layers = ogr.handle(configuration_options=configs)
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
 def test_utf8(self):
     """Tests utf8 characters in attributes
     """
     path = test_file('china_provinces.shp')
     layer = self.generic_import(path)
     ogr = OGRImport(path)
     datastore, _ = ogr.open_target_datastore(ogr.target_store)
     sql = (
         "select NAME_CH from {0} where NAME_PY = 'An Zhou'"
         .format(layer.name)
     )
     result = datastore.ExecuteSQL(sql)
     feature = result.GetFeature(0)
     self.assertEqual(feature.GetField('name_ch'), '安州')
Example #15
0
    def test_duplicate_imports(self):
        """
        Tests importing the same layer twice to ensure incrementing file names is properly handled.
        """
        filename = os.path.join(os.path.dirname(__file__), '..',
                                'importer-test-files',
                                'boxes_with_date_iso_date.zip')

        gi = OGRImport(filename)
        layers1 = gi.handle({'index': 0, 'name': 'test'})
        layers2 = gi.handle({'index': 0, 'name': 'test'})

        self.assertEqual(layers1[0][0], 'test')
        self.assertEqual(layers2[0][0], 'test0')
Example #16
0
 def test_utf8(self):
     """
     Tests utf8 characters in attributes
     """
     filename = os.path.join(os.path.dirname(__file__), '..',
                             'importer-test-files', 'china_provinces.shp')
     layer = self.generic_import('china_provinces.shp')
     gi = OGRImport(filename)
     ds, insp = gi.open_target_datastore(gi.target_store)
     sql = str("select NAME_CH from %s where NAME_PY = 'An Zhou'" %
               (layer.name))
     res = ds.ExecuteSQL(sql)
     feat = res.GetFeature(0)
     self.assertEqual(feat.GetField('name_ch'), "安州")
Example #17
0
    def test_import_file_uses_configured_layername(self):
        """ Checks that if a custom layer name is provided it replaces the UploadLayer name and is used
            as the table name for data stored in PostGIS.
        """
        # --- Prereq's to importing layers
        # my_states.gpkg is a straightforward 1-layer vector package.
        test_filename = 'my_states.gpkg'
        test_filepath = os.path.join(_TEST_FILES_DIR, test_filename)

        # Make temporary file (the upload/configure process removes the file & we want to keep our test file)
        tmppath = os.path.join('/tmp', test_filename)
        shutil.copyfile(test_filepath, tmppath)

        # upload & configure_upload expect closed file objects
        #    This is heritage from originally being closely tied to a view passing request.Files
        of = open(tmppath, 'rb')
        of.close()
        files = [of]
        upload = self.upload(files, self.admin_user)
        self.configure_upload(upload, files)

        # should be a single UploadFile resulting from configure_upload()
        upload_file = upload.uploadfile_set.first()
        # should be a single UploadLayer related to upload_file
        upload_layer = upload_file.uploadlayer_set.first()

        # --- Actually do imports (just import_file(), not handlers)
        custom_layername = 'my_custom_layer'
        configuration_options = {
            'upload_layer_id': upload_layer.id,
            'index': 0,
            'layer_name': custom_layername
        }
        oi = OGRImport(upload_file.file.name, upload_file=upload_file)
        oi.import_file(configuration_options=configuration_options)

        # --- Check that PostGIS has a table matching the name of the layer set during configure_upload()
        expected_tablename = custom_layername
        default_tablename = upload_layer.layer_name
        with connections['datastore'].cursor() as cursor:
            sql = """
                SELECT tablename
                FROM pg_catalog.pg_tables
                WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';
            """
            cursor.execute(sql)
            tables = [row[0] for row in cursor.fetchall()]
            self.assertIn(expected_tablename, tables)
            self.assertNotIn(default_tablename, tables)
Example #18
0
 def test_wfs(self):
     """Tests the import from a WFS Endpoint
     """
     wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs'
     ogr = OGRImport(wfs)
     configs = [
         {'layer_name': 'og:bugsites'},
         {'layer_name': 'topp:states'}
     ]
     layers = ogr.handle(configuration_options=configs)
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
    def test_import_file_uses_configured_layername(self):
        """ Checks that if a custom layer name is provided it replaces the UploadLayer name and is used
            as the table name for data stored in PostGIS.
        """
        # --- Prereq's to importing layers
        # my_states.gpkg is a straightforward 1-layer vector package.
        test_filename = 'my_states.gpkg'
        test_filepath = os.path.join(_TEST_FILES_DIR, test_filename)

        # Make temporary file (the upload/configure process removes the file & we want to keep our test file)
        tmppath = os.path.join('/tmp', test_filename)
        shutil.copyfile(test_filepath, tmppath)

        # upload & configure_upload expect closed file objects
        #    This is heritage from originally being closely tied to a view passing request.Files
        of = open(tmppath, 'rb')
        of.close()
        files = [of]
        upload = self.upload(files, self.admin_user)
        self.configure_upload(upload, files)

        # should be a single UploadFile resulting from configure_upload()
        upload_file = upload.uploadfile_set.first()
        # should be a single UploadLayer related to upload_file
        upload_layer = upload_file.uploadlayer_set.first()

        # --- Actually do imports (just import_file(), not handlers)
        custom_layername = 'my_custom_layer'
        configuration_options = {'upload_layer_id': upload_layer.id, 'index': 0, 'layer_name': custom_layername}
        oi = OGRImport(upload_file.file.name, upload_file=upload_file)
        oi.import_file(configuration_options=configuration_options)

        # --- Check that PostGIS has a table matching the name of the layer set during configure_upload()
        expected_tablename = custom_layername
        default_tablename = upload_layer.layer_name
        with connections['datastore'].cursor() as cursor:
            sql = """
                SELECT tablename
                FROM pg_catalog.pg_tables
                WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';
            """
            cursor.execute(sql)
            tables = [row[0] for row in cursor.fetchall()]
            self.assertIn(expected_tablename, tables)
            self.assertNotIn(default_tablename, tables)
 def test_wfs(self):
     """Tests the import from a WFS Endpoint
     """
     wfs = 'WFS:http://demo.geo-solutions.it/geoserver/tiger/wfs'
     ogr = OGRImport(wfs)
     configs = [
         {
             'layer_name': 'tiger:giant_polygon'
         },
         {
             'layer_name': 'tiger:poi'
         },
     ]
     layers = ogr.handle(configuration_options=configs)
     for result in layers:
         layer = Layer.objects.get(name=result[0])
         self.assertEqual(layer.srid, 'EPSG:4326')
         self.assertEqual(layer.store, self.datastore.name)
         self.assertEqual(layer.storeType, 'dataStore')
Example #21
0
    def test_import_file_skips_duplicate_configured_layername(self):
        """ Checks that if a custom layer name is provided, but is already used the unique layer name
            created by configure_upload() & stored in UploadLayer is used as the table name for data stored
            in PostGIS.
        """
        # --- Prereq's to importing layers
        # my_states.gpkg is a straightforward 1-layer vector package.
        test_filename = 'my_states.gpkg'
        test_filepath = os.path.join(_TEST_FILES_DIR, test_filename)

        # Upload two copies so we can have a duplicate name with the second.
        # Make temporary file (the upload/configure process removes the file & we want to keep our test file)
        filename, ext = test_filename.split('.')
        tmppath1 = os.path.join('/tmp', '{}-1.{}'.format(filename, ext))
        tmppath2 = os.path.join('/tmp', '{}-2.{}'.format(filename, ext))

        shutil.copyfile(test_filepath, tmppath1)
        shutil.copyfile(test_filepath, tmppath2)

        # upload & configure_upload expect closed file objects
        #    This is heritage from originally being closely tied to a view passing request.Files
        of1 = open(tmppath1, 'rb')
        of1.close()
        files = [of1]
        upload1 = self.upload(files, self.admin_user)
        self.configure_upload(upload1, files)

        of2 = open(tmppath2, 'rb')
        of2.close()
        files = [of2]
        upload2 = self.upload(files, self.admin_user)
        self.configure_upload(upload2, files)

        upload_file1 = upload1.uploadfile_set.first()
        upload_layer1 = upload_file1.uploadlayer_set.first()

        upload_file2 = upload2.uploadfile_set.first()
        upload_layer2 = upload_file2.uploadlayer_set.first()

        # --- Import file1's layer using default name
        configuration_options = {'upload_layer_id': upload_layer1.id, 'index': 0}
        oi = OGRImport(upload_file1.file.name, upload_file=upload_file1)
        oi.import_file(configuration_options=configuration_options)

        # --- Try importing file2's layer using the same name as file1's layer.
        configuration_options = {
            'upload_layer_id': upload_layer2.id, 'index': 0, 'layer_name': upload_layer1.layer_name}
        oi = OGRImport(upload_file2.file.name, upload_file=upload_file2)
        oi.import_file(configuration_options=configuration_options)

        # --- Check that PostGIS has a table matching the default layer name for upload_layer2
        expected_tablename = upload_layer2.layer_name
        with connections['datastore'].cursor() as cursor:
            sql = """
                SELECT tablename
                FROM pg_catalog.pg_tables
                WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';
            """
            cursor.execute(sql)
            tables = [row[0] for row in cursor.fetchall()]
            self.assertIn(expected_tablename, tables)
    def test_utf8(self):
        """
        Tests utf8 characters in attributes
        """
<<<<<<< HEAD
        filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'china_provinces.zip')
        layer = self.generic_import('china_provinces.zip')
=======
        filename = os.path.join(
            os.path.dirname(__file__),
            '..',
            'importer-test-files',
            'china_provinces.shp')
        layer = self.generic_import('china_provinces.shp')
>>>>>>> upstream/multiupload_cherrypick
        gi = OGRImport(filename)
        ds, insp = gi.open_target_datastore(gi.target_store)
        sql = str(
            "select NAME_CH from %s where NAME_PY = 'An Zhou'" %
            (layer.name))
        res = ds.ExecuteSQL(sql)
        feat = res.GetFeature(0)
        self.assertEqual(feat.GetField('name_ch'), "安州")

    def test_non_converted_date(self):
        """
        Test converting a field as date.
        """
        results = self.generic_import(
            'TM_WORLD_BORDERS_2005.zip', configuration_options=[{'index': 0,
                                                                 'start_date':
    def test_import_file_skips_duplicate_configured_layername(self):
        """ Checks that if a custom layer name is provided, but is already used the unique layer name
            created by configure_upload() & stored in UploadLayer is used as the table name for data stored
            in PostGIS.
        """
        # --- Prereq's to importing layers
        # my_states.gpkg is a straightforward 1-layer vector package.
        test_filename = 'my_states.gpkg'
        test_filepath = os.path.join(_TEST_FILES_DIR, test_filename)

        # Upload two copies so we can have a duplicate name with the second.
        # Make temporary file (the upload/configure process removes the file & we want to keep our test file)
        filename, ext = test_filename.split('.')
        tmppath1 = os.path.join('/tmp', '{}-1.{}'.format(filename, ext))
        tmppath2 = os.path.join('/tmp', '{}-2.{}'.format(filename, ext))

        shutil.copyfile(test_filepath, tmppath1)
        shutil.copyfile(test_filepath, tmppath2)

        # upload & configure_upload expect closed file objects
        #    This is heritage from originally being closely tied to a view passing request.Files
        of1 = open(tmppath1, 'rb')
        of1.close()
        files = [of1]
        upload1 = self.upload(files, self.admin_user)
        self.configure_upload(upload1, files)

        of2 = open(tmppath2, 'rb')
        of2.close()
        files = [of2]
        upload2 = self.upload(files, self.admin_user)
        self.configure_upload(upload2, files)

        upload_file1 = upload1.uploadfile_set.first()
        upload_layer1 = upload_file1.uploadlayer_set.first()

        upload_file2 = upload2.uploadfile_set.first()
        upload_layer2 = upload_file2.uploadlayer_set.first()

        # --- Import file1's layer using default name
        configuration_options = {'upload_layer_id': upload_layer1.id, 'index': 0}
        oi = OGRImport(upload_file1.file.name, upload_file=upload_file1)
        oi.import_file(configuration_options=configuration_options)

        # --- Try importing file2's layer using the same name as file1's layer.
        configuration_options = {
            'upload_layer_id': upload_layer2.id, 'index': 0, 'layer_name': upload_layer1.layer_name}
        oi = OGRImport(upload_file2.file.name, upload_file=upload_file2)
        oi.import_file(configuration_options=configuration_options)

        # --- Check that PostGIS has a table matching the default layer name for upload_layer2
        expected_tablename = upload_layer2.layer_name
        with connections['datastore'].cursor() as cursor:
            sql = """
                SELECT tablename
                FROM pg_catalog.pg_tables
                WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';
            """
            cursor.execute(sql)
            tables = [row[0] for row in cursor.fetchall()]
            self.assertIn(expected_tablename, tables)