def get_shapefile_info(shapefile_path): from django.contrib.gis import gdal ds = gdal.DataSource(shapefile_path) layer = ds[0] fields = [] for i in range(0, len(layer.fields)): fields.append( dict(name=layer.fields[i], type=layer.field_types[i].__name__, width=layer.field_widths[i], precision=layer.field_precisions[i])) srid = None if layer.srs is not None: if layer.srs.srid is None: try: srid = get_srid(layer.srs.proj4) except: pass else: srid = layer.srs.srid return DynamicObject( dict( name=layer.name, # num_features=layer.num_feat, type=layer.geom_type.name, # fields=fields, # extent=layer.extent.tuple, srid=srid))
def to_python(self, value): value = super(GeometryFileField, self).to_python(value) if value is None: return value filename = value.name tmpdir = None if zipfile.is_zipfile(value): tmpdir = tempfile.mkdtemp() zf = zipfile.ZipFile(value) # Extract all files from the temporary directory using only the # base file name, avoids security issues with relative paths in the # zip. for item in zf.namelist(): filename = os.path.join(tmpdir, os.path.basename(item)) with open(filename, 'wb') as f: f.write(zf.read(item)) if filename.endswith('.shp'): break # Attempt to union all geometries from GDAL data source. try: geoms = gdal.DataSource(filename)[0].get_geoms() geom = reduce(lambda g1, g2: g1.union(g2), geoms) if not geom.srs: raise gdal.OGRException('Cannot determine SRS') except (gdal.OGRException, gdal.OGRIndexError): geom = None finally: if tmpdir and os.path.isdir(tmpdir): shutil.rmtree(tmpdir) return geom
def handle(self, *args, **options): data_source, model_name = options.pop('data_source'), options.pop('model_name') if not gdal.HAS_GDAL: raise CommandError('GDAL is required to inspect geospatial data sources.') # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.OGRException as msg: raise CommandError(msg) # Returning the output of ogrinspect with the given arguments # and options. from django.contrib.gis.utils.ogrinspect import _ogrinspect, mapping # Filter options to params accepted by `_ogrinspect` ogr_options = dict((k, v) for k, v in options.items() if k in inspect.getargspec(_ogrinspect).args and v is not None) output = [s for s in _ogrinspect(ds, model_name, **ogr_options)] if options['mapping']: # Constructing the keyword arguments for `mapping`, and # calling it on the data source. kwargs = {'geom_name': options['geom_name'], 'layer_key': options['layer_key'], 'multi_geom': options['multi_geom'], } mapping_dict = mapping(ds, **kwargs) # This extra legwork is so that the dictionary definition comes # out in the same order as the fields in the model definition. rev_mapping = dict((v, k) for k, v in mapping_dict.items()) output.extend(['', '# Auto-generated `LayerMapping` dictionary for %s model' % model_name, '%s_mapping = {' % model_name.lower()]) output.extend(" '%s' : '%s'," % (rev_mapping[ogr_fld], ogr_fld) for ogr_fld in ds[options['layer_key']].fields) output.extend([" '%s' : '%s'," % (options['geom_name'], mapping_dict[options['geom_name']]), '}']) return '\n'.join(output) + '\n'
def _get_datasource(self): handler = HandlerFactory.get(self.file) ext_set = set(ShapefileReader.REQUIRED_EXTENSIONS) files_name = None for name in handler.namelist: fname, ext = os.path.splitext(name) if ext not in ShapefileReader.REQUIRED_EXTENSIONS: continue if not files_name: shapefile_name = fname elif fname != shapefile_name: raise CompressedShapeError(_(u'The files are not all from ' u'the same shapefile.')) ext_set.discard(ext) if ext_set: raise CompressedShapeError(_(u'The following files are missing ' u'in the shapefile: {}') .format(list(ext_set))) handler.extract(self._tmp_dir) shapefile_path = os.path.join(self._tmp_dir, shapefile_name+'.shp') return gdal.DataSource(shapefile_path)
def upload(request): ctoken = {} ctoken.update(csrf(request)) if request.method == 'POST': form = UploadForm(request.POST, request.FILES) if form.is_valid(): shp = form.handle(request.FILES['file_obj']) ds = gdal.DataSource(shp) ct = CoordTransform(SpatialReference('EPSG:4326'), SpatialReference('EPSG:900913')) dados = [] for layer in ds[0]: aux = layer.geom aux.transform(ct) dados.append(aux) form = UploadForm() return render_to_response( 'googlev3_upload.html', RequestContext(request, { 'form': form, 'dados': dados, 'token': ctoken })) else: form = UploadForm() return render_to_response('googlev3_upload.html', RequestContext(request, {'form': form}))
def handle(self, *args, **options): data_source, model_name = options.pop('data_source'), options.pop('model_name') # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.GDALException as msg: raise CommandError(msg) # Returning the output of ogrinspect with the given arguments # and options. from django.contrib.gis.utils.ogrinspect import _ogrinspect, mapping # Filter options to params accepted by `_ogrinspect` ogr_options = {k: v for k, v in options.items() if k in get_func_args(_ogrinspect) and v is not None} output = [s for s in _ogrinspect(ds, model_name, **ogr_options)] if options['mapping']: # Constructing the keyword arguments for `mapping`, and # calling it on the data source. kwargs = { 'geom_name': options['geom_name'], 'layer_key': options['layer_key'], 'multi_geom': options['multi_geom'], } mapping_dict = mapping(ds, **kwargs) # This extra legwork is so that the dictionary definition comes # out in the same order as the fields in the model definition. rev_mapping = {v: k for k, v in mapping_dict.items()}
def _from_file(self, fileobj, tmpdir): if zipfile.is_zipfile(fileobj): with zipfile.ZipFile(fileobj) as zf: extracted = [] for item in zf.infolist(): fname = os.path.abspath(os.path.join( tmpdir, item.filename)) if fname.startswith(tmpdir): zf.extract(item, tmpdir) extracted.append(fname) for path in extracted: if path.endswith('.shp'): fname = path else: # NOTE: is_zipfile() seeks to end of file or at least 110 bytes. fileobj.seek(0) with tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as fp: shutil.copyfileobj(fileobj, fp) fname = fp.name # Attempt to union all geometries from GDAL data source. try: geoms = gdal.DataSource(fname)[0].get_geoms() geom = reduce(lambda g1, g2: g1.union(g2), geoms) if not geom.srs: raise gdal.OGRException('Cannot determine SRS') except (gdal.OGRException, gdal.OGRIndexError): raise forms.ValidationError( GeometryField.default_error_messages['invalid_geom'], code='invalid_geom') return geom
def handle(self, *args, **options): path = pathjoin(dirname(__file__), pardir, pardir, 'azavea-geo-data', 'Neighborhoods_Philadelphia', 'Neighborhoods_Philadelphia.shp') shapefile = gdal.DataSource(path) for layer in shapefile: for feature in layer: try: region = Region.objects.get(layer='neighborhoods', name=feature.get('listname')) print 'Updating "%s"' % region.name except Region.DoesNotExist: region = Region(layer='neighborhoods', name=feature.get('listname')) print 'Loading "%s"' % region.name # Transform the projection from 102729 (NAD 1983 StatePlane # Pennsylvania South FIPS 3702 Feet) to 4326 (WSG 1984). # # Source: http://svn.osgeo.org/postgis/tags/0.7.4/spatial_ref_sys.sql region.shape = feature.geom.geos region.shape.srid = 102729 region.shape.transform(4326) region.save()
def handle(self, *args, **options): try: data_source, model_name = args except ValueError: raise CommandError('Invalid arguments, must provide: %s' % self.args) if not gdal.HAS_GDAL: raise CommandError( 'GDAL is required to inspect geospatial data sources.') # Removing options with `None` values. options = dict([(k, v) for k, v in options.items() if not v is None]) # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.OGRException as msg: raise CommandError(msg) # Whether the user wants to generate the LayerMapping dictionary as well. show_mapping = options.pop('mapping', False) # Getting rid of settings that `_ogrinspect` doesn't like. verbosity = options.pop('verbosity', False) settings = options.pop('settings', False) # Returning the output of ogrinspect with the given arguments # and options. from django.contrib.gis.utils.ogrinspect import _ogrinspect, mapping output = [s for s in _ogrinspect(ds, model_name, **options)] if show_mapping: # Constructing the keyword arguments for `mapping`, and # calling it on the data source. kwargs = { 'geom_name': options['geom_name'], 'layer_key': options['layer_key'], 'multi_geom': options['multi_geom'], } mapping_dict = mapping(ds, **kwargs) # This extra legwork is so that the dictionary definition comes # out in the same order as the fields in the model definition. rev_mapping = dict([(v, k) for k, v in mapping_dict.items()]) output.extend([ '', '# Auto-generated `LayerMapping` dictionary for %s model' % model_name, '%s_mapping = {' % model_name.lower() ]) output.extend([ " '%s' : '%s'," % (rev_mapping[ogr_fld], ogr_fld) for ogr_fld in ds[options['layer_key']].fields ]) output.extend([ " '%s' : '%s'," % (options['geom_name'], mapping_dict[options['geom_name']]), '}' ]) return '\n'.join(output) + '\n'
def geoms_from_shapefile(filename, id=None): ds = gdal.DataSource(filename) layer = ds[0] id_column = lambda f: f.get(id) if id else f.fid geoms = {} for f in layer: try: geoms[id_column(f)] = geos.GEOSGeometry(f.geom.wkt) except Exception, e: print e
def handle(self, *args, **options): data_source, model_name = options.pop("data_source"), options.pop("model_name") # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.GDALException as msg: raise CommandError(msg) # Returning the output of ogrinspect with the given arguments # and options. from django.contrib.gis.utils.ogrinspect import _ogrinspect, mapping # Filter options to params accepted by `_ogrinspect` ogr_options = { k: v for k, v in options.items() if k in get_func_args(_ogrinspect) and v is not None } output = [s for s in _ogrinspect(ds, model_name, **ogr_options)] if options["mapping"]: # Constructing the keyword arguments for `mapping`, and # calling it on the data source. kwargs = { "geom_name": options["geom_name"], "layer_key": options["layer_key"], "multi_geom": options["multi_geom"], } mapping_dict = mapping(ds, **kwargs) # This extra legwork is so that the dictionary definition comes # out in the same order as the fields in the model definition. rev_mapping = {v: k for k, v in mapping_dict.items()} output.extend( [ "", "", "# Auto-generated `LayerMapping` dictionary for %s model" % model_name, "%s_mapping = {" % model_name.lower(), ] ) output.extend( " '%s': '%s'," % (rev_mapping[ogr_fld], ogr_fld) for ogr_fld in ds[options["layer_key"]].fields ) output.extend( [ " '%s': '%s'," % (options["geom_name"], mapping_dict[options["geom_name"]]), "}", ] ) return "\n".join(output) + "\n"
def getShapefileLayers(self): shapefiles = self.serializer.path_directory shapefiles = [ shapefile for shapefile in os.listdir(shapefiles) if shapefile[-3:] == "shp" ] datasources = [ gdal.DataSource(os.path.join(self.serializer.path_directory, s)) for s in shapefiles ] layers = [ds[0] for ds in datasources] return layers
def validate(self, zip_path): """ Validate the uploaded, zipped shapefile by unpacking to a temporary sandbox. """ # create a temporary file to write the zip archive to tmp = tempfile.NamedTemporaryFile(suffix='.zip', mode='w') # write zip to tmp sandbox self.write_file(zip_path, tmp.name) if not zipfile.is_zipfile(tmp.name): return False, 'That file is not a valid Zip Archive' # create zip object zfile = zipfile.ZipFile(tmp.name) # ensure proper file contents by extensions inside if not self.check_zip_contents('shp', zfile): return False, 'Found Zip Archive but no file with a .shp extension found inside.' #elif not self.check_zip_contents('prj', zfile): # return False, 'You must supply a .prj file with the Shapefile to indicate the projection.' elif not self.check_zip_contents('dbf', zfile): return False, 'You must supply a .dbf file with the Shapefile to supply attribute data.' elif not self.check_zip_contents('shx', zfile): return False, 'You must supply a .shx file for the Shapefile to have a valid index.' # unpack contents into tmp directory tmp_dir = tempfile.gettempdir() zfile.extractall(tmp_dir) files_list = zfile.namelist() #looking for the shape. whe must have 3 files with the same name shape_candidate = self.get_shape_candidate(files_list) if not shape_candidate: return False, "No suitable shapefile found" ds_name = shape_candidate # ogr needs the full path to the unpacked 'file.shp' full_path = '%s%s%s.shp' % (tmp_dir, os.path.sep, ds_name) ds = gdal.DataSource(full_path) # shapefiles have just one layer, so grab the first... layer = ds[0] # one way of testing a sane shapefile... # further tests should be able to be plugged in here... if layer.test_capability('RandomRead'): if str(ds.driver) == 'ESRI Shapefile': return True, None else: return False, "Sorry, we've experienced a problem on our server. Please try again later." else: return False, 'Cannot read the shapefile, data is corrupted inside the zip, please try to upload again'
def validate(self, filefield_data): """ Validate the uploaded, zipped shapefile by unpacking to a temporary sandbox. """ # create a temporary file to write the zip archive to tmp = tempfile.NamedTemporaryFile(suffix='.zip', mode='w') # write zip to tmp sandbox self.write_file(tmp.name, filefield_data) if not zipfile.is_zipfile(tmp.name): return False, 'That file is not a valid Zip Archive' # create zip object zfile = zipfile.ZipFile(tmp.name) # ensure proper file contents by extensions inside if not self.check_zip_contents('shp', zfile): return False, 'Found Zip Archive but no file with a .shp extension found inside.' elif not self.check_zip_contents('prj', zfile): return False, 'You must supply a .prj file with the Shapefile to indicate the projection.' elif not self.check_zip_contents('dbf', zfile): return False, 'You must supply a .dbf file with the Shapefile to supply attribute data.' elif not self.check_zip_contents('shx', zfile): return False, 'You must supply a .shx file for the Shapefile to have a valid index.' # unpack contents into tmp directory tmp_dir = tempfile.gettempdir() for info in zfile.infolist(): data = zfile.read(info.filename) shp_part = '%s%s%s' % (tmp_dir, os.path.sep, info.filename) fout = open(shp_part, "wb") fout.write(data) fout.close() # get the datasource name without extension ds_name = os.path.splitext(zfile.namelist()[0])[0] # ogr needs the full path to the unpacked 'file.shp' ds = gdal.DataSource('%s%s%s.shp' % (tmp_dir, os.path.sep, ds_name)) # shapefiles have just one layer, so grab the first... layer = ds[0] # one way of testing a sane shapefile... # further tests should be able to be plugged in here... if layer.test_capability('RandomRead'): if str(ds.driver) == 'ESRI Shapefile': return True, None else: return False, "Sorry, we've experienced a problem on our server. Please try again later." else: return False, 'Cannot read the shapefile, data is corrupted inside the zip, please try to upload again'
def save(self, commit=True): modal = super(ShapeForm, self).save(commit=False) print(self.cleaned_data['shape_file']) if not self.cleaned_data['shape_file'] == None: shape_file = self.cleaned_data['shape_file'] working_dir = mkdtemp() try: shape_zip = ZipFile(shape_file) shape_zip.extractall(working_dir) except: shutil.rmtree(working_dir) print("Could not extract zip file.") shapes_list = glob.glob(working_dir + '/*.shp') try: ds = gdal.DataSource(shapes_list[0]) layer = ds[0] polygons = [] for feature in layer: state = self.cleaned_data['state'] geom = feature.geom.geos source = self.cleaned_data['source'] start_date = self.cleaned_data['start_date'] or date( 1945, 1, 1) end_date = self.cleaned_data['end_date'] or date( 2018, 1, 1) if type(geom) == geos.Polygon: polygons.append(geom) if len(polygons) > 0: multipoly = geos.MultiPolygon(polygons) modal, created = Shape.objects.get_or_create( state=state, shape=multipoly, source=source, start_date=start_date, end_date=end_date) except: shutil.rmtree(working_dir) print("Error when converting shape file.") if commit == True: m.save() return modal
def handle(self, *args, **options): file_names = options['geojson'] ds = gdal.DataSource(file_names[0]) layer = ds[0] # assert(all(field in layer.fields for field in ['state', 'source', 'start_date', 'end_date'])) for feature in layer: state = feature.get('name') geom = feature.geom.geos source = 'https://github.com/johan/world.geo.json' start_date = feature.get('start_date') or date(1945, 1, 1) if type(geom) == geos.Polygon: geom = geos.MultiPolygon(geom) state, created = State.objects.get_or_create(name=state, color="test") print(state, start_date, created) Shape.objects.get_or_create(state=state, shape=geom, source=source, start_date=start_date, end_date=date(2018, 1, 1)) print(State.objects.count()) print(Shape.objects.count())
def handle(self, *args, **options): try: data_source, model_name = args except ValueError: raise CommandError('Invalid arguments, must provide: %s' % self.args) if not gdal.HAS_GDAL: raise CommandError( 'GDAL is required to inspect geospatial data sources.') # Removing options with `None` values. options = dict([(k, v) for k, v in options.items() if not v is None]) # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.OGRException, msg: raise CommandError(msg)
def _from_file(self, fileobj, tmpdir): if zipfile.is_zipfile(fileobj): with zipfile.ZipFile(fileobj) as zf: for item in zf.infolist(): fname = os.path.abspath(os.path.join( tmpdir, item.filename)) if fname.startswith(tmpdir): zf.extract(item, tmpdir) else: # NOTE: is_zipfile() seeks to end of file or at least 110 bytes. fileobj.seek(0) with tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as fp: shutil.copyfileobj(fileobj, fp) fname = fp.name # Attempt to union all geometries from GDAL data source. try: geoms = gdal.DataSource(fname)[0].get_geoms() geom = reduce(lambda g1, g2: g1.union(g2), geoms) if not geom.srs: raise gdal.OGRException('Cannot determine SRS') except (gdal.OGRException, gdal.OGRIndexError): geom = None return geom
def handle(self, filefield_data, user): """ Upload the file data, in chunks, to the media/upload/loadshp. Then unpack it, read the features and return them """ # ensure the upload directory exists upload_dir = os.path.join(settings.MEDIA_ROOT, 'upload', 'loadshp', user.username) if not os.path.exists(upload_dir): os.makedirs(upload_dir) # contruct the full filepath and filename downloaded_file = os.path.normpath( os.path.join(upload_dir, filefield_data.name)) # if we've already got an upload with the same name, append the daymonthyear_minute if os.path.exists(downloaded_file): name, ext = os.path.splitext(downloaded_file) append = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') downloaded_file = '%s_%s%s' % (name, append, ext) # write the zip archive to final location self.write_file(downloaded_file, filefield_data) #unpack the files zfile = zipfile.ZipFile(downloaded_file) tmp_dir = tempfile.gettempdir() for info in zfile.infolist(): data = zfile.read(info.filename) shp_part = '%s%s%s' % (tmp_dir, os.path.sep, info.filename) fout = open(shp_part, "wb") fout.write(data) fout.close() # get the datasource name without extension ds_name = os.path.splitext(zfile.namelist()[0])[0] ds = gdal.DataSource('%s%s%s.shp' % (tmp_dir, os.path.sep, ds_name)) return ds[0]
def save(self, commit=True): model = super(TerritoryForm, self).save(commit=False) if not self.cleaned_data["shape_file"] is None: shape_file = self.cleaned_data["shape_file"] working_dir = mkdtemp() try: shape_zip = ZipFile(shape_file) shape_zip.extractall(working_dir) except: shutil.rmtree(working_dir) raise ValidationError("Could not extract zipfile.") basename = os.path.splitext(os.path.basename(shape_file.name))[0] shapes_list = glob.glob(working_dir + "/" + basename + "/*.shp") try: ds = gdal.DataSource(shapes_list[0]) layer = ds[0] polygons = [] for feature in layer: geom = feature.geom.geos if isinstance(geom, geos.Polygon): polygons.append(geom) if len(polygons) > 0: multipoly = geos.MultiPolygon(polygons) model.geo = multipoly.geojson except: shutil.rmtree(working_dir) raise ValidationError("Error converting shapefile") if commit: model.save() return model
def handle_args(self, *args, **options): try: data_source, model_name = args except ValueError: raise CommandError('Invalid arguments, must provide: %s' % self.args) if not gdal.HAS_GDAL: raise CommandError( 'GDAL is required to inspect geospatial data sources.') # TODO: Support non file-based OGR datasources. if not os.path.isfile(data_source): raise CommandError('The given data source cannot be found: "%s"' % data_source) # Removing options with `None` values. options = dict([(k, v) for k, v in options.items() if not v is None]) # Getting the OGR DataSource from the string parameter. try: ds = gdal.DataSource(data_source) except gdal.OGRException, msg: raise CommandError(msg)
def test_shape_mixed(self): """ Test that a project made of intervention of different geom create multiple files. Check that those files are each of a different type (Point/LineString) and that the project and the intervention are correctly referenced in it. """ # Create topology line topo_line = TopologyFactory.create(no_path=True) line = PathFactory.create( geom=LineString(Point(10, 10), Point(11, 10))) PathAggregationFactory.create(topo_object=topo_line, path=line) # Create a topology point lng, lat = tuple( Point(1, 1, srid=settings.SRID).transform(settings.API_SRID, clone=True)) closest_path = PathFactory( geom=LineString(Point(0, 0), Point(1, 0), srid=settings.SRID)) topo_point = TopologyHelper._topologypoint(lng, lat, None).reload() self.assertEquals(topo_point.paths.get(), closest_path) # Create one intervention by geometry (point/linestring) it_point = InterventionFactory.create(topology=topo_point) it_line = InterventionFactory.create(topology=topo_line) # reload it_point = type(it_point).objects.get(pk=it_point.pk) it_line = type(it_line).objects.get(pk=it_line.pk) proj = ProjectFactory.create() proj.interventions.add(it_point) proj.interventions.add(it_line) # instanciate the class based view 'abnormally' to use create_shape directly # to avoid making http request, authent and reading from a zip pfl = ZipShapeSerializer() devnull = open(os.devnull, "wb") pfl.serialize(Project.objects.all(), stream=devnull, delete=False, fields=ProjectFormatList.columns) self.assertEquals(len(pfl.layers), 2) ds_point = gdal.DataSource(pfl.layers.values()[0]) layer_point = ds_point[0] ds_line = gdal.DataSource(pfl.layers.values()[1]) layer_line = ds_line[0] self.assertEquals(layer_point.geom_type.name, 'MultiPoint') self.assertEquals(layer_line.geom_type.name, 'LineString') for layer in [layer_point, layer_line]: self.assertEquals(layer.srs.name, 'RGF93_Lambert_93') self.assertItemsEqual(layer.fields, [ u'id', u'name', u'period', u'type', u'domain', u'constraint', u'global_cos', u'interventi', u'interven_1', u'comments', u'contractor', u'project_ow', u'project_ma', u'founders', u'related_st', u'insertion_', u'update_dat', u'cities', u'districts', u'restricted' ]) self.assertEquals(len(layer_point), 1) self.assertEquals(len(layer_line), 1) for feature in layer_point: self.assertEquals(str(feature['id']), str(proj.pk)) self.assertTrue(feature.geom.geos.equals(it_point.geom)) for feature in layer_line: self.assertEquals(str(feature['id']), str(proj.pk)) self.assertTrue(feature.geom.geos.equals(it_line.geom)) # Clean-up temporary shapefiles for layer_file in pfl.layers.values(): for subfile in shapefile_files(layer_file): os.remove(subfile)
def getShapefileLayers(self): shapefiles = self.serializer.layers.values() datasources = [gdal.DataSource(s) for s in shapefiles] layers = [ds[0] for ds in datasources] return layers
def test_shape_mixed(self): """ Test that a project made of intervention of different geom create multiple files. Check that those files are each of a different type (Point/LineString) and that the project and the intervention are correctly referenced in it. """ # Create topology line line = PathFactory.create( geom=LineString(Point(10, 10), Point(11, 10))) topo_line = TopologyFactory.create(paths=[line]) # Create a topology point lng, lat = tuple( Point(1, 1, srid=settings.SRID).transform(settings.API_SRID, clone=True)) closest_path = PathFactory( geom=LineString(Point(0, 0), Point(1, 0), srid=settings.SRID)) topo_point = TopologyHelper._topologypoint(lng, lat, None).reload() self.assertEqual(topo_point.paths.get(), closest_path) # Create one intervention by geometry (point/linestring) it_point = InterventionFactory.create(target=topo_point) it_line = InterventionFactory.create(target=topo_line) # reload it_point = type(it_point).objects.get(pk=it_point.pk) it_line = type(it_line).objects.get(pk=it_line.pk) proj = ProjectFactory.create() proj.interventions.add(it_point) proj.interventions.add(it_line) # instanciate the class based view 'abnormally' to use create_shape directly # to avoid making http request, authent and reading from a zip pfl = ZipShapeSerializer() shapefiles = pfl.path_directory devnull = open(os.devnull, "wb") pfl.serialize(Project.objects.all(), stream=devnull, delete=False, fields=ProjectFormatList.columns) shapefiles = [ shapefile for shapefile in os.listdir(shapefiles) if shapefile[-3:] == "shp" ] datasources = [ gdal.DataSource(os.path.join(pfl.path_directory, s)) for s in shapefiles ] layers = [ds[0] for ds in datasources] self.assertEqual(len(datasources), 2) geom_type_layer = {layer.name: layer for layer in layers} geom_types = geom_type_layer.keys() self.assertIn('MultiPoint', geom_types) self.assertIn('MultiLineString', geom_types) for layer in layers: self.assertEqual(layer.srs.name, 'RGF93_Lambert_93') self.assertCountEqual(layer.fields, [ 'id', 'name', 'period', 'type', 'domain', 'constraint', 'global_cos', 'interventi', 'comments', 'contractor', 'project_ow', 'project_ma', 'founders', 'related_st', 'insertion_', 'update_dat', 'cities', 'districts', 'restricted' ]) self.assertEqual(len(layers[0]), 1) self.assertEqual(len(layers[1]), 1) for feature in geom_type_layer['MultiPoint']: self.assertEqual(str(feature['id']), str(proj.pk)) self.assertEqual(len(feature.geom.geos), 1) self.assertAlmostEqual(feature.geom.geos[0].x, it_point.geom.x) self.assertAlmostEqual(feature.geom.geos[0].y, it_point.geom.y) for feature in geom_type_layer['MultiLineString']: self.assertEqual(str(feature['id']), str(proj.pk)) self.assertTrue(feature.geom.geos.equals(it_line.geom))
def validate(self, filefield_data): """ Validate the uploaded, zipped shapefile by unpacking to a temporary sandbox. """ # create a temporary file to write the zip archive to tmp = tempfile.NamedTemporaryFile(suffix='.zip', mode='w', delete=False) # write zip to tmp sandbox self.write_file(tmp.name, filefield_data) if not zipfile.is_zipfile(tmp.name): return False, 'That file is not a valid Zip Archive' # create zip object zfile = zipfile.ZipFile(tmp.name) # ensure proper file contents by extensions inside if not self.check_zip_contents('shp', zfile): return False, 'Found Zip Archive but no file with a .shp extension found inside.' elif not self.check_zip_contents('dbf', zfile): return False, 'You must supply a .dbf file with the Shapefile to supply attribute data.' elif not self.check_zip_contents('shx', zfile): return False, 'You must supply a .shx file for the Shapefile to have a valid index.' # unpack contents into tmp directory tmp_dir = tempfile.gettempdir() for info in zfile.infolist(): data = zfile.read(info.filename) shp_part = '%s%s%s' % (tmp_dir, os.path.sep, info.filename) fout = open(shp_part, "wb") fout.write(data) fout.close() # get the datasource name without extension ds_name = os.path.splitext(zfile.namelist()[0])[0] # ogr needs the full path to the unpacked 'file.shp' ds = gdal.DataSource('%s%s%s.shp' % (tmp_dir, os.path.sep, ds_name)) # shapefiles have just one layer, so grab the first... layer = ds[0] # one way of testing a sane shapefile... # further tests should be able to be plugged in here... if layer.test_capability('RandomRead'): if str(ds.driver) != 'ESRI Shapefile': return False, "Sorry, we've experienced a problem on our server. Please try again later." else: return False, 'Cannot read the shapefile, data is corrupted inside the zip, please try to upload again' # Must have a prj or have data in the -180,-90,180,90 window (assumed to be latlong) if not self.check_zip_contents('prj', zfile) and (layer.extent.min_x < -180.0 or layer.extent.max_x > 180.0 \ or layer.extent.min_y < -90.0 or layer.extent.max_y > 90.0): return False, 'You must supply a .prj file with the Shapefile to indicate the projection.' else: g = layer[0].geom if g.srs: g.transform_to(4326) ext = g.envelope if ext.min_x < -180.0 or ext.max_x > 180.0 or ext.min_y < -90.0 or ext.max_y > 90.0: return False, 'There was an error reprojecting your geometry. You must supply a .prj file or reproject to WGS84.' if layer.geom_type.name not in self.supported_geomtypes: return False, "Sorry, %s geometries are not supported. Try uploading a zipped shapefile with %s geometries" % \ (layer.geom_type.name, ', '.join(self.supported_geomtypes)) if not self.multi_feature and layer.num_feat != 1: return False, "We can only support shapefiles with a single feature" return True, "Shapefile is good to go"