def getClusterContent(self, custom_filterstring=""):
x = self.params["x"]
y = self.params["y"]
cluster = Point(x, y, srid=self.input_srid)
# request lnglat point
cell_geos = self.maptools.getCellForPointAsGeos(
cluster, self.zoom, self.gridSize, self.db_srid)
cluster.transform(self.db_srid)
query_geometry = None
for geometry in self.cache["geometries"]:
geos = GEOSGeometry(geometry)
if cluster.within(geos):
query_geometry = geos.intersection(cell_geos)
break
filterstring = self.constructFilterstring(
json.loads(self.cache["filters"]))
filterstring += custom_filterstring
kmeans_list = self.params["ids"]
kmeans_string = (",").join(str(k) for k in kmeans_list)
gis_fields_str = self.get_gis_fields_str()
sql = '''
SELECT {fields} FROM (
SELECT kmeans(ARRAY[ST_X({geo_column}), ST_Y({geo_column})], {base_k}) OVER ()
AS kmeans, "{geo_table}".*
FROM {schema_name}.{geo_table} WHERE {geo_column} IS NOT NULL
AND ST_Intersects({geo_column}, ST_GeometryFromText('{geometry}', {srid}) ) {filterstring}
) AS ksub
WHERE kmeans IN ({kmeans_string})
'''.format(geo_column=geo_column_str,
base_k=BASE_K,
geo_table=geo_table,
schema_name=self.schema_name,
geometry=query_geometry.ewkt,
srid=self.db_srid,
filterstring=filterstring,
kmeans_string=kmeans_string,
fields=gis_fields_str)
entries_queryset = Gis.objects.raw(sql)
return entries_queryset
def getClusterContent(self, custom_filterstring=""):
x = self.params["x"]
y = self.params["y"]
cluster = Point(x, y, srid=self.input_srid)
# request lnglat point
cell_geos = self.maptools.getCellForPointAsGeos(
cluster, self.zoom, self.gridSize, self.db_srid)
cluster.transform(self.db_srid)
query_geometry = None
for geometry in self.cache["geometries"]:
geos = GEOSGeometry(geometry)
if cluster.within(geos):
query_geometry = geos.intersection(cell_geos)
break
filterstring = self.constructFilterstring(
json.loads(self.cache["filters"]))
filterstring += custom_filterstring
kmeans_list = self.params["ids"]
kmeans_string = (",").join(str(k) for k in kmeans_list)
entries_queryset = Gis.objects.raw(
'''
SELECT * FROM (
SELECT kmeans(ARRAY[ST_X(%s), ST_Y(%s)], %s) OVER () AS kmeans, "%s".*
FROM "%s" WHERE %s IS NOT NULL AND ST_Intersects(%s, ST_GeometryFromText('%s', %s) ) %s
) AS ksub
WHERE kmeans IN (%s)
''' %
(geo_column_str, geo_column_str, BASE_K, geo_table, geo_table,
geo_column_str, geo_column_str, query_geometry.ewkt, self.db_srid,
filterstring, kmeans_string))
return entries_queryset
def getClusterContent(self, custom_filterstring=""):
x = self.params["x"]
y = self.params["y"]
cluster = Point(x, y, srid=self.input_srid)
# request lnglat point
cell_geos = self.maptools.getCellForPointAsGeos(cluster, self.zoom, self.gridSize, self.db_srid)
cluster.transform(self.db_srid)
query_geometry = None
for geometry in self.cache["geometries"]:
geos = GEOSGeometry(geometry)
if cluster.within(geos):
query_geometry = geos.intersection(cell_geos)
break
filterstring = self.constructFilterstring(json.loads(self.cache["filters"]))
filterstring += custom_filterstring
kmeans_list = self.params["ids"]
kmeans_string = (",").join(str(k) for k in kmeans_list)
entries_queryset = Gis.objects.raw('''
SELECT * FROM (
SELECT kmeans(ARRAY[ST_X(%s), ST_Y(%s)], %s) OVER () AS kmeans, "%s".*
FROM "%s" WHERE %s IS NOT NULL AND ST_Intersects(%s, ST_GeometryFromText('%s', %s) ) %s
) AS ksub
WHERE kmeans IN (%s)
''' %(geo_column_str, geo_column_str, BASE_K, geo_table,
geo_table, geo_column_str, geo_column_str, query_geometry.ewkt, self.db_srid, filterstring,
kmeans_string ))
return entries_queryset
def check_geodata(request):
json_layer_stringified = request.GET.get('aoi')
mpoly = GEOSGeometry(json_layer_stringified)
user_profile = Profile.objects.get(pk=request.user.pk)
user_download_area = user_profile.download_region.all()
if user_download_area.exists():
country_layer = user_download_area.filter(
geom__intersects=mpoly).aggregate(area=Union('geom'))["area"]
if country_layer is not None:
aoi_intersection_layer = mpoly.intersection(country_layer)
return JsonResponse(aoi_intersection_layer.geojson, safe=False)
else:
return JsonResponse({
"status": False,
"message":
"Please draw a rectangle inside your allowed download region.",
"title": 'Area of Interest not valid!'
})
else:
return JsonResponse(mpoly.geojson, safe=False)
def metadata_post_save(instance, *args, **kwargs):
logger.debug("handling UUID In pre_save_dataset")
if isinstance(instance, Dataset) and hasattr(settings, 'LAYER_UUID_HANDLER') and settings.LAYER_UUID_HANDLER != '':
logger.debug("using custom uuid handler In pre_save_dataset")
from ..layers.utils import get_uuid_handler
_uuid = get_uuid_handler()(instance).create_uuid()
if _uuid != instance.uuid:
instance.uuid = _uuid
Dataset.objects.filter(id=instance.id).update(uuid=_uuid)
# Fixup bbox
if instance.bbox_polygon is None:
instance.set_bbox_polygon((-180, -90, 180, 90), 'EPSG:4326')
instance.set_bounds_from_bbox(
instance.bbox_polygon,
instance.srid or instance.bbox_polygon.srid
)
# Set a default user for accountstream to work correctly.
if instance.owner is None:
instance.owner = get_valid_user()
if not instance.uuid:
instance.uuid = str(uuid.uuid1())
# set default License if no specified
if instance.license is None:
license = License.objects.filter(name="Not Specified")
if license and len(license) > 0:
instance.license = license[0]
instance.thumbnail_url = instance.get_thumbnail_url()
instance.csw_insert_date = datetime.datetime.now(timezone.get_current_timezone())
instance.set_missing_info()
ResourceBase.objects.filter(id=instance.id).update(
uuid=instance.uuid,
srid=instance.srid,
alternate=instance.alternate,
bbox_polygon=instance.bbox_polygon,
thumbnail_url=instance.get_thumbnail_url(),
csw_insert_date=datetime.datetime.now(timezone.get_current_timezone())
)
try:
if not instance.regions or instance.regions.count() == 0:
srid1, wkt1 = instance.geographic_bounding_box.split(";")
srid1 = re.findall(r'\d+', srid1)
poly1 = GEOSGeometry(wkt1, srid=int(srid1[0]))
poly1.transform(4326)
queryset = Region.objects.all().order_by('name')
global_regions = []
regions_to_add = []
for region in queryset:
try:
srid2, wkt2 = region.geographic_bounding_box.split(";")
srid2 = re.findall(r'\d+', srid2)
poly2 = GEOSGeometry(wkt2, srid=int(srid2[0]))
poly2.transform(4326)
if poly2.intersection(poly1):
regions_to_add.append(region)
if region.level == 0 and region.parent is None:
global_regions.append(region)
except Exception:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
if regions_to_add or global_regions:
if regions_to_add and len(
regions_to_add) > 0 and len(regions_to_add) <= 30:
instance.regions.add(*regions_to_add)
else:
instance.regions.add(*global_regions)
except Exception:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
finally:
# refresh catalogue metadata records
from ..catalogue.models import catalogue_post_save
catalogue_post_save(instance=instance, sender=instance.__class__)
def resourcebase_post_save(instance, *args, **kwargs):
"""
Used to fill any additional fields after the save.
Has to be called by the children
"""
# we need to remove stale links
for link in instance.link_set.all():
if link.name == "External Document":
if link.resource.doc_url != link.url:
link.delete()
else:
if urlsplit(settings.SITEURL).hostname not in link.url:
link.delete()
try:
# set default License if no specified
if instance.license is None:
license = License.objects.filter(name="Not Specified")
if license and len(license) > 0:
instance.license = license[0]
ResourceBase.objects.filter(id=instance.id).update(
thumbnail_url=instance.get_thumbnail_url(),
detail_url=instance.get_absolute_url(),
csw_insert_date=now(),
license=instance.license)
instance.refresh_from_db()
except BaseException:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
finally:
instance.set_missing_info()
try:
if instance.regions and instance.regions.all():
"""
try:
queryset = instance.regions.all().order_by('name')
for region in queryset:
print ("%s : %s" % (region.name, region.geographic_bounding_box))
except:
tb = traceback.format_exc()
else:
tb = None
finally:
if tb:
logger.debug(tb)
"""
pass
else:
srid1, wkt1 = instance.geographic_bounding_box.split(";")
srid1 = re.findall(r'\d+', srid1)
poly1 = GEOSGeometry(wkt1, srid=int(srid1[0]))
poly1.transform(4326)
queryset = Region.objects.all().order_by('name')
global_regions = []
regions_to_add = []
for region in queryset:
try:
srid2, wkt2 = region.geographic_bounding_box.split(";")
srid2 = re.findall(r'\d+', srid2)
poly2 = GEOSGeometry(wkt2, srid=int(srid2[0]))
poly2.transform(4326)
if poly2.intersection(poly1):
regions_to_add.append(region)
if region.level == 0 and region.parent is None:
global_regions.append(region)
except BaseException:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
if regions_to_add or global_regions:
if regions_to_add and len(
regions_to_add) > 0 and len(regions_to_add) <= 30:
instance.regions.add(*regions_to_add)
else:
instance.regions.add(*global_regions)
except BaseException:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
def resourcebase_post_save(instance, *args, **kwargs):
"""
Used to fill any additional fields after the save.
Has to be called by the children
"""
# we need to remove stale links
for link in instance.link_set.all():
if link.name == "External Document":
if link.resource.doc_url != link.url:
link.delete()
else:
if urlsplit(settings.SITEURL).hostname not in link.url:
link.delete()
try:
ResourceBase.objects.filter(id=instance.id).update(
thumbnail_url=instance.get_thumbnail_url(),
detail_url=instance.get_absolute_url(),
csw_insert_date=datetime.datetime.now())
except BaseException:
pass
try:
instance.thumbnail_url = instance.get_thumbnail_url()
instance.detail_url = instance.get_absolute_url()
instance.csw_insert_date = datetime.datetime.now()
finally:
instance.set_missing_info()
try:
if instance.regions and instance.regions.all():
"""
try:
queryset = instance.regions.all().order_by('name')
for region in queryset:
print ("%s : %s" % (region.name, region.geographic_bounding_box))
except:
tb = traceback.format_exc()
else:
tb = None
finally:
if tb:
logger.debug(tb)
"""
pass
else:
srid1, wkt1 = instance.geographic_bounding_box.split(";")
srid1 = re.findall(r'\d+', srid1)
poly1 = GEOSGeometry(wkt1, srid=int(srid1[0]))
poly1.transform(4326)
queryset = Region.objects.all().order_by('name')
global_regions = []
regions_to_add = []
for region in queryset:
try:
srid2, wkt2 = region.geographic_bounding_box.split(";")
srid2 = re.findall(r'\d+', srid2)
poly2 = GEOSGeometry(wkt2, srid=int(srid2[0]))
poly2.transform(4326)
if poly2.intersection(poly1):
regions_to_add.append(region)
if region.level == 0 and region.parent is None:
global_regions.append(region)
except BaseException:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
if regions_to_add or global_regions:
if regions_to_add and len(
regions_to_add) > 0 and len(regions_to_add) <= 30:
instance.regions.add(*regions_to_add)
else:
instance.regions.add(*global_regions)
except BaseException:
tb = traceback.format_exc()
if tb:
logger.debug(tb)
# set default License if no specified
if instance.license is None:
no_license = License.objects.filter(name="Not Specified")
if no_license and len(no_license) > 0:
instance.license = no_license[0]
def process(self, input_filename, output_filename,
geo_reference=None, generate_metadata=True):
# open the dataset and create an In-Memory Dataset as copy
# to perform optimizations
ds = create_mem_copy(gdal.Open(input_filename))
gt = ds.GetGeoTransform()
footprint_wkt = None
if not geo_reference:
if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
# TODO: maybe use a better check
raise ValueError("No geospatial reference for unreferenced "
"dataset given.")
else:
logger.debug("Applying geo reference '%s'."
% type(geo_reference).__name__)
ds, footprint_wkt = geo_reference.apply(ds)
# apply optimizations
for optimization in self.get_optimizations(ds):
logger.debug("Applying optimization '%s'."
% type(optimization).__name__)
try:
new_ds = optimization(ds)
if new_ds is not ds:
# cleanup afterwards
cleanup_temp(ds)
ds = new_ds
except:
cleanup_temp(ds)
raise
# generate the footprint from the dataset
if not footprint_wkt:
logger.debug("Generating footprint.")
footprint_wkt = self._generate_footprint_wkt(ds)
# check that footprint is inside of extent of generated image
# regenerate otherwise
else:
tmp_extent = getExtentFromRectifiedDS(ds)
tmp_bbox = Polygon.from_bbox((tmp_extent[0], tmp_extent[1],
tmp_extent[2], tmp_extent[3]))
tmp_footprint = GEOSGeometry(footprint_wkt)
if not tmp_bbox.contains(tmp_footprint):
footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt
if self.footprint_alpha:
logger.debug("Applying optimization 'AlphaBandOptimization'.")
opt = AlphaBandOptimization()
opt(ds, footprint_wkt)
output_filename = self.generate_filename(output_filename)
logger.debug("Writing file to disc using options: %s."
% ", ".join(self.format_selection.creation_options))
logger.debug("Metadata tags to be written: %s"
% ", ".join(ds.GetMetadata_List("") or []))
# save the file to the disc
driver = gdal.GetDriverByName(self.format_selection.driver_name)
ds = driver.CreateCopy(output_filename, ds,
options=self.format_selection.creation_options)
for optimization in self.get_post_optimizations(ds):
logger.debug("Applying post-optimization '%s'."
% type(optimization).__name__)
optimization(ds)
# generate metadata if requested
footprint = None
if generate_metadata:
normalized_space = Polygon.from_bbox((-180, -90, 180, 90))
non_normalized_space = Polygon.from_bbox((180, -90, 360, 90))
footprint = GEOSGeometry(footprint_wkt)
#.intersection(normalized_space)
outer = non_normalized_space.intersection(footprint)
if len(outer):
footprint = MultiPolygon(
*map(lambda p:
Polygon(*map(lambda ls:
LinearRing(*map(lambda point:
(point[0] - 360, point[1]), ls.coords
)), tuple(p)
)), (outer,)
)
).union(normalized_space.intersection(footprint))
else:
if isinstance(footprint, Polygon):
footprint = MultiPolygon(footprint)
logger.info("Calculated Footprint: '%s'" % footprint.wkt)
# use the provided footprint
#geom = OGRGeometry(footprint_wkt)
#exterior = []
#for x, y in geom.exterior_ring.tuple:
# exterior.append(y); exterior.append(x)
#polygon = [exterior]
num_bands = ds.RasterCount
# finally close the dataset and write it to the disc
ds = None
return PreProcessResult(output_filename, footprint, num_bands)
def process(self,
input_filename,
output_filename,
geo_reference=None,
generate_metadata=True,
merge_with=None,
original_footprint=None):
# open the dataset and create an In-Memory Dataset as copy
# to perform optimizations
ds = create_mem_copy(gdal.Open(input_filename))
gt = ds.GetGeoTransform()
footprint_wkt = None
if not geo_reference:
if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
if ds.GetGCPCount() > 0:
geo_reference = InternalGCPs()
else:
raise ValueError("No geospatial reference for "
"unreferenced dataset given.")
if geo_reference:
logger.debug("Applying geo reference '%s'." %
type(geo_reference).__name__)
# footprint is always in EPSG:4326
ds, footprint_wkt = geo_reference.apply(ds)
# apply optimizations
for optimization in self.get_optimizations(ds):
logger.debug("Applying optimization '%s'." %
type(optimization).__name__)
try:
new_ds = optimization(ds)
if new_ds is not ds:
# cleanup afterwards
cleanup_temp(ds)
ds = new_ds
except:
cleanup_temp(ds)
raise
# generate the footprint from the dataset
if not footprint_wkt:
logger.debug("Generating footprint.")
footprint_wkt = self._generate_footprint_wkt(ds)
# check that footprint is inside of extent of generated image
# regenerate otherwise
else:
tmp_extent = getExtentFromRectifiedDS(ds)
tmp_bbox = Polygon.from_bbox(
(tmp_extent[0], tmp_extent[1], tmp_extent[2], tmp_extent[3]))
# transform image bbox to EPSG:4326 if necessary
proj = ds.GetProjection()
srs = osr.SpatialReference()
try:
srs.ImportFromWkt(proj)
srs.AutoIdentifyEPSG()
ptype = "PROJCS" if srs.IsProjected() else "GEOGCS"
srid = int(srs.GetAuthorityCode(ptype))
if srid != '4326':
out_srs = osr.SpatialReference()
out_srs.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(srs, out_srs)
tmp_bbox2 = ogr.CreateGeometryFromWkt(tmp_bbox.wkt)
tmp_bbox2.Transform(transform)
tmp_bbox = GEOSGeometry(tmp_bbox2.ExportToWkt())
except (RuntimeError, TypeError), e:
logger.warn("Projection: %s" % proj)
logger.warn("Failed to identify projection's EPSG code."
"%s: %s" % (type(e).__name__, str(e)))
tmp_footprint = GEOSGeometry(footprint_wkt)
if not tmp_bbox.contains(tmp_footprint):
logger.debug("Re-generating footprint because not inside of "
"generated image.")
footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt
def process(self, input_filename, output_filename,
geo_reference=None, generate_metadata=True):
# open the dataset and create an In-Memory Dataset as copy
# to perform optimizations
ds = create_mem_copy(gdal.Open(input_filename))
gt = ds.GetGeoTransform()
footprint_wkt = None
if not geo_reference:
if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
# TODO: maybe use a better check
raise ValueError("No geospatial reference for unreferenced "
"dataset given.")
else:
logger.debug("Applying geo reference '%s'."
% type(geo_reference).__name__)
ds, footprint_wkt = geo_reference.apply(ds)
# apply optimizations
for optimization in self.get_optimizations(ds):
logger.debug("Applying optimization '%s'."
% type(optimization).__name__)
try:
new_ds = optimization(ds)
if new_ds is not ds:
# cleanup afterwards
cleanup_temp(ds)
ds = new_ds
except:
cleanup_temp(ds)
raise
# generate the footprint from the dataset
if not footprint_wkt:
logger.debug("Generating footprint.")
footprint_wkt = self._generate_footprint_wkt(ds)
# check that footprint is inside of extent of generated image
# regenerate otherwise
else:
tmp_extent = getExtentFromRectifiedDS(ds)
tmp_bbox = Polygon.from_bbox((tmp_extent[0], tmp_extent[1],
tmp_extent[2], tmp_extent[3]))
tmp_footprint = GEOSGeometry(footprint_wkt)
if not tmp_bbox.contains(tmp_footprint):
footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt
if self.footprint_alpha:
logger.debug("Applying optimization 'AlphaBandOptimization'.")
opt = AlphaBandOptimization()
opt(ds, footprint_wkt)
output_filename = self.generate_filename(output_filename)
logger.debug("Writing file to disc using options: %s."
% ", ".join(self.format_selection.creation_options))
logger.debug("Metadata tags to be written: %s"
% ", ".join(ds.GetMetadata_List("") or []))
# save the file to the disc
driver = gdal.GetDriverByName(self.format_selection.driver_name)
ds = driver.CreateCopy(output_filename, ds,
options=self.format_selection.creation_options)
for optimization in self.get_post_optimizations(ds):
logger.debug("Applying post-optimization '%s'."
% type(optimization).__name__)
optimization(ds)
# generate metadata if requested
footprint = None
if generate_metadata:
normalized_space = Polygon.from_bbox((-180, -90, 180, 90))
non_normalized_space = Polygon.from_bbox((180, -90, 360, 90))
footprint = GEOSGeometry(footprint_wkt)
#.intersection(normalized_space)
outer = non_normalized_space.intersection(footprint)
if len(outer):
footprint = MultiPolygon(
*map(lambda p:
Polygon(*map(lambda ls:
LinearRing(*map(lambda point:
(point[0] - 360, point[1]), ls.coords
)), tuple(p)
)), (outer,)
)
).union(normalized_space.intersection(footprint))
else:
if isinstance(footprint, Polygon):
footprint = MultiPolygon(footprint)
logger.info("Calculated Footprint: '%s'" % footprint.wkt)
# use the provided footprint
#geom = OGRGeometry(footprint_wkt)
#exterior = []
#for x, y in geom.exterior_ring.tuple:
# exterior.append(y); exterior.append(x)
#polygon = [exterior]
num_bands = ds.RasterCount
# finally close the dataset and write it to the disc
ds = None
return PreProcessResult(output_filename, footprint, num_bands)
def process(self, input_filename, output_filename,
geo_reference=None, generate_metadata=True,
merge_with=None, original_footprint=None):
# open the dataset and create an In-Memory Dataset as copy
# to perform optimizations
ds = create_mem_copy(gdal.Open(input_filename))
gt = ds.GetGeoTransform()
footprint_wkt = None
if not geo_reference:
if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
if ds.GetGCPCount() > 0:
geo_reference = InternalGCPs()
else:
raise ValueError("No geospatial reference for "
"unreferenced dataset given.")
if geo_reference:
logger.debug("Applying geo reference '%s'."
% type(geo_reference).__name__)
# footprint is always in EPSG:4326
ds, footprint_wkt = geo_reference.apply(ds)
# apply optimizations
for optimization in self.get_optimizations(ds):
logger.debug("Applying optimization '%s'."
% type(optimization).__name__)
try:
new_ds = optimization(ds)
if new_ds is not ds:
# cleanup afterwards
cleanup_temp(ds)
ds = new_ds
except:
cleanup_temp(ds)
raise
# generate the footprint from the dataset
if not footprint_wkt:
logger.debug("Generating footprint.")
footprint_wkt = self._generate_footprint_wkt(ds)
# check that footprint is inside of extent of generated image
# regenerate otherwise
else:
tmp_extent = getExtentFromRectifiedDS(ds)
tmp_bbox = Polygon.from_bbox((tmp_extent[0], tmp_extent[1],
tmp_extent[2], tmp_extent[3]))
# transform image bbox to EPSG:4326 if necessary
proj = ds.GetProjection()
srs = osr.SpatialReference()
try:
srs.ImportFromWkt(proj)
srs.AutoIdentifyEPSG()
ptype = "PROJCS" if srs.IsProjected() else "GEOGCS"
srid = int(srs.GetAuthorityCode(ptype))
if srid != '4326':
out_srs = osr.SpatialReference()
out_srs.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(srs, out_srs)
tmp_bbox2 = ogr.CreateGeometryFromWkt(tmp_bbox.wkt)
tmp_bbox2.Transform(transform)
tmp_bbox = GEOSGeometry(tmp_bbox2.ExportToWkt())
except (RuntimeError, TypeError), e:
logger.warn("Projection: %s" % proj)
logger.warn("Failed to identify projection's EPSG code."
"%s: %s" % ( type(e).__name__ , str(e) ) )
tmp_footprint = GEOSGeometry(footprint_wkt)
if not tmp_bbox.contains(tmp_footprint):
footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt