def get_source_dataset(self, coverage, data_items, range_type):
if len(data_items) == 1:
return gdal.OpenShared(abspath(connect(data_items[0])))
else:
vrt = VRTBuilder(coverage.size_x,
coverage.size_y,
vrt_filename=temp_vsimem_filename())
# sort in ascending order according to semantic
data_items = sorted(data_items, key=(lambda d: d.semantic))
gcps = []
compound_index = 0
for data_item in data_items:
path = abspath(connect(data_item))
# iterate over all bands of the data item
for set_index, item_index in self._data_item_band_indices(
data_item):
if set_index != compound_index + 1:
raise ValueError
compound_index = set_index
band = range_type[set_index]
vrt.add_band(band.data_type)
vrt.add_simple_source(set_index, path, item_index)
return vrt.dataset
def get_source_dataset(self, coverage, data_items, range_type):
if len(data_items) == 1:
return gdal.OpenShared(abspath(connect(data_items[0])))
else:
vrt = VRTBuilder(
coverage.size_x, coverage.size_y,
vrt_filename=temp_vsimem_filename()
)
# sort in ascending order according to semantic
data_items = sorted(data_items, key=(lambda d: d.semantic))
gcps = []
compound_index = 0
for data_item in data_items:
path = abspath(connect(data_item))
# iterate over all bands of the data item
for set_index, item_index in self._data_item_band_indices(data_item):
if set_index != compound_index + 1:
raise ValueError
compound_index = set_index
band = range_type[set_index]
vrt.add_band(band.data_type)
vrt.add_simple_source(
set_index, path, item_index
)
return vrt.dataset
def connect(self, coverage, data_items, layer, options):
mask_item = data_items[0]
try:
is_reversed = (
layer.metadata.get("eoxs_geometry_reversed") == "true"
)
except ms.MapServerError:
is_reversed = False
# check if the geometry is "reversed"
if is_reversed:
# TODO: better use the coverages Extent?
geom_types = (ogr.wkbPolygon, ogr.wkbMultiPolygon)
output_polygon = ogr.Geometry(wkt=str(coverage.footprint.wkt))
for mask_item in data_items:
ds = ogr.Open(connect(mask_item))
for i in range(ds.GetLayerCount()):
ogr_layer = ds.GetLayer(i)
if not ogr_layer:
continue
feature = ogr_layer.GetNextFeature()
while feature:
# TODO: reproject if necessary
geometry = feature.GetGeometryRef()
if geometry.GetGeometryType() not in geom_types:
continue
if geometry:
output_polygon = output_polygon.Difference(geometry)
feature = ogr_layer.GetNextFeature()
# since we have the geometry already in memory, add it to the layer
# as WKT
shape = ms.shapeObj.fromWKT(output_polygon.ExportToWkt())
shape.initValues(1)
shape.setValue(0, coverage.identifier)
layer.addFeature(shape)
else:
layer.connectiontype = ms.MS_OGR
layer.connection = connect(data_items[0])
# TODO: more than one mask_item?
layer.setProjection("EPSG:4326")
layer.setMetaData("ows_srs", "EPSG:4326")
layer.setMetaData("wms_srs", "EPSG:4326")
def connect(self, coverage, data_items, layer, options):
mask_item = data_items[0]
try:
is_reversed = (
layer.metadata.get("eoxs_geometry_reversed") == "true")
except ms.MapServerError:
is_reversed = False
# check if the geometry is "reversed"
if is_reversed:
# TODO: better use the coverages Extent?
geom_types = (ogr.wkbPolygon, ogr.wkbMultiPolygon)
output_polygon = ogr.Geometry(wkt=str(coverage.footprint.wkt))
for mask_item in data_items:
ds = ogr.Open(connect(mask_item))
for i in range(ds.GetLayerCount()):
ogr_layer = ds.GetLayer(i)
if not ogr_layer:
continue
feature = ogr_layer.GetNextFeature()
while feature:
# TODO: reproject if necessary
geometry = feature.GetGeometryRef()
if geometry.GetGeometryType() not in geom_types:
continue
if geometry:
output_polygon = output_polygon.Difference(
geometry)
feature = ogr_layer.GetNextFeature()
# since we have the geometry already in memory, add it to the layer
# as WKT
shape = ms.shapeObj.fromWKT(output_polygon.ExportToWkt())
shape.initValues(1)
shape.setValue(0, coverage.identifier)
layer.addFeature(shape)
else:
layer.connectiontype = ms.MS_OGR
layer.connection = connect(data_items[0])
# TODO: more than one mask_item?
layer.setProjection("EPSG:4326")
layer.setMetaData("ows_srs", "EPSG:4326")
layer.setMetaData("wms_srs", "EPSG:4326")
def connect(self, coverage, data_items, layer, options):
filtered = filter(lambda d: d.semantic.startswith("bands"), data_items)
data = connect(filtered[0])
if isinstance(coverage, models.ReferenceableDataset):
vrt_path = join("/vsimem", uuid4().hex)
reftools.create_rectified_vrt(data, vrt_path)
data = vrt_path
layer.setMetaData("eoxs_ref_data", data)
if not layer.metadata.get("eoxs_wrap_dateline") == "true":
layer.data = data
else:
e = wrap_extent_around_dateline(coverage.extent, coverage.srid)
vrt_path = join("/vsimem", uuid4().hex)
ds = gdal.Open(data)
vrt_ds = create_simple_vrt(ds, vrt_path)
size_x = ds.RasterXSize
size_y = ds.RasterYSize
dx = abs(e[0] - e[2]) / size_x
dy = abs(e[1] - e[3]) / size_y
vrt_ds.SetGeoTransform([e[0], dx, 0, e[3], 0, -dy])
vrt_ds = None
layer.data = vrt_path
def connect(self, coverage, data_items, layer):
filtered = filter(lambda d: d.semantic.startswith("bands"), data_items)
data = connect(filtered[0])
if isinstance(coverage, models.ReferenceableDataset):
vrt_path = join("/vsimem", uuid4().hex)
reftools.create_rectified_vrt(data, vrt_path)
data = vrt_path
layer.setMetaData("eoxs_ref_data", data)
if not layer.metadata.get("eoxs_wrap_dateline") == "true":
layer.data = data
else:
e = wrap_extent_around_dateline(coverage.extent, coverage.srid)
vrt_path = join("/vsimem", uuid4().hex)
ds = gdal.Open(data)
vrt_ds = create_simple_vrt(ds, vrt_path)
size_x = ds.RasterXSize
size_y = ds.RasterYSize
dx = abs(e[0] - e[2]) / size_x
dy = abs(e[1] - e[3]) / size_y
vrt_ds.SetGeoTransform([e[0], dx, 0, e[3], 0, -dy])
vrt_ds = None
layer.data = vrt_path
def coverage(request, identifier):
cov = Coverage.objects.get(identifier=identifier)
filename = connect(cov.data_items.filter(semantic__startswith="band")[0])
size = getsize(filename)
if size > (100 * 1024 * 1024):
tmpfile = NamedTemporaryFile(suffix='.tif')
cmd_args = [
'gdal_translate', '-outsize', '7%', '7%', filename, tmpfile.name
]
process = check_output(cmd_args)
#returnfile = FileWrapper(tmpfile)
returnfile = tmpfile
size = getsize(tmpfile.name)
else:
returnfile = open(filename)
response = HttpResponse(returnfile)
response['Content-Type'] = 'image/tif'
response['Content-Length'] = size
response['Content-Disposition'] = 'attachment; filename=%s' % basename(
filename)
response['Cache-Control'] = 'max-age=%d' % MAX_AGE
return response
def coverage(request, identifier):
cov = Coverage.objects.get(identifier=identifier)
filename = connect(cov.data_items.filter(semantic__startswith="band")[0])
size = getsize(filename)
if size>(100*1024*1024):
tmpfile = NamedTemporaryFile(suffix='.tif')
cmd_args = ['gdal_translate', '-outsize', '7%', '7%', filename, tmpfile.name]
process = check_output(cmd_args)
#returnfile = FileWrapper(tmpfile)
returnfile = tmpfile
size = getsize(tmpfile.name)
else:
returnfile = open(filename)
response = HttpResponse(returnfile)
response['Content-Type'] = 'image/tif'
response['Content-Length'] = size
response['Content-Disposition'] = 'attachment; filename=%s' % basename(filename)
response['Cache-Control'] = 'max-age=%d' % MAX_AGE
return response
def apply(self, coverage, data_items, layer):
sld_items = filter(lambda d: (
d.semantic.startswith("style") and d.format.upper() == "SLD"
), data_items)
for sld_item in sld_items:
sld_filename = connect(sld_item)
with open(sld_filename) as f:
layer.map.applySLD(f.read())
def apply(self, coverage, data_items, layer):
sld_items = filter(
lambda d:
(d.semantic.startswith("style") and d.format.upper() == "SLD"),
data_items)
for sld_item in sld_items:
sld_filename = connect(sld_item)
with open(sld_filename) as f:
layer.map.applySLD(f.read())
def _read_metadata_from_data(self, data_item, retrieved_metadata, cache):
metadata_component = MetadataComponent(env)
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
retrieved_metadata.setdefault(key, value)
ds = None
def _read_data(self, coverage, subset, rangesubset):
range_type = coverage.range_type
# Open file
filename = connect(coverage.data_items.all()[0])
root = etree.parse(filename).getroot()
output_data = OrderedDict()
# Read data
band = range_type[0]
if not rangesubset or band in rangesubset:
data = map(float, root.xpath("data/value/text()"))
print data, root
data = data[int(subset.low):int(subset.high)]
output_data[band.identifier] = data
return output_data
def _read_data(self, coverage, subset, rangesubset):
range_type = coverage.range_type
# Open file
filename = connect(coverage.data_items.all()[0])
root = etree.parse(filename).getroot()
output_data = OrderedDict()
# Read data
band = range_type[0]
if not rangesubset or band in rangesubset:
data = map(float, root.xpath("data/value/text()"))
print data, root
data = data[int(subset.low):int(subset.high)]
output_data[band.identifier] = data
return output_data
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
# replace any already registered dataset
if kwargs["replace"]:
try:
# get a list of all collections the coverage was in.
coverage = models.Coverage.objects.get(
identifier=retrieved_metadata["identifier"]
)
additional_ids = [
c.identifier
for c in models.Collection.objects.filter(
eo_objects__in=[coverage.pk]
)
]
coverage.delete()
self.print_msg(
"Replacing previous dataset '%s'."
% retrieved_metadata["identifier"]
)
collection_ids = kwargs["collection_ids"] or []
for identifier in additional_ids:
if identifier not in collection_ids:
collection_ids.append(identifier)
kwargs["collection_ids"] = collection_ids
except models.Coverage.DoesNotExist:
self.print_msg(
"Could not replace previous dataset '%s'."
% retrieved_metadata["identifier"]
)
try:
# TODO: allow types of different apps
CoverageType = getattr(models, retrieved_metadata["coverage_type"])
except AttributeError:
raise CommandError(
"Type '%s' is not supported." % kwargs["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
def execute(self, collections, begin_time, end_time, coord_list, srid):
""" The main execution function for the process.
"""
eo_ids = collections.split(',')
containment = "overlaps"
subsets = Subsets((Trim("t", begin_time, end_time),))
if len(eo_ids) == 0:
raise
# fetch a list of all requested EOObjects
available_ids = models.EOObject.objects.filter(
identifier__in=eo_ids
).values_list("identifier", flat=True)
# match the requested EOIDs against the available ones. If any are
# requested, that are not available, raise and exit.
failed = [ eo_id for eo_id in eo_ids if eo_id not in available_ids ]
if failed:
raise NoSuchDatasetSeriesOrCoverageException(failed)
collections_qs = subsets.filter(models.Collection.objects.filter(
identifier__in=eo_ids
), containment="overlaps")
# create a set of all indirectly referenced containers by iterating
# recursively. The containment is set to "overlaps", to also include
# collections that might have been excluded with "contains" but would
# have matching coverages inserted.
def recursive_lookup(super_collection, collection_set):
sub_collections = models.Collection.objects.filter(
collections__in=[super_collection.pk]
).exclude(
pk__in=map(lambda c: c.pk, collection_set)
)
sub_collections = subsets.filter(sub_collections, "overlaps")
# Add all to the set
collection_set |= set(sub_collections)
for sub_collection in sub_collections:
recursive_lookup(sub_collection, collection_set)
collection_set = set(collections_qs)
for collection in set(collection_set):
recursive_lookup(collection, collection_set)
collection_pks = map(lambda c: c.pk, collection_set)
# Get all either directly referenced coverages or coverages that are
# within referenced containers. Full subsetting is applied here.
coverages_qs = subsets.filter(models.Coverage.objects.filter(
Q(identifier__in=eo_ids) | Q(collections__in=collection_pks)
), containment=containment)
coordinates = coord_list.split(';')
points = []
for coordinate in coordinates:
x,y = coordinate.split(',')
# parameter parsing
point = Point(float(x), float(y))
point.srid = srid
points.append(point)
points = MultiPoint(points)
points.srid = srid
eo_objects = coverages_qs.filter(
footprint__intersects=points
).order_by('begin_time')
output = StringIO()
writer = csv.writer(output, quoting=csv.QUOTE_NONE)
header = ["id", "time", "val"]
writer.writerow(header)
for eo_object in eo_objects:
coverage = eo_object.cast()
#layer = models.DatasetSeries.objects.get(identifier__in=coverage.identifier)
layer = eo_object.collections.all()[0]
time = isoformat(coverage.begin_time)
data_item = coverage.data_items.get(semantic__startswith="bands")
filename = connect(data_item)
ds = gdal.Open(filename)
if ds.GetProjection():
gt = ds.GetGeoTransform()
sr = SpatialReference(ds.GetProjection())
points_t = points.transform(sr, clone=True)
else:
bbox = coverage.footprint.extent
gt = [
bbox[0], (bbox[2] - bbox[0])/ds.RasterXSize, 0,
bbox[3], 0, (bbox[1] - bbox[3])/ds.RasterYSize
]
for index, point in enumerate(points, start=1):
print index, point
if not coverage.footprint.contains(point):
continue
#point.transform(sr)
# Works only if gt[2] and gt[4] equal zero!
px = int((point[0] - gt[0]) / gt[1]) #x pixel
py = int((point[1] - gt[3]) / gt[5]) #y pixel
pixelVal = ds.GetRasterBand(1).ReadAsArray(px,py,1,1)[0,0]
if pixelVal != -9999:
writer.writerow([ str(layer.identifier), time, pixelVal])
return {
"processed": output.getvalue()
}
def execute(self, collection, begin_time, end_time, coord_list, srid):
""" The main execution function for the process.
"""
col_name = collection
collection = models.Collection.objects.get(identifier=collection)
eo_objects = collection.eo_objects.filter(
begin_time__lte=end_time, end_time__gte=begin_time
)
coordinates = coord_list.split(';')
points = []
for coordinate in coordinates:
x,y = coordinate.split(',')
# parameter parsing
point = Point(float(x), float(y))
point.srid = srid
points.append(point)
points = MultiPoint(points)
points.srid = srid
eo_objects = eo_objects.filter(
footprint__intersects=points
)
output = StringIO()
writer = csv.writer(output, quoting=csv.QUOTE_ALL)
#header = ["id", "begin", "end"] + ["point%d" % i for i in range(len(points))]
header = ["id", "Green", "Red", "NIR", "MIR" ]
writer.writerow(header)
for eo_object in eo_objects:
coverage = eo_object.cast()
#values = [coverage.identifier, isoformat(coverage.begin_time), isoformat(coverage.end_time)] + [None] * len(points)
values = [collection] + [None] * 4
data_item = coverage.data_items.get(semantic__startswith="bands")
filename = connect(data_item)
ds = gdal.Open(filename)
sr = SpatialReference(ds.GetProjection())
#points_t = points.transform(sr, clone=True)
for index, point in enumerate(points, start=1):
if not coverage.footprint.contains(point):
continue
gt = ds.GetGeoTransform()
point.transform(sr)
# Works only if gt[2] and gt[4] equal zero!
px = int((point[0] - gt[0]) / gt[1]) #x pixel
py = int((point[1] - gt[3]) / gt[5]) #y pixel
#array = ds.ReadRaster(px, py, 1, 1)
#structval = ds.ReadRaster(px,py,1,1,buf_type=gdal.GDT_Int16) #TODO: Check Range Type to adapt buf_type!
pixelVal = ds.ReadAsArray(px,py,1,1)[:,0,0]
#pixel_value = array[0][0]
#print structval
#pixel_value = struct.unpack('IIII' , structval) #use the 'short' format code (2 bytes) not int (4 bytes)
#values[index] = pixel_value[0]
#writer.writerow([ col_name+"_p"+str(index), pixelVal[0], pixelVal[1], pixelVal[2], pixelVal[3] ])
writer.writerow([ "P_"+str(index), pixelVal[0], pixelVal[1], pixelVal[2], pixelVal[3] ])
return {
"processed": output.getvalue()
}
def create_diff_label(self, master_id, slave_id, bbox, num_bands, crs, unit):
""" The main execution function for the process.
"""
#srid = crss.parseEPSGCode(str(crs), (crss.fromShortCode, crss.fromURN, crss.fromURL))
master = models.RectifiedDataset.objects.get(identifier=master_id)
slave = models.RectifiedDataset.objects.get(identifier=slave_id)
filename_master = connect(master.data_items.get(semantic__startswith="bands"))
filename_slave = connect(slave.data_items.get(semantic__startswith="bands"))
ds_master = gdal.Open(filename_master, gdalconst.GA_ReadOnly)
ds_slave = gdal.Open(filename_slave, gdalconst.GA_ReadOnly)
master_bbox = master.footprint.extent
slave_bbox = slave.footprint.extent
res_x_master = (master_bbox[2] - master_bbox[0]) / ds_master.RasterXSize
res_y_master = (master_bbox[3] - master_bbox[1]) / ds_master.RasterYSize
res_x_slave = (slave_bbox[2] - slave_bbox[0]) / ds_slave.RasterXSize
res_y_slave = (slave_bbox[3] - slave_bbox[1]) / ds_slave.RasterYSize
size_x = int((bbox[2]-bbox[0])/res_x_master)
size_y = int((bbox[3]-bbox[1])/res_y_master)
builder = VRTBuilder(size_x, size_y, (num_bands*2), master.range_type.bands.all()[0].data_type)
dst_rect_master = (
int( math.floor((master_bbox[0] - bbox[0]) / res_x_master) ), # x offset
int( math.floor((bbox[3] - master_bbox[3]) / res_y_master) ), # y offset
ds_master.RasterXSize, # x size
ds_master.RasterYSize # y size
)
dst_rect_slave = (
int( math.floor((slave_bbox[0] - bbox[0]) / res_x_slave) ), # x offset
int( math.floor((bbox[3] - slave_bbox[3]) / res_y_slave) ), # y offset
ds_slave.RasterXSize, # x size
ds_slave.RasterYSize # y size
)
for i in range(1, num_bands+1):
builder.add_simple_source(i, str(filename_master), i, src_rect=(0, 0, ds_master.RasterXSize, ds_master.RasterYSize), dst_rect=dst_rect_master)
builder.add_simple_source(num_bands+i , str(filename_slave), i, src_rect=(0, 0, ds_slave.RasterXSize, ds_slave.RasterYSize), dst_rect=dst_rect_slave)
ext = Rect(0,0,size_x, size_y)
pix_master = builder.dataset.GetRasterBand(1).ReadAsArray()
pix_slave = builder.dataset.GetRasterBand(num_bands +1).ReadAsArray()
if num_bands == 1:
pix_master = np.dstack((pix_master, builder.dataset.GetRasterBand(1).ReadAsArray()))
pix_slave = np.dstack((pix_slave, builder.dataset.GetRasterBand(2).ReadAsArray()))
else:
for i in range(2, num_bands+1):
pix_master = np.dstack((pix_master, builder.dataset.GetRasterBand(i).ReadAsArray()))
pix_slave = np.dstack((pix_slave, builder.dataset.GetRasterBand(num_bands+i).ReadAsArray()))
def _diff(a,b):
d = np.array(a[:,:,0],'float32') - np.array(b[:,:,0],'float32')
return d
pix_res = _diff(pix_master, pix_slave)
res_max = np.max(pix_res)
res_min = np.min(pix_res)
# Make a figure and axes with dimensions as desired.
fig = pyplot.figure(figsize=(8,1))
fig.patch.set_alpha(0.8)
ax1 = fig.add_axes([0.05, 0.75, 0.9, 0.15])
def savefig_pix(fig,fname,width,height,dpi=100):
rdpi = 1.0/float(dpi)
fig.set_size_inches(width*rdpi,height*rdpi)
fig.savefig(fname,dpi=dpi)
# Set the colormap and norm to correspond to the data for which
# the colorbar will be used.
cmap = mpl.cm.RdBu
#norm = mpl.colors.Normalize(vmin=res_min, vmax=res_max)
res_ = max(abs(res_max), abs(res_min))
norm = mpl.colors.Normalize(vmin=-res_, vmax=res_)
cb1 = mpl.colorbar.ColorbarBase(ax1, cmap=cmap,
norm=norm,
orientation='horizontal')
mis = master_id.split("_")
master_id_label = " ".join( (mis[0], mis[1], mis[2], isoformat(master.begin_time)) )
sis = slave_id.split("_")
slave_id_label = " ".join( (sis[0], sis[1], sis[2], isoformat(slave.begin_time)) )
if unit:
label = "Difference from %s \n to %s; Unit: %s"%(slave_id_label, master_id_label, unit)
else:
label = "Difference from %s \n to %s"%(slave_id_label, master_id_label)
cb1.set_label(label)
# the output image
basename = "%s_%s"%( self.identifier,uuid4().hex )
filename_png = "/tmp/%s.png" %( basename )
try:
fig.savefig(filename_png, dpi=80)
with open(filename_png) as f:
output = f.read()
except Exception as e:
if os.path.isfile(filename_png):
os.remove(filename_png)
raise e
else:
os.remove(filename_png)
return output
def diff_process(self, master_id, slave_id, bbox, num_bands, crs):
""" The main execution function for the process.
"""
#srid = crss.parseEPSGCode(str(crs), (crss.fromShortCode, crss.fromURN, crss.fromURL))
master = models.RectifiedDataset.objects.get(identifier=master_id)
slave = models.RectifiedDataset.objects.get(identifier=slave_id)
filename_master = connect(master.data_items.get(semantic__startswith="bands"))
filename_slave = connect(slave.data_items.get(semantic__startswith="bands"))
ds_master = gdal.Open(filename_master, gdalconst.GA_ReadOnly)
ds_slave = gdal.Open(filename_slave, gdalconst.GA_ReadOnly)
master_bbox = master.footprint.extent
slave_bbox = slave.footprint.extent
res_x_master = (master_bbox[2] - master_bbox[0]) / ds_master.RasterXSize
res_y_master = (master_bbox[3] - master_bbox[1]) / ds_master.RasterYSize
res_x_slave = (slave_bbox[2] - slave_bbox[0]) / ds_slave.RasterXSize
res_y_slave = (slave_bbox[3] - slave_bbox[1]) / ds_slave.RasterYSize
size_x = int((bbox[2]-bbox[0])/res_x_master)
size_y = int((bbox[3]-bbox[1])/res_y_master)
builder = VRTBuilder(size_x, size_y, (num_bands*2), master.range_type.bands.all()[0].data_type)
dst_rect_master = (
int( math.floor((master_bbox[0] - bbox[0]) / res_x_master) ), # x offset
int( math.floor((bbox[3] - master_bbox[3]) / res_y_master) ), # y offset
ds_master.RasterXSize, # x size
ds_master.RasterYSize # y size
)
dst_rect_slave = (
int( math.floor((slave_bbox[0] - bbox[0]) / res_x_slave) ), # x offset
int( math.floor((bbox[3] - slave_bbox[3]) / res_y_slave) ), # y offset
ds_slave.RasterXSize, # x size
ds_slave.RasterYSize # y size
)
for i in range(1, num_bands+1):
builder.add_simple_source(i, str(filename_master), i, src_rect=(0, 0, ds_master.RasterXSize, ds_master.RasterYSize), dst_rect=dst_rect_master)
builder.add_simple_source(num_bands+i , str(filename_slave), i, src_rect=(0, 0, ds_slave.RasterXSize, ds_slave.RasterYSize), dst_rect=dst_rect_slave)
ext = Rect(0,0,size_x, size_y)
pix_master = builder.dataset.GetRasterBand(1).ReadAsArray()
pix_slave = builder.dataset.GetRasterBand(num_bands +1).ReadAsArray()
if num_bands == 1:
pix_master = np.dstack((pix_master, builder.dataset.GetRasterBand(1).ReadAsArray()))
pix_slave = np.dstack((pix_slave, builder.dataset.GetRasterBand(2).ReadAsArray()))
else:
for i in range(2, num_bands+1):
pix_master = np.dstack((pix_master, builder.dataset.GetRasterBand(i).ReadAsArray()))
pix_slave = np.dstack((pix_slave, builder.dataset.GetRasterBand(num_bands+i).ReadAsArray()))
def _diff(a,b):
d = np.array(a[:,:,0],'float32') - np.array(b[:,:,0],'float32')
return d
pix_res = _diff(pix_master, pix_slave)
res_max = np.max(pix_res)
res_min = np.min(pix_res)
res_ = max(abs(res_max), abs(res_min))
# the output image
basename = "%s_%s"%( self.identifier,uuid4().hex )
filename_png = "/tmp/%s.png" %( basename )
try:
#fig = pyplot.imshow(pix_res,interpolation='nearest')
fig = pyplot.imshow(pix_res,vmin=-res_, vmax=res_, interpolation='nearest')
fig.set_cmap('RdBu')
fig.write_png(filename_png, True)
with open(filename_png) as f:
output = f.read()
except Exception as e:
if os.path.isfile(filename_png):
os.remove(filename_png)
raise e
else:
os.remove(filename_png)
#return base64.b64encode(output)
return output
def connect(self, coverage, data_items, layer, options):
layer.tileindex = os.path.abspath(connect(data_items[0]))
layer.tileitem = "location"
def handle_with_cache(self, cache, *args, **kwargs):
#----------------------------------------------------------------------
# check the inputs
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
# TODO: not required, as the keys are already
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time"
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
#----------------------------------------------------------------------
# parent dataset series
# extract the parents
ignore_missing_parent = bool(kwargs.get('ignore_missing_parent',False))
parents = []
for parent_id in kwargs.get('parents',[]):
try :
ds = models.DatasetSeries.objects.get(identifier=parent_id)
parents.append( ds )
except models.DatasetSeries.DoesNotExist :
msg ="There is no Dataset Series matching the given" \
" identifier: '%s' "%parent_id
if ignore_missing_parent :
self.print_wrn( msg )
else :
raise CommandError( msg )
#----------------------------------------------------------------------
# meta-data
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(metadata)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
try:
CoverageType = getattr(models, kwargs["coverage_type"])
except:
pass
# TODO: split into module path/coverage and get correct coverage class
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
retrieved_metadata["projection"] = models.Projection.objects.get(format=format, definition=definition)
#coverage.identifier = identifier # TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
#------------------------------------------------------------------
# link to the parent dataset
for parent in parents :
self.print_msg( "Linking: '%s' ---> '%s' " % (
coverage.identifier, parent.identifier ) )
parent.insert( coverage )
def connect(self, coverage, data_items, layer, options):
layer.tileindex = os.path.abspath(connect(data_items[0]))
layer.tileitem = "location"
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
try:
ds = gdal.Open(connect(data_item, cache))
except:
with open(connect(data_item, cache)) as f:
ds = f.read()
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
# replace any already registered dataset
if kwargs["replace"]:
try:
# get a list of all collections the coverage was in.
coverage = models.Coverage.objects.get(
identifier=retrieved_metadata["identifier"]
)
additional_ids = [
c.identifier
for c in models.Collection.objects.filter(
eo_objects__in=[coverage.pk]
)
]
coverage.delete()
self.print_msg(
"Replacing previous dataset '%s'."
% retrieved_metadata["identifier"]
)
collection_ids = kwargs["collection_ids"] or []
for identifier in additional_ids:
if identifier not in collection_ids:
collection_ids.append(identifier)
kwargs["collection_ids"] = collection_ids
except models.Coverage.DoesNotExist:
self.print_msg(
"Could not replace previous dataset '%s'."
% retrieved_metadata["identifier"]
)
try:
coverage_type = retrieved_metadata["coverage_type"]
# TODO: allow types of different apps
if len(coverage_type.split(".")) > 1:
module_name, _, coverage_type = coverage_type.rpartition(".")
module = import_module(module_name)
CoverageType = getattr(module, coverage_type)
else:
CoverageType = getattr(models, coverage_type)
except AttributeError:
raise CommandError(
"Type '%s' is not supported."
% retrieved_metadata["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
polygon_mask_cloud = kwargs["pm_cloud"]
polygon_mask_snow = kwargs["pm_snow"]
wms_view = kwargs["md_wms_view"]
wms_alias = kwargs["md_wms_alias"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
#----------------------------------------------------------------------
# meta-data
vector_masks_src = []
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = etree.parse(f)
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
vector_masks_src = values.get("vmasks", [])
#----------------------------------------------------------------------
# polygon masks
def _get_geometry(file_name):
""" load geometry from a feature collection """
# TODO: improve the feature selection
ds = ogr.Open(file_name)
ly = ds.GetLayer(0)
ft = ly.GetFeature(0)
g0 = ft.GetGeometryRef()
g0.TransformTo(SR_WGS84) # geometries always stored in WGS84
return geos.GEOSGeometry(buffer(g0.ExportToWkb()), srid=4326)
if polygon_mask_cloud is not None:
vector_masks_src.append({
'type': 'CLOUD',
'subtype': None,
'mask': _get_geometry(polygon_mask_cloud),
})
if polygon_mask_snow is not None:
vector_masks_src.append({
'type': 'SNOW',
'subtype': None,
'mask': _get_geometry(polygon_mask_snow),
})
#----------------------------------------------------------------------
# handle vector masks
vector_masks = []
VMASK_TYPE = dict((v, k) for (k, v) in models.VectorMask.TYPE_CHOICES)
for vm_src in vector_masks_src:
if vm_src["type"] not in VMASK_TYPE:
raise CommandError("Invalid mask type '%s'! Allowed "
"mask-types are: %s", vm_src["type"],
"|".join(VMASK_TYPE.keys()))
vm = models.VectorMask()
vm.type = VMASK_TYPE[vm_src["type"]]
vm.subtype = vm_src["subtype"]
vm.geometry = vm_src["mask"]
#TODO: improve the semantic handling
if vm.subtype:
vm.semantic = ("%s_%s"%(vm_src["type"],
vm_src["subtype"].replace(" ", "_"))).lower()
else:
vm.semantic = vm_src["type"].lower()
vector_masks.append(vm)
#----------------------------------------------------------------------
# meta-data
metadata_items = []
# prerendered WMS view
if wms_view is not None:
metadata_items.append(
models.MetadataItem(semantic="wms_view", value=wms_view)
)
# alias of the WMS view
if wms_alias is not None:
metadata_items.append(
models.MetadataItem(semantic="wms_alias", value=wms_alias)
)
#----------------------------------------------------------------------
# coverage
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
try:
# TODO: allow types of different apps
CoverageType = getattr(models, retrieved_metadata["coverage_type"])
except AttributeError:
raise CommandError(
"Type '%s' is not supported." % kwargs["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
for vm in vector_masks:
vm.coverage = coverage
vm.full_clean()
vm.save()
for md in metadata_items:
md.eo_object = coverage
md.full_clean()
md.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
