def _read_metadata_from_data(self, data_item, retrieved_metadata, cache):
metadata_component = MetadataComponent(env)
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
retrieved_metadata.setdefault(key, value)
ds = None
def _read_metadata(self, data_item, retrieved_metadata, cache):
""" Read all available metadata of a ``data_item`` into the
``retrieved_metadata`` :class:`dict`.
"""
metadata_component = MetadataComponent(env)
with open(retrieve(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in self.metadata_keys:
retrieved_metadata.setdefault(key, value)
def _read_metadata(self, data_item, retrieved_metadata, cache):
""" Read all available metadata of a ``data_item`` into the
``retrieved_metadata`` :class:`dict`.
"""
metadata_component = MetadataComponent(env)
with open(retrieve(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in self.metadata_keys:
retrieved_metadata.setdefault(key, value)
def handle_with_cache(self, cache, *args, **kwargs):
#----------------------------------------------------------------------
# check the inputs
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
# TODO: not required, as the keys are already
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time"
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
#----------------------------------------------------------------------
# parent dataset series
# extract the parents
ignore_missing_parent = bool(kwargs.get('ignore_missing_parent',False))
parents = []
for parent_id in kwargs.get('parents',[]):
try :
ds = models.DatasetSeries.objects.get(identifier=parent_id)
parents.append( ds )
except models.DatasetSeries.DoesNotExist :
msg ="There is no Dataset Series matching the given" \
" identifier: '%s' "%parent_id
if ignore_missing_parent :
self.print_wrn( msg )
else :
raise CommandError( msg )
#----------------------------------------------------------------------
# meta-data
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(metadata)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
try:
CoverageType = getattr(models, kwargs["coverage_type"])
except:
pass
# TODO: split into module path/coverage and get correct coverage class
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
retrieved_metadata["projection"] = models.Projection.objects.get(format=format, definition=definition)
#coverage.identifier = identifier # TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
#------------------------------------------------------------------
# link to the parent dataset
for parent in parents :
self.print_msg( "Linking: '%s' ---> '%s' " % (
coverage.identifier, parent.identifier ) )
parent.insert( coverage )
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
# replace any already registered dataset
if kwargs["replace"]:
try:
# get a list of all collections the coverage was in.
coverage = models.Coverage.objects.get(
identifier=retrieved_metadata["identifier"]
)
additional_ids = [
c.identifier
for c in models.Collection.objects.filter(
eo_objects__in=[coverage.pk]
)
]
coverage.delete()
self.print_msg(
"Replacing previous dataset '%s'."
% retrieved_metadata["identifier"]
)
collection_ids = kwargs["collection_ids"] or []
for identifier in additional_ids:
if identifier not in collection_ids:
collection_ids.append(identifier)
kwargs["collection_ids"] = collection_ids
except models.Coverage.DoesNotExist:
self.print_msg(
"Could not replace previous dataset '%s'."
% retrieved_metadata["identifier"]
)
try:
# TODO: allow types of different apps
CoverageType = getattr(models, retrieved_metadata["coverage_type"])
except AttributeError:
raise CommandError(
"Type '%s' is not supported." % kwargs["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
polygon_mask_cloud = kwargs["pm_cloud"]
polygon_mask_snow = kwargs["pm_snow"]
wms_view = kwargs["md_wms_view"]
wms_alias = kwargs["md_wms_alias"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
#----------------------------------------------------------------------
# meta-data
vector_masks_src = []
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = etree.parse(f)
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
vector_masks_src = values.get("vmasks", [])
#----------------------------------------------------------------------
# polygon masks
def _get_geometry(file_name):
""" load geometry from a feature collection """
# TODO: improve the feature selection
ds = ogr.Open(file_name)
ly = ds.GetLayer(0)
ft = ly.GetFeature(0)
g0 = ft.GetGeometryRef()
g0.TransformTo(SR_WGS84) # geometries always stored in WGS84
return geos.GEOSGeometry(buffer(g0.ExportToWkb()), srid=4326)
if polygon_mask_cloud is not None:
vector_masks_src.append({
'type': 'CLOUD',
'subtype': None,
'mask': _get_geometry(polygon_mask_cloud),
})
if polygon_mask_snow is not None:
vector_masks_src.append({
'type': 'SNOW',
'subtype': None,
'mask': _get_geometry(polygon_mask_snow),
})
#----------------------------------------------------------------------
# handle vector masks
vector_masks = []
VMASK_TYPE = dict((v, k) for (k, v) in models.VectorMask.TYPE_CHOICES)
for vm_src in vector_masks_src:
if vm_src["type"] not in VMASK_TYPE:
raise CommandError("Invalid mask type '%s'! Allowed "
"mask-types are: %s", vm_src["type"],
"|".join(VMASK_TYPE.keys()))
vm = models.VectorMask()
vm.type = VMASK_TYPE[vm_src["type"]]
vm.subtype = vm_src["subtype"]
vm.geometry = vm_src["mask"]
#TODO: improve the semantic handling
if vm.subtype:
vm.semantic = ("%s_%s"%(vm_src["type"],
vm_src["subtype"].replace(" ", "_"))).lower()
else:
vm.semantic = vm_src["type"].lower()
vector_masks.append(vm)
#----------------------------------------------------------------------
# meta-data
metadata_items = []
# prerendered WMS view
if wms_view is not None:
metadata_items.append(
models.MetadataItem(semantic="wms_view", value=wms_view)
)
# alias of the WMS view
if wms_alias is not None:
metadata_items.append(
models.MetadataItem(semantic="wms_alias", value=wms_alias)
)
#----------------------------------------------------------------------
# coverage
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
# TODO: other opening methods than GDAL
ds = gdal.Open(connect(data_item, cache))
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
try:
# TODO: allow types of different apps
CoverageType = getattr(models, retrieved_metadata["coverage_type"])
except AttributeError:
raise CommandError(
"Type '%s' is not supported." % kwargs["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
for vm in vector_masks:
vm.coverage = coverage
vm.full_clean()
vm.save()
for md in metadata_items:
md.eo_object = coverage
md.full_clean()
md.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
def handle_with_cache(self, cache, *args, **kwargs):
metadata_component = MetadataComponent(env)
datas = kwargs["data"]
semantics = kwargs.get("semantics")
metadatas = kwargs["metadata"]
range_type_name = kwargs["range_type_name"]
if range_type_name is None:
raise CommandError("No range type name specified.")
range_type = models.RangeType.objects.get(name=range_type_name)
metadata_keys = set((
"identifier", "extent", "size", "projection",
"footprint", "begin_time", "end_time", "coverage_type",
))
all_data_items = []
retrieved_metadata = {}
retrieved_metadata.update(
self._get_overrides(**kwargs)
)
for metadata in metadatas:
storage, package, format, location = self._get_location_chain(
metadata
)
data_item = backends.DataItem(
location=location, format=format or "", semantic="metadata",
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
with open(connect(data_item, cache)) as f:
content = f.read()
reader = metadata_component.get_reader_by_test(content)
if reader:
values = reader.read(content)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
if key in metadata_keys:
retrieved_metadata.setdefault(key, value)
if len(datas) < 1:
raise CommandError("No data files specified.")
if semantics is None:
# TODO: check corner cases.
# e.g: only one data item given but multiple bands in range type
# --> bands[1:<bandnum>]
if len(datas) == 1:
if len(range_type) == 1:
semantics = ["bands[1]"]
else:
semantics = ["bands[1:%d]" % len(range_type)]
else:
semantics = ["bands[%d]" % i for i in range(len(datas))]
for data, semantic in zip(datas, semantics):
storage, package, format, location = self._get_location_chain(data)
data_item = backends.DataItem(
location=location, format=format or "", semantic=semantic,
storage=storage, package=package,
)
data_item.full_clean()
data_item.save()
all_data_items.append(data_item)
try:
ds = gdal.Open(connect(data_item, cache))
except:
with open(connect(data_item, cache)) as f:
ds = f.read()
reader = metadata_component.get_reader_by_test(ds)
if reader:
values = reader.read(ds)
format = values.pop("format", None)
if format:
data_item.format = format
data_item.full_clean()
data_item.save()
for key, value in values.items():
retrieved_metadata.setdefault(key, value)
ds = None
if len(metadata_keys - set(retrieved_metadata.keys())):
raise CommandError(
"Missing metadata keys %s."
% ", ".join(metadata_keys - set(retrieved_metadata.keys()))
)
# replace any already registered dataset
if kwargs["replace"]:
try:
# get a list of all collections the coverage was in.
coverage = models.Coverage.objects.get(
identifier=retrieved_metadata["identifier"]
)
additional_ids = [
c.identifier
for c in models.Collection.objects.filter(
eo_objects__in=[coverage.pk]
)
]
coverage.delete()
self.print_msg(
"Replacing previous dataset '%s'."
% retrieved_metadata["identifier"]
)
collection_ids = kwargs["collection_ids"] or []
for identifier in additional_ids:
if identifier not in collection_ids:
collection_ids.append(identifier)
kwargs["collection_ids"] = collection_ids
except models.Coverage.DoesNotExist:
self.print_msg(
"Could not replace previous dataset '%s'."
% retrieved_metadata["identifier"]
)
try:
coverage_type = retrieved_metadata["coverage_type"]
# TODO: allow types of different apps
if len(coverage_type.split(".")) > 1:
module_name, _, coverage_type = coverage_type.rpartition(".")
module = import_module(module_name)
CoverageType = getattr(module, coverage_type)
else:
CoverageType = getattr(models, coverage_type)
except AttributeError:
raise CommandError(
"Type '%s' is not supported."
% retrieved_metadata["coverage_type"]
)
try:
coverage = CoverageType()
coverage.range_type = range_type
proj = retrieved_metadata.pop("projection")
if isinstance(proj, int):
retrieved_metadata["srid"] = proj
else:
definition, format = proj
# Try to identify the SRID from the given input
try:
sr = osr.SpatialReference(definition, format)
retrieved_metadata["srid"] = sr.srid
except Exception, e:
prj = models.Projection.objects.get(
format=format, definition=definition
)
retrieved_metadata["projection"] = prj
# TODO: bug in models for some coverages
for key, value in retrieved_metadata.items():
setattr(coverage, key, value)
coverage.visible = kwargs["visible"]
coverage.full_clean()
coverage.save()
for data_item in all_data_items:
data_item.dataset = coverage
data_item.full_clean()
data_item.save()
# link with collection(s)
if kwargs["collection_ids"]:
ignore_missing_collection = kwargs["ignore_missing_collection"]
call_command("eoxs_collection_link",
collection_ids=kwargs["collection_ids"],
add_ids=[coverage.identifier],
ignore_missing_collection=ignore_missing_collection
)
