def _read_metadata_from_data(self, data_item, retrieved_metadata, cache): metadata_component = MetadataComponent(env) ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None
def _read_metadata(self, data_item, retrieved_metadata, cache): """ Read all available metadata of a ``data_item`` into the ``retrieved_metadata`` :class:`dict`. """ metadata_component = MetadataComponent(env) with open(retrieve(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in self.metadata_keys: retrieved_metadata.setdefault(key, value)
def handle_with_cache(self, cache, *args, **kwargs): #---------------------------------------------------------------------- # check the inputs metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) # TODO: not required, as the keys are already metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time" )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) #---------------------------------------------------------------------- # parent dataset series # extract the parents ignore_missing_parent = bool(kwargs.get('ignore_missing_parent',False)) parents = [] for parent_id in kwargs.get('parents',[]): try : ds = models.DatasetSeries.objects.get(identifier=parent_id) parents.append( ds ) except models.DatasetSeries.DoesNotExist : msg ="There is no Dataset Series matching the given" \ " identifier: '%s' "%parent_id if ignore_missing_parent : self.print_wrn( msg ) else : raise CommandError( msg ) #---------------------------------------------------------------------- # meta-data for metadata in metadatas: storage, package, format, location = self._get_location_chain(metadata) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) # TODO: other opening methods than GDAL ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) try: CoverageType = getattr(models, kwargs["coverage_type"]) except: pass # TODO: split into module path/coverage and get correct coverage class try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: retrieved_metadata["projection"] = models.Projection.objects.get(format=format, definition=definition) #coverage.identifier = identifier # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() #------------------------------------------------------------------ # link to the parent dataset for parent in parents : self.print_msg( "Linking: '%s' ---> '%s' " % ( coverage.identifier, parent.identifier ) ) parent.insert( coverage )
def handle_with_cache(self, cache, *args, **kwargs): metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time", "coverage_type", )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) for metadata in metadatas: storage, package, format, location = self._get_location_chain( metadata ) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) # TODO: other opening methods than GDAL ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) # replace any already registered dataset if kwargs["replace"]: try: # get a list of all collections the coverage was in. coverage = models.Coverage.objects.get( identifier=retrieved_metadata["identifier"] ) additional_ids = [ c.identifier for c in models.Collection.objects.filter( eo_objects__in=[coverage.pk] ) ] coverage.delete() self.print_msg( "Replacing previous dataset '%s'." % retrieved_metadata["identifier"] ) collection_ids = kwargs["collection_ids"] or [] for identifier in additional_ids: if identifier not in collection_ids: collection_ids.append(identifier) kwargs["collection_ids"] = collection_ids except models.Coverage.DoesNotExist: self.print_msg( "Could not replace previous dataset '%s'." % retrieved_metadata["identifier"] ) try: # TODO: allow types of different apps CoverageType = getattr(models, retrieved_metadata["coverage_type"]) except AttributeError: raise CommandError( "Type '%s' is not supported." % kwargs["coverage_type"] ) try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: prj = models.Projection.objects.get( format=format, definition=definition ) retrieved_metadata["projection"] = prj # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.visible = kwargs["visible"] coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() # link with collection(s) if kwargs["collection_ids"]: ignore_missing_collection = kwargs["ignore_missing_collection"] call_command("eoxs_collection_link", collection_ids=kwargs["collection_ids"], add_ids=[coverage.identifier], ignore_missing_collection=ignore_missing_collection )
def handle_with_cache(self, cache, *args, **kwargs): metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] polygon_mask_cloud = kwargs["pm_cloud"] polygon_mask_snow = kwargs["pm_snow"] wms_view = kwargs["md_wms_view"] wms_alias = kwargs["md_wms_alias"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time", "coverage_type", )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) #---------------------------------------------------------------------- # meta-data vector_masks_src = [] for metadata in metadatas: storage, package, format, location = self._get_location_chain( metadata ) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = etree.parse(f) reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) vector_masks_src = values.get("vmasks", []) #---------------------------------------------------------------------- # polygon masks def _get_geometry(file_name): """ load geometry from a feature collection """ # TODO: improve the feature selection ds = ogr.Open(file_name) ly = ds.GetLayer(0) ft = ly.GetFeature(0) g0 = ft.GetGeometryRef() g0.TransformTo(SR_WGS84) # geometries always stored in WGS84 return geos.GEOSGeometry(buffer(g0.ExportToWkb()), srid=4326) if polygon_mask_cloud is not None: vector_masks_src.append({ 'type': 'CLOUD', 'subtype': None, 'mask': _get_geometry(polygon_mask_cloud), }) if polygon_mask_snow is not None: vector_masks_src.append({ 'type': 'SNOW', 'subtype': None, 'mask': _get_geometry(polygon_mask_snow), }) #---------------------------------------------------------------------- # handle vector masks vector_masks = [] VMASK_TYPE = dict((v, k) for (k, v) in models.VectorMask.TYPE_CHOICES) for vm_src in vector_masks_src: if vm_src["type"] not in VMASK_TYPE: raise CommandError("Invalid mask type '%s'! Allowed " "mask-types are: %s", vm_src["type"], "|".join(VMASK_TYPE.keys())) vm = models.VectorMask() vm.type = VMASK_TYPE[vm_src["type"]] vm.subtype = vm_src["subtype"] vm.geometry = vm_src["mask"] #TODO: improve the semantic handling if vm.subtype: vm.semantic = ("%s_%s"%(vm_src["type"], vm_src["subtype"].replace(" ", "_"))).lower() else: vm.semantic = vm_src["type"].lower() vector_masks.append(vm) #---------------------------------------------------------------------- # meta-data metadata_items = [] # prerendered WMS view if wms_view is not None: metadata_items.append( models.MetadataItem(semantic="wms_view", value=wms_view) ) # alias of the WMS view if wms_alias is not None: metadata_items.append( models.MetadataItem(semantic="wms_alias", value=wms_alias) ) #---------------------------------------------------------------------- # coverage if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) # TODO: other opening methods than GDAL ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) try: # TODO: allow types of different apps CoverageType = getattr(models, retrieved_metadata["coverage_type"]) except AttributeError: raise CommandError( "Type '%s' is not supported." % kwargs["coverage_type"] ) try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: prj = models.Projection.objects.get( format=format, definition=definition ) retrieved_metadata["projection"] = prj # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.visible = kwargs["visible"] coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() for vm in vector_masks: vm.coverage = coverage vm.full_clean() vm.save() for md in metadata_items: md.eo_object = coverage md.full_clean() md.save() # link with collection(s) if kwargs["collection_ids"]: ignore_missing_collection = kwargs["ignore_missing_collection"] call_command("eoxs_collection_link", collection_ids=kwargs["collection_ids"], add_ids=[coverage.identifier], ignore_missing_collection=ignore_missing_collection )
def handle_with_cache(self, cache, *args, **kwargs): metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time", "coverage_type", )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) for metadata in metadatas: storage, package, format, location = self._get_location_chain( metadata ) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) try: ds = gdal.Open(connect(data_item, cache)) except: with open(connect(data_item, cache)) as f: ds = f.read() reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) # replace any already registered dataset if kwargs["replace"]: try: # get a list of all collections the coverage was in. coverage = models.Coverage.objects.get( identifier=retrieved_metadata["identifier"] ) additional_ids = [ c.identifier for c in models.Collection.objects.filter( eo_objects__in=[coverage.pk] ) ] coverage.delete() self.print_msg( "Replacing previous dataset '%s'." % retrieved_metadata["identifier"] ) collection_ids = kwargs["collection_ids"] or [] for identifier in additional_ids: if identifier not in collection_ids: collection_ids.append(identifier) kwargs["collection_ids"] = collection_ids except models.Coverage.DoesNotExist: self.print_msg( "Could not replace previous dataset '%s'." % retrieved_metadata["identifier"] ) try: coverage_type = retrieved_metadata["coverage_type"] # TODO: allow types of different apps if len(coverage_type.split(".")) > 1: module_name, _, coverage_type = coverage_type.rpartition(".") module = import_module(module_name) CoverageType = getattr(module, coverage_type) else: CoverageType = getattr(models, coverage_type) except AttributeError: raise CommandError( "Type '%s' is not supported." % retrieved_metadata["coverage_type"] ) try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: prj = models.Projection.objects.get( format=format, definition=definition ) retrieved_metadata["projection"] = prj # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.visible = kwargs["visible"] coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() # link with collection(s) if kwargs["collection_ids"]: ignore_missing_collection = kwargs["ignore_missing_collection"] call_command("eoxs_collection_link", collection_ids=kwargs["collection_ids"], add_ids=[coverage.identifier], ignore_missing_collection=ignore_missing_collection )