def _read_metadata_from_data(self, data_item, retrieved_metadata, cache): metadata_component = MetadataComponent(env) ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None
def _read_metadata(self, data_item, retrieved_metadata, cache): """ Read all available metadata of a ``data_item`` into the ``retrieved_metadata`` :class:`dict`. """ metadata_component = MetadataComponent(env) with open(retrieve(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in self.metadata_keys: retrieved_metadata.setdefault(key, value)
def handle_with_cache(self, cache, *args, **kwargs): metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time", "coverage_type", )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) for metadata in metadatas: storage, package, format, location = self._get_location_chain( metadata ) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) try: ds = gdal.Open(connect(data_item, cache)) except: with open(connect(data_item, cache)) as f: ds = f.read() reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) # replace any already registered dataset if kwargs["replace"]: try: # get a list of all collections the coverage was in. coverage = models.Coverage.objects.get( identifier=retrieved_metadata["identifier"] ) additional_ids = [ c.identifier for c in models.Collection.objects.filter( eo_objects__in=[coverage.pk] ) ] coverage.delete() self.print_msg( "Replacing previous dataset '%s'." % retrieved_metadata["identifier"] ) collection_ids = kwargs["collection_ids"] or [] for identifier in additional_ids: if identifier not in collection_ids: collection_ids.append(identifier) kwargs["collection_ids"] = collection_ids except models.Coverage.DoesNotExist: self.print_msg( "Could not replace previous dataset '%s'." % retrieved_metadata["identifier"] ) try: coverage_type = retrieved_metadata["coverage_type"] # TODO: allow types of different apps if len(coverage_type.split(".")) > 1: module_name, _, coverage_type = coverage_type.rpartition(".") module = import_module(module_name) CoverageType = getattr(module, coverage_type) else: CoverageType = getattr(models, coverage_type) except AttributeError: raise CommandError( "Type '%s' is not supported." % retrieved_metadata["coverage_type"] ) try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: prj = models.Projection.objects.get( format=format, definition=definition ) retrieved_metadata["projection"] = prj # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.visible = kwargs["visible"] coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() # link with collection(s) if kwargs["collection_ids"]: ignore_missing_collection = kwargs["ignore_missing_collection"] call_command("eoxs_collection_link", collection_ids=kwargs["collection_ids"], add_ids=[coverage.identifier], ignore_missing_collection=ignore_missing_collection )