class DatasetRecord(object): """DatasetRecord database interface class.""" DATASET_METADATA_FIELDS = [ 'dataset_path', 'datetime_processed', 'dataset_size', 'll_x', 'll_y', 'lr_x', 'lr_y', 'ul_x', 'ul_y', 'ur_x', 'ur_y', 'x_pixels', 'y_pixels', 'xml_text' ] def __init__(self, collection, acquisition, dataset): self.collection = collection self.datacube = collection.datacube self.db = IngestDBWrapper(self.datacube.db_connection) dataset_key = collection.get_dataset_key(dataset) self.dataset_bands = collection.new_bands[dataset_key] self.dataset = dataset self.mdd = dataset.metadata_dict self.dataset_dict = {} for field in self.DATASET_METADATA_FIELDS: self.dataset_dict[field] = self.mdd[field] self.dataset_dict['acquisition_id'] = acquisition.acquisition_id self.dataset_dict['crs'] = self.mdd['projection'] self.dataset_dict['level_name'] = self.mdd['processing_level'] self.dataset_dict['level_id'] = \ self.db.get_level_id(self.dataset_dict['level_name']) self.dataset_dict['dataset_id'] = \ self.db.get_dataset_id(self.dataset_dict) if self.dataset_dict['dataset_id'] is None: # create a new dataset record in the database self.dataset_dict['dataset_id'] = \ self.db.insert_dataset_record(self.dataset_dict) self.needs_update = False else: # check that the old dataset record can be updated self.__check_update_ok() self.needs_update = True self.dataset_id = self.dataset_dict['dataset_id'] def remove_mosaics(self, dataset_filter): """Remove mosaics associated with the dataset. This will mark mosaic files for removal, delete mosaic database records if they exist, and update the tile class of overlapping tiles (from other datasets) to reflect the lack of a mosaic. The 'dataset_filter' is a list of dataset_ids to filter on. It should be the list of dataset_ids that have been locked (including this dataset). It is used to avoid operating on the tiles of an unlocked dataset. """ # remove new mosaics (those with database records) overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED, TC_MOSAIC), output_tile_class_filter=(TC_MOSAIC, ), dataset_filter=dataset_filter) for tile_record_list in overlap_dict.values(): for tr in tile_record_list: self.db.remove_tile_record(tr['tile_id']) self.collection.mark_tile_for_removal(tr['tile_pathname']) # build a dictionary of overlaps (ignoring mosaics) overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED), output_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED), dataset_filter=dataset_filter) # update tile classes for overlap tiles from other datasets for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 2: raise DatasetError("Attempt to update a mosaic of three or " + "more datasets. Handling for this case " + "is not yet implemented.") for tr in tile_record_list: if tr['dataset_id'] != self.dataset_id: self.db.update_tile_class(tr['tile_id'], TC_SINGLE_SCENE) # remove old mosaics (those without database records) for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 1: # tile_record_list is sorted by acquisition start time, so # the first record should be the one the mosaic filename is # based on. tr = tile_record_list[0] mosaic_pathname = \ self.__make_mosaic_pathname(tr['tile_pathname']) if os.path.isfile(mosaic_pathname): self.collection.mark_tile_for_removal(mosaic_pathname) def remove_tiles(self): """Remove the tiles associated with the dataset. This will remove ALL the tiles belonging to this dataset, deleting database records and marking tile files for removal on commit. Mosaics should be removed BEFORE calling this (as it will delete the tiles needed to figure out the overlaps, but may not delete all the mosaics). """ tile_list = self.db.get_dataset_tile_ids(self.dataset_id) for tile_id in tile_list: tile_pathname = self.db.get_tile_pathname(tile_id) self.db.remove_tile_record(tile_id) self.collection.mark_tile_for_removal(tile_pathname) def update(self): """Update the dataset record in the database. This first checks that the new dataset is more recent than the record in the database. If not it raises a dataset error. """ self.__check_update_ok() self.db.update_dataset_record(self.dataset_dict) def make_tiles(self, tile_type_id, band_stack): """Tile the dataset, returning a list of tile_content objects.""" tile_list = [] tile_footprint_list = self.get_coverage(tile_type_id) for tile_footprint in tile_footprint_list: tile_contents = self.collection.create_tile_contents( tile_type_id, tile_footprint, band_stack) tile_contents.reproject() if tile_contents.has_data(): tile_list.append(tile_contents) else: tile_contents.remove() return tile_list def store_tiles(self, tile_list): """Store tiles in the database and file store. 'tile_list' is a list of tile_contents objects. This method will create the corrisponding database records and mark tiles for creation when the transaction commits. """ tile_record_list = [] for tile_contents in tile_list: tile_record = self.create_tile_record(tile_contents) tile_record_list.append(tile_record) return tile_record_list def create_mosaics(self, dataset_filter): """Create mosaics associated with the dataset. 'dataset_filter' is a list of dataset_ids to filter on. It should be the list of dataset_ids that have been locked (including this dataset). It is used to avoid operating on the tiles of an unlocked dataset. """ # Build a dictionary of overlaps (ignoring mosaics, including pending). overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED), output_tile_class_filter=(TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED), dataset_filter=dataset_filter) # Make mosaics and update tile classes as needed. for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 2: raise DatasetError("Attempt to create a mosaic of three or " + "more datasets. Handling for this case " + "is not yet implemented.") elif len(tile_record_list) == 2: self.__make_one_mosaic(tile_record_list) for tr in tile_record_list: self.db.update_tile_class(tr['tile_id'], TC_SUPERSEDED) else: for tr in tile_record_list: self.db.update_tile_class(tr['tile_id'], TC_SINGLE_SCENE) def get_removal_overlaps(self): """Returns a list of overlapping dataset ids for mosaic removal.""" tile_class_filter = (TC_SINGLE_SCENE, TC_SUPERSEDED, TC_MOSAIC) return self.get_overlaps(tile_class_filter) def get_creation_overlaps(self): """Returns a list of overlapping dataset_ids for mosaic creation.""" tile_class_filter = (TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED) return self.get_overlaps(tile_class_filter) def get_overlaps(self, tile_class_filter): """Returns a list of overlapping dataset ids, including this dataset. A dataset is overlapping if it contains tiles that overlap with tiles belonging to this dataset. Only tiles in the tile_class_filter are considered. """ dataset_list = self.db.get_overlapping_dataset_ids( self.dataset_id, tile_class_filter=tile_class_filter) if not dataset_list: dataset_list = [self.dataset_id] return dataset_list def create_tile_record(self, tile_contents): """Factory method to create an instance of the TileRecord class. The created object will be responsible for inserting tile table records into the database for reprojected or mosaiced tiles.""" self.collection.mark_tile_for_creation(tile_contents) return TileRecord(self.collection, self, tile_contents) def mark_as_tiled(self): """Flag the dataset record as tiled in the database. This flag does not exist in the current database schema, so this method does nothing at the moment.""" pass def list_tile_types(self): """Returns a list of the tile type ids for this dataset.""" return self.dataset_bands.keys() def get_tile_bands(self, tile_type_id): """Returns a dictionary containing the band info for one tile type. The tile_type_id must valid for this dataset, available from list_tile_types above. """ return self.dataset_bands[tile_type_id] def get_coverage(self, tile_type_id): """Given the coordinate reference system of the dataset and that of the tile_type_id, return a list of tiles within the dataset footprint""" tile_type_info = self.collection.datacube.tile_type_dict[tile_type_id] #Get geospatial information from the dataset. dataset_crs = self.mdd['projection'] dataset_geotransform = self.mdd['geo_transform'] pixels = self.mdd['x_pixels'] lines = self.mdd['y_pixels'] #Look up the datacube's projection information for this tile_type tile_crs = tile_type_info['crs'] #Get the transformation between the two projections transformation = self.define_transformation(dataset_crs, tile_crs) #Determine the bounding quadrilateral of the dataset extent #in tile coordinates dataset_bbox = self.get_bbox(transformation, dataset_geotransform, pixels, lines) #Determine maximum inner rectangle, which is guaranteed to need tiling #and the minimum outer rectangle outside which no tiles will exist. cube_origin = (tile_type_info['x_origin'], tile_type_info['y_origin']) cube_tile_size = (tile_type_info['x_size'], tile_type_info['y_size']) coverage = self.get_touched_tiles(dataset_bbox, cube_origin, cube_tile_size) return coverage # # worker methods # def __check_update_ok(self): """Checks if an update is possible, raises a DatasetError otherwise.""" tile_class_filter = (TC_SINGLE_SCENE, TC_SUPERSEDED) if self.db.dataset_older_than_database( self.dataset_dict['dataset_id'], self.dataset_dict['datetime_processed'], tile_class_filter): raise DatasetError("Dataset to be ingested is older than " + "the version in the database.") def __make_one_mosaic(self, tile_record_list): """Create a single mosaic. This create the mosaic contents, creates the database record, and marks the mosaic contents for creation on transaction commit. """ mosaic = MosaicContents(tile_record_list, self.datacube.tile_type_dict, self.dataset_dict['level_name'], self.collection.get_temp_tile_directory()) mosaic.create_record(self.db) self.collection.mark_tile_for_creation(mosaic) def __make_mosaic_pathname(self, tile_pathname): """Return the pathname of the mosaic corrisponding to a tile.""" (tile_dir, tile_basename) = os.path.split(tile_pathname) mosaic_dir = os.path.join(tile_dir, 'mosaic_cache') if self.dataset_dict['level_name'] == 'PQA': mosaic_basename = tile_basename else: mosaic_basename = re.sub(r'\.\w+$', '.vrt', tile_basename) return os.path.join(mosaic_dir, mosaic_basename) # # Worker methods for coverage. # # These are public so that they can be called by test_dataset_record. # def define_transformation(self, dataset_crs, tile_crs): """Return the transformation between dataset_crs and tile_crs projections""" osr.UseExceptions() try: dataset_spatial_reference = self.create_spatial_ref(dataset_crs) tile_spatial_reference = self.create_spatial_ref(tile_crs) if dataset_spatial_reference is None: raise DatasetError('Unknown projecton %s' % str(dataset_crs)) if tile_spatial_reference is None: raise DatasetError('Unknown projecton %s' % str(tile_crs)) return osr.CoordinateTransformation(dataset_spatial_reference, tile_spatial_reference) except Exception: raise DatasetError('Coordinate transformation error ' + 'for transforming %s to %s' % (str(dataset_crs), str(tile_crs))) @staticmethod def create_spatial_ref(crs): """Create a spatial reference system for projecton crs. Called by define_transformation()""" # pylint: disable=broad-except osr.UseExceptions() try: spatial_ref = osr.SpatialReference() except Exception: raise DatasetError('No spatial reference done for %s' % str(crs)) try: spatial_ref.ImportFromWkt(crs) return spatial_ref except Exception: pass try: matchobj = re.match(r'EPSG:(\d+)', crs) epsg_code = int(matchobj.group(1)) spatial_ref.ImportFromEPSG(epsg_code) return spatial_ref except Exception: return None @staticmethod def get_bbox(transform, geotrans, pixels, lines): """Return the coordinates of the dataset footprint in clockwise order from upper-left""" xul, yul, dummy_z = \ transform.TransformPoint(geotrans[0], geotrans[3], 0) xur, yur, dummy_z = \ transform.TransformPoint(geotrans[0] + geotrans[1] * pixels, geotrans[3] + geotrans[4] * pixels, 0) xll, yll, dummy_z = \ transform.TransformPoint(geotrans[0] + geotrans[2] * lines, geotrans[3] + geotrans[5] * lines, 0) xlr, ylr, dummy_z = \ transform.TransformPoint( geotrans[0] + geotrans[1] * pixels + geotrans[2] * lines, geotrans[3] + geotrans[4] * pixels + geotrans[5] * lines, 0) return [(xul, yul), (xur, yur), (xlr, ylr), (xll, yll)] def get_touched_tiles(self, dataset_bbox, cube_origin, cube_tile_size): """Return a list of tuples (itile, jtile) comprising all tiles footprints that intersect the dataset bounding box""" definite_tiles, possible_tiles = \ self.get_definite_and_possible_tiles(dataset_bbox, cube_origin, cube_tile_size) coverage_set = definite_tiles #Check possible tiles: #Check if the tile perimeter intersects the dataset bbox perimeter: intersected_tiles = \ self.get_intersected_tiles(possible_tiles, dataset_bbox, cube_origin, cube_tile_size) coverage_set = coverage_set.union(intersected_tiles) possible_tiles = possible_tiles.difference(intersected_tiles) #Otherwise the tile might be wholly contained in the dataset bbox contained_tiles = \ self.get_contained_tiles(possible_tiles, dataset_bbox, cube_origin, cube_tile_size) coverage_set = coverage_set.union(contained_tiles) return coverage_set @staticmethod def get_definite_and_possible_tiles(bbox, cube_origin, cube_tile_size): """Return two lists of tile footprints: from the largest rectangle wholly contained within the dataset bbox and the smallest rectangle containing the bbox.""" #pylint: disable=too-many-locals #unpack the bbox vertices in clockwise order from upper-left xyul, xyur, xylr, xyll = bbox xul, yul = xyul xur, yur = xyur xlr, ylr = xylr xll, yll = xyll #unpack the origin of the tiled datacube (e.g. lat=0, lon=0) and the #datacube tile size xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size #Define the largest rectangle wholly contained within footprint xmin = max(xll, xul) xmax = min(xlr, xur) ymin = max(yll, ylr) ymax = min(yul, yur) xmin_index = int(floor((xmin - xorigin) / xsize)) xmax_index = int(floor((xmax - xorigin) / xsize)) ymin_index = int(floor((ymin - yorigin) / ysize)) ymax_index = int(floor((ymax - yorigin) / ysize)) definite_tiles = set([(itile, jtile) for itile in range(xmin_index, xmax_index + 1) for jtile in range(ymin_index, ymax_index + 1)]) #Define the smallest rectangle which is guaranteed to include all tiles #in the foorprint. xmin = min(xll, xul) xmax = max(xlr, xur) ymin = min(yll, ylr) ymax = max(yul, yur) xmin_index = int(floor((xmin - xorigin) / xsize)) xmax_index = int(floor((xmax - xorigin) / xsize)) ymin_index = int(floor((ymin - yorigin) / ysize)) ymax_index = int(floor((ymax - yorigin) / ysize)) possible_tiles = set([(itile, jtile) for itile in range(xmin_index, xmax_index + 1) for jtile in range(ymin_index, ymax_index + 1) ]).difference(definite_tiles) return (definite_tiles, possible_tiles) def get_intersected_tiles(self, candidate_tiles, dset_bbox, cube_origin, cube_tile_size): """Return the subset of candidate_tiles that have an intersection with the dataset bounding box""" #pylint: disable=too-many-locals xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size keep_list = [] for itile, jtile in candidate_tiles: intersection_exists = False (x0, y0) = (xorigin + itile * xsize, yorigin + (jtile + 1) * ysize) tile_bbox = [(x0, y0), (x0 + xsize, y0), (x0 + xsize, y0 - ysize), (x0, y0 - ysize)] tile_vtx_number = len(tile_bbox) dset_vtx_number = len(dset_bbox) for tile_vtx in range(tile_vtx_number): x1, y1 = tile_bbox[tile_vtx] x2, y2 = tile_bbox[(tile_vtx + 1) % tile_vtx_number] for dset_vtx in range(dset_vtx_number): x3, y3 = dset_bbox[dset_vtx] x4, y4 = dset_bbox[(dset_vtx + 1) % dset_vtx_number] xcoords = [x1, x2, x3, x4] ycoords = [y1, y2, y3, y4] intersection_exists = \ self.check_intersection(xcoords, ycoords) if intersection_exists: keep_list.append((itile, jtile)) break if intersection_exists: break return set(keep_list) @staticmethod def get_contained_tiles(candidate_tiles, dset_bbox, cube_origin, cube_tile_size): """Return the subset of candidate tiles that lie wholly within the dataset bounding box""" #pylint: disable=too-many-locals xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size keep_list = [] for itile, jtile in candidate_tiles: tile_vtx_inside = [] (x0, y0) = (xorigin + itile * xsize, yorigin + (jtile + 1) * ysize) tile_bbox = [(x0, y0), (x0 + xsize, y0), (x0 + xsize, y0 - ysize), (x0, y0 - ysize)] dset_vtx_number = len(dset_bbox) for x, y in tile_bbox: #Check if this vertex lies within the dataset bounding box: winding_number = 0 for dset_vtx in range(dset_vtx_number): x1, y1 = dset_bbox[dset_vtx] x2, y2 = dset_bbox[(dset_vtx + 1) % dset_vtx_number] if y >= y1 and y < y2: if (x - x1) * (y2 - y1) > (x2 - x1) * (y - y1): winding_number += 1 elif y <= y1 and y > y2: if (x - x1) * (y2 - y1) < (x2 - x1) * (y - y1): winding_number += 1 tile_vtx_inside.append(winding_number % 2 == 1) if tile_vtx_inside.count(True) == len(tile_bbox): keep_list.append((itile, jtile)) assert tile_vtx_inside.count(True) == 4 or \ tile_vtx_inside.count(True) == 0, \ "Tile partially inside dataset bounding box but has" \ "no intersection" return set(keep_list) @staticmethod def check_intersection(xpts, ypts): """Determines if the line segments (xpts[0], ypts[0]) to (xpts[1], ypts[1]) and (xpts[2], ypts[2]) to (xpts[3], ypts[3]) intersect""" pvec = (xpts[0], ypts[0]) qvec = (xpts[2], ypts[2]) rvec = (xpts[1] - xpts[0], ypts[1] - ypts[0]) svec = (xpts[3] - xpts[2], ypts[3] - ypts[2]) rvec_cross_svec = rvec[0] * svec[1] - rvec[1] * svec[0] if rvec_cross_svec == 0: return False qminusp_cross_svec = \ (qvec[0] - pvec[0]) * svec[1] - (qvec[1] - pvec[1]) * svec[0] qminusp_cross_rvec = \ (qvec[0] - pvec[0]) * rvec[1] - (qvec[1] - pvec[1]) * rvec[0] tparameter = qminusp_cross_svec / rvec_cross_svec uparameter = qminusp_cross_rvec / rvec_cross_svec if tparameter > 0 and tparameter < 1 and \ uparameter > 0 and uparameter < 1: return True
class DatasetRecord(object): """DatasetRecord database interface class.""" DATASET_METADATA_FIELDS = ['dataset_path', 'datetime_processed', 'dataset_size', 'll_x', 'll_y', 'lr_x', 'lr_y', 'ul_x', 'ul_y', 'ur_x', 'ur_y', 'x_pixels', 'y_pixels', 'xml_text' ] def __init__(self, collection, acquisition, dataset): self.collection = collection self.datacube = collection.datacube self.db = IngestDBWrapper(self.datacube.db_connection) dataset_key = collection.get_dataset_key(dataset) self.dataset_bands = collection.new_bands[dataset_key] self.dataset = dataset self.mdd = dataset.metadata_dict self.dataset_dict = {} for field in self.DATASET_METADATA_FIELDS: self.dataset_dict[field] = self.mdd[field] self.dataset_dict['acquisition_id'] = acquisition.acquisition_id self.dataset_dict['crs'] = self.mdd['projection'] self.dataset_dict['level_name'] = self.mdd['processing_level'] self.dataset_dict['level_id'] = \ self.db.get_level_id(self.dataset_dict['level_name']) self.dataset_dict['dataset_id'] = \ self.db.get_dataset_id(self.dataset_dict) if self.dataset_dict['dataset_id'] is None: # create a new dataset record in the database self.dataset_dict['dataset_id'] = \ self.db.insert_dataset_record(self.dataset_dict) self.needs_update = False else: # check that the old dataset record can be updated self.__check_update_ok() self.needs_update = True self.dataset_id = self.dataset_dict['dataset_id'] def remove_mosaics(self, dataset_filter): """Remove mosaics associated with the dataset. This will mark mosaic files for removal, delete mosaic database records if they exist, and update the tile class of overlapping tiles (from other datasets) to reflect the lack of a mosaic. The 'dataset_filter' is a list of dataset_ids to filter on. It should be the list of dataset_ids that have been locked (including this dataset). It is used to avoid operating on the tiles of an unlocked dataset. """ # remove new mosaics (those with database records) overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED, TC_MOSAIC), output_tile_class_filter=(TC_MOSAIC,), dataset_filter=dataset_filter ) for tile_record_list in overlap_dict.values(): for tr in tile_record_list: self.db.remove_tile_record(tr['tile_id']) self.collection.mark_tile_for_removal(tr['tile_pathname']) # build a dictionary of overlaps (ignoring mosaics) overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED), output_tile_class_filter=(TC_SINGLE_SCENE, TC_SUPERSEDED), dataset_filter=dataset_filter ) # update tile classes for overlap tiles from other datasets for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 2: raise DatasetError("Attempt to update a mosaic of three or " + "more datasets. Handling for this case " + "is not yet implemented.") for tr in tile_record_list: if tr['dataset_id'] != self.dataset_id: self.db.update_tile_class(tr['tile_id'], TC_SINGLE_SCENE) # remove old mosaics (those without database records) for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 1: # tile_record_list is sorted by acquisition start time, so # the first record should be the one the mosaic filename is # based on. tr = tile_record_list[0] mosaic_pathname = \ self.__make_mosaic_pathname(tr['tile_pathname']) if os.path.isfile(mosaic_pathname): self.collection.mark_tile_for_removal(mosaic_pathname) def remove_tiles(self): """Remove the tiles associated with the dataset. This will remove ALL the tiles belonging to this dataset, deleting database records and marking tile files for removal on commit. Mosaics should be removed BEFORE calling this (as it will delete the tiles needed to figure out the overlaps, but may not delete all the mosaics). """ tile_list = self.db.get_dataset_tile_ids(self.dataset_id) for tile_id in tile_list: tile_pathname = self.db.get_tile_pathname(tile_id) self.db.remove_tile_record(tile_id) self.collection.mark_tile_for_removal(tile_pathname) def update(self): """Update the dataset record in the database. This first checks that the new dataset is more recent than the record in the database. If not it raises a dataset error. """ self.__check_update_ok() self.db.update_dataset_record(self.dataset_dict) def make_tiles(self, tile_type_id, band_stack): """Tile the dataset, returning a list of tile_content objects.""" tile_list = [] tile_footprint_list = self.get_coverage(tile_type_id) for tile_footprint in tile_footprint_list: tile_contents = self.collection.create_tile_contents( tile_type_id, tile_footprint, band_stack ) tile_contents.reproject() if tile_contents.has_data(): tile_list.append(tile_contents) else: tile_contents.remove() return tile_list def store_tiles(self, tile_list): """Store tiles in the database and file store. 'tile_list' is a list of tile_contents objects. This method will create the corrisponding database records and mark tiles for creation when the transaction commits. """ tile_record_list = [] for tile_contents in tile_list: tile_record = self.create_tile_record(tile_contents) tile_record_list.append(tile_record) return tile_record_list def create_mosaics(self, dataset_filter): """Create mosaics associated with the dataset. 'dataset_filter' is a list of dataset_ids to filter on. It should be the list of dataset_ids that have been locked (including this dataset). It is used to avoid operating on the tiles of an unlocked dataset. """ # Build a dictionary of overlaps (ignoring mosaics, including pending). overlap_dict = self.db.get_overlapping_tiles_for_dataset( self.dataset_id, input_tile_class_filter=(TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED), output_tile_class_filter=(TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED), dataset_filter=dataset_filter ) # Make mosaics and update tile classes as needed. for tile_record_list in overlap_dict.values(): if len(tile_record_list) > 2: raise DatasetError("Attempt to create a mosaic of three or " + "more datasets. Handling for this case " + "is not yet implemented.") elif len(tile_record_list) == 2: self.__make_one_mosaic(tile_record_list) for tr in tile_record_list: self.db.update_tile_class(tr['tile_id'], TC_SUPERSEDED) else: for tr in tile_record_list: self.db.update_tile_class(tr['tile_id'], TC_SINGLE_SCENE) def get_removal_overlaps(self): """Returns a list of overlapping dataset ids for mosaic removal.""" tile_class_filter = (TC_SINGLE_SCENE, TC_SUPERSEDED, TC_MOSAIC) return self.get_overlaps(tile_class_filter) def get_creation_overlaps(self): """Returns a list of overlapping dataset_ids for mosaic creation.""" tile_class_filter = (TC_PENDING, TC_SINGLE_SCENE, TC_SUPERSEDED) return self.get_overlaps(tile_class_filter) def get_overlaps(self, tile_class_filter): """Returns a list of overlapping dataset ids, including this dataset. A dataset is overlapping if it contains tiles that overlap with tiles belonging to this dataset. Only tiles in the tile_class_filter are considered. """ dataset_list = self.db.get_overlapping_dataset_ids( self.dataset_id, tile_class_filter=tile_class_filter ) if not dataset_list: dataset_list = [self.dataset_id] return dataset_list def create_tile_record(self, tile_contents): """Factory method to create an instance of the TileRecord class. The created object will be responsible for inserting tile table records into the database for reprojected or mosaiced tiles.""" self.collection.mark_tile_for_creation(tile_contents) return TileRecord(self.collection, self, tile_contents) def mark_as_tiled(self): """Flag the dataset record as tiled in the database. This flag does not exist in the current database schema, so this method does nothing at the moment.""" pass def list_tile_types(self): """Returns a list of the tile type ids for this dataset.""" return self.dataset_bands.keys() def get_tile_bands(self, tile_type_id): """Returns a dictionary containing the band info for one tile type. The tile_type_id must valid for this dataset, available from list_tile_types above. """ return self.dataset_bands[tile_type_id] def get_coverage(self, tile_type_id): """Given the coordinate reference system of the dataset and that of the tile_type_id, return a list of tiles within the dataset footprint""" tile_type_info = self.collection.datacube.tile_type_dict[tile_type_id] #Get geospatial information from the dataset. dataset_crs = self.mdd['projection'] dataset_geotransform = self.mdd['geo_transform'] pixels = self.mdd['x_pixels'] lines = self.mdd['y_pixels'] #Look up the datacube's projection information for this tile_type tile_crs = tile_type_info['crs'] #Get the transformation between the two projections transformation = self.define_transformation(dataset_crs, tile_crs) #Determine the bounding quadrilateral of the dataset extent #in tile coordinates dataset_bbox = self.get_bbox(transformation, dataset_geotransform, pixels, lines) #Determine maximum inner rectangle, which is guaranteed to need tiling #and the minimum outer rectangle outside which no tiles will exist. cube_origin = (tile_type_info['x_origin'], tile_type_info['y_origin']) cube_tile_size = (tile_type_info['x_size'], tile_type_info['y_size']) coverage = self.get_touched_tiles(dataset_bbox, cube_origin, cube_tile_size) return coverage # # worker methods # def __check_update_ok(self): """Checks if an update is possible, raises a DatasetError otherwise.""" tile_class_filter = (TC_SINGLE_SCENE, TC_SUPERSEDED) if self.db.dataset_older_than_database( self.dataset_dict['dataset_id'], self.dataset_dict['datetime_processed'], tile_class_filter): raise DatasetError("Dataset to be ingested is older than " + "the version in the database.") def __make_one_mosaic(self, tile_record_list): """Create a single mosaic. This create the mosaic contents, creates the database record, and marks the mosaic contents for creation on transaction commit. """ mosaic = MosaicContents( tile_record_list, self.datacube.tile_type_dict, self.dataset_dict['level_name'], self.collection.get_temp_tile_directory() ) mosaic.create_record(self.db) self.collection.mark_tile_for_creation(mosaic) def __make_mosaic_pathname(self, tile_pathname): """Return the pathname of the mosaic corrisponding to a tile.""" (tile_dir, tile_basename) = os.path.split(tile_pathname) mosaic_dir = os.path.join(tile_dir, 'mosaic_cache') if self.dataset_dict['level_name'] == 'PQA': mosaic_basename = tile_basename else: mosaic_basename = re.sub(r'\.\w+$', '.vrt', tile_basename) return os.path.join(mosaic_dir, mosaic_basename) # # Worker methods for coverage. # # These are public so that they can be called by test_dataset_record. # def define_transformation(self, dataset_crs, tile_crs): """Return the transformation between dataset_crs and tile_crs projections""" osr.UseExceptions() try: dataset_spatial_reference = self.create_spatial_ref(dataset_crs) tile_spatial_reference = self.create_spatial_ref(tile_crs) if dataset_spatial_reference is None: raise DatasetError('Unknown projecton %s' % str(dataset_crs)) if tile_spatial_reference is None: raise DatasetError('Unknown projecton %s' % str(tile_crs)) return osr.CoordinateTransformation(dataset_spatial_reference, tile_spatial_reference) except Exception: raise DatasetError('Coordinate transformation error ' + 'for transforming %s to %s' % (str(dataset_crs), str(tile_crs))) @staticmethod def create_spatial_ref(crs): """Create a spatial reference system for projecton crs. Called by define_transformation()""" # pylint: disable=broad-except osr.UseExceptions() try: spatial_ref = osr.SpatialReference() except Exception: raise DatasetError('No spatial reference done for %s' % str(crs)) try: spatial_ref.ImportFromWkt(crs) return spatial_ref except Exception: pass try: matchobj = re.match(r'EPSG:(\d+)', crs) epsg_code = int(matchobj.group(1)) spatial_ref.ImportFromEPSG(epsg_code) return spatial_ref except Exception: return None @staticmethod def get_bbox(transform, geotrans, pixels, lines): """Return the coordinates of the dataset footprint in clockwise order from upper-left""" xul, yul, dummy_z = \ transform.TransformPoint(geotrans[0], geotrans[3], 0) xur, yur, dummy_z = \ transform.TransformPoint(geotrans[0] + geotrans[1] * pixels, geotrans[3] + geotrans[4] * pixels, 0) xll, yll, dummy_z = \ transform.TransformPoint(geotrans[0] + geotrans[2] * lines, geotrans[3] + geotrans[5] * lines, 0) xlr, ylr, dummy_z = \ transform.TransformPoint( geotrans[0] + geotrans[1] * pixels + geotrans[2] * lines, geotrans[3] + geotrans[4] * pixels + geotrans[5] * lines, 0) return [(xul, yul), (xur, yur), (xlr, ylr), (xll, yll)] def get_touched_tiles(self, dataset_bbox, cube_origin, cube_tile_size): """Return a list of tuples (itile, jtile) comprising all tiles footprints that intersect the dataset bounding box""" definite_tiles, possible_tiles = \ self.get_definite_and_possible_tiles(dataset_bbox, cube_origin, cube_tile_size) coverage_set = definite_tiles #Check possible tiles: #Check if the tile perimeter intersects the dataset bbox perimeter: intersected_tiles = \ self.get_intersected_tiles(possible_tiles, dataset_bbox, cube_origin, cube_tile_size) coverage_set = coverage_set.union(intersected_tiles) possible_tiles = possible_tiles.difference(intersected_tiles) #Otherwise the tile might be wholly contained in the dataset bbox contained_tiles = \ self.get_contained_tiles(possible_tiles, dataset_bbox, cube_origin, cube_tile_size) coverage_set = coverage_set.union(contained_tiles) return coverage_set @staticmethod def get_definite_and_possible_tiles(bbox, cube_origin, cube_tile_size): """Return two lists of tile footprints: from the largest rectangle wholly contained within the dataset bbox and the smallest rectangle containing the bbox.""" #pylint: disable=too-many-locals #unpack the bbox vertices in clockwise order from upper-left xyul, xyur, xylr, xyll = bbox xul, yul = xyul xur, yur = xyur xlr, ylr = xylr xll, yll = xyll #unpack the origin of the tiled datacube (e.g. lat=0, lon=0) and the #datacube tile size xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size #Define the largest rectangle wholly contained within footprint xmin = max(xll, xul) xmax = min(xlr, xur) ymin = max(yll, ylr) ymax = min(yul, yur) xmin_index = int(floor((xmin - xorigin) / xsize)) xmax_index = int(floor((xmax - xorigin) / xsize)) ymin_index = int(floor((ymin - yorigin) / ysize)) ymax_index = int(floor((ymax - yorigin) / ysize)) definite_tiles = set([(itile, jtile) for itile in range(xmin_index, xmax_index + 1) for jtile in range(ymin_index, ymax_index + 1)]) #Define the smallest rectangle which is guaranteed to include all tiles #in the foorprint. xmin = min(xll, xul) xmax = max(xlr, xur) ymin = min(yll, ylr) ymax = max(yul, yur) xmin_index = int(floor((xmin - xorigin) / xsize)) xmax_index = int(floor((xmax - xorigin) / xsize)) ymin_index = int(floor((ymin - yorigin) / ysize)) ymax_index = int(floor((ymax - yorigin) / ysize)) possible_tiles = set([(itile, jtile) for itile in range(xmin_index, xmax_index + 1) for jtile in range(ymin_index, ymax_index + 1) ]).difference(definite_tiles) return (definite_tiles, possible_tiles) def get_intersected_tiles(self, candidate_tiles, dset_bbox, cube_origin, cube_tile_size): """Return the subset of candidate_tiles that have an intersection with the dataset bounding box""" #pylint: disable=too-many-locals xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size keep_list = [] for itile, jtile in candidate_tiles: intersection_exists = False (x0, y0) = (xorigin + itile * xsize, yorigin + (jtile + 1) * ysize) tile_bbox = [(x0, y0), (x0 + xsize, y0), (x0 + xsize, y0 - ysize), (x0, y0 - ysize)] tile_vtx_number = len(tile_bbox) dset_vtx_number = len(dset_bbox) for tile_vtx in range(tile_vtx_number): x1, y1 = tile_bbox[tile_vtx] x2, y2 = tile_bbox[(tile_vtx + 1) % tile_vtx_number] for dset_vtx in range(dset_vtx_number): x3, y3 = dset_bbox[dset_vtx] x4, y4 = dset_bbox[(dset_vtx + 1) % dset_vtx_number] xcoords = [x1, x2, x3, x4] ycoords = [y1, y2, y3, y4] intersection_exists = \ self.check_intersection(xcoords, ycoords) if intersection_exists: keep_list.append((itile, jtile)) break if intersection_exists: break return set(keep_list) @staticmethod def get_contained_tiles(candidate_tiles, dset_bbox, cube_origin, cube_tile_size): """Return the subset of candidate tiles that lie wholly within the dataset bounding box""" #pylint: disable=too-many-locals xorigin, yorigin = cube_origin xsize, ysize = cube_tile_size keep_list = [] for itile, jtile in candidate_tiles: tile_vtx_inside = [] (x0, y0) = (xorigin + itile * xsize, yorigin + (jtile + 1) * ysize) tile_bbox = [(x0, y0), (x0 + xsize, y0), (x0 + xsize, y0 - ysize), (x0, y0 - ysize)] dset_vtx_number = len(dset_bbox) for x, y in tile_bbox: #Check if this vertex lies within the dataset bounding box: winding_number = 0 for dset_vtx in range(dset_vtx_number): x1, y1 = dset_bbox[dset_vtx] x2, y2 = dset_bbox[(dset_vtx + 1) % dset_vtx_number] if y >= y1 and y < y2: if (x - x1) * (y2 - y1) > (x2 - x1) * (y - y1): winding_number += 1 elif y <= y1 and y > y2: if (x - x1) * (y2 - y1) < (x2 - x1) * (y - y1): winding_number += 1 tile_vtx_inside.append(winding_number % 2 == 1) if tile_vtx_inside.count(True) == len(tile_bbox): keep_list.append((itile, jtile)) assert tile_vtx_inside.count(True) == 4 or \ tile_vtx_inside.count(True) == 0, \ "Tile partially inside dataset bounding box but has" \ "no intersection" return set(keep_list) @staticmethod def check_intersection(xpts, ypts): """Determines if the line segments (xpts[0], ypts[0]) to (xpts[1], ypts[1]) and (xpts[2], ypts[2]) to (xpts[3], ypts[3]) intersect""" pvec = (xpts[0], ypts[0]) qvec = (xpts[2], ypts[2]) rvec = (xpts[1] - xpts[0], ypts[1] - ypts[0]) svec = (xpts[3] - xpts[2], ypts[3] - ypts[2]) rvec_cross_svec = rvec[0] * svec[1] - rvec[1] * svec[0] if rvec_cross_svec == 0: return False qminusp_cross_svec = \ (qvec[0] - pvec[0]) * svec[1] - (qvec[1] - pvec[1]) * svec[0] qminusp_cross_rvec = \ (qvec[0] - pvec[0]) * rvec[1] - (qvec[1] - pvec[1]) * rvec[0] tparameter = qminusp_cross_svec / rvec_cross_svec uparameter = qminusp_cross_rvec / rvec_cross_svec if tparameter > 0 and tparameter < 1 and \ uparameter > 0 and uparameter < 1: return True