def test_crosscheck_pq8_fc8(self): """Cross-check metadata between Landsat 8 pc and fc datasets.""" pq_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.PQ8_DIR, self.PQ8_SCENE)) fc_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.FC8_DIR, self.FC8_SCENE)) self.cross_check(pq_ds, fc_ds, self.CROSSCHECK_KEYS_TWO)
def test_crosscheck_ortho_pq(self): """Cross-check metadata between ortho and pq datasets.""" ortho_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.ORTHO_DIR, self.ORTHO_SCENE)) pq_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.PQ_DIR, self.PQ_SCENE)) self.cross_check(ortho_ds, pq_ds, self.CROSSCHECK_KEYS_TWO)
def test_crosscheck_ortho8_fc8(self): """Cross-check metadata between Landsat 8 ortho and fc datasets.""" ortho_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.ORTHO8_DIR, self.ORTHO8_SCENE)) fc_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.FC8_DIR, self.FC8_SCENE)) self.cross_check(ortho_ds, fc_ds, self.CROSSCHECK_KEYS_TWO)
def test_crosscheck_ortho8_nbar8(self): """Cross-check metadata between Landsat 8 ortho and nbar datasets.""" ortho_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.ORTHO8_DIR, self.ORTHO8_SCENE)) nbar_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.NBAR8_DIR, self.NBAR8_SCENE)) self.cross_check(ortho_ds, nbar_ds, self.CROSSCHECK_KEYS_TWO)
def test_fc8_scene(self): """Test for a Landsat 8 Fractional Cover scene.""" fc_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.FC8_DIR, self.FC8_SCENE)) mdd = fc_ds.metadata_dict self.dump_metadata('fc8_metadata.txt', mdd, self.SMALL_METADATA_KEYS) self.dump_string('fc8_xml.xml', mdd['xml_text']) self.assertIsNone(mdd['mtl_text']) self.check_file('fc8_metadata.txt') self.check_file('fc8_xml.xml')
def test_nbar8_scene(self): """Test for a Landsat 8 NBAR scene.""" nbar_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.NBAR8_DIR, self.NBAR8_SCENE)) mdd = nbar_ds.metadata_dict self.dump_metadata('nbar8_metadata.txt', mdd, self.SMALL_METADATA_KEYS) self.dump_string('nbar8_xml.xml', mdd['xml_text']) self.assertIsNone(mdd['mtl_text']) self.check_file('nbar8_metadata.txt') self.check_file('nbar8_xml.xml')
def test_pq8_scene(self): """Test for a Landsat 8 Pixel Quality scene.""" pq_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.PQ8_DIR, self.PQ8_SCENE)) mdd = pq_ds.metadata_dict self.dump_metadata('pq8_metadata.txt', mdd, self.SMALL_METADATA_KEYS) self.dump_string('pq8_xml.xml', mdd['xml_text']) self.assertIsNone(mdd['mtl_text']) self.check_file('pq8_metadata.txt') self.check_file('pq8_xml.xml')
def check_buildvrt(self, idataset): """Test the LandsatBandstack.buildvrt() method by comparing output to a file on disk""" assert idataset in range(len(DATASETS_TO_INGEST)) print 'Testing Dataset %s' %DATASETS_TO_INGEST[idataset] dset = LandsatDataset(DATASETS_TO_INGEST[idataset]) # Create a DatasetRecord instance so that we can access its # list_tile_types() method. In doing this we need to create a # collection object and entries on the acquisition and dataset # tables of the database. self.collection.begin_transaction() acquisition = \ self.collection.create_acquisition_record(dset) dset_record = acquisition.create_dataset_record(dset) self.collection.commit_transaction() tile_type_list = dset_record.list_tile_types() #Assume dataset has tile_type = 1 only: tile_type_id = 1 dataset_bands_dict = dset_record.get_tile_bands(tile_type_id) ls_bandstack = dset.stack_bands(dataset_bands_dict) temp_dir = self.collection.get_temp_tile_directory() ls_bandstack.buildvrt(temp_dir) # Get benchmark vrt for comparision vrt_benchmark = os.path.join(self.BENCHMARK_DIR, os.path.basename(ls_bandstack.vrt_name)) diff_cmd = ["diff", "-I", "[Ff]ilename", "%s" %vrt_benchmark, "%s" %ls_bandstack.vrt_name ] result = execute(diff_cmd, shell=False) if result['stdout'] != '': self.fail("Differences between vrt files:\n" + result['stdout']) if result['stderr'] != '': self.fail("Error in system diff command:\n" + result['stderr'])
def test_ortho_scene(self): """Test for an ORTHO (level 1) scene.""" ortho_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.ORTHO_DIR, self.ORTHO_SCENE)) mdd = ortho_ds.metadata_dict self.dump_metadata('ortho_metadata.txt', mdd, self.SMALL_METADATA_KEYS) self.dump_string('ortho_xml.xml', mdd['xml_text']) self.dump_string('ortho_mtl.txt', mdd['mtl_text']) self.check_file('ortho_metadata.txt') self.check_file('ortho_xml.xml') self.check_file('ortho_mtl.txt')
def check_buildvrt(self, idataset): """Test the LandsatBandstack.buildvrt() method by comparing output to a file on disk""" assert idataset in range(len(DATASETS_TO_INGEST)) print 'Testing Dataset %s' % DATASETS_TO_INGEST[idataset] dset = LandsatDataset(DATASETS_TO_INGEST[idataset]) # Create a DatasetRecord instance so that we can access its # list_tile_types() method. In doing this we need to create a # collection object and entries on the acquisition and dataset # tables of the database. self.collection.begin_transaction() acquisition = \ self.collection.create_acquisition_record(dset) dset_record = acquisition.create_dataset_record(dset) self.collection.commit_transaction() tile_type_list = dset_record.list_tile_types() #Assume dataset has tile_type = 1 only: tile_type_id = 1 dataset_bands_dict = dset_record.get_tile_bands(tile_type_id) ls_bandstack = dset.stack_bands(dataset_bands_dict) temp_dir = self.collection.get_temp_tile_directory() ls_bandstack.buildvrt(temp_dir) # Get benchmark vrt for comparision vrt_benchmark = os.path.join(self.BENCHMARK_DIR, os.path.basename(ls_bandstack.vrt_name)) diff_cmd = [ "diff", "-I", "[Ff]ilename", "%s" % vrt_benchmark, "%s" % ls_bandstack.vrt_name ] result = execute(diff_cmd, shell=False) if result['stdout'] != '': self.fail("Differences between vrt files:\n" + result['stdout']) if result['stderr'] != '': self.fail("Error in system diff command:\n" + result['stderr'])
def test_build_metadata_dict(self): """Test for the build_metadata_dict method. This method is actually defined in AbstractDataset, but an AbstractDataset cannot be instantiated, so it is tested here. """ ortho_ds = LandsatDataset( os.path.join(self.INPUT_DIR, self.ORTHO_DIR, self.ORTHO_SCENE)) mdd = ortho_ds.metadata_dict self.assertEqual(set(self.METADATA_KEYS), set(mdd.keys())) for k in mdd.keys(): mdd_value = mdd[k] accessor_name = 'get_' + k accessor_value = getattr(ortho_ds, accessor_name)() self.assertEqual(mdd_value, accessor_value)
def test_reproject(self): """Test the Landsat tiling process method by comparing output to a file on disk.""" # pylint: disable=too-many-locals #For each processing_level, and dataset keep record of those #tile footprints in the benchmark set. bench_footprints = {} for iacquisition in range(len(TestIngest.DATASETS_TO_INGEST['PQA'])): for processing_level in ['PQA', 'NBAR', 'ORTHO']: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite if iacquisition > 0: continue if processing_level in ['NBAR']: continue dataset_path = \ TestIngest.DATASETS_TO_INGEST[processing_level]\ [iacquisition] LOGGER.info('Testing Dataset %s', dataset_path) dset = LandsatDataset(dataset_path) #return # Create a DatasetRecord instance so that we can access its # list_tile_types() method. In doing this we need to create a # collection object and entries on the acquisition and dataset # tables of the database. self.collection.begin_transaction() acquisition = \ self.collection.create_acquisition_record(dset) dset_record = acquisition.create_dataset_record(dset) self.collection.commit_transaction() # List the benchmark footprints associated with this datset ftprints = \ self.get_benchmark_footprints(dset_record.mdd, TestIngest.BENCHMARK_DIR) bench_footprints.setdefault(processing_level, {}) bench_footprints[processing_level].setdefault(iacquisition, {}) bench_footprints[processing_level][iacquisition] = ftprints LOGGER.info('bench_footprints=%s', str(ftprints)) # Get tile types dummy_tile_type_list = dset_record.list_tile_types() # Assume dataset has tile_type = 1 only: tile_type_id = 1 dataset_bands_dict = dset_record.get_tile_bands(tile_type_id) ls_bandstack = dset.stack_bands(dataset_bands_dict) temp_dir = os.path.join(self.ingester.datacube.tile_root, 'ingest_temp') # Form scene vrt ls_bandstack.buildvrt(temp_dir) # Reproject scene data onto selected tile coverage tile_footprint_list = dset_record.get_coverage(tile_type_id) LOGGER.info('coverage=%s', str(tile_footprint_list)) for tile_footprint in tile_footprint_list: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite if tile_footprint not in [(117, -35), (115, -34)]: continue tile_contents = \ self.collection.create_tile_contents(tile_type_id, tile_footprint, ls_bandstack) LOGGER.info('reprojecting for %s tile %s', processing_level, str(tile_footprint)) tile_contents.reproject() # Because date-time of PQA datasets is coming directly from # the PQA dataset, rather NBAR, match on ymd string of # datetime, rather than the micorseconds version in the # NBAR data. tile_benchmark = \ self.get_benchmark_tile(dset_record.mdd, TestIngest.BENCHMARK_DIR, tile_footprint) LOGGER.info('tile_benchmark is %s', tile_benchmark) if tile_contents.has_data(): LOGGER.info('Tile %s has data', str(tile_footprint)) # The tile might have data but, if PQA does not, then # the benchmark tile will not exist if tile_footprint not in bench_footprints \ [processing_level][iacquisition]: assert tile_footprint not in \ bench_footprints['PQA'][iacquisition], \ "Old ingester found PQA tile and should have "\ "found cooresponding tile for %s"\ %processing_level LOGGER.info('%s tile %s has data in new ingester', processing_level, str(tile_footprint)) continue # Tile exists in old ingester and new ingester LOGGER.info('Calling load and check ...') ([data1, data2], dummy_nlayers) = \ TestLandsatTiler.load_and_check( tile_benchmark, tile_contents.temp_tile_output_path, tile_contents.band_stack.band_dict, tile_contents.band_stack.band_dict) LOGGER.info('Checking arrays ...') assert (data1 == data2).all(), \ "Reprojected tile differs " \ "from %s" %tile_benchmark LOGGER.info('...OK') else: LOGGER.info('No data in %s', str(tile_footprint)) assert tile_footprint not in \ bench_footprints[processing_level][iacquisition], \ "%s tile %s does not have data " \ %(processing_level, str(tile_footprint)) LOGGER.info('-' * 80)
def test_reproject(self): """Test the Landsat tiling process method by comparing output to a file on disk.""" # pylint: disable=too-many-locals #For each processing_level, and dataset keep record of those #tile footprints in the benchmark set. for iacquisition in range(len(TestIngest.DATASETS_TO_INGEST['PQA'])): for processing_level in ['PQA', 'NBAR', 'ORTHO']: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite #if iacquisition > 0: # continue #if processing_level in ['NBAR']: # continue dataset_path = \ TestIngest.DATASETS_TO_INGEST[processing_level]\ [iacquisition] LOGGER.info('Testing Dataset %s', dataset_path) dset = LandsatDataset(dataset_path) #return # Create a DatasetRecord instance so that we can access its # list_tile_types() method. In doing this we need to create a # collection object and entries on the acquisition and dataset # tables of the database. self.collection.begin_transaction() acquisition = \ self.collection.create_acquisition_record(dset) dset_record = acquisition.create_dataset_record(dset) # Get tile types dummy_tile_type_list = dset_record.list_tile_types() # Assume dataset has tile_type = 1 only: tile_type_id = 1 dataset_bands_dict = dset_record.get_tile_bands(tile_type_id) ls_bandstack = dset.stack_bands(dataset_bands_dict) # Form scene vrt ls_bandstack.buildvrt(self.collection. get_temp_tile_directory()) # Reproject scene data onto selected tile coverage tile_footprint_list = dset_record.get_coverage(tile_type_id) LOGGER.info('coverage=%s', str(tile_footprint_list)) for tile_footprint in tile_footprint_list: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite #if tile_footprint not in [(117, -35), (115, -34)]: # continue tile_contents = \ self.collection.create_tile_contents(tile_type_id, tile_footprint, ls_bandstack) LOGGER.info('reprojecting for %s tile %s...', processing_level, str(tile_footprint)) tile_contents.reproject() LOGGER.info('...done') if self.POPULATE_EXPECTED: continue #Do comparision with expected results tile_benchmark = self.swap_dir_in_path(tile_contents. tile_output_path, 'output', 'expected') if tile_contents.has_data(): LOGGER.info('Tile %s has data', str(tile_footprint)) LOGGER.info("Comparing test output with benchmark:\n"\ "benchmark: %s\ntest output: %s", tile_benchmark, tile_contents.temp_tile_output_path) # Do comparision with expected directory LOGGER.info('Calling load and check ...') ([data1, data2], dummy_nlayers) = \ TestLandsatTiler.load_and_check( tile_benchmark, tile_contents.temp_tile_output_path, tile_contents.band_stack.band_dict, tile_contents.band_stack.band_dict) LOGGER.info('Checking arrays ...') if not (data1 == data2).all(): self.fail("Reprojected tile differs " \ "from %s" %tile_benchmark) LOGGER.info('...OK') else: LOGGER.info('No data in %s', str(tile_footprint)) LOGGER.info('-' * 80) self.collection.commit_transaction()
def test_reproject(self): """Test the Landsat tiling process method by comparing output to a file on disk.""" # pylint: disable=too-many-locals #For each processing_level, and dataset keep record of those #tile footprints in the benchmark set. for iacquisition in range(len(TestIngest.DATASETS_TO_INGEST['PQA'])): for processing_level in ['PQA', 'NBAR', 'ORTHO']: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite #if iacquisition > 0: # continue #if processing_level in ['NBAR']: # continue dataset_path = \ TestIngest.DATASETS_TO_INGEST[processing_level]\ [iacquisition] LOGGER.info('Testing Dataset %s', dataset_path) dset = LandsatDataset(dataset_path) #return # Create a DatasetRecord instance so that we can access its # list_tile_types() method. In doing this we need to create a # collection object and entries on the acquisition and dataset # tables of the database. self.collection.begin_transaction() acquisition = \ self.collection.create_acquisition_record(dset) dset_record = acquisition.create_dataset_record(dset) # Get tile types dummy_tile_type_list = dset_record.list_tile_types() # Assume dataset has tile_type = 1 only: tile_type_id = 1 dataset_bands_dict = dset_record.get_tile_bands(tile_type_id) ls_bandstack = dset.stack_bands(dataset_bands_dict) # Form scene vrt ls_bandstack.buildvrt( self.collection.get_temp_tile_directory()) # Reproject scene data onto selected tile coverage tile_footprint_list = dset_record.get_coverage(tile_type_id) LOGGER.info('coverage=%s', str(tile_footprint_list)) for tile_footprint in tile_footprint_list: #Skip all but PQA and ORTHO for first dataset. #TODO program this in as a paramter of the suite #if tile_footprint not in [(117, -35), (115, -34)]: # continue tile_contents = \ self.collection.create_tile_contents(tile_type_id, tile_footprint, ls_bandstack) LOGGER.info('reprojecting for %s tile %s...', processing_level, str(tile_footprint)) tile_contents.reproject() LOGGER.info('...done') if self.POPULATE_EXPECTED: continue #Do comparision with expected results tile_benchmark = self.swap_dir_in_path( tile_contents.tile_output_path, 'output', 'expected') if tile_contents.has_data(): LOGGER.info('Tile %s has data', str(tile_footprint)) LOGGER.info("Comparing test output with benchmark:\n"\ "benchmark: %s\ntest output: %s", tile_benchmark, tile_contents.temp_tile_output_path) # Do comparision with expected directory LOGGER.info('Calling load and check ...') ([data1, data2], dummy_nlayers) = \ TestLandsatTiler.load_and_check( tile_benchmark, tile_contents.temp_tile_output_path, tile_contents.band_stack.band_dict, tile_contents.band_stack.band_dict) LOGGER.info('Checking arrays ...') if not (data1 == data2).all(): self.fail("Reprojected tile differs " \ "from %s" %tile_benchmark) LOGGER.info('...OK') else: LOGGER.info('No data in %s', str(tile_footprint)) LOGGER.info('-' * 80) self.collection.commit_transaction()