def test_expand_metadata_without_bands(self): # We have imagery files but no bands (eg: RAW data) f = write_files({'first.txt': 'test', 'second.txt': 'test2'}) class FauxDriver(drivers.DatasetDriver): def to_band(self, dataset, path): return None def get_ga_label(self, dataset): return 'DATASET_ID_1234' def get_id(self): return 'faux' d = ptype.DatasetMetadata() # Iterator is falsey, but returns files. This triggered a bug previously. # noinspection PyTypeChecker d = package.expand_driver_metadata(FauxDriver(), d, f.iterdir()) self.assert_same( d, ptype.DatasetMetadata(id_=d.id_, ga_label='DATASET_ID_1234', product_type='faux', size_bytes=9))
def test_prepare_metadata(self): f = write_files({'first.txt': 'test', 'second.txt': 'test2'}) class FauxDriver(drivers.DatasetDriver): def to_band(self, dataset, path): numbers = { 'first': ptype.BandMetadata(path=path, number='1'), 'second': None } return numbers.get(path.stem) def get_ga_label(self, dataset): return 'DATASET_ID_1234' def get_id(self): return 'faux' d = ptype.DatasetMetadata() d = package.expand_driver_metadata(FauxDriver(), d, list(f.iterdir())) self.assert_same( d, ptype.DatasetMetadata( id_=d.id_, ga_label='DATASET_ID_1234', product_type='faux', size_bytes=9, image=ptype.ImageMetadata( bands={ '1': ptype.BandMetadata(path=f.joinpath('first.txt'), number='1') })))
def test_extract_md(_run_pdsinfo_exe): input_dir = write_files({'P1540064AAAAAAAAAAAAAA14219032341001.PDS': ''}) # def run_pdsinfo(file_): # assert file_ == input_dir # # return _run_pdsinfo_exe.return_value = b"""APID 64: count 610338 invalid 0 missing 6255 first packet: 2014/08/07 03:16:28.750910 last packet: 2014/08/07 03:21:28.604695 missing seconds: 2 day packets: 545223/64311 night packets: 0/0 engineering packets: 804/0 """ md = pds.extract_md(ptype.DatasetMetadata(), input_dir) expected = ptype.DatasetMetadata( platform=ptype.PlatformMetadata(code='AQUA'), instrument=ptype.InstrumentMetadata(name='MODIS'), format_=ptype.FormatMetadata(name='PDS'), acquisition=ptype.AcquisitionMetadata( aos=datetime.datetime(2014, 8, 7, 3, 16, 28, 750910), los=datetime.datetime(2014, 8, 7, 3, 21, 28, 604695) ), image=ptype.ImageMetadata( day_percentage_estimate=100.0 ) ) md.id_, expected.id_ = None, None assert_same(expected, md)
def test_create_typical_browse_metadata(): class TestDriver(drivers.DatasetDriver): def browse_image_bands(self, d): return '5', '1', '3' d = write_files({}) dataset = browseimage.create_typical_browse_metadata( TestDriver(), ptype.DatasetMetadata(), d) expected = ptype.DatasetMetadata( browse={ 'full': ptype.BrowseMetadata(path=d.joinpath('browse.fr.jpg'), file_type='image/jpg', red_band='5', green_band='1', blue_band='3'), 'medium': ptype.BrowseMetadata( path=d.joinpath('browse.jpg'), # Default medium size. shape=ptype.Point(1024, None), file_type='image/jpg', red_band='5', green_band='1', blue_band='3') }) expected.id_, dataset.id_ = None, None assert_same(expected, dataset)
def test_unchanged_without_id(self): # No MDF directory, only files. Don't try to extract information from the files. d = write_files({ '446.000.2013254233714881.ASA': 'a', '447.000.2013254233711482.ASA': 'a', }) id_ = uuid.uuid1() date = datetime.datetime.utcnow() metadata = mdf.extract_md( ptype.DatasetMetadata(id_=id_, creation_dt=date), d) # Should be unchanged: No USGS ID found. assert_same(metadata, ptype.DatasetMetadata(id_=id_, creation_dt=date))
def test_extract_groundstation(self): d = write_files({'LANDSAT-7.ALICE': {'dataset': {}}}) d = d.joinpath('LANDSAT-7.ALICE') md = adsfolder.extract_md(ptype.DatasetMetadata(), d) self.assertIsNotNone(md.acquisition) self.assertEqual(md.acquisition.groundstation, ptype.GroundstationMetadata(code='ASA')) d = d.joinpath('dataset') md = adsfolder.extract_md(ptype.DatasetMetadata(), d) self.assertIsNotNone(md.acquisition) self.assertEqual(md.acquisition.groundstation, ptype.GroundstationMetadata(code='ASA'))
def test_eods_fill_metadata(self): dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012" bandname = '10' bandfile = dataset_folder + '_B' + bandname + '.tif' input_folder = write_files({ dataset_folder: { 'metadata.xml': """<EODS_DATASET> <ACQUISITIONINFORMATION> <EVENT> <AOS>20141012T03:23:36</AOS> <LOS>20141012T03:29:10</LOS> </EVENT> </ACQUISITIONINFORMATION> <EXEXTENT> <TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM> <TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO> </EXEXTENT> </EODS_DATASET>""", 'scene01': { bandfile: '' } } }) expected = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_, ga_label=dataset_folder, ga_level='P54', product_type='EODS_NBAR', platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='GeoTiff'), acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36), los=datetime.datetime(2014, 10, 12, 3, 29, 10), groundstation=ptype.GroundstationMetadata(code='LGS')), extent=ptype.ExtentMetadata( center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6), from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54), to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18) ), image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78), satellite_ref_point_end=ptype.Point(x=101, y=78), bands={bandname: ptype.BandMetadata(number=bandname, path=Path(input_folder, dataset_folder, 'scene01', bandfile))}) ) dataset = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_ ) received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder)) self.assert_same(expected, received)
def test_parse_passinfo_file(self): d = write_files({ 'subdirectory': { }, 'passinfo': [ "STATION ALICE\n", "STRING S1A1C1D1R1\n", "PASSID LANDSAT-5.110912\n", "SATELLITE LANDSAT-5\n", "ORBIT 110912\n", "LINKID X\n", "BITRATE 84900000\n", "SENSOR TM\n", "START 2005-01-06T23:32:14\n", "STOP 2005-01-06T23:39:12\n", "DURATION 423\n", "COMMENTS Bit\n", "logfile acs acs.log\n", "logfile ref ref.log\n", "logfile demod demod.log\n", "logfile 1050110912.eph 1050110912.eph.log\n", "telemetry telemetry.data\n" ]}) # It should find a passinfo file one directory up. md = extraction.extract_md(ptype.DatasetMetadata(), d.joinpath('subdirectory')) self.assertEqual(md.acquisition.groundstation.code, "ASA") self.assertEqual(md.acquisition.platform_orbit, 110912) self.assertEqual(md.platform.code, "LANDSAT_5") self.assertEqual(md.instrument.name, "TM") self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14)) self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_parse_passinfo(self): pm = ptype.DatasetMetadata() lines = [ "STATION ALICE\n", "STRING S1A1C1D1R1\n", "PASSID LANDSAT-5.110912\n", "SATELLITE LANDSAT-5\n", "ORBIT 110912\n", "LINKID X\n", "BITRATE 84900000\n", "SENSOR TM\n", "START 2005-01-06T23:32:14\n", "STOP 2005-01-06T23:39:12\n", "DURATION 423\n", "COMMENTS Bit\n", "logfile acs acs.log\n", "logfile ref ref.log\n", "logfile demod demod.log\n", "logfile 1050110912.eph 1050110912.eph.log\n", "telemetry telemetry.data\n" ] md = extraction._parse_passinfo_md(pm, lines) self.assertEqual(md.acquisition.groundstation.code, "ASA") self.assertEqual(md.acquisition.platform_orbit, 110912) self.assertEqual(md.platform.code, "LANDSAT_5") self.assertEqual(md.instrument.name, "TM") self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14)) self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_unusual_filenames(self): # Some passinfo filenames have orbit numbers appended d = write_files({'passinfo.24775': [ "STATION TERSS\n", "STRING S1A1C1D1R1\n", "PASSID LANDSAT-5.110912\n", "SATELLITE LANDSAT-5\n", "ORBIT 110912\n", "LINKID X\n", "BITRATE 84900000\n", "SENSOR TM\n", "START 2005-01-06T23:32:14\n", "STOP 2005-01-06T23:39:12\n", "DURATION 423\n", "COMMENTS Bit\n", "logfile acs acs.log\n", "logfile ref ref.log\n", "logfile demod demod.log\n", "logfile 1050110912.eph 1050110912.eph.log\n", "telemetry telemetry.data\n" ]}) md = extraction.extract_md(ptype.DatasetMetadata(), d) # Station "TERSS" is hobart. self.assertEqual(md.acquisition.groundstation.code, "HOA") self.assertEqual(md.acquisition.platform_orbit, 110912) self.assertEqual(md.platform.code, "LANDSAT_5") self.assertEqual(md.instrument.name, "TM") self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14)) self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_pqa_fill(self): input_folder = write_files({ 'pqa.tif': '', 'pq_metadata.yaml': dedent( """ algorithm_information: software_version: 1.0 pq_doi: ancillary: {} tests_run: {} """) }) dataset = ptype.DatasetMetadata( id_=_EXPECTED_PQA.id_, lineage=ptype.LineageMetadata( source_datasets={ 'nbar': _EXPECTED_NBAR, 'level1': test_ls8.EXPECTED_OUT } ) ) received_dataset = drivers.PqaDriver().fill_metadata(dataset, input_folder) self.assert_same(_EXPECTED_PQA, received_dataset)
def test_aqua_pds_label(self): ds = ptype.DatasetMetadata( id_=UUID('d083fa45-1edd-11e5-8f9e-1040f381a756'), product_type='satellite_telemetry_data', creation_dt=datetime.datetime(2015, 6, 11, 5, 51, 50), platform=ptype.PlatformMetadata(code='AQUA'), instrument=ptype.InstrumentMetadata(name='MODIS'), format_=ptype.FormatMetadata(name='PDS'), rms_string='S1A1C1D1R1', acquisition=ptype.AcquisitionMetadata( aos=datetime.datetime(2014, 8, 7, 3, 16, 28, 750910), los=datetime.datetime(2014, 8, 7, 3, 16, 30, 228023), platform_orbit=65208 ), image=ptype.ImageMetadata(day_percentage_estimate=100.0), lineage=ptype.LineageMetadata( machine=ptype.MachineMetadata(), source_datasets={} ) ) self.assertEqual( "AQUA_MODIS_STD-PDS_P00_65208.S1A1C1D1R1_0_0_20140807T031628Z20140807T031630", drivers.RawDriver().get_ga_label(ds) )
def test_parse_from_driver(self): d = write_files({ 'NPP.VIIRS.11361.ALICE': { 'RNSCA-RVIRS_npp_d20140106_t0444094_e0451182_' 'b11361_c20140106045941812000_nfts_drl.h5': '' } }) metadata = drivers.RawDriver().fill_metadata( ptype.DatasetMetadata(), d.joinpath('NPP.VIIRS.11361.ALICE') ) self.assertEqual(metadata.platform.code, 'NPP') self.assertEqual(metadata.instrument.name, 'VIIRS') self.assertEqual(metadata.ga_level, 'P00') self.assertEqual(metadata.format_.name, 'HDF5') # Groundstation should be found from surrounding adsfolder. self.assertEqual( metadata.acquisition.groundstation, ptype.GroundstationMetadata(code='ASA') ) self.assertEqual(metadata.acquisition.aos, datetime.datetime(2014, 1, 6, 4, 44, 9)) self.assertEqual(metadata.acquisition.los, datetime.datetime(2014, 1, 6, 4, 59, 41)) self.assertEqual(metadata.acquisition.platform_orbit, 11361)
def init_existing_dataset(directory, source_datasets, uuid_=None, source_hostname=None): """ Package an existing dataset folder (with mostly unknown provenance). This is intended for old datasets where little information was recorded. For brand new datasets, it's better to use init_locally_processed_dataset() to capture local machine information. :param uuid_: The existing dataset_id, if any. :param source_hostname: Hostname where processed, if known. :rtype: ptype.DatasetMetadata """ md = ptype.DatasetMetadata( id_=uuid_, # Default creation time is creation of an image. creation_dt=datetime.datetime.utcfromtimestamp( directory.stat().st_ctime), lineage=ptype.LineageMetadata( machine=ptype.MachineMetadata(hostname=source_hostname), source_datasets=source_datasets)) _note_package_vers(md) return md
def _build_ls8_raw(): _reset_runtime_id() raw = ptype.DatasetMetadata( id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'), creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"), size_bytes=5680940 * 1024, ga_label= 'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1', product_type='satellite_telemetry_data', usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'), platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='MD'), acquisition=ptype.AcquisitionMetadata( aos=dateutil.parser.parse('2014-10-12T00:52:52'), los=dateutil.parser.parse('2014-10-12T00:58:37'), groundstation=ptype.GroundstationMetadata( code='ASA', antenna_coord=ptype.Coord(lat=-23.759, lon=133.8824, height=579.312)), heading='D', platform_orbit=8846), extent=None, grid_spatial=None, browse=None, image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point( 101, 70), satellite_ref_point_end=ptype.Point(101, 83)), lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata())) return raw
def test_parse_passinfo_ls7_file(self): d = write_files({'passinfo': [ "STATION ALICE\n", "STRING S1A1C2D3R3\n", "PASSID LANDSAT-7.30486\n", "SATELLITE LANDSAT-7\n", "ORBIT 30486\n", "LINKID L\n", "BITRATE 150000000\n", "SENSOR ETM\n", "START 2005-01-07T02:00:28.000\n", "STOP 2005-01-07T02:07:19.000\n", "DURATION 528\n", "COMMENTS Bit\n", "logfile acs acs.log\n", "logfile ref ref.log\n", "logfile 1070030486.eph 1070030486.eph.log\n", "logfile demod demod.log\n", "telemetry telemetry.data\n", ]}) md = extraction.extract_md(ptype.DatasetMetadata(), d) self.assertEqual(md.acquisition.groundstation.code, "ASA") self.assertEqual(md.acquisition.platform_orbit, 30486) self.assertEqual(md.platform.code, "LANDSAT_7") self.assertEqual(md.instrument.name, "ETM") self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 7, 2, 0, 28)) self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 7, 2, 7, 19))
def test_files_with_usgs_id(self): # No MDF directory, only files. Can we still extract enough info? d = write_files({ '446.000.2013254233714881.ASA': 'a', '447.000.2013254233711482.ASA': 'a', 'LC80880750762013254ASA00_IDF.xml': 'a', 'LC80880750762013254ASA00_MD5.txt': 'a', }) def _test_mdf_output(metadata): self.assertEqual(metadata.usgs.interval_id, 'LC80880750762013254ASA00') self.assertEqual(metadata.platform.code, 'LANDSAT_8') self.assertEqual(metadata.instrument.name, 'OLI_TIRS') self.assertEqual(metadata.format_.name, 'MD') self.assertEqual(metadata.ga_level, 'P00') self.assertEqual(metadata.image.satellite_ref_point_start, ptype.Point(88, 75)) self.assertEqual(metadata.image.satellite_ref_point_end, ptype.Point(88, 76)) self.assertEqual(metadata.acquisition.groundstation.code, 'ASA') # No longer bother with vague center date. # self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11)) self.assertEqual( metadata.acquisition.aos, datetime.datetime(2013, 9, 11, 23, 36, 11, 482000)) self.assertEqual( metadata.acquisition.los, datetime.datetime(2013, 9, 11, 23, 37, 14, 881000)) metadata = mdf.extract_md(ptype.DatasetMetadata(), d) _test_mdf_output(metadata)
def test_no_directory(self): d = write_files({ 'L7EB2013259012832ASN213I00.data': 'nothing', 'L7EB2013259012832ASN213Q00.data': 'nothing' }) self.assertEqual((None, set()), mdf.find_mdf_files(d)) # Make sure that metadata is not modified when no MDF is found. starting_md = ptype.DatasetMetadata() id_ = starting_md.id_ creation_dt = starting_md.creation_dt expected_dt = ptype.DatasetMetadata(id_=id_, creation_dt=creation_dt) output = mdf.extract_md(starting_md, d) self.assertEqual(expected_dt, output)
def test_expand_metadata(self): id_ = uuid.uuid1() create_dt = datetime.datetime.utcnow() d = ptype.DatasetMetadata( id_=id_, creation_dt=create_dt, platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), image=ptype.ImageMetadata( bands={ '4': BandMetadata( path=PosixPath( '/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'), number='4', ) }), acquisition=ptype.AcquisitionMetadata( groundstation=ptype.GroundstationMetadata('ALSP'))) assert_same( metadata.expand_common_metadata(d), ptype.DatasetMetadata( id_=id_, creation_dt=create_dt, platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), image=ptype.ImageMetadata( bands={ '4': BandMetadata(path=PosixPath( '/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'), type_='reflective', label='Visible Red', number='4', cell_size=25.0), }), acquisition=ptype. AcquisitionMetadata(groundstation=ptype.GroundstationMetadata( # Should have translated groundstation 'ALSP' to standard GSI form 'ASA' code='ASA', label='Alice Springs', eods_domain_code='002'))))
def test_extract_ls7_rcc_filename_fields(self): md = rccfile._extract_rcc_filename_fields( ptype.DatasetMetadata(), 'L7EB2012028010752ASA111I.data') self.assertEqual(md.platform.code, 'LANDSAT_7') self.assertEqual(md.instrument.name, 'ETM') self.assertEqual(md.instrument.operation_mode, 'BUMPER') self.assertEqual(md.acquisition.groundstation.code, 'ASA') # Is AOS correct here? The RCC spec is not clear exactly when the timestamp is taken. self.assertEqual(md.acquisition.aos, datetime.datetime(2012, 1, 28, 1, 7, 52))
def test_gqa(): md = ptype.DatasetMetadata(lineage=ptype.LineageMetadata( machine=ptype.MachineMetadata())) gqa.populate_from_gqa(md, _GQA_PATH) print(repr(md.gqa)) assert md.gqa == { "cep90": 0.41, "colors": { "blue": 600, "green": 30164, "red": 336, "teal": 1340, "yellow": 399 }, "final_gcp_count": 32582, "ref_date": datetime.date(2000, 9, 4), "ref_source": "GLS_v2", "ref_source_path": "/g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF", "residual": { "abs": { "x": 0.2, "y": 0.23 }, "abs_iterative_mean": { "x": 0.15, "y": 0.17 }, "iterative_mean": { "x": 0.02, "y": 0 }, "iterative_stddev": { "x": 0.32, "y": 0.52 }, "mean": { "x": 0.01, "y": -0.03 }, "stddev": { "x": 1.27, "y": 3.94 } } } assert 'gqa' in md.lineage.machine.software_versions assert md.lineage.machine.software_versions['gqa'] == { 'repo_url': "https://github.com/GeoscienceAustralia/gqa.git", 'version': "0.4+20.gb0d00dc" }
def test_extract_ls5_rcc_filename_fields(self): md = rccfile._extract_rcc_filename_fields( ptype.DatasetMetadata(), 'L5TT2003339014237ASA111I00.data') self.assertEqual(md.platform.code, 'LANDSAT_5') self.assertEqual(md.instrument.name, 'TM') self.assertEqual(md.instrument.operation_mode, 'SAM') self.assertEqual(md.acquisition.groundstation.code, 'ASA') self.assertEqual(md.format_.version, 0) # Is AOS correct here? The RCC spec is not clear exactly when the timestamp is taken. self.assertEqual(md.acquisition.aos, datetime.datetime(2003, 12, 5, 1, 42, 37))
def test_override_gqa_version(): md = ptype.DatasetMetadata(lineage=ptype.LineageMetadata( machine=ptype.MachineMetadata())) md.lineage.machine.note_software_version('gqa', '0.0.0') gqa.populate_from_gqa(md, _GQA_PATH) # It should have overridden it (more specific). assert 'gqa' in md.lineage.machine.software_versions assert md.lineage.machine.software_versions['gqa'] == { 'repo_url': "https://github.com/GeoscienceAustralia/gqa.git", 'version': "0.4+20.gb0d00dc" }
def test_directory(self): d = write_files({ 'LC80880750762013254ASA00': { '446.000.2013254233714881.ASA': 'a', '447.000.2013254233711482.ASA': 'a', 'LC80880750762013254ASA00_IDF.xml': 'a', 'LC80880750762013254ASA00_MD5.txt': 'a', } }) def _test_mdf_output(metadata): self.assertEqual(metadata.usgs.interval_id, 'LC80880750762013254ASA00') self.assertEqual(metadata.platform.code, 'LANDSAT_8') self.assertEqual(metadata.instrument.name, 'OLI_TIRS') self.assertEqual(metadata.format_.name, 'MD') self.assertEqual(metadata.ga_level, 'P00') self.assertEqual(metadata.image.satellite_ref_point_start, ptype.Point(88, 75)) self.assertEqual(metadata.image.satellite_ref_point_end, ptype.Point(88, 76)) self.assertEqual(metadata.acquisition.groundstation.code, 'ASA') # No longer bother with vague center date. # self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11)) self.assertEqual( metadata.acquisition.aos, datetime.datetime(2013, 9, 11, 23, 36, 11, 482000)) self.assertEqual( metadata.acquisition.los, datetime.datetime(2013, 9, 11, 23, 37, 14, 881000)) metadata = mdf.extract_md(ptype.DatasetMetadata(), d) _test_mdf_output(metadata) # It should also work when given the specific MDF folder. metadata = mdf.extract_md(ptype.DatasetMetadata(), d.joinpath('LC80880750762013254ASA00')) _test_mdf_output(metadata)
def _check_rcc_parse(self, d): md = rccfile.extract_md(ptype.DatasetMetadata(), d) self.assertEqual(md.platform.code, 'LANDSAT_7') self.assertEqual(md.instrument.name, 'ETM') self.assertEqual(md.acquisition.groundstation.code, 'ASN') self.assertEqual(md.ga_level, 'P00') self.assertEqual(md.format_.name, 'RCC') self.assertEqual(md.usgs.interval_id, 'L7EB2013259012832ASN213') self.assertEqual(md.acquisition.aos, datetime.datetime(2013, 9, 16, 1, 28, 32)) # From the old onreceipt codebase, # Default L7 LOS is: AOS + (I.data fileSize) * 8.0 / 75000000.0 self.assertEqual(md.acquisition.los, datetime.datetime(2013, 9, 16, 1, 28, 32))
def testExtractHDF5FilenameFields(self): metadata = extraction._extract_hdf5_filename_fields( ptype.DatasetMetadata(), 'RNSCA-RVIRS_npp_d20130422_t0357358_e0410333_b07686_c20130422041225898000_nfts_drl.h5' ) self.assertEqual(metadata.platform.code, 'NPP') self.assertEqual(metadata.instrument.name, 'VIIRS') self.assertEqual(metadata.acquisition.platform_orbit, 7686) self.assertEqual(metadata.acquisition.aos, datetime.datetime(2013, 4, 22, 3, 57, 35)) self.assertEqual(metadata.acquisition.los, datetime.datetime(2013, 4, 22, 4, 12, 25))
def test_directory_parse(self): dataset_id = 'LC80880750762013254ASA00' metadata = mdf._extract_mdf_id_fields(ptype.DatasetMetadata(), dataset_id) self.assertEqual(metadata.usgs.interval_id, dataset_id) self.assertEqual(metadata.platform.code, 'LANDSAT_8') self.assertEqual(metadata.instrument.name, 'OLI_TIRS') self.assertEqual(metadata.image.satellite_ref_point_start, ptype.Point(88, 75)) self.assertEqual(metadata.image.satellite_ref_point_end, ptype.Point(88, 76)) self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
def test_directory(self): d = write_files({ 'LANDSAT-8.11308': { 'LC80880750762013254ASA00': { '446.000.2013254233714881.ASA': 'a', '447.000.2013254233711482.ASA': 'a', 'LC80880750762013254ASA00_IDF.xml': 'a', 'LC80880750762013254ASA00_MD5.txt': 'a', } } }) # Read orbit from folder name d = d.joinpath('LANDSAT-8.11308') ds = ptype.DatasetMetadata() metadata = adsfolder.extract_md(ds, d) self.assertEqual(metadata.acquisition.platform_orbit, 11308) # Read orbit from parent folder name. d = d.joinpath('LC80880750762013254ASA00') ds = ptype.DatasetMetadata() metadata = adsfolder.extract_md(ds, d) self.assertEqual(metadata.acquisition.platform_orbit, 11308)
def test_parse_l5_rcc_filenames(self): d = write_files({'L5TB2003339014237ASA111I00.data': 'nothing'}) md = rccfile.extract_md(ptype.DatasetMetadata(), d) self.assertEqual(md.platform.code, 'LANDSAT_5') self.assertEqual(md.instrument.name, 'TM') self.assertEqual(md.acquisition.groundstation.code, 'ASA') self.assertEqual(md.format_.name, 'RCC') self.assertEqual(md.usgs.interval_id, 'L5TB2003339014237ASA111') self.assertEqual(md.acquisition.aos, datetime.datetime(2003, 12, 5, 1, 42, 37)) # From the old onreceipt codebase, # Default L5 LOS is: AOS + (I.data fileSize) * 8.0 / 84900000.0 self.assertEqual(md.acquisition.los, datetime.datetime(2003, 12, 5, 1, 42, 37))
def test_nbar_fill_metadata(self): input_folder = write_files({ 'reflectance_brdf_1.tif': '', 'reflectance_brdf_2.tif': '', 'reflectance_brdf_3.tif': '', 'reflectance_brdf_4.tif': '', 'reflectance_brdf_5.tif': '', 'reflectance_brdf_6.tif': '', 'reflectance_brdf_7.tif': '', 'reflectance_terrain_1.tif': '', 'reflectance_terrain_2.tif': '', 'reflectance_terrain_3.tif': '', 'reflectance_terrain_4.tif': '', 'reflectance_terrain_5.tif': '', 'reflectance_terrain_6.tif': '', 'reflectance_terrain_7.tif': '', 'reflectance_lambertian_1.tif': '', 'reflectance_lambertian_2.tif': '', 'reflectance_lambertian_3.tif': '', 'reflectance_lambertian_4.tif': '', 'reflectance_lambertian_5.tif': '', 'reflectance_lambertian_6.tif': '', 'reflectance_lambertian_7.tif': '', 'nbar_metadata.yaml': dedent( """ algorithm_information: software_version: 1.0 algorithm_version: 1.0 arg25_doi: nbar_doi: nbar_terrain_corrected_doi: ancillary_data: {} """), }) dataset = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_, lineage=ptype.LineageMetadata( source_datasets={ 'level1': test_ls8.EXPECTED_OUT } ) ) received_dataset = drivers.NbarDriver('terrain').fill_metadata(dataset, input_folder) self.assert_same(_EXPECTED_NBAR, received_dataset)