def test_prepare_metadata(self): f = write_files({'first.txt': 'test', 'second.txt': 'test2'}) class FauxDriver(drivers.DatasetDriver): def to_band(self, dataset, path): numbers = { 'first': ptype.BandMetadata(path=path, number='1'), 'second': None } return numbers.get(path.stem) def get_ga_label(self, dataset): return 'DATASET_ID_1234' def get_id(self): return 'faux' d = ptype.DatasetMetadata() d = package.expand_driver_metadata(FauxDriver(), d, list(f.iterdir())) self.assert_same( d, ptype.DatasetMetadata( id_=d.id_, ga_label='DATASET_ID_1234', product_type='faux', size_bytes=9, image=ptype.ImageMetadata( bands={ '1': ptype.BandMetadata(path=f.joinpath('first.txt'), number='1') })))
def to_band(self, dataset, path): """ :type dataset: ptype.DatasetMetadata :type path: pathlib.Path :rtype: ptype.BandMetadata >>> OrthoDriver().to_band(None, Path('/tmp/out/LT51030782005002ASA00_B3.TIF')) BandMetadata(path=PosixPath('/tmp/out/LT51030782005002ASA00_B3.TIF'), number='3') >>> OrthoDriver().to_band(None, Path('/tmp/out/LC81090852015088LGN00_B10.tif')) BandMetadata(path=PosixPath('/tmp/out/LC81090852015088LGN00_B10.tif'), number='10') >>> OrthoDriver().to_band(None, Path('/data/output/LE70900782007292ASA00_B6_VCID_2.TIF')) BandMetadata(path=PosixPath('/data/output/LE70900782007292ASA00_B6_VCID_2.TIF'), number='6_vcid_2') >>> # No bands for non-tiff files. >>> OrthoDriver().to_band(None, Path('/tmp/out/LC81090852015088LGN00_MTL.txt')) >>> OrthoDriver().to_band(None, Path('/tmp/out/passinfo')) >>> # A DEM image -- not included as a band. >>> OrthoDriver().to_band(None, Path('LT05_L1TP_108078_20060703_20170309_01_T1_DEM.TIF')) """ if path.suffix.lower() != '.tif': return None name = path.stem.lower() # A DEM image -- the only tif without a 'B' prefix. # We don't include it in the list of bands according to Lan-Wei, as it's not part of a typical USGS package. if name.endswith('_dem'): return None # Images end in a band number (eg '_B12.tif'). Extract it. position = name.rfind('_b') if position == -1: raise ValueError('Unexpected tif image in level1: %r' % path) band_number = name[position + 2:] return ptype.BandMetadata(path=path, number=band_number)
def to_band(self, dataset, path): """ :type dataset: ptype.DatasetMetadata :type path: Path :param path: The filename of the input file. :rtype: ptype.BandMetadata """ if path.suffix != '.tif': return None return ptype.BandMetadata(path=path, number='pqa')
def test_eods_fill_metadata(self): dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012" bandname = '10' bandfile = dataset_folder + '_B' + bandname + '.tif' input_folder = write_files({ dataset_folder: { 'metadata.xml': """<EODS_DATASET> <ACQUISITIONINFORMATION> <EVENT> <AOS>20141012T03:23:36</AOS> <LOS>20141012T03:29:10</LOS> </EVENT> </ACQUISITIONINFORMATION> <EXEXTENT> <TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM> <TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO> </EXEXTENT> </EODS_DATASET>""", 'scene01': { bandfile: '' } } }) expected = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_, ga_label=dataset_folder, ga_level='P54', product_type='EODS_NBAR', platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='GeoTiff'), acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36), los=datetime.datetime(2014, 10, 12, 3, 29, 10), groundstation=ptype.GroundstationMetadata(code='LGS')), extent=ptype.ExtentMetadata( center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6), from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54), to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18) ), image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78), satellite_ref_point_end=ptype.Point(x=101, y=78), bands={bandname: ptype.BandMetadata(number=bandname, path=Path(input_folder, dataset_folder, 'scene01', bandfile))}) ) dataset = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_ ) received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder)) self.assert_same(expected, received)
def to_band(self, dataset, path): """ :type dataset: ptype.DatasetMetadata :type path: Path :rtype: ptype.BandMetadata >>> p = Path('/tmp/something/reflectance_terrain_3.tif') >>> NbarDriver('terrain').to_band(None, p).number '3' >>> NbarDriver('terrain').to_band(None, p).path PosixPath('/tmp/something/reflectance_terrain_3.tif') >>> p = Path('/tmp/something/LS8_OLITIRS_NBAR_P54_GANBART01-002_112_079_20140126_B4.tif') >>> NbarDriver('terrain').to_band(None, p).number '4' """ return ptype.BandMetadata(path=path, number=_read_band_number(path))
def test_pqa_to_band(self): input_folder = write_files({ 'pqa.tif': '', 'process.log': '', 'passinfo': '', }) # Creates a single band. self.assertEqual( ptype.BandMetadata(path=input_folder.joinpath('pqa.tif'), number='pqa'), drivers.PqaDriver().to_band(None, input_folder.joinpath('pqa.tif')) ) # Other files should not be bands. self.assertIsNone(drivers.PqaDriver().to_band(None, input_folder.joinpath('process.log'))) self.assertIsNone(drivers.PqaDriver().to_band(None, input_folder.joinpath('passinfo')))
def _read_bands(mtl_, folder_path): """ :type mtl_: dict of (str, obj) :type folder_path: pathlib.Path >>> import pathlib >>> _read_bands({'PRODUCT_METADATA': { ... 'file_name_band_9': "LC81010782014285LGN00_B9.TIF"} ... }, folder_path=pathlib.Path('product/')) {'9': BandMetadata(path=PosixPath('product/LC81010782014285LGN00_B9.TIF'), number='9')} """ bs = _read_mtl_band_filenames(mtl_) # TODO: shape, size, md5 return dict([(number, ptype.BandMetadata(path=folder_path / filename, number=number)) for (number, filename) in bs.items()])
def to_band(self, dataset, path): """ :type dataset: ptype.DatasetMetadata :type final_path: pathlib.Path :rtype: ptype.BandMetadata """ if path.suffix.lower() != '.tif': return None name = path.stem # Images end in a band number (eg '_B12.tif'). Extract it. position = name.rfind('_') if position == -1: raise ValueError('Unexpected tif image in eods: %r' % path) if re.match(r"[Bb]\d+", name[position + 1:]): band_number = name[position + 2:] else: band_number = name[position + 1:] return ptype.BandMetadata(path=path, number=band_number)
def to_band(self, dataset, path): numbers = { 'first': ptype.BandMetadata(path=path, number='1'), 'second': None } return numbers.get(path.stem)
def _build_ls8_ortho(): _reset_runtime_id() return ptype.DatasetMetadata( id_=uuid.UUID('17b92c16-51d3-11e4-909d-005056bb6972'), ga_label='LS8_OLITIRS_OTH_P51_GALPGS01-002_101_078_20141012', product_type='GAORTHO01', creation_dt=dateutil.parser.parse('2014-10-12 05:46:20'), size_bytes=2386550 * 1024, platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS', type_="Multi-Spectral", operation_mode='PUSH-BROOM'), format_=ptype.FormatMetadata(name='GeoTiff', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969), ur=ptype.Coord(lat=-24.96826, lon=136.24838), lr=ptype.Coord(lat=-26.96338, lon=136.26962), ll=ptype.Coord(lat=-26.96528, lon=133.96233)), from_dt=dateutil.parser.parse("2014-10-12T00:55:54"), center_dt=dateutil.parser.parse("2014-10-12T00:56:06"), to_dt=dateutil.parser.parse("2014-10-12T00:56:18"), ), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='sample', resolution=25.0, size=9161), ptype.DimensionMetadata(name='line', resolution=25.0, size=9161) ], projection=ptype.ProjectionMetadata( centre_point=ptype.Point(511512.500000, 7127487.500000), geo_ref_points=ptype.PointPolygon( ul=ptype.Point(397012.5, 7237987.5), ur=ptype.Point(626012.5, 7237987.5), ll=ptype.Point(397012.5, 7016987.5), lr=ptype.Point(626012.5, 7016987.5)), datum='GDA94', ellipsoid='GRS80', point_in_pixel='UL', map_projection='UTM', resampling_option='CUBIC_CONVOLUTION', zone=-53)), browse={ 'medium': ptype.BrowseMetadata(path=Path( 'product/LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012.jpg' ), file_type='image/jpg', cell_size=219.75, red_band=7, green_band=5, blue_band=1), 'full': ptype.BrowseMetadata(path=Path( 'LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012_FR.jpg'), file_type='image/jpg', cell_size=25.0, red_band=7, green_band=5, blue_band=1) }, image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(101, 78), cloud_cover_percentage=0, cloud_cover_details=None, sun_elevation=58.00268508, sun_azimuth=59.41814014, ground_control_points_model=420, geometric_rmse_model=4.610, geometric_rmse_model_x=3.527, geometric_rmse_model_y=2.968, # TODO: What are these two? viewing_incidence_angle_long_track=0, viewing_incidence_angle_x_track=0, bands={ 'coastal_aerosol': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B1.TIF'), number=1, type_='reflective', cell_size=25.0, ), 'visible_blue': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B2.TIF'), number=2, type_='reflective', cell_size=25.0, ), 'visible_green': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B3.TIF'), number=3, type_='reflective', cell_size=25.0, ), 'visible_red': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B4.TIF'), number=4, type_='reflective', cell_size=25.0, ), 'near_infrared': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B5.TIF'), number=5, type_='reflective', cell_size=25.0, ), 'short_wave_infrared1': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B6.TIF'), number=6, type_='reflective', cell_size=25.0, ), 'short_wave_infrared2': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B7.TIF'), number=7, type_='reflective', cell_size=25.0, ), 'panchromatic': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B8.TIF'), number=8, type_='panchromatic', cell_size=12.50, shape=ptype.Point(17761, 18241), ), 'cirrus': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B9.TIF'), number=9, type_='atmosphere', ), 'thermal_infrared1': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B10.TIF'), number=10, type_='thermal', cell_size=25.0, shape=ptype.Point(8881, 9121), ), 'thermal_infrared2': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B11.TIF'), number=11, type_='thermal', cell_size=25.0, shape=ptype.Point(8881, 9121), ), 'quality': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_BQA.TIF'), number='QA', type_='quality', ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata( name='Pinkmatter Landsat Processor', version='3.3.3104', parameters={ 'resampling': 'CC', 'radiometric_correction': 'CPF', 'orientation': 'NUP', 'hemisphere': 'S', }), machine=ptype.MachineMetadata( hostname='rhe-jm-prod08.prod.lan', type_id='jobmanager', uname= 'Linux rhe-jm-dev08.dev.lan 2.6.32-279.22.1.el6.x86_64 #1 SMP Sun Oct ' '12 ' '09:21:40 EST 2014 x86_64 x86_64 x86_64 GNU/Linux'), ancillary={ 'cpf': ptype.AncillaryMetadata( name='L8CPF20141001_20141231.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile' '/L8CPF20141001_20141231.01'), 'bpf_tirs': ptype.AncillaryMetadata( name='LT8BPF20141012002432_20141012020301.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10' '/LT8BPF20141012002432_20141012020301.01'), 'bpf_oli': ptype.AncillaryMetadata( name='LO8BPF20141012002825_20141012011100.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10' '/LT8BPF20141012002432_20141012020301.01'), 'rlut': ptype.AncillaryMetadata(name='L8RLUT20130211_20431231v09.h5') }, source_datasets={'satellite_telemetry_data': _build_ls8_raw()}))
def _build_ls8_nbar(): _reset_runtime_id() nbar = ptype.DatasetMetadata( id_=uuid.UUID("249ae098-bd88-11e4-beaa-1040f381a756"), size_bytes=622208 * 1024, ga_label='LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012', product_type='GANBAR01', platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS', type_="Multi-Spectral", operation_mode='PUSH-BROOM'), # acquisition=ptype.AcquisitionMetadata(), format_=ptype.FormatMetadata(name='GeoTiff', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969), ur=ptype.Coord(lat=-24.96826, lon=136.24838), lr=ptype.Coord(lat=-26.96338, lon=136.26962), ll=ptype.Coord(lat=-26.96528, lon=133.96233)), from_dt=dateutil.parser.parse("2014-10-12T00:55:54"), to_dt=dateutil.parser.parse("2014-10-12T00:56:18"), ), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='sample', resolution=25.0, size=9161), ptype.DimensionMetadata(name='line', resolution=25.0, size=9161) ], projection=ptype.ProjectionMetadata( centre_point=ptype.Point(511512.500000, 7127487.500000), geo_ref_points=ptype.PointPolygon( ul=ptype.Point(397012.5, 7237987.5), ur=ptype.Point(626012.5, 7237987.5), ll=ptype.Point(397012.5, 7016987.5), lr=ptype.Point(626012.5, 7016987.5)), datum='GDA94', ellipsoid='GRS80', point_in_pixel='UL', map_projection='UTM', resampling_option='CUBIC_CONVOLUTION', zone=-53)), browse={ 'medium': ptype.BrowseMetadata(path=Path( 'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.tif'), file_type='image/jpg', cell_size=219.75, red_band=7, green_band=5, blue_band=2), 'full': ptype.BrowseMetadata(path=Path( 'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_FR.tif'), file_type='image/jpg', cell_size=25.0, red_band=7, green_band=5, blue_band=2) }, image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(101, 78), cloud_cover_percentage=0.01, cloud_cover_details=None, # TODO: What are these two? viewing_incidence_angle_long_track=0, viewing_incidence_angle_x_track=0, bands={ '1': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B1.tif' ), ), '2': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B2.tif' ), ), '3': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B3.tif' ), ), '4': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B4.tif' ), ), '5': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B5.tif' ), ), '6': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B6.tif' ), ), '7': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B7.tif' ), ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata(name='GANBAR', version='3.2.1', parameters={}), machine=ptype.MachineMetadata(), source_datasets={'level1': _build_ls8_ortho()}, ancillary={})) return nbar
def _build_ls7_wofs(): return ptype.DatasetMetadata( ga_label='LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897', product_type='GAWATER', size_bytes=616 * 1024, platform=ptype.PlatformMetadata(code='LS7'), instrument=ptype.InstrumentMetadata(name='ETM', type_='Multi-Spectral'), format_=ptype.FormatMetadata('GeoTIFF', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(140.0000000, -26.0000000), ll=ptype.Coord(140.0000000, -27.0000000), ur=ptype.Coord(141.0000000, -26.0000000), lr=ptype.Coord(141.0000000, -27.0000000)), # TODO: Should we store the center coordinate? from_dt=dateutil.parser.parse('2013-07-24 00:32:27.952897'), to_dt=dateutil.parser.parse('2013-07-24 00:33:15.899670')), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='x', resolution=27.1030749476, size=4000), ptype.DimensionMetadata(name='y', resolution=27.1030749476, size=4000) ], # TODO: Should WOfS have tile coordinates here? # georectified=ptype.GeoRectifiedSpacialMetadata( # geo_ref_points=PointPolygon( # ul=ptype.Point(255012.500, 7229987.500), # ur=ptype.Point(497012.500, 7229987.500), # ll=ptype.Point(255012.500, 7019987.500), # lr=ptype.Point(497012.500, 7229987.500) # ), # checkpoint_availability=0, # datum='GDA94', # ellipsoid='GRS80', # point_in_pixel='UL', # projection='UTM', # zone=-54 # ) ), image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(98, 78), satellite_ref_point_end=ptype.Point(98, 79), cloud_cover_percentage=0.76494375, cloud_cover_details='122391 count', sun_elevation=33.0061002772, sun_azimuth=38.2433049177, bands={ 'W': ptype.BandMetadata(path=Path( 'LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897.tif'), # TODO: Nodata value? ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata(name='WOfS', version='1.3', parameters={}), machine=ptype.MachineMetadata(), source_datasets={ # TODO: LS7 dataset? }))