def test_prepare_metadata(self):
f = write_files({'first.txt': 'test', 'second.txt': 'test2'})
class FauxDriver(drivers.DatasetDriver):
def to_band(self, dataset, path):
numbers = {
'first': ptype.BandMetadata(path=path, number='1'),
'second': None
}
return numbers.get(path.stem)
def get_ga_label(self, dataset):
return 'DATASET_ID_1234'
def get_id(self):
return 'faux'
d = ptype.DatasetMetadata()
d = package.expand_driver_metadata(FauxDriver(), d, list(f.iterdir()))
self.assert_same(
d,
ptype.DatasetMetadata(
id_=d.id_,
ga_label='DATASET_ID_1234',
product_type='faux',
size_bytes=9,
image=ptype.ImageMetadata(
bands={
'1':
ptype.BandMetadata(path=f.joinpath('first.txt'),
number='1')
})))
def to_band(self, dataset, path):
"""
:type dataset: ptype.DatasetMetadata
:type path: pathlib.Path
:rtype: ptype.BandMetadata
>>> OrthoDriver().to_band(None, Path('/tmp/out/LT51030782005002ASA00_B3.TIF'))
BandMetadata(path=PosixPath('/tmp/out/LT51030782005002ASA00_B3.TIF'), number='3')
>>> OrthoDriver().to_band(None, Path('/tmp/out/LC81090852015088LGN00_B10.tif'))
BandMetadata(path=PosixPath('/tmp/out/LC81090852015088LGN00_B10.tif'), number='10')
>>> OrthoDriver().to_band(None, Path('/data/output/LE70900782007292ASA00_B6_VCID_2.TIF'))
BandMetadata(path=PosixPath('/data/output/LE70900782007292ASA00_B6_VCID_2.TIF'), number='6_vcid_2')
>>> # No bands for non-tiff files.
>>> OrthoDriver().to_band(None, Path('/tmp/out/LC81090852015088LGN00_MTL.txt'))
>>> OrthoDriver().to_band(None, Path('/tmp/out/passinfo'))
>>> # A DEM image -- not included as a band.
>>> OrthoDriver().to_band(None, Path('LT05_L1TP_108078_20060703_20170309_01_T1_DEM.TIF'))
"""
if path.suffix.lower() != '.tif':
return None
name = path.stem.lower()
# A DEM image -- the only tif without a 'B' prefix.
# We don't include it in the list of bands according to Lan-Wei, as it's not part of a typical USGS package.
if name.endswith('_dem'):
return None
# Images end in a band number (eg '_B12.tif'). Extract it.
position = name.rfind('_b')
if position == -1:
raise ValueError('Unexpected tif image in level1: %r' % path)
band_number = name[position + 2:]
return ptype.BandMetadata(path=path, number=band_number)
def to_band(self, dataset, path):
"""
:type dataset: ptype.DatasetMetadata
:type path: Path
:param path: The filename of the input file.
:rtype: ptype.BandMetadata
"""
if path.suffix != '.tif':
return None
return ptype.BandMetadata(path=path, number='pqa')
def test_eods_fill_metadata(self):
dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012"
bandname = '10'
bandfile = dataset_folder + '_B' + bandname + '.tif'
input_folder = write_files({
dataset_folder: {
'metadata.xml': """<EODS_DATASET>
<ACQUISITIONINFORMATION>
<EVENT>
<AOS>20141012T03:23:36</AOS>
<LOS>20141012T03:29:10</LOS>
</EVENT>
</ACQUISITIONINFORMATION>
<EXEXTENT>
<TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM>
<TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO>
</EXEXTENT>
</EODS_DATASET>""",
'scene01': {
bandfile: ''
}
}
})
expected = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_,
ga_label=dataset_folder,
ga_level='P54',
product_type='EODS_NBAR',
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTiff'),
acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36),
los=datetime.datetime(2014, 10, 12, 3, 29, 10),
groundstation=ptype.GroundstationMetadata(code='LGS')),
extent=ptype.ExtentMetadata(
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6),
from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54),
to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18)
),
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78),
satellite_ref_point_end=ptype.Point(x=101, y=78),
bands={bandname: ptype.BandMetadata(number=bandname,
path=Path(input_folder, dataset_folder,
'scene01', bandfile))})
)
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_
)
received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder))
self.assert_same(expected, received)
def to_band(self, dataset, path):
"""
:type dataset: ptype.DatasetMetadata
:type path: Path
:rtype: ptype.BandMetadata
>>> p = Path('/tmp/something/reflectance_terrain_3.tif')
>>> NbarDriver('terrain').to_band(None, p).number
'3'
>>> NbarDriver('terrain').to_band(None, p).path
PosixPath('/tmp/something/reflectance_terrain_3.tif')
>>> p = Path('/tmp/something/LS8_OLITIRS_NBAR_P54_GANBART01-002_112_079_20140126_B4.tif')
>>> NbarDriver('terrain').to_band(None, p).number
'4'
"""
return ptype.BandMetadata(path=path, number=_read_band_number(path))
def test_pqa_to_band(self):
input_folder = write_files({
'pqa.tif': '',
'process.log': '',
'passinfo': '',
})
# Creates a single band.
self.assertEqual(
ptype.BandMetadata(path=input_folder.joinpath('pqa.tif'), number='pqa'),
drivers.PqaDriver().to_band(None, input_folder.joinpath('pqa.tif'))
)
# Other files should not be bands.
self.assertIsNone(drivers.PqaDriver().to_band(None, input_folder.joinpath('process.log')))
self.assertIsNone(drivers.PqaDriver().to_band(None, input_folder.joinpath('passinfo')))
def _read_bands(mtl_, folder_path):
"""
:type mtl_: dict of (str, obj)
:type folder_path: pathlib.Path
>>> import pathlib
>>> _read_bands({'PRODUCT_METADATA': {
... 'file_name_band_9': "LC81010782014285LGN00_B9.TIF"}
... }, folder_path=pathlib.Path('product/'))
{'9': BandMetadata(path=PosixPath('product/LC81010782014285LGN00_B9.TIF'), number='9')}
"""
bs = _read_mtl_band_filenames(mtl_)
# TODO: shape, size, md5
return dict([(number,
ptype.BandMetadata(path=folder_path / filename,
number=number))
for (number, filename) in bs.items()])
def to_band(self, dataset, path):
"""
:type dataset: ptype.DatasetMetadata
:type final_path: pathlib.Path
:rtype: ptype.BandMetadata
"""
if path.suffix.lower() != '.tif':
return None
name = path.stem
# Images end in a band number (eg '_B12.tif'). Extract it.
position = name.rfind('_')
if position == -1:
raise ValueError('Unexpected tif image in eods: %r' % path)
if re.match(r"[Bb]\d+", name[position + 1:]):
band_number = name[position + 2:]
else:
band_number = name[position + 1:]
return ptype.BandMetadata(path=path, number=band_number)
def to_band(self, dataset, path):
numbers = {
'first': ptype.BandMetadata(path=path, number='1'),
'second': None
}
return numbers.get(path.stem)
def _build_ls8_ortho():
_reset_runtime_id()
return ptype.DatasetMetadata(
id_=uuid.UUID('17b92c16-51d3-11e4-909d-005056bb6972'),
ga_label='LS8_OLITIRS_OTH_P51_GALPGS01-002_101_078_20141012',
product_type='GAORTHO01',
creation_dt=dateutil.parser.parse('2014-10-12 05:46:20'),
size_bytes=2386550 * 1024,
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
type_="Multi-Spectral",
operation_mode='PUSH-BROOM'),
format_=ptype.FormatMetadata(name='GeoTiff', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
ur=ptype.Coord(lat=-24.96826,
lon=136.24838),
lr=ptype.Coord(lat=-26.96338,
lon=136.26962),
ll=ptype.Coord(lat=-26.96528,
lon=133.96233)),
from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
center_dt=dateutil.parser.parse("2014-10-12T00:56:06"),
to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='sample',
resolution=25.0,
size=9161),
ptype.DimensionMetadata(name='line',
resolution=25.0,
size=9161)
],
projection=ptype.ProjectionMetadata(
centre_point=ptype.Point(511512.500000, 7127487.500000),
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(397012.5, 7237987.5),
ur=ptype.Point(626012.5, 7237987.5),
ll=ptype.Point(397012.5, 7016987.5),
lr=ptype.Point(626012.5, 7016987.5)),
datum='GDA94',
ellipsoid='GRS80',
point_in_pixel='UL',
map_projection='UTM',
resampling_option='CUBIC_CONVOLUTION',
zone=-53)),
browse={
'medium':
ptype.BrowseMetadata(path=Path(
'product/LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012.jpg'
),
file_type='image/jpg',
cell_size=219.75,
red_band=7,
green_band=5,
blue_band=1),
'full':
ptype.BrowseMetadata(path=Path(
'LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012_FR.jpg'),
file_type='image/jpg',
cell_size=25.0,
red_band=7,
green_band=5,
blue_band=1)
},
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(101, 78),
cloud_cover_percentage=0,
cloud_cover_details=None,
sun_elevation=58.00268508,
sun_azimuth=59.41814014,
ground_control_points_model=420,
geometric_rmse_model=4.610,
geometric_rmse_model_x=3.527,
geometric_rmse_model_y=2.968,
# TODO: What are these two?
viewing_incidence_angle_long_track=0,
viewing_incidence_angle_x_track=0,
bands={
'coastal_aerosol':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B1.TIF'),
number=1,
type_='reflective',
cell_size=25.0,
),
'visible_blue':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B2.TIF'),
number=2,
type_='reflective',
cell_size=25.0,
),
'visible_green':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B3.TIF'),
number=3,
type_='reflective',
cell_size=25.0,
),
'visible_red':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B4.TIF'),
number=4,
type_='reflective',
cell_size=25.0,
),
'near_infrared':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B5.TIF'),
number=5,
type_='reflective',
cell_size=25.0,
),
'short_wave_infrared1':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B6.TIF'),
number=6,
type_='reflective',
cell_size=25.0,
),
'short_wave_infrared2':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B7.TIF'),
number=7,
type_='reflective',
cell_size=25.0,
),
'panchromatic':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B8.TIF'),
number=8,
type_='panchromatic',
cell_size=12.50,
shape=ptype.Point(17761, 18241),
),
'cirrus':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B9.TIF'),
number=9,
type_='atmosphere',
),
'thermal_infrared1':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B10.TIF'),
number=10,
type_='thermal',
cell_size=25.0,
shape=ptype.Point(8881, 9121),
),
'thermal_infrared2':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B11.TIF'),
number=11,
type_='thermal',
cell_size=25.0,
shape=ptype.Point(8881, 9121),
),
'quality':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_BQA.TIF'),
number='QA',
type_='quality',
)
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(
name='Pinkmatter Landsat Processor',
version='3.3.3104',
parameters={
'resampling': 'CC',
'radiometric_correction': 'CPF',
'orientation': 'NUP',
'hemisphere': 'S',
}),
machine=ptype.MachineMetadata(
hostname='rhe-jm-prod08.prod.lan',
type_id='jobmanager',
uname=
'Linux rhe-jm-dev08.dev.lan 2.6.32-279.22.1.el6.x86_64 #1 SMP Sun Oct '
'12 '
'09:21:40 EST 2014 x86_64 x86_64 x86_64 GNU/Linux'),
ancillary={
'cpf':
ptype.AncillaryMetadata(
name='L8CPF20141001_20141231.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile'
'/L8CPF20141001_20141231.01'),
'bpf_tirs':
ptype.AncillaryMetadata(
name='LT8BPF20141012002432_20141012020301.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
'/LT8BPF20141012002432_20141012020301.01'),
'bpf_oli':
ptype.AncillaryMetadata(
name='LO8BPF20141012002825_20141012011100.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
'/LT8BPF20141012002432_20141012020301.01'),
'rlut':
ptype.AncillaryMetadata(name='L8RLUT20130211_20431231v09.h5')
},
source_datasets={'satellite_telemetry_data': _build_ls8_raw()}))
def _build_ls8_nbar():
_reset_runtime_id()
nbar = ptype.DatasetMetadata(
id_=uuid.UUID("249ae098-bd88-11e4-beaa-1040f381a756"),
size_bytes=622208 * 1024,
ga_label='LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012',
product_type='GANBAR01',
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
type_="Multi-Spectral",
operation_mode='PUSH-BROOM'),
# acquisition=ptype.AcquisitionMetadata(),
format_=ptype.FormatMetadata(name='GeoTiff', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
ur=ptype.Coord(lat=-24.96826,
lon=136.24838),
lr=ptype.Coord(lat=-26.96338,
lon=136.26962),
ll=ptype.Coord(lat=-26.96528,
lon=133.96233)),
from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='sample',
resolution=25.0,
size=9161),
ptype.DimensionMetadata(name='line',
resolution=25.0,
size=9161)
],
projection=ptype.ProjectionMetadata(
centre_point=ptype.Point(511512.500000, 7127487.500000),
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(397012.5, 7237987.5),
ur=ptype.Point(626012.5, 7237987.5),
ll=ptype.Point(397012.5, 7016987.5),
lr=ptype.Point(626012.5, 7016987.5)),
datum='GDA94',
ellipsoid='GRS80',
point_in_pixel='UL',
map_projection='UTM',
resampling_option='CUBIC_CONVOLUTION',
zone=-53)),
browse={
'medium':
ptype.BrowseMetadata(path=Path(
'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.tif'),
file_type='image/jpg',
cell_size=219.75,
red_band=7,
green_band=5,
blue_band=2),
'full':
ptype.BrowseMetadata(path=Path(
'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_FR.tif'),
file_type='image/jpg',
cell_size=25.0,
red_band=7,
green_band=5,
blue_band=2)
},
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(101, 78),
cloud_cover_percentage=0.01,
cloud_cover_details=None,
# TODO: What are these two?
viewing_incidence_angle_long_track=0,
viewing_incidence_angle_x_track=0,
bands={
'1':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B1.tif'
), ),
'2':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B2.tif'
), ),
'3':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B3.tif'
), ),
'4':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B4.tif'
), ),
'5':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B5.tif'
), ),
'6':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B6.tif'
), ),
'7':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B7.tif'
), )
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(name='GANBAR',
version='3.2.1',
parameters={}),
machine=ptype.MachineMetadata(),
source_datasets={'level1': _build_ls8_ortho()},
ancillary={}))
return nbar
def _build_ls7_wofs():
return ptype.DatasetMetadata(
ga_label='LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897',
product_type='GAWATER',
size_bytes=616 * 1024,
platform=ptype.PlatformMetadata(code='LS7'),
instrument=ptype.InstrumentMetadata(name='ETM',
type_='Multi-Spectral'),
format_=ptype.FormatMetadata('GeoTIFF', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(140.0000000, -26.0000000),
ll=ptype.Coord(140.0000000, -27.0000000),
ur=ptype.Coord(141.0000000, -26.0000000),
lr=ptype.Coord(141.0000000, -27.0000000)),
# TODO: Should we store the center coordinate?
from_dt=dateutil.parser.parse('2013-07-24 00:32:27.952897'),
to_dt=dateutil.parser.parse('2013-07-24 00:33:15.899670')),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='x',
resolution=27.1030749476,
size=4000),
ptype.DimensionMetadata(name='y',
resolution=27.1030749476,
size=4000)
],
# TODO: Should WOfS have tile coordinates here?
# georectified=ptype.GeoRectifiedSpacialMetadata(
# geo_ref_points=PointPolygon(
# ul=ptype.Point(255012.500, 7229987.500),
# ur=ptype.Point(497012.500, 7229987.500),
# ll=ptype.Point(255012.500, 7019987.500),
# lr=ptype.Point(497012.500, 7229987.500)
# ),
# checkpoint_availability=0,
# datum='GDA94',
# ellipsoid='GRS80',
# point_in_pixel='UL',
# projection='UTM',
# zone=-54
# )
),
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(98, 78),
satellite_ref_point_end=ptype.Point(98, 79),
cloud_cover_percentage=0.76494375,
cloud_cover_details='122391 count',
sun_elevation=33.0061002772,
sun_azimuth=38.2433049177,
bands={
'W':
ptype.BandMetadata(path=Path(
'LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897.tif'),
# TODO: Nodata value?
)
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(name='WOfS',
version='1.3',
parameters={}),
machine=ptype.MachineMetadata(),
source_datasets={
# TODO: LS7 dataset?
}))
