def _build_ls8_raw():
_reset_runtime_id()
raw = ptype.DatasetMetadata(
id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'),
creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"),
size_bytes=5680940 * 1024,
ga_label=
'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1',
product_type='satellite_telemetry_data',
usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'),
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='MD'),
acquisition=ptype.AcquisitionMetadata(
aos=dateutil.parser.parse('2014-10-12T00:52:52'),
los=dateutil.parser.parse('2014-10-12T00:58:37'),
groundstation=ptype.GroundstationMetadata(
code='ASA',
antenna_coord=ptype.Coord(lat=-23.759,
lon=133.8824,
height=579.312)),
heading='D',
platform_orbit=8846),
extent=None,
grid_spatial=None,
browse=None,
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(
101, 70),
satellite_ref_point_end=ptype.Point(101,
83)),
lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata()))
return raw
def _populate_grid_spatial(md, mtl_):
product_md = _get(mtl_, 'PRODUCT_METADATA')
# We don't have a single set of dimensions. Depends on the band?
# md.grid_spatial.dimensions = []
if not md.grid_spatial:
md.grid_spatial = ptype.GridSpatialMetadata()
if not md.grid_spatial.projection:
md.grid_spatial.projection = ptype.ProjectionMetadata()
md.grid_spatial.projection.geo_ref_points = ptype.PointPolygon(
ul=ptype.Point(x=_get(product_md, 'corner_ul_projection_x_product'),
y=_get(product_md, 'corner_ul_projection_y_product')),
ur=ptype.Point(x=_get(product_md, 'corner_ur_projection_x_product'),
y=_get(product_md, 'corner_ur_projection_y_product')),
ll=ptype.Point(x=_get(product_md, 'corner_ll_projection_x_product'),
y=_get(product_md, 'corner_ll_projection_y_product')),
lr=ptype.Point(x=_get(product_md, 'corner_lr_projection_x_product'),
y=_get(product_md, 'corner_lr_projection_y_product')))
# centre_point=None,
projection_md = _get(mtl_, 'PROJECTION_PARAMETERS')
md.grid_spatial.projection.datum = _get(projection_md, 'datum')
md.grid_spatial.projection.ellipsoid = _get(projection_md, 'ellipsoid')
# Where does this come from? 'ul' etc.
# point_in_pixel=None,
md.grid_spatial.projection.map_projection = _get(projection_md,
'map_projection')
md.grid_spatial.projection.resampling_option = _get(
projection_md, 'resampling_option')
md.grid_spatial.projection.datum = _get(projection_md, 'datum')
md.grid_spatial.projection.ellipsoid = _get(projection_md, 'ellipsoid')
md.grid_spatial.projection.zone = _get(projection_md, 'utm_zone')
md.grid_spatial.projection.orientation = _get(projection_md, 'orientation')
def _extract_mdf_id_fields(base_md, mdf_usgs_id):
"""
V I N ppp RRR rrr YYYY ddd GSI vv
:type base_md: ptype.DatasetMetadata
:type mdf_usgs_id: str
:rtype: ptype.DatasetMetadata
"""
m = re.search((r"(?P<vehicle>L)"
r"(?P<instrument>[OTC])"
r"(?P<vehicle_number>\d)"
r"(?P<path>\d{3})"
r"(?P<row_start>\d{3})"
r"(?P<row_end>\d{3})"
r"(?P<acq_date>\d{7})"
r"(?P<gsi>\w{3})"
r"(?P<version>\d{2})"), mdf_usgs_id)
fields = m.groupdict()
if not base_md.usgs:
base_md.usgs = ptype.UsgsMetadata()
base_md.usgs.interval_id = mdf_usgs_id
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
base_md.platform.code = "LANDSAT_" + fields["vehicle_number"]
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.instrument.name = LS8_SENSORS[fields["instrument"]]
path = int(fields["path"])
if not base_md.image:
base_md.image = ptype.ImageMetadata()
base_md.image.satellite_ref_point_start = ptype.Point(
path, int(fields["row_start"]))
base_md.image.satellite_ref_point_end = ptype.Point(
path, int(fields["row_end"]))
# base_md.version = int(fields["version"]) or base_md.version
# Probably less than useful without time.
# if not base_md.extent:
# base_md.extent = ptype.ExtentMetadata()
#
# base_md.extent.center_dt = base_md.extent.center_dt or datetime.strptime(fields["acq_date"], "%Y%j").date()
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=fields["gsi"])
return base_md
def populate_from_image_metadata(md):
"""
Populate by extracting metadata from existing band files.
:type md: eodatasets.type.DatasetMetadata
:rtype: eodatasets.type.DatasetMetadata
"""
for band_id, band in md.image.bands.items():
i = gdal.Open(str(band.path))
if not i:
# TODO: log? throw?
continue
spacial_ref = osr.SpatialReference(i.GetProjectionRef())
# Extract actual image coords
# md.grid_spatial.projection.
band.shape = ptype.Point(i.RasterXSize, i.RasterYSize)
band.cell_size = ptype.Point(abs(i.GetGeoTransform()[1]),
abs(i.GetGeoTransform()[5]))
# TODO separately: create standardised WGS84 coords. for md.extent
# wkt_contents = spacial_ref.ExportToPrettyWkt()
# TODO: if srs IsGeographic()? Otherwise srs IsProjected()?
if not md.grid_spatial:
md.grid_spatial = ptype.GridSpatialMetadata()
if not md.grid_spatial.projection:
md.grid_spatial.projection = ptype.ProjectionMetadata()
md.grid_spatial.projection.geo_ref_points = _get_gdal_image_coords(i)
md.grid_spatial.projection.unit = spacial_ref.GetLinearUnitsName()
md.grid_spatial.projection.zone = spacial_ref.GetUTMZone()
# ?
md.grid_spatial.projection.datum = 'GDA94'
md.grid_spatial.projection.ellipsoid = 'GRS80'
# TODO: DATUM/Reference system etc.
if not md.extent:
md.extent = ptype.ExtentMetadata()
md.extent.coord = reproject_coords(
md.grid_spatial.projection.geo_ref_points, spacial_ref)
# Get positional info, projection etc.
# Is projection/etc same as previous?
# -- If all match, set on wider image.
i = None
return md
def test_eods_fill_metadata(self):
dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012"
bandname = '10'
bandfile = dataset_folder + '_B' + bandname + '.tif'
input_folder = write_files({
dataset_folder: {
'metadata.xml': """<EODS_DATASET>
<ACQUISITIONINFORMATION>
<EVENT>
<AOS>20141012T03:23:36</AOS>
<LOS>20141012T03:29:10</LOS>
</EVENT>
</ACQUISITIONINFORMATION>
<EXEXTENT>
<TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM>
<TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO>
</EXEXTENT>
</EODS_DATASET>""",
'scene01': {
bandfile: ''
}
}
})
expected = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_,
ga_label=dataset_folder,
ga_level='P54',
product_type='EODS_NBAR',
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTiff'),
acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36),
los=datetime.datetime(2014, 10, 12, 3, 29, 10),
groundstation=ptype.GroundstationMetadata(code='LGS')),
extent=ptype.ExtentMetadata(
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6),
from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54),
to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18)
),
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78),
satellite_ref_point_end=ptype.Point(x=101, y=78),
bands={bandname: ptype.BandMetadata(number=bandname,
path=Path(input_folder, dataset_folder,
'scene01', bandfile))})
)
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_
)
received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder))
self.assert_same(expected, received)
def test_directory_parse(self):
dataset_id = 'LC80880750762013254ASA00'
metadata = mdf._extract_mdf_id_fields(ptype.DatasetMetadata(),
dataset_id)
self.assertEqual(metadata.usgs.interval_id, dataset_id)
self.assertEqual(metadata.platform.code, 'LANDSAT_8')
self.assertEqual(metadata.instrument.name, 'OLI_TIRS')
self.assertEqual(metadata.image.satellite_ref_point_start,
ptype.Point(88, 75))
self.assertEqual(metadata.image.satellite_ref_point_end,
ptype.Point(88, 76))
self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
def test_create_typical_browse_metadata():
class TestDriver(drivers.DatasetDriver):
def browse_image_bands(self, d):
return '5', '1', '3'
d = write_files({})
dataset = browseimage.create_typical_browse_metadata(
TestDriver(), ptype.DatasetMetadata(), d)
expected = ptype.DatasetMetadata(
browse={
'full':
ptype.BrowseMetadata(path=d.joinpath('browse.fr.jpg'),
file_type='image/jpg',
red_band='5',
green_band='1',
blue_band='3'),
'medium':
ptype.BrowseMetadata(
path=d.joinpath('browse.jpg'),
# Default medium size.
shape=ptype.Point(1024, None),
file_type='image/jpg',
red_band='5',
green_band='1',
blue_band='3')
})
expected.id_, dataset.id_ = None, None
assert_same(expected, dataset)
def _test_mdf_output(metadata):
self.assertEqual(metadata.usgs.interval_id,
'LC80880750762013254ASA00')
self.assertEqual(metadata.platform.code, 'LANDSAT_8')
self.assertEqual(metadata.instrument.name, 'OLI_TIRS')
self.assertEqual(metadata.format_.name, 'MD')
self.assertEqual(metadata.ga_level, 'P00')
self.assertEqual(metadata.image.satellite_ref_point_start,
ptype.Point(88, 75))
self.assertEqual(metadata.image.satellite_ref_point_end,
ptype.Point(88, 76))
self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
# No longer bother with vague center date.
# self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11))
self.assertEqual(
metadata.acquisition.aos,
datetime.datetime(2013, 9, 11, 23, 36, 11, 482000))
self.assertEqual(
metadata.acquisition.los,
datetime.datetime(2013, 9, 11, 23, 37, 14, 881000))
def create_dataset_browse_images(
dataset_driver,
dataset,
target_directory,
after_file_creation=lambda file_path: None):
"""
:type dataset_driver: drivers.DatasetDriver
:type dataset: ptype.DatasetMetadata
:type target_directory: Path
:type after_file_creation: Path -> None
:rtype: ptype.DatasetMetadata
"""
if not dataset.image or not dataset.image.bands:
# A dataset without defined bands doesn't get a browse image (eg. raw file)
_LOG.info('No bands defined. Skipping browse image.')
return dataset
# Create browse image metadata if missing.
if not dataset.browse:
create_typical_browse_metadata(dataset_driver, dataset, target_directory)
# Create browse images based on the metadata.
for browse_id, browse_metadata in dataset.browse.items():
x_constraint = None
if browse_metadata.shape:
x_constraint = browse_metadata.shape.x
bands = dataset.image.bands
necessary_bands = (browse_metadata.red_band, browse_metadata.green_band, browse_metadata.blue_band)
if not all([bands.get(band) for band in necessary_bands]):
raise ValueError(
'Some browse bands missing. Requires {!r}, has {!r}'
''.format(necessary_bands, bands.keys())
)
r_path, g_path, b_path = [bands[p].path for p in necessary_bands]
cols, rows, output_res = _create_thumbnail(
r_path,
g_path,
b_path,
browse_metadata.path,
x_constraint=x_constraint
)
# Update with the exact shape information.
browse_metadata.shape = ptype.Point(cols, rows)
browse_metadata.cell_size = output_res
after_file_creation(browse_metadata.path)
return dataset
def _populate_image(md, mtl_):
product_md = _get(mtl_, 'PRODUCT_METADATA')
image_md = _get(mtl_, 'IMAGE_ATTRIBUTES')
if not md.image:
md.image = ptype.ImageMetadata()
md.image.satellite_ref_point_start = ptype.Point(
_get(product_md, 'wrs_path'), _get(product_md, 'wrs_row'))
md.image.cloud_cover_percentage = _get(image_md, 'cloud_cover')
md.image.sun_elevation = _get(image_md, 'sun_elevation')
md.image.sun_azimuth = _get(image_md, 'sun_azimuth')
md.image.sun_earth_distance = _get(image_md, 'earth_sun_distance')
md.image.ground_control_points_model = _get(image_md,
'ground_control_points_model')
# md.image. = _get(image_md, 'earth_sun_distance')
md.image.geometric_rmse_model = _get(image_md, 'geometric_rmse_model')
md.image.geometric_rmse_model_y = _get(image_md, 'geometric_rmse_model_y')
md.image.geometric_rmse_model_x = _get(image_md, 'geometric_rmse_model_x')
if not md.image.bands:
md.image.bands = {}
def create_typical_browse_metadata(dataset_driver, dataset,
destination_directory):
"""
Create browse metadata.
:type dataset_driver: eodatasets.package.DatasetDriver
:type dataset: ptype.DatasetMetadata
:type destination_directory: Path
:return:
"""
rgb_bands = dataset_driver.browse_image_bands(dataset)
if len(rgb_bands) == 3:
r, g, b = rgb_bands
elif len(rgb_bands) == 1:
band = rgb_bands[0]
r, g, b = band, band, band
else:
raise ValueError('Unexpected number of bands (%s). Received %r' %
(len(rgb_bands), rgb_bands))
dataset.browse = {
'medium':
ptype.BrowseMetadata(
path=destination_directory.joinpath('browse.jpg'),
file_type='image/jpg',
# cell_size=output_res,
shape=ptype.Point(1024, None),
red_band=r,
green_band=g,
blue_band=b),
'full':
ptype.BrowseMetadata(
path=destination_directory.joinpath('browse.fr.jpg'),
file_type='image/jpg',
# cell_size=output_res,
red_band=r,
green_band=g,
blue_band=b)
}
return dataset
lat=-19.72798,
lon=115.56872
),
lr=ptype.Coord(
lat=-19.73177,
lon=117.83040
)
),
# TODO: python dt is one digit less precise than mtl (02:03:36.9270519Z). Does this matter?
center_dt=datetime.datetime(2005, 1, 7, 2, 3, 36, 927051)
),
grid_spatial=ptype.GridSpatialMetadata(
projection=ptype.ProjectionMetadata(
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(
x=350012.500,
y=8028987.500
),
ur=ptype.Point(
x=587012.500,
y=8028987.500
),
ll=ptype.Point(
x=350012.500,
y=7817987.500
),
lr=ptype.Point(
x=587012.500,
y=7817987.500
)
),
datum='GDA94',
def _get_point(gt, px, py):
x = gt[0] + (px * gt[1]) + (py * gt[2])
y = gt[3] + (px * gt[4]) + (py * gt[5])
return ptype.Point(x, y)
def _build_ls7_wofs():
return ptype.DatasetMetadata(
ga_label='LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897',
product_type='GAWATER',
size_bytes=616 * 1024,
platform=ptype.PlatformMetadata(code='LS7'),
instrument=ptype.InstrumentMetadata(name='ETM',
type_='Multi-Spectral'),
format_=ptype.FormatMetadata('GeoTIFF', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(140.0000000, -26.0000000),
ll=ptype.Coord(140.0000000, -27.0000000),
ur=ptype.Coord(141.0000000, -26.0000000),
lr=ptype.Coord(141.0000000, -27.0000000)),
# TODO: Should we store the center coordinate?
from_dt=dateutil.parser.parse('2013-07-24 00:32:27.952897'),
to_dt=dateutil.parser.parse('2013-07-24 00:33:15.899670')),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='x',
resolution=27.1030749476,
size=4000),
ptype.DimensionMetadata(name='y',
resolution=27.1030749476,
size=4000)
],
# TODO: Should WOfS have tile coordinates here?
# georectified=ptype.GeoRectifiedSpacialMetadata(
# geo_ref_points=PointPolygon(
# ul=ptype.Point(255012.500, 7229987.500),
# ur=ptype.Point(497012.500, 7229987.500),
# ll=ptype.Point(255012.500, 7019987.500),
# lr=ptype.Point(497012.500, 7229987.500)
# ),
# checkpoint_availability=0,
# datum='GDA94',
# ellipsoid='GRS80',
# point_in_pixel='UL',
# projection='UTM',
# zone=-54
# )
),
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(98, 78),
satellite_ref_point_end=ptype.Point(98, 79),
cloud_cover_percentage=0.76494375,
cloud_cover_details='122391 count',
sun_elevation=33.0061002772,
sun_azimuth=38.2433049177,
bands={
'W':
ptype.BandMetadata(path=Path(
'LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897.tif'),
# TODO: Nodata value?
)
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(name='WOfS',
version='1.3',
parameters={}),
machine=ptype.MachineMetadata(),
source_datasets={
# TODO: LS7 dataset?
}))
def _build_ls8_nbar():
_reset_runtime_id()
nbar = ptype.DatasetMetadata(
id_=uuid.UUID("249ae098-bd88-11e4-beaa-1040f381a756"),
size_bytes=622208 * 1024,
ga_label='LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012',
product_type='GANBAR01',
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
type_="Multi-Spectral",
operation_mode='PUSH-BROOM'),
# acquisition=ptype.AcquisitionMetadata(),
format_=ptype.FormatMetadata(name='GeoTiff', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
ur=ptype.Coord(lat=-24.96826,
lon=136.24838),
lr=ptype.Coord(lat=-26.96338,
lon=136.26962),
ll=ptype.Coord(lat=-26.96528,
lon=133.96233)),
from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='sample',
resolution=25.0,
size=9161),
ptype.DimensionMetadata(name='line',
resolution=25.0,
size=9161)
],
projection=ptype.ProjectionMetadata(
centre_point=ptype.Point(511512.500000, 7127487.500000),
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(397012.5, 7237987.5),
ur=ptype.Point(626012.5, 7237987.5),
ll=ptype.Point(397012.5, 7016987.5),
lr=ptype.Point(626012.5, 7016987.5)),
datum='GDA94',
ellipsoid='GRS80',
point_in_pixel='UL',
map_projection='UTM',
resampling_option='CUBIC_CONVOLUTION',
zone=-53)),
browse={
'medium':
ptype.BrowseMetadata(path=Path(
'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.tif'),
file_type='image/jpg',
cell_size=219.75,
red_band=7,
green_band=5,
blue_band=2),
'full':
ptype.BrowseMetadata(path=Path(
'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_FR.tif'),
file_type='image/jpg',
cell_size=25.0,
red_band=7,
green_band=5,
blue_band=2)
},
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(101, 78),
cloud_cover_percentage=0.01,
cloud_cover_details=None,
# TODO: What are these two?
viewing_incidence_angle_long_track=0,
viewing_incidence_angle_x_track=0,
bands={
'1':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B1.tif'
), ),
'2':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B2.tif'
), ),
'3':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B3.tif'
), ),
'4':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B4.tif'
), ),
'5':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B5.tif'
), ),
'6':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B6.tif'
), ),
'7':
ptype.BandMetadata(path=Path(
'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B7.tif'
), )
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(name='GANBAR',
version='3.2.1',
parameters={}),
machine=ptype.MachineMetadata(),
source_datasets={'level1': _build_ls8_ortho()},
ancillary={}))
return nbar
ga_level='P54',
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTIFF'),
acquisition=ptype.AcquisitionMetadata(groundstation=ptype.GroundstationMetadata(code='LGN')),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(
ul=ptype.Coord(lat=-24.98805, lon=133.97954),
ur=ptype.Coord(lat=-24.9864, lon=136.23866),
ll=ptype.Coord(lat=-26.99236, lon=133.96208),
lr=ptype.Coord(lat=-26.99055, lon=136.25985)
),
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)
),
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(x=101, y=78),
bands={}
),
lineage=ptype.LineageMetadata(
source_datasets={'level1': test_ls8.EXPECTED_OUT},
algorithm=ptype.AlgorithmMetadata(name='terrain', version='1.0'),
),
grid_spatial=ptype.GridSpatialMetadata(
projection=ptype.ProjectionMetadata(
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(
x=397012.5,
y=7235987.5
),
ur=ptype.Point(
x=625012.5,
def _build_ls8_ortho():
_reset_runtime_id()
return ptype.DatasetMetadata(
id_=uuid.UUID('17b92c16-51d3-11e4-909d-005056bb6972'),
ga_label='LS8_OLITIRS_OTH_P51_GALPGS01-002_101_078_20141012',
product_type='GAORTHO01',
creation_dt=dateutil.parser.parse('2014-10-12 05:46:20'),
size_bytes=2386550 * 1024,
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
type_="Multi-Spectral",
operation_mode='PUSH-BROOM'),
format_=ptype.FormatMetadata(name='GeoTiff', version=1),
extent=ptype.ExtentMetadata(
reference_system='WGS84',
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
ur=ptype.Coord(lat=-24.96826,
lon=136.24838),
lr=ptype.Coord(lat=-26.96338,
lon=136.26962),
ll=ptype.Coord(lat=-26.96528,
lon=133.96233)),
from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
center_dt=dateutil.parser.parse("2014-10-12T00:56:06"),
to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
),
grid_spatial=ptype.GridSpatialMetadata(
dimensions=[
ptype.DimensionMetadata(name='sample',
resolution=25.0,
size=9161),
ptype.DimensionMetadata(name='line',
resolution=25.0,
size=9161)
],
projection=ptype.ProjectionMetadata(
centre_point=ptype.Point(511512.500000, 7127487.500000),
geo_ref_points=ptype.PointPolygon(
ul=ptype.Point(397012.5, 7237987.5),
ur=ptype.Point(626012.5, 7237987.5),
ll=ptype.Point(397012.5, 7016987.5),
lr=ptype.Point(626012.5, 7016987.5)),
datum='GDA94',
ellipsoid='GRS80',
point_in_pixel='UL',
map_projection='UTM',
resampling_option='CUBIC_CONVOLUTION',
zone=-53)),
browse={
'medium':
ptype.BrowseMetadata(path=Path(
'product/LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012.jpg'
),
file_type='image/jpg',
cell_size=219.75,
red_band=7,
green_band=5,
blue_band=1),
'full':
ptype.BrowseMetadata(path=Path(
'LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012_FR.jpg'),
file_type='image/jpg',
cell_size=25.0,
red_band=7,
green_band=5,
blue_band=1)
},
image=ptype.ImageMetadata(
satellite_ref_point_start=ptype.Point(101, 78),
cloud_cover_percentage=0,
cloud_cover_details=None,
sun_elevation=58.00268508,
sun_azimuth=59.41814014,
ground_control_points_model=420,
geometric_rmse_model=4.610,
geometric_rmse_model_x=3.527,
geometric_rmse_model_y=2.968,
# TODO: What are these two?
viewing_incidence_angle_long_track=0,
viewing_incidence_angle_x_track=0,
bands={
'coastal_aerosol':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B1.TIF'),
number=1,
type_='reflective',
cell_size=25.0,
),
'visible_blue':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B2.TIF'),
number=2,
type_='reflective',
cell_size=25.0,
),
'visible_green':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B3.TIF'),
number=3,
type_='reflective',
cell_size=25.0,
),
'visible_red':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B4.TIF'),
number=4,
type_='reflective',
cell_size=25.0,
),
'near_infrared':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B5.TIF'),
number=5,
type_='reflective',
cell_size=25.0,
),
'short_wave_infrared1':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B6.TIF'),
number=6,
type_='reflective',
cell_size=25.0,
),
'short_wave_infrared2':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B7.TIF'),
number=7,
type_='reflective',
cell_size=25.0,
),
'panchromatic':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B8.TIF'),
number=8,
type_='panchromatic',
cell_size=12.50,
shape=ptype.Point(17761, 18241),
),
'cirrus':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B9.TIF'),
number=9,
type_='atmosphere',
),
'thermal_infrared1':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B10.TIF'),
number=10,
type_='thermal',
cell_size=25.0,
shape=ptype.Point(8881, 9121),
),
'thermal_infrared2':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_B11.TIF'),
number=11,
type_='thermal',
cell_size=25.0,
shape=ptype.Point(8881, 9121),
),
'quality':
ptype.BandMetadata(
path=Path('product/LC81010782014285LGN00_BQA.TIF'),
number='QA',
type_='quality',
)
}),
lineage=ptype.LineageMetadata(
algorithm=ptype.AlgorithmMetadata(
name='Pinkmatter Landsat Processor',
version='3.3.3104',
parameters={
'resampling': 'CC',
'radiometric_correction': 'CPF',
'orientation': 'NUP',
'hemisphere': 'S',
}),
machine=ptype.MachineMetadata(
hostname='rhe-jm-prod08.prod.lan',
type_id='jobmanager',
uname=
'Linux rhe-jm-dev08.dev.lan 2.6.32-279.22.1.el6.x86_64 #1 SMP Sun Oct '
'12 '
'09:21:40 EST 2014 x86_64 x86_64 x86_64 GNU/Linux'),
ancillary={
'cpf':
ptype.AncillaryMetadata(
name='L8CPF20141001_20141231.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile'
'/L8CPF20141001_20141231.01'),
'bpf_tirs':
ptype.AncillaryMetadata(
name='LT8BPF20141012002432_20141012020301.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
'/LT8BPF20141012002432_20141012020301.01'),
'bpf_oli':
ptype.AncillaryMetadata(
name='LO8BPF20141012002825_20141012011100.01',
uri=
'/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
'/LT8BPF20141012002432_20141012020301.01'),
'rlut':
ptype.AncillaryMetadata(name='L8RLUT20130211_20431231v09.h5')
},
source_datasets={'satellite_telemetry_data': _build_ls8_raw()}))
def fill_metadata(self, dataset, path, additional_files=()):
"""
:type additional_files: tuple[Path]
:type dataset: ptype.DatasetMetadata
:type path: Path
:rtype: ptype.DatasetMetadata
"""
fields = re.match((r"(?P<vehicle>LS[578])"
r"_(?P<instrument>OLI_TIRS|OLI|TIRS|TM|ETM)"
r"_(?P<type>NBAR|PQ|FC)"
r"_(?P<level>[^-_]*)"
r"_(?P<product>[^-_]*)"
r"-(?P<groundstation>[0-9]{3})"
r"_(?P<path>[0-9]{3})"
r"_(?P<row>[0-9]{3})"
r"_(?P<date>[12][0-9]{7})"
r"(_(?P<version>[0-9]+))?"
"$"), path.stem).groupdict()
dataset.product_type = "EODS_" + fields["type"]
dataset.ga_level = fields["level"]
dataset.ga_label = path.stem
dataset.format_ = ptype.FormatMetadata(name='GeoTiff')
if not dataset.platform:
dataset.platform = ptype.PlatformMetadata()
dataset.platform.code = "LANDSAT_" + fields["vehicle"][2]
if not dataset.instrument:
dataset.instrument = ptype.InstrumentMetadata()
dataset.instrument.name = fields["instrument"]
if not dataset.image:
dataset.image = ptype.ImageMetadata(bands={})
dataset.image.satellite_ref_point_start = ptype.Point(
int(fields["path"]), int(fields["row"]))
dataset.image.satellite_ref_point_end = ptype.Point(
int(fields["path"]), int(fields["row"]))
for image_path in path.joinpath("scene01").iterdir():
band = self.to_band(dataset, image_path)
if band:
dataset.image.bands[band.number] = band
md_image.populate_from_image_metadata(dataset)
if not dataset.acquisition:
dataset.acquisition = ptype.AcquisitionMetadata()
for _station in _GROUNDSTATION_LIST:
if _station["eods_domain_code"] == fields["groundstation"]:
dataset.acquisition.groundstation = ptype.GroundstationMetadata(
code=_station["code"])
break
if not dataset.extent:
dataset.extent = ptype.ExtentMetadata()
def els2date(els):
if not els:
return None
return parse(els[0].text)
doc = ElementTree.parse(str(path.joinpath('metadata.xml')))
start_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTFROM"))
end_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTTO"))
# check if the dates in the metadata file are at least as accurate as what we have
filename_time = datetime.datetime.strptime(fields["date"], "%Y%m%d")
time_diff = start_time - filename_time
# Is the EODS metadata extremely off?
if abs(time_diff).days != 0:
raise ValueError(
'EODS time information differs too much from source files: %s'
% time_diff)
dataset.acquisition.aos = els2date(
doc.findall("./ACQUISITIONINFORMATION/EVENT/AOS"))
dataset.acquisition.los = els2date(
doc.findall("./ACQUISITIONINFORMATION/EVENT/LOS"))
dataset.extent.center_dt = start_time + (end_time - start_time) / 2
dataset.extent.from_dt = start_time
dataset.extent.to_dt = end_time
return dataset
creation_dt=datetime.datetime(2014, 11, 12, 15, 8, 35),
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTIFF'),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata(code='LGN')),
usgs=ptype.UsgsMetadata(scene_id='LC81010782014285LGN00'),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.98805, lon=133.97954),
ur=ptype.Coord(lat=-24.9864, lon=136.23866),
ll=ptype.Coord(lat=-26.99236, lon=133.96208),
lr=ptype.Coord(lat=-26.99055, lon=136.25985)),
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)),
grid_spatial=ptype.GridSpatialMetadata(projection=ptype.ProjectionMetadata(
geo_ref_points=ptype.
PointPolygon(ul=ptype.Point(x=397012.5, y=7235987.5),
ur=ptype.Point(x=625012.5, y=7235987.5),
ll=ptype.Point(x=397012.5, y=7013987.5),
lr=ptype.Point(x=625012.5, y=7013987.5)),
datum='GDA94',
ellipsoid='GRS80',
map_projection='UTM',
orientation='NORTH_UP',
resampling_option='CUBIC_CONVOLUTION',
zone=-53)),
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101,
y=78),
cloud_cover_percentage=0.01,
sun_azimuth=59.57807899,
sun_elevation=57.89670734,
sun_earth_distance=0.998137,
