def _build_ls8_raw():
_reset_runtime_id()
raw = ptype.DatasetMetadata(
id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'),
creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"),
size_bytes=5680940 * 1024,
ga_label=
'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1',
product_type='satellite_telemetry_data',
usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'),
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='MD'),
acquisition=ptype.AcquisitionMetadata(
aos=dateutil.parser.parse('2014-10-12T00:52:52'),
los=dateutil.parser.parse('2014-10-12T00:58:37'),
groundstation=ptype.GroundstationMetadata(
code='ASA',
antenna_coord=ptype.Coord(lat=-23.759,
lon=133.8824,
height=579.312)),
heading='D',
platform_orbit=8846),
extent=None,
grid_spatial=None,
browse=None,
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(
101, 70),
satellite_ref_point_end=ptype.Point(101,
83)),
lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata()))
return raw
def _extract_mdf_id_fields(base_md, mdf_usgs_id):
"""
V I N ppp RRR rrr YYYY ddd GSI vv
:type base_md: ptype.DatasetMetadata
:type mdf_usgs_id: str
:rtype: ptype.DatasetMetadata
"""
m = re.search((r"(?P<vehicle>L)"
r"(?P<instrument>[OTC])"
r"(?P<vehicle_number>\d)"
r"(?P<path>\d{3})"
r"(?P<row_start>\d{3})"
r"(?P<row_end>\d{3})"
r"(?P<acq_date>\d{7})"
r"(?P<gsi>\w{3})"
r"(?P<version>\d{2})"), mdf_usgs_id)
fields = m.groupdict()
if not base_md.usgs:
base_md.usgs = ptype.UsgsMetadata()
base_md.usgs.interval_id = mdf_usgs_id
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
base_md.platform.code = "LANDSAT_" + fields["vehicle_number"]
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.instrument.name = LS8_SENSORS[fields["instrument"]]
path = int(fields["path"])
if not base_md.image:
base_md.image = ptype.ImageMetadata()
base_md.image.satellite_ref_point_start = ptype.Point(
path, int(fields["row_start"]))
base_md.image.satellite_ref_point_end = ptype.Point(
path, int(fields["row_end"]))
# base_md.version = int(fields["version"]) or base_md.version
# Probably less than useful without time.
# if not base_md.extent:
# base_md.extent = ptype.ExtentMetadata()
#
# base_md.extent.center_dt = base_md.extent.center_dt or datetime.strptime(fields["acq_date"], "%Y%j").date()
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=fields["gsi"])
return base_md
def _extract_rcc_filename_fields(base_md, filename):
"""
Landsat 5 and 7 RCC format specifications:
http://landsat.usgs.gov/documents/LS_DFCB_01.pdf
http://landsat.usgs.gov/documents/LS_DFCB_06.pdf
:type base_md: ptype.DatasetMetadata
:type filename: str
:rtype: ptype.DatasetMetadata
"""
m = re.search(
r'(?P<satsens>\w{4})(?P<date>\d{13})(?P<gsi>[^\d]+).*?(?P<version>\d\d)?\.data',
filename)
fields = m.groupdict()
if not base_md.platform or not base_md.platform.code:
# TODO: Do we have a cleaner way to do this? A list of mappings?
satsens_ = fields['satsens']
vehicle = satsens_[0]
vehicle_num = satsens_[1]
instrument_short = satsens_[2]
smode_short = satsens_[3]
platform_code, instrument_name, operation_mode = _expand_platform_info(
vehicle, vehicle_num, instrument_short, smode_short)
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.platform.code = platform_code
base_md.instrument.name = instrument_name
base_md.instrument.operation_mode = operation_mode
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
if not base_md.acquisition.aos:
base_md.acquisition.aos = datetime.datetime.strptime(
fields['date'], "%Y%j%H%M%S")
base_md.gsi = fields['gsi']
if not base_md.acquisition.groundstation:
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=fields['gsi'])
if not base_md.usgs:
base_md.usgs = ptype.UsgsMetadata()
base_md.usgs.interval_id = _usgs_id_from_filename(filename)
# RCC is raw: P00
base_md.ga_level = 'P00'
version = int(fields['version']) if fields.get('version') else None
base_md.format_ = ptype.FormatMetadata(name='RCC', version=version)
return base_md
def _populate_from_mtl_dict(md, mtl_, folder):
"""
:param mtl_: Parsed mtl file
:param folder: Folder containing imagery (and mtl). For fixing relative paths in the MTL.
:type md: eodatasets.type.DatasetMetadata
:type mtl_: dict of (str, obj)
:rtype: eodatasets.type.DatasetMetadata
"""
if not md.usgs:
md.usgs = ptype.UsgsMetadata()
md.usgs.scene_id = _get(mtl_, 'METADATA_FILE_INFO', 'landsat_scene_id')
md.creation_dt = _get(mtl_, 'METADATA_FILE_INFO', 'file_date')
# TODO: elsewhere we've used 'GAORTHO01' etc. Here it's 'L1T' etc.
md.product_level = _translate_to_old_usgs_code(
_get(mtl_, 'PRODUCT_METADATA', 'data_type'))
# md.size_bytes=None,
satellite_id = _get(mtl_, 'PRODUCT_METADATA', 'spacecraft_id')
if not md.platform:
md.platform = ptype.PlatformMetadata()
md.platform.code = satellite_id
md.format_ = ptype.FormatMetadata(
name=_get(mtl_, 'PRODUCT_METADATA', 'output_format'))
product_md = _get(mtl_, 'PRODUCT_METADATA')
sensor_id = _get(mtl_, 'PRODUCT_METADATA', 'sensor_id')
if not md.instrument:
md.instrument = ptype.InstrumentMetadata()
md.instrument.name = sensor_id
# md.instrument.type_
md.instrument.operation_mode = _get(product_md, 'sensor_mode')
if not md.acquisition:
md.acquisition = ptype.AcquisitionMetadata()
md.acquisition.groundstation = ptype.GroundstationMetadata(
code=_get(mtl_, "METADATA_FILE_INFO", "station_id"))
# md.acquisition.groundstation.antenna_coord
# aos, los, groundstation, heading, platform_orbit
_populate_extent(md, product_md)
_populate_grid_spatial(md, mtl_)
_populate_image(md, mtl_)
_populate_lineage(md, mtl_)
return md
def test_parse_variations(self):
new_examples = {
'L7EB2007303000923ASA222Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2007303000923ASA222'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2007, 10, 30, 0, 9, 23),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7EB2015118010116ASA213Q00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2015118010116ASA213'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2015, 4, 28, 1, 1, 16),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7EB2011239021036ASA111Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2011239021036ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2011, 8, 27, 2, 10, 36),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2005120001242ASA111I.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2005120001242ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 4, 30, 0, 12, 42),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TT1995117002206ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TT1995117002206ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(1995, 4, 27, 0, 22, 6),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TT1990118013106ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TT1990118013106ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(1990, 4, 28, 1, 31, 6),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7ET2005302020634ASA123Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7ET2005302020634ASA123'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 10, 29, 2, 6, 34),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2011299000126ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2011299000126ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2011, 10, 26, 0, 1, 26),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2010119010045ASA214I.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2010119010045ASA214'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2010, 4, 29, 1, 0, 45),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7ET2000296234136ASA111Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7ET2000296234136ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2000, 10, 22, 23, 41, 36),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
}
for file_name, expected_output in new_examples.items():
output = rccfile._extract_rcc_filename_fields(
ptype.DatasetMetadata(), file_name)
# The ids will be different — clear them before comparison.
output.id_ = None
expected_output.id_ = None
self.assert_same(expected_output, output)
code='LANDSAT_7'
),
instrument=ptype.InstrumentMetadata(
name='ETM',
operation_mode='SAM'
),
format_=ptype.FormatMetadata(
name='GeoTIFF'
),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata(
code='ASA'
)
),
usgs=ptype.UsgsMetadata(
scene_id='LE71140732005007ASA00'
),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(
ul=ptype.Coord(
lat=-17.82157,
lon=115.58472
),
ur=ptype.Coord(
lat=-17.82497,
lon=117.82111
),
ll=ptype.Coord(
lat=-19.72798,
lon=115.56872
),
ga_level='P00',
product_type='satellite_telemetry_data',
creation_dt=datetime.datetime(2015, 4, 22, 0, 7, 48),
size_bytes=5871413760,
checksum_path=Path('package.sha1'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM', operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 6, 1, 1, 51, 10),
los=datetime.datetime(2005, 6, 1, 2, 0, 25),
groundstation=ptype.GroundstationMetadata(code='ASA'),
platform_orbit=113025
),
usgs=ptype.UsgsMetadata(
interval_id='L5TB2005152015110ASA111'
),
lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata(
hostname='niggle.local',
runtime_id=UUID('b2af5545-e894-11e4-b3b0-1040f381a756'),
type_id='jobmanager',
version='2.4.0',
uname='Darwin niggle.local 14.3.0 Darwin Kernel Version 14.3.0: Mon Mar 23 11:59:05 PDT 2015; '
'root:xnu-2782.20.48~5/RELEASE_X86_64 x86_64'
),
source_datasets={}
)
)
_LS7_RAW = ptype.DatasetMetadata(
import eodatasets.type as ptype
from tests.metadata.mtl import assert_expected_mtl
FILENAME = 'ls8_mtl.txt'
EXPECTED_OUT = ptype.DatasetMetadata(
id_=uuid.UUID('3ff71eb0-d5c5-11e4-aebb-1040f381a756'),
product_level='L1T',
creation_dt=datetime.datetime(2014, 11, 12, 15, 8, 35),
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTIFF'),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata(code='LGN')),
usgs=ptype.UsgsMetadata(scene_id='LC81010782014285LGN00'),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.98805, lon=133.97954),
ur=ptype.Coord(lat=-24.9864, lon=136.23866),
ll=ptype.Coord(lat=-26.99236, lon=133.96208),
lr=ptype.Coord(lat=-26.99055, lon=136.25985)),
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)),
grid_spatial=ptype.GridSpatialMetadata(projection=ptype.ProjectionMetadata(
geo_ref_points=ptype.
PointPolygon(ul=ptype.Point(x=397012.5, y=7235987.5),
ur=ptype.Point(x=625012.5, y=7235987.5),
ll=ptype.Point(x=397012.5, y=7013987.5),
lr=ptype.Point(x=625012.5, y=7013987.5)),
datum='GDA94',
ellipsoid='GRS80',
map_projection='UTM',