def test_extract_groundstation(self):
d = write_files({'LANDSAT-7.ALICE': {'dataset': {}}})
d = d.joinpath('LANDSAT-7.ALICE')
md = adsfolder.extract_md(ptype.DatasetMetadata(), d)
self.assertIsNotNone(md.acquisition)
self.assertEqual(md.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA'))
d = d.joinpath('dataset')
md = adsfolder.extract_md(ptype.DatasetMetadata(), d)
self.assertIsNotNone(md.acquisition)
self.assertEqual(md.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA'))
def test_parse_from_driver(self):
d = write_files({
'NPP.VIIRS.11361.ALICE': {
'RNSCA-RVIRS_npp_d20140106_t0444094_e0451182_'
'b11361_c20140106045941812000_nfts_drl.h5': ''
}
})
metadata = drivers.RawDriver().fill_metadata(
ptype.DatasetMetadata(),
d.joinpath('NPP.VIIRS.11361.ALICE')
)
self.assertEqual(metadata.platform.code, 'NPP')
self.assertEqual(metadata.instrument.name, 'VIIRS')
self.assertEqual(metadata.ga_level, 'P00')
self.assertEqual(metadata.format_.name, 'HDF5')
# Groundstation should be found from surrounding adsfolder.
self.assertEqual(
metadata.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA')
)
self.assertEqual(metadata.acquisition.aos,
datetime.datetime(2014, 1, 6, 4, 44, 9))
self.assertEqual(metadata.acquisition.los,
datetime.datetime(2014, 1, 6, 4, 59, 41))
self.assertEqual(metadata.acquisition.platform_orbit, 11361)
def _build_ls8_raw():
_reset_runtime_id()
raw = ptype.DatasetMetadata(
id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'),
creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"),
size_bytes=5680940 * 1024,
ga_label=
'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1',
product_type='satellite_telemetry_data',
usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'),
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='MD'),
acquisition=ptype.AcquisitionMetadata(
aos=dateutil.parser.parse('2014-10-12T00:52:52'),
los=dateutil.parser.parse('2014-10-12T00:58:37'),
groundstation=ptype.GroundstationMetadata(
code='ASA',
antenna_coord=ptype.Coord(lat=-23.759,
lon=133.8824,
height=579.312)),
heading='D',
platform_orbit=8846),
extent=None,
grid_spatial=None,
browse=None,
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(
101, 70),
satellite_ref_point_end=ptype.Point(101,
83)),
lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata()))
return raw
def _extract_mdf_id_fields(base_md, mdf_usgs_id):
"""
V I N ppp RRR rrr YYYY ddd GSI vv
:type base_md: ptype.DatasetMetadata
:type mdf_usgs_id: str
:rtype: ptype.DatasetMetadata
"""
m = re.search((r"(?P<vehicle>L)"
r"(?P<instrument>[OTC])"
r"(?P<vehicle_number>\d)"
r"(?P<path>\d{3})"
r"(?P<row_start>\d{3})"
r"(?P<row_end>\d{3})"
r"(?P<acq_date>\d{7})"
r"(?P<gsi>\w{3})"
r"(?P<version>\d{2})"), mdf_usgs_id)
fields = m.groupdict()
if not base_md.usgs:
base_md.usgs = ptype.UsgsMetadata()
base_md.usgs.interval_id = mdf_usgs_id
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
base_md.platform.code = "LANDSAT_" + fields["vehicle_number"]
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.instrument.name = LS8_SENSORS[fields["instrument"]]
path = int(fields["path"])
if not base_md.image:
base_md.image = ptype.ImageMetadata()
base_md.image.satellite_ref_point_start = ptype.Point(
path, int(fields["row_start"]))
base_md.image.satellite_ref_point_end = ptype.Point(
path, int(fields["row_end"]))
# base_md.version = int(fields["version"]) or base_md.version
# Probably less than useful without time.
# if not base_md.extent:
# base_md.extent = ptype.ExtentMetadata()
#
# base_md.extent.center_dt = base_md.extent.center_dt or datetime.strptime(fields["acq_date"], "%Y%j").date()
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=fields["gsi"])
return base_md
def _extract_rcc_filename_fields(base_md, filename):
"""
Landsat 5 and 7 RCC format specifications:
http://landsat.usgs.gov/documents/LS_DFCB_01.pdf
http://landsat.usgs.gov/documents/LS_DFCB_06.pdf
:type base_md: ptype.DatasetMetadata
:type filename: str
:rtype: ptype.DatasetMetadata
"""
m = re.search(
r'(?P<satsens>\w{4})(?P<date>\d{13})(?P<gsi>[^\d]+).*?(?P<version>\d\d)?\.data',
filename)
fields = m.groupdict()
if not base_md.platform or not base_md.platform.code:
# TODO: Do we have a cleaner way to do this? A list of mappings?
satsens_ = fields['satsens']
vehicle = satsens_[0]
vehicle_num = satsens_[1]
instrument_short = satsens_[2]
smode_short = satsens_[3]
platform_code, instrument_name, operation_mode = _expand_platform_info(
vehicle, vehicle_num, instrument_short, smode_short)
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.platform.code = platform_code
base_md.instrument.name = instrument_name
base_md.instrument.operation_mode = operation_mode
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
if not base_md.acquisition.aos:
base_md.acquisition.aos = datetime.datetime.strptime(
fields['date'], "%Y%j%H%M%S")
base_md.gsi = fields['gsi']
if not base_md.acquisition.groundstation:
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=fields['gsi'])
if not base_md.usgs:
base_md.usgs = ptype.UsgsMetadata()
base_md.usgs.interval_id = _usgs_id_from_filename(filename)
# RCC is raw: P00
base_md.ga_level = 'P00'
version = int(fields['version']) if fields.get('version') else None
base_md.format_ = ptype.FormatMetadata(name='RCC', version=version)
return base_md
def test_expand_metadata(self):
id_ = uuid.uuid1()
create_dt = datetime.datetime.utcnow()
d = ptype.DatasetMetadata(
id_=id_,
creation_dt=create_dt,
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
image=ptype.ImageMetadata(
bands={
'4':
BandMetadata(
path=PosixPath(
'/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'),
number='4',
)
}),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata('ALSP')))
assert_same(
metadata.expand_common_metadata(d),
ptype.DatasetMetadata(
id_=id_,
creation_dt=create_dt,
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
image=ptype.ImageMetadata(
bands={
'4':
BandMetadata(path=PosixPath(
'/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'),
type_='reflective',
label='Visible Red',
number='4',
cell_size=25.0),
}),
acquisition=ptype.
AcquisitionMetadata(groundstation=ptype.GroundstationMetadata(
# Should have translated groundstation 'ALSP' to standard GSI form 'ASA'
code='ASA',
label='Alice Springs',
eods_domain_code='002'))))
def test_eods_fill_metadata(self):
dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012"
bandname = '10'
bandfile = dataset_folder + '_B' + bandname + '.tif'
input_folder = write_files({
dataset_folder: {
'metadata.xml': """<EODS_DATASET>
<ACQUISITIONINFORMATION>
<EVENT>
<AOS>20141012T03:23:36</AOS>
<LOS>20141012T03:29:10</LOS>
</EVENT>
</ACQUISITIONINFORMATION>
<EXEXTENT>
<TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM>
<TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO>
</EXEXTENT>
</EODS_DATASET>""",
'scene01': {
bandfile: ''
}
}
})
expected = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_,
ga_label=dataset_folder,
ga_level='P54',
product_type='EODS_NBAR',
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTiff'),
acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36),
los=datetime.datetime(2014, 10, 12, 3, 29, 10),
groundstation=ptype.GroundstationMetadata(code='LGS')),
extent=ptype.ExtentMetadata(
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6),
from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54),
to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18)
),
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78),
satellite_ref_point_end=ptype.Point(x=101, y=78),
bands={bandname: ptype.BandMetadata(number=bandname,
path=Path(input_folder, dataset_folder,
'scene01', bandfile))})
)
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_
)
received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder))
self.assert_same(expected, received)
def _populate_from_mtl_dict(md, mtl_, folder):
"""
:param mtl_: Parsed mtl file
:param folder: Folder containing imagery (and mtl). For fixing relative paths in the MTL.
:type md: eodatasets.type.DatasetMetadata
:type mtl_: dict of (str, obj)
:rtype: eodatasets.type.DatasetMetadata
"""
if not md.usgs:
md.usgs = ptype.UsgsMetadata()
md.usgs.scene_id = _get(mtl_, 'METADATA_FILE_INFO', 'landsat_scene_id')
md.creation_dt = _get(mtl_, 'METADATA_FILE_INFO', 'file_date')
# TODO: elsewhere we've used 'GAORTHO01' etc. Here it's 'L1T' etc.
md.product_level = _translate_to_old_usgs_code(
_get(mtl_, 'PRODUCT_METADATA', 'data_type'))
# md.size_bytes=None,
satellite_id = _get(mtl_, 'PRODUCT_METADATA', 'spacecraft_id')
if not md.platform:
md.platform = ptype.PlatformMetadata()
md.platform.code = satellite_id
md.format_ = ptype.FormatMetadata(
name=_get(mtl_, 'PRODUCT_METADATA', 'output_format'))
product_md = _get(mtl_, 'PRODUCT_METADATA')
sensor_id = _get(mtl_, 'PRODUCT_METADATA', 'sensor_id')
if not md.instrument:
md.instrument = ptype.InstrumentMetadata()
md.instrument.name = sensor_id
# md.instrument.type_
md.instrument.operation_mode = _get(product_md, 'sensor_mode')
if not md.acquisition:
md.acquisition = ptype.AcquisitionMetadata()
md.acquisition.groundstation = ptype.GroundstationMetadata(
code=_get(mtl_, "METADATA_FILE_INFO", "station_id"))
# md.acquisition.groundstation.antenna_coord
# aos, los, groundstation, heading, platform_orbit
_populate_extent(md, product_md)
_populate_grid_spatial(md, mtl_)
_populate_image(md, mtl_)
_populate_lineage(md, mtl_)
return md
def _parse_passinfo_md(base_md, lines):
"""
:type base_md: ptype.DatasetMetadata
:type lines: list of str
:rtype: ptype.DatasetMetadata
"""
fields = {}
for l in lines:
tmp = l.split()
if len(tmp) >= 2:
k, v = tmp[:2]
fields[k.upper()] = v.upper()
if not base_md.platform:
base_md.platform = ptype.PlatformMetadata()
base_md.platform.code = standardise_satellite(fields.get('SATELLITE'))
if not base_md.instrument:
base_md.instrument = ptype.InstrumentMetadata()
base_md.instrument.name = fields.get('SENSOR')
if not base_md.acquisition:
base_md.acquisition = ptype.AcquisitionMetadata()
base_md.acquisition.platform_orbit = int(fields.get('ORBIT'))
start_dt = _parse_common_date(fields.get('START'))
if start_dt:
base_md.acquisition.aos = start_dt
stop_dt = _parse_common_date(fields.get('STOP'))
if stop_dt:
base_md.acquisition.los = stop_dt
if not base_md.acquisition.groundstation:
gsi = station_to_gsi(fields.get('STATION'))
if gsi is not None:
base_md.acquisition.groundstation = ptype.GroundstationMetadata(
code=gsi)
return base_md
def get_groundstation(gsi):
"""
:param gsi:
:rtype: ptype.GroundstationMetadata
>>> get_groundstation('ASA')
GroundstationMetadata(code='ASA', label='Alice Springs', eods_domain_code='002')
>>> # Aliases should work too
>>> get_groundstation('ALICE')
GroundstationMetadata(code='ASA', label='Alice Springs', eods_domain_code='002')
>>> get_groundstation('UNKNOWN_GSI')
"""
gsi = normalise_gsi(gsi)
stations = [g for g in _GROUNDSTATION_LIST if g['code'].upper() == gsi]
if not stations:
_LOG.warning('Station GSI not known: %r', gsi)
return None
station = stations[0]
return ptype.GroundstationMetadata(code=str(station['code']),
label=str(station['label']),
eods_domain_code=str(
station['eods_domain_code']))
def extract_md(md, directory):
"""
Extract metadata from typical ads3 directory names.
Folder names contain orbit numbers.
Eg:
LANDSAT-7.76773.S3A1C2D2R2
AQUA.60724.S1A1C2D2R2
TERRA.73100.S1A2C2D4R4
LANDSAT-8.3108
NPP.VIIRS.10014.ALICE
:type md: ptype.DatasetMetadata
:type directory: pathlib.Path
:rtype: ptype.DatasetMetadata
"""
directory = directory.absolute()
parent_dir = directory.parent
orbit = _extract_orbit(directory.name) or _extract_orbit(parent_dir.name)
rms_string = _extract_rms_string(directory.name) or _extract_rms_string(parent_dir.name)
gsi = _extract_gsi(directory.name) or _extract_gsi(parent_dir.name)
if rms_string:
md.rms_string = rms_string
if not md.acquisition:
md.acquisition = ptype.AcquisitionMetadata()
if not md.acquisition.platform_orbit:
md.acquisition.platform_orbit = orbit
if not md.acquisition.groundstation and gsi:
md.acquisition.groundstation = ptype.GroundstationMetadata(code=gsi)
return md
def test_parse_variations(self):
new_examples = {
'L7EB2007303000923ASA222Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2007303000923ASA222'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2007, 10, 30, 0, 9, 23),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7EB2015118010116ASA213Q00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2015118010116ASA213'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2015, 4, 28, 1, 1, 16),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7EB2011239021036ASA111Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7EB2011239021036ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2011, 8, 27, 2, 10, 36),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2005120001242ASA111I.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2005120001242ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 4, 30, 0, 12, 42),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TT1995117002206ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TT1995117002206ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(1995, 4, 27, 0, 22, 6),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TT1990118013106ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TT1990118013106ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(1990, 4, 28, 1, 31, 6),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7ET2005302020634ASA123Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7ET2005302020634ASA123'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 10, 29, 2, 6, 34),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2011299000126ASA111I00.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2011299000126ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC', version=0),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2011, 10, 26, 0, 1, 26),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L5TB2010119010045ASA214I.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L5TB2010119010045ASA214'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM',
operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2010, 4, 29, 1, 0, 45),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
'L7ET2000296234136ASA111Q.data':
ptype.DatasetMetadata(
ga_level='P00',
usgs=ptype.UsgsMetadata(interval_id='L7ET2000296234136ASA111'),
platform=ptype.PlatformMetadata(code='LANDSAT_7'),
instrument=ptype.InstrumentMetadata(name='ETM',
operation_mode='SAM'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2000, 10, 22, 23, 41, 36),
groundstation=ptype.GroundstationMetadata(code='ASA'))),
}
for file_name, expected_output in new_examples.items():
output = rccfile._extract_rcc_filename_fields(
ptype.DatasetMetadata(), file_name)
# The ids will be different — clear them before comparison.
output.id_ = None
expected_output.id_ = None
self.assert_same(expected_output, output)
id_=uuid.UUID('3ff71eb0-d5c5-11e4-aebb-1040f381a756'),
product_level='L1G',
creation_dt=datetime.datetime(2015, 4, 7, 1, 58, 25),
platform=ptype.PlatformMetadata(
code='LANDSAT_7'
),
instrument=ptype.InstrumentMetadata(
name='ETM',
operation_mode='SAM'
),
format_=ptype.FormatMetadata(
name='GeoTIFF'
),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata(
code='ASA'
)
),
usgs=ptype.UsgsMetadata(
scene_id='LE71140732005007ASA00'
),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(
ul=ptype.Coord(
lat=-17.82157,
lon=115.58472
),
ur=ptype.Coord(
lat=-17.82497,
lon=117.82111
),
def fill_metadata(self, dataset, path, additional_files=()):
"""
:type additional_files: tuple[Path]
:type dataset: ptype.DatasetMetadata
:type path: Path
:rtype: ptype.DatasetMetadata
"""
fields = re.match((r"(?P<vehicle>LS[578])"
r"_(?P<instrument>OLI_TIRS|OLI|TIRS|TM|ETM)"
r"_(?P<type>NBAR|PQ|FC)"
r"_(?P<level>[^-_]*)"
r"_(?P<product>[^-_]*)"
r"-(?P<groundstation>[0-9]{3})"
r"_(?P<path>[0-9]{3})"
r"_(?P<row>[0-9]{3})"
r"_(?P<date>[12][0-9]{7})"
r"(_(?P<version>[0-9]+))?"
"$"), path.stem).groupdict()
dataset.product_type = "EODS_" + fields["type"]
dataset.ga_level = fields["level"]
dataset.ga_label = path.stem
dataset.format_ = ptype.FormatMetadata(name='GeoTiff')
if not dataset.platform:
dataset.platform = ptype.PlatformMetadata()
dataset.platform.code = "LANDSAT_" + fields["vehicle"][2]
if not dataset.instrument:
dataset.instrument = ptype.InstrumentMetadata()
dataset.instrument.name = fields["instrument"]
if not dataset.image:
dataset.image = ptype.ImageMetadata(bands={})
dataset.image.satellite_ref_point_start = ptype.Point(
int(fields["path"]), int(fields["row"]))
dataset.image.satellite_ref_point_end = ptype.Point(
int(fields["path"]), int(fields["row"]))
for image_path in path.joinpath("scene01").iterdir():
band = self.to_band(dataset, image_path)
if band:
dataset.image.bands[band.number] = band
md_image.populate_from_image_metadata(dataset)
if not dataset.acquisition:
dataset.acquisition = ptype.AcquisitionMetadata()
for _station in _GROUNDSTATION_LIST:
if _station["eods_domain_code"] == fields["groundstation"]:
dataset.acquisition.groundstation = ptype.GroundstationMetadata(
code=_station["code"])
break
if not dataset.extent:
dataset.extent = ptype.ExtentMetadata()
def els2date(els):
if not els:
return None
return parse(els[0].text)
doc = ElementTree.parse(str(path.joinpath('metadata.xml')))
start_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTFROM"))
end_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTTO"))
# check if the dates in the metadata file are at least as accurate as what we have
filename_time = datetime.datetime.strptime(fields["date"], "%Y%m%d")
time_diff = start_time - filename_time
# Is the EODS metadata extremely off?
if abs(time_diff).days != 0:
raise ValueError(
'EODS time information differs too much from source files: %s'
% time_diff)
dataset.acquisition.aos = els2date(
doc.findall("./ACQUISITIONINFORMATION/EVENT/AOS"))
dataset.acquisition.los = els2date(
doc.findall("./ACQUISITIONINFORMATION/EVENT/LOS"))
dataset.extent.center_dt = start_time + (end_time - start_time) / 2
dataset.extent.from_dt = start_time
dataset.extent.to_dt = end_time
return dataset
from tests.metadata.mtl import test_ls8, test_ls7_definitive, test_ls5_definitive
_LS5_RAW = ptype.DatasetMetadata(
id_=UUID('c86809b3-e894-11e4-8958-1040f381a756'),
ga_level='P00',
product_type='satellite_telemetry_data',
creation_dt=datetime.datetime(2015, 4, 22, 0, 7, 48),
size_bytes=5871413760,
checksum_path=Path('package.sha1'),
platform=ptype.PlatformMetadata(code='LANDSAT_5'),
instrument=ptype.InstrumentMetadata(name='TM', operation_mode='BUMPER'),
format_=ptype.FormatMetadata(name='RCC'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2005, 6, 1, 1, 51, 10),
los=datetime.datetime(2005, 6, 1, 2, 0, 25),
groundstation=ptype.GroundstationMetadata(code='ASA'),
platform_orbit=113025
),
usgs=ptype.UsgsMetadata(
interval_id='L5TB2005152015110ASA111'
),
lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata(
hostname='niggle.local',
runtime_id=UUID('b2af5545-e894-11e4-b3b0-1040f381a756'),
type_id='jobmanager',
version='2.4.0',
uname='Darwin niggle.local 14.3.0 Darwin Kernel Version 14.3.0: Mon Mar 23 11:59:05 PDT 2015; '
'root:xnu-2782.20.48~5/RELEASE_X86_64 x86_64'
),
source_datasets={}
from pathlib import Path
import eodatasets.type as ptype
from tests.metadata.mtl import assert_expected_mtl
FILENAME = 'ls8_mtl.txt'
EXPECTED_OUT = ptype.DatasetMetadata(
id_=uuid.UUID('3ff71eb0-d5c5-11e4-aebb-1040f381a756'),
product_level='L1T',
creation_dt=datetime.datetime(2014, 11, 12, 15, 8, 35),
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTIFF'),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata(code='LGN')),
usgs=ptype.UsgsMetadata(scene_id='LC81010782014285LGN00'),
extent=ptype.ExtentMetadata(
coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.98805, lon=133.97954),
ur=ptype.Coord(lat=-24.9864, lon=136.23866),
ll=ptype.Coord(lat=-26.99236, lon=133.96208),
lr=ptype.Coord(lat=-26.99055, lon=136.25985)),
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)),
grid_spatial=ptype.GridSpatialMetadata(projection=ptype.ProjectionMetadata(
geo_ref_points=ptype.
PointPolygon(ul=ptype.Point(x=397012.5, y=7235987.5),
ur=ptype.Point(x=625012.5, y=7235987.5),
ll=ptype.Point(x=397012.5, y=7013987.5),
lr=ptype.Point(x=625012.5, y=7013987.5)),
datum='GDA94',
ellipsoid='GRS80',