def test_expand_metadata_without_bands(self):
# We have imagery files but no bands (eg: RAW data)
f = write_files({'first.txt': 'test', 'second.txt': 'test2'})
class FauxDriver(drivers.DatasetDriver):
def to_band(self, dataset, path):
return None
def get_ga_label(self, dataset):
return 'DATASET_ID_1234'
def get_id(self):
return 'faux'
d = ptype.DatasetMetadata()
# Iterator is falsey, but returns files. This triggered a bug previously.
# noinspection PyTypeChecker
d = package.expand_driver_metadata(FauxDriver(), d, f.iterdir())
self.assert_same(
d,
ptype.DatasetMetadata(id_=d.id_,
ga_label='DATASET_ID_1234',
product_type='faux',
size_bytes=9))
def test_prepare_metadata(self):
f = write_files({'first.txt': 'test', 'second.txt': 'test2'})
class FauxDriver(drivers.DatasetDriver):
def to_band(self, dataset, path):
numbers = {
'first': ptype.BandMetadata(path=path, number='1'),
'second': None
}
return numbers.get(path.stem)
def get_ga_label(self, dataset):
return 'DATASET_ID_1234'
def get_id(self):
return 'faux'
d = ptype.DatasetMetadata()
d = package.expand_driver_metadata(FauxDriver(), d, list(f.iterdir()))
self.assert_same(
d,
ptype.DatasetMetadata(
id_=d.id_,
ga_label='DATASET_ID_1234',
product_type='faux',
size_bytes=9,
image=ptype.ImageMetadata(
bands={
'1':
ptype.BandMetadata(path=f.joinpath('first.txt'),
number='1')
})))
def test_extract_md(_run_pdsinfo_exe):
input_dir = write_files({'P1540064AAAAAAAAAAAAAA14219032341001.PDS': ''})
# def run_pdsinfo(file_):
# assert file_ == input_dir
#
# return
_run_pdsinfo_exe.return_value = b"""APID 64: count 610338 invalid 0 missing 6255
first packet: 2014/08/07 03:16:28.750910
last packet: 2014/08/07 03:21:28.604695
missing seconds: 2
day packets: 545223/64311
night packets: 0/0
engineering packets: 804/0
"""
md = pds.extract_md(ptype.DatasetMetadata(), input_dir)
expected = ptype.DatasetMetadata(
platform=ptype.PlatformMetadata(code='AQUA'),
instrument=ptype.InstrumentMetadata(name='MODIS'),
format_=ptype.FormatMetadata(name='PDS'),
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2014, 8, 7, 3, 16, 28, 750910),
los=datetime.datetime(2014, 8, 7, 3, 21, 28, 604695)
),
image=ptype.ImageMetadata(
day_percentage_estimate=100.0
)
)
md.id_, expected.id_ = None, None
assert_same(expected, md)
def test_create_typical_browse_metadata():
class TestDriver(drivers.DatasetDriver):
def browse_image_bands(self, d):
return '5', '1', '3'
d = write_files({})
dataset = browseimage.create_typical_browse_metadata(
TestDriver(), ptype.DatasetMetadata(), d)
expected = ptype.DatasetMetadata(
browse={
'full':
ptype.BrowseMetadata(path=d.joinpath('browse.fr.jpg'),
file_type='image/jpg',
red_band='5',
green_band='1',
blue_band='3'),
'medium':
ptype.BrowseMetadata(
path=d.joinpath('browse.jpg'),
# Default medium size.
shape=ptype.Point(1024, None),
file_type='image/jpg',
red_band='5',
green_band='1',
blue_band='3')
})
expected.id_, dataset.id_ = None, None
assert_same(expected, dataset)
def test_unchanged_without_id(self):
# No MDF directory, only files. Don't try to extract information from the files.
d = write_files({
'446.000.2013254233714881.ASA': 'a',
'447.000.2013254233711482.ASA': 'a',
})
id_ = uuid.uuid1()
date = datetime.datetime.utcnow()
metadata = mdf.extract_md(
ptype.DatasetMetadata(id_=id_, creation_dt=date), d)
# Should be unchanged: No USGS ID found.
assert_same(metadata, ptype.DatasetMetadata(id_=id_, creation_dt=date))
def test_extract_groundstation(self):
d = write_files({'LANDSAT-7.ALICE': {'dataset': {}}})
d = d.joinpath('LANDSAT-7.ALICE')
md = adsfolder.extract_md(ptype.DatasetMetadata(), d)
self.assertIsNotNone(md.acquisition)
self.assertEqual(md.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA'))
d = d.joinpath('dataset')
md = adsfolder.extract_md(ptype.DatasetMetadata(), d)
self.assertIsNotNone(md.acquisition)
self.assertEqual(md.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA'))
def test_eods_fill_metadata(self):
dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012"
bandname = '10'
bandfile = dataset_folder + '_B' + bandname + '.tif'
input_folder = write_files({
dataset_folder: {
'metadata.xml': """<EODS_DATASET>
<ACQUISITIONINFORMATION>
<EVENT>
<AOS>20141012T03:23:36</AOS>
<LOS>20141012T03:29:10</LOS>
</EVENT>
</ACQUISITIONINFORMATION>
<EXEXTENT>
<TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM>
<TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO>
</EXEXTENT>
</EODS_DATASET>""",
'scene01': {
bandfile: ''
}
}
})
expected = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_,
ga_label=dataset_folder,
ga_level='P54',
product_type='EODS_NBAR',
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='GeoTiff'),
acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36),
los=datetime.datetime(2014, 10, 12, 3, 29, 10),
groundstation=ptype.GroundstationMetadata(code='LGS')),
extent=ptype.ExtentMetadata(
center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6),
from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54),
to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18)
),
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78),
satellite_ref_point_end=ptype.Point(x=101, y=78),
bands={bandname: ptype.BandMetadata(number=bandname,
path=Path(input_folder, dataset_folder,
'scene01', bandfile))})
)
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_
)
received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder))
self.assert_same(expected, received)
def test_parse_passinfo_file(self):
d = write_files({
'subdirectory': {
},
'passinfo': [
"STATION ALICE\n",
"STRING S1A1C1D1R1\n",
"PASSID LANDSAT-5.110912\n",
"SATELLITE LANDSAT-5\n",
"ORBIT 110912\n",
"LINKID X\n",
"BITRATE 84900000\n",
"SENSOR TM\n",
"START 2005-01-06T23:32:14\n",
"STOP 2005-01-06T23:39:12\n",
"DURATION 423\n",
"COMMENTS Bit\n",
"logfile acs acs.log\n",
"logfile ref ref.log\n",
"logfile demod demod.log\n",
"logfile 1050110912.eph 1050110912.eph.log\n",
"telemetry telemetry.data\n"
]})
# It should find a passinfo file one directory up.
md = extraction.extract_md(ptype.DatasetMetadata(), d.joinpath('subdirectory'))
self.assertEqual(md.acquisition.groundstation.code, "ASA")
self.assertEqual(md.acquisition.platform_orbit, 110912)
self.assertEqual(md.platform.code, "LANDSAT_5")
self.assertEqual(md.instrument.name, "TM")
self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14))
self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_parse_passinfo(self):
pm = ptype.DatasetMetadata()
lines = [
"STATION ALICE\n",
"STRING S1A1C1D1R1\n",
"PASSID LANDSAT-5.110912\n",
"SATELLITE LANDSAT-5\n",
"ORBIT 110912\n",
"LINKID X\n",
"BITRATE 84900000\n",
"SENSOR TM\n",
"START 2005-01-06T23:32:14\n",
"STOP 2005-01-06T23:39:12\n",
"DURATION 423\n",
"COMMENTS Bit\n",
"logfile acs acs.log\n",
"logfile ref ref.log\n",
"logfile demod demod.log\n",
"logfile 1050110912.eph 1050110912.eph.log\n",
"telemetry telemetry.data\n"
]
md = extraction._parse_passinfo_md(pm, lines)
self.assertEqual(md.acquisition.groundstation.code, "ASA")
self.assertEqual(md.acquisition.platform_orbit, 110912)
self.assertEqual(md.platform.code, "LANDSAT_5")
self.assertEqual(md.instrument.name, "TM")
self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14))
self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_unusual_filenames(self):
# Some passinfo filenames have orbit numbers appended
d = write_files({'passinfo.24775': [
"STATION TERSS\n",
"STRING S1A1C1D1R1\n",
"PASSID LANDSAT-5.110912\n",
"SATELLITE LANDSAT-5\n",
"ORBIT 110912\n",
"LINKID X\n",
"BITRATE 84900000\n",
"SENSOR TM\n",
"START 2005-01-06T23:32:14\n",
"STOP 2005-01-06T23:39:12\n",
"DURATION 423\n",
"COMMENTS Bit\n",
"logfile acs acs.log\n",
"logfile ref ref.log\n",
"logfile demod demod.log\n",
"logfile 1050110912.eph 1050110912.eph.log\n",
"telemetry telemetry.data\n"
]})
md = extraction.extract_md(ptype.DatasetMetadata(), d)
# Station "TERSS" is hobart.
self.assertEqual(md.acquisition.groundstation.code, "HOA")
self.assertEqual(md.acquisition.platform_orbit, 110912)
self.assertEqual(md.platform.code, "LANDSAT_5")
self.assertEqual(md.instrument.name, "TM")
self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 6, 23, 32, 14))
self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 6, 23, 39, 12))
def test_pqa_fill(self):
input_folder = write_files({
'pqa.tif': '',
'pq_metadata.yaml': dedent(
"""
algorithm_information:
software_version: 1.0
pq_doi:
ancillary: {}
tests_run: {}
""")
})
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_PQA.id_,
lineage=ptype.LineageMetadata(
source_datasets={
'nbar': _EXPECTED_NBAR,
'level1': test_ls8.EXPECTED_OUT
}
)
)
received_dataset = drivers.PqaDriver().fill_metadata(dataset, input_folder)
self.assert_same(_EXPECTED_PQA, received_dataset)
def test_aqua_pds_label(self):
ds = ptype.DatasetMetadata(
id_=UUID('d083fa45-1edd-11e5-8f9e-1040f381a756'),
product_type='satellite_telemetry_data',
creation_dt=datetime.datetime(2015, 6, 11, 5, 51, 50),
platform=ptype.PlatformMetadata(code='AQUA'),
instrument=ptype.InstrumentMetadata(name='MODIS'),
format_=ptype.FormatMetadata(name='PDS'),
rms_string='S1A1C1D1R1',
acquisition=ptype.AcquisitionMetadata(
aos=datetime.datetime(2014, 8, 7, 3, 16, 28, 750910),
los=datetime.datetime(2014, 8, 7, 3, 16, 30, 228023),
platform_orbit=65208
),
image=ptype.ImageMetadata(day_percentage_estimate=100.0),
lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata(),
source_datasets={}
)
)
self.assertEqual(
"AQUA_MODIS_STD-PDS_P00_65208.S1A1C1D1R1_0_0_20140807T031628Z20140807T031630",
drivers.RawDriver().get_ga_label(ds)
)
def test_parse_from_driver(self):
d = write_files({
'NPP.VIIRS.11361.ALICE': {
'RNSCA-RVIRS_npp_d20140106_t0444094_e0451182_'
'b11361_c20140106045941812000_nfts_drl.h5': ''
}
})
metadata = drivers.RawDriver().fill_metadata(
ptype.DatasetMetadata(),
d.joinpath('NPP.VIIRS.11361.ALICE')
)
self.assertEqual(metadata.platform.code, 'NPP')
self.assertEqual(metadata.instrument.name, 'VIIRS')
self.assertEqual(metadata.ga_level, 'P00')
self.assertEqual(metadata.format_.name, 'HDF5')
# Groundstation should be found from surrounding adsfolder.
self.assertEqual(
metadata.acquisition.groundstation,
ptype.GroundstationMetadata(code='ASA')
)
self.assertEqual(metadata.acquisition.aos,
datetime.datetime(2014, 1, 6, 4, 44, 9))
self.assertEqual(metadata.acquisition.los,
datetime.datetime(2014, 1, 6, 4, 59, 41))
self.assertEqual(metadata.acquisition.platform_orbit, 11361)
def init_existing_dataset(directory,
source_datasets,
uuid_=None,
source_hostname=None):
"""
Package an existing dataset folder (with mostly unknown provenance).
This is intended for old datasets where little information was recorded.
For brand new datasets, it's better to use init_locally_processed_dataset() to capture
local machine information.
:param uuid_: The existing dataset_id, if any.
:param source_hostname: Hostname where processed, if known.
:rtype: ptype.DatasetMetadata
"""
md = ptype.DatasetMetadata(
id_=uuid_,
# Default creation time is creation of an image.
creation_dt=datetime.datetime.utcfromtimestamp(
directory.stat().st_ctime),
lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata(hostname=source_hostname),
source_datasets=source_datasets))
_note_package_vers(md)
return md
def _build_ls8_raw():
_reset_runtime_id()
raw = ptype.DatasetMetadata(
id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'),
creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"),
size_bytes=5680940 * 1024,
ga_label=
'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1',
product_type='satellite_telemetry_data',
usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'),
platform=ptype.PlatformMetadata(code='LANDSAT-8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
format_=ptype.FormatMetadata(name='MD'),
acquisition=ptype.AcquisitionMetadata(
aos=dateutil.parser.parse('2014-10-12T00:52:52'),
los=dateutil.parser.parse('2014-10-12T00:58:37'),
groundstation=ptype.GroundstationMetadata(
code='ASA',
antenna_coord=ptype.Coord(lat=-23.759,
lon=133.8824,
height=579.312)),
heading='D',
platform_orbit=8846),
extent=None,
grid_spatial=None,
browse=None,
image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(
101, 70),
satellite_ref_point_end=ptype.Point(101,
83)),
lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata()))
return raw
def test_parse_passinfo_ls7_file(self):
d = write_files({'passinfo': [
"STATION ALICE\n",
"STRING S1A1C2D3R3\n",
"PASSID LANDSAT-7.30486\n",
"SATELLITE LANDSAT-7\n",
"ORBIT 30486\n",
"LINKID L\n",
"BITRATE 150000000\n",
"SENSOR ETM\n",
"START 2005-01-07T02:00:28.000\n",
"STOP 2005-01-07T02:07:19.000\n",
"DURATION 528\n",
"COMMENTS Bit\n",
"logfile acs acs.log\n",
"logfile ref ref.log\n",
"logfile 1070030486.eph 1070030486.eph.log\n",
"logfile demod demod.log\n",
"telemetry telemetry.data\n",
]})
md = extraction.extract_md(ptype.DatasetMetadata(), d)
self.assertEqual(md.acquisition.groundstation.code, "ASA")
self.assertEqual(md.acquisition.platform_orbit, 30486)
self.assertEqual(md.platform.code, "LANDSAT_7")
self.assertEqual(md.instrument.name, "ETM")
self.assertEqual(md.acquisition.aos, datetime.datetime(2005, 1, 7, 2, 0, 28))
self.assertEqual(md.acquisition.los, datetime.datetime(2005, 1, 7, 2, 7, 19))
def test_files_with_usgs_id(self):
# No MDF directory, only files. Can we still extract enough info?
d = write_files({
'446.000.2013254233714881.ASA': 'a',
'447.000.2013254233711482.ASA': 'a',
'LC80880750762013254ASA00_IDF.xml': 'a',
'LC80880750762013254ASA00_MD5.txt': 'a',
})
def _test_mdf_output(metadata):
self.assertEqual(metadata.usgs.interval_id,
'LC80880750762013254ASA00')
self.assertEqual(metadata.platform.code, 'LANDSAT_8')
self.assertEqual(metadata.instrument.name, 'OLI_TIRS')
self.assertEqual(metadata.format_.name, 'MD')
self.assertEqual(metadata.ga_level, 'P00')
self.assertEqual(metadata.image.satellite_ref_point_start,
ptype.Point(88, 75))
self.assertEqual(metadata.image.satellite_ref_point_end,
ptype.Point(88, 76))
self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
# No longer bother with vague center date.
# self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11))
self.assertEqual(
metadata.acquisition.aos,
datetime.datetime(2013, 9, 11, 23, 36, 11, 482000))
self.assertEqual(
metadata.acquisition.los,
datetime.datetime(2013, 9, 11, 23, 37, 14, 881000))
metadata = mdf.extract_md(ptype.DatasetMetadata(), d)
_test_mdf_output(metadata)
def test_no_directory(self):
d = write_files({
'L7EB2013259012832ASN213I00.data': 'nothing',
'L7EB2013259012832ASN213Q00.data': 'nothing'
})
self.assertEqual((None, set()), mdf.find_mdf_files(d))
# Make sure that metadata is not modified when no MDF is found.
starting_md = ptype.DatasetMetadata()
id_ = starting_md.id_
creation_dt = starting_md.creation_dt
expected_dt = ptype.DatasetMetadata(id_=id_, creation_dt=creation_dt)
output = mdf.extract_md(starting_md, d)
self.assertEqual(expected_dt, output)
def test_expand_metadata(self):
id_ = uuid.uuid1()
create_dt = datetime.datetime.utcnow()
d = ptype.DatasetMetadata(
id_=id_,
creation_dt=create_dt,
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
image=ptype.ImageMetadata(
bands={
'4':
BandMetadata(
path=PosixPath(
'/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'),
number='4',
)
}),
acquisition=ptype.AcquisitionMetadata(
groundstation=ptype.GroundstationMetadata('ALSP')))
assert_same(
metadata.expand_common_metadata(d),
ptype.DatasetMetadata(
id_=id_,
creation_dt=create_dt,
platform=ptype.PlatformMetadata(code='LANDSAT_8'),
instrument=ptype.InstrumentMetadata(name='OLI_TIRS'),
image=ptype.ImageMetadata(
bands={
'4':
BandMetadata(path=PosixPath(
'/tmp/fake-folder/LC81010782014285LGN00_B4.TIF'),
type_='reflective',
label='Visible Red',
number='4',
cell_size=25.0),
}),
acquisition=ptype.
AcquisitionMetadata(groundstation=ptype.GroundstationMetadata(
# Should have translated groundstation 'ALSP' to standard GSI form 'ASA'
code='ASA',
label='Alice Springs',
eods_domain_code='002'))))
def test_extract_ls7_rcc_filename_fields(self):
md = rccfile._extract_rcc_filename_fields(
ptype.DatasetMetadata(), 'L7EB2012028010752ASA111I.data')
self.assertEqual(md.platform.code, 'LANDSAT_7')
self.assertEqual(md.instrument.name, 'ETM')
self.assertEqual(md.instrument.operation_mode, 'BUMPER')
self.assertEqual(md.acquisition.groundstation.code, 'ASA')
# Is AOS correct here? The RCC spec is not clear exactly when the timestamp is taken.
self.assertEqual(md.acquisition.aos,
datetime.datetime(2012, 1, 28, 1, 7, 52))
def test_gqa():
md = ptype.DatasetMetadata(lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata()))
gqa.populate_from_gqa(md, _GQA_PATH)
print(repr(md.gqa))
assert md.gqa == {
"cep90": 0.41,
"colors": {
"blue": 600,
"green": 30164,
"red": 336,
"teal": 1340,
"yellow": 399
},
"final_gcp_count": 32582,
"ref_date": datetime.date(2000, 9, 4),
"ref_source": "GLS_v2",
"ref_source_path":
"/g/data/v10/eoancillarydata/GCP/GQA_v2/wrs2/091/081/LE70910812000248ASA00_B5.TIF",
"residual": {
"abs": {
"x": 0.2,
"y": 0.23
},
"abs_iterative_mean": {
"x": 0.15,
"y": 0.17
},
"iterative_mean": {
"x": 0.02,
"y": 0
},
"iterative_stddev": {
"x": 0.32,
"y": 0.52
},
"mean": {
"x": 0.01,
"y": -0.03
},
"stddev": {
"x": 1.27,
"y": 3.94
}
}
}
assert 'gqa' in md.lineage.machine.software_versions
assert md.lineage.machine.software_versions['gqa'] == {
'repo_url': "https://github.com/GeoscienceAustralia/gqa.git",
'version': "0.4+20.gb0d00dc"
}
def test_extract_ls5_rcc_filename_fields(self):
md = rccfile._extract_rcc_filename_fields(
ptype.DatasetMetadata(), 'L5TT2003339014237ASA111I00.data')
self.assertEqual(md.platform.code, 'LANDSAT_5')
self.assertEqual(md.instrument.name, 'TM')
self.assertEqual(md.instrument.operation_mode, 'SAM')
self.assertEqual(md.acquisition.groundstation.code, 'ASA')
self.assertEqual(md.format_.version, 0)
# Is AOS correct here? The RCC spec is not clear exactly when the timestamp is taken.
self.assertEqual(md.acquisition.aos,
datetime.datetime(2003, 12, 5, 1, 42, 37))
def test_override_gqa_version():
md = ptype.DatasetMetadata(lineage=ptype.LineageMetadata(
machine=ptype.MachineMetadata()))
md.lineage.machine.note_software_version('gqa', '0.0.0')
gqa.populate_from_gqa(md, _GQA_PATH)
# It should have overridden it (more specific).
assert 'gqa' in md.lineage.machine.software_versions
assert md.lineage.machine.software_versions['gqa'] == {
'repo_url': "https://github.com/GeoscienceAustralia/gqa.git",
'version': "0.4+20.gb0d00dc"
}
def test_directory(self):
d = write_files({
'LC80880750762013254ASA00': {
'446.000.2013254233714881.ASA': 'a',
'447.000.2013254233711482.ASA': 'a',
'LC80880750762013254ASA00_IDF.xml': 'a',
'LC80880750762013254ASA00_MD5.txt': 'a',
}
})
def _test_mdf_output(metadata):
self.assertEqual(metadata.usgs.interval_id,
'LC80880750762013254ASA00')
self.assertEqual(metadata.platform.code, 'LANDSAT_8')
self.assertEqual(metadata.instrument.name, 'OLI_TIRS')
self.assertEqual(metadata.format_.name, 'MD')
self.assertEqual(metadata.ga_level, 'P00')
self.assertEqual(metadata.image.satellite_ref_point_start,
ptype.Point(88, 75))
self.assertEqual(metadata.image.satellite_ref_point_end,
ptype.Point(88, 76))
self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
# No longer bother with vague center date.
# self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11))
self.assertEqual(
metadata.acquisition.aos,
datetime.datetime(2013, 9, 11, 23, 36, 11, 482000))
self.assertEqual(
metadata.acquisition.los,
datetime.datetime(2013, 9, 11, 23, 37, 14, 881000))
metadata = mdf.extract_md(ptype.DatasetMetadata(), d)
_test_mdf_output(metadata)
# It should also work when given the specific MDF folder.
metadata = mdf.extract_md(ptype.DatasetMetadata(),
d.joinpath('LC80880750762013254ASA00'))
_test_mdf_output(metadata)
def _check_rcc_parse(self, d):
md = rccfile.extract_md(ptype.DatasetMetadata(), d)
self.assertEqual(md.platform.code, 'LANDSAT_7')
self.assertEqual(md.instrument.name, 'ETM')
self.assertEqual(md.acquisition.groundstation.code, 'ASN')
self.assertEqual(md.ga_level, 'P00')
self.assertEqual(md.format_.name, 'RCC')
self.assertEqual(md.usgs.interval_id, 'L7EB2013259012832ASN213')
self.assertEqual(md.acquisition.aos,
datetime.datetime(2013, 9, 16, 1, 28, 32))
# From the old onreceipt codebase,
# Default L7 LOS is: AOS + (I.data fileSize) * 8.0 / 75000000.0
self.assertEqual(md.acquisition.los,
datetime.datetime(2013, 9, 16, 1, 28, 32))
def testExtractHDF5FilenameFields(self):
metadata = extraction._extract_hdf5_filename_fields(
ptype.DatasetMetadata(),
'RNSCA-RVIRS_npp_d20130422_t0357358_e0410333_b07686_c20130422041225898000_nfts_drl.h5'
)
self.assertEqual(metadata.platform.code, 'NPP')
self.assertEqual(metadata.instrument.name, 'VIIRS')
self.assertEqual(metadata.acquisition.platform_orbit, 7686)
self.assertEqual(metadata.acquisition.aos,
datetime.datetime(2013, 4, 22, 3, 57, 35))
self.assertEqual(metadata.acquisition.los,
datetime.datetime(2013, 4, 22, 4, 12, 25))
def test_directory_parse(self):
dataset_id = 'LC80880750762013254ASA00'
metadata = mdf._extract_mdf_id_fields(ptype.DatasetMetadata(),
dataset_id)
self.assertEqual(metadata.usgs.interval_id, dataset_id)
self.assertEqual(metadata.platform.code, 'LANDSAT_8')
self.assertEqual(metadata.instrument.name, 'OLI_TIRS')
self.assertEqual(metadata.image.satellite_ref_point_start,
ptype.Point(88, 75))
self.assertEqual(metadata.image.satellite_ref_point_end,
ptype.Point(88, 76))
self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
def test_directory(self):
d = write_files({
'LANDSAT-8.11308': {
'LC80880750762013254ASA00': {
'446.000.2013254233714881.ASA': 'a',
'447.000.2013254233711482.ASA': 'a',
'LC80880750762013254ASA00_IDF.xml': 'a',
'LC80880750762013254ASA00_MD5.txt': 'a',
}
}
})
# Read orbit from folder name
d = d.joinpath('LANDSAT-8.11308')
ds = ptype.DatasetMetadata()
metadata = adsfolder.extract_md(ds, d)
self.assertEqual(metadata.acquisition.platform_orbit, 11308)
# Read orbit from parent folder name.
d = d.joinpath('LC80880750762013254ASA00')
ds = ptype.DatasetMetadata()
metadata = adsfolder.extract_md(ds, d)
self.assertEqual(metadata.acquisition.platform_orbit, 11308)
def test_parse_l5_rcc_filenames(self):
d = write_files({'L5TB2003339014237ASA111I00.data': 'nothing'})
md = rccfile.extract_md(ptype.DatasetMetadata(), d)
self.assertEqual(md.platform.code, 'LANDSAT_5')
self.assertEqual(md.instrument.name, 'TM')
self.assertEqual(md.acquisition.groundstation.code, 'ASA')
self.assertEqual(md.format_.name, 'RCC')
self.assertEqual(md.usgs.interval_id, 'L5TB2003339014237ASA111')
self.assertEqual(md.acquisition.aos,
datetime.datetime(2003, 12, 5, 1, 42, 37))
# From the old onreceipt codebase,
# Default L5 LOS is: AOS + (I.data fileSize) * 8.0 / 84900000.0
self.assertEqual(md.acquisition.los,
datetime.datetime(2003, 12, 5, 1, 42, 37))
def test_nbar_fill_metadata(self):
input_folder = write_files({
'reflectance_brdf_1.tif': '',
'reflectance_brdf_2.tif': '',
'reflectance_brdf_3.tif': '',
'reflectance_brdf_4.tif': '',
'reflectance_brdf_5.tif': '',
'reflectance_brdf_6.tif': '',
'reflectance_brdf_7.tif': '',
'reflectance_terrain_1.tif': '',
'reflectance_terrain_2.tif': '',
'reflectance_terrain_3.tif': '',
'reflectance_terrain_4.tif': '',
'reflectance_terrain_5.tif': '',
'reflectance_terrain_6.tif': '',
'reflectance_terrain_7.tif': '',
'reflectance_lambertian_1.tif': '',
'reflectance_lambertian_2.tif': '',
'reflectance_lambertian_3.tif': '',
'reflectance_lambertian_4.tif': '',
'reflectance_lambertian_5.tif': '',
'reflectance_lambertian_6.tif': '',
'reflectance_lambertian_7.tif': '',
'nbar_metadata.yaml': dedent(
"""
algorithm_information:
software_version: 1.0
algorithm_version: 1.0
arg25_doi:
nbar_doi:
nbar_terrain_corrected_doi:
ancillary_data: {}
"""),
})
dataset = ptype.DatasetMetadata(
id_=_EXPECTED_NBAR.id_,
lineage=ptype.LineageMetadata(
source_datasets={
'level1': test_ls8.EXPECTED_OUT
}
)
)
received_dataset = drivers.NbarDriver('terrain').fill_metadata(dataset, input_folder)
self.assert_same(_EXPECTED_NBAR, received_dataset)
