def test_package():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset),
'pqa',
'--parent', str(parent_dataset),
str(source_dataset), str(output_path)
])
output_dataset = output_path.joinpath('LS8_OLITIRS_PQ_P55_GAPQ01-032_090_081_20140726')
assert_file_structure(output_path, {
'LS8_OLITIRS_PQ_P55_GAPQ01-032_090_081_20140726': {
'browse.jpg': '',
'browse.fr.jpg': '',
'product': {
'LS8_OLITIRS_PQ_P55_GAPQ01-032_090_081_20140726.tif': '',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
# Load metadata file and compare it to expected.
output_checksum_path = output_dataset.joinpath('ga-metadata.yaml')
assert output_checksum_path.exists()
md = yaml.load(output_checksum_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
# Check metadata is as expected.
EXPECTED_METADATA['size_bytes'] = directory_size(output_dataset / 'product')
add_default_software_versions(EXPECTED_METADATA)
assert_same(
md,
EXPECTED_METADATA
)
# TODO: Assert correct checksums? They shouldn't change in theory. But they may with gdal versions etc.
# Check all files are listed in checksum file.
output_checksum_path = output_dataset.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'browse.fr.jpg',
'browse.jpg',
'ga-metadata.yaml',
'product/LS8_OLITIRS_PQ_P55_GAPQ01-032_090_081_20140726.tif',
]
def test_package():
work_path = temp_dir()
output_path = work_path.joinpath('out')
output_path.mkdir(parents=True)
ancil_path = work_path.joinpath('ancil')
# We have to override the ancillary directory lookup as they won't exist on test systems.
ancil_files = (
FakeAncilFile(ancil_path, 'cpf', 'L8CPF20140101_20140331.05'),
FakeAncilFile(ancil_path, 'bpf_oli', 'LO8BPF20140127130115_20140127144056.01'),
FakeAncilFile(ancil_path, 'bpf_tirs', 'LT8BPF20140116023714_20140116032836.02'),
FakeAncilFile(ancil_path, 'tirs_ssm_position', '20160529.l8_tirs_estimated_ssm_position.txt'),
FakeAncilFile(ancil_path, 'rlut', 'L8RLUT20130211_20431231v09.h5', folder_offset=('2013',)),
)
work_order_path = prepare_work_order(ancil_files, work_order_template_path)
# Run!
args = [
hardlink_arg(output_path, source_dataset),
'level1',
'--newly-processed',
'--parent', str(parent_dataset),
'--add-file', str(work_order_path)
]
for additional_file in additional_files.iterdir():
args.extend(['--add-file', str(additional_file)])
args.extend([
str(source_dataset),
str(output_path)
])
run_packaging_cli(args)
output_dataset = output_path.joinpath('LS8_OLITIRS_OTH_P51_GALPGS01-002_112_079_20140126')
assert_file_structure(output_path, {
'LS8_OLITIRS_OTH_P51_GALPGS01-002_112_079_20140126': {
'browse.jpg': '',
'browse.fr.jpg': '',
'product': {
# Newer versions of GDAL create an IMD file with some of the embedded metadata.
'LC81120792014026ASA00_B10.IMD': 'optional',
'LC81120792014026ASA00_B10.TIF': '',
'LC81120792014026ASA00_B11.IMD': 'optional',
'LC81120792014026ASA00_B11.TIF': '',
'LC81120792014026ASA00_B1.IMD': 'optional',
'LC81120792014026ASA00_B1.TIF': '',
'LC81120792014026ASA00_B2.IMD': 'optional',
'LC81120792014026ASA00_B2.TIF': '',
'LC81120792014026ASA00_B3.IMD': 'optional',
'LC81120792014026ASA00_B3.TIF': '',
'LC81120792014026ASA00_B4.IMD': 'optional',
'LC81120792014026ASA00_B4.TIF': '',
'LC81120792014026ASA00_B5.IMD': 'optional',
'LC81120792014026ASA00_B5.TIF': '',
'LC81120792014026ASA00_B6.IMD': 'optional',
'LC81120792014026ASA00_B6.TIF': '',
'LC81120792014026ASA00_B7.IMD': 'optional',
'LC81120792014026ASA00_B7.TIF': '',
'LC81120792014026ASA00_B8.IMD': 'optional',
'LC81120792014026ASA00_B8.TIF': '',
'LC81120792014026ASA00_B9.IMD': 'optional',
'LC81120792014026ASA00_B9.TIF': '',
'LC81120792014026ASA00_BQA.IMD': 'optional',
'LC81120792014026ASA00_BQA.TIF': '',
'LC81120792014026ASA00_GCP.txt': '',
'LC81120792014026ASA00_MTL.txt': '',
'LO8_20140126_112_079_L1T.xml': '',
},
'additional': {
'work_order.xml': '',
'lpgs_out.xml': '',
'20141201_20010425_B6_gqa_results.yaml': ''
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
# TODO: Check metadata fields are sensible.
output_metadata_path = output_dataset.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
prepare_datasets_for_comparison(
EXPECTED_METADATA,
md,
ancil_files,
output_dataset.joinpath('product')
)
assert_same(md, EXPECTED_METADATA)
# TODO: Asset all files are checksummed.
output_checksum_path = output_dataset.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
expected_filenames = sorted(f for f in as_file_list(output_dataset) if f != 'package.sha1')
assert set(checksummed_filenames) == set(expected_filenames)
assert checksummed_filenames == expected_filenames
def test_metadata():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset), 'raw',
str(source_dataset),
str(output_path)
])
assert_file_structure(
output_path, {
'NPP_VIIRS_STD-HDF5_P00_18966.ASA_0_0_20150626T053709Z20150626T055046':
{
'product': {
'RNSCA-RVIRS_npp_d20150626_t0537097_e0549423_b18966_'
'c20150626055046759000_nfts_drl.h5':
'',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
output_path = output_path.joinpath(
'NPP_VIIRS_STD-HDF5_P00_18966.ASA_0_0_20150626T053709Z20150626T055046')
# TODO: Check metadata fields are sensible.
output_metadata_path = output_path.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
import sys
sys.stderr.write('\n\n\n\n%r\n\n\n' % md)
expected = {
'ga_label':
'NPP_VIIRS_STD-HDF5_P00_18966.ASA_0_0_20150626T053709Z20150626T055046',
'image': {
'bands': {}
},
'size_bytes':
0,
'creation_dt':
datetime.datetime.utcfromtimestamp(source_dataset.stat().st_ctime),
'id':
None,
'platform': {
'code': 'NPP'
},
'instrument': {
'name': 'VIIRS'
},
'ga_level':
'P00',
'format': {
'name': 'HDF5'
},
'checksum_path':
'package.sha1',
'product_type':
'satellite_telemetry_data',
'acquisition': {
'groundstation': {
'eods_domain_code': '002',
'label': 'Alice Springs',
'code': 'ASA'
},
'platform_orbit': 18966,
'los': datetime.datetime(2015, 6, 26, 5, 50, 46),
'aos': datetime.datetime(2015, 6, 26, 5, 37, 9)
},
'lineage': {
'machine': {},
'source_datasets': {}
},
}
add_default_software_versions(expected)
assert_same(md, expected)
# Check all files are listed in checksum file.
output_checksum_path = output_path.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'ga-metadata.yaml',
'product/RNSCA-RVIRS_npp_d20150626_t0537097_e0549423_b18966_'
'c20150626055046759000_nfts_drl.h5',
]
def test_metadata():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset), 'raw',
str(source_dataset),
str(output_path)
])
# EODS LS7 dataset id:
# 'LS7_ETM_STD-RCC_P00_LANDSAT-7.65771.ALSP_0_0_20110827T021036Z20110827T021707'
# ... slightly different to NCI?
assert_file_structure(
output_path, {
'LS7_ETM_STD-RCC_P00_L7EB2011239021036ASA111_0_0_20110827T021036Z20110827T021707':
{
'product': {
'ephem.log': '',
'acs.log': '',
'L7EB2011239021036ASA111Q.data': '',
'passinfo': '',
'L7EB2011239021036ASA111I.data': '',
'ref.log': '',
'demod.log': ''
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
output_path = output_path.joinpath(
'LS7_ETM_STD-RCC_P00_L7EB2011239021036ASA111_0_0_20110827T021036Z20110827T021707'
)
# TODO: Check metadata fields are sensible.
output_metadata_path = output_path.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
expected = {
'id':
None,
'size_bytes':
164368,
'platform': {
'code': 'LANDSAT_7'
},
'instrument': {
'operation_mode': 'BUMPER',
'name': 'ETM'
},
'ga_level':
'P00',
'usgs': {
'interval_id': 'L7EB2011239021036ASA111'
},
'product_type':
'satellite_telemetry_data',
'format': {
'name': 'RCC'
},
# Default creation date is the same as the input folder ctime.
'creation_dt':
datetime.datetime.utcfromtimestamp(source_dataset.stat().st_ctime),
'ga_label':
'LS7_ETM_STD-RCC_P00_L7EB2011239021036ASA111_0_0_'
'20110827T021036Z20110827T021707',
'acquisition': {
'aos': datetime.datetime(2011, 8, 27, 2, 10, 36),
'groundstation': {
'code': 'ASA',
'label': 'Alice Springs',
'eods_domain_code': '002'
},
'los': datetime.datetime(2011, 8, 27, 2, 17, 7),
'platform_orbit': 65771
},
'image': {
'bands': {}
},
'lineage': {
'source_datasets': {},
'machine': {}
},
'checksum_path':
'package.sha1'
}
add_default_software_versions(expected)
assert_same(md, expected)
# Check all files are listed in checksum file.
output_checksum_path = output_path.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'ga-metadata.yaml',
'product/L7EB2011239021036ASA111I.data',
'product/L7EB2011239021036ASA111Q.data',
'product/acs.log',
'product/demod.log',
'product/ephem.log',
'product/passinfo',
'product/ref.log',
]
def test_metadata():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset), 'raw',
str(source_dataset),
str(output_path)
])
# EODS LS7 dataset id:
# 'LS7_ETM_STD-RCC_P00_LANDSAT-7.65771.ALSP_0_0_20110827T021036Z20110827T021707'
# ... slightly different to NCI?
assert_file_structure(
output_path, {
'AQUA_MODIS_STD-PDS_P00_65208.S1A1C1D1R1_0_0_20140807T031628Z20140807T031630':
{
'product': {
'P1540064AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540064AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540141AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540141AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540157AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540157AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540261AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540261AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540262AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540262AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540290AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540290AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540342AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540342AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540402AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540402AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540404AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540404AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540405AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540405AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540406AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540406AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540407AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540407AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540414AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540414AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540415AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540415AAAAAAAAAAAAAA14219032341001.PDS': '',
'P1540957AAAAAAAAAAAAAA14219032341000.PDS': '',
'P1540957AAAAAAAAAAAAAA14219032341001.PDS': '',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
output_path = output_path.joinpath(
'AQUA_MODIS_STD-PDS_P00_65208.S1A1C1D1R1_0_0_20140807T031628Z20140807T031630'
)
# TODO: Check metadata fields are sensible.
output_metadata_path = output_path.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
expected = {
'lineage': {
'machine': {},
'source_datasets': {}
},
'product_type':
'satellite_telemetry_data',
'format': {
'name': 'PDS'
},
'image': {
'bands': {},
'day_percentage_estimate': 100.0
},
# Default creation date is the same as the input folder ctime.
'creation_dt':
datetime.datetime.utcfromtimestamp(source_dataset.stat().st_ctime),
'rms_string':
'S1A1C1D1R1',
'instrument': {
'name': 'MODIS'
},
'ga_label':
'AQUA_MODIS_STD-PDS_P00_65208.S1A1C1D1R1_0_0_20140807T031628Z20140807T031630',
'platform': {
'code': 'AQUA'
},
'size_bytes':
2144280,
'checksum_path':
'package.sha1',
'id':
None,
'acquisition': {
'los': datetime.datetime(2014, 8, 7, 3, 16, 30, 228023),
'platform_orbit': 65208,
'aos': datetime.datetime(2014, 8, 7, 3, 16, 28, 750910)
}
}
add_default_software_versions(expected)
assert_same(md, expected)
# Check all files are listed in checksum file.
output_checksum_path = output_path.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'ga-metadata.yaml',
'product/P1540064AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540064AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540141AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540141AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540157AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540157AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540261AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540261AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540262AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540262AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540290AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540290AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540342AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540342AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540402AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540402AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540404AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540404AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540405AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540405AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540406AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540406AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540407AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540407AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540414AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540414AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540415AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540415AAAAAAAAAAAAAA14219032341001.PDS',
'product/P1540957AAAAAAAAAAAAAA14219032341000.PDS',
'product/P1540957AAAAAAAAAAAAAA14219032341001.PDS',
]
def test_package():
work_path = temp_dir()
output_path = work_path.joinpath('out')
output_path.mkdir(parents=True)
ancil_base = work_path.joinpath('ancil')
# We have to override the ancillary directory lookup as they won't exist on test systems.
ancil_files = (
FakeAncilFile(ancil_base, 'cpf', 'LE07CPF_20110101_20110331_01.02'),
FakeAncilFile(ancil_base, 'ephemeris', 'L72013231ASADEF.S00'),
)
work_order = prepare_work_order(ancil_files, wo_template)
# Run!
args = [
hardlink_arg(output_path, source_dataset), 'level1',
'--newly-processed', '--parent',
str(parent_dataset), '--add-file',
str(work_order)
]
for additional_file in additional_files.iterdir():
args.extend(['--add-file', str(additional_file)])
args.extend([str(source_dataset), str(output_path)])
run_packaging_cli(args)
output_dataset = output_path.joinpath(
'LS7_ETM_SYS_P31_GALPGS01-002_092_082_20110214')
assert_file_structure(
output_path, {
'LS7_ETM_SYS_P31_GALPGS01-002_092_082_20110214': {
'browse.jpg': '',
'browse.fr.jpg': '',
'product': {
'LE07_L1GS_092082_20110214_20170221_01_T2_ANG.txt': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B1.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B2.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B3.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B4.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B5.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B6_VCID_1.TIF':
'',
'LE07_L1GS_092082_20110214_20170221_01_T2_B6_VCID_2.TIF':
'',
'LE07_L1GS_092082_20110214_20170221_01_T2_B7.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B8.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_BQA.TIF': '',
'LE07_L1GS_092082_20110214_20170221_01_T2_B1.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B2.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B3.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B4.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B5.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B6_VCID_1.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B6_VCID_2.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B7.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_B8.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_BQA.IMD':
'optional',
'LE07_L1GS_092082_20110214_20170221_01_T2_MTL.txt': '',
'LE7_20110214_092_082_L1GS.xml': '',
},
'additional': {
'work_order.xml': '',
'20130818_20000119_B5_gqa_results.yaml': '',
'lpgs_out.xml': '',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
# TODO: Check metadata fields are sensible.
output_metadata_path = output_dataset.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
prepare_datasets_for_comparison(EXPECTED_METADATA, md, ancil_files,
output_dataset.joinpath('product'))
assert_same(md, EXPECTED_METADATA)
# TODO: Asset all files are checksummed.
output_checksum_path = output_dataset.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
expected_filenames = sorted(f for f in as_file_list(output_dataset)
if f != 'package.sha1')
assert set(checksummed_filenames) == set(expected_filenames)
assert checksummed_filenames == expected_filenames
def test_metadata():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset), 'raw',
str(source_dataset),
str(output_path)
])
assert_file_structure(
output_path, {
'LS8_OLITIRS_STD-MD_P00_LC81140740812015123LGN00_114_074-081_'
'20150503T031224Z20150503T031438': {
'product': {
'270.000.2015123031324364.LGS': '',
'271.000.2015123031330204.LGS': '',
'271.001.2015123031352904.LGS': '',
'271.002.2015123031415490.LGS': '',
'271.003.2015123031438105.LGS': '',
'LC81140740812015123LGN00_IDF.xml': '',
'LC81140740812015123LGN00_MD5.txt': '',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
output_path = output_path.joinpath(
'LS8_OLITIRS_STD-MD_P00_LC81140740812015123LGN00_114_074-081_'
'20150503T031224Z20150503T031438')
# TODO: Check metadata fields are sensible.
output_metadata_path = output_path.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
expected = {
'id':
None,
'ga_label':
'LS8_OLITIRS_STD-MD_P00_LC81140740812015123LGN00_114_074-081_'
'20150503T031224Z20150503T031438',
# Default creation date is the same as the input folder ctime.
'creation_dt':
datetime.datetime.utcfromtimestamp(source_dataset.stat().st_ctime),
'size_bytes':
4485,
'product_type':
'satellite_telemetry_data',
'usgs': {
'interval_id': 'LC81140740812015123LGN00'
},
'format': {
'name': 'MD'
},
'ga_level':
'P00',
'checksum_path':
'package.sha1',
'platform': {
'code': 'LANDSAT_8'
},
'instrument': {
'name': 'OLI_TIRS'
},
'acquisition': {
'los': datetime.datetime(2015, 5, 3, 3, 14, 38, 105000),
'aos': datetime.datetime(2015, 5, 3, 3, 12, 24, 364000),
'groundstation': {
'code': 'LGN',
'label': 'Landsat Ground Network',
'eods_domain_code': '032'
}
},
'image': {
'satellite_ref_point_start': {
'x': 114,
'y': 74
},
'satellite_ref_point_end': {
'x': 114,
'y': 81
},
'bands': {},
},
'lineage': {
'source_datasets': {},
'machine': {}
}
}
add_default_software_versions(expected)
assert_same(md, expected)
# Check all files are listed in checksum file.
output_checksum_path = output_path.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'ga-metadata.yaml',
'product/270.000.2015123031324364.LGS',
'product/271.000.2015123031330204.LGS',
'product/271.001.2015123031352904.LGS',
'product/271.002.2015123031415490.LGS',
'product/271.003.2015123031438105.LGS',
'product/LC81140740812015123LGN00_IDF.xml',
'product/LC81140740812015123LGN00_MD5.txt',
]
def test_metadata():
output_path = temp_dir()
run_packaging_cli([
hardlink_arg(output_path, source_dataset), 'raw',
str(source_dataset),
str(output_path)
])
assert_file_structure(
output_path, {
'LS5_TM_STD-RCC_P00_L5TB2011240002022ASA123_0_0_20110828T002022Z20110828T002858':
{
'product': {
'L5TB2011240002022ASA123I00.data': '',
'acs.log': '',
'demod.log': '',
'ephem.log': '',
'passinfo': '',
'ref.log': '',
},
'ga-metadata.yaml': '',
'package.sha1': ''
}
})
output_path = output_path.joinpath(
'LS5_TM_STD-RCC_P00_L5TB2011240002022ASA123_0_0_20110828T002022Z20110828T002858'
)
# TODO: Check metadata fields are sensible.
output_metadata_path = output_path.joinpath('ga-metadata.yaml')
assert output_metadata_path.exists()
md = yaml.load(output_metadata_path.open('r'))
# ID is different every time: check not none, and clear it.
assert md['id'] is not None
md['id'] = None
expected = {
'id':
None,
'ga_level':
'P00',
# Default creation date is the same as the input folder ctime.
'creation_dt':
datetime.datetime.utcfromtimestamp(source_dataset.stat().st_ctime),
'platform': {
'code': 'LANDSAT_5'
},
'format': {
'version': 0,
'name': 'RCC'
},
'size_bytes':
226667,
'product_type':
'satellite_telemetry_data',
'usgs': {
'interval_id': 'L5TB2011240002022ASA123'
},
'instrument': {
'name': 'TM',
'operation_mode': 'BUMPER'
},
'acquisition': {
'aos': datetime.datetime(2011, 8, 28, 0, 20, 22),
'los': datetime.datetime(2011, 8, 28, 0, 28, 58),
'platform_orbit': 146212,
'groundstation': {
'eods_domain_code': '002',
'label': 'Alice Springs',
'code': 'ASA'
},
},
'ga_label':
'LS5_TM_STD-RCC_P00_L5TB2011240002022ASA123_0_0_'
'20110828T002022Z20110828T002858',
'checksum_path':
'package.sha1',
'lineage': {
'machine': {},
'source_datasets': {}
},
'image': {
'bands': {}
}
}
add_default_software_versions(expected)
assert_same(md, expected)
# Check all files are listed in checksum file.
output_checksum_path = output_path.joinpath('package.sha1')
assert output_checksum_path.exists()
checksummed_filenames = load_checksum_filenames(output_checksum_path)
assert checksummed_filenames == [
'ga-metadata.yaml',
'product/L5TB2011240002022ASA123I00.data',
'product/acs.log',
'product/demod.log',
'product/ephem.log',
'product/passinfo',
'product/ref.log',
]