def test_dataset_add(dataset_add_configs, index_empty, clirunner):
p = dataset_add_configs
index = index_empty
r = clirunner(['dataset', 'add', p.datasets], expect_success=False)
assert r.exit_code != 0
assert 'Found no products' in r.output
clirunner(['metadata_type', 'add', p.metadata])
clirunner(['product', 'add', p.products])
clirunner(['dataset', 'add', p.datasets])
clirunner(['dataset', 'add', p.datasets_bad1])
ds = load_dataset_definition(p.datasets)
ds_bad1 = load_dataset_definition(p.datasets_bad1)
r = clirunner(['dataset', 'search'], expect_success=True)
assert ds.id in r.output
assert ds_bad1.id not in r.output
assert ds.sources['ab'].id in r.output
assert ds.sources['ac'].sources['cd'].id in r.output
r = clirunner(['dataset', 'info', '-f', 'csv', ds.id])
assert ds.id in r.output
r = clirunner(['dataset', 'info', '-f', 'yaml', '--show-sources', ds.id])
assert ds.sources['ae'].id in r.output
r = clirunner([
'dataset', 'info', '-f', 'yaml', '--show-derived', ds.sources['ae'].id
])
assert ds.id in r.output
ds_ = SimpleDocNav(gen_dataset_test_dag(1, force_tree=True))
assert ds_.id == ds.id
x = index.datasets.get(ds.id, include_sources=True)
assert str(x.sources['ab'].id) == ds.sources['ab'].id
assert str(
x.sources['ac'].sources['cd'].id) == ds.sources['ac'].sources['cd'].id
check_skip_lineage_test(clirunner, index)
check_no_product_match(clirunner, index)
check_with_existing_lineage(clirunner, index)
check_inconsistent_lineage(clirunner, index)
check_missing_metadata_doc(clirunner)
check_missing_lineage(clirunner, index)
check_no_confirm(clirunner, p.datasets)
check_bad_yaml(clirunner, index)
# check --product=nosuchproduct
r = clirunner(['dataset', 'add', '--product', 'nosuchproduct', p.datasets],
expect_success=False)
assert "ERROR Supplied product name" in r.output
assert r.exit_code != 0
# Check that deprecated option is accepted
r = clirunner(['dataset', 'add', '--auto-match', p.datasets])
assert 'WARNING --auto-match option is deprecated' in r.output
def test_dataset_archive_restore(dataset_add_configs, index_empty, clirunner):
p = dataset_add_configs
index = index_empty
clirunner(['metadata', 'add', p.metadata])
clirunner(['product', 'add', p.products])
clirunner(['dataset', 'add', p.datasets])
ds = load_dataset_definition(p.datasets)
assert index.datasets.has(ds.id) is True
# First do dry run
r = clirunner(['dataset', 'archive', '--dry-run', ds.id])
r = clirunner([
'dataset', 'archive', '--dry-run', '--archive-derived',
ds.sources['ae'].id
])
assert ds.id in r.output
assert ds.sources['ae'].id in r.output
assert index.datasets.has(ds.id) is True
# Run for real
r = clirunner(['dataset', 'archive', ds.id])
r = clirunner(['dataset', 'info', ds.id])
assert 'status: archived' in r.output
# restore dry run
r = clirunner(['dataset', 'restore', '--dry-run', ds.id])
r = clirunner(['dataset', 'info', ds.id])
assert 'status: archived' in r.output
# restore for real
r = clirunner(['dataset', 'restore', ds.id])
r = clirunner(['dataset', 'info', ds.id])
assert 'status: active' in r.output
# archive derived
d_id = ds.sources['ac'].sources['cd'].id
r = clirunner(['dataset', 'archive', '--archive-derived', d_id])
r = clirunner(
['dataset', 'info', ds.id, ds.sources['ab'].id, ds.sources['ac'].id])
assert 'status: active' not in r.output
assert 'status: archived' in r.output
# restore derived
r = clirunner(['dataset', 'restore', '--restore-derived', d_id])
r = clirunner(
['dataset', 'info', ds.id, ds.sources['ab'].id, ds.sources['ac'].id])
assert 'status: active' in r.output
assert 'status: archived' not in r.output
def dataset_archive_prep(dataset_add_configs, index_empty, clirunner):
p = dataset_add_configs
index = index_empty
clirunner(['metadata', 'add', p.metadata])
clirunner(['product', 'add', p.products])
clirunner(['dataset', 'add', p.datasets])
ds = load_dataset_definition(p.datasets)
assert index.datasets.has(ds.id) is True
return p, index, ds
def test_dataset_add_http(dataset_add_configs, index: Index, default_metadata_type: MetadataType, httpserver,
clirunner):
# pytest-localserver also looks good, it's been around for ages, but httpserver is the new cool
p = dataset_add_configs
httpserver.expect_request('/metadata_types.yaml').respond_with_data(open(p.metadata).read())
httpserver.expect_request('/products.yaml').respond_with_data(open(p.products).read())
httpserver.expect_request('/datasets.yaml').respond_with_data(open(p.datasets).read())
# check that the request is served
# assert requests.get(httpserver.url_for("/dataset.yaml")).yaml() == {'foo': 'bar'}
clirunner(['metadata', 'add', httpserver.url_for('/metadata_types.yaml')])
clirunner(['product', 'add', httpserver.url_for('/products.yaml')])
# clirunner(['dataset', 'add', p.datasets])
clirunner(['dataset', 'add', httpserver.url_for('/datasets.yaml')])
ds = load_dataset_definition(p.datasets)
assert index.datasets.has(ds.id)
def test_dataset_add(dataset_add_configs, index_empty, clirunner):
p = dataset_add_configs
index = index_empty
r = clirunner(['dataset', 'add', p.datasets], expect_success=False)
assert r.exit_code != 0
assert 'Found no products' in r.output
clirunner(['metadata', 'add', p.metadata])
clirunner(['product', 'add', p.products])
clirunner(['dataset', 'add', p.datasets])
clirunner(['dataset', 'add', p.datasets_bad1])
clirunner(['dataset', 'add', p.datasets_eo3])
ds = load_dataset_definition(p.datasets)
ds_bad1 = load_dataset_definition(p.datasets_bad1)
# Check .hl.Doc2Dataset
doc2ds = Doc2Dataset(index)
_ds, _err = doc2ds(ds.doc, 'file:///something')
assert _err is None
assert str(_ds.id) == ds.id
assert _ds.metadata_doc == ds.doc
# Check dataset search
r = clirunner(['dataset', 'search'], expect_success=True)
assert ds.id in r.output
assert ds_bad1.id not in r.output
assert ds.sources['ab'].id in r.output
assert ds.sources['ac'].sources['cd'].id in r.output
r = clirunner(['dataset', 'info', '-f', 'csv', ds.id])
assert ds.id in r.output
r = clirunner(['dataset', 'info', '-f', 'yaml', '--show-sources', ds.id])
assert ds.sources['ae'].id in r.output
r = clirunner([
'dataset', 'info', '-f', 'yaml', '--show-derived', ds.sources['ae'].id
])
assert ds.id in r.output
ds_ = SimpleDocNav(gen_dataset_test_dag(1, force_tree=True))
assert ds_.id == ds.id
x = index.datasets.get(ds.id, include_sources=True)
assert str(x.sources['ab'].id) == ds.sources['ab'].id
assert str(
x.sources['ac'].sources['cd'].id) == ds.sources['ac'].sources['cd'].id
check_skip_lineage_test(clirunner, index)
check_no_product_match(clirunner, index)
check_with_existing_lineage(clirunner, index)
check_inconsistent_lineage(clirunner, index)
check_missing_metadata_doc(clirunner)
check_missing_lineage(clirunner, index)
check_no_confirm(clirunner, p.datasets)
check_bad_yaml(clirunner, index)
# check --product=nosuchproduct
r = clirunner(['dataset', 'add', '--product', 'nosuchproduct', p.datasets],
expect_success=False)
assert "ERROR Supplied product name" in r.output
assert r.exit_code != 0
# Check that deprecated option is accepted
r = clirunner(['dataset', 'add', '--auto-match', p.datasets])
assert 'WARNING --auto-match option is deprecated' in r.output
# test dataset add eo3
r = clirunner(['dataset', 'add', p.datasets_eo3])
assert r.exit_code == 0
ds_eo3 = load_dataset_definition(p.datasets_eo3)
assert ds_eo3.location is not None
_ds = index.datasets.get(ds_eo3.id, include_sources=True)
assert sorted(_ds.sources) == ['a', 'bc1', 'bc2']
assert _ds.crs == 'EPSG:3857'
assert _ds.extent is not None
assert _ds.extent.crs == _ds.crs
assert _ds.uris == [ds_eo3.location]
assert 'location' not in _ds.metadata_doc
def ls5_dataset_w_children(index, clirunner, example_ls5_dataset_path, indexed_ls5_scene_products):
clirunner(['dataset', 'add', str(example_ls5_dataset_path)])
doc = load_dataset_definition(example_ls5_dataset_path)
return index.datasets.get(doc.id, include_sources=True)
