def test_dataset_add(dataset_add_configs, index_empty, clirunner): p = dataset_add_configs index = index_empty r = clirunner(['dataset', 'add', p.datasets], expect_success=False) assert r.exit_code != 0 assert 'Found no products' in r.output clirunner(['metadata_type', 'add', p.metadata]) clirunner(['product', 'add', p.products]) clirunner(['dataset', 'add', p.datasets]) clirunner(['dataset', 'add', p.datasets_bad1]) ds = load_dataset_definition(p.datasets) ds_bad1 = load_dataset_definition(p.datasets_bad1) r = clirunner(['dataset', 'search'], expect_success=True) assert ds.id in r.output assert ds_bad1.id not in r.output assert ds.sources['ab'].id in r.output assert ds.sources['ac'].sources['cd'].id in r.output r = clirunner(['dataset', 'info', '-f', 'csv', ds.id]) assert ds.id in r.output r = clirunner(['dataset', 'info', '-f', 'yaml', '--show-sources', ds.id]) assert ds.sources['ae'].id in r.output r = clirunner([ 'dataset', 'info', '-f', 'yaml', '--show-derived', ds.sources['ae'].id ]) assert ds.id in r.output ds_ = SimpleDocNav(gen_dataset_test_dag(1, force_tree=True)) assert ds_.id == ds.id x = index.datasets.get(ds.id, include_sources=True) assert str(x.sources['ab'].id) == ds.sources['ab'].id assert str( x.sources['ac'].sources['cd'].id) == ds.sources['ac'].sources['cd'].id check_skip_lineage_test(clirunner, index) check_no_product_match(clirunner, index) check_with_existing_lineage(clirunner, index) check_inconsistent_lineage(clirunner, index) check_missing_metadata_doc(clirunner) check_missing_lineage(clirunner, index) check_no_confirm(clirunner, p.datasets) check_bad_yaml(clirunner, index) # check --product=nosuchproduct r = clirunner(['dataset', 'add', '--product', 'nosuchproduct', p.datasets], expect_success=False) assert "ERROR Supplied product name" in r.output assert r.exit_code != 0 # Check that deprecated option is accepted r = clirunner(['dataset', 'add', '--auto-match', p.datasets]) assert 'WARNING --auto-match option is deprecated' in r.output
def test_dataset_archive_restore(dataset_add_configs, index_empty, clirunner): p = dataset_add_configs index = index_empty clirunner(['metadata', 'add', p.metadata]) clirunner(['product', 'add', p.products]) clirunner(['dataset', 'add', p.datasets]) ds = load_dataset_definition(p.datasets) assert index.datasets.has(ds.id) is True # First do dry run r = clirunner(['dataset', 'archive', '--dry-run', ds.id]) r = clirunner([ 'dataset', 'archive', '--dry-run', '--archive-derived', ds.sources['ae'].id ]) assert ds.id in r.output assert ds.sources['ae'].id in r.output assert index.datasets.has(ds.id) is True # Run for real r = clirunner(['dataset', 'archive', ds.id]) r = clirunner(['dataset', 'info', ds.id]) assert 'status: archived' in r.output # restore dry run r = clirunner(['dataset', 'restore', '--dry-run', ds.id]) r = clirunner(['dataset', 'info', ds.id]) assert 'status: archived' in r.output # restore for real r = clirunner(['dataset', 'restore', ds.id]) r = clirunner(['dataset', 'info', ds.id]) assert 'status: active' in r.output # archive derived d_id = ds.sources['ac'].sources['cd'].id r = clirunner(['dataset', 'archive', '--archive-derived', d_id]) r = clirunner( ['dataset', 'info', ds.id, ds.sources['ab'].id, ds.sources['ac'].id]) assert 'status: active' not in r.output assert 'status: archived' in r.output # restore derived r = clirunner(['dataset', 'restore', '--restore-derived', d_id]) r = clirunner( ['dataset', 'info', ds.id, ds.sources['ab'].id, ds.sources['ac'].id]) assert 'status: active' in r.output assert 'status: archived' not in r.output
def dataset_archive_prep(dataset_add_configs, index_empty, clirunner): p = dataset_add_configs index = index_empty clirunner(['metadata', 'add', p.metadata]) clirunner(['product', 'add', p.products]) clirunner(['dataset', 'add', p.datasets]) ds = load_dataset_definition(p.datasets) assert index.datasets.has(ds.id) is True return p, index, ds
def test_dataset_add_http(dataset_add_configs, index: Index, default_metadata_type: MetadataType, httpserver, clirunner): # pytest-localserver also looks good, it's been around for ages, but httpserver is the new cool p = dataset_add_configs httpserver.expect_request('/metadata_types.yaml').respond_with_data(open(p.metadata).read()) httpserver.expect_request('/products.yaml').respond_with_data(open(p.products).read()) httpserver.expect_request('/datasets.yaml').respond_with_data(open(p.datasets).read()) # check that the request is served # assert requests.get(httpserver.url_for("/dataset.yaml")).yaml() == {'foo': 'bar'} clirunner(['metadata', 'add', httpserver.url_for('/metadata_types.yaml')]) clirunner(['product', 'add', httpserver.url_for('/products.yaml')]) # clirunner(['dataset', 'add', p.datasets]) clirunner(['dataset', 'add', httpserver.url_for('/datasets.yaml')]) ds = load_dataset_definition(p.datasets) assert index.datasets.has(ds.id)
def test_dataset_add(dataset_add_configs, index_empty, clirunner): p = dataset_add_configs index = index_empty r = clirunner(['dataset', 'add', p.datasets], expect_success=False) assert r.exit_code != 0 assert 'Found no products' in r.output clirunner(['metadata', 'add', p.metadata]) clirunner(['product', 'add', p.products]) clirunner(['dataset', 'add', p.datasets]) clirunner(['dataset', 'add', p.datasets_bad1]) clirunner(['dataset', 'add', p.datasets_eo3]) ds = load_dataset_definition(p.datasets) ds_bad1 = load_dataset_definition(p.datasets_bad1) # Check .hl.Doc2Dataset doc2ds = Doc2Dataset(index) _ds, _err = doc2ds(ds.doc, 'file:///something') assert _err is None assert str(_ds.id) == ds.id assert _ds.metadata_doc == ds.doc # Check dataset search r = clirunner(['dataset', 'search'], expect_success=True) assert ds.id in r.output assert ds_bad1.id not in r.output assert ds.sources['ab'].id in r.output assert ds.sources['ac'].sources['cd'].id in r.output r = clirunner(['dataset', 'info', '-f', 'csv', ds.id]) assert ds.id in r.output r = clirunner(['dataset', 'info', '-f', 'yaml', '--show-sources', ds.id]) assert ds.sources['ae'].id in r.output r = clirunner([ 'dataset', 'info', '-f', 'yaml', '--show-derived', ds.sources['ae'].id ]) assert ds.id in r.output ds_ = SimpleDocNav(gen_dataset_test_dag(1, force_tree=True)) assert ds_.id == ds.id x = index.datasets.get(ds.id, include_sources=True) assert str(x.sources['ab'].id) == ds.sources['ab'].id assert str( x.sources['ac'].sources['cd'].id) == ds.sources['ac'].sources['cd'].id check_skip_lineage_test(clirunner, index) check_no_product_match(clirunner, index) check_with_existing_lineage(clirunner, index) check_inconsistent_lineage(clirunner, index) check_missing_metadata_doc(clirunner) check_missing_lineage(clirunner, index) check_no_confirm(clirunner, p.datasets) check_bad_yaml(clirunner, index) # check --product=nosuchproduct r = clirunner(['dataset', 'add', '--product', 'nosuchproduct', p.datasets], expect_success=False) assert "ERROR Supplied product name" in r.output assert r.exit_code != 0 # Check that deprecated option is accepted r = clirunner(['dataset', 'add', '--auto-match', p.datasets]) assert 'WARNING --auto-match option is deprecated' in r.output # test dataset add eo3 r = clirunner(['dataset', 'add', p.datasets_eo3]) assert r.exit_code == 0 ds_eo3 = load_dataset_definition(p.datasets_eo3) assert ds_eo3.location is not None _ds = index.datasets.get(ds_eo3.id, include_sources=True) assert sorted(_ds.sources) == ['a', 'bc1', 'bc2'] assert _ds.crs == 'EPSG:3857' assert _ds.extent is not None assert _ds.extent.crs == _ds.crs assert _ds.uris == [ds_eo3.location] assert 'location' not in _ds.metadata_doc
def ls5_dataset_w_children(index, clirunner, example_ls5_dataset_path, indexed_ls5_scene_products): clirunner(['dataset', 'add', str(example_ls5_dataset_path)]) doc = load_dataset_definition(example_ls5_dataset_path) return index.datasets.get(doc.id, include_sources=True)