def test_index_dataset(): mock_db = MockDb() mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) ids = {d.metadata['id'] for d in mock_db.dataset.values()} assert ids == {_nbar_uuid, _ortho_uuid, _telemetry_uuid} # Three datasets (ours and the two embedded source datasets) assert len(mock_db.dataset) == 3 # Our three datasets should be linked together # Nbar -> Ortho -> Telemetry assert len(mock_db.dataset_source) == 2 assert mock_db.dataset_source == { ('ortho', _nbar_uuid, _ortho_uuid), ('satellite_telemetry_data', _ortho_uuid, _telemetry_uuid) } # Nothing ingested, because we reported the first as already ingested. dataset = datasets.add(_EXAMPLE_NBAR_DATASET) assert len(mock_db.dataset) == 3 assert len(mock_db.dataset_source) == 2 ds2 = deepcopy(_EXAMPLE_NBAR_DATASET) ds2.metadata_doc['product_type'] = 'zzzz' with pytest.raises(ValueError): dataset = datasets.add(ds2)
def test_index_dataset(): mock_db = MockDb() mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) ids = {d.id for d in mock_db.dataset.values()} assert ids == {_nbar_uuid, _ortho_uuid, _telemetry_uuid} # Three datasets (ours and the two embedded source datasets) assert len(mock_db.dataset) == 3 # Our three datasets should be linked together # Nbar -> Ortho -> Telemetry assert len(mock_db.dataset_source) == 2 assert mock_db.dataset_source == { ('ortho', _nbar_uuid, _ortho_uuid), ('satellite_telemetry_data', _ortho_uuid, _telemetry_uuid) } # Nothing ingested, because we reported the first as already ingested. dataset = datasets.add(_EXAMPLE_NBAR_DATASET) assert len(mock_db.dataset) == 3 assert len(mock_db.dataset_source) == 2 ds2 = deepcopy(_EXAMPLE_NBAR_DATASET) ds2.metadata_doc['product_type'] = 'zzzz' with pytest.raises(ValueError): dataset = datasets.add(ds2)
def test_index_already_ingested_source_dataset(): mock_db = MockDb() mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET.sources['ortho']) assert len(mock_db.dataset) == 2 assert len(mock_db.dataset_source) == 1 dataset = datasets.add(_EXAMPLE_NBAR_DATASET) assert len(mock_db.dataset) == 3 assert len(mock_db.dataset_source) == 2
def test_index_two_levels_already_ingested(): mock_db = MockDb() mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET.sources['ortho'].sources['satellite_telemetry_data']) assert len(mock_db.dataset) == 1 assert len(mock_db.dataset_source) == 0 dataset = datasets.add(_EXAMPLE_NBAR_DATASET) assert len(mock_db.dataset) == 3 assert len(mock_db.dataset_source) == 2
def test_index_already_ingested_dataset(): mock_db = MockDb() mock_db.already_ingested = {_ortho_uuid, _telemetry_uuid, _nbar_uuid} mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) # Nothing ingested, because we reported the first as already ingested. assert len(mock_db.dataset) == 0 assert len(mock_db.dataset_source) == 0
def add_dataset(pr, dt, metadict, file): """Add a dataset to the datacube database It's added to 2 tables: - dataset: with all the metadata - dataset_location Args: pr (ProductResource): A ProductResource object, contained in the return of ``add_product`` dt (DatasetType): A DatasetType object, contained in the return of ``add_product`` metadict (dict): Dictionary containing dataset metadata, generally generated by ``metadict_from_netcdf`` file (str): Path of the file to add to the index Return: No return, the function is used for its side effect of adding a dataset to the datacube """ db = PostgresDb.from_config(CONFIG) dataset_resource = DatasetResource(db, pr) dataset = Dataset(dt, metadict, sources={}) dataset_resource.add(dataset) uid = metadict['id'] dataset_resource.add_location(uid, file)
def test_index_already_ingested_source_dataset(): mock_db = MockDb() mock_db.already_ingested = {_ortho_uuid, _telemetry_uuid} mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) # Only the first dataset ingested assert len(mock_db.dataset) == 1 assert mock_db.dataset[0][1] == _nbar_uuid # It should have been linked to the ortho. assert len(mock_db.dataset_source) == 1 assert mock_db.dataset_source == { ('ortho', _nbar_uuid, _ortho_uuid), }
def test_index_dataset(): mock_db = MockDb() mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) ids = {d[0]['id'] for d in mock_db.dataset} assert ids == {_nbar_uuid, _ortho_uuid, _telemetry_uuid} # Three datasets (ours and the two embedded source datasets) assert len(mock_db.dataset) == 3 # Our three datasets should be linked together # Nbar -> Ortho -> Telemetry assert len(mock_db.dataset_source) == 2 assert mock_db.dataset_source == { ('ortho', _nbar_uuid, _ortho_uuid), ('satellite_telemetry_data', _ortho_uuid, _telemetry_uuid) }
def test_index_two_levels_already_ingested(): mock_db = MockDb() # RAW was already ingested. mock_db.already_ingested = {_telemetry_uuid} mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE) datasets = DatasetResource(mock_db, mock_types) dataset = datasets.add(_EXAMPLE_NBAR_DATASET) ids = {d[0]['id'] for d in mock_db.dataset} assert ids == {_nbar_uuid, _ortho_uuid} # Two datasets (the telemetry data already ingested) assert len(mock_db.dataset) == 2 # Our three datasets should be linked together # Nbar -> Ortho -> Telemetry assert len(mock_db.dataset_source) == 2 assert mock_db.dataset_source == { ('ortho', _nbar_uuid, _ortho_uuid), ('satellite_telemetry_data', _ortho_uuid, _telemetry_uuid) }