def test_publish_update_locally_is_true(pytest_dataset: Metadata): date_before = pytest_dataset.local_metadata.get('metadata_modified') # update local metadata new_metadata = pytest_dataset.local_metadata.copy() # generate random strings with 3 characters random_string = "".join( random.choice(string.ascii_uppercase) for _ in range(3)) # update metadata tags with random_string new_metadata["tags"] = [random_string] ryaml.dump(new_metadata, open(new_metadata.filepath, "w", encoding="utf-8")) # publish changes pytest_dataset.publish(update_locally=True) # get new tags from local metadata new_tags = pytest_dataset.local_metadata.get('tags') # get new `metadata_modified` value from local config file date_after = pytest_dataset.local_metadata.get('metadata_modified') assert new_tags == [random_string], "Tags were not updated locally" assert date_after > date_before, "Date after should be greater than date before"
def test_publish_all_is_true( pytest_dataset: Metadata, pytest_table: Metadata, ): res = pytest_table.publish(all=True) assert isinstance(res, dict) assert res != {} assert pytest_dataset.exists_in_ckan()
def test_validate_is_not_succesful( invalid_dataset_metadata: Metadata, invalid_table_metadata: Metadata, ): with pytest.raises(BaseDosDadosException): invalid_table_metadata.validate() with pytest.raises(BaseDosDadosException): invalid_dataset_metadata.validate()
def test_publish_is_not_successful( invalid_dataset_metadata: Metadata, invalid_table_metadata: Metadata, ): with pytest.raises(AssertionError, match="Could not publish"): invalid_dataset_metadata.publish() with pytest.raises(BaseDosDadosException, match="Could not publish"): invalid_table_metadata.publish()
def valid_metadata_table(metadatadir): table_metadata = Metadata( dataset_id="br_ibge_pib", table_id="municipio", metadata_path=metadatadir, ) table_metadata.create(if_exists="replace") dataset_metadata.CKAN_API_KEY = "valid-key" return table_metadata
def out_of_date_metadata_obj(metadatadir): out_of_date_metadata = Metadata(dataset_id="br_me_caged", metadata_path=metadatadir) out_of_date_metadata.create(if_exists="replace") out_of_date_config = out_of_date_metadata.local_metadata out_of_date_config["metadata_modified"] = "old_date" ryaml.dump(out_of_date_config, open(out_of_date_metadata.filepath, "w", encoding="utf-8")) return out_of_date_metadata
def test_create_partition_columns_from_user_input( existent_metadata: Metadata, existent_metadata_path: Path, ): shutil.rmtree(existent_metadata_path, ignore_errors=True) existent_metadata.create(partition_columns=["id_municipio"]) assert existent_metadata_path.exists() metadata_dict = existent_metadata.local_metadata assert metadata_dict.get("partitions") == "id_municipio"
def test_create_partition_columns_from_existent_table( existent_metadata: Metadata, existent_metadata_path: Path, ): shutil.rmtree(existent_metadata_path, ignore_errors=True) existent_metadata.create() assert existent_metadata_path.exists() metadata_dict = existent_metadata.local_metadata assert metadata_dict.get("partitions") == "ano, sigla_uf, id_municipio"
def updated_metadata_obj(metadatadir): updated_metadata = Metadata(dataset_id="br_me_caged", metadata_path=metadatadir) updated_metadata.create(if_exists="replace") updated_config = updated_metadata.local_metadata updated_config["metadata_modified"] = updated_metadata.ckan_metadata[ "metadata_modified"] ryaml.dump(updated_config, open(updated_metadata.filepath, "w", encoding="utf-8")) return updated_metadata
def test_create_force_columns_is_false( existent_metadata: Metadata, existent_metadata_path: Path, ): shutil.rmtree(existent_metadata_path, ignore_errors=True) existent_metadata.create(columns=["column1", "column2"], force_columns=False) assert (existent_metadata_path / METADATA_FILES["table"]).exists() table_metadata_dict = existent_metadata.local_metadata assert table_metadata_dict["columns"][0]["name"] != "column1" assert table_metadata_dict["columns"][1]["name"] != "column2"
def pytest_dataset(metadatadir): shutil.rmtree(metadatadir, ignore_errors=True) pytest_dataset = Metadata(dataset_id="pytest", metadata_path=metadatadir) pytest_dataset.create(if_exists="replace") # fill dataset metadata for it to be publishable pytest_dataset_metadata = pytest_dataset.local_metadata pytest_dataset_metadata["organization"] = "acaps" # set valid organization # materialize metadata file ryaml.dump(pytest_dataset_metadata, open(pytest_dataset.filepath, "w", encoding="utf-8")) return pytest_dataset
def invalid_organization_dataset(metadatadir): invalid_organization_dataset = Metadata( dataset_id="br_ibge_pib", metadata_path=metadatadir, ) invalid_organization_dataset.create(if_exists="replace") invalid_config = invalid_organization_dataset.local_metadata invalid_config["organization"] = "not-a-valid-organization" with open(invalid_organization_dataset.filepath, "w", encoding="utf-8") as file: ryaml.dump(invalid_config, file) return invalid_organization_dataset
def existent_metadata(metadatadir): table_metadata_obj = Metadata( dataset_id="br_me_rais", table_id="microdados_vinculos", metadata_path=metadatadir, ) return table_metadata_obj
def invalid_table_metadata(metadatadir): invalid_dataset_metadata = Metadata( dataset_id="br_ibge_pib", table_id="municipio", metadata_path=metadatadir, ) invalid_dataset_metadata.create(if_exists="replace") invalid_config = invalid_dataset_metadata.local_metadata invalid_config["table_id"] = None with open(invalid_dataset_metadata.filepath, "w", encoding="utf-8") as file: ryaml.dump(invalid_config, file) return invalid_dataset_metadata
def invalid_dataset_metadata(metadatadir): invalid_dataset_metadata = Metadata( dataset_id="br_ibge_pib", metadata_path=metadatadir, ) invalid_dataset_metadata.create(if_exists="replace") invalid_config = invalid_dataset_metadata.local_metadata invalid_config["title"] = {"this_title": "is_not_valid"} print(invalid_dataset_metadata.filepath) with open(invalid_dataset_metadata.filepath, "w", encoding="utf-8") as file: ryaml.dump(invalid_config, file) return invalid_dataset_metadata
def test_publish_if_exists_pass(valid_metadata_dataset: Metadata): assert isinstance(valid_metadata_dataset.publish(if_exists="pass"), dict) assert valid_metadata_dataset.publish(if_exists="pass") == {}
def test_publish_if_exists_replace(valid_metadata_dataset: Metadata): res = valid_metadata_dataset.publish(if_exists="replace") assert isinstance(res, dict) assert res != {}
def test_publish_if_exists_raise(valid_metadata_dataset: Metadata): with pytest.raises(BaseDosDadosException, match="already exists in CKAN"): valid_metadata_dataset.publish(if_exists="raise")
def valid_metadata_dataset(metadatadir): dataset_metadata = Metadata(dataset_id="br_ibge_pib", metadata_path=metadatadir) dataset_metadata.create(if_exists="replace") dataset_metadata.CKAN_API_KEY = "valid-key" return dataset_metadata
def test_validate_is_succesful(valid_metadata_dataset: Metadata, valid_metadata_table: Metadata): assert valid_metadata_dataset.validate() == True assert valid_metadata_table.validate() == True
def test_publish_is_successful( valid_metadata_dataset: Metadata, valid_metadata_table: Metadata, ): assert isinstance(valid_metadata_dataset.publish(), dict) assert isinstance(valid_metadata_table.publish(), dict)
def pytest_table(metadatadir): shutil.rmtree(metadatadir, ignore_errors=True) pytest_table = Metadata(dataset_id="pytest", table_id="pytest") pytest_table.create(if_exists="replace") return pytest_table
def table_metadata(metadatadir): return Metadata(dataset_id=DATASET_ID, table_id=TABLE_ID, metadata_path=metadatadir)
def dataset_metadata(metadatadir): return Metadata(dataset_id=DATASET_ID, metadata_path=metadatadir)