def test_ckan_storage_constraints(options): # Export/Import source = Package("data/storage/constraints.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata assert target.get_resource("constraints").schema == { "fields": [ {"name": "required", "type": "string"}, # constraint removal {"name": "minLength", "type": "string"}, # constraint removal {"name": "maxLength", "type": "string"}, # constraint removal {"name": "pattern", "type": "string"}, # constraint removal {"name": "enum", "type": "string"}, # constraint removal {"name": "minimum", "type": "integer"}, # constraint removal {"name": "maximum", "type": "integer"}, # constraint removal ], } # Assert data assert target.get_resource("constraints").read_rows() == [ { "required": "passing", "minLength": "passing", "maxLength": "passing", "pattern": "passing", "enum": "passing", "minimum": 5, "maximum": 5, }, ] # Cleanup storage storage.delete_package(target.resource_names)
def test_ckan_storage_types(options): # Export/Import source = Package("data/storage/types.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata assert target.get_resource("types").schema == { "fields": [ {"name": "any", "type": "string"}, # type fallback {"name": "array", "type": "array"}, {"name": "boolean", "type": "boolean"}, {"name": "date", "type": "string"}, # type fallback {"name": "date_year", "type": "string"}, # type fallback {"name": "datetime", "type": "datetime"}, {"name": "duration", "type": "string"}, # type fallback {"name": "geojson", "type": "object"}, # type downgrade {"name": "geopoint", "type": "string"}, # type fallback {"name": "integer", "type": "integer"}, {"name": "number", "type": "number"}, {"name": "object", "type": "object"}, {"name": "string", "type": "string"}, {"name": "time", "type": "time"}, {"name": "year", "type": "integer"}, # type downgrade {"name": "yearmonth", "type": "string"}, # type fallback ], } # Assert data assert target.get_resource("types").read_rows() == [ { "any": "中国人", "array": ["Mike", "John"], "boolean": True, "date": "2015-01-01", "date_year": "2015", "datetime": datetime.datetime(2015, 1, 1, 3, 0), "duration": "P1Y1M", "geojson": {"type": "Point", "coordinates": [33, 33.33]}, "geopoint": "30,70", "integer": 1, "number": 7, "object": {"chars": 560}, "string": "english", "time": datetime.time(3, 0), "year": 2015, "yearmonth": "2015-01", }, ] # Cleanup storage storage.delete_package(target.resource_names)
def test_ckan_storage_integrity(options): # Export/Import source = Package("data/storage/integrity.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata (main) assert target.get_resource("integrity_main").schema == { "fields": [ {"name": "id", "type": "integer"}, {"name": "parent", "type": "integer"}, {"name": "description", "type": "string"}, ], # primary key removal # foreign keys removal } # Assert metadata (link) assert target.get_resource("integrity_link").schema == { "fields": [ {"name": "main_id", "type": "integer"}, {"name": "some_id", "type": "integer"}, # constraint removal {"name": "description", "type": "string"}, # constraint removal ], # primary key removal # foreign keys removal } # Assert data (main) assert target.get_resource("integrity_main").read_rows() == [ {"id": 1, "parent": None, "description": "english"}, {"id": 2, "parent": 1, "description": "中国人"}, ] # Assert data (link) assert target.get_resource("integrity_link").read_rows() == [ {"main_id": 1, "some_id": 1, "description": "note1"}, {"main_id": 2, "some_id": 2, "description": "note2"}, ] # Cleanup storage storage.delete_package(target.resource_names)
class TestWriteExists: def setup_method(self): existing_resources_ids = [ "bd79c992-40f0-454a-a0ff-887f84a792fb", "79843e49-7974-411c-8eb5-fb2d1111d707", ] self.target_resource_id = existing_resources_ids[1] filter_qs = urllib.parse.urlencode( {"filters": json.dumps({"name": existing_resources_ids})}) self.dataset_id = "my-dataset-id" self.base_url = "https://demo.ckan.org" _mock_json_call( f"{self.base_url}/api/3/action/package_show?id={self.dataset_id}", "data/ckan_mock_responses/package_show.json", ) # First response gets the _table_metadata results _mock_json_call( f"{self.base_url}/api/3/action/datastore_search?resource_id=_table_metadata&{filter_qs}", "data/ckan_mock_responses/datastore_search_table_metadata_01.json", ) # Second response is empty, indicating the results are exhausted _mock_json_call( f"{self.base_url}/api/3/action/datastore_search?offset=100&{filter_qs}&resource_id=_table_metadata", "data/ckan_mock_responses/datastore_search_table_metadata_02.json", ) _mock_json_call( f"{self.base_url}/api/3/action/datastore_delete", "data/ckan_mock_responses/doesnt_matter.json", method=responses.POST, ) _mock_json_call( f"{self.base_url}/api/3/action/datastore_create", "data/ckan_mock_responses/doesnt_matter.json", method=responses.POST, ) _mock_json_call( f"{self.base_url}/api/3/action/datastore_upsert", "data/ckan_mock_responses/doesnt_matter.json", method=responses.POST, ) self.package = Package("data/storage/types.json") self.package.resources[0].name = self.target_resource_id @responses.activate def test_write_resource_no_force(self): with pytest.raises(exceptions.FrictionlessException) as excinfo: self.package.to_ckan( base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY", ) error = excinfo.value.error assert error.code == "storage-error" assert error.note.count( f'Table "{self.target_resource_id}" already exists') @responses.activate def test_write_resource_with_force(self): self.package.to_ckan( base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY", force=True, ) assert responses.assert_call_count( f"{self.base_url}/api/3/action/datastore_upsert", 1) @responses.activate def test_write_package_no_force(self): with pytest.raises(exceptions.FrictionlessException) as excinfo: self.package.to_ckan( base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY", ) error = excinfo.value.error assert error.code == "storage-error" assert error.note.count( f'Table "{self.target_resource_id}" already exists') @responses.activate def test_write_package_with_force(self): self.package.to_ckan( base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY", force=True, ) assert responses.assert_call_count( f"{self.base_url}/api/3/action/datastore_upsert", 1)
def test_write_constraints(self): package = Package("data/storage/constraints.json") package.resources[0].name = self.target_resource_id package.to_ckan(base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY") # metadata assert json.loads(responses.calls[2].request.body) == { "fields": [ { "id": "required", "type": "text" }, # constraint removal { "id": "minLength", "type": "text" }, # constraint removal { "id": "maxLength", "type": "text" }, # constraint removal { "id": "pattern", "type": "text" }, # constraint removal { "id": "enum", "type": "text" }, # constraint removal { "id": "minimum", "type": "int" }, # constraint removal { "id": "maximum", "type": "int" }, # constraint removal ], "resource_id": self.target_resource_id, "force": True, "primary_key": [], } # data assert json.loads(responses.calls[3].request.body) == { "resource_id": self.target_resource_id, "method": "insert", "force": True, "records": [{ "required": "passing", "minLength": "passing", "maxLength": "passing", "pattern": "passing", "enum": "passing", "minimum": 5, "maximum": 5, }], }
def test_write_integrity(self): package = Package("data/storage/integrity.json") package.resources[0].name = self.existing_resources_ids[0] package.resources[1].name = self.existing_resources_ids[1] package.resources[1].schema["foreignKeys"][0]["reference"][ "resource"] = package.resources[0].name package.to_ckan(base_url=self.base_url, dataset_id=self.dataset_id, api_key="env:CKAN_API_KEY") # metadata assert json.loads(responses.calls[2].request.body) == { "fields": [ { "id": "id", "type": "int" }, # constraint removal { "id": "parent", "type": "int" }, { "id": "description", "type": "text" }, ], "resource_id": self.existing_resources_ids[0], "force": True, "primary_key": ["id"], # foreign keys removal } assert json.loads(responses.calls[6].request.body) == { "fields": [ { "id": "main_id", "type": "int" }, # constraint removal { "id": "some_id", "type": "int" }, # constraint removal { "id": "description", "type": "text" }, # constraint removal ], "resource_id": self.existing_resources_ids[1], "force": True, "primary_key": ["main_id", "some_id"], # foreign keys removal } # data assert json.loads(responses.calls[3].request.body) == { "resource_id": self.existing_resources_ids[0], "method": "insert", "force": True, "records": [ { "id": 1, "parent": None, "description": "english" }, { "id": 2, "parent": 1, "description": "中国人" }, ], } assert json.loads(responses.calls[7].request.body) == { "resource_id": self.existing_resources_ids[1], "method": "insert", "force": True, "records": [ { "main_id": 1, "some_id": 1, "description": "note1" }, { "main_id": 2, "some_id": 2, "description": "note2" }, ], }