def test_ckan_storage_constraints(options): # Export/Import source = Package("data/storage/constraints.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata assert target.get_resource("constraints").schema == { "fields": [ {"name": "required", "type": "string"}, # constraint removal {"name": "minLength", "type": "string"}, # constraint removal {"name": "maxLength", "type": "string"}, # constraint removal {"name": "pattern", "type": "string"}, # constraint removal {"name": "enum", "type": "string"}, # constraint removal {"name": "minimum", "type": "integer"}, # constraint removal {"name": "maximum", "type": "integer"}, # constraint removal ], } # Assert data assert target.get_resource("constraints").read_rows() == [ { "required": "passing", "minLength": "passing", "maxLength": "passing", "pattern": "passing", "enum": "passing", "minimum": 5, "maximum": 5, }, ] # Cleanup storage storage.delete_package(target.resource_names)
def test_read_package_bad_package_id(): dataset_id = "bad-dataset-id" base_url = "https://demo.ckan.org" _mock_json_call( f"{base_url}/api/3/action/package_show?id={dataset_id}", "data/ckan_mock_responses/ckan_error.json", status=404, ) with pytest.raises(exceptions.FrictionlessException) as excinfo: Package.from_ckan( base_url=base_url, dataset_id=dataset_id, api_key="env:CKAN_API_KEY", ) error = excinfo.value.error assert error.code == "storage-error" assert error.note.count("Not found")
def test_ckan_storage_types(options): # Export/Import source = Package("data/storage/types.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata assert target.get_resource("types").schema == { "fields": [ {"name": "any", "type": "string"}, # type fallback {"name": "array", "type": "array"}, {"name": "boolean", "type": "boolean"}, {"name": "date", "type": "string"}, # type fallback {"name": "date_year", "type": "string"}, # type fallback {"name": "datetime", "type": "datetime"}, {"name": "duration", "type": "string"}, # type fallback {"name": "geojson", "type": "object"}, # type downgrade {"name": "geopoint", "type": "string"}, # type fallback {"name": "integer", "type": "integer"}, {"name": "number", "type": "number"}, {"name": "object", "type": "object"}, {"name": "string", "type": "string"}, {"name": "time", "type": "time"}, {"name": "year", "type": "integer"}, # type downgrade {"name": "yearmonth", "type": "string"}, # type fallback ], } # Assert data assert target.get_resource("types").read_rows() == [ { "any": "中国人", "array": ["Mike", "John"], "boolean": True, "date": "2015-01-01", "date_year": "2015", "datetime": datetime.datetime(2015, 1, 1, 3, 0), "duration": "P1Y1M", "geojson": {"type": "Point", "coordinates": [33, 33.33]}, "geopoint": "30,70", "integer": 1, "number": 7, "object": {"chars": 560}, "string": "english", "time": datetime.time(3, 0), "year": 2015, "yearmonth": "2015-01", }, ] # Cleanup storage storage.delete_package(target.resource_names)
def test_read_package_valid(self): resources_ids = [ "bd79c992-40f0-454a-a0ff-887f84a792fb", "79843e49-7974-411c-8eb5-fb2d1111d707", ] filter_qs = urllib.parse.urlencode( {"filters": json.dumps({"name": resources_ids})}) dataset_id = "my-dataset-id" base_url = "https://demo.ckan.org" _mock_json_call( f"{base_url}/api/3/action/package_show?id={dataset_id}", "data/ckan_mock_responses/package_show.json", ) # First response gets the _table_metadata results _mock_json_call( f"{base_url}/api/3/action/datastore_search?resource_id=_table_metadata&{filter_qs}", "data/ckan_mock_responses/datastore_search_table_metadata_01.json", ) # Second response is empty, indicating the results are exhausted _mock_json_call( f"{base_url}/api/3/action/datastore_search?offset=100&{filter_qs}&resource_id=_table_metadata", "data/ckan_mock_responses/datastore_search_table_metadata_02.json", ) for resource_id in resources_ids: _mock_json_call( f"{base_url}/api/3/action/datastore_search?limit=0&resource_id={resource_id}", "data/ckan_mock_responses/datastore_search_describe.json", ) # First response gets the data rows _mock_json_call( f"{base_url}/api/3/action/datastore_search?resource_id={resource_id}", "data/ckan_mock_responses/datastore_search_rows.json", ) # Second response is empty, indicating the results are exhausted _mock_json_call( f"{base_url}/api/3/action/datastore_search?resource_id={resource_id}&offset=100", "data/ckan_mock_responses/datastore_search_rows_empty.json", ) package = Package.from_ckan( base_url=base_url, dataset_id=dataset_id, api_key="env:CKAN_API_KEY", ) resource = package.get_resource(resources_ids[1]) resource.onerror = "raise" # be strict about errors under test self._assert_resource_valid(resource)
def test_ckan_storage_integrity(options): # Export/Import source = Package("data/storage/integrity.json") storage = source.to_ckan(force=True, **options) target = Package.from_ckan(**options) # Assert metadata (main) assert target.get_resource("integrity_main").schema == { "fields": [ {"name": "id", "type": "integer"}, {"name": "parent", "type": "integer"}, {"name": "description", "type": "string"}, ], # primary key removal # foreign keys removal } # Assert metadata (link) assert target.get_resource("integrity_link").schema == { "fields": [ {"name": "main_id", "type": "integer"}, {"name": "some_id", "type": "integer"}, # constraint removal {"name": "description", "type": "string"}, # constraint removal ], # primary key removal # foreign keys removal } # Assert data (main) assert target.get_resource("integrity_main").read_rows() == [ {"id": 1, "parent": None, "description": "english"}, {"id": 2, "parent": 1, "description": "中国人"}, ] # Assert data (link) assert target.get_resource("integrity_link").read_rows() == [ {"main_id": 1, "some_id": 1, "description": "note1"}, {"main_id": 2, "some_id": 2, "description": "note2"}, ] # Cleanup storage storage.delete_package(target.resource_names)