def test_ckan_storage_constraints(options):
# Export/Import
source = Package("data/storage/constraints.json")
storage = source.to_ckan(force=True, **options)
target = Package.from_ckan(**options)
# Assert metadata
assert target.get_resource("constraints").schema == {
"fields": [
{"name": "required", "type": "string"}, # constraint removal
{"name": "minLength", "type": "string"}, # constraint removal
{"name": "maxLength", "type": "string"}, # constraint removal
{"name": "pattern", "type": "string"}, # constraint removal
{"name": "enum", "type": "string"}, # constraint removal
{"name": "minimum", "type": "integer"}, # constraint removal
{"name": "maximum", "type": "integer"}, # constraint removal
],
}
# Assert data
assert target.get_resource("constraints").read_rows() == [
{
"required": "passing",
"minLength": "passing",
"maxLength": "passing",
"pattern": "passing",
"enum": "passing",
"minimum": 5,
"maximum": 5,
},
]
# Cleanup storage
storage.delete_package(target.resource_names)
def test_ckan_storage_types(options):
# Export/Import
source = Package("data/storage/types.json")
storage = source.to_ckan(force=True, **options)
target = Package.from_ckan(**options)
# Assert metadata
assert target.get_resource("types").schema == {
"fields": [
{"name": "any", "type": "string"}, # type fallback
{"name": "array", "type": "array"},
{"name": "boolean", "type": "boolean"},
{"name": "date", "type": "string"}, # type fallback
{"name": "date_year", "type": "string"}, # type fallback
{"name": "datetime", "type": "datetime"},
{"name": "duration", "type": "string"}, # type fallback
{"name": "geojson", "type": "object"}, # type downgrade
{"name": "geopoint", "type": "string"}, # type fallback
{"name": "integer", "type": "integer"},
{"name": "number", "type": "number"},
{"name": "object", "type": "object"},
{"name": "string", "type": "string"},
{"name": "time", "type": "time"},
{"name": "year", "type": "integer"}, # type downgrade
{"name": "yearmonth", "type": "string"}, # type fallback
],
}
# Assert data
assert target.get_resource("types").read_rows() == [
{
"any": "中国人",
"array": ["Mike", "John"],
"boolean": True,
"date": "2015-01-01",
"date_year": "2015",
"datetime": datetime.datetime(2015, 1, 1, 3, 0),
"duration": "P1Y1M",
"geojson": {"type": "Point", "coordinates": [33, 33.33]},
"geopoint": "30,70",
"integer": 1,
"number": 7,
"object": {"chars": 560},
"string": "english",
"time": datetime.time(3, 0),
"year": 2015,
"yearmonth": "2015-01",
},
]
# Cleanup storage
storage.delete_package(target.resource_names)
def test_ckan_storage_integrity(options):
# Export/Import
source = Package("data/storage/integrity.json")
storage = source.to_ckan(force=True, **options)
target = Package.from_ckan(**options)
# Assert metadata (main)
assert target.get_resource("integrity_main").schema == {
"fields": [
{"name": "id", "type": "integer"},
{"name": "parent", "type": "integer"},
{"name": "description", "type": "string"},
],
# primary key removal
# foreign keys removal
}
# Assert metadata (link)
assert target.get_resource("integrity_link").schema == {
"fields": [
{"name": "main_id", "type": "integer"},
{"name": "some_id", "type": "integer"}, # constraint removal
{"name": "description", "type": "string"}, # constraint removal
],
# primary key removal
# foreign keys removal
}
# Assert data (main)
assert target.get_resource("integrity_main").read_rows() == [
{"id": 1, "parent": None, "description": "english"},
{"id": 2, "parent": 1, "description": "中国人"},
]
# Assert data (link)
assert target.get_resource("integrity_link").read_rows() == [
{"main_id": 1, "some_id": 1, "description": "note1"},
{"main_id": 2, "some_id": 2, "description": "note2"},
]
# Cleanup storage
storage.delete_package(target.resource_names)
class TestWriteExists:
def setup_method(self):
existing_resources_ids = [
"bd79c992-40f0-454a-a0ff-887f84a792fb",
"79843e49-7974-411c-8eb5-fb2d1111d707",
]
self.target_resource_id = existing_resources_ids[1]
filter_qs = urllib.parse.urlencode(
{"filters": json.dumps({"name": existing_resources_ids})})
self.dataset_id = "my-dataset-id"
self.base_url = "https://demo.ckan.org"
_mock_json_call(
f"{self.base_url}/api/3/action/package_show?id={self.dataset_id}",
"data/ckan_mock_responses/package_show.json",
)
# First response gets the _table_metadata results
_mock_json_call(
f"{self.base_url}/api/3/action/datastore_search?resource_id=_table_metadata&{filter_qs}",
"data/ckan_mock_responses/datastore_search_table_metadata_01.json",
)
# Second response is empty, indicating the results are exhausted
_mock_json_call(
f"{self.base_url}/api/3/action/datastore_search?offset=100&{filter_qs}&resource_id=_table_metadata",
"data/ckan_mock_responses/datastore_search_table_metadata_02.json",
)
_mock_json_call(
f"{self.base_url}/api/3/action/datastore_delete",
"data/ckan_mock_responses/doesnt_matter.json",
method=responses.POST,
)
_mock_json_call(
f"{self.base_url}/api/3/action/datastore_create",
"data/ckan_mock_responses/doesnt_matter.json",
method=responses.POST,
)
_mock_json_call(
f"{self.base_url}/api/3/action/datastore_upsert",
"data/ckan_mock_responses/doesnt_matter.json",
method=responses.POST,
)
self.package = Package("data/storage/types.json")
self.package.resources[0].name = self.target_resource_id
@responses.activate
def test_write_resource_no_force(self):
with pytest.raises(exceptions.FrictionlessException) as excinfo:
self.package.to_ckan(
base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY",
)
error = excinfo.value.error
assert error.code == "storage-error"
assert error.note.count(
f'Table "{self.target_resource_id}" already exists')
@responses.activate
def test_write_resource_with_force(self):
self.package.to_ckan(
base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY",
force=True,
)
assert responses.assert_call_count(
f"{self.base_url}/api/3/action/datastore_upsert", 1)
@responses.activate
def test_write_package_no_force(self):
with pytest.raises(exceptions.FrictionlessException) as excinfo:
self.package.to_ckan(
base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY",
)
error = excinfo.value.error
assert error.code == "storage-error"
assert error.note.count(
f'Table "{self.target_resource_id}" already exists')
@responses.activate
def test_write_package_with_force(self):
self.package.to_ckan(
base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY",
force=True,
)
assert responses.assert_call_count(
f"{self.base_url}/api/3/action/datastore_upsert", 1)
def test_write_constraints(self):
package = Package("data/storage/constraints.json")
package.resources[0].name = self.target_resource_id
package.to_ckan(base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY")
# metadata
assert json.loads(responses.calls[2].request.body) == {
"fields": [
{
"id": "required",
"type": "text"
}, # constraint removal
{
"id": "minLength",
"type": "text"
}, # constraint removal
{
"id": "maxLength",
"type": "text"
}, # constraint removal
{
"id": "pattern",
"type": "text"
}, # constraint removal
{
"id": "enum",
"type": "text"
}, # constraint removal
{
"id": "minimum",
"type": "int"
}, # constraint removal
{
"id": "maximum",
"type": "int"
}, # constraint removal
],
"resource_id":
self.target_resource_id,
"force":
True,
"primary_key": [],
}
# data
assert json.loads(responses.calls[3].request.body) == {
"resource_id":
self.target_resource_id,
"method":
"insert",
"force":
True,
"records": [{
"required": "passing",
"minLength": "passing",
"maxLength": "passing",
"pattern": "passing",
"enum": "passing",
"minimum": 5,
"maximum": 5,
}],
}
def test_write_integrity(self):
package = Package("data/storage/integrity.json")
package.resources[0].name = self.existing_resources_ids[0]
package.resources[1].name = self.existing_resources_ids[1]
package.resources[1].schema["foreignKeys"][0]["reference"][
"resource"] = package.resources[0].name
package.to_ckan(base_url=self.base_url,
dataset_id=self.dataset_id,
api_key="env:CKAN_API_KEY")
# metadata
assert json.loads(responses.calls[2].request.body) == {
"fields": [
{
"id": "id",
"type": "int"
}, # constraint removal
{
"id": "parent",
"type": "int"
},
{
"id": "description",
"type": "text"
},
],
"resource_id":
self.existing_resources_ids[0],
"force":
True,
"primary_key": ["id"],
# foreign keys removal
}
assert json.loads(responses.calls[6].request.body) == {
"fields": [
{
"id": "main_id",
"type": "int"
}, # constraint removal
{
"id": "some_id",
"type": "int"
}, # constraint removal
{
"id": "description",
"type": "text"
}, # constraint removal
],
"resource_id":
self.existing_resources_ids[1],
"force":
True,
"primary_key": ["main_id", "some_id"],
# foreign keys removal
}
# data
assert json.loads(responses.calls[3].request.body) == {
"resource_id":
self.existing_resources_ids[0],
"method":
"insert",
"force":
True,
"records": [
{
"id": 1,
"parent": None,
"description": "english"
},
{
"id": 2,
"parent": 1,
"description": "中国人"
},
],
}
assert json.loads(responses.calls[7].request.body) == {
"resource_id":
self.existing_resources_ids[1],
"method":
"insert",
"force":
True,
"records": [
{
"main_id": 1,
"some_id": 1,
"description": "note1"
},
{
"main_id": 2,
"some_id": 2,
"description": "note2"
},
],
}