def __init__(self, base_url, api_key=None, **kw):
"""
:param base_url:
Base URL of the Ckan instance, passed to high-level client
:param api_key:
API key to be used, passed to high-level client
:param organization_merge_strategy: One of:
- 'create' (default) if the organization doesn't exist, create it.
Otherwise, leave it alone.
- 'update' if the organization doesn't exist, create it.
Otherwise, update with new values.
:param group_merge_strategy: One of:
- 'create' (default) if the group doesn't exist, create it.
Otherwise, leave it alone.
- 'update' if the group doesn't exist, create it.
Otherwise, update with new values.
:param dataset_preserve_names:
if ``True`` (the default) will preserve old names of existing
datasets
:param dataset_preserve_organization:
if ``True`` (the default) will preserve old organizations of
existing datasets.
:param dataset_group_merge_strategy:
- 'add' add groups, keep old ones (default)
- 'replace' replace all existing groups
- 'preserve' leave groups alone
"""
self._client = CkanHighlevelClient(base_url, api_key)
self._conf = {
'organization_merge_strategy': 'create',
'group_merge_strategy': 'create',
'dataset_preserve_names': True,
'dataset_preserve_organization': True,
'dataset_group_merge_strategy': 'add',
}
self._conf.update(kw)
def test_merge_strategies(ckan_client_arguments):
args = ckan_client_arguments
client = CkanHighlevelClient(*args[0], **args[1])
sync_client = SynchronizationClient(*args[0], **args[1])
data = copy.deepcopy(SAMPLE_DATA)
# Sync data -- should create new datasets only
sync_client.sync('test_merge', data)
assert client.get_dataset_by_name('dataset-1').title == 'Dataset #1'
assert client.get_organization_by_name(
'org-1').title == 'Organization #1' # noqa
assert client.get_group_by_name('grp-1').title == 'Group #1' # noqa
# Make sure we preserve names if told so
# ------------------------------------------------------------
sync_client._conf['dataset_preserve_names'] = True
data['dataset']['dataset-1']['name'] = 'dummy-dataset-one'
data['dataset']['dataset-1']['title'] = 'Dataset #1.1'
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-1')
assert dataset.name == 'dataset-1'
assert dataset.title == 'Dataset #1.1'
# Make sure we update names if told so
# ------------------------------------------------------------
sync_client._conf['dataset_preserve_names'] = False
data['dataset']['dataset-1']['name'] = 'dummy-dataset-one'
data['dataset']['dataset-1']['title'] = 'Dataset #1.2'
sync_client.sync('test_merge', data)
with pytest.raises(HTTPError) as excinfo:
# It got renamed!
client.get_dataset_by_name('dataset-1')
assert excinfo.value.status_code == 404
# Get using the old id
dataset = client.get_dataset(dataset.id)
assert dataset.name == 'dummy-dataset-one'
assert dataset.title == 'Dataset #1.2'
# Get using the new name
dataset = client.get_dataset_by_name('dummy-dataset-one')
assert dataset.name == 'dummy-dataset-one'
assert dataset.title == 'Dataset #1.2'
# Prepare for merging groups
# ============================================================
grp1_id = client.get_group_by_name('grp-1').id
grp2_id = client.get_group_by_name('grp-2').id
# grp3_id = client.get_group_by_name('grp-3').id
# Merge groups with 'replace' strategy
# ------------------------------------------------------------
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.groups == set([grp1_id, grp2_id])
sync_client._conf['dataset_group_merge_strategy'] = 'replace'
data['dataset']['dataset-2']['groups'] = ['grp-1']
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.groups == set([grp1_id])
# Merge groups with 'add' strategy
# ------------------------------------------------------------
sync_client._conf['dataset_group_merge_strategy'] = 'add'
data['dataset']['dataset-2']['groups'] = ['grp-2']
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.groups == set([grp1_id, grp2_id])
# Merge groups with 'preserve' strategy
# ------------------------------------------------------------
sync_client._conf['dataset_group_merge_strategy'] = 'preserve'
data['dataset']['dataset-2']['groups'] = ['grp-3']
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.groups == set([grp1_id, grp2_id])
# Prepare for merging Organizations
# ============================================================
org1_id = client.get_organization_by_name('org-1').id
org2_id = client.get_organization_by_name('org-2').id
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.owner_org == org1_id
# Update preserving organization
# ------------------------------------------------------------
sync_client._conf['dataset_preserve_organization'] = True
data['dataset']['dataset-2']['owner_org'] = 'org-2'
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.owner_org == org1_id
# Update *not* preserving organization
# ------------------------------------------------------------
sync_client._conf['dataset_preserve_organization'] = False
data['dataset']['dataset-2']['owner_org'] = 'org-2'
sync_client.sync('test_merge', data)
dataset = client.get_dataset_by_name('dataset-2')
assert dataset.owner_org == org2_id
def test_merge_organizations(ckan_client_arguments):
args = ckan_client_arguments
client = CkanHighlevelClient(*args[0], **args[1])
sync_client = SynchronizationClient(*args[0], **args[1])
# Create a couple initial organizations
# ------------------------------------------------------------
client.create_organization(
CkanOrganization({
'name': 'tmo-1',
'title': 'TMO 1'
}))
client.create_organization(
CkanOrganization({
'name': 'tmo-2',
'title': 'TMO 2'
}))
# Test merging with "create" strategy
# ------------------------------------------------------------
data = {
'organization': {
'tmo-2': {
'name': 'tmo-2',
'title': 'TMO 2.1'
},
'tmo-3': {
'name': 'tmo-3',
'title': 'TMO 3.1'
},
},
'group': {},
'dataset': {}
}
sync_client._conf['organization_merge_strategy'] = 'create'
sync_client.sync('test_merge_organizations', data)
assert client.get_organization_by_name('tmo-1').title == 'TMO 1'
assert client.get_organization_by_name('tmo-2').title == 'TMO 2'
assert client.get_organization_by_name('tmo-3').title == 'TMO 3.1'
# Test merging with "update" strategy
# ------------------------------------------------------------
data = {
'organization': {
'tmo-2': {
'name': 'tmo-2',
'title': 'TMO 2.2'
},
'tmo-4': {
'name': 'tmo-4',
'title': 'TMO 4.2'
},
},
'group': {},
'dataset': {}
}
sync_client._conf['organization_merge_strategy'] = 'update'
sync_client.sync('test_merge_organizations', data)
assert client.get_organization_by_name('tmo-1').title == 'TMO 1'
assert client.get_organization_by_name('tmo-2').title == 'TMO 2.2'
assert client.get_organization_by_name('tmo-3').title == 'TMO 3.1'
assert client.get_organization_by_name('tmo-4').title == 'TMO 4.2'
def test_dataset_import_export(ckan_instance):
api_key = ckan_instance.get_sysadmin_api_key()
with ckan_instance.serve():
client = CkanHighlevelClient(ckan_instance.server_url, api_key=api_key)
assert client.list_datasets() == []
from ckan_api_client.high_level import CkanHighlevelClient
from ckan_api_client.objects import CkanDataset
API_Key = "cdc0284b-47c9-48ff-8a17-5861259c5a03"
# ckan_url = "https://demo.ckan.org"
ckan_url = "http://localhost:5000"
# ckan_url = "http://ckan.dev.pfe.co.nz"
ua = 'ckanapiexample/1.0 (+http://pfr.co.nz/)'
client = CkanHighlevelClient(ckan_url, api_key=API_Key)
# Put the details of the dataset we're going to create into a dict.
dataset_dict = {
"name": "lure-dispenser-comparison-trial2",
"title": "Lure Dispenser comparison trial, thrips, Australia, Perth",
"notes":
"Assess three thrips Lure delivery mechanisms:\n\n* P Paint pen\n* D Deer wick\n* C Control (no wick)",
"private": False,
"owner_org": "plant-and-food-research-nz",
"author": "Mette Nielson",
}
dataset_dict2 = {
"name": "lure-dispenser-comparison-trial",
"title": "Lure Dispenser comparison trial, thrips, Australia, Perth",
"notes":
"Assess three thrips Lure delivery mechanisms:\n\n* P Paint pen\n* D Deer wick\n* C Control (no wick)",
"private": False,
"owner_org": "plant-and-food-research-nz",
"state": "active",
"project_code": "P/1234",
