def test_updating_groups(request, ckan_client_ll):
client = ckan_client_ll # shorten name
dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': "Test dataset",
'groups': []
}
dummy_groups = []
for x in xrange(10):
code = gen_random_id()
group = client.post_group({
'name': 'group-{0}'.format(code),
'title': 'Group {0}'.format(code),
})
dummy_groups.append(group)
request.addfinalizer(lambda: client.delete_group(group['id']))
dataset['groups'] = [x['id'] for x in dummy_groups[:5]]
created = client.post_dataset(dataset)
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
assert sorted(created['groups']) == sorted(dataset['groups'])
# Let's try updating the dataset w/o groups
updated = client.update_dataset(dataset_id, {'title': "My dataset"})
assert sorted(updated['groups']) == sorted(dataset['groups'])
# Let's try updating the dataset with empty groups
updated = client.update_dataset(dataset_id, {'groups': []})
assert sorted(updated['groups']) \
== sorted(dataset['groups']) # WTF? -- should be empty
def test_updating_groups(request, ckan_client_ll):
client = ckan_client_ll # shorten name
dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': "Test dataset",
'groups': []
}
dummy_groups = []
for x in xrange(10):
code = gen_random_id()
group = client.post_group({
'name': 'group-{0}'.format(code),
'title': 'Group {0}'.format(code),
})
dummy_groups.append(group)
request.addfinalizer(lambda: client.delete_group(group['id']))
dataset['groups'] = [x['id'] for x in dummy_groups[:5]]
created = client.post_dataset(dataset)
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
assert sorted(created['groups']) == sorted(dataset['groups'])
# Let's try updating the dataset w/o groups
updated = client.update_dataset(dataset_id, {'title': "My dataset"})
assert sorted(updated['groups']) == sorted(dataset['groups'])
# Let's try updating the dataset with empty groups
updated = client.update_dataset(dataset_id, {'groups': []})
assert sorted(updated['groups']) \
== sorted(dataset['groups']) # WTF? -- should be empty
def test_extras_bad_behavior(request, ckan_client_ll):
dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': "Test dataset",
'extras': {'a': 'aa', 'b': 'bb', 'c': 'cc'},
}
created = ckan_client_ll.post_dataset(dataset)
dataset_id = created['id']
request.addfinalizer(lambda: ckan_client_ll.delete_dataset(dataset_id))
assert created['extras'] == {'a': 'aa', 'b': 'bb', 'c': 'cc'}
# -- Update #1: omitting extras will.. flush it!
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
# 'extras' intentionally omitted
})
assert updated['extras'] == {}
# -- Update #2: re-add some extras
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {'a': 'aa', 'b': 'bb', 'c': 'cc'},
})
assert updated['extras'] == {'a': 'aa', 'b': 'bb', 'c': 'cc'}
# -- Update #3: partial extras will just update
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {'a': 'UPDATED'},
})
assert updated['extras'] == {'a': 'UPDATED', 'b': 'bb', 'c': 'cc'}
# -- Update #4: empty extras has no effect
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {},
})
assert updated['extras'] == {'a': 'UPDATED', 'b': 'bb', 'c': 'cc'}
# -- Update #5: this is f****d up, man..
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
})
assert updated['extras'] == {}
def _new_dataset(groups):
code = gen_random_id()
dataset = {
'name': 'dataset-{0}'.format(code),
'title': 'Dataset {0}'.format(code),
'groups': [grp[1], grp[2]],
}
created = client.post_dataset(dataset)
assert created['name'] == dataset['name']
assert created['title'] == dataset['title']
assert sorted(created['groups']) == sorted([grp[1], grp[2]])
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
return created
def _new_dataset(groups):
code = gen_random_id()
dataset = {
'name': 'dataset-{0}'.format(code),
'title': 'Dataset {0}'.format(code),
'groups': [grp[1], grp[2]],
}
created = client.post_dataset(dataset)
assert created['name'] == dataset['name']
assert created['title'] == dataset['title']
assert sorted(created['groups']) == sorted([grp[1], grp[2]])
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
return created
def test_group_crud(ckan_client_ll):
client = ckan_client_ll
code = gen_random_id()
group = {
'name': 'group-{0}'.format(code),
'title': 'Group {0}'.format(code),
}
created = client.post_group(group)
check_group(created, group)
group_id = created['id']
# Retrieve & check
retrieved = client.get_group(group_id)
assert retrieved == created
# Update & check
updated = client.put_group({'id': group_id, 'title': 'My Group'})
assert updated['name'] == group['name']
assert updated['title'] == 'My Group'
# Check differences
expected = copy.deepcopy(created)
expected['title'] = 'My Group'
check_group(updated, expected)
# Retrieve & double-check
retrieved = client.get_group(group_id)
assert retrieved == updated
# Delete
# ------------------------------------------------------------
# Note: it's impossible to actually delete a group.
# The only hint it has been deleted is its "state"
# is set to "deleted".
# ------------------------------------------------------------
client.delete_group(group_id)
with pytest.raises(HTTPError) as excinfo:
client.get_group(group_id)
assert excinfo.value.status_code in (404, 403) # workaround
def test_groups_bad_behavior(request, ckan_client_ll):
"""
Test to pinpoint "bad behavior" when updating groups associated
with a dataset.
See the GROUP_TEST_CASES (initial state, update, result) below
for more information on the behavior to expect..
"""
client = ckan_client_ll
OMITTED = object()
GROUP_TEST_CASES = [
# -- If we omit the key entirely, it will be flushed
([], OMITTED, []),
([1, 2, 3], OMITTED, []),
# -- If we pass None, the API will fail with 500:
# -- Error - <type 'exceptions.TypeError'>: object of type
# -- 'NoneType' has no len()
# ([1, 2, 3], None, []),
# -- For other cases, it seems to work when passed IDs
# -- Do **not** attempt passing objects, as behavior here
# -- is more uncertain..
([1, 2, 3], [], []),
([1], [1, 2, 3, 4], [1, 2, 3, 4]),
([1, 2, 3], [1, 2], [1, 2]),
([1, 2, 3], [1, 2, 4], [1, 2, 4]),
([1, 2], [1, 2, 3, 4], [1, 2, 3, 4]),
]
# -- Create a bunch of groups
grp = []
for x in xrange(5):
code = gen_random_id()
group = client.post_group({
'name': 'group-{0}'.format(code),
'title': 'Group {0}'.format(code),
})
grp.append(group['id'])
request.addfinalizer(lambda: client.delete_group(group['id']))
def _new_dataset(groups):
code = gen_random_id()
dataset = {
'name': 'dataset-{0}'.format(code),
'title': 'Dataset {0}'.format(code),
'groups': [grp[1], grp[2]],
}
created = client.post_dataset(dataset)
assert created['name'] == dataset['name']
assert created['title'] == dataset['title']
assert sorted(created['groups']) == sorted([grp[1], grp[2]])
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
return created
def _to_ids(x):
if x is None or x is OMITTED:
return x
return [grp[y - 1] for y in x]
for state, update, result in GROUP_TEST_CASES:
state, update, result = map(_to_ids, (state, update, result))
dataset = _new_dataset(groups=state)
# -- Intentionally using low-level put_dataset() here
_data = {'id': dataset['id'], 'groups': update} \
if update is not OMITTED else {'id': dataset['id']}
upd = client.put_dataset(_data)
upd2 = client.get_dataset(dataset['id'])
assert sorted(upd['groups']) == sorted(result)
assert sorted(upd2['groups']) == sorted(result)
def test_extras_bad_behavior(request, ckan_client_ll):
dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': "Test dataset",
'extras': {
'a': 'aa',
'b': 'bb',
'c': 'cc'
},
}
created = ckan_client_ll.post_dataset(dataset)
dataset_id = created['id']
request.addfinalizer(lambda: ckan_client_ll.delete_dataset(dataset_id))
assert created['extras'] == {'a': 'aa', 'b': 'bb', 'c': 'cc'}
# -- Update #1: omitting extras will.. flush it!
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
# 'extras' intentionally omitted
})
assert updated['extras'] == {}
# -- Update #2: re-add some extras
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {
'a': 'aa',
'b': 'bb',
'c': 'cc'
},
})
assert updated['extras'] == {'a': 'aa', 'b': 'bb', 'c': 'cc'}
# -- Update #3: partial extras will just update
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {
'a': 'UPDATED'
},
})
assert updated['extras'] == {'a': 'UPDATED', 'b': 'bb', 'c': 'cc'}
# -- Update #4: empty extras has no effect
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
'extras': {},
})
assert updated['extras'] == {'a': 'UPDATED', 'b': 'bb', 'c': 'cc'}
# -- Update #5: this is f****d up, man..
updated = ckan_client_ll.put_dataset({
'id': dataset_id,
'name': dataset['name'],
'title': dataset['title'],
})
assert updated['extras'] == {}
def test_dataset_simple_crud(ckan_client_ll):
# Let's try creating a dataset
now = datetime.datetime.now()
now_str = now.strftime('%F %T')
_dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': 'Dataset {0}'.format(now_str),
'url': 'http://example.com/dataset1',
'author': 'Author 1',
'author_email': '*****@*****.**',
'maintainer': 'Maintainer 1',
'maintainer_email': '*****@*****.**',
'license_id': 'cc-by',
'notes': "Dataset 1 notes",
'private': False,
}
dataset = ckan_client_ll.post_dataset(_dataset)
dataset_id = dataset['id']
# Let's check dataset data first
for key, val in _dataset.iteritems():
assert dataset[key] == val
# Check that retrieved dataset is identical
dataset = ckan_client_ll.get_dataset(dataset_id)
for key, val in _dataset.iteritems():
assert dataset[key] == val
# Check against data loss on update..
retrieved_dataset = dataset
updates = {
'author': 'Another Author',
'author_email': '*****@*****.**',
}
new_dataset = copy.deepcopy(dataset)
new_dataset.update(updates)
new_dataset['id'] = dataset_id
# Get the updated dataset
updated_dataset = ckan_client_ll.put_dataset(new_dataset)
updated_dataset_2 = ckan_client_ll.get_dataset(dataset_id)
# They should be equal!
assert updated_dataset == updated_dataset_2
# And the updated dataset shouldn't have data loss
expected_dataset = copy.deepcopy(retrieved_dataset)
expected_dataset.update(updates)
IGNORED_FIELDS = ['revision_id', 'metadata_modified']
for f in IGNORED_FIELDS:
updated_dataset.pop(f, None)
expected_dataset.pop(f, None)
assert updated_dataset == expected_dataset
# Delete the dataset
ckan_client_ll.delete_dataset(dataset_id)
def test_groups_bad_behavior(request, ckan_client_ll):
"""
Test to pinpoint "bad behavior" when updating groups associated
with a dataset.
See the GROUP_TEST_CASES (initial state, update, result) below
for more information on the behavior to expect..
"""
client = ckan_client_ll
OMITTED = object()
GROUP_TEST_CASES = [
# -- If we omit the key entirely, it will be flushed
([], OMITTED, []),
([1, 2, 3], OMITTED, []),
# -- If we pass None, the API will fail with 500:
# -- Error - <type 'exceptions.TypeError'>: object of type
# -- 'NoneType' has no len()
# ([1, 2, 3], None, []),
# -- For other cases, it seems to work when passed IDs
# -- Do **not** attempt passing objects, as behavior here
# -- is more uncertain..
([1, 2, 3], [], []),
([1], [1, 2, 3, 4], [1, 2, 3, 4]),
([1, 2, 3], [1, 2], [1, 2]),
([1, 2, 3], [1, 2, 4], [1, 2, 4]),
([1, 2], [1, 2, 3, 4], [1, 2, 3, 4]),
]
# -- Create a bunch of groups
grp = []
for x in xrange(5):
code = gen_random_id()
group = client.post_group({
'name': 'group-{0}'.format(code),
'title': 'Group {0}'.format(code),
})
grp.append(group['id'])
request.addfinalizer(lambda: client.delete_group(group['id']))
def _new_dataset(groups):
code = gen_random_id()
dataset = {
'name': 'dataset-{0}'.format(code),
'title': 'Dataset {0}'.format(code),
'groups': [grp[1], grp[2]],
}
created = client.post_dataset(dataset)
assert created['name'] == dataset['name']
assert created['title'] == dataset['title']
assert sorted(created['groups']) == sorted([grp[1], grp[2]])
dataset_id = created['id']
request.addfinalizer(lambda: client.delete_dataset(dataset_id))
return created
def _to_ids(x):
if x is None or x is OMITTED:
return x
return [grp[y - 1] for y in x]
for state, update, result in GROUP_TEST_CASES:
state, update, result = map(_to_ids, (state, update, result))
dataset = _new_dataset(groups=state)
# -- Intentionally using low-level put_dataset() here
_data = {'id': dataset['id'], 'groups': update} \
if update is not OMITTED else {'id': dataset['id']}
upd = client.put_dataset(_data)
upd2 = client.get_dataset(dataset['id'])
assert sorted(upd['groups']) == sorted(result)
assert sorted(upd2['groups']) == sorted(result)
def test_dataset_simple_crud(ckan_client_ll):
# Let's try creating a dataset
now = datetime.datetime.now()
now_str = now.strftime('%F %T')
_dataset = {
'name': 'dataset-{0}'.format(gen_random_id()),
'title': 'Dataset {0}'.format(now_str),
'url': 'http://example.com/dataset1',
'author': 'Author 1',
'author_email': '*****@*****.**',
'maintainer': 'Maintainer 1',
'maintainer_email': '*****@*****.**',
'license_id': 'cc-by',
'notes': "Dataset 1 notes",
'private': False,
}
dataset = ckan_client_ll.post_dataset(_dataset)
dataset_id = dataset['id']
# Let's check dataset data first
for key, val in _dataset.iteritems():
assert dataset[key] == val
# Check that retrieved dataset is identical
dataset = ckan_client_ll.get_dataset(dataset_id)
for key, val in _dataset.iteritems():
assert dataset[key] == val
# Check against data loss on update..
retrieved_dataset = dataset
updates = {
'author': 'Another Author',
'author_email': '*****@*****.**',
}
new_dataset = copy.deepcopy(dataset)
new_dataset.update(updates)
new_dataset['id'] = dataset_id
# Get the updated dataset
updated_dataset = ckan_client_ll.put_dataset(new_dataset)
updated_dataset_2 = ckan_client_ll.get_dataset(dataset_id)
# They should be equal!
assert updated_dataset == updated_dataset_2
# And the updated dataset shouldn't have data loss
expected_dataset = copy.deepcopy(retrieved_dataset)
expected_dataset.update(updates)
IGNORED_FIELDS = ['revision_id', 'metadata_modified']
for f in IGNORED_FIELDS:
updated_dataset.pop(f, None)
expected_dataset.pop(f, None)
assert updated_dataset == expected_dataset
# Delete the dataset
ckan_client_ll.delete_dataset(dataset_id)