def test_resource_version_clear(self):
resource = factories.Resource(name='First name')
user = factories.Sysadmin()
context = get_context(user)
for i in range(0, 3):
resource_version_create(
context, {
'resource_id': resource['id'],
'name': '{}.0'.format(i),
'notes': 'Version notes {}'.format(i)
})
assert len(
resource_version_list(context,
{'resource_id': resource['id']})) == 3
resource_version_clear(context, {'resource_id': resource['id']})
assert len(
resource_version_list(context,
{'resource_id': resource['id']})) == 0
def test_update_unknown_key(self):
resource = factories.Resource(url_type="datastore")
data = {
"resource_id":
resource["id"],
"force":
True,
"primary_key":
u"id",
"fields": [
{
"id": "id",
"type": "text"
},
{
"id": "book",
"type": "text"
},
{
"id": "author",
"type": "text"
},
],
"records": [],
}
helpers.call_action("datastore_create", **data)
data = {
"resource_id": resource["id"],
"method": "update",
"records": [{
"id": "1",
"author": "tolkien"
}], # unknown
}
with pytest.raises(ValidationError) as context:
helpers.call_action("datastore_upsert", **data)
assert u"key \"[\\'1\\']\" not found" in str(context.value)
def test_logged_in_users_cannot_delete_resources_they_do_not_own(self):
# setup our dataset
owner = factories.User()
owner_org = factories.Organization(users=[{
'name': owner['id'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=owner_org['id'])
resource = factories.Resource(package_id=dataset['id'])
# access as another user
user = factories.User()
env = {'REMOTE_USER': user['name'].encode('ascii')}
app = helpers._get_test_app()
response = app.post(url_for(controller='package',
action='resource_delete',
id=dataset['name'],
resource_id=resource['id']),
extra_environ=env,
expect_errors=True)
assert_equal(401, response.status_int)
response.mustcontain('Unauthorized to delete package')
def test_calculate_record_count_is_false(self):
resource = factories.Resource()
data = {
"resource_id":
resource["id"],
"force":
True,
"fields": [
{
"id": "name",
"type": "text"
},
{
"id": "age",
"type": "text"
},
],
"records": [
{
"name": "Sunita",
"age": "51"
},
{
"name": "Bowan",
"age": "68"
},
],
}
helpers.call_action("datastore_create", **data)
data = {
"resource_id": resource["id"],
"filters": {
"name": "Bowan"
},
"force": True,
}
helpers.call_action("datastore_delete", **data)
last_analyze = when_was_last_analyze(resource["id"])
assert last_analyze is None
def test_collaborators_can_delete_resources(self, role):
org1 = factories.Organization()
dataset = factories.Dataset(owner_org=org1['id'])
resource = factories.Resource(package_id=dataset['id'])
user = factories.User()
helpers.call_action(
'package_collaborator_create',
id=dataset['id'], user_id=user['id'], capacity=role)
context = {
'user': user['name'],
'ignore_auth': False,
}
created_resource = helpers.call_action(
'resource_delete',
context=context,
id=resource['id'])
def test_column_names(self, Session):
csv_filepath = get_sample_filepath("column_names.csv")
resource_id = "test1"
factories.Resource(id=resource_id)
loader.load_csv(
csv_filepath,
resource_id=resource_id,
mimetype="text/csv",
logger=PrintLogger(),
)
assert self._get_column_names(Session, "test1")[2:] == [
u"d@t$e",
u"t^e&m*pe!r(a)t?u:r%%e",
r"p\l/a[c{e%",
]
assert self._get_records(Session, "test1")[0] == (
1,
u"2011-01-01",
u"1",
u"Galway",
)
def test_datastore_upsert_private_editor(self):
org = factories.Organization()
dataset = factories.Dataset(private=True, owner_org=org['id'])
resource = factories.Resource(package_id=dataset['id'])
user = factories.User()
context = self._get_context(user)
assert_raises(toolkit.NotAuthorized,
helpers.call_auth,
'datastore_upsert',
context=context,
resource_id=resource['id'])
helpers.call_action('dataset_collaborator_create',
id=dataset['id'],
user_id=user['id'],
capacity='editor')
assert helpers.call_auth('datastore_upsert',
context=context,
resource_id=resource['id'])
def test_add_default_views_to_resource_no_dataset_passed(self):
# New resources have no views
dataset_dict = factories.Dataset()
resource_dict = factories.Resource(
package_id=dataset_dict['id'],
url='http://some.image.png',
format='png',
)
# Change default views config setting
config['ckan.views.default_views'] = 'image_view'
context = {'user': helpers.call_action('get_site_user')['name']}
created_views = helpers.call_action(
'resource_create_default_resource_views',
context,
resource=resource_dict)
assert_equals(len(created_views), 1)
assert_equals(created_views[0]['view_type'], 'image_view')
def test_dump_limit(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'records': [
{
u'book': 'annakarenina'
},
{
u'book': 'warandpeace'
},
],
}
helpers.call_action('datastore_create', **data)
app = self._get_test_app()
response = app.get('/datastore/dump/{0}?limit=1'.format(
str(resource['id'])))
content = response.body.decode('utf-8')
expected_content = (u'_id,book\r\n' u'1,annakarenina\n')
assert_equals(content, expected_content)
def test_resource_view_create_public_member(self):
org = factories.Organization()
dataset = factories.Dataset(owner_org=org['id'])
resource = factories.Resource(package_id=dataset['id'])
user = factories.User()
context = self._get_context(user)
with pytest.raises(logic.NotAuthorized):
helpers.call_auth('resource_view_create',
context=context,
resource_id=resource['id'])
helpers.call_action('package_collaborator_create',
id=dataset['id'],
user_id=user['id'],
capacity='member')
with pytest.raises(logic.NotAuthorized):
helpers.call_auth('resource_view_create',
context=context,
resource_id=resource['id'])
def test_fts_on_field_calculates_ranks_only_on_that_specific_field(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'records': [
{'from': 'Brazil', 'to': 'Brazil'},
{'from': 'Brazil', 'to': 'Italy'}
],
}
result = helpers.call_action('datastore_create', **data)
search_data = {
'resource_id': resource['id'],
'fields': 'from',
'q': {
'from': 'Brazil'
},
}
result = helpers.call_action('datastore_search', **search_data)
ranks = [r['rank from'] for r in result['records']]
assert_equals(len(result['records']), 2)
assert_equals(len(set(ranks)), 1)
def test_duplicated_submits(self):
def submit(res, user):
return helpers.call_action(
"xloader_submit",
context=dict(user=user["name"]),
resource_id=res["id"],
)
user = factories.User()
with mock.patch(
"ckanext.xloader.action.enqueue_job",
return_value=mock.MagicMock(id=123),
) as enqueue_mock:
enqueue_mock.reset_mock()
# creating the resource causes it to be queued
res = factories.Resource(user=user, format="csv")
assert 1 == enqueue_mock.call_count
# a second request to queue it will be stopped, because of the
# existing task for this resource - shown by task_status_show
submit(res, user)
assert 1 == enqueue_mock.call_count
def test_basic_insert(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'primary_key': 'id',
'fields': [{'id': 'id', 'type': 'text'},
{'id': 'book', 'type': 'text'},
{'id': 'author', 'type': 'text'}],
}
helpers.call_action('datastore_create', **data)
data = {
'resource_id': resource['id'],
'force': True,
'method': 'insert',
'records': [
{'id': '1',
'book': u'El Niño',
'author': 'Torres'}],
}
helpers.call_action('datastore_upsert', **data)
search_result = _search(resource['id'])
assert_equal(search_result['total'], 1)
assert_equal(search_result['fields'], [
{u'id': '_id', u'type': 'int'},
{u'id': u'id', u'type': u'text'},
{u'id': u'book', u'type': u'text'},
{u'id': u'author', u'type': u'text'}
])
assert_equal(
search_result['records'][0],
{u'book':
u'El Ni\xf1o',
u'_id': 1,
u'id': u'1',
u'author': u'Torres'})
def test_update_unspecified_key(self):
resource = factories.Resource(url_type="datastore")
data = {
"resource_id":
resource["id"],
"force":
True,
"primary_key":
u"id",
"fields": [
{
"id": "id",
"type": "text"
},
{
"id": "book",
"type": "text"
},
{
"id": "author",
"type": "text"
},
],
"records": [],
}
helpers.call_action("datastore_create", **data)
data = {
"resource_id": resource["id"],
"method": "update",
"records": [{
"author": "tolkien"
}], # no id
}
with pytest.raises(ValidationError) as context:
helpers.call_action("datastore_upsert", **data)
assert u'fields "id" are missing' in str(context.value)
def test_upsert_doesnt_crash_with_json_field(self):
resource = factories.Resource()
data = {
'resource_id':
resource['id'],
'force':
True,
'primary_key':
'id',
'fields': [{
'id': 'id',
'type': 'text'
}, {
'id': 'book',
'type': 'json'
}, {
'id': 'author',
'type': 'text'
}],
}
helpers.call_action('datastore_create', **data)
data = {
'resource_id':
resource['id'],
'force':
True,
'method':
'insert',
'records': [{
'id': '1',
'book': {
'code': 'A',
'title': u'ñ'
},
'author': 'tolstoy'
}],
}
helpers.call_action('datastore_upsert', **data)
def test_alias(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'aliases': 'books',
'records': [
{
u'book': 'annakarenina'
},
{
u'book': 'warandpeace'
},
],
}
helpers.call_action('datastore_create', **data)
app = self._get_test_app()
# get with alias instead of id
response = app.get('/datastore/dump/books')
assert_equals('_id,book\r\n'
'1,annakarenina\n'
'2,warandpeace\n', response.body)
def test_update_unknown_key(self):
resource = factories.Resource(url_type='datastore')
data = {
'resource_id': resource['id'],
'force': True,
'primary_key': u'id',
'fields': [{'id': 'id', 'type': 'text'},
{'id': 'book', 'type': 'text'},
{'id': 'author', 'type': 'text'}],
'records': [],
}
helpers.call_action('datastore_create', **data)
data = {
'resource_id': resource['id'],
'method': 'update',
'records': [{'id': '1', # unknown
'author': 'tolkien'}]
}
with assert_raises(ValidationError) as context:
helpers.call_action('datastore_upsert', **data)
assert_in(u'key "[\\\'1\\\']" not found', str(context.exception))
def test_sysadmins_can_delete_any_resource(self):
owner_org = factories.Organization()
dataset = factories.Dataset(owner_org=owner_org['id'])
resource = factories.Resource(package_id=dataset['id'])
sysadmin = factories.Sysadmin()
app = helpers._get_test_app()
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
response = app.post(
url_for(controller='package',
action='resource_delete',
id=dataset['name'],
resource_id=resource['id']),
extra_environ=env,
)
response = response.follow()
assert_equal(200, response.status_int)
response.mustcontain('This dataset has no data')
assert_raises(p.toolkit.ObjectNotFound,
helpers.call_action,
'resource_show',
id=resource['id'])
def test_create_doesnt_add_more_indexes_when_updating_data(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'records': [{
'book': 'annakarenina',
'author': 'tolstoy'
}]
}
result = helpers.call_action('datastore_create', **data)
previous_index_names = self._get_index_names(resource['id'])
data = {
'resource_id': resource['id'],
'force': True,
'records': [{
'book': 'warandpeace',
'author': 'tolstoy'
}]
}
result = helpers.call_action('datastore_create', **data)
current_index_names = self._get_index_names(resource['id'])
assert_equal(previous_index_names, current_index_names)
def test_resource_delete_editor(self):
'''Normally organization admins can delete resources
Our plugin prevents this by blocking delete organization.
Ensure the delete button is not displayed (as only resource delete
is checked for showing this)
'''
user = factories.User()
owner_org = factories.Organization(users=[{
'name': user['id'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=owner_org['id'])
resource = factories.Resource(package_id=dataset['id'])
with assert_raises(logic.NotAuthorized) as e:
logic.check_access('resource_delete', {'user': user['name']},
{'id': resource['id']})
assert_equal(
e.exception.message,
'User %s not authorized to delete resource %s' %
(user['name'], resource['id']))
def test_add_default_views_to_resource_no_dataset_passed(self):
# New resources have no views
dataset_dict = factories.Dataset()
resource_dict = factories.Resource(
package_id=dataset_dict["id"],
url="http://some.image.png",
format="png",
)
# Change default views config setting
config["ckan.views.default_views"] = "image_view"
context = {"user": helpers.call_action("get_site_user")["name"]}
created_views = helpers.call_action(
"resource_create_default_resource_views",
context,
resource=resource_dict,
)
assert len(created_views) == 1
assert created_views[0]["view_type"] == "image_view"
def upload_json_resource(dataset_name, resource_name):
sysadmin = factories.Sysadmin()
resource = factories.Resource(name=resource_name)
file_path = os.path.join(
os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))),
'data.geojson')
site_base_url = get_site_base_url()
data_dict = {
'package_id': dataset_name,
'name': resource['name'],
'url': 'test_url',
'format': 'geojson'
}
# Upload resource
response = requests.post(
'{0}/api/action/resource_create'.format(site_base_url),
data=data_dict,
headers={'X-CKAN-API-Key': sysadmin['apikey']},
files=[('upload', file(file_path))])
return response.json()['result']
def test_collaborators_can_delete_resources(self, role):
org1 = factories.Organization()
dataset = factories.Dataset(owner_org=org1["id"])
resource = factories.Resource(package_id=dataset["id"])
user = factories.User()
helpers.call_action(
"package_collaborator_create",
id=dataset["id"],
user_id=user["id"],
capacity=role,
)
context = {
"user": user["name"],
"ignore_auth": False,
}
helpers.call_action("resource_delete",
context=context,
id=resource["id"])
def test_basic_access(self):
"""Normally organization admins can delete resources
Our plugin prevents this by blocking delete organization.
Ensure the delete button is not displayed (as only resource delete
is checked for showing this)
"""
owner = factories.User()
access = factories.User()
owner_org = factories.Organization(
users=[{'name': owner['id'], 'capacity': 'admin'}]
)
dataset = factories.Dataset(owner_org=owner_org['id'], private=True)
resource = factories.Resource(package_id=dataset['id'])
assert logic.check_access('package_show', {'user': owner['name']}, {'id': dataset['id']})
assert logic.check_access('resource_show', {'user': owner['name']}, {'id': resource['id']})
with pytest.raises(logic.NotAuthorized):
logic.check_access('package_show', {'user': access['name']}, {'id': dataset['id']})
with pytest.raises(logic.NotAuthorized):
logic.check_access('resource_show', {'user': access['name']}, {'id': resource['id']})
def test_fts_works_on_non_textual_fields(self):
resource = factories.Resource()
data = {
'resource_id': resource['id'],
'force': True,
'records': [
{'from': 'Brazil', 'year': {'foo': 2014}},
{'from': 'Brazil', 'year': {'foo': 1986}}
],
}
result = helpers.call_action('datastore_create', **data)
search_data = {
'resource_id': resource['id'],
'fields': 'year',
'plain': False,
'q': {
'year': '20:*'
},
}
result = helpers.call_action('datastore_search', **search_data)
assert_equals(len(result['records']), 1)
assert_equals(result['records'][0]['year'], {'foo': 2014})
def test_sets_datastore_active_on_resource_on_delete(self):
resource = factories.Resource(datastore_active=True)
assert_equal(resource['datastore_active'], True)
data = {
'resource_id': resource['id'],
'force': True,
'records': [{
'book': 'annakarenina',
'author': 'tolstoy'
}]
}
helpers.call_action('datastore_create', **data)
helpers.call_action('datastore_delete',
resource_id=resource['id'],
force=True)
resource = helpers.call_action('resource_show', id=resource['id'])
assert_equal(resource['datastore_active'], False)
def test_simple(self):
csv_filepath = get_sample_filepath('simple.csv')
resource_id = 'test1'
factories.Resource(id=resource_id)
loader.load_csv(csv_filepath, resource_id=resource_id,
mimetype='text/csv', logger=PrintLogger())
assert_equal(self._get_records(
'test1', limit=1, exclude_full_text_column=False),
[(1, "'-01':2,3 '1':4 '2011':1 'galway':5", u'2011-01-01', u'1', u'Galway')])
assert_equal(self._get_records('test1'),
[(1, u'2011-01-01', u'1', u'Galway'),
(2, u'2011-01-02', u'-1', u'Galway'),
(3, u'2011-01-03', u'0', u'Galway'),
(4, u'2011-01-01', u'6', u'Berkeley'),
(5, None, None, u'Berkeley'),
(6, u'2011-01-03', u'5', None)])
assert_equal(
self._get_column_names('test1'),
[u'_id', u'_full_text', u'date', u'temperature', u'place'])
assert_equal(
self._get_column_types('test1'),
[u'int4', u'tsvector', u'text', u'text', u'text'])
def test_non_existing_field(self):
resource = factories.Resource(url_type='datastore')
data = {
'resource_id': resource['id'],
'force': True,
'primary_key': u'id',
'fields': [{'id': 'id', 'type': 'text'},
{'id': 'book', 'type': 'text'},
{'id': 'author', 'type': 'text'}],
'records': [],
}
helpers.call_action('datastore_create', **data)
data = {
'resource_id': resource['id'],
'method': 'insert',
'records': [{'id': '1',
'dummy': 'tolkien'}] # key not known
}
with assert_raises(ValidationError) as context:
helpers.call_action('datastore_upsert', **data)
assert_in(u'row "1" has extra keys "dummy"', str(context.exception))
def test_not_authorized_if_user_has_no_permissions_on_dataset_2(self):
org = factories.Organization()
user = factories.User()
member = {"username": user["name"], "role": "admin", "id": org["id"]}
helpers.call_action("organization_member_create", **member)
user_2 = factories.User()
dataset = factories.Dataset(owner_org=org["id"])
resource = factories.Resource(package_id=dataset["id"])
context = {"user": user_2["name"], "model": core_model}
with pytest.raises(logic.NotAuthorized):
helpers.call_auth(
"resource_create_default_resource_views",
context=context,
resource=resource,
)
def test_boston_311(self):
csv_filepath = get_sample_filepath('boston_311_sample.csv')
resource_id = 'test1'
factories.Resource(id=resource_id)
loader.load_table(csv_filepath, resource_id=resource_id,
mimetype='csv', logger=PrintLogger())
records = self._get_records('test1')
print records
assert_equal(
records,
[(1, Decimal('101002153891'), datetime.datetime(2017, 7, 6, 23, 38, 43), datetime.datetime(2017, 7, 21, 8, 30), u'', u'ONTIME', u'Open', u' ', u'Street Light Outages', u'Public Works Department', u'Street Lights', u'Street Light Outages', u'PWDx_Street Light Outages', u'PWDx', u'', u'', u'480 Harvard St Dorchester MA 02124', Decimal('8'), Decimal('7'), Decimal('4'), u'B3', u'Greater Mattapan', Decimal('9'), u'Ward 14', Decimal('1411'), u'480 Harvard St', Decimal('2124'), Decimal('42.288'), Decimal('-71.0927'), u'Citizens Connect App'),
(2, Decimal('101002153890'), datetime.datetime(2017, 7, 6, 23, 29, 13), datetime.datetime(2017, 9, 11, 8, 30), u'', u'ONTIME', u'Open', u' ', u'Graffiti Removal', u'Property Management', u'Graffiti', u'Graffiti Removal', u'PROP_GRAF_GraffitiRemoval', u'PROP', u' https://mayors24.cityofboston.gov/media/boston/report/photos/595f0000048560f46d94b9fa/report.jpg', u'', u'522 Saratoga St East Boston MA 02128', Decimal('1'), Decimal('9'), Decimal('1'), u'A7', u'East Boston', Decimal('1'), u'Ward 1', Decimal('110'), u'522 Saratoga St', Decimal('2128'), Decimal('42.3807'), Decimal('-71.0259'), u'Citizens Connect App'),
(3, Decimal('101002153889'), datetime.datetime(2017, 7, 6, 23, 24, 20), datetime.datetime(2017, 9, 11, 8, 30), u'', u'ONTIME', u'Open', u' ', u'Graffiti Removal', u'Property Management', u'Graffiti', u'Graffiti Removal', u'PROP_GRAF_GraffitiRemoval', u'PROP', u' https://mayors24.cityofboston.gov/media/boston/report/photos/595efedb048560f46d94b9ef/report.jpg', u'', u'965 Bennington St East Boston MA 02128', Decimal('1'), Decimal('9'), Decimal('1'), u'A7', u'East Boston', Decimal('1'), u'Ward 1', Decimal('112'), u'965 Bennington St', Decimal('2128'), Decimal('42.386'), Decimal('-71.008'), u'Citizens Connect App')]
)
print self._get_column_names('test1')
assert_equal(
self._get_column_names('test1'),
[u'_id', u'_full_text', u'CASE_ENQUIRY_ID', u'open_dt', u'target_dt', u'closed_dt', u'OnTime_Status', u'CASE_STATUS', u'CLOSURE_REASON', u'CASE_TITLE', u'SUBJECT', u'REASON', u'TYPE', u'QUEUE', u'Department', u'SubmittedPhoto', u'ClosedPhoto', u'Location', u'Fire_district', u'pwd_district', u'city_council_district', u'police_district', u'neighborhood', u'neighborhood_services_district', u'ward', u'precinct', u'LOCATION_STREET_NAME', u'LOCATION_ZIPCODE', u'Latitude', u'Longitude', u'Source'])
print self._get_column_types('test1')
assert_equal(self._get_column_types('test1'),
[u'int4', u'tsvector',
u'numeric', u'timestamp', u'timestamp', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'text', u'numeric', u'numeric', u'numeric', u'text', u'text', u'numeric', u'text', u'numeric', u'text', u'numeric', u'numeric', u'numeric', u'text'])
