def test_extras_rollup(self):
""" Test https://github.com/GSA/datagov-deploy/issues/2166 """
datasets = self.get_datasets_from_waf1_sample()
package = datasets[0]
pkg = package.as_dict()
extras = pkg['extras']
extras_rollup = json.loads(extras['extras_rollup'])
log.info("Rolled up at create: {}".format(extras_rollup))
assert extras_rollup
log.info("extras_rollup package info: %s", package)
sysadmin = Sysadmin(name='testUpdate')
user_name = sysadmin['name'].encode('ascii')
context = {'user': user_name}
new_extras = [{
'key': key,
'value': value
} for key, value in extras.iteritems()]
get_action('package_update')(context, {
"id": package.id,
"title": "Test change",
"extras": new_extras
})
updated_package = model.Package.get(package.id)
extras_rollup = json.loads(updated_package.extras['extras_rollup'])
assert 'extras_rollup' not in extras_rollup
def test_resource_form_create(self, app):
dataset = Dataset()
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
value = {
'delimiter': ';',
'headers': 2,
'skip_rows': ['#'],
}
json_value = json.dumps(value)
app.post(url_for("{}_resource.new".format(dataset["type"]),
id=dataset["id"]),
extra_environ=env,
data={
"url": "https://example.com/data.csv",
"validation_options": json_value,
"save": "go-dataset-complete",
"id": ""
})
dataset = call_action('package_show', id=dataset['id'])
assert_equals(
json.loads(dataset['resources'][0]['validation_options']), value)
def test_manage_permissions_page(self):
app = _get_test_app()
url = url_for('manage_permissions')
admin = Sysadmin()
ACCESS_PERMISSIONS.clear()
page = app.get(
url, extra_environ={'REMOTE_USER': admin['name'].encode('utf-8')})
checkboxes = page.html.select('.perm-checkbox input')
pl = ACCESS_PERMISSIONS.permissions_list()
nt.assert_equal(pl, [])
nt.assert_equal(pl, checkboxes)
test_perm_list = ['organization_create', 'package_create']
for perm in test_perm_list:
ACCESS_PERMISSIONS.create_permission(perm)
page = app.get(
url, extra_environ={'REMOTE_USER': admin['name'].encode('utf-8')})
checkboxes = page.html.select('.perm-checkbox input')
pl = ACCESS_PERMISSIONS.permissions_list()
nt.assert_equal(pl, test_perm_list)
nt.assert_equal(len(pl), len(checkboxes))
def setup(cls):
# Start data json sources server we can test harvesting against it
reset_db()
harvest_model.setup()
cls.user = Sysadmin()
if p.toolkit.check_ckan_version(min_version='2.8.0'):
raise SkipTest('Just for CKAN 2.3')
def setup(cls):
reset_db()
harvest_model.setup()
sysadmin = Sysadmin(name='dummy')
user_name = sysadmin['name'].encode('ascii')
call_action('organization_create',
context={'user': user_name},
name='test-org')
def _get_resource_update_page_as_sysadmin(app, id, resource_id):
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
response = app.get(
url='/dataset/{}/resource_edit/{}'.format(id, resource_id),
extra_environ=env,
)
return env, response
def test_resource_form_create_url(self, app):
dataset = Dataset()
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
value = 'https://example.com/schemas.json'
app.post(url_for("{}_resource.new".format(dataset["type"]),
id=dataset["id"]),
extra_environ=env,
data={
"url": "https://example.com/data.csv",
"schema_json": value,
"save": "go-dataset-complete",
"id": ""
})
dataset = call_action('package_show', id=dataset['id'])
assert_equals(dataset['resources'][0]['schema'], value)
def test_resource_form_create_invalid(self, mock_open):
user = Sysadmin()
org = Organization(user=user)
dataset = Dataset(owner_org=org['id'])
upload = ('upload', 'invalid.csv', INVALID_CSV)
invalid_stream = io.BufferedReader(io.BytesIO(INVALID_CSV))
params = {
'name': 'test_resource_form_create_invalid',
'url': 'https://example.com/data.csv'
}
with mock.patch('io.open', return_value=invalid_stream):
response = _get_response_body(_post(self.app, NEW_RESOURCE_URL.format(dataset['id']),
params, upload=[upload]))
assert 'validation' in response
assert 'missing-value' in response
assert 'Row 2 has a missing value in column 4' in response
def test_resource_form_create_json(self, app):
dataset = Dataset()
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
value = {'fields': [{'name': 'code'}, {'name': 'department'}]}
json_value = json.dumps(value)
app.post(url_for("{}_resource.new".format(dataset["type"]),
id=dataset["id"]),
extra_environ=env,
data={
"url": "https://example.com/data.csv",
"schema_json": json_value,
"save": "go-dataset-complete",
"id": ""
})
dataset = call_action('package_show', id=dataset['id'])
assert_equals(json.loads(dataset['resources'][0]['schema']), value)
def test_resource_form_fields_are_persisted(self, app):
user = Sysadmin()
dataset = Dataset()
resource = Resource(
package_id=dataset['id'],
url='https://example.com/data.csv',
validation_status='success',
validation_timestamp=datetime.datetime.now().isoformat())
env = {'REMOTE_USER': user['name'].encode('ascii')}
test_desc = 'Test Description'
response = app.post(
url_for("{}_resource.edit".format(dataset["type"]),
id=dataset["id"],
resource_id=resource["id"],
description=test_desc))
dataset = call_action('package_show', id=dataset['id'])
assert_equals(json.loads(dataset['resources'][0]['description']),
test_desc)
app = self._get_test_app()
env, response = _get_resource_update_page_as_sysadmin(
app, dataset['id'], dataset['resources'][0]['id'])
form = response.forms['resource-edit']
form['description'] = 'test desc'
# submit_and_follow(app, form, env, 'save')
dataset = call_action('package_show', id=dataset['id'])
assert_equals(dataset['resources'][0]['validation_status'], 'success')
assert 'validation_timestamp' in dataset['resources'][0]
assert_equals(dataset['resources'][0]['description'], 'test desc')
def test_resource_form_create_valid(self, mock_open, app):
dataset = Dataset()
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
upload = ('upload', 'valid.csv', VALID_CSV)
valid_stream = io.BufferedReader(io.BytesIO(VALID_CSV))
with mock.patch('io.open', return_value=valid_stream):
app.post(url_for("{}_resource.new".format(dataset["type"]),
id=dataset["id"]),
extra_environ=env,
data={
"url": "https://example.com/data.csv",
"upload_files": [upload],
"save": "go-dataset-complete",
"id": ""
})
dataset = call_action('package_show', id=dataset['id'])
assert_equals(dataset['resources'][0]['validation_status'], 'success')
assert 'validation_timestamp' in dataset['resources'][0]
def setup(cls):
# Start data json sources server we can test harvesting against it
reset_db()
harvest_model.setup()
cls.user = Sysadmin()
def _get_extra_env_as_sysadmin():
user = Sysadmin()
return {'REMOTE_USER': user['name'].encode('ascii')}