def test_get_creds_if_must_sync(self):
pkg = Dataset()
creds = api._get_creds_if_must_sync(pkg)
self.assertEqual(None, creds)
pkg = Dataset(owner_org=self.org['id'])
creds = api._get_creds_if_must_sync(pkg)
self.assertNotEqual(None, creds)
old_creds = creds
creds.integration = False
creds = api._get_creds_if_must_sync(pkg)
self.assertEqual(None, creds)
old_creds.integration = True
def test_resource_form_update_upload(self, app):
value = {'fields': [{'name': 'code'}, {'name': 'department'}]}
dataset = Dataset(resources=[{
'url': 'https://example.com/data.csv',
'schema': value
}])
app = self._get_test_app()
env, response = _get_resource_update_page_as_sysadmin(
app, dataset['id'], dataset['resources'][0]['id'])
form = response.forms['resource-edit']
assert_equals(form['schema_json'].value, json.dumps(value, indent=2))
value = {
'fields': [{
'name': 'code'
}, {
'name': 'department'
}, {
'name': 'date'
}]
}
form['url'] = 'https://example.com/data.csv'
dataset = call_action('package_show', id=dataset['id'])
assert_equals(dataset['resources'][0]['schema'], value)
def test_delete_extra(self):
changes = []
original = Dataset(extras=[
{
u"key": u"subject",
u"value": u"science"
},
{
u"key": u"topic",
u"value": u"wind"
},
])
new = helpers.call_action(
u"package_patch",
id=original["id"],
extras=[{
u"key": u"topic",
u"value": u"wind"
}],
)
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
assert changes[0]["type"] == u"extra_fields"
assert changes[0]["method"] == u"remove_one"
assert changes[0]["key"] == u"subject"
def test_resource_form_update_invalid(self, mock_open):
dataset = Dataset(
resources=[{
'name': 'test_resource_form_update_invalid',
'url': 'https://example.com/data.csv'
}]
)
resource_id = dataset['resources'][0]['id']
response = _get_resource_update_page_as_sysadmin(
self.app, dataset['id'], resource_id)
upload = ('upload', 'invalid.csv', INVALID_CSV)
params = {
'name': 'test_resource_form_update_invalid',
'url': 'https://example.com/data.csv'
}
invalid_stream = io.BufferedReader(io.BytesIO(INVALID_CSV))
with mock.patch('io.open', return_value=invalid_stream):
response = _get_response_body(_post(
self.app, EDIT_RESOURCE_URL.format(dataset['id'], resource_id),
params, resource_id=resource_id, upload=[upload]))
assert 'validation' in response
assert 'missing-value' in response
assert 'Row 2 has a missing value in column 4' in response
def test_resource_form_update_valid(self, mock_open):
dataset = Dataset(resources=[
{
'url': 'https://example.com/data.csv'
}
])
upload = ('upload', 'valid.csv', VALID_CSV)
valid_stream = io.BufferedReader(io.BytesIO(VALID_CSV))
params = {
'name': 'test_resource_form_update_valid',
'url': 'https://example.com/data.csv'
}
resource_id = dataset['resources'][0]['id']
with mock.patch('io.open', return_value=valid_stream):
_post(self.app, EDIT_RESOURCE_URL.format(dataset['id'], resource_id),
params, resource_id=resource_id, upload=[upload])
dataset = call_action('package_show', id=dataset['id'])
assert dataset['resources'][0]['validation_status'] == 'success'
assert 'validation_timestamp' in dataset['resources'][0]
def test_resource_form_includes_json_fields(self, app):
dataset = Dataset()
response = _get_resource_new_page_as_sysadmin(app, dataset['id'])
form = _get_form(response)
assert form.find("textarea", attrs={'name': 'validation_options'})
def test_delete_resource(self):
changes = []
original = Dataset(
resources=[
{
u"url": u"http://example.com/image.png",
u"format": u"png",
u"name": u"Image 1",
},
{
u"url": u"http://example.com/image.png",
u"format": u"png",
u"name": u"Image 2",
},
]
)
new = copy.deepcopy(original)
del new["resources"][0]
new = helpers.call_action(u"package_update", **new)
check_resource_changes(changes, original, new, u"fake")
assert len(changes) == 1, changes
assert changes[0]["type"] == u"delete_resource"
assert changes[0]["resource_name"] == u"Image 1"
def test_change_multiple_extras(self):
changes = []
original = Dataset(
extras=[
{u"key": u"subject", u"value": u"science"},
{u"key": u"topic", u"value": u"wind"},
]
)
new = helpers.call_action(
u"package_patch",
id=original["id"],
extras=[
{u"key": u"subject", u"value": u"scientific"},
{u"key": u"topic", u"value": u"rain"},
],
)
check_metadata_changes(changes, original, new)
assert len(changes) == 2, changes
for change in changes:
assert change["type"] == u"extra_fields"
assert change["method"] == u"change_with_old_value"
if change["key"] == u"subject":
assert change["new_value"] == u"scientific"
else:
assert change["key"] == u"topic"
assert change["new_value"] == u"rain"
def test_change_extra(self):
changes = []
original = Dataset(extras=[{
u'key': u'subject',
u'value': u'science'
}, {
u'key': u'topic',
u'value': u'wind'
}])
new = helpers.call_action(u'package_patch',
id=original['id'],
extras=[{
u'key': u'subject',
u'value': u'scientific'
}, {
u'key': u'topic',
u'value': u'wind'
}])
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
eq(changes[0]['type'], u'extra_fields')
eq(changes[0]['method'], u'change_with_old_value')
eq(changes[0]['key'], u'subject')
eq(changes[0]['old_value'], u'science')
eq(changes[0]['new_value'], u'scientific')
def test_delete_multiple_resources(self):
changes = []
original = Dataset(resources=[{
u'url': u'http://example.com/image.png',
u'format': u'png',
u'name': u'Image 1'
}, {
u'url': u'http://example.com/image.png',
u'format': u'png',
u'name': u'Image 2'
}, {
u'url': u'http://example.com/image.png',
u'format': u'png',
u'name': u'Image 3'
}])
new = copy.deepcopy(original)
del new['resources'][1]
del new['resources'][0]
new = helpers.call_action(u'package_update', **new)
check_resource_changes(changes, original, new, u'fake')
assert len(changes) == 2, changes
eq(changes[0]['type'], u'delete_resource')
if changes[0]['resource_name'] == u'Image 1':
eq(changes[1]['resource_name'], u'Image 2')
else:
eq(changes[0]['resource_name'], u'Image 2')
eq(changes[1]['resource_name'], u'Image 1')
def test_add_multiple_resources(self):
changes = []
original = Dataset()
new = helpers.call_action(u'package_patch',
id=original['id'],
resources=[{
u'url': u'http://example.com/image.png',
u'format': u'png',
u'name': u'Image 1'
}, {
u'url': u'http://example.com/image2.png',
u'format': u'png',
u'name': u'Image 2'
}])
check_resource_changes(changes, original, new, u'fake')
assert len(changes) == 2, changes
eq(changes[0]['type'], u'new_resource')
eq(changes[1]['type'], u'new_resource')
if changes[0]['resource_name'] == u'Image 1':
eq(changes[1]['resource_name'], u'Image 2')
else:
eq(changes[1]['resource_name'], u'Image 1')
eq(changes[0]['resource_name'], u'Image 2')
def test_delete_multiple_extras(self):
changes = []
original = Dataset(extras=[{
u'key': u'subject',
u'value': u'science'
}, {
u'key': u'topic',
u'value': u'wind'
}, {
u'key': u'geography',
u'value': u'global'
}])
new = helpers.call_action(u'package_patch',
id=original['id'],
extras=[{
u'key': u'topic',
u'value': u'wind'
}])
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
eq(changes[0]['type'], u'extra_fields')
eq(changes[0]['method'], u'remove_multiple')
eq(set(changes[0]['key_list']), set((u'subject', u'geography')))
def test_resource_form_update(self, app):
value = {
'delimiter': ';',
'headers': 2,
'skip_rows': ['#'],
}
dataset = Dataset()
resource = Resource(package_id=dataset['id'],
url='https://example.com/data.csv',
validation_options=value)
value = {
'delimiter': ';',
'headers': 2,
'skip_rows': ['#'],
'skip_tests': ['blank-rows'],
}
json_value = json.dumps(value)
call_action("resource_update",
id=resource["id"],
name="somethingnew",
validation_options=json_value)
dataset = call_action('package_show', id=dataset['id'])
assert_equals(
json.loads(dataset['resources'][0]['validation_options']), value)
def test_resource_form_create(self, app):
dataset = Dataset()
user = Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
value = {
'delimiter': ';',
'headers': 2,
'skip_rows': ['#'],
}
json_value = json.dumps(value)
app.post(url_for("{}_resource.new".format(dataset["type"]),
id=dataset["id"]),
extra_environ=env,
data={
"url": "https://example.com/data.csv",
"validation_options": json_value,
"save": "go-dataset-complete",
"id": ""
})
dataset = call_action('package_show', id=dataset['id'])
assert_equals(
json.loads(dataset['resources'][0]['validation_options']), value)
def test_notify(self, sync):
pkg = Dataset(type='related-dataset')
self.assertFalse(api.notify(pkg['id']))
pkg = Dataset()
self.assertFalse(api.notify(pkg['id']))
pkg = Dataset(owner_org=self.org['id'], state='draft')
self.assertFalse(api.notify(pkg['id']))
pkg = Dataset(owner_org=self.org['id'], state='deleted')
self.assertTrue(api.notify(pkg['id']))
pkg = Dataset(owner_org=self.org['id'])
self.assertTrue(api.notify(pkg['id']))
sync.assert_called_with(pkg)
def test_change_multiple_resources(self):
changes = []
original = Dataset(resources=[
{
u"url": u"http://example.com/image.png",
u"format": u"png",
u"name": u"Image 1",
},
{
u"url": u"http://example.com/image.png",
u"format": u"png",
u"name": u"Image 2",
},
{
u"url": u"http://example.com/image.png",
u"format": u"png",
u"name": u"Image 3",
},
])
new = copy.deepcopy(original)
new["resources"][0]["name"] = u"changed-1"
new["resources"][1]["name"] = u"changed-2"
new = helpers.call_action(u"package_update", **new)
check_resource_changes(changes, original, new, u"fake")
assert len(changes) == 2, changes
assert changes[0]["type"] == u"resource_name"
assert changes[1]["type"] == u"resource_name"
if changes[0]["old_resource_name"] == u"Image 1":
assert changes[0]["new_resource_name"] == u"changed-1"
else:
assert changes[0]["old_resource_name"] == u"Image 2"
assert changes[0]["new_resource_name"] == u"changed-2"
def test_resource_form_update_json(self, app):
value = {'fields': [{'name': 'code'}, {'name': 'department'}]}
dataset = Dataset()
resource = Resource(package_id=dataset['id'],
url='https://example.com/data.csv',
schema=value)
value = {
"fields": [{
"name": "code"
}, {
"name": "department"
}, {
"name": "date"
}]
}
json_value = json.dumps(value)
call_action("resource_update",
id=resource["id"],
name="somethingnew",
schema_json=json_value)
dataset = call_action('package_show', id=dataset['id'])
assert_equals(dataset['resources'][0]['schema'], value)
def test_delete_multiple_resources(self):
changes = []
original = Dataset(resources=[
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 1",
},
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 2",
},
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 3",
},
])
new = copy.deepcopy(original)
del new["resources"][1]
del new["resources"][0]
new = helpers.call_action("package_update", **new)
check_resource_changes(changes, original, new, "fake")
assert len(changes) == 2, changes
assert changes[0]["type"] == "delete_resource"
if changes[0]["resource_name"] == "Image 1":
assert changes[1]["resource_name"] == "Image 2"
else:
assert changes[0]["resource_name"] == "Image 2"
assert changes[1]["resource_name"] == "Image 1"
def test_change_extra(self):
changes = []
original = Dataset(extras=[
{
"key": "subject",
"value": "science"
},
{
"key": "topic",
"value": "wind"
},
])
new = helpers.call_action(
"package_patch",
id=original["id"],
extras=[
{
"key": "subject",
"value": "scientific"
},
{
"key": "topic",
"value": "wind"
},
],
)
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
assert changes[0]["type"] == "extra_fields"
assert changes[0]["method"] == "change_with_old_value"
assert changes[0]["key"] == "subject"
assert changes[0]["old_value"] == "science"
assert changes[0]["new_value"] == "scientific"
def test_add_multiple_resources(self):
changes = []
original = Dataset()
new = helpers.call_action(
"package_patch",
id=original["id"],
resources=[
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 1",
},
{
"url": "http://example.com/image2.png",
"format": "png",
"name": "Image 2",
},
],
)
check_resource_changes(changes, original, new, "fake")
assert len(changes) == 2, changes
assert changes[0]["type"] == "new_resource"
assert changes[1]["type"] == "new_resource"
if changes[0]["resource_name"] == "Image 1":
assert changes[1]["resource_name"] == "Image 2"
else:
assert changes[1]["resource_name"] == "Image 1"
assert changes[0]["resource_name"] == "Image 2"
def test_change_resource_description(self):
changes = []
original = Dataset(resources=[
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 1",
"description": "First image",
},
{
"url": "http://example.com/image.png",
"format": "png",
"name": "Image 2",
"description": "Second image",
},
])
new = copy.deepcopy(original)
new["resources"][1]["description"] = "changed"
new = helpers.call_action("package_update", **new)
check_resource_changes(changes, original, new, "fake")
assert len(changes) == 1, changes
assert changes[0]["type"] == "resource_desc"
assert changes[0]["method"] == "change"
assert changes[0]["resource_name"] == "Image 2"
def test_remove_multiple_tags(self):
changes = []
original = Dataset(tags=[
{
"name": "rivers"
},
{
"name": "oceans"
},
{
"name": "streams"
},
])
new = helpers.call_action("package_patch",
id=original["id"],
tags=[{
"name": "rivers"
}])
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
assert changes[0]["type"] == "tags"
assert changes[0]["method"] == "remove_multiple"
assert set(changes[0]["tags"]) == set(("oceans", "streams"))
def test_delete_multiple_extras(self):
changes = []
original = Dataset(extras=[
{
"key": "subject",
"value": "science"
},
{
"key": "topic",
"value": "wind"
},
{
"key": "geography",
"value": "global"
},
])
new = helpers.call_action(
"package_patch",
id=original["id"],
extras=[{
"key": "topic",
"value": "wind"
}],
)
check_metadata_changes(changes, original, new)
assert len(changes) == 1, changes
assert changes[0]["type"] == "extra_fields"
assert changes[0]["method"] == "remove_multiple"
assert set(changes[0]["key_list"]) == set(("subject", "geography"))
def test_daily_frequency_subscriptions_of_an_org_are_not_included(self):
org = Organization()
factories.Subscription(organization_id=org['id'], frequency='daily')
Dataset(owner_org=org['id'])
notifies = get_weekly_notifications()
eq(_get_activities(notifies), [])
def test_resource_form_includes_json_fields(self, app):
dataset = Dataset()
app = self._get_test_app()
env, response = _get_resource_new_page_as_sysadmin(app, dataset['id'])
form = response.forms['resource-edit']
assert_in('validation_options', form.fields)
assert_equals(form.fields['validation_options'][0].tag, 'textarea')
def test_sync(self, update, create):
pkg = Dataset()
self.api.sync(pkg)
self.assertTrue(create.called)
self.assertFalse(update.called)
create.reset_mock()
self.api.sync(pkg)
self.assertFalse(create.called)
self.assertTrue(update.called)
update.reset_mock()
pkg = Dataset(state='deleted')
self.api.sync(pkg)
self.assertFalse(create.called)
self.assertFalse(update.called)
def test_bad_email(self):
dataset = Dataset()
response = self._get_test_app().post('/subscribe/signup',
params={
'email': 'invalid email',
'dataset': dataset['id']
},
status=400)
response.mustcontain(u'Email supplied is invalid')
def test_empty_email(self):
dataset = Dataset()
response = self._get_test_app().post('/subscribe/signup',
params={
'email': '',
'dataset': dataset['id']
},
status=400)
response.mustcontain(u'No email address supplied')
def test_remove_url(self):
changes = []
original = Dataset(url="first url")
new = helpers.call_action("package_patch", id=original["id"], url="")
check_metadata_changes(changes, original, new)
assert changes[0]["type"] == "url"
assert changes[0]["method"] == "remove"
def test_immediate_frequency_subscriptions_of_an_group_are_not_included(
self):
group = Group()
factories.Subscription(group_id=group['id'], frequency='immediate')
Dataset(groups=[{'id': group['id']}])
notifies = get_weekly_notifications()
eq(_get_activities(notifies), [])
