def test_with_nested_initial_elements(self):
fake = Fake.objects.create(nested=[
Nested(name=faker.name(), sub=SubNested(name=faker.word())),
Nested(name=faker.name(), sub=SubNested(name=faker.word())),
])
order = [n.id for n in fake.nested]
data = [
{'id': str(fake.nested[0].id)},
{'id': str(fake.nested[1].id)},
{'name': faker.name(), 'sub': {'name': faker.name()}},
]
form = self.factory({'nested': data}, fake, sub=True)
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(len(fake.nested), 3)
for idx, id in enumerate(order):
nested = fake.nested[idx]
self.assertEqual(nested.id, id)
self.assertIsNotNone(nested.sub)
self.assertIsNotNone(nested.sub.name)
self.assertIsNotNone(fake.nested[2].id)
self.assertEqual(fake.nested[2].sub.name, data[2]['sub']['name'])
def test_suggest_tags_api_with_unicode(self):
'''It should suggest tags'''
with self.autoindex():
for i in range(3):
tags = [
faker.word(),
faker.word(), 'testé', 'testé-{0}'.format(i)
]
ReuseFactory(tags=tags, visible=True)
DatasetFactory(tags=tags, visible=True)
response = self.get(url_for('api.suggest_tags'),
qs={
'q': 'testé',
'size': '5'
})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
self.assertEqual(response.json[0]['text'], 'teste')
for suggestion in response.json:
self.assertIn('text', suggestion)
self.assertIn('score', suggestion)
self.assertTrue(suggestion['text'].startswith('teste'))
def test_with_nested_initial_elements(self):
fake = Fake.objects.create(nested=[
Nested(name=faker.name(), sub=SubNested(name=faker.word())),
Nested(name=faker.name(), sub=SubNested(name=faker.word())),
])
order = [n.id for n in fake.nested]
data = [
{'id': str(fake.nested[0].id)},
{'id': str(fake.nested[1].id)},
{'name': faker.name(), 'sub': {'name': faker.name()}},
]
form = self.factory({'nested': data}, fake, sub=True)
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(len(fake.nested), 3)
for idx, id in enumerate(order):
nested = fake.nested[idx]
self.assertEqual(nested.id, id)
self.assertIsNotNone(nested.sub)
self.assertIsNotNone(nested.sub.name)
self.assertIsNotNone(fake.nested[2].id)
self.assertEqual(fake.nested[2].sub.name, data[2]['sub']['name'])
def test_suggest_tags_api_with_unicode(self, api, autoindex):
'''It should suggest tags'''
with autoindex:
for i in range(3):
tags = [
faker.word(),
faker.word(), 'testé', 'testé-{0}'.format(i)
]
ReuseFactory(tags=tags, visible=True)
DatasetFactory(tags=tags, visible=True)
response = api.get(url_for('api.suggest_tags'),
qs={
'q': 'testé',
'size': '5'
})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
assert response.json[0]['text'] == 'teste'
for suggestion in response.json:
assert 'text' in suggestion
assert 'score' in suggestion
assert suggestion['text'].startswith('teste')
def test_type_ok(self, type, expected):
label = faker.word()
key = faker.word()
description = faker.sentence()
hf = HarvestFilter(label, key, type, description)
assert hf.as_dict() == {
'label': label,
'key': key,
'type': expected,
'description': description,
}
def test_suggest_organizations_api(self, api):
'''It should suggest organizations'''
for i in range(3):
OrganizationFactory(
name='test-{0}'.format(i) if i % 2 else faker.word(),
metrics={"followers": i})
max_follower_organization = OrganizationFactory(
name='test-4',
metrics={"followers": 10}
)
response = api.get(url_for('api.suggest_organizations'),
qs={'q': 'tes', 'size': '5'})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
for suggestion in response.json:
assert 'id' in suggestion
assert 'slug' in suggestion
assert 'name' in suggestion
assert 'image_url' in suggestion
assert 'acronym' in suggestion
assert 'tes' in suggestion['name']
assert response.json[0]['id'] == str(max_follower_organization.id)
def test_suggest_users_api_first_name(self):
'''It should suggest users based on first name'''
with self.autoindex():
for i in range(4):
UserFactory(first_name='test-{0}'.format(i) if i %
2 else faker.word())
response = self.get(url_for('api.suggest_users'),
qs={
'q': 'tes',
'size': '5'
})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('first_name', suggestion)
self.assertIn('last_name', suggestion)
self.assertIn('avatar_url', suggestion)
self.assertIn('slug', suggestion)
self.assertIn('score', suggestion)
self.assertIn('test', suggestion['first_name'])
def test_suggest_datasets_api_unicode(self):
'''It should suggest datasets with special characters'''
with self.autoindex():
for i in range(4):
DatasetFactory(title='testé-{0}'.format(i) if i %
2 else faker.word(),
resources=[ResourceFactory()])
response = self.get(url_for('api.suggest_datasets'),
qs={
'q': 'testé',
'size': '5'
})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('title', suggestion)
self.assertIn('slug', suggestion)
self.assertIn('score', suggestion)
self.assertIn('image_url', suggestion)
self.assertTrue(suggestion['title'].startswith('test'))
def test_suggest_zones_on_extra_list_key(self):
'''It should suggest zones based on any list key'''
with self.autoindex():
for i in range(4):
GeoZoneFactory(
name='in' if i % 2 else 'not-in',
keys={
str(i):
['test-{0}'.format(i) if i % 2 else faker.word()]
},
is_current=True)
response = self.get(url_for('api.suggest_zones'),
qs={
'q': 'test',
'size': '5'
})
self.assert200(response)
self.assertEqual(len(response.json), 2)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('name', suggestion)
self.assertIn('code', suggestion)
self.assertIn('level', suggestion)
self.assertIn('keys', suggestion)
self.assertIsInstance(suggestion['keys'], dict)
self.assertEqual(suggestion['name'], 'in')
def test_suggest_organizations_with_apostrophe(self):
'''It should suggest organizations with words'''
with self.autoindex():
for i in range(4):
OrganizationFactory(
name='Ministère de l\'intérieur {0}'.format(i) if i %
2 else faker.word())
response = self.get(url_for('api.suggest_organizations'),
qs={
'q': 'Ministère intérieur',
'size': '5'
})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('slug', suggestion)
self.assertIn('name', suggestion)
self.assertIn('score', suggestion)
self.assertIn('image_url', suggestion)
self.assertTrue(
suggestion['name'].startswith('Ministère de l\'intérieur'))
def test_suggest_organizations_api(self, api, autoindex):
'''It should suggest organizations'''
with autoindex:
for i in range(4):
OrganizationFactory(name='test-{0}'.format(i) if i %
2 else faker.word())
response = api.get(url_for('api.suggest_organizations'),
qs={
'q': 'tes',
'size': '5'
})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
for suggestion in response.json:
assert 'id' in suggestion
assert 'slug' in suggestion
assert 'name' in suggestion
assert 'score' in suggestion
assert 'image_url' in suggestion
assert 'acronym' in suggestion
assert suggestion['name'].startswith('test')
def test_suggest_zones_on_name(self):
'''It should suggest zones based on its name'''
with self.autoindex():
for i in range(4):
GeoZoneFactory(name='test-{0}'.format(i) if i %
2 else faker.word(),
is_current=True)
response = self.get(url_for('api.suggest_zones'),
qs={
'q': 'test',
'size': '5'
})
self.assert200(response)
self.assertEqual(len(response.json), 2)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('name', suggestion)
self.assertIn('code', suggestion)
self.assertIn('level', suggestion)
self.assertIn('keys', suggestion)
self.assertIsInstance(suggestion['keys'], dict)
self.assertTrue(suggestion['name'].startswith('test'))
def test_suggest_reuses_api_unicode(self, api, autoindex):
'''It should suggest reuses with special characters'''
with autoindex:
for i in range(4):
ReuseFactory(title='testé-{0}'.format(i) if i %
2 else faker.word(),
visible=True)
response = api.get(url_for('api.suggest_reuses'),
qs={
'q': 'testé',
'size': '5'
})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
for suggestion in response.json:
assert 'id' in suggestion
assert 'slug' in suggestion
assert 'title' in suggestion
assert 'score' in suggestion
assert 'image_url' in suggestion
assert suggestion['title'].startswith('test')
def test_suggest_organizations_with_apostrophe(self, api, autoindex):
'''It should suggest organizations with words'''
with autoindex:
for i in range(4):
OrganizationFactory(
name='Ministère de l\'intérieur {0}'.format(i) if i %
2 else faker.word())
response = api.get(url_for('api.suggest_organizations'),
qs={
'q': 'Ministère intérieur',
'size': '5'
})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
for suggestion in response.json:
assert 'id' in suggestion
assert 'slug' in suggestion
assert 'name' in suggestion
assert 'score' in suggestion
assert 'image_url' in suggestion
assert suggestion['name'].startswith('Ministère de l\'intérieur')
def test_create_source_with_false_feature(self, api):
'''It should handled negative values'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'features': {
'test': False,
'toggled': False,
}
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['config'] == {
'features': {
'test': False,
'toggled': False,
}
}
def test_base_rendering(self):
response = internal(faker.word(), 32)
assert200(response)
assert response.mimetype == 'image/png'
assert response.is_streamed
etag, weak = response.get_etag()
assert etag is not None
def test_suggest_reuses_api(self, api):
'''It should suggest reuses'''
for i in range(3):
ReuseFactory(title='test-{0}'.format(i) if i % 2 else faker.word(),
visible=True,
metrics={"followers": i})
max_follower_reuse = ReuseFactory(title='test-4',
visible=True,
metrics={"followers": 10})
response = api.get(url_for('api.suggest_reuses'),
qs={
'q': 'tes',
'size': '5'
})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
for suggestion in response.json:
assert 'id' in suggestion
assert 'slug' in suggestion
assert 'title' in suggestion
assert 'image_url' in suggestion
assert 'test' in suggestion['title']
assert response.json[0]['id'] == str(max_follower_reuse.id)
class TestDataset(TerritoryDataset):
order = 1
id = faker.word()
title = faker.sentence()
organization_id = str(org.id)
description = faker.paragraph()
temporal_coverage = {'start': 2007, 'end': 2012}
url_template = 'http://somehere.com/{code}'
def create_chunks(self, uuid, nb=3, last=None):
for i in range(nb):
storages.chunks.write(chunk_filename(uuid, i), faker.word())
storages.chunks.write(chunk_filename(uuid, META), json.dumps({
'uuid': str(uuid),
'filename': faker.file_name(),
'totalparts': nb + 1,
'lastchunk': last or datetime.now(),
}))
class FakeFactory(MongoEngineFactory):
class Meta:
model = Fake
title = factory.LazyAttribute(lambda o: faker.sentence())
description = factory.LazyAttribute(lambda o: faker.paragraph())
tags = factory.LazyAttribute(
lambda o: [faker.word() for _ in range(1, randint(1, 4))])
sub = factory.SubFactory(NestedFactory)
def test_create_with_non_submitted_elements(self):
form = self.factory({'name': faker.word()})
form.validate()
self.assertEqual(form.errors, {})
fake = form.save()
self.assertIsNotNone(fake.name)
self.assertIsNone(fake.nested)
def test_user_api_full_text_search_last_name(self):
'''It should find users based on last name'''
for i in range(4):
UserFactory(last_name='test-{0}'.format(i) if i %
2 else faker.word())
response = self.get(url_for('api.users', q='test'))
self.assert200(response)
self.assertEqual(len(response.json['data']), 2)
def hit_factory():
return {
"_score": float(faker.random_number(2)),
"_type": "fake",
"_id": faker.md5(),
"_source": {
"title": faker.sentence(),
"tags": [faker.word() for _ in range(faker.random_digit())]
},
"_index": "udata-test"
}
def hit_factory():
return {
"_score": float(faker.random_number(2)),
"_type": "fake",
"_id": faker.md5(),
"_source": {
"title": faker.sentence(),
"tags": [faker.word() for _ in range(faker.random_digit())]
},
"_index": "udata-test"
}
def test_suggest_format_api_no_match(self):
'''It should not provide format suggestion if no match'''
with self.autoindex():
DatasetFactory(resources=[
ResourceFactory(format=faker.word()) for _ in range(3)
])
response = self.get(url_for('api.suggest_formats'),
qs={'q': 'test', 'size': '5'})
self.assert200(response)
self.assertEqual(len(response.json), 0)
def test_create_source_with_owner(self, api):
'''It should create and attach a new source to an owner'''
user = api.login()
data = {'name': faker.word(), 'url': faker.url(), 'backend': 'factory'}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['validation']['state'] == VALIDATION_PENDING
assert source['owner']['id'] == str(user.id)
assert source['organization'] is None
def test_create_source_with_config(self, api):
'''It should create a new source with configuration'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'filters': [
{
'key': 'test',
'value': 1
},
{
'key': 'test',
'value': 42
},
{
'key': 'tag',
'value': 'my-tag'
},
],
'features': {
'test': True,
'toggled': True,
}
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['config'] == {
'filters': [
{
'key': 'test',
'value': 1
},
{
'key': 'test',
'value': 42
},
{
'key': 'tag',
'value': 'my-tag'
},
],
'features': {
'test': True,
'toggled': True,
}
}
def test_tag_filter(ckan):
# create 2 datasets with a different tag each
tag = faker.word()
package = package_factory(ckan, tags=[{'name': tag}])
package_factory(ckan, tags=[{'name': faker.word()}])
source = HarvestSourceFactory(
backend='ckan',
url=ckan.BASE_URL,
config={'filters': [{
'key': 'tags',
'value': tag
}]})
actions.run(source.slug)
source.reload()
job = source.get_last_job()
assert len(job.items) == 1
assert Dataset.objects.count() == 1
assert Dataset.objects.first().extras['harvest:remote_id'] == package['id']
def test_suggest_formats_api(self):
'''It should suggest formats'''
with self.autoindex():
DatasetFactory(resources=[
ResourceFactory(format=f)
for f in (faker.word(), faker.word(), 'test', 'test-1')
])
response = self.get(url_for('api.suggest_formats'),
qs={'q': 'test', 'size': '5'})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
# Shortest match first.
self.assertEqual(response.json[0]['text'], 'test')
for suggestion in response.json:
self.assertIn('text', suggestion)
self.assertIn('score', suggestion)
self.assertTrue(suggestion['text'].startswith('test'))
def test_get_values(self):
buckets = [{
'key': faker.word(),
'doc_count': faker.random_number(2)
} for _ in range(10)]
result = self.factory(aggregations=bucket_agg_factory(buckets))
assert len(result) == 10
for row in result:
assert isinstance(row[0], basestring)
assert isinstance(row[1], int)
assert isinstance(row[2], bool)
def test_suggest_tags_api_with_unicode(self, api, autoindex):
'''It should suggest tags'''
with autoindex:
for i in range(3):
tags = [faker.word(), faker.word(), 'testé',
'testé-{0}'.format(i)]
ReuseFactory(tags=tags, visible=True)
DatasetFactory(tags=tags, visible=True)
response = api.get(url_for('api.suggest_tags'),
qs={'q': 'testé', 'size': '5'})
assert200(response)
assert len(response.json) <= 5
assert len(response.json) > 1
assert response.json[0]['text'] == 'teste'
for suggestion in response.json:
assert 'text' in suggestion
assert 'score' in suggestion
assert suggestion['text'].startswith('teste')
def test_with_non_submitted_initial_elements(self):
fake = Fake.objects.create(nested=Nested(name=faker.name()))
initial_id = fake.nested.id
initial_name = fake.nested.name
form = self.factory({'name': faker.word()}, fake)
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(fake.nested.id, initial_id)
self.assertEqual(fake.nested.name, initial_name)
def test_user_api_full_text_search_unicode(self):
'''It should find user with special characters'''
for i in range(4):
UserFactory(first_name='test-{0}'.format(i) if i %
2 else faker.word())
user = UserFactory(first_name='test', last_name='testé')
response = self.get(url_for('api.users', q='test testé'))
self.assert200(response)
self.assertEqual(len(response.json['data']), 3)
self.assertEqual(response.json['data'][0]['id'], str(user.id))
def test_with_non_submitted_initial_elements(self):
fake = Fake.objects.create(nested=[
Nested(name=faker.name()),
Nested(name=faker.name()),
])
initial = [(n.id, n.name) for n in fake.nested]
form = self.factory({'name': faker.word()}, fake)
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(len(fake.nested), len(initial))
for idx, (id, name) in enumerate(initial):
nested = fake.nested[idx]
self.assertEqual(nested.id, id)
self.assertEqual(nested.name, name)
def test_all_fields(self):
uri = 'https://test.org/dataset'
node = URIRef(uri)
g = Graph()
id = faker.uuid4()
title = faker.sentence()
acronym = faker.word()
description = faker.paragraph()
tags = faker.words(nb=3)
start = faker.past_date(start_date='-30d')
end = faker.future_date(end_date='+30d')
g.set((node, RDF.type, DCAT.Dataset))
g.set((node, DCT.identifier, Literal(id)))
g.set((node, DCT.title, Literal(title)))
g.set((node, SKOS.altLabel, Literal(acronym)))
g.set((node, DCT.description, Literal(description)))
g.set((node, DCT.accrualPeriodicity, FREQ.daily))
pot = BNode()
g.add((node, DCT.temporal, pot))
g.set((pot, RDF.type, DCT.PeriodOfTime))
g.set((pot, SCHEMA.startDate, Literal(start)))
g.set((pot, SCHEMA.endDate, Literal(end)))
for tag in tags:
g.add((node, DCAT.keyword, Literal(tag)))
dataset = dataset_from_rdf(g)
dataset.validate()
assert isinstance(dataset, Dataset)
assert dataset.title == title
assert dataset.acronym == acronym
assert dataset.description == description
assert dataset.frequency == 'daily'
assert set(dataset.tags) == set(tags)
assert isinstance(dataset.temporal_coverage, db.DateRange)
assert dataset.temporal_coverage.start == start
assert dataset.temporal_coverage.end == end
extras = dataset.extras
assert 'dct:identifier' in extras
assert extras['dct:identifier'] == id
assert 'uri' in extras
assert extras['uri'] == uri
def test_suggest_zones_on_name(self):
'''It should suggest zones based on its name'''
with self.autoindex():
for i in range(4):
GeoZoneFactory(name='test-{0}'.format(i)
if i % 2 else faker.word())
response = self.get(
url_for('api.suggest_zones'), qs={'q': 'test', 'size': '5'})
self.assert200(response)
self.assertEqual(len(response.json), 2)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('name', suggestion)
self.assertIn('code', suggestion)
self.assertIn('level', suggestion)
self.assertIn('keys', suggestion)
self.assertIsInstance(suggestion['keys'], dict)
self.assertTrue(suggestion['name'].startswith('test'))
def test_suggest_users_api_last_name(self):
'''It should suggest users based on last name'''
with self.autoindex():
for i in range(4):
UserFactory(
last_name='test-{0}'.format(i) if i % 2 else faker.word())
response = self.get(url_for('api.suggest_users'),
qs={'q': 'tes', 'size': '5'})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('first_name', suggestion)
self.assertIn('last_name', suggestion)
self.assertIn('avatar_url', suggestion)
self.assertIn('score', suggestion)
self.assertIn('test', suggestion['last_name'])
def test_suggest_reuses_api_unicode(self):
'''It should suggest reuses with special characters'''
with self.autoindex():
for i in range(4):
ReuseFactory(
title='testé-{0}'.format(i) if i % 2 else faker.word(),
datasets=[DatasetFactory()])
response = self.get(url_for('api.suggest_reuses'),
qs={'q': 'testé', 'size': '5'})
self.assert200(response)
self.assertLessEqual(len(response.json), 5)
self.assertGreater(len(response.json), 1)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('slug', suggestion)
self.assertIn('title', suggestion)
self.assertIn('score', suggestion)
self.assertIn('image_url', suggestion)
self.assertTrue(suggestion['title'].startswith('test'))
def test_suggest_zones_on_extra_list_key(self):
'''It should suggest zones based on any list key'''
with self.autoindex():
for i in range(4):
GeoZoneFactory(
name='in' if i % 2 else 'not-in',
keys={str(i): ['test-{0}'.format(i)
if i % 2 else faker.word()]}
)
response = self.get(url_for('api.suggest_zones'),
qs={'q': 'test', 'size': '5'})
self.assert200(response)
self.assertEqual(len(response.json), 2)
for suggestion in response.json:
self.assertIn('id', suggestion)
self.assertIn('name', suggestion)
self.assertIn('code', suggestion)
self.assertIn('level', suggestion)
self.assertIn('keys', suggestion)
self.assertIsInstance(suggestion['keys'], dict)
self.assertEqual(suggestion['name'], 'in')
def test_stream_nested_from_adapter(self):
fake = FakeFactory.build()
for i in range(3):
fake.nested.append(NestedFake(key=faker.word(), value=i))
fake.save()
response = self.get(url_for('testcsv.from_nested'))
self.assert200(response)
self.assertEqual(response.mimetype, 'text/csv')
self.assertEqual(response.charset, 'utf-8')
csvfile = StringIO.StringIO(response.data)
reader = csv.get_reader(csvfile)
header = reader.next()
self.assertEqual(header, ['title', 'description', 'key', 'alias'])
rows = list(reader)
self.assertEqual(len(rows), len(fake.nested))
for row, obj in zip(rows, fake.nested):
self.assertEqual(len(row), len(header))
self.assertEqual(row[0], fake.title)
self.assertEqual(row[1], fake.description)
self.assertEqual(row[2], obj.key)
self.assertEqual(row[3], str(obj.value))
def test_render_twice_the_same(self):
identifier = faker.word()
assert_stream_equal(internal(identifier, 32),
internal(identifier, 32))