def youckan_api_response(**kwargs):
'''A YouCKAN ME API response factory'''
data = {
'profile': {
'website': faker.url(),
'city': faker.city(),
'about': faker.text(),
'avatar': faker.url() + 'avatar.png',
},
'first_name': faker.first_name(),
'last_name': faker.last_name(),
'email': faker.email(),
'is_active': True,
'is_superuser': False,
'date_joined': datetime.now().isoformat(),
'slug': None,
}
for key in data.keys():
if key in kwargs:
data[key] = kwargs[key]
data['fullname'] = ' '.join((data['first_name'], data['last_name']))
if not data['slug']:
data['slug'] = slugify.slugify(data['fullname'].lower())
return data
def test_preview_from_config(self):
org = OrganizationFactory()
source_url = faker.url()
count = 10
job = actions.preview_from_config('Test source',
source_url,
'factory',
organization=org,
config={'count': count})
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert len(job.items) == count
for item in job.items:
assert item.status == 'done'
assert item.errors == []
assert item.started is not None
assert item.ended is not None
assert item.dataset is not None
dataset = item.dataset
assert dataset.organization == org
assert 'harvest:remote_id' in dataset.extras
assert 'harvest:last_update' in dataset.extras
assert 'harvest:source_id' in dataset.extras
assert len(HarvestJob.objects) == 0
assert len(Dataset.objects) == 0
def test_create_source_with_false_feature(self, api):
'''It should handled negative values'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'features': {
'test': False,
'toggled': False,
}
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['config'] == {
'features': {
'test': False,
'toggled': False,
}
}
def process(self, item):
'''Generate a random dataset from a fake identifier'''
# Get or create a harvested dataset with this identifier.
# Harvest metadata are already filled on creation.
dataset = self.get_dataset(item.remote_id)
# Here you comes your implementation. You should :
# - fetch the remote dataset (if necessary)
# - validate the fetched payload
# - map its content to the dataset fields
# - store extra significant data in the `extra` attribute
# - map resources data
dataset.title = faker.sentence()
dataset.description = faker.text()
dataset.tags = list(set(faker.words(nb=faker.pyint())))
# Resources
for i in range(faker.pyint()):
dataset.resources.append(
Resource(title=faker.sentence(),
description=faker.text(),
url=faker.url(),
filetype='remote',
mime=faker.mime_type(category='text'),
format=faker.file_extension(category='text'),
filesize=faker.pyint()))
return dataset
def test_update(self):
resource = ResourceFactory()
self.dataset.resources.append(resource)
self.dataset.save()
now = datetime.now()
data = {
'title': faker.sentence(),
'description': faker.text(),
'url': faker.url(),
'published': now.isoformat(),
'extras': {
'extra:id': 'id',
}
}
with self.api_user():
response = self.put(
url_for('api.resource',
dataset=self.dataset,
rid=str(resource.id)), data)
self.assert200(response)
self.dataset.reload()
self.assertEqual(len(self.dataset.resources), 1)
updated = self.dataset.resources[0]
self.assertEqual(updated.title, data['title'])
self.assertEqual(updated.description, data['description'])
self.assertEqual(updated.url, data['url'])
self.assertEqual(updated.extras, {'extra:id': 'id'})
self.assertEqualDates(updated.published, now)
def test_preview_from_config(self):
org = OrganizationFactory()
source_url = faker.url()
count = 10
job = actions.preview_from_config('Test source',
source_url,
'factory',
organization=org,
config={'count': count})
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert len(job.items) == count
for item in job.items:
assert item.status == 'done'
assert item.errors == []
assert item.started is not None
assert item.ended is not None
assert item.dataset is not None
dataset = item.dataset
assert dataset.organization == org
assert 'harvest:remote_id' in dataset.extras
assert 'harvest:last_update' in dataset.extras
assert 'harvest:source_id' in dataset.extras
assert len(HarvestJob.objects) == 0
assert len(Dataset.objects) == 0
def test_bulk_update(self):
resources = ResourceFactory.build_batch(2)
self.dataset.resources.extend(resources)
self.dataset.save()
now = datetime.now()
ids = [r.id for r in self.dataset.resources]
data = [{
'id': str(id),
'title': faker.sentence(),
'description': faker.text(),
} for id in ids]
data.append({
'title': faker.sentence(),
'description': faker.text(),
'url': faker.url(),
})
with self.api_user():
response = self.put(url_for('api.resources', dataset=self.dataset),
data)
self.assert200(response)
self.dataset.reload()
self.assertEqual(len(self.dataset.resources), 3)
for idx, id in enumerate(ids):
resource = self.dataset.resources[idx]
rdata = data[idx]
self.assertEqual(str(resource.id), rdata['id'])
self.assertEqual(resource.title, rdata['title'])
self.assertEqual(resource.description, rdata['description'])
self.assertIsNotNone(resource.url)
new_resource = self.dataset.resources[-1]
self.assertEqualDates(new_resource.published, now)
def resource_factory():
return {
"resource_group_id": str(uuid4()),
"cache_last_updated": None,
"revision_timestamp": "2013-10-01T15:59:56.322481",
"webstore_last_updated": "2013-10-01T17:59:56.238951",
"id": str(uuid4()),
"size": "1375",
"state": "active",
"hash": "689afc083c6316259955f499580bdf41bfc5e495",
"description": faker.paragraph(),
"format": "CSV",
"tracking_summary": {
"total": 0,
"recent": 0
},
"mimetype_inner": None,
"mimetype": "text/csv",
"cache_url": None,
"name": faker.sentence(),
"created": "2013-08-01T09:43:09.031465",
"url": faker.url(),
"webstore_url": "active",
"last_modified": "2013-10-01T17:59:55.552785",
"position": 0,
"revision_id": str(uuid4()),
"resource_type": "file.upload"
}
def test_users(self):
'''It should provide a list of users'''
with self.autoindex():
user = UserFactory(about=faker.paragraph(),
website=faker.url(),
avatar_url=faker.url(),
metrics={'datasets': 10})
response = self.get(url_for('api.users'))
self.assert200(response)
[json] = response.json['data']
self.assertEqual(json['id'], str(user.id))
self.assertEqual(json['slug'], user.slug)
self.assertEqual(json['first_name'], user.first_name)
self.assertEqual(json['last_name'], user.last_name)
self.assertEqual(json['website'], user.website)
self.assertEqual(json['about'], user.about)
self.assertEqual(json['metrics'], user.metrics)
def test_users(self):
'''It should provide a list of users'''
with self.autoindex():
user = UserFactory(
about=faker.paragraph(),
website=faker.url(),
avatar_url=faker.url(),
metrics={'datasets': 10})
response = self.get(url_for('api.users'))
self.assert200(response)
[json] = response.json['data']
self.assertEquals(json['id'], str(user.id))
self.assertEquals(json['slug'], user.slug)
self.assertEquals(json['first_name'], user.first_name)
self.assertEquals(json['last_name'], user.last_name)
self.assertEquals(json['website'], user.website)
self.assertEquals(json['about'], user.about)
self.assertEquals(json['metrics'], user.metrics)
class ReuseFactory(ModelFactory):
class Meta:
model = Reuse
title = factory.Faker('sentence')
description = factory.Faker('text')
url = factory.LazyAttribute(
lambda o: '/'.join([faker.url(), faker.unique_string()]))
type = FuzzyChoice(REUSE_TYPES.keys())
def test_user_api_update_with_website(self):
'''It should raise a 400'''
self.login(AdminFactory())
user = UserFactory()
data = user.to_dict()
data['website'] = 'foo'
response = self.put(url_for('api.user', user=user), data)
self.assert400(response)
data['website'] = faker.url()
response = self.put(url_for('api.user', user=user), data)
self.assert200(response)
def test_create_source_with_config(self):
source_url = faker.url()
config = {'filters': [{'key': 'test', 'value': 42}]}
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory',
config=config)
assert source.config == config
def test_update_404(self):
data = {
'title': faker.sentence(),
'description': faker.text(),
'url': faker.url(),
}
with self.api_user():
response = self.put(url_for('api.resource',
dataset=self.dataset,
rid=str(ResourceFactory().id)), data)
self.assert404(response)
def test_user_api_update_with_website(self):
'''It should raise a 400'''
self.login(AdminFactory())
user = UserFactory()
data = user.to_dict()
data['website'] = 'foo'
response = self.put(url_for('api.user', user=user), data)
self.assert400(response)
data['website'] = faker.url()
response = self.put(url_for('api.user', user=user), data)
self.assert200(response)
def test_update_source(self):
source = HarvestSourceFactory()
data = source.to_dict()
new_url = faker.url()
data['url'] = new_url
with assert_emit(signals.harvest_source_updated):
source = actions.update_source(source.id, data)
self.assertEqual(source.url, new_url)
source.reload()
self.assertEqual(source.url, new_url)
def test_create_source_with_owner(self, api):
'''It should create and attach a new source to an owner'''
user = api.login()
data = {'name': faker.word(), 'url': faker.url(), 'backend': 'factory'}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['validation']['state'] == VALIDATION_PENDING
assert source['owner']['id'] == str(user.id)
assert source['organization'] is None
def test_update_source(self):
source = HarvestSourceFactory()
data = source.to_dict()
new_url = faker.url()
data['url'] = new_url
with assert_emit(signals.harvest_source_updated):
source = actions.update_source(source.id, data)
assert source.url == new_url
source.reload()
assert source.url == new_url
def test_create_source_with_config(self, api):
'''It should create a new source with configuration'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'filters': [
{
'key': 'test',
'value': 1
},
{
'key': 'test',
'value': 42
},
{
'key': 'tag',
'value': 'my-tag'
},
],
'features': {
'test': True,
'toggled': True,
}
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['config'] == {
'filters': [
{
'key': 'test',
'value': 1
},
{
'key': 'test',
'value': 42
},
{
'key': 'tag',
'value': 'my-tag'
},
],
'features': {
'test': True,
'toggled': True,
}
}
def test_with_valid_url_is_stripped(self):
Fake, FakeForm = self.factory()
fake = Fake()
url = faker.url()
form = FakeForm(MultiDict({'url': url + ' '}))
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(fake.url, url)
def test_with_valid_url_is_stripped(self):
Fake, FakeForm = self.factory()
fake = Fake()
url = faker.url()
form = FakeForm(MultiDict({'url': url + ' '}))
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(fake.url, url)
def test_create_source_with_org_not_member(self, api):
'''It should create and attach a new source to an organization'''
user = api.login()
member = Member(user=user, role='editor')
org = OrganizationFactory(members=[member])
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'organization': str(org.id)
}
response = api.post(url_for('api.harvest_sources'), data)
assert403(response)
class ReuseFactory(ModelFactory):
class Meta:
model = Reuse
title = factory.Faker('sentence')
description = factory.Faker('text')
url = factory.LazyAttribute(
lambda o: '/'.join([faker.url(), faker.unique_string()]))
type = FuzzyChoice(REUSE_TYPES.keys())
topic = FuzzyChoice(REUSE_TOPICS.keys())
class Params:
visible = factory.Trait(
datasets=factory.LazyAttribute(lambda o: [DatasetFactory()]))
def test_simple():
org = OrganizationFactory()
source = HarvestSourceFactory(backend='{{ cookiecutter.identifier }}',
url=faker.url(),
organization=org)
# TODO: mock remote endpoints responses
actions.run(source.slug)
source.reload()
job = source.get_last_job()
assert len(job.items) > 0
def test_create_source_with_config(self):
source_url = faker.url()
config = {
'filters': [{'key': 'test', 'value': 42}],
'features': {'key': True},
}
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory',
config=config)
assert source.config == config
def test_create_source_with_not_boolean_feature(self, api):
'''It should handled negative values'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'features': {
'test': 'not a boolean',
}
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert400(response)
def test_create_source_with_config_with_custom_key(self, api):
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'custom': 'value'
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['config'] == {'custom': 'value'}
def test_create_source_with_unknown_feature(self, api):
'''Can only use known features in config'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'features': {
'unknown': True
},
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert400(response)
def test_update_source(self, api):
'''It should update a source'''
user = api.login()
source = HarvestSourceFactory(owner=user)
new_url = faker.url()
data = {
'name': source.name,
'description': source.description,
'url': new_url,
'backend': 'factory',
}
api_url = url_for('api.harvest_source', ident=str(source.id))
response = api.put(api_url, data)
assert200(response)
source = response.json
assert source['url'] == new_url
def test_create_source_with_bad_filter_format(self, api):
'''Filters should have the right format'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'filters': [
{
'key': 'unknown',
'notvalue': 'any'
},
]
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert400(response)
def test_create_source_with_org(self, api):
'''It should create and attach a new source to an organization'''
user = api.login()
member = Member(user=user, role='admin')
org = OrganizationFactory(members=[member])
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'organization': str(org.id)
}
response = api.post(url_for('api.harvest_sources'), data)
assert201(response)
source = response.json
assert source['validation']['state'] == VALIDATION_PENDING
assert source['owner'] is None
assert source['organization']['id'] == str(org.id)
def test_create_source_with_unknown_filter(self, api):
'''Can only use known filters in config'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'filters': [
{
'key': 'unknown',
'value': 'any'
},
]
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert400(response)
def test_create_source_with_bad_filter_type(self, api):
'''Can only use the xpected filter type'''
api.login()
data = {
'name': faker.word(),
'url': faker.url(),
'backend': 'factory',
'config': {
'filters': [
{
'key': 'test',
'value': 'not-an-integer'
},
]
}
}
response = api.post(url_for('api.harvest_sources'), data)
assert400(response)
def test_harvest_source_id(self):
nb_datasets = 3
source = HarvestSourceFactory(config={'nb_datasets': nb_datasets})
backend = FakeBackend(source)
job = backend.harvest()
assert len(job.items) == nb_datasets
source_url = faker.url()
source.url = source_url
source.save()
job = backend.harvest()
datasets = Dataset.objects()
# no new datasets have been created
assert len(datasets) == nb_datasets
for dataset in datasets:
assert dataset.extras['harvest:source_id'] == str(source.id)
parsed = urlparse(source_url).netloc.split(':')[0]
assert parsed == dataset.extras['harvest:domain']
def test_create_source(self):
source_url = faker.url()
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source', source_url,
'factory')
self.assertEqual(source.name, 'Test source')
self.assertEqual(source.slug, 'test-source')
self.assertEqual(source.url, source_url)
self.assertEqual(source.backend, 'factory')
self.assertEqual(source.frequency, 'manual')
self.assertTrue(source.active)
self.assertIsNone(source.owner)
self.assertIsNone(source.organization)
self.assertEqual(source.validation.state, VALIDATION_PENDING)
self.assertIsNone(source.validation.on)
self.assertIsNone(source.validation.by)
self.assertIsNone(source.validation.comment)
def test_create_source(self):
source_url = faker.url()
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source', source_url,
'factory')
assert source.name == 'Test source'
assert source.slug == 'test-source'
assert source.url == source_url
assert source.backend == 'factory'
assert source.frequency == 'manual'
assert source.active
assert source.owner is None
assert source.organization is None
assert source.validation.state == VALIDATION_PENDING
assert source.validation.on is None
assert source.validation.by is None
assert source.validation.comment is None
def test_create_source(self):
source_url = faker.url()
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory')
assert source.name == 'Test source'
assert source.slug == 'test-source'
assert source.url == source_url
assert source.backend == 'factory'
assert source.frequency == 'manual'
assert source.active
assert source.owner is None
assert source.organization is None
assert source.validation.state == VALIDATION_PENDING
assert source.validation.on is None
assert source.validation.by is None
assert source.validation.comment is None
def test_defaults(self):
source = HarvestSource.objects.create(name='Test', url=faker.url())
assert source.name == 'Test'
assert source.slug == 'test'
def test_initial_values(self):
Fake, FakeForm = self.factory()
fake = Fake(url=faker.url())
form = FakeForm(None, obj=fake)
self.assertEqual(form.url.data, fake.url)
def test_source_from_config(self, api):
api.login()
data = {'name': faker.word(), 'url': faker.url(), 'backend': 'factory'}
response = api.post(url_for('api.preview_harvest_source_config'), data)
assert200(response)