def test_update(self):
"""
admin + custodian of project for site 1
:return:
"""
rec = self.record_1
previous_data = clone(rec.data)
updated_data = clone(previous_data)
updated_data['Longitude'] = '118.78'
urls = [reverse('api:genericRecord-detail', kwargs={'pk': rec.pk})]
data = {
"data": updated_data,
}
access = {
"forbidden": [self.anonymous_client, self.readonly_client, self.custodian_2_client],
"allowed": [self.admin_client, self.custodian_1_client]
}
for client in access['forbidden']:
for url in urls:
self.assertIn(
client.patch(url, data, format='json').status_code,
[status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]
)
for client in access['allowed']:
for url in urls:
rec.data = previous_data
rec.save()
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_200_OK
)
rec.refresh_from_db()
self.assertEqual(rec.data, updated_data)
def test_update(self):
"""
admin + custodian of project for site 1
:return:
"""
rec = self.record_1
previous_data = clone(rec.data)
updated_data = clone(previous_data)
updated_data['Longitude'] = '118.78'
urls = [reverse('api:record-detail', kwargs={'pk': rec.pk})]
data = {
"data": updated_data,
}
access = {
"forbidden": [self.anonymous_client, self.readonly_client, self.custodian_2_client],
"allowed": [self.admin_client, self.custodian_1_client]
}
for client in access['forbidden']:
for url in urls:
self.assertIn(
client.patch(url, data, format='json').status_code,
[status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]
)
for client in access['allowed']:
for url in urls:
rec.data = previous_data
rec.save()
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_200_OK
)
rec.refresh_from_db()
self.assertEqual(rec.data, updated_data)
def test_bulk_create_forbidden(self):
"""
Bulk create is not authorized for Datasets
:return:
"""
"""
Cannot create multiple dataset
:return:
"""
project = self.project_1
urls = [reverse('api:dataset-list')]
data = [{
"name": "New1 for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': clone(GENERIC_DATA_PACKAGE)
}, {
"name": "New2 for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': clone(GENERIC_DATA_PACKAGE)
}]
access = {
"forbidden": [
self.anonymous_client, self.readonly_client,
self.custodian_2_client, self.admin_client,
self.custodian_1_client, self.data_engineer_1_client,
self.data_engineer_2_client
],
"allowed": []
}
for client in access['forbidden']:
for url in urls:
self.assertIn(
client.post(url, data, format='json').status_code, [
status.HTTP_400_BAD_REQUEST,
status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN
])
for client in access['allowed']:
for url in urls:
# name must be unique
for ds in data:
ds['name'] += '1'
count = Dataset.objects.count()
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED)
self.assertEqual(Dataset.objects.count(), count + 1)
def test_update(self):
"""
Test that the species name is extracted from the data and saved in DB
:return:
"""
# clear all records
ds = self.ds_1
record = self.record_1
new_data = clone(record.data)
# change species name
name = 'Chubby Bat'
self.assertEqual(ds.record_queryset.filter(species_name=name).count(), 0)
new_data[ds.schema.species_name_field.name] = name
data = {
"dataset": record.dataset.pk,
"data": new_data
}
count = ds.record_queryset.count()
url = reverse('api:speciesObservation-detail', kwargs={"pk": record.pk})
client = self.custodian_1_client
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_200_OK
)
self.assertEquals(ds.record_queryset.count(), count)
self.assertEqual(ds.record_queryset.filter(species_name=name).count(), 1)
def test_create(self):
"""
Test that the name_id is retrieved from the species facade from the species_name
:return:
"""
ds = self.ds_1
record = self.record_1
column = ds.schema.species_name_field.name
new_data = clone(record.data)
for species_name, name_id in list(helpers.LightSpeciesFacade().name_id_by_species_name().items())[:2]:
ds.record_queryset.delete()
self.assertEquals(ds.record_queryset.count(), 0)
new_data[column] = species_name
data = {
"dataset": record.dataset.pk,
"data": new_data
}
url = reverse('api:speciesObservation-list')
client = self.custodian_1_client
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED
)
self.assertEquals(ds.record_queryset.count(), 1)
self.assertEquals(ds.record_queryset.first().name_id, name_id)
def test_update_column_not_in_schema(self):
"""
Test that if we introduce a column not in the the dataset it will not validate
:return:
"""
record = self.record_1
incorrect_data = clone(record.data)
incorrect_data['Extra Column'] = "Extra Value"
data = {
"dataset": record.dataset.pk,
"data": incorrect_data
}
url = reverse('api:record-detail', kwargs={"pk": record.pk})
# set strict mode
url = helpers.set_strict_mode(url)
client = self.custodian_1_client
count = Record.objects.count()
self.assertEqual(
client.put(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(Record.objects.count(), count)
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(Record.objects.count(), count)
def test_update_column_not_in_schema(self):
"""
Test that if we introduce a column not in the the dataset it will not validate
:return:
"""
record = self.record_1
incorrect_data = clone(record.data)
incorrect_data['Extra Column'] = "Extra Value"
data = {
"dataset": record.dataset.pk,
"data": incorrect_data
}
url = reverse('api:genericRecord-detail', kwargs={"pk": record.pk})
client = self.custodian_1_client
count = GenericRecord.objects.count()
self.assertEqual(
client.put(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEquals(GenericRecord.objects.count(), count)
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEquals(GenericRecord.objects.count(), count)
def test_create_happy_path(self):
ds = self.ds_1
record = self.record_1
url = reverse('api:dataset-data', kwargs={'pk': ds.pk})
# clear records
self.ds_1.record_queryset.delete()
self.assertEqual(self.ds_1.record_queryset.count(), 0)
r_1_data = clone(record.data)
site_1 = self.record_1.site
r_2_data = clone(record.data)
# change the site for the second record
site_2 = Site.objects.filter(name="Site1").first()
# need to test if the site belongs to the dataset project or the update won't happen
self.assertIsNotNone(site_2)
self.assertTrue(site_2.project == record.dataset.project)
self.assertNotEquals(site_1, site_2)
helpers.set_site(r_2_data, ds, site_2)
data = [
{
"data": r_1_data
},
{
"data": r_2_data
}
]
client = self.custodian_1_client
resp = client.post(url, data, format='json')
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
self.assertEqual(len(resp.data), 2)
self.assertEqual(ds.record_queryset.count(), 2)
# test site/date/geom extraction
for record in ds.record_queryset:
self.assertIsNotNone(record.site)
self.assertIsNotNone(record.datetime)
self.assertIsNotNone(record.geometry)
self.assertEqual(ds.record_queryset.filter(site=site_1).count(), 1)
self.assertEqual(ds.record_queryset.filter(site=site_2).count(), 1)
for r in ds.record_queryset:
self.assertEqual(r.geometry, record.geometry)
self.assertEqual(r.datetime.date(), record.datetime.date())
def test_create_happy_path(self):
ds = self.ds_1
record = self.record_1
self.assertTrue(record.species_name)
self.assertTrue(record.name_id > 0)
url = reverse('api:dataset-data', kwargs={'pk': ds.pk})
# clear records
self.ds_1.record_queryset.delete()
self.assertEqual(self.ds_1.record_queryset.count(), 0)
r_1_data = clone(record.data)
r_2_data = clone(record.data)
data = [
{
"data": r_1_data
},
{
"data": r_2_data
}
]
client = self.custodian_1_client
resp = client.post(url, data, format='json')
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
self.assertEqual(len(resp.data), 2)
self.assertEqual(ds.record_queryset.count(), 2)
# test site/date/geom/species_name/name_id extraction
for r in ds.record_queryset:
self.assertIsNotNone(r.site)
self.assertEqual(r.site, record.site)
self.assertIsNotNone(r.geometry)
self.assertEqual(r.geometry, record.geometry)
self.assertIsNotNone(r.datetime)
self.assertEqual(r.datetime.date(), record.datetime.date())
self.assertTrue(r.species_name)
self.assertEqual(r.species_name, record.species_name)
self.assertTrue(r.name_id > 0)
self.assertEqual(r.name_id, record.name_id)
def test_geometry_error(self):
"""
An observation must have a valid geometry
:return:
"""
ds = self.ds_1
record = self.record_1
lat_column = ds.schema.latitude_field.name
new_data = clone(record.data)
url_post = reverse('api:observation-list')
url_update = reverse('api:observation-detail', kwargs={'pk': record.pk})
valid_values = [-34.125]
for value in valid_values:
new_data[lat_column] = value
data = {
"dataset": record.dataset.pk,
"data": new_data
}
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.post(url_post, data, format='json').status_code,
status.HTTP_201_CREATED
)
self.assertEquals(ds.record_queryset.count(), count + 1)
invalid_values = [None, '', 'abcd']
for value in invalid_values:
new_data[lat_column] = value
data = {
"dataset": record.dataset.pk,
"data": new_data
}
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.post(url_post, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(
client.put(url_update, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(
client.patch(url_update, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEquals(ds.record_queryset.count(), count)
def test_generic_create_happy_path(self):
data_package = clone(GENERIC_DATA_PACKAGE)
url = reverse('api:dataset-list')
project = self.project_1
client = self.data_engineer_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': data_package
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED)
def test_species_observation_create_happy_path(self):
data_package = clone(SPECIES_OBSERVATION_DATA_PACKAGE)
url = reverse('api:dataset-list')
project = self.project_1
client = self.data_engineer_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_SPECIES_OBSERVATION,
"project": project.pk,
'data_package': data_package
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED)
def test_no_schema_error(self):
data_package = clone(GENERIC_DATA_PACKAGE)
data_package['resources'][0]['schema'] = {}
url = reverse('api:dataset-list')
project = self.project_1
client = self.data_engineer_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': {}
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST)
def test_no_schema_error(self):
data_package = clone(GENERIC_DATA_PACKAGE)
data_package['resources'][0]['schema'] = {}
url = reverse('api:dataset-list')
project = self.project_1
client = self.custodian_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': {}
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
def test_species_observation_happy_path(self):
data_package = clone(SPECIES_OBSERVATION_DATA_PACKAGE)
url = reverse('api:dataset-list')
project = self.project_1
client = self.custodian_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_SPECIES_OBSERVATION,
"project": project.pk,
'data_package': data_package
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED
)
def test_generic_happy_path(self):
data_package = clone(GENERIC_DATA_PACKAGE)
url = reverse('api:dataset-list')
project = self.project_1
client = self.custodian_1_client
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': data_package
}
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_201_CREATED
)
def test_species_name(self):
ds = self.ds_1
record = self.record_1
column = ds.schema.species_name_field.name
new_data = clone(record.data)
url_post = reverse('api:speciesObservation-list')
url_update = reverse('api:speciesObservation-detail', kwargs={'pk': record.pk})
valid_values = ['Canis Lupus', 'chubby bat', 'anything']
for value in valid_values:
new_data[column] = value
data = {
"dataset": record.dataset.pk,
"data": new_data
}
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.post(url_post, data, format='json').status_code,
status.HTTP_201_CREATED
)
self.assertEquals(ds.record_queryset.count(), count + 1)
invalid_values = [None, '', 125]
for value in invalid_values:
new_data[column] = value
data = {
"dataset": record.dataset.pk,
"data": new_data
}
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.post(url_post, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(
client.put(url_update, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEqual(
client.patch(url_update, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEquals(ds.record_queryset.count(), count)
def test_create_column_not_in_schema(self):
"""
Test that if we introduce a column not in the the dataset it will not validate
:return:
"""
ds = self.ds_1
record = self.record_1
incorrect_data = clone(record.data)
incorrect_data['Extra Column'] = "Extra Value"
data = {
"dataset": record.dataset.pk,
"data": incorrect_data
}
url = reverse('api:observation-list')
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.post(url, data, format='json').status_code,
status.HTTP_400_BAD_REQUEST
)
self.assertEquals(ds.record_queryset.count(), count)
def test_update(self):
"""
Test that the name_id is retrieved from the species facade from the species_name
:return:
"""
ds = self.ds_1
record = self.record_1
column = ds.schema.species_name_field.name
new_data = clone(record.data)
for species_name, name_id in list(helpers.LightSpeciesFacade().name_id_by_species_name().items())[:2]:
new_data[column] = species_name
data = {
"dataset": record.dataset.pk,
"data": new_data
}
url = reverse('api:speciesObservation-detail', kwargs={"pk": record.pk})
client = self.custodian_1_client
self.assertEqual(
client.put(url, data, format='json').status_code,
status.HTTP_200_OK
)
record.refresh_from_db()
self.assertEqual(record.name_id, name_id)
def test_update(self):
"""
Test that the date and geometry are extracted from the data
and saved in DB
:return:
"""
# clear all records
ds = self.ds_1
record = self.record_1
new_data = clone(record.data)
# change date
date = '20/4/2016'
expected_date = datetime.date(2016, 4, 20)
new_data[ds.schema.observation_date_field.name] = date
# change lat/lon
lon = 111.111
lat = 22.222
new_data[ds.schema.longitude_field.name] = lon
new_data[ds.schema.latitude_field.name] = lat
expected_geojson = Point(lon, lat).geojson
data = {
"dataset": record.dataset.pk,
"data": new_data
}
url = reverse('api:observation-detail', kwargs={"pk": record.pk})
client = self.custodian_1_client
count = ds.record_queryset.count()
self.assertEqual(
client.patch(url, data, format='json').status_code,
status.HTTP_200_OK
)
self.assertEquals(ds.record_queryset.count(), count)
dtz = timezone.localtime(ds.record_queryset.first().datetime)
self.assertEquals(dtz.date(), expected_date)
self.assertEquals(ds.record_queryset.first().geometry.geojson, expected_geojson)
def test_create(self):
"""
Admin and data engineers
:return:
"""
data_package = clone(GENERIC_DATA_PACKAGE)
project = self.project_1
urls = [reverse('api:dataset-list')]
data = {
"name": "New for Unit test",
"type": Dataset.TYPE_GENERIC,
"project": project.pk,
'data_package': data_package
}
access = {
"forbidden": [
self.anonymous_client, self.readonly_client,
self.custodian_2_client, self.custodian_1_client,
self.data_engineer_2_client
],
"allowed": [self.admin_client, self.data_engineer_1_client]
}
for client in access['forbidden']:
for url in urls:
self.assertIn(
client.post(url, data, format='json').status_code,
[status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN])
for client in access['allowed']:
for url in urls:
# name must be unique
data['name'] += '1'
count = Dataset.objects.count()
resp = client.post(url, data, format='json')
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
self.assertEqual(Dataset.objects.count(), count + 1)
