def test_without_datum_zone(self):
"""
A schema without datum/zone can be accepted as it will use the project's
:return:
"""
schema_fields = [{
"name": "Easting",
"type": "number",
"format": "default",
"constraints": {
"required": True,
}
}, {
"name": "Northing",
"type": "number",
"format": "default",
"constraints": {
"required": True,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_easting_northing)
self.assertTrue(parser.is_easting_northing_only)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_lat_long_only)
self.assertFalse(parser.is_site_code)
self.assertFalse(parser.is_site_code_only)
self.assertIsNotNone(parser.easting_field)
self.assertIsNotNone(parser.northing_field)
def test_with_foreign_key(self):
"""
Alternatively to naming or tagging the column we can use a foreign key
"""
schema_fields = [
{
"name": "Island",
"type": "string",
"format": "default",
"constraints": {
"required": True,
}
},
]
schema = helpers.create_schema_from_fields(schema_fields)
schema = helpers.add_model_field_foreign_key_to_schema(
schema, {
'schema_field': 'Island',
'model': 'Site',
'model_field': 'code'
})
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_site_code_only)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_lat_long_only)
self.assertFalse(parser.is_easting_northing)
self.assertFalse(parser.is_easting_northing_only)
self.assertIsNotNone(parser.site_code_field)
self.assertEqual(parser.site_code_field.name, 'Island')
def test_biosys_type_without_required(self):
"""
If it is a site code only schema, the column must be required
"""
schema_fields = [
{
"name": "Site",
"type": "string",
"format": "default",
"biosys": {
"type": "siteCode"
},
},
]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
expected_error = "The field named 'Site' must have the 'required' constraint set to true."
self.assertEqual(parser.errors[0], expected_error)
self.assertTrue(parser.is_site_code_only)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNotNone(parser.site_code_field)
self.assertEqual(parser.site_code_field.name, 'Site')
def test_happy_path(self):
schema_fields = [{
"name": "Latitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Longitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_lat_long)
self.assertTrue(parser.is_lat_long_only)
self.assertFalse(parser.is_easting_northing)
self.assertFalse(parser.is_easting_northing_only)
self.assertFalse(parser.is_site_code)
self.assertFalse(parser.is_site_code_only)
self.assertIsNotNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
def schema_with_easting_northing():
schema_fields = [{
"name": "What",
"type": "string",
"constraints": helpers.REQUIRED_CONSTRAINTS
}, {
"name": "When",
"type": "date",
"constraints": helpers.REQUIRED_CONSTRAINTS,
"format": "any",
"biosys": {
'type': 'observationDate'
}
}, {
"name": "Northing",
"type": "number",
"constraints": helpers.REQUIRED_CONSTRAINTS,
}, {
"name": "Easting",
"type": "number",
"constraints": helpers.REQUIRED_CONSTRAINTS
}, {
"name": "Datum",
"type": "string",
"constraints": helpers.REQUIRED_CONSTRAINTS
}, {
"name": "Zone",
"type": "string",
"constraints": helpers.REQUIRED_CONSTRAINTS
}]
return helpers.create_schema_from_fields(schema_fields)
def test_lat_long_and_site_code_fk_not_required(self):
"""
Schema with lat/long and a site code foreign key
"""
schema_fields = [
{
"name": "What",
"type": "string",
"constraints": helpers.REQUIRED_CONSTRAINTS
},
{
"name": "When",
"type": "date",
"constraints": helpers.REQUIRED_CONSTRAINTS,
"format": "any",
"biosys": {
'type': 'observationDate'
}
},
{
"name": "Latitude",
"type": "number",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS,
"biosys": {
"type": 'latitude'
}
},
{
"name": "Longitude",
"type": "number",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS,
"biosys": {
"type": 'longitude'
}
},
{
"name": "Site Code",
"type": "string",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
},
]
schema = helpers.create_schema_from_fields(schema_fields)
schema = helpers.add_model_field_foreign_key_to_schema(
schema, {
'schema_field': 'Site Code',
'model': 'Site',
'model_field': 'code'
})
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertFalse(parser.is_site_code_only)
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_lat_long_only)
self.assertFalse(parser.is_easting_northing)
self.assertFalse(parser.is_easting_northing_only)
self.assertIsNotNone(parser.site_code_field)
self.assertEqual(parser.site_code_field.name, 'Site Code')
def test_two_biosys_type_is_error(self):
"""
Can't have two fields of the same type
:return:
"""
schema_fields = [{
"name": "Lat",
"type": "number",
"format": "default",
"biosys": {
"type": 'latitude'
},
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Another lat",
"type": "number",
"format": "default",
"biosys": {
"type": 'latitude'
},
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Long",
"type": "number",
"format": "default",
"biosys": {
"type": 'longitude'
},
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
expected_errors = [
"More than one Biosys type latitude field found: ['Lat', 'Another lat']",
]
self.assertIn(parser.errors[0], expected_errors)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
def test_biosys_tag_precedence(self):
"""
Two fields one name 'Latitude' and another one tagged as biosys type latitude
The biosys one is chosen
"""
schema_fields = [{
"name": "The Observation Latitude",
"type": "number",
"format": "default",
"biosys": {
"type": 'latitude'
},
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Latitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Longitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNotNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
self.assertEqual(parser.latitude_field.name,
'The Observation Latitude')
self.assertEqual(parser.longitude_field.name, 'Longitude')
def test_required_date_with_format_any(self):
"""
field of type date, required with format should
"""
schema_fields = [{
"name": "DateAny",
"type": "date",
"format": "any",
"constraints": helpers.REQUIRED_CONSTRAINTS
}, {
"name": "What",
"type": "string",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
}]
schema = helpers.create_schema_from_fields(schema_fields)
dataset = self.assert_create_dataset(schema)
records = [
['DateAny', 'What'],
[None, 'something'],
['', 'something'],
[' ', 'something'],
]
resp = self._upload_records_from_rows(records,
dataset_pk=dataset.pk,
strict=True)
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
received = resp.json()
self.assertIsInstance(received, list)
self.assertEqual(len(received), 3)
# this what an report should look like
expected_row_report = {
'row': 3,
'errors': {
'DateAny':
'Field "DateAny" has constraint "required" which is not satisfied for value "None"'
},
'warnings': {}
}
for row_report in received:
self.assertIn('errors', row_report)
errors = row_report.get('errors')
self.assertIn('DateAny', errors)
msg = errors.get('DateAny')
self.assertEqual(msg, expected_row_report['errors']['DateAny'])
def test_with_column_name_only(self):
schema_fields = [
{
"name": "Site Code",
"type": "string",
"format": "default",
"constraints": {
"required": True,
}
},
]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_site_code_only)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNotNone(parser.site_code_field)
self.assertEqual(parser.site_code_field.name, 'Site Code')
def test_biosys_tag_happy_path(self):
"""
columns not named latitude or longitude but with biosys tags
"""
schema_fields = [{
"name": "Lat",
"type": "number",
"format": "default",
"biosys": {
"type": 'latitude'
},
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Long",
"type": "number",
"format": "default",
"biosys": {
"type": 'longitude'
},
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNotNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
self.assertFalse(parser.is_site_code)
self.assertFalse(parser.is_site_code_only)
self.assertEqual(parser.latitude_field.name, 'Lat')
self.assertEqual(parser.longitude_field.name, 'Long')
def test_not_required_date_with_format_any(self):
"""
field of type date, not required with format any should not raise an error when received a empty string
see https://decbugs.com/view.php?id=6928
"""
schema_fields = [
{
"name": "DateAny",
"type": "date",
"format": "any",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
},
{
"name": "DateTimeAny",
"type": "datetime",
"format": "any",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
},
{
"name": "DateDefault",
"type": "date",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
},
{
"name": "DateTimeDefault",
"type": "datetime",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
}
]
schema = helpers.create_schema_from_fields(schema_fields)
dataset = self.assert_create_dataset(schema)
records = [
['DateAny', 'DateTimeAny', 'DateDefault', 'DateTimeDefault'],
[None, None, None, None],
['', '', '', ''],
[' ', ' ', ' ', ' '],
]
resp = self._upload_records_from_rows(records, dataset_pk=dataset.pk, strict=True)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
def test_two_latitude_name_is_error(self):
"""
Can't have two fields Latitude (no biosys tag)
"""
schema_fields = [{
"name": "Latitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
}, {
"name": "Latitude",
"type": "number",
"format": "default",
}, {
"name": "Longitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
expected_errors = [
"More than one field named Latitude found.",
]
self.assertIn(parser.errors[0], expected_errors)
self.assertFalse(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
def test_with_wrong_column_name(self):
"""
The coloumn name must be Site Code not Site
:return:
"""
schema_fields = [
{
"name": "Site",
"type": "string",
"format": "default",
"constraints": {
"required": True,
}
},
]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
expected_error = "The schema must contain some geometry fields: latitude/longitude or easting/northing " \
"or alternatively a reference to the Site Code."
self.assertEqual(parser.errors[0], expected_error)
def test_biosys_tag_without_required(self):
"""
columns not named latitude or longitude but with biosys tags must be required.
"""
schema_fields = [{
"name": "Lat",
"type": "number",
"format": "default",
"biosys": {
"type": 'latitude'
},
}, {
"name": "Long",
"type": "number",
"format": "default",
"biosys": {
"type": 'longitude'
},
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 2)
expected_errors = [
"The field named 'Lat' must have the 'required' constraint set to true.",
"The field named 'Long' must have the 'required' constraint set to true."
]
self.assertIn(parser.errors[0], expected_errors)
self.assertIn(parser.errors[1], expected_errors)
self.assertNotEqual(parser.errors[0], parser.errors[1])
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
self.assertIsNotNone(parser.latitude_field)
self.assertIsNotNone(parser.longitude_field)
self.assertEqual(parser.latitude_field.name, 'Lat')
self.assertEqual(parser.longitude_field.name, 'Long')
def test_with_datum_zone(self):
"""
A schema with datum/zone
:return:
"""
schema_fields = [{
"name": "Easting",
"type": "number",
"format": "default",
"constraints": {
"required": True,
}
}, {
"name": "Northing",
"type": "number",
"format": "default",
"constraints": {
"required": True,
}
}, {
"name": "Datum",
"type": "string"
}, {
"name": "Zone",
"type": "integer"
}]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertTrue(parser.is_valid())
self.assertEqual(len(parser.errors), 0)
self.assertTrue(parser.is_easting_northing)
self.assertFalse(parser.is_lat_long)
self.assertIsNotNone(parser.easting_field)
self.assertIsNotNone(parser.northing_field)
self.assertIsNotNone(parser.datum_field)
self.assertIsNotNone(parser.zone_field)
def test_fk_with_dataset_code(self):
"""
This is the same test as above but this time the foreign key is declared with the parent dataset code instead
of name
:return:
"""
# Create a parent dataset with some records
parent_dataset = self._create_dataset_and_records_from_rows([
['Survey ID', 'Where', 'When', 'Who'],
['ID-001', 'King\'s Park', '2018-07-15', 'Tim Reynolds'],
['ID-002', 'Cottesloe', '2018-07-11', 'SLB'],
['ID-003', 'Somewhere', '2018-07-13', 'Phil Bill']
])
parent_dataset.data_package['resources'][0]['schema']['primaryKey'] = 'Survey ID'
parent_dataset.save()
parent_dataset.code = 'Survey'
parent_dataset.save()
parent_records = parent_dataset.record_set.all()
self.assertEqual(parent_records.count(), 3)
# Create a child/related schema
child_schema = helpers.create_schema_from_fields([
{
"name": "Survey ID",
"type": "string",
"constraints": helpers.REQUIRED_CONSTRAINTS
},
{
"name": "What",
"type": "string",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS
},
{
"name": "Comments",
"type": "string",
"constraints": helpers.NOT_REQUIRED_CONSTRAINTS,
}
])
# declaring a foreign key pointing to the dataset name
foreign_keys = [{
'fields': 'Survey ID',
'reference': {
'fields': 'Survey ID',
'resource': parent_dataset.code
}
}]
child_schema['foreignKeys'] = foreign_keys
child_dataset = self._create_dataset_with_schema(
self.project_1,
self.data_engineer_1_client,
child_schema
)
self.assertIsNotNone(child_dataset)
# post some records for survey ID--001
rows = [
['Survey ID', 'What', 'Comments'],
['ID-001', 'Canis lupus', 'doggy, doggy'],
['ID-001', 'A frog', 'kiss'],
['ID-001', 'A tooth brush', 'I should stop drinking'],
]
self._upload_records_from_rows(rows, child_dataset.id, strict=False)
children_records = child_dataset.record_set.all()
self.assertEqual(children_records.count(), 3)
# test serialisation of parent records
id_001 = parent_records.filter(data__contains={'Survey ID': 'ID-001'}).first()
expected_children_ids = [r.id for r in children_records]
expected_parent_id = None
self.assertIsNotNone(id_001)
url = reverse('api:record-detail', kwargs={'pk': id_001.pk})
client = self.custodian_1_client
resp = client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
data = resp.json()
self.assertEqual(sorted(data['children']), sorted(expected_children_ids))
self.assertEqual(data['parent'], expected_parent_id)
id_002 = parent_records.filter(data__contains={'Survey ID': 'ID-002'}).first()
expected_children_ids = []
expected_parent_id = None
self.assertIsNotNone(id_002)
url = reverse('api:record-detail', kwargs={'pk': id_002.pk})
client = self.custodian_1_client
resp = client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
data = resp.json()
self.assertEqual(sorted(data['children']), sorted(expected_children_ids))
self.assertEqual(data['parent'], expected_parent_id)
id_003 = parent_records.filter(data__contains={'Survey ID': 'ID-003'}).first()
expected_children_ids = []
expected_parent_id = None
self.assertIsNotNone(id_003)
url = reverse('api:record-detail', kwargs={'pk': id_003.pk})
client = self.custodian_1_client
resp = client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
data = resp.json()
self.assertEqual(sorted(data['children']), sorted(expected_children_ids))
self.assertEqual(data['parent'], expected_parent_id)
#
# test serialisation of children records
# they all have the same parent and no children
#
expected_children_ids = None
expected_parent_id = id_001.pk
client = self.custodian_1_client
for record in children_records:
url = reverse('api:record-detail', kwargs={'pk': record.pk})
resp = client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
data = resp.json()
self.assertEqual(data['children'], expected_children_ids)
self.assertEqual(data['parent'], expected_parent_id)
def test_excel_type(self):
schema_fields = [
{
"name": "string",
"type": "string",
"format": "default"
},
{
"name": "number",
"type": "number",
"format": "default"
},
{
"name": "integer",
"type": "integer",
"format": "default"
},
{
"name": "date",
"type": "date",
"format": "any"
},
{
"name": "datetime",
"type": "datetime",
"format": "any"
},
{
"name": "boolean",
"type": "boolean",
"format": "default"
}
]
schema = helpers.create_schema_from_fields(schema_fields)
project = self.project_1
client = self.data_engineer_1_client
dataset = self._create_dataset_with_schema(project, client, schema)
# create one record
record_data = {
"string": "string",
"number": 12.3,
"integer": 456,
"date": "21/06/2017",
"datetime": "13/04/2017 15:55",
"boolean": 'yes'
}
payload = {
'dataset': dataset.pk,
'data': record_data
}
url = reverse('api:record-list')
resp = client.post(url, data=payload, format='json')
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
record = Record.objects.filter(id=resp.json().get('id')).first()
self.assertIsNotNone(record)
url = reverse('api:record-list')
query = {
'dataset__id': dataset.pk,
'output': 'xlsx'
}
try:
resp = client.get(url, query)
except Exception as e:
self.fail("Export should not raise an exception: {}".format(e))
self.assertEqual(resp.status_code, status.HTTP_200_OK)
# load workbook
wb = load_workbook(six.BytesIO(resp.content))
ws = wb[dataset.name]
rows = list(ws.rows)
self.assertEqual(len(rows), 2)
cells = rows[1]
string, number, integer, date, datetime, boolean = cells
# excel type are string, number or boolean
self.assertEqual(string.data_type, Cell.TYPE_STRING)
self.assertEqual(number.data_type, Cell.TYPE_NUMERIC)
self.assertEqual(integer.data_type, Cell.TYPE_NUMERIC)
self.assertEqual(date.data_type, Cell.TYPE_NUMERIC)
self.assertEqual(datetime.data_type, Cell.TYPE_NUMERIC)
self.assertEqual(boolean.data_type, Cell.TYPE_BOOL)
def test_without_required(self):
"""
If the schema contains only lat/long they must have a required constraint
"""
schema_fields = [
{
"name": "Latitude",
"type": "number",
"format": "default",
"constraints": {
"minimum": -90.0,
"maximum": 90.0,
}
},
{
"name": "Longitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -180.0,
"maximum": 180.0,
}
},
]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
self.assertEqual(
parser.errors[0],
"The field named 'Latitude' must have the 'required' constraint set to true."
)
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
schema_fields = [
{
"name": "Latitude",
"type": "number",
"format": "default",
"constraints": {
"required": True,
"minimum": -90.0,
"maximum": 90.0,
}
},
{
"name": "Longitude",
"type": "number",
"format": "default",
"constraints": {
"minimum": -180.0,
"maximum": 180.0,
}
},
]
schema = helpers.create_schema_from_fields(schema_fields)
parser = GeometryParser(schema)
self.assertFalse(parser.is_valid())
self.assertEqual(len(parser.errors), 1)
self.assertEqual(
parser.errors[0],
"The field named 'Longitude' must have the 'required' constraint set to true."
)
self.assertTrue(parser.is_lat_long)
self.assertFalse(parser.is_easting_northing)
