def test_without_datum_zone(self): """ A schema without datum/zone can be accepted as it will use the project's :return: """ schema_fields = [{ "name": "Easting", "type": "number", "format": "default", "constraints": { "required": True, } }, { "name": "Northing", "type": "number", "format": "default", "constraints": { "required": True, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_easting_northing) self.assertTrue(parser.is_easting_northing_only) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_lat_long_only) self.assertFalse(parser.is_site_code) self.assertFalse(parser.is_site_code_only) self.assertIsNotNone(parser.easting_field) self.assertIsNotNone(parser.northing_field)
def test_with_foreign_key(self): """ Alternatively to naming or tagging the column we can use a foreign key """ schema_fields = [ { "name": "Island", "type": "string", "format": "default", "constraints": { "required": True, } }, ] schema = helpers.create_schema_from_fields(schema_fields) schema = helpers.add_model_field_foreign_key_to_schema( schema, { 'schema_field': 'Island', 'model': 'Site', 'model_field': 'code' }) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_site_code_only) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_lat_long_only) self.assertFalse(parser.is_easting_northing) self.assertFalse(parser.is_easting_northing_only) self.assertIsNotNone(parser.site_code_field) self.assertEqual(parser.site_code_field.name, 'Island')
def test_biosys_type_without_required(self): """ If it is a site code only schema, the column must be required """ schema_fields = [ { "name": "Site", "type": "string", "format": "default", "biosys": { "type": "siteCode" }, }, ] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) expected_error = "The field named 'Site' must have the 'required' constraint set to true." self.assertEqual(parser.errors[0], expected_error) self.assertTrue(parser.is_site_code_only) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNotNone(parser.site_code_field) self.assertEqual(parser.site_code_field.name, 'Site')
def test_happy_path(self): schema_fields = [{ "name": "Latitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Longitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_lat_long) self.assertTrue(parser.is_lat_long_only) self.assertFalse(parser.is_easting_northing) self.assertFalse(parser.is_easting_northing_only) self.assertFalse(parser.is_site_code) self.assertFalse(parser.is_site_code_only) self.assertIsNotNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field)
def schema_with_easting_northing(): schema_fields = [{ "name": "What", "type": "string", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "When", "type": "date", "constraints": helpers.REQUIRED_CONSTRAINTS, "format": "any", "biosys": { 'type': 'observationDate' } }, { "name": "Northing", "type": "number", "constraints": helpers.REQUIRED_CONSTRAINTS, }, { "name": "Easting", "type": "number", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "Datum", "type": "string", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "Zone", "type": "string", "constraints": helpers.REQUIRED_CONSTRAINTS }] return helpers.create_schema_from_fields(schema_fields)
def test_lat_long_and_site_code_fk_not_required(self): """ Schema with lat/long and a site code foreign key """ schema_fields = [ { "name": "What", "type": "string", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "When", "type": "date", "constraints": helpers.REQUIRED_CONSTRAINTS, "format": "any", "biosys": { 'type': 'observationDate' } }, { "name": "Latitude", "type": "number", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS, "biosys": { "type": 'latitude' } }, { "name": "Longitude", "type": "number", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS, "biosys": { "type": 'longitude' } }, { "name": "Site Code", "type": "string", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }, ] schema = helpers.create_schema_from_fields(schema_fields) schema = helpers.add_model_field_foreign_key_to_schema( schema, { 'schema_field': 'Site Code', 'model': 'Site', 'model_field': 'code' }) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertFalse(parser.is_site_code_only) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_lat_long_only) self.assertFalse(parser.is_easting_northing) self.assertFalse(parser.is_easting_northing_only) self.assertIsNotNone(parser.site_code_field) self.assertEqual(parser.site_code_field.name, 'Site Code')
def test_two_biosys_type_is_error(self): """ Can't have two fields of the same type :return: """ schema_fields = [{ "name": "Lat", "type": "number", "format": "default", "biosys": { "type": 'latitude' }, "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Another lat", "type": "number", "format": "default", "biosys": { "type": 'latitude' }, "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Long", "type": "number", "format": "default", "biosys": { "type": 'longitude' }, "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) expected_errors = [ "More than one Biosys type latitude field found: ['Lat', 'Another lat']", ] self.assertIn(parser.errors[0], expected_errors) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field)
def test_biosys_tag_precedence(self): """ Two fields one name 'Latitude' and another one tagged as biosys type latitude The biosys one is chosen """ schema_fields = [{ "name": "The Observation Latitude", "type": "number", "format": "default", "biosys": { "type": 'latitude' }, "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Latitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Longitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNotNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field) self.assertEqual(parser.latitude_field.name, 'The Observation Latitude') self.assertEqual(parser.longitude_field.name, 'Longitude')
def test_required_date_with_format_any(self): """ field of type date, required with format should """ schema_fields = [{ "name": "DateAny", "type": "date", "format": "any", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "What", "type": "string", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }] schema = helpers.create_schema_from_fields(schema_fields) dataset = self.assert_create_dataset(schema) records = [ ['DateAny', 'What'], [None, 'something'], ['', 'something'], [' ', 'something'], ] resp = self._upload_records_from_rows(records, dataset_pk=dataset.pk, strict=True) self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) received = resp.json() self.assertIsInstance(received, list) self.assertEqual(len(received), 3) # this what an report should look like expected_row_report = { 'row': 3, 'errors': { 'DateAny': 'Field "DateAny" has constraint "required" which is not satisfied for value "None"' }, 'warnings': {} } for row_report in received: self.assertIn('errors', row_report) errors = row_report.get('errors') self.assertIn('DateAny', errors) msg = errors.get('DateAny') self.assertEqual(msg, expected_row_report['errors']['DateAny'])
def test_with_column_name_only(self): schema_fields = [ { "name": "Site Code", "type": "string", "format": "default", "constraints": { "required": True, } }, ] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_site_code_only) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNotNone(parser.site_code_field) self.assertEqual(parser.site_code_field.name, 'Site Code')
def test_biosys_tag_happy_path(self): """ columns not named latitude or longitude but with biosys tags """ schema_fields = [{ "name": "Lat", "type": "number", "format": "default", "biosys": { "type": 'latitude' }, "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Long", "type": "number", "format": "default", "biosys": { "type": 'longitude' }, "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNotNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field) self.assertFalse(parser.is_site_code) self.assertFalse(parser.is_site_code_only) self.assertEqual(parser.latitude_field.name, 'Lat') self.assertEqual(parser.longitude_field.name, 'Long')
def test_not_required_date_with_format_any(self): """ field of type date, not required with format any should not raise an error when received a empty string see https://decbugs.com/view.php?id=6928 """ schema_fields = [ { "name": "DateAny", "type": "date", "format": "any", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }, { "name": "DateTimeAny", "type": "datetime", "format": "any", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }, { "name": "DateDefault", "type": "date", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }, { "name": "DateTimeDefault", "type": "datetime", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS } ] schema = helpers.create_schema_from_fields(schema_fields) dataset = self.assert_create_dataset(schema) records = [ ['DateAny', 'DateTimeAny', 'DateDefault', 'DateTimeDefault'], [None, None, None, None], ['', '', '', ''], [' ', ' ', ' ', ' '], ] resp = self._upload_records_from_rows(records, dataset_pk=dataset.pk, strict=True) self.assertEqual(resp.status_code, status.HTTP_200_OK)
def test_two_latitude_name_is_error(self): """ Can't have two fields Latitude (no biosys tag) """ schema_fields = [{ "name": "Latitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Latitude", "type": "number", "format": "default", }, { "name": "Longitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) expected_errors = [ "More than one field named Latitude found.", ] self.assertIn(parser.errors[0], expected_errors) self.assertFalse(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field)
def test_with_wrong_column_name(self): """ The coloumn name must be Site Code not Site :return: """ schema_fields = [ { "name": "Site", "type": "string", "format": "default", "constraints": { "required": True, } }, ] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) expected_error = "The schema must contain some geometry fields: latitude/longitude or easting/northing " \ "or alternatively a reference to the Site Code." self.assertEqual(parser.errors[0], expected_error)
def test_biosys_tag_without_required(self): """ columns not named latitude or longitude but with biosys tags must be required. """ schema_fields = [{ "name": "Lat", "type": "number", "format": "default", "biosys": { "type": 'latitude' }, }, { "name": "Long", "type": "number", "format": "default", "biosys": { "type": 'longitude' }, }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 2) expected_errors = [ "The field named 'Lat' must have the 'required' constraint set to true.", "The field named 'Long' must have the 'required' constraint set to true." ] self.assertIn(parser.errors[0], expected_errors) self.assertIn(parser.errors[1], expected_errors) self.assertNotEqual(parser.errors[0], parser.errors[1]) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) self.assertIsNotNone(parser.latitude_field) self.assertIsNotNone(parser.longitude_field) self.assertEqual(parser.latitude_field.name, 'Lat') self.assertEqual(parser.longitude_field.name, 'Long')
def test_with_datum_zone(self): """ A schema with datum/zone :return: """ schema_fields = [{ "name": "Easting", "type": "number", "format": "default", "constraints": { "required": True, } }, { "name": "Northing", "type": "number", "format": "default", "constraints": { "required": True, } }, { "name": "Datum", "type": "string" }, { "name": "Zone", "type": "integer" }] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertTrue(parser.is_valid()) self.assertEqual(len(parser.errors), 0) self.assertTrue(parser.is_easting_northing) self.assertFalse(parser.is_lat_long) self.assertIsNotNone(parser.easting_field) self.assertIsNotNone(parser.northing_field) self.assertIsNotNone(parser.datum_field) self.assertIsNotNone(parser.zone_field)
def test_fk_with_dataset_code(self): """ This is the same test as above but this time the foreign key is declared with the parent dataset code instead of name :return: """ # Create a parent dataset with some records parent_dataset = self._create_dataset_and_records_from_rows([ ['Survey ID', 'Where', 'When', 'Who'], ['ID-001', 'King\'s Park', '2018-07-15', 'Tim Reynolds'], ['ID-002', 'Cottesloe', '2018-07-11', 'SLB'], ['ID-003', 'Somewhere', '2018-07-13', 'Phil Bill'] ]) parent_dataset.data_package['resources'][0]['schema']['primaryKey'] = 'Survey ID' parent_dataset.save() parent_dataset.code = 'Survey' parent_dataset.save() parent_records = parent_dataset.record_set.all() self.assertEqual(parent_records.count(), 3) # Create a child/related schema child_schema = helpers.create_schema_from_fields([ { "name": "Survey ID", "type": "string", "constraints": helpers.REQUIRED_CONSTRAINTS }, { "name": "What", "type": "string", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS }, { "name": "Comments", "type": "string", "constraints": helpers.NOT_REQUIRED_CONSTRAINTS, } ]) # declaring a foreign key pointing to the dataset name foreign_keys = [{ 'fields': 'Survey ID', 'reference': { 'fields': 'Survey ID', 'resource': parent_dataset.code } }] child_schema['foreignKeys'] = foreign_keys child_dataset = self._create_dataset_with_schema( self.project_1, self.data_engineer_1_client, child_schema ) self.assertIsNotNone(child_dataset) # post some records for survey ID--001 rows = [ ['Survey ID', 'What', 'Comments'], ['ID-001', 'Canis lupus', 'doggy, doggy'], ['ID-001', 'A frog', 'kiss'], ['ID-001', 'A tooth brush', 'I should stop drinking'], ] self._upload_records_from_rows(rows, child_dataset.id, strict=False) children_records = child_dataset.record_set.all() self.assertEqual(children_records.count(), 3) # test serialisation of parent records id_001 = parent_records.filter(data__contains={'Survey ID': 'ID-001'}).first() expected_children_ids = [r.id for r in children_records] expected_parent_id = None self.assertIsNotNone(id_001) url = reverse('api:record-detail', kwargs={'pk': id_001.pk}) client = self.custodian_1_client resp = client.get(url) self.assertEqual(resp.status_code, status.HTTP_200_OK) data = resp.json() self.assertEqual(sorted(data['children']), sorted(expected_children_ids)) self.assertEqual(data['parent'], expected_parent_id) id_002 = parent_records.filter(data__contains={'Survey ID': 'ID-002'}).first() expected_children_ids = [] expected_parent_id = None self.assertIsNotNone(id_002) url = reverse('api:record-detail', kwargs={'pk': id_002.pk}) client = self.custodian_1_client resp = client.get(url) self.assertEqual(resp.status_code, status.HTTP_200_OK) data = resp.json() self.assertEqual(sorted(data['children']), sorted(expected_children_ids)) self.assertEqual(data['parent'], expected_parent_id) id_003 = parent_records.filter(data__contains={'Survey ID': 'ID-003'}).first() expected_children_ids = [] expected_parent_id = None self.assertIsNotNone(id_003) url = reverse('api:record-detail', kwargs={'pk': id_003.pk}) client = self.custodian_1_client resp = client.get(url) self.assertEqual(resp.status_code, status.HTTP_200_OK) data = resp.json() self.assertEqual(sorted(data['children']), sorted(expected_children_ids)) self.assertEqual(data['parent'], expected_parent_id) # # test serialisation of children records # they all have the same parent and no children # expected_children_ids = None expected_parent_id = id_001.pk client = self.custodian_1_client for record in children_records: url = reverse('api:record-detail', kwargs={'pk': record.pk}) resp = client.get(url) self.assertEqual(resp.status_code, status.HTTP_200_OK) data = resp.json() self.assertEqual(data['children'], expected_children_ids) self.assertEqual(data['parent'], expected_parent_id)
def test_excel_type(self): schema_fields = [ { "name": "string", "type": "string", "format": "default" }, { "name": "number", "type": "number", "format": "default" }, { "name": "integer", "type": "integer", "format": "default" }, { "name": "date", "type": "date", "format": "any" }, { "name": "datetime", "type": "datetime", "format": "any" }, { "name": "boolean", "type": "boolean", "format": "default" } ] schema = helpers.create_schema_from_fields(schema_fields) project = self.project_1 client = self.data_engineer_1_client dataset = self._create_dataset_with_schema(project, client, schema) # create one record record_data = { "string": "string", "number": 12.3, "integer": 456, "date": "21/06/2017", "datetime": "13/04/2017 15:55", "boolean": 'yes' } payload = { 'dataset': dataset.pk, 'data': record_data } url = reverse('api:record-list') resp = client.post(url, data=payload, format='json') self.assertEqual(resp.status_code, status.HTTP_201_CREATED) record = Record.objects.filter(id=resp.json().get('id')).first() self.assertIsNotNone(record) url = reverse('api:record-list') query = { 'dataset__id': dataset.pk, 'output': 'xlsx' } try: resp = client.get(url, query) except Exception as e: self.fail("Export should not raise an exception: {}".format(e)) self.assertEqual(resp.status_code, status.HTTP_200_OK) # load workbook wb = load_workbook(six.BytesIO(resp.content)) ws = wb[dataset.name] rows = list(ws.rows) self.assertEqual(len(rows), 2) cells = rows[1] string, number, integer, date, datetime, boolean = cells # excel type are string, number or boolean self.assertEqual(string.data_type, Cell.TYPE_STRING) self.assertEqual(number.data_type, Cell.TYPE_NUMERIC) self.assertEqual(integer.data_type, Cell.TYPE_NUMERIC) self.assertEqual(date.data_type, Cell.TYPE_NUMERIC) self.assertEqual(datetime.data_type, Cell.TYPE_NUMERIC) self.assertEqual(boolean.data_type, Cell.TYPE_BOOL)
def test_without_required(self): """ If the schema contains only lat/long they must have a required constraint """ schema_fields = [ { "name": "Latitude", "type": "number", "format": "default", "constraints": { "minimum": -90.0, "maximum": 90.0, } }, { "name": "Longitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -180.0, "maximum": 180.0, } }, ] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) self.assertEqual( parser.errors[0], "The field named 'Latitude' must have the 'required' constraint set to true." ) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_easting_northing) schema_fields = [ { "name": "Latitude", "type": "number", "format": "default", "constraints": { "required": True, "minimum": -90.0, "maximum": 90.0, } }, { "name": "Longitude", "type": "number", "format": "default", "constraints": { "minimum": -180.0, "maximum": 180.0, } }, ] schema = helpers.create_schema_from_fields(schema_fields) parser = GeometryParser(schema) self.assertFalse(parser.is_valid()) self.assertEqual(len(parser.errors), 1) self.assertEqual( parser.errors[0], "The field named 'Longitude' must have the 'required' constraint set to true." ) self.assertTrue(parser.is_lat_long) self.assertFalse(parser.is_easting_northing)