def test_unknown_field(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv([['address','name','age','point x','point y'], ['123 Beach St','a','b','5','5'], ['222 Main St','a','b','8','8']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() self.assertFalse(rslt) ierrors = json.loads(ie.errors) # Should be x/y point error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], False) self.assertEqual(etpl, errors.UNMATCHED_FIELDS) self.assertEqual(set(ierrors[0]['data']), set(['name','age']))
def test_empty_file_error(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv([['header_field1','header_fields2','header_field3']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() # No rows added and validation failed self.assertEqual(TreeImportRow.objects.count(), base_rows) self.assertFalse(rslt) ierrors = json.loads(ie.errors) # The only error is a bad file error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], True) self.assertEqual(etpl, errors.EMPTY_FILE)
def test_missing_point_field(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv([['address','plot width','plot_length'], ['123 Beach St','5','5'], ['222 Main St','8','8']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() self.assertFalse(rslt) ierrors = json.loads(ie.errors) # Should be x/y point error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], True) self.assertEqual(etpl, errors.MISSING_POINTS)
def test_unknown_field(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv([['address', 'name', 'age', 'point x', 'point y'], ['123 Beach St', 'a', 'b', '5', '5'], ['222 Main St', 'a', 'b', '8', '8']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() self.assertFalse(rslt) ierrors = json.loads(ie.errors) # Should be x/y point error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], False) self.assertEqual(etpl, errors.UNMATCHED_FIELDS) self.assertEqual(set(ierrors[0]['data']), set(['name', 'age']))
def test_missing_point_field(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv([['address', 'plot width', 'plot_length'], ['123 Beach St', '5', '5'], ['222 Main St', '8', '8']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() self.assertFalse(rslt) ierrors = json.loads(ie.errors) # Should be x/y point error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], True) self.assertEqual(etpl, errors.MISSING_POINTS)
def test_empty_file_error(self): bie = ImportEvent(file_name="fn") bie.save() ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) ie.save() base_rows = TreeImportRow.objects.count() c = self.write_csv( [['header_field1', 'header_fields2', 'header_field3']]) create_rows_for_event(ie, c) rslt = ie.validate_main_file() # No rows added and validation failed self.assertEqual(TreeImportRow.objects.count(), base_rows) self.assertFalse(rslt) ierrors = json.loads(ie.errors) # The only error is a bad file error self.assertTrue(len(ierrors), 1) etpl = (ierrors[0]['code'], ierrors[0]['msg'], True) self.assertEqual(etpl, errors.EMPTY_FILE)
def setUp(self): self.user = User(username='******') self.user.save() bie = ImportEvent(file_name="fn") bie.save() self.ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) self.ie.save()
class ValidationTest(TestCase): def setUp(self): self.user = User(username='******') self.user.save() bie = ImportEvent(file_name="fn") bie.save() self.ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) self.ie.save() def mkrow(self,data): return TreeImportRow.objects.create( data=json.dumps(data), import_event=self.ie, idx=1) def assertHasError(self, thing, err, data=None, df=None): errors = '' errn,msg,fatal = err if thing.errors: errors = json.loads(thing.errors) for e in errors: if e['code'] == errn: if data is not None: edata = e['data'] if df: edata = df(edata) self.assertEqual(edata, data) return raise AssertionError('Error code %s not found in %s' % (errn,errors)) def assertNotHasError(self, thing, err, data=None): errn,msg,fatal = err if thing.errors: errors = json.loads(thing.errors) for e in errors: if e['code'] == errn: raise AssertionError('Error code %s found in %s' % (errn,errors)) def test_species_dbh_and_height(self): s1_gsc = Species(symbol='S1G__', scientific_name='',family='', genus='g1', species='s1', cultivar_name='c1', v_max_height=30, v_max_dbh=19) s1_gs = Species(symbol='S1GS_', scientific_name='',family='', genus='g1', species='s1', cultivar_name='', v_max_height=22, v_max_dbh=12) s1_gsc.save() s1_gs.save() row = {'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1', 'diameter': '15', 'tree height': '18'} i = self.mkrow(row) r = i.validate_row() self.assertHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertNotHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) row['tree height'] = 25 i = self.mkrow(row) r = i.validate_row() self.assertHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) row['cultivar'] = 'c1' i = self.mkrow(row) r = i.validate_row() self.assertNotHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertNotHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) def test_proximity(self): setupTreemapEnv() user = User.objects.get(username="******") bie1 = ImportEvent(file_name="bie1") bie2 = ImportEvent(file_name="bie2") bie3 = ImportEvent(file_name="bie3") bie4 = ImportEvent(file_name="bie4") for bie in [bie1, bie2, bie3, bie4]: bie.save() p1 = mkPlot(user, geom=Point(25.0000001,25.0000001)) p1.import_event = bie1 p1.save() p2 = mkPlot(user, geom=Point(25.0000002,25.0000002)) p2.import_event = bie2 p2.save() p3 = mkPlot(user, geom=Point(25.0000003,25.0000003)) p3.import_event = bie3 p3.save() p4 = mkPlot(user, geom=Point(27.0000001,27.0000001)) p4.import_event = bie4 p4.save() n1 = { p.pk for p in [p1,p2,p3] } n2 = { p4.pk } i = self.mkrow({'point x': '25.00000025', 'point y': '25.00000025'}) r = i.validate_row() self.assertHasError(i, errors.NEARBY_TREES, n1, set) i = self.mkrow({'point x': '27.00000015', 'point y': '27.00000015'}) r = i.validate_row() self.assertHasError(i, errors.NEARBY_TREES, n2, set) i = self.mkrow({'point x': '30.00000015', 'point y': '30.00000015'}) r = i.validate_row() self.assertNotHasError(i, errors.NEARBY_TREES) def test_species_id(self): s1_gsc = Species(symbol='S1G__', scientific_name='',family='', genus='g1', species='s1', cultivar_name='c1') s1_gs = Species(symbol='S1GS_', scientific_name='',family='', genus='g1', species='s1', cultivar_name='') s1_g = Species(symbol='S1GSC', scientific_name='',family='', genus='g1', species='', cultivar_name='') s2_gsc = Species(symbol='S2GSC', scientific_name='',family='', genus='g2', species='s2', cultivar_name='c2') s2_gs = Species(symbol='S2GS_', scientific_name='',family='', genus='g2', species='s2', cultivar_name='') for s in [s1_gsc, s1_gs, s1_g, s2_gsc, s2_gs]: s.save() # Simple genus, species, cultivar matches i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g1'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1', 'cultivar': 'c1'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) # Test no species info at all i = self.mkrow({'point x': '16', 'point y': '20'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) # Test mismatches i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's2', 'cultivar': 'c1'}) r = i.validate_row() self.assertHasError(i, errors.INVALID_SPECIES) i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g2'}) r = i.validate_row() self.assertHasError(i, errors.INVALID_SPECIES) def test_otm_id(self): # silly invalid-int-errors should be caught i = self.mkrow({'point x': '16', 'point y': '20', 'opentreemap id number': '44b'}) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INT_ERROR, None) i = self.mkrow({'point x': '16', 'point y': '20', 'opentreemap id number': '-22'}) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.POS_INT_ERROR) # With no plots in the system, all ids should fail i = self.mkrow({'point x': '16', 'point y': '20', 'opentreemap id number': '44'}) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_OTM_ID) # Add in plot setupTreemapEnv() # We need the whole darn thing # just so we can add a plot :( # SetupTME provides a special user for us to use # as well as particular neighborhood user = User.objects.get(username="******") p = mkPlot(user, geom=Point(25,25)) p.save() # With an existing plot it should be fine i = self.mkrow({'point x': '16', 'point y': '20', 'opentreemap id number': p.pk}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_OTM_ID) self.assertNotHasError(i, errors.INT_ERROR) def test_geom_validation(self): def mkpt(x,y): return self.mkrow({'point x': str(x), 'point y': str(y)}) # Invalid numbers i = mkpt('300a','20b') r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.FLOAT_ERROR) # Crazy lat/lngs i = mkpt(300,20) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_GEOM) i = mkpt(50,93) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_GEOM) # Out of neighborhood (neighborhood created in setUp) ngeom = MultiPolygon(Polygon( ((0, 0), (0, 50), (50, 50), (50, 0), (0, 0)))) neighborhood = Neighborhood( name='test neighborhood', region_id=34, city='blah', county='blarg', geometry=ngeom) neighborhood.save() i = mkpt(55,55) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.GEOM_OUT_OF_BOUNDS) i = mkpt(-5,-5) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.GEOM_OUT_OF_BOUNDS) # This should work... i = mkpt(25,25) r = i.validate_row() # Can't assert that r is true because other validation # logic may have tripped it self.assertNotHasError(i, errors.GEOM_OUT_OF_BOUNDS) self.assertNotHasError(i, errors.INVALID_GEOM) self.assertNotHasError(i, errors.FLOAT_ERROR) # If we add an exclusion zone, it should fail egeom = MultiPolygon(Polygon( ((10,10),(10,30),(30,30),(30,10),(10,10)))) e = ExclusionMask(geometry=egeom, type='blah blah') e.save() i = mkpt(25,25) r = i.validate_row() self.assertNotHasError(i, errors.GEOM_OUT_OF_BOUNDS) self.assertNotHasError(i, errors.INVALID_GEOM) self.assertNotHasError(i, errors.FLOAT_ERROR) self.assertHasError(i, errors.EXCL_ZONE)
class ValidationTest(TestCase): def setUp(self): self.user = User(username='******') self.user.save() bie = ImportEvent(file_name="fn") bie.save() self.ie = TreeImportEvent(file_name='file', owner=self.user, base_import_event=bie) self.ie.save() def mkrow(self, data): return TreeImportRow.objects.create(data=json.dumps(data), import_event=self.ie, idx=1) def assertHasError(self, thing, err, data=None, df=None): errors = '' errn, msg, fatal = err if thing.errors: errors = json.loads(thing.errors) for e in errors: if e['code'] == errn: if data is not None: edata = e['data'] if df: edata = df(edata) self.assertEqual(edata, data) return raise AssertionError('Error code %s not found in %s' % (errn, errors)) def assertNotHasError(self, thing, err, data=None): errn, msg, fatal = err if thing.errors: errors = json.loads(thing.errors) for e in errors: if e['code'] == errn: raise AssertionError('Error code %s found in %s' % (errn, errors)) def test_species_dbh_and_height(self): s1_gsc = Species(symbol='S1G__', scientific_name='', family='', genus='g1', species='s1', cultivar_name='c1', v_max_height=30, v_max_dbh=19) s1_gs = Species(symbol='S1GS_', scientific_name='', family='', genus='g1', species='s1', cultivar_name='', v_max_height=22, v_max_dbh=12) s1_gsc.save() s1_gs.save() row = { 'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1', 'diameter': '15', 'tree height': '18' } i = self.mkrow(row) r = i.validate_row() self.assertHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertNotHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) row['tree height'] = 25 i = self.mkrow(row) r = i.validate_row() self.assertHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) row['cultivar'] = 'c1' i = self.mkrow(row) r = i.validate_row() self.assertNotHasError(i, errors.SPECIES_DBH_TOO_HIGH) self.assertNotHasError(i, errors.SPECIES_HEIGHT_TOO_HIGH) def test_proximity(self): setupTreemapEnv() user = User.objects.get(username="******") bie1 = ImportEvent(file_name="bie1") bie2 = ImportEvent(file_name="bie2") bie3 = ImportEvent(file_name="bie3") bie4 = ImportEvent(file_name="bie4") for bie in [bie1, bie2, bie3, bie4]: bie.save() p1 = mkPlot(user, geom=Point(25.0000001, 25.0000001)) p1.import_event = bie1 p1.save() p2 = mkPlot(user, geom=Point(25.0000002, 25.0000002)) p2.import_event = bie2 p2.save() p3 = mkPlot(user, geom=Point(25.0000003, 25.0000003)) p3.import_event = bie3 p3.save() p4 = mkPlot(user, geom=Point(27.0000001, 27.0000001)) p4.import_event = bie4 p4.save() n1 = {p.pk for p in [p1, p2, p3]} n2 = {p4.pk} i = self.mkrow({'point x': '25.00000025', 'point y': '25.00000025'}) r = i.validate_row() self.assertHasError(i, errors.NEARBY_TREES, n1, set) i = self.mkrow({'point x': '27.00000015', 'point y': '27.00000015'}) r = i.validate_row() self.assertHasError(i, errors.NEARBY_TREES, n2, set) i = self.mkrow({'point x': '30.00000015', 'point y': '30.00000015'}) r = i.validate_row() self.assertNotHasError(i, errors.NEARBY_TREES) def test_species_id(self): s1_gsc = Species(symbol='S1G__', scientific_name='', family='', genus='g1', species='s1', cultivar_name='c1') s1_gs = Species(symbol='S1GS_', scientific_name='', family='', genus='g1', species='s1', cultivar_name='') s1_g = Species(symbol='S1GSC', scientific_name='', family='', genus='g1', species='', cultivar_name='') s2_gsc = Species(symbol='S2GSC', scientific_name='', family='', genus='g2', species='s2', cultivar_name='c2') s2_gs = Species(symbol='S2GS_', scientific_name='', family='', genus='g2', species='s2', cultivar_name='') for s in [s1_gsc, s1_gs, s1_g, s2_gsc, s2_gs]: s.save() # Simple genus, species, cultivar matches i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g1'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) i = self.mkrow({ 'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1' }) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) i = self.mkrow({ 'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's1', 'cultivar': 'c1' }) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) # Test no species info at all i = self.mkrow({'point x': '16', 'point y': '20'}) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_SPECIES) # Test mismatches i = self.mkrow({ 'point x': '16', 'point y': '20', 'genus': 'g1', 'species': 's2', 'cultivar': 'c1' }) r = i.validate_row() self.assertHasError(i, errors.INVALID_SPECIES) i = self.mkrow({'point x': '16', 'point y': '20', 'genus': 'g2'}) r = i.validate_row() self.assertHasError(i, errors.INVALID_SPECIES) def test_otm_id(self): # silly invalid-int-errors should be caught i = self.mkrow({ 'point x': '16', 'point y': '20', 'opentreemap id number': '44b' }) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INT_ERROR, None) i = self.mkrow({ 'point x': '16', 'point y': '20', 'opentreemap id number': '-22' }) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.POS_INT_ERROR) # With no plots in the system, all ids should fail i = self.mkrow({ 'point x': '16', 'point y': '20', 'opentreemap id number': '44' }) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_OTM_ID) # Add in plot setupTreemapEnv() # We need the whole darn thing # just so we can add a plot :( # SetupTME provides a special user for us to use # as well as particular neighborhood user = User.objects.get(username="******") p = mkPlot(user, geom=Point(25, 25)) p.save() # With an existing plot it should be fine i = self.mkrow({ 'point x': '16', 'point y': '20', 'opentreemap id number': p.pk }) r = i.validate_row() self.assertNotHasError(i, errors.INVALID_OTM_ID) self.assertNotHasError(i, errors.INT_ERROR) def test_geom_validation(self): def mkpt(x, y): return self.mkrow({'point x': str(x), 'point y': str(y)}) # Invalid numbers i = mkpt('300a', '20b') r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.FLOAT_ERROR) # Crazy lat/lngs i = mkpt(300, 20) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_GEOM) i = mkpt(50, 93) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.INVALID_GEOM) # Out of neighborhood (neighborhood created in setUp) ngeom = MultiPolygon( Polygon(((0, 0), (0, 50), (50, 50), (50, 0), (0, 0)))) neighborhood = Neighborhood(name='test neighborhood', region_id=34, city='blah', county='blarg', geometry=ngeom) neighborhood.save() i = mkpt(55, 55) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.GEOM_OUT_OF_BOUNDS) i = mkpt(-5, -5) r = i.validate_row() self.assertFalse(r) self.assertHasError(i, errors.GEOM_OUT_OF_BOUNDS) # This should work... i = mkpt(25, 25) r = i.validate_row() # Can't assert that r is true because other validation # logic may have tripped it self.assertNotHasError(i, errors.GEOM_OUT_OF_BOUNDS) self.assertNotHasError(i, errors.INVALID_GEOM) self.assertNotHasError(i, errors.FLOAT_ERROR) # If we add an exclusion zone, it should fail egeom = MultiPolygon( Polygon(((10, 10), (10, 30), (30, 30), (30, 10), (10, 10)))) e = ExclusionMask(geometry=egeom, type='blah blah') e.save() i = mkpt(25, 25) r = i.validate_row() self.assertNotHasError(i, errors.GEOM_OUT_OF_BOUNDS) self.assertNotHasError(i, errors.INVALID_GEOM) self.assertNotHasError(i, errors.FLOAT_ERROR) self.assertHasError(i, errors.EXCL_ZONE)