def test_schema_changes(self): arcpy.Copy_management(check_for_changes_gdb, test_gdb) with self.assertRaises(ValidationException): core.check_schema(Crate('ZipCodes', test_gdb, check_for_changes_gdb, 'FieldLength')) result = core.check_schema(Crate('ZipCodes', test_gdb, check_for_changes_gdb, 'ZipCodes')) self.assertEqual(result, True)
def test_check_schema_match(self): with self.assertRaises(ValidationException): core.check_schema(Crate('FieldLength', check_for_changes_gdb, check_for_changes_gdb, 'FieldLength2')) with self.assertRaises(ValidationException): core.check_schema(Crate('FieldType', check_for_changes_gdb, check_for_changes_gdb, 'FieldType2')) self.assertEqual(core.check_schema(Crate('ZipCodes', check_for_changes_gdb, check_for_changes_gdb2, 'ZipCodes')), True)
def test_schema_changes(test_gdb): with pytest.raises(ValidationException): core.check_schema(Crate('ZipCodes', test_gdb, test_gdb, 'FieldLength')) result = core.check_schema( Crate('ZipCodes', test_gdb, test_gdb, 'ZipCodes')) assert result == True
def test_check_schema_match(test_gdb): with pytest.raises(ValidationException): core.check_schema( Crate('FieldLength', test_gdb, test_gdb, 'FieldLength2')) with pytest.raises(ValidationException): core.check_schema(Crate('FieldType', test_gdb, test_gdb, 'FieldType2')) assert core.check_schema(Crate('ZipCodes', test_gdb, test_gdb, 'ZipCodes2')) == True
def test_check_schema_no_objectid_in_source(test_gdb): skip_if_no_local_sde() result = core.check_schema( Crate('UPDATE_TESTS.dbo.NO_OBJECTID_TEST', UPDATE_TESTS_SDE, test_gdb, r'NO_OBJECTID_TEST')) assert result == True
def test_check_schema_no_objectid_in_source(self): skip_if_no_local_sde() result = core.check_schema( Crate('UPDATE_TESTS.dbo.NO_OBJECTID_TEST', update_tests_sde, check_for_changes_gdb, r'NO_OBJECTID_TEST')) self.assertEqual(result, True)
def test_check_schema_ignore_length_for_all_except_text(self): skip_if_no_local_sde() # only worry about length on text fields result = core.check_schema( Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells', update_tests_sde, check_for_changes_gdb, 'DNROilGasWells')) self.assertEqual(result, True)
def test_check_schema_ignore_length_for_all_except_text(test_gdb): skip_if_no_local_sde() # only worry about length on text fields result = core.check_schema( Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells', UPDATE_TESTS_SDE, test_gdb, 'DNROilGasWells')) assert result == True
def test_schema_changes_in_sde(self): skip_if_no_local_sde() arcpy.Copy_management(check_for_changes_gdb, test_gdb) result = core.check_schema( Crate('FieldTypeFloat', test_gdb, update_tests_sde, 'FieldTypeFloat')) self.assertEqual(result, True)
def test_schema_changes_in_sde(test_gdb): skip_if_no_local_sde() result = core.check_schema( Crate('FieldTypeFloat', test_gdb, UPDATE_TESTS_SDE, 'FieldTypeFloat')) assert result == True
def test_schema_ignore_non_standard_shape_length_fields(test_gdb): result = core.check_schema( Crate('DirectionalSurveyHeaderSource', test_gdb, test_gdb, 'DirectionalSurveyHeaderDestination')) assert result
def test_schema_changes_field_case_differences(test_gdb): with pytest.raises(ValidationException): core.check_schema(Crate('lower', test_gdb, test_gdb, 'UPPER'))
def test_check_schema_no_objectid_in_source(self): skip_if_no_local_sde() result = core.check_schema(Crate('UPDATE_TESTS.dbo.NO_OBJECTID_TEST', update_tests_sde, check_for_changes_gdb, r'NO_OBJECTID_TEST')) self.assertEqual(result, True)
def test_check_schema_ignore_length_for_all_except_text(self): skip_if_no_local_sde() # only worry about length on text fields result = core.check_schema(Crate(r'UPDATE_TESTS.DBO.Hello\UPDATE_TESTS.DBO.DNROilGasWells', update_tests_sde, check_for_changes_gdb, 'DNROilGasWells')) self.assertEqual(result, True)
def test_schema_changes_in_sde(self): skip_if_no_local_sde() arcpy.Copy_management(check_for_changes_gdb, test_gdb) result = core.check_schema(Crate('FieldTypeFloat', test_gdb, update_tests_sde, 'FieldTypeFloat')) self.assertEqual(result, True)
def validate_crate(self, crate): fltr = 'TRANSTECH <> 60' bb_service = 'UBBMAP.UBBADMIN.BB_Service' providerTableName = 'UBBMAP.UBBADMIN.BB_Providers_Table' providerTableFieldName = 'Code' coverageProviders = [] # list to hold all providers in coverage data tableProviders = [] # list of all providers in providers table coverageFieldName = 'UTProvCode' nonNullFields = ['"' + coverageFieldName + '" IS NULL OR "' + coverageFieldName + '" = \'\'', '"MAXADUP" IS NULL OR "MAXADUP" = 0', '"MAXADDOWN" IS NULL OR "MAXADDOWN" = 0', '"TRANSTECH" IS NULL OR "TRANSTECH" = 0'] errors = [] if crate.source_name != 'BB_Service': return NotImplemented #: this will raise if it doesn't pass... check_schema(crate) arcpy.env.workspace = crate.source_workspace arcpy.env.geographicTransformations = 'NAD_1983_To_WGS_1984_5' self.log.info("checking non-null fields") # create layer for selecting self.log.info('creating layer') layerName = bb_service + 'Layer' arcpy.MakeFeatureLayer_management(bb_service, layerName, fltr) # loop through fields for query in nonNullFields: self.log.info('query: ' + query) arcpy.SelectLayerByAttribute_management(layerName, 'NEW_SELECTION', query) cnt = arcpy.GetCount_management(layerName) if int(str(cnt)) > 0: errors.append('ERROR: null or empty values found in ' + bb_service + ':' + query) # get search cursor self.log.info('building list of providers in coverage feature class') cur = arcpy.SearchCursor(bb_service, fltr, '', coverageFieldName) row = cur.next() while row: code = row.getValue(coverageFieldName) # add to list of providers if not [code, bb_service] in coverageProviders: coverageProviders.append([code, bb_service]) row = cur.next() del cur self.log.info('Finished with ' + bb_service) # get cursor for provider table self.log.info('building list of providers in providers table') prows = arcpy.SearchCursor(providerTableName, "Exclude IS NULL OR Exclude = ''") row = prows.next() while row: tableProviders.append(row.getValue(providerTableFieldName)) row = prows.next() del prows, row # loop through coverage providers and make sure that they are in the provider table list self.log.info('looking for providers that show up in coverage data but not providers table') missingProviders = [] for row in coverageProviders: if not row[0] in tableProviders: missingProviders.append(row[0]) # check for data errors if len(errors) > 0: self.log.info('ERRORS IN DATA:') for e in errors: self.log.info(e) else: self.log.info('NO ERRORS IN DATA') # check for mis matching providers if len(missingProviders) > 0: self.log.info('MISSING PROVIDERS THAT ARE IN THE COVERAGE DATA BUT NOT IN THE PROVIDERS TABLE:') errors.append('missing providers in the coverage data: {}'.format(missingProviders)) for mp in missingProviders: self.log.info(str(mp)) else: self.log.info('NO PROVIDERS FOUND IN THE COVERAGE DATA THAT ARE NOT IN THE PROVIDERS TABLE.') if len(missingProviders) > 0 or len(errors) > 0: validation_message = 'Errors were found during validation: {}'.format(errors) self.log.info(validation_message) raise ValidationException(validation_message) return True