def setUp(self): filenames = main.readInput('datafiles.json') profiles = main.extractProfiles(filenames) # identify and import tests testNames = main.importQC('qctests') testNames.sort() for testName in testNames: exec('from qctests import ' + testName) # Set up any keyword arguments needed by tests. kwargs = {'profiles': profiles} testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] firstProfile = True delete = [] currentFile = '' self.profiles = [] for iprofile, pinfo in enumerate(profiles): # Load the profile data. if pinfo.file_name != currentFile: if currentFile != '': f.close() currentFile = pinfo.file_name f = open(currentFile) if f.tell() != pinfo.file_position: f.seek(pinfo.file_position) self.profiles.append(wod.WodProfile(f))
def setUp(self): filenames = main.readInput('datafiles.json') profiles = main.extractProfiles(filenames) # identify and import tests testNames = main.importQC('qctests') testNames.sort() for testName in testNames: exec('from qctests import ' + testName) # Set up any keyword arguments needed by tests. kwargs = {'profiles' : profiles} testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] firstProfile = True delete = [] currentFile = '' self.profiles = [] for iprofile, pinfo in enumerate(profiles): # Load the profile data. if pinfo.file_name != currentFile: if currentFile != '': f.close() currentFile = pinfo.file_name f = open(currentFile) if f.tell() != pinfo.file_position: f.seek(pinfo.file_position) self.profiles.append(wod.WodProfile(f))
def extractProfiles_test(self): ''' simple check to make sure only WodProfile objects are getting returned ''' profiles = main.extractProfiles(["data/quota_subset.dat"]) for i in profiles: assert isinstance(i, wod.WodProfile), i + ' is not a WodProfile'
def referenceResults_example_test(self): ''' make sure main.referenceResults is extacting the correct references from data/example.dat ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) ref = main.referenceResults([p]) assert ref[0][0] == False, 'incorrect extraction of overall reference result for data/example.dat' assert numpy.array_equal(ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0][ 'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def profileData_example_test(self): ''' continue examining data extracted from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0]) assert numpy.array_equal(p.z_level_qc(), [0,0,0,0]) assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230]) assert numpy.array_equal(p.t_level_qc(), [0,0,0,0]) assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410]) assert numpy.array_equal(p.s_level_qc(), [0,0,0,0])
def referenceResults_example_test(self): ''' make sure main.referenceResults is extacting the correct references from data/example.dat ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) ref = main.referenceResults([p]) assert ref[0][ 0] == False, 'incorrect extraction of overall reference result for data/example.dat' assert numpy.array_equal( ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def profileData_example_test(self): ''' continue examining data extracted from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0]) assert numpy.array_equal(p.z_level_qc(), [0, 0, 0, 0]) assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230]) assert numpy.array_equal(p.t_level_qc(), [0, 0, 0, 0]) assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410]) assert numpy.array_equal(p.s_level_qc(), [0, 0, 0, 0])
def extractProfiles_example_test(self): ''' check the example from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf is extracted correctly. data is in `data/example.dat` ''' profile = main.extractProfiles(['data/example.dat'])[0] assert profile.latitude() == 61.930, 'incorrect latitude extraction' assert profile.longitude() == -172.270, 'incorrect longitude extraction' assert profile.uid() == 67064, 'incorrect UID extraction' assert profile.n_levels() == 4, 'incorrect # levels extraction' assert profile.year() == 1934, 'incorrect year extraction' assert profile.month() == 8, 'incorrect month extraction' assert profile.day() == 7, 'incorrect day extraction' assert profile.time() == 10.37, 'incorrect time extraction' assert profile.probe_type() == 7, 'incorrect probe type extraction'
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) data.ds.threadFile = fName for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
def extractProfiles_example_test(self): ''' check the example from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf is extracted correctly. data is in `data/example.dat` ''' profile = main.extractProfiles(['data/example.dat'])[0] assert profile.latitude() == 61.930, 'incorrect latitude extraction' assert profile.longitude( ) == -172.270, 'incorrect longitude extraction' assert profile.uid() == 67064, 'incorrect UID extraction' assert profile.n_levels() == 4, 'incorrect # levels extraction' assert profile.year() == 1934, 'incorrect year extraction' assert profile.month() == 8, 'incorrect month extraction' assert profile.day() == 7, 'incorrect day extraction' assert profile.time() == 10.37, 'incorrect time extraction' assert profile.probe_type() == 7, 'incorrect probe type extraction'
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
######################################## if len(sys.argv) > 2: # Identify and import tests testNames = main.importQC('qctests') testNames.sort() print('{} quality control checks have been found'.format(len(testNames))) testNames = main.checkQCTestRequirements(testNames) print('{} quality control checks are able to be run:'.format( len(testNames))) for testName in testNames: print(' {}'.format(testName)) # Identify data files and create a profile list. filenames = main.readInput('datafiles.json') profiles = main.extractProfiles(filenames) data.ds.profiles = profiles print('\n{} file(s) will be read containing {} profiles'.format( len(filenames), len(profiles))) # Parallel processing. print('\nPlease wait while QC is performed\n') processFile.parallel = main.parallel_function(processFile, sys.argv[2]) parallel_result = processFile.parallel(filenames) # Recombine results truth, results, profileIDs = main.combineArrays(parallel_result) # Print summary statistics and write output file. main.printSummary(truth, results, testNames) main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1])
# main ######################################## if len(sys.argv)>2: # Identify and import tests testNames = main.importQC('qctests') testNames.sort() print('{} quality control checks have been found'.format(len(testNames))) testNames = main.checkQCTestRequirements(testNames) print('{} quality control checks are able to be run:'.format(len(testNames))) for testName in testNames: print(' {}'.format(testName)) # Identify data files and create a profile list. filenames = main.readInput('datafiles.json') profiles = main.extractProfiles(filenames) data.ds.profiles = profiles print('\n{} file(s) will be read containing {} profiles'.format(len(filenames), len(profiles))) # Parallel processing. print('\nPlease wait while QC is performed\n') processFile.parallel = main.parallel_function(processFile, sys.argv[2]) parallel_result = processFile.parallel(filenames) # Recombine results truth, results, profileIDs = main.combineArrays(parallel_result) # Print summary statistics and write output file. main.printSummary(truth, results, testNames) main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1]) else: