def setUp(self):
filenames = main.readInput('datafiles.json')
profiles = main.extractProfiles(filenames)
# identify and import tests
testNames = main.importQC('qctests')
testNames.sort()
for testName in testNames:
exec('from qctests import ' + testName)
# Set up any keyword arguments needed by tests.
kwargs = {'profiles': profiles}
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
firstProfile = True
delete = []
currentFile = ''
self.profiles = []
for iprofile, pinfo in enumerate(profiles):
# Load the profile data.
if pinfo.file_name != currentFile:
if currentFile != '': f.close()
currentFile = pinfo.file_name
f = open(currentFile)
if f.tell() != pinfo.file_position: f.seek(pinfo.file_position)
self.profiles.append(wod.WodProfile(f))
def setUp(self):
filenames = main.readInput('datafiles.json')
profiles = main.extractProfiles(filenames)
# identify and import tests
testNames = main.importQC('qctests')
testNames.sort()
for testName in testNames:
exec('from qctests import ' + testName)
# Set up any keyword arguments needed by tests.
kwargs = {'profiles' : profiles}
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
firstProfile = True
delete = []
currentFile = ''
self.profiles = []
for iprofile, pinfo in enumerate(profiles):
# Load the profile data.
if pinfo.file_name != currentFile:
if currentFile != '': f.close()
currentFile = pinfo.file_name
f = open(currentFile)
if f.tell() != pinfo.file_position: f.seek(pinfo.file_position)
self.profiles.append(wod.WodProfile(f))
def extractProfiles_test(self):
'''
simple check to make sure only WodProfile objects are getting returned
'''
profiles = main.extractProfiles(["data/quota_subset.dat"])
for i in profiles:
assert isinstance(i, wod.WodProfile), i + ' is not a WodProfile'
def extractProfiles_test(self):
'''
simple check to make sure only WodProfile objects are getting returned
'''
profiles = main.extractProfiles(["data/quota_subset.dat"])
for i in profiles:
assert isinstance(i, wod.WodProfile), i + ' is not a WodProfile'
def referenceResults_example_test(self):
'''
make sure main.referenceResults is extacting the correct references from data/example.dat
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
ref = main.referenceResults([p])
assert ref[0][0] == False, 'incorrect extraction of overall reference result for data/example.dat'
assert numpy.array_equal(ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def catchFlags_example_test(self):
'''
make sure main.catchFlags is flagging temperatures of 99.9 as missing,
using the artificial data generated in temp.dat above.
'''
profile = main.extractProfiles(['temp.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
main.catchFlags(p)
assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def catchFlags_example_test(self):
'''
make sure main.catchFlags is flagging temperatures of 99.9 as missing,
using the artificial data generated in temp.dat above.
'''
profile = main.extractProfiles(['temp.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
main.catchFlags(p)
assert p.profile_data[0]['variables'][0][
'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def profileData_example_test(self):
'''
continue examining data extracted from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0])
assert numpy.array_equal(p.z_level_qc(), [0,0,0,0])
assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230])
assert numpy.array_equal(p.t_level_qc(), [0,0,0,0])
assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410])
assert numpy.array_equal(p.s_level_qc(), [0,0,0,0])
def referenceResults_example_test(self):
'''
make sure main.referenceResults is extacting the correct references from data/example.dat
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
ref = main.referenceResults([p])
assert ref[0][
0] == False, 'incorrect extraction of overall reference result for data/example.dat'
assert numpy.array_equal(
ref[1][0], [False, False, False, False]
), 'incorrect extraction of verbose reference results for data/example.dat'
def profileData_example_test(self):
'''
continue examining data extracted from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0])
assert numpy.array_equal(p.z_level_qc(), [0, 0, 0, 0])
assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230])
assert numpy.array_equal(p.t_level_qc(), [0, 0, 0, 0])
assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410])
assert numpy.array_equal(p.s_level_qc(), [0, 0, 0, 0])
def extractProfiles_example_test(self):
'''
check the example from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
is extracted correctly.
data is in `data/example.dat`
'''
profile = main.extractProfiles(['data/example.dat'])[0]
assert profile.latitude() == 61.930, 'incorrect latitude extraction'
assert profile.longitude() == -172.270, 'incorrect longitude extraction'
assert profile.uid() == 67064, 'incorrect UID extraction'
assert profile.n_levels() == 4, 'incorrect # levels extraction'
assert profile.year() == 1934, 'incorrect year extraction'
assert profile.month() == 8, 'incorrect month extraction'
assert profile.day() == 7, 'incorrect day extraction'
assert profile.time() == 10.37, 'incorrect time extraction'
assert profile.probe_type() == 7, 'incorrect probe type extraction'
def processFile(fName):
# run each test on each profile, and record its summary & verbose performance
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
profileIDs = []
firstProfile = True
currentFile = ''
f = None
# keep a list of only the profiles in this thread
data.ds.threadProfiles = main.extractProfiles([fName])
data.ds.threadFile = fName
for iprofile, pinfo in enumerate(data.ds.threadProfiles):
# Load the profile data.
p, currentFile, f = main.profileData(pinfo, currentFile, f)
# Check that there are temperature data in the profile, otherwise skip.
if p.var_index() is None:
continue
main.catchFlags(p)
if np.sum(p.t().mask == False) == 0:
continue
# Run each test.
for itest, test in enumerate(testNames):
result = run(test, [p])
if firstProfile:
testResults.append(result[0])
testVerbose.append(result[1])
else:
testResults[itest].append(result[0][0])
testVerbose[itest].append(result[1][0])
firstProfile = False
# Read the reference result.
truth = main.referenceResults([p])
trueResults.append(truth[0][0])
trueVerbose.append(truth[1][0])
profileIDs.append(p.uid())
# testResults[i][j] now contains a flag indicating the exception raised by test i on profile j
return trueResults, testResults, profileIDs
def extractProfiles_example_test(self):
'''
check the example from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
is extracted correctly.
data is in `data/example.dat`
'''
profile = main.extractProfiles(['data/example.dat'])[0]
assert profile.latitude() == 61.930, 'incorrect latitude extraction'
assert profile.longitude(
) == -172.270, 'incorrect longitude extraction'
assert profile.uid() == 67064, 'incorrect UID extraction'
assert profile.n_levels() == 4, 'incorrect # levels extraction'
assert profile.year() == 1934, 'incorrect year extraction'
assert profile.month() == 8, 'incorrect month extraction'
assert profile.day() == 7, 'incorrect day extraction'
assert profile.time() == 10.37, 'incorrect time extraction'
assert profile.probe_type() == 7, 'incorrect probe type extraction'
def processFile(fName):
# run each test on each profile, and record its summary & verbose performance
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
profileIDs = []
firstProfile = True
currentFile = ''
f = None
# keep a list of only the profiles in this thread
data.ds.threadProfiles = main.extractProfiles([fName])
for iprofile, pinfo in enumerate(data.ds.threadProfiles):
# Load the profile data.
p, currentFile, f = main.profileData(pinfo, currentFile, f)
# Check that there are temperature data in the profile, otherwise skip.
if p.var_index() is None:
continue
main.catchFlags(p)
if np.sum(p.t().mask == False) == 0:
continue
# Run each test.
for itest, test in enumerate(testNames):
result = run(test, [p])
if firstProfile:
testResults.append(result[0])
testVerbose.append(result[1])
else:
testResults[itest].append(result[0][0])
testVerbose[itest].append(result[1][0])
firstProfile = False
# Read the reference result.
truth = main.referenceResults([p])
trueResults.append(truth[0][0])
trueVerbose.append(truth[1][0])
profileIDs.append(p.uid())
# testResults[i][j] now contains a flag indicating the exception raised by test i on profile j
return trueResults, testResults, profileIDs
########################################
if len(sys.argv) > 2:
# Identify and import tests
testNames = main.importQC('qctests')
testNames.sort()
print('{} quality control checks have been found'.format(len(testNames)))
testNames = main.checkQCTestRequirements(testNames)
print('{} quality control checks are able to be run:'.format(
len(testNames)))
for testName in testNames:
print(' {}'.format(testName))
# Identify data files and create a profile list.
filenames = main.readInput('datafiles.json')
profiles = main.extractProfiles(filenames)
data.ds.profiles = profiles
print('\n{} file(s) will be read containing {} profiles'.format(
len(filenames), len(profiles)))
# Parallel processing.
print('\nPlease wait while QC is performed\n')
processFile.parallel = main.parallel_function(processFile, sys.argv[2])
parallel_result = processFile.parallel(filenames)
# Recombine results
truth, results, profileIDs = main.combineArrays(parallel_result)
# Print summary statistics and write output file.
main.printSummary(truth, results, testNames)
main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1])
# main
########################################
if len(sys.argv)>2:
# Identify and import tests
testNames = main.importQC('qctests')
testNames.sort()
print('{} quality control checks have been found'.format(len(testNames)))
testNames = main.checkQCTestRequirements(testNames)
print('{} quality control checks are able to be run:'.format(len(testNames)))
for testName in testNames:
print(' {}'.format(testName))
# Identify data files and create a profile list.
filenames = main.readInput('datafiles.json')
profiles = main.extractProfiles(filenames)
data.ds.profiles = profiles
print('\n{} file(s) will be read containing {} profiles'.format(len(filenames), len(profiles)))
# Parallel processing.
print('\nPlease wait while QC is performed\n')
processFile.parallel = main.parallel_function(processFile, sys.argv[2])
parallel_result = processFile.parallel(filenames)
# Recombine results
truth, results, profileIDs = main.combineArrays(parallel_result)
# Print summary statistics and write output file.
main.printSummary(truth, results, testNames)
main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1])
else: