def test_known_value(self):
val = qc.get_sst(0.5, -179.5, 1, 1, self.sst)
self.assertAlmostEqual(val, 28.2904, delta=0.0001)
val = qc.get_sst(-0.5, -179.5, 1, 1, self.sst)
self.assertAlmostEqual(val, 28.4300, delta=0.0001)
val = qc.get_sst(0.5, -179.5, 1, 1, self.mat)
self.assertAlmostEqual(val, 27.4615, delta=0.0001)
val = qc.get_sst(-0.5, -179.5, 1, 1, self.mat)
self.assertAlmostEqual(val, 27.2693, delta=0.0001)
def add_imma1_record_to_db(cursor, year, x, climsst, climnmat):
'''
Given a database cursor and an IMMA report it will populate the appropriate table in the data base.
:param cursor: data base cursor for SQLite3 data base
:param year: year of the table to which the observation is to be added
:param x: IMMA report to be added to the data base
:param climsst: SST climatology
:param climnmat: NMAT climatology
:type cursor: unknown
:type year: integer
:type x: IMMA report
:type climsst: numpy array
:type climnmat: numpy array
A feature of the current version is that, in line with MDS3 and earlier, the hour defaults to zero UTC when
hour is missing from the report.
'''
rep = qc.MarineReport.report_from_imma(x)
inyear = year
#Deck 201 GMT midnights are assigned to the wrong day, see Carella, Kent, Berry 2015 Appendix A3
#Reports from deck 201 before 1899 taken at the GMT midnight were moved one day before the reported date.
if rep.dck == 201 and rep.year < 1899 and rep.hour == 0:
rep.shift_day(-1)
inyear = rep.year #make sure it goes in the right table
#Deck 701 prior to 1857ish has lots of obs with no hour set
if rep.dck == 701 and rep.year < 1860 and rep.hour == None:
rep.hour = 12.
try:
sst_climav = qc.get_sst(rep.lat,rep.lon,rep.month,rep.day,climsst)
except:
sst_climav = None
if sst_climav != None:
sst_climav = float(sst_climav)
try:
mat_climav = qc.get_sst(rep.lat,rep.lon,rep.month,rep.day,climnmat)
except:
mat_climav = None
if mat_climav != None:
mat_climav = float(mat_climav)
rep.setnorm(sst_climav, mat_climav)
add_marine_report_to_db(cursor,inyear,rep)
return
def get_clim(rep, clim):
'''
Get the climatological value for this particular observation
KW Also now used to pull out the climatological stdev for the ob
:param rep: a MarineReport
:param clim: a masked array containing the climatological averages
:type rep: MarineReport
:type clim: numpy array
'''
try:
rep_clim = qc.get_sst(rep.getvar('LAT'),
rep.getvar('LON'),
rep.getvar('MO'),
rep.getvar('DY'),
clim)
rep_clim = float(rep_clim)
except:
rep_clim = None
return rep_clim
def test_works_with_dummy_arrays(self):
val = qc.get_sst(0, 0, 1, 1, self.dummyclim)
self.assertEqual(val, 1.13)
def test_south_pole_is_missing(self):
val = qc.get_sst(-89.9, 0.0, 1, 1, self.sst)
self.assertEqual(val, None)
def test_north_pole_is_freezing_in_january(self):
val = qc.get_sst(89.9, 0.0, 1, 1, self.sst)
self.assertAlmostEqual(val, -1.8, delta=0.0001)
val = qc.get_sst(89.9, 0.0, 1, 1, self.mat)
self.assertAlmostEqual(val, 0, delta=0.0001)
def test_stdev_known_value(self):
val = qc.get_sst(-9.5, -179.5, 1, 1, self.sdv)
self.assertAlmostEqual(val, 2.19, delta=0.0001)
val = qc.get_sst(-10.5, -179.5, 12, 31, self.sdv)
self.assertAlmostEqual(val, 0.44, delta=0.0001)
def test_stdev_is_none_at_pole(self):
val = qc.get_sst(89.9, 0.0, 1, 1, self.sdv)
self.assertEqual(val, None)
def test_single_field(self):
"""
Test that single fields ignore the time info
"""
val = qc.get_sst(89.9, 0.0, 12, 31, self.dummyclim_single_field)
self.assertEqual(val, 3.13)
def test_december(self):
"""
other tests test Jan 1st, this test tests the other end of the calendar
"""
val = qc.get_sst(89.9, 0.0, 12, 31, self.sdv)
self.assertEqual(val, None)
def get_buddy_limits(anoms_by_grid, pentad_stdev):
'''
This is an internal routine for the buddy check. It takes gridded obs in a dictionary and a field of standard
deviations and returns the neighbour-average for each populated gridcell as well as bounds based on the number
of neighbours.
:param anoms_by_grid: gridded anomalies and counts of observations
:param pentad_stdev: grid of climatological standard deviations
:type anoms_by_grid: dictionary containing two-element lists [anom, number of observations]. The keys of the dictionary are :class:`.Gridpt` objects
:type pentad_stdev: numpy array
:return: gridded buddy anomalies and gridded buddy multipliers
:rtype: dictionaries containing buddy anomalies and multipliers referenced by :class:`Gridpt`
A neighbour average and allowed range are calculated for each populated grid cell. Neighbours are initially
sought within 1 degree and 2 pentads. If neighbours are found then for N observations the allowed ranges R are:
* If N > 100: R = SD*2.5
* If 15 < N < 100: R = SD*3.0
* If 5 < N < 15: R = SD*3.5
* If 0 < N < 5: R = SD*4.0
Where SD is the input standard deviation for that grid box. M, the mean of the anomalies, is also calculated.
If no neighbours are found then the search area is expanded to 2 degrees and 2 pentads and then to 1 degrees
and 4 pentads and then 2 degrees and 4 pentads. If neighbours are found then the allowed ranges R are:
* R = SD*4.0
If no neighbours are found at all then the range is set to be infinite.
The function returns M and R for each grid box
'''
#this takes a dictionary (anoms_by_grid) whose keys
#are instances of type Gridpt.
#each key relates to a single 1x1xpentad grid box and
#the entries are each a two-element list
#the first element is the grid box average and the
#second is the number of obs in the average.
#for each of these populated grid boxes, neighbours
#are sought and a neighbour average and
#tolerance is calculated.
buddy_anom_by_grid = {}
buddy_multiplier_by_grid = {}
#for each populated grid box
for key in anoms_by_grid:
stdev = qc.get_sst(key.latitude_approx, key.longitude_approx,
key.month, key.day, pentad_stdev)
if stdev == None or stdev < 0.0:
stdev = 1.0
match_not_found = True
#if there is neighbour in that range then calculate a mean
if match_not_found:
temp_anom, temp_nobs = \
get_neighbour_anomalies([1, 1, 2],
key,
anoms_by_grid)
if len(temp_anom) > 0:
buddy_anom_by_grid[key] = np.mean(temp_anom)
total_nobs = np.sum(temp_nobs)
buddy_multiplier_by_grid[key] = \
get_threshold_multiplier(total_nobs,
[0, 5, 15, 100],
[4.0, 3.5, 3.0, 2.5])*stdev
match_not_found = False
assert total_nobs != 0, "total number of observations is zero"
#otherwise move out further in space and time to 2 degrees and 2 pentads
if match_not_found:
temp_anom, temp_nobs = \
get_neighbour_anomalies([2, 2, 2],
key,
anoms_by_grid)
if len(temp_anom) > 0:
buddy_anom_by_grid[key] = np.mean(temp_anom)
total_nobs = np.sum(temp_nobs)
buddy_multiplier_by_grid[key] = \
get_threshold_multiplier(total_nobs,
[0],
[4.0])*stdev
match_not_found = False
assert total_nobs != 0, "total number of observations is zero"
#otherwise move out further in space and time to 2 degrees and 2 pentads
if match_not_found:
temp_anom, temp_nobs = \
get_neighbour_anomalies([1, 1, 4],
key,
anoms_by_grid)
if len(temp_anom) > 0:
buddy_anom_by_grid[key] = np.mean(temp_anom)
total_nobs = np.sum(temp_nobs)
buddy_multiplier_by_grid[key] = \
get_threshold_multiplier(total_nobs,
[0, 5, 15, 100],
[4.0, 3.5, 3.0, 2.5])*stdev
match_not_found = False
assert total_nobs != 0, "total number of observations is zero"
#final step out is to 2 degrees and 4 pentads
if match_not_found:
temp_anom, temp_nobs = \
get_neighbour_anomalies([2, 2, 4],
key,
anoms_by_grid)
if len(temp_anom) > 0:
buddy_anom_by_grid[key] = np.mean(temp_anom)
total_nobs = np.sum(temp_nobs)
buddy_multiplier_by_grid[key] = \
get_threshold_multiplier(total_nobs,
[0],
[4.0])*stdev
match_not_found = False
assert total_nobs != 0 , "total number of observations is zero"
#if there are no neighbours then any observation will get a pass
if match_not_found:
buddy_anom_by_grid[key] = 0.0
buddy_multiplier_by_grid[key] = 500.0
return buddy_anom_by_grid, buddy_multiplier_by_grid
def get_new_buddy_limits(anoms_by_grid, stdev1, stdev2, stdev3, sigma_m=1.0):
"""
This returns the buddy mean and buddy uncertainty given a whole set of input fields.
:param anoms_by_grid: a set of gridded anomalies
:param stdev1: a field of standard deviations of the difference between a single grid box and a buddy average
:param stdev2: a field of standard deviations of the sampling error for a single 1x1x5-day grid cell
:param stdev3: a field of standard deviations of the sampling error from estimating a buddy average by one grid box
:param sigma_m: the estimated measurement error uncertainty on a single observation
:type anoms_by_grid: dictionary containing two-element lists [anom, number of observations].
The keys of the dictionary are :class:`.Gridpt` objects
:type stdev1: numpy masked array
:type stdev2: numpy masked array
:type stdev3: numpy masked array
:type sigma_m: float
:return: buddy average anomaly and expected standard deviation of difference between an ob and the buddy average
:rtype: dictionaries containing two-element lists [anom, number of observations].
The keys of the dictionary are :class:`.Gridpt` objects
This is the updated estimate of the buddy limits used internally in the new buddy check and
bayesian buddy check. An observation will be compared to the buddy average (the first returned
value) with the limits set by the standard deviation (the second returned value).
"""
buddy_anom_by_grid = {}
buddy_multiplier_by_grid = {}
#for each populated grid box
for key in anoms_by_grid:
stdev1_ex = qc.get_sst(key.latitude_approx, key.longitude_approx,
key.month, key.day, stdev1)
stdev2_ex = qc.get_sst(key.latitude_approx, key.longitude_approx,
key.month, key.day, stdev2)
stdev3_ex = qc.get_sst(key.latitude_approx, key.longitude_approx,
key.month, key.day, stdev3)
if stdev1_ex == None or stdev1_ex < 0.0:
stdev1_ex = 1.0
if stdev2_ex == None or stdev2_ex < 0.0:
stdev2_ex = 1.0
if stdev3_ex == None or stdev3_ex < 0.0:
stdev3_ex = 1.0
#if there is neighbour in that range then calculate a mean
temp_anom, temp_nobs = \
get_neighbour_anomalies([2, 2, 4],
key,
anoms_by_grid)
if len(temp_anom) > 0:
buddy_anom_by_grid[key] = np.mean(temp_anom)
total_nobs = np.sum(temp_nobs)
tot = 0.0
ntot = 0.0
for n in temp_nobs:
tot += ( sigma_m**2. / n ) #measurement error for each 1x1x5day cell
tot += 2 * ( stdev2_ex**2. / n) #sampling error for each 1x1x5day cell
ntot += 1.0
sigma_buddy = tot / (ntot**2.)
sigma_buddy += stdev3_ex**2. / ntot
"""The uncertainty in the buddy average as a prediction of the measurement is as given
the sum of the uncertainty in the buddy average, the uncertainty associated with using
the buddy average to estimate the point value and the uncertainty in a single measurement"""
buddy_multiplier_by_grid[key] = math.sqrt( sigma_m**2. +
stdev1_ex**2. +
2 * stdev2_ex**2. +
sigma_buddy )
else:
buddy_anom_by_grid[key] = 0.0
buddy_multiplier_by_grid[key] = 500.
return buddy_anom_by_grid, buddy_multiplier_by_grid
