def estamt(network, minlenshf=24, **hom_params):
"""
COPIED FROM ucpmonthly.v24a.f:
The major steps in determining the best adjustment value for each station
and changepoint. Entire network undergoes each of the following processes.
In order:
1) Remove unusable data. Align move swith respect to non-missing data
and compress out changes that are too close AND the data between them.
2) ISTEP=2 processing begins the adjustment process by removing the
non-significant changepoints to lengthen segments.
3) NPASS (:= ISTEP=3) finishes the adjustment process by testing for the
minimum number of months in a segment and number of neighbors with which
the difference series can be examined.
4) Final adjusted output is written.
"""
## FILTER 4
## Since the amplitude estimate MUST rely upon a minimum of MINLEN months to
## get even close to a reliable estimate at this point, it is assumed that
## the changepoints are as good as the station history files. Therefore,
## align moves with respect to non-missing data and compress out changes
## that are too close AND the data between them (i.e., less than MINLEN
## apart)
# station_list = network.stations.keys()
all_station_list = network.stations.keys()
# station_list = ["215887", ]
station_list = all_station_list
# for each station...
for id in station_list:
station_index = station_list.index(id)
station_series = network.raw_series[id]
station_data = station_series.monthly_series[:]
missing_val = station_series.MISSING_VAL
# ... gen arrays for alignment
move, amt, mday = [], [], []
changepoints = station_series.changepoints
cps = sorted(changepoints.keys())
for cp in cps:
print " Hist move: ", len(move) + 1, station_index + 1, imo2iym(cp)
move.append(cp)
amt.append(changepoints[cp]["jsum"])
mday.append(31)
movnum = len(changepoints)
if movnum > 0:
## At this point, the Fortran code executes alignmoves() in
## SHAPinp.v6c.f to reconcile the fact that station history files
## report dates of moves. It also removes segments that eare too short
## - less than minlenshf. Instead of implementing alignmoves(). Right
## now, I'll only implement this second functionality.
# alignmoves()
####################################################################
# Seek to find first and last month indices
first_set = False
for month in range(len(station_data)):
# Skip first year
if month < 12:
continue
if station_data[month] != missing_val:
if not first_set:
first = month
first_set = True
last = month
cps = sorted(changepoints.keys())
cps.insert(0, first)
cps.append(last)
for (cp1, cp2) in zip(cps[:], cps[1:]):
if (cp2 - cp1) < minlenshf:
months_to_delete = range(cp1 + 1, cp2 + 1)
network.raw_series[id].delete_months(months_to_delete)
if cp2 == last:
del_key = cp1
else:
del_key = cp2
if del_key in network.raw_series[id].changepoints:
# print len(network.raw_series[id].changepoints.keys()),
del network.raw_series[id].changepoints[del_key]
# print len(network.raw_series[id].changepoints.keys()),
# raw_input("pause")
del_str = "Del 1st segment: " if cp1 == first else "Delete segment: "
print id, station_index + 1, del_str, imo2iym(cp1), cp1, imo2iym(cp2), cp2
new_changepoints = network.raw_series[id].changepoints
new_cps = sorted(new_changepoints.keys())
print " First data value: ", imo2iym(first)
for cp in new_cps:
print " End seg:", new_cps.index(cp), " ym: ", imo2iym(cp), cp, new_changepoints[cp]["jsum"]
print " End segment ym: ", imo2iym(last), last
# Finally, add first and last value to the list of changepoints.
first_stats = dict(ahigh=0.0, astd=0.0, jsum=0)
last_stats = dict(ahigh=0.0, astd=0.0, jsum=0)
network.raw_series[id].changepoints[first] = first_stats
network.raw_series[id].changepoints[last] = last_stats
####################################################################
## Series of debug print statements summarizing the final list of
## changepoints. Not necessary at the moment
############################################################################
# The subnetwork processing became a multi-step process plus a "post-process
# pass" to manage:
# 1) problems with documented changepoints with NO undocumented support
# 2) determine the best amplitude estimation for each confirmed changepoint
for step in [2, 3]:
## Setup output strings based on the step used. Only cosmetic differences
## really.
iminlen = hom_params["minlen"]
numclim = 3
## STEP 1 - NEVER USED (technically the history-consideration done previously
if step == 1:
continue
elif step == 2:
## STEP 2 - NOT SIG REMOVAL
## equivalent to ipass loopback for istep == 2 in Fortran PHA
print " ---------------- NOT SIG REMOVAL --------------- "
tstr = "Not sig: "
outid = "NS"
ipass = 1
elif step == 3:
## STEP 3 - ADJUSTMENT OF DISCONTINUITIES
# equivalent to ipass loopback for istep == in FORTRAN PHA
print " ---------------- ADJUST DISCONTINUITY STEP --------------- "
print "Adjpass, iminlen, numclim", "--", iminlen, numclim
print " ---------------- NPASS --------------- "
tstr = "Dstep Dtrend: "
outid = "WM"
ipass = ipass + 1
final_results = dict()
print " NET STN FILT TECH ------ AFTER ------ ------ BEFORE ------"
# Process each station and its network of neighbors
for id in station_list:
station_index = station_list.index(id)
station_cp_dict = network.raw_series[id].changepoints
sorted_cps = sorted(station_cp_dict.keys())
## If there are no breakpoints...
if not sorted_cps:
final_results[id] = dict()
continue
station_series = network.raw_series[id]
missing_val = station_series.MISSING_VAL
# compute monthly anomalies for this station data
station_anomalies = station_series.monthly_anomaly_series
# What are the first and last valid months in this station's data set?
# We've saved them as the first and last changepoint before...
first = sorted_cps[0]
last = sorted_cps[-1]
# What are the pairs to this station that we need to consider?
station_pairs = []
for other_id in all_station_list:
pair = tuple(sorted([id, other_id]))
if pair in hom_params["pairs"]:
station_pairs.append(pair)
print station_pairs
# List the remaining changepoints after the "confirmfilt" process
for cp in sorted_cps:
cp_stats = station_cp_dict[cp]
hit_count = cp_stats["jsum"]
iy, im = imo2iym(cp)
print (
"%3d %5d %6s Estamt chgin: -- %4d %2d %4d %3d" % (ipass, station_index, id, iy, im, cp, hit_count)
)
## ACCUMULATE PAIRED CHANGEPOINTS AND AMPLITUDE ESTIMATES
# Loop over "brackets" of changepoints - that is, for changepoints
# [a, b, c, d], consider the two brackets [a,b,c] and [b,c,d] with
# the center value of the changepoints. Note that in the Fortran PHA,
# we go through these brackets in reverse order - right to left.
brackets = zip(sorted_cps[-3::-1], sorted_cps[-2::-1], sorted_cps[::-1])
final_results[id] = dict()
for bracket in brackets:
# for bracket in brackets[:1]:
(left, cp, right) = bracket[:]
ly, lm = imo2iym(left)
cpy, cpm = imo2iym(cp)
ry, rm = imo2iym(right)
print "Oriented: ", "--", "--", "--", left, cp, cp + 1, right
# setup the output string for this bracket's tests
chgptstr = " Win1: %5d %4d%2d %5d %4d%2dto Win2: %5d %4d%2d %5d %4d%2d" % (
left,
ly,
lm,
cp,
cpy,
cpm,
cp,
cpy,
cpm,
right,
ry,
rm,
)
## THIS SECTION ACCUMULATES TARGET-NEIGHBOR COMPARISONS
# See if there are enough homogeneous data in the target;
# check each window
valid_count_right = len(get_valid_data(station_data[cp + 1 : right + 1], missing_val))
valid_count_left = len(get_valid_data(station_data[left : cp + 1], missing_val))
# if the segment length (valid count) is too short, skip this
# changepoint (for now)
if valid_count_left < iminlen:
print "Adjpass seg2 short ", station_index, id, chgptstr, valid_count_left
continue
if valid_count_right < iminlen:
print "Adjpass seg1 short ", station_index, id, chgptstr, valid_count_right
continue
## We've pass the too-little-data pitfall. Now, we are actually going
## to go back through our paired neighbors and compute some final
## statistics about these changepoints. We'll store them in a
## dictionary for later, just like the pair_results dictionary
## from splitmerge
pair_results = dict()
# for (id1, id2) in [("215887", "200779")]:
for (id1, id2) in station_pairs:
# Reset the left, cp, and right indices to the original
# bracket we're considering. We are going to be changing them
# while we look at this pair
(left, cp, right) = bracket[:]
## Figure out which station is the neighbor (not the target
## we're currently considering). At the same time, note that if
## the target is the 2nd changepoint, the adjustments will be
## flipped in sign, so we need to have a correction factor ready
correction = 1.0
if id == id1:
neighb_id = id2
else:
neighb_id = id1
# correction = -1.0
# Add this pair to pair_results if it's not already there
(ida, idb) = sorted([id1, id2])
pair_str = "%s-%s" % (ida, idb)
if pair_str not in pair_results:
pair_results[neighb_id] = dict()
print pair_str
neighb_index = all_station_list.index(neighb_id)
neighb_cp_dict = network.raw_series[neighb_id].changepoints
neighb_series = network.raw_series[neighb_id]
neighb_anomalies = neighb_series.monthly_anomaly_series
## Generature a difference data set for this pair of stations
diff_data = diff(station_anomalies, neighb_anomalies)
## It's possible that in the [left, right] bracket we're looking
## at, there's a changepoint in the paired neighbor. We need
## to adjust the endpoints of the bracket to exclude those
## breakpoints
# Check right-hand side first and break out if ...
right_seg_len = len(get_valid_data(diff_data[cp + 1 : right + 1]))
# right_seg_len = len(diff_data[cp+1:right+1])
for month in range(cp + 1, right + 1):
if month == last:
continue
# ... we hit a changepoint in the neighbor ...
if month in neighb_cp_dict:
neighb_hits = neighb_cp_dict[month]["jsum"]
right_seg_len = len(get_valid_data(diff_data[cp + 1 : month + 1]))
# right_seg_len = len(diff_data[cp+1:month+1])
print (
"CHG2: ",
neighb_index,
neighb_id,
"num,edit,2b,2e,imo,nhits",
right_seg_len,
"--",
cp + 1,
right,
month,
neighb_hits,
)
right = month
break
# ... and the final right-segment is too short
print left, cp, right
if right_seg_len < iminlen:
print (
"Low2: ",
neighb_index,
neighb_id,
"num,edit,2b,2e,imo,nhits",
right_seg_len,
"--",
cp + 1,
right,
month,
"--",
)
continue
# Now, check the left-hand side and break out if ...
left_seg_len = len(get_valid_data(diff_data[left : cp + 1]))
for month in range(cp - 1, left, -1):
if month == first:
continue
# ... we hit a changepoint in the neighbor ...
if month in neighb_cp_dict:
neighb_hits = neighb_cp_dict[month]["jsum"]
left_seg_len = len(get_valid_data(diff_data[month:cp]))
# left_seg_len = len(diff_data[month:cp])
print (
"CHG1: ",
neighb_index,
neighb_id,
"num,edit,1b,1e,imo,nhits",
left_seg_len,
"--",
cp + 1,
left,
month,
neighb_hits,
)
left = month
break
# ... and the final left-segment is too short
if left_seg_len < iminlen:
print (
"Low1: ",
neighb_index,
neighb_id,
"num,edit,1b,1e,imo,nhits",
left_seg_len,
"--",
cp + 1,
left,
month,
"--",
)
continue
## We can now estimate the raw changepoint amplitude using minbic.
## However, we'll short-circuit a lot of the work by telling it to only
## use the KTHTPR0 model (simple step-change model)
(seg_x, seg_data) = range(left + 1, right + 1), diff_data[left + 1 : right + 1]
bp_index = cp - (left + 1)
# print left, cp, right, "|", bp_index
# print left_seg_len, right_seg_len
bic_result = minbic(seg_x, seg_data, bp_index, missing_val, models=[("KTHTPR0", kthtpr0)])
## Also check the first difference correlations between the
## monthly anomalies
station_first_diff = compute_first_diff(station_anomalies, missing_val)
neighb_first_diff = compute_first_diff(neighb_anomalies, missing_val)
corr = compute_corr(station_anomalies, neighb_anomalies)
## Write out the results of this testing process so far
cmodel = bic_result["cmodel"]
bic = bic_result["bic"]
test_stat = bic_result["test_stat"]
crit_val = bic_result["crit_val"]
offset = bic_result["offset"]
slopes = bic_result["slopes"]
left_slope, right_slope = slopes
print (
"%s %6s-%6s %s %7.2f %7.2f %7.2f %7.2f %7.3f %7.3f -- %d --"
% (
tstr,
id,
neighb_id,
chgptstr,
crit_val,
test_stat,
offset,
corr,
left_slope,
right_slope,
right_seg_len,
)
)
## Analysis is done.
## Keep the adjustment (offset) for each neighbor/segment,
## set/reset trend for each neighbor/segment
## the first segment is the left-segment,
## the second segment is the right-segment
##
## Note that we reset left/right potentially to avoid conflicts
## within the paired neighbor data. However, our estimates of
## trends/offsets associated with the "right" adjacent changepoint
## actually refers to that original right changepoint. We'll
## reset left, cp, and right from the bracket before continuing
(left, cp, right) = bracket[:]
# Do the left segment first
left_dict = dict()
left_dict["adj"] = offset * correction
left_dict["cor"] = corr
left_dict["bic"] = bic
left_dict["cmodel"] = cmodel
left_dict["trend"] = left_slope
left_dict["spanob"] = left_seg_len
pair_results[neighb_id][cp] = left_dict
# Do the right segment now
right_dict = dict()
right_dict["adj"] = offset * correction
right_dict["cor"] = corr
right_dict["bic"] = bic
right_dict["cmodel"] = cmodel
right_dict["trend"] = right_slope
right_dict["spanob"] = right_seg_len
if right not in pair_results[neighb_id]:
pair_results[neighb_id][right] = right_dict
else:
# We've already recorded this segment before for the last
# changepoint. Update the slopes/spanob count (length of
# preceding segment) if the slopes are different and the
# length is different.
new_trend = slopes[1]
new_spanob = right_seg_len
old_trend = pair_results[neighb_id][right]["trend"]
old_spanob = pair_results[neighb_id][right]["spanob"]
if old_trend != new_trend:
print (
" Seg2 diff: %s %4d old: %7.2f %4d new: %7.2f %4d"
% (pair_str, right, old_trend, old_spanob, new_trend, new_spanob)
)
# if the new count is greater than the old one, the slope
# is probably more robust so update those entries.
if new_spanob > old_spanob:
pair_results[neighb_id][right]["trend"] = new_trend
pair_results[neighb_id][right]["spanob"] = new_spanob
## We're done with this pair/changepoint. Summary output -
if step == 2:
print "itarg,ipair,ichg,numc,iqt,adj,trends: -- -- -- --", cmodel, offset, slopes
# raw_input("pause")
####################################################################
## ADJUSTMENT DETERMINATION SECTION
# Recall the paired-changepoint analyses we just performed, and
# determine if the potential adjustment is statistically valid
(left, cp, right) = bracket[:]
pair_data = []
for neighb_id in pair_results:
if not cp in pair_results[neighb_id]:
continue
cp_stats = pair_results[neighb_id][cp]
adjacent_stats = pair_results[neighb_id][right]
trends = (cp_stats["trend"], adjacent_stats["trend"])
pair_dict = dict(
neighb_id=neighb_id, adj=cp_stats["adj"], cor=cp_stats["cor"], trends=trends, used=True
)
pair_data.append(pair_dict)
npairs = len(pair_data)
if npairs < numclim:
print "Adjpass numc low --", station_index, id, left, cp, right, npairs
continue
# Process -
# 1) Remove both adjustment and trend outliers
# 2) Calculate median adjustment
#
# filter around inter-quartile range
qscale = hom_params["qscale"]
pair_data = sorted(pair_data, key=operator.itemgetter("adj"))
pair_chgs = [p["adj"] for p in pair_data]
chg_25th, chg_median, chg_75th = tukey_med(pair_chgs)
chg_iqr = chg_75th - chg_25th
chg_low = chg_25th - (chg_median - chg_25th) * 1.0 * qscale
chg_high = chg_75th + (chg_75th - chg_median) * 1.0 * qscale
print (
" TRIM p25, p75, pct50, rng, lo, hi: %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"
% (chg_25th, chg_75th, chg_median, chg_iqr, chg_low, chg_high)
)
# If any of the estimated changepoints are outside the statistically
# robust range we just computed, then flag them as we print them and
for data in pair_data:
neighb_id = data["neighb_id"]
neighb_index = all_station_list.index(neighb_id)
adj = data["adj"]
cor = data["cor"]
trends = data["trends"]
if not (chg_low < adj < chg_high):
data["used"] = False
flag = "U" if data["used"] else "X"
print ("%s %4d %7.2f %8.4f %8.4f %7.2f" % (flag, neighb_index, adj, trends[0], trends[1], cor))
valid_adj_count = len([d for d in pair_data if d["used"]])
if valid_adj_count < numclim:
if step == 2:
print (
"Insuff trimmed mean -- %4d %s %5d %5d %5d %5d"
% (station_index, id, left, cp, right, valid_adj_count)
)
continue
## BUG: The code here re-computes the inter-quartile range by
## scaling qscale by 1.0. Curiously, it doesn't reject any
## pairs based on this new range.
chg_iqr = chg_75th - chg_25th
chg_low = chg_25th - (chg_median - chg_25th) * qscale
chg_high = chg_75th + (chg_75th - chg_median) * qscale
## Tweak the inter-quartile range to check if the adjustment is
## Check whether the computed adjustment is significant. That is,
## if 0 is included within the inter-quartile range we computed, then
## we can't reject the null hypothesis that the changepoint is significant
if chg_high * chg_low > 0.0:
# signs are the same, so 0 isn't included in the range.
procstr = "CONSHF"
sigadj = chg_median
else:
procstr = "ZERSHF"
sigadj = 0.0
final_results[id][cp] = dict(adj=sigadj, std=chg_iqr * 1.0 * qscale, num=npairs)
print ("%2d %s-%s %s %7.2f" % (station_index, id, procstr, chgptstr, sigadj))
## Print some final output about what changepoints remain for this station
final_station_results = final_results[id]
final_cps = sorted(final_station_results.keys())
for cp in final_cps:
adj = final_station_results[cp]["adj"]
std = final_station_results[cp]["std"]
cp_stats = station_cp_dict[cp]
hit_count = cp_stats["jsum"]
iy, im = imo2iym(cp)
print (
"-- %5d %s Estamt chgout: -- %4d%2d %5d %5d %7.2f %7.2f"
% (station_index + 1, id, iy, im, cp, hit_count, adj, std)
)
# raw_input("pause")
## Remove the accumulated non-significant changepoints (either non-sig because
## there was too much missing data, the target segment was too short, or the
## trimmed mean test could not reject the null hypothesis of no change
for id in station_list:
station_index = station_list.index(id)
final_station_results = final_results[id]
final_cps = sorted(final_station_results.keys())
for cp in final_cps:
iy, im = imo2iym(cp)
cp_index = final_cps.index(cp)
adj = final_station_results[cp]["adj"]
std = final_station_results[cp]["std"]
if adj == 0.0:
print ("%s %5d Remove chgpt %5d %4d %2d %4d" % (id, station_index, cp_index, iy, im, cp))
del network.raw_series[id].changepoints[cp]
else:
# Update the network's record of changepoints with this new list
network.raw_series[id].changepoints[cp]["ahigh"] = adj
network.raw_series[id].changepoints[cp]["astd"] = std
# the changepoint at first month has been removed; add it back in
network.raw_series[id].changepoints[first] = dict(ahigh=0.0, astd=0.0, jsum=0)
def splitmerge(network, pairs=None, beg_year=1, end_year=2, **kwargs):
## EXPERIMENTAL PLACEHOLDERS - will eventually be replaced with a master
## loop to do all the id pairs.
id_list = network.stations.keys()
pair_results = dict()
def dict_to_tuples(d):
keys = d.keys()
return [(key, d[key]) for key in keys]
## Generate station pairs for use in splitmerge by iteratively going through the
## station_list and adding stations in order of decreasing correlation. Skip a
## neighbor if the pair is already present; want 20 stations or until all the
## correlated neighbors are used up.
# pairs = []
# for id1 in id_list:
# neighbors = dict_to_tuples(network.correlations[id1])
# sorted_neighbors = sorted(neighbors, key=operator.itemgetter(1))
# added_pairs = 0
# while sorted_neighbors and (added_pairs < 5):
# id2, _ = sorted_neighbors.pop()
# ordered_pair = tuple(sorted((id1, id2)))
# if not ordered_pair in pairs:
# pairs.append(ordered_pair)
# added_pairs += 1
for (id1, id2) in pairs:
print "Pair %s with %s" % (id1, id2)
pair_str = "%6s-%6s" % (id1, id2)
#if pair_str != "051528-298107":
# continue
raw_series = network.raw_series
stations = network.stations
series_copy = deepcopy(raw_series)
min_ann = 5
num_years = end_year - beg_year
num_months = num_years*12
for s in series_copy.itervalues():
data = s.series
scaled = scale_series(data, 0.1, s.MISSING_VAL)
anomalies = compute_monthly_anomalies(scaled, s.MISSING_VAL)
s.set_series(anomalies, s.years)
## Retrieve the data for each of the stations.
station1 = stations[id1]
series1 = series_copy[id1]
data1 = series1.monthly_series
station2 = stations[id2]
series2 = series_copy[id2]
data2 = series2.monthly_series
#print data1[:50]
#print data2[:50]
#print "################################################################"
## Compute the difference series
diff_data = diff(data1, data2)
MISS = series1.MISSING_VAL # Missing value placeholder
## Quickly pass through the data to find where it starts. We need to do this
## because it's possible that beg_year is earlier than the first year of
## valid data in either data1 or data2. Furthermore, the original PHA code
## deliberately clipped off the first year of good data, so emulate that
## effect here as well.
##
## Ultimately, we save the extreme early and extreme late month with valid
## data to use as our first guess at the undocumented changepoints.
first = 0
first_set = False
last = 0
for (i, d1, d2) in zip(xrange(num_months), data1, data2):
if d1!=MISS and d2!=MISS:
if first < 12:
first = i
#first_set = True
#if not first_set:
# first = i
# first_set = True
last = i
## Set the initial breakpoints and the list of already-found, homogenous
## segments.
breakpoints = [first, last, ]
homog_segs = []
#####################################################################
## BEGIN SPLITMERGE PROCESS TO GENERATE FIRST GUESS AT UNDOCUMENTED
## CHANGEPOINTS
iter = 0 # counts how many times we've repeated the splitmerge process
enter_BIC = False # break out of iterations into the BIC process?
last_breakpoints = []
while (iter < 10) and not enter_BIC:
seg_bounds = zip(breakpoints[:-1], breakpoints[1:])
last_breakpoints = deepcopy(breakpoints)
new_breakpoints = deepcopy(breakpoints)
new_homog_segs = []
print "Parse segments (isplit = 1), ipass: "******"Too short: ", imo2iym(l), imo2iym(r)
continue
## If we've previously found that this segment is homogenous (has no
## potential changepoint), then we can skip it as well and proceed to
## the next one.
# Set the within() method to check if this segment is within any
# previously found homogenous ones. Use lambda, since we can't pass
# keyword or positional arguments to map().
within_this_seg = lambda seg: within((l, r), seg)
within_stable_segs = map(within_this_seg, homog_segs)
if any(within_stable_segs):
print "Stable segment: ", imo2iym(l), imo2iym(r)
if l == first:
new_breakpoints.append(first)
continue
## The standard normal homogeneity test - which is the statistical test
## we'll use to see if there is a potential changepoint in this segment
## - requires us to normalize our paired difference series. We can do
## that in snht(), but we'll do it right now so we can inspect those
## standardized values later.
z = standardize(segment, MISS)
## Apply standard normal homogeneity test.
## For mechanics, see Alexandersson and Moberg 1997, Int'l Jrnl of
## Climatology (pp 25-34)
likelihood_ratios = snht(z, MISS, standardized=True)
z_count = len(get_valid_data(z))
## We're left with the likelihood ratio for each value being a potential
## changepoint. Find the max ratio, and if that value is significant, let
## it be the newest potential changepoint.
ind_max_ratio = 0
max_ratio = 0.0
clip_ratios = likelihood_ratios[2:-2] # clip the beginning and end,
# they can't be changepoints.
for (ind, ratio) in zip(xrange(len(clip_ratios)), clip_ratios):
if ratio > max_ratio:
ind_max_ratio = ind
max_ratio = ratio
## Now we find the critical value for this data set, and check our max
## likelihood ratio against it
crit_val = lrt_lookup(z_count)
# The possible changepoint is the index of the max ratio we found.
# We have to shift it the following ways to align it to the original
# data -
# 1) shift by 2 re-aligns it from clip_ratios to likelihood_ratios
# 2) shift by adjust re-aligns it to this segment in diff_data
# 3) shift by l re-aligns it to the first index in diff_data
possible_changepoint = l + ind_max_ratio + 2 + adjust
y_new, m_new = imo2iym(possible_changepoint) # year, month
## If this is the first iteration, we indicate as such, and add the new
## changepoint
if iter == 0:
print "%6s-%6s MD FIRST series %4d %2d to %4d %2d | at %4d %2d ts: %4.2f limit >: %3.2f" % (id1,id2,y1,m1,y2,m2,y_new,m_new,max_ratio,crit_val)
breakpoints.append(possible_changepoint)
breakpoints = sorted(breakpoints)
else:
## Else, if we found a new possible changepoint, add it to our list.
if max_ratio > crit_val:
print "%6s-%6s MD Inhomogenity for series %4d %2d to %4d %2d | at %4d %2d ts: %4.2f limit >: %3.2f %4d" % (id1,id2,y1,m1,y2,m2,y_new,m_new,max_ratio,crit_val,z_count)
new_breakpoints.append(possible_changepoint)
## If not, record that we found a homogeneous segment.
else:
print "%6s-%6s MD Homogeneous series %4d %2d to %4d %2d | at %4d %2d ts: %4.2f limit >: %3.2f %4d" % (id1,id2,y1,m1,y2,m2,y_new,m_new,max_ratio,crit_val,z_count)
new_homog_segs.append((l, r))
## Now we need to update our account of which segments were homogeneous,
## because we need to know during the next iteration. We will do this,
## as well as condense stable segments that lie adjacent to each other
## i.e, if we have the segments [(1,5), (5, 10,),, (12, 15)], then we
## really have [(1,10), (12, 15)].
homog_segs.extend(new_homog_segs)
if homog_segs:
homog_segs = sorted(homog_segs, key=operator.itemgetter(0))
final_homog_segs = [homog_segs[0], ] # this will be like a stack
for seg in homog_segs[1:]:
last_seg = final_homog_segs[-1]
if last_seg[1] == seg[0]:
new_seg = (last_seg[0], seg[1])
final_homog_segs.pop()
final_homog_segs.append(new_seg)
else:
final_homog_segs.append(seg)
homog_segs = final_homog_segs
## So we have new segments that can be generated from these new
## breakpoints. Now, the PHA routine enters a "merge" process
## to see whether or not to keep these newly found changepoints or throw
## them out as false alarms.
##
## We do this by "leapfrogging" every other breakpoint. This gives us
## a set of segments that all have another breakpoint in them. We want
## to see if these segments are homogeneous, because if they are, it
## means that the breakpoint we previously found in the segment has
## been superseded.
new_breakpoints = sorted(new_breakpoints)
seg_bounds = zip(new_breakpoints[:-2], new_breakpoints[2:])
remove_breakpoints = set()
merged_breakpoints = set()
if iter > 0:
print "Merge segments (isplit = 0), ipass: "******"Stable segment: ", imo2iym(l), imo2iym(r)
# if l == first:
# new_breakpoints.append(first)
# seg_lookup.append(((l, r), 'stable'))
# continue
# Set the within() method to check if this segment is within any
# previously found homogenous ones. Use lambda, since we can't pass
# keyword or positional arguments to map().
within_this_seg = lambda seg: within((l, r), seg)
within_stable_segs = map(within_this_seg, homog_segs)
if any(within_stable_segs):
print "Stable segment: ", imo2iym(l), imo2iym(r)
#if l == first:
# new_breakpoints.append(first)
merged_breakpoints.update([l, r])
continue
## Apply the same adjustments and the same standard normal homogeneity
## test that we did in the previous splitting process. There is no
## difference here until we consider what to do if we find a new
## homogeneous segment.
adjust = int(seg_bounds.index((l, r)) > 0)
segment = diff_data[l+adjust:r+1]
z = standardize(segment, MISS)
likelihood_ratios = snht(z, MISS, standardized=True)
z_count = len(get_valid_data(z))
ind_max_ratio = 0
max_ratio = 0.0
clip_ratios = likelihood_ratios[2:-2] # We clip the beginning and end
for (ind, ratio) in zip(xrange(len(clip_ratios)), clip_ratios):
if ratio > max_ratio:
ind_max_ratio = ind
max_ratio = ratio
crit_val = lrt_lookup(z_count)
possible_changepoint = l + ind_max_ratio + 2 + adjust
y_new, m_new = imo2iym(possible_changepoint)
if z_count < 2:
y1, m1 = imo2iym(l)
y2, m2 = imo2iym(r)
print "%6s-%6s MD No found peaks %4d %2d to %4d %2d" % (id1,id2,y1,m1,y2,m2)
print "%6s-%6s MD Compress 1 out peak at %4d %2d" % (id1,id2,y_new,m_new)
#remove_breakpoints.add_
## If we found a new breakpoint that is statistically significant, then
## great! Let's keep it.
if max_ratio > crit_val:
print "%6s-%6s MD Peak kept in merge at %4d %2d | ts: %4.2f limit >: %3.2f" % (id1,id2,y_new,m_new,max_ratio,crit_val)
merged_breakpoints.add(l)
merged_breakpoints.add(new_bp)
merged_breakpoints.add(r)
## If not, then this segment was homogeneous, so the breakpoint which
## already exists in it is no good.
else:
print "%6s-%6s MD Compress 2 out peak at %4d %2d | ts: %4.2f limit >: %3.2f" % (id1,id2,y_new,m_new,max_ratio,crit_val)
# Crap, if there are any potential breakpoints in this segment,
# we need to remove them because this segment is homogeneous. Let's
# remember this homogeneous segment for now and come back once
# we've found all of them.
merged_breakpoints.update([l, r])
remove_breakpoints.add(new_bp)
## At this point, we have a set of all the breakpoints we've accumulated
## during this iteration of split/merge, as well as a set of breakpoints
## which we've found to be of no further use. We can difference update
## our set of breakpoints to remove these guys, and let those merged
## breakpoints be the set of newest breakpoints for the next splitmerge
## iteration.
merged_breakpoints.difference_update(remove_breakpoints)
breakpoints = list(merged_breakpoints)
breakpoints = sorted(breakpoints)
## Did we actually find new breakpoints? If not, then we're done
## with splitmerge and can move on to the BIC process.
enter_BIC = (breakpoints == last_breakpoints)
iter = iter + 1
## Okay wow, we've potentially made it to the BIC stage now... !
if first not in breakpoints:
breakpoints.insert(0, first)
ym_breakpoints = map(imo2iym, breakpoints)
#print ym_breakpoints
## ENTERING MINBIC
bp_dictionary = dict()
####################################
##### MULTIPROCESS
from multiprocessing import Pool
global counter
multi_bp_dict = {}
counter = 0
def cb(r):
global counter
#print counter, r
counter += 1
start = time.clock()
po = Pool(processes=4)
for left,bp,right in zip(breakpoints[0:], breakpoints[1:], breakpoints[2:]):
if left != first:
left = left + 1
# recall that we only consider data after the first full year. we will be
# computing regressions with the independent variable indexed from this
# starting point, so we need to shift these indices. we also need to shift them
# by +1 if this is any segment beyond the first one, so that we don't include
# changepoints in more than one analysis.
# TOTAL_SHIFT = -12 + 1 = -11
#
# However, this shift is only necessary while looking at the array indices that
# we generate using range(). the data should already be aligned correctly.
total_shift = -12 + 1
left_shift, bp_shift, right_shift = left+total_shift, bp+total_shift, right+total_shift
y1, m1 = imo2iym(left)
yb, mb = imo2iym(bp)
y2, m2 = imo2iym(right)
#print "Entering MINBIC - %4d %2d %4d %2d %4d %2d" % (y1, m1, yb,
# mb, y2, m2)
(seg_x, seg_data) = range(left_shift, right_shift+1), diff_data[left:right+1]
bp_index = bp-left
#print len(seg_x), len(seg_data), bp_index
#bp_analysis = minbic(seg_x, seg_data, bp_index, MISS)
multi_bp_dict[bp] = po.apply_async(minbic,(seg_x,seg_data,bp_index,MISS,),callback=cb)
po.close()
po.join()
for bp in multi_bp_dict:
r = multi_bp_dict[bp]
multi_bp_dict[bp] = r.get()
#print "counter - %d" % counter
elapsed = (time.clock() - start)
print "ELAPSED TIME - %2.3e" % elapsed
#print new_bp_dict
####################################
##### NORMAL
# start = time.clock()
# for left,bp,right in zip(breakpoints[0:], breakpoints[1:], breakpoints[2:]):
#
# if left != first:
# left = left + 1
# # recall that we only consider data after the first full year. we will be
# # computing regressions with the independent variable indexed from this
# # starting point, so we need to shift these indices. we also need to shift them
# # by +1 if this is any segment beyond the first one, so that we don't include
# # changepoints in more than one analysis.
# # TOTAL_SHIFT = -12 + 1 = -11
# #
# # However, this shift is only necessary while looking at the array indices that
# # we generate using range(). the data should already be aligned correctly.
# total_shift = -12 + 1
# left_shift, bp_shift, right_shift = left+total_shift, bp+total_shift, right+total_shift
# y1, m1 = imo2iym(left)
# yb, mb = imo2iym(bp)
# y2, m2 = imo2iym(right)
# print "Entering MINBIC - %4d %2d %4d %2d %4d %2d" % (y1, m1, yb,
# mb, y2, m2)
# (seg_x, seg_data) = range(left_shift, right_shift+1), diff_data[left:right+1]
# bp_index = bp-left
# #print len(seg_x), len(seg_data), bp_index
# bp_analysis = minbic(seg_x, seg_data, bp_index, MISS)
#
# bp_dictionary[bp] = bp_analysis
# elapsed2 = (time.clock() - start)
# print "ELAPSED TIME = %3.2e" % elapsed2
##################################3
## Print the adjustment summaries
bp_dictionary = multi_bp_dict
sorted_bps = sorted(bp_dictionary.keys())
ndelete = []
valid_bps = {}
for bp in sorted_bps:
stats = bp_dictionary[bp]
cmodel=stats['cmodel']
iqtype=stats['iqtype']
asigx=stats['offset']
azscr=stats['offset_z']
rslp=stats['slopes']
end1 = bp
y_end1, m_end1 = imo2iym(end1)
beg2 = bp+1
y_beg2, m_beg2 = imo2iym(beg2)
# If cmodel is *SLR*, then there is no breakpoint
if 'SLR' in cmodel:
print ("%s-%s -- -- MD TESTSEG SKIP: %7.2f %5d %5d %3d %5d %5d %3d" %
(id1, id2, asigx, end1, y_end1, m_end1, beg2, y_beg2, m_beg2))
# Don't store it!
else:
print ("%6s-%6s -- -- MD TESTSEG ADJ: %7.2f %7.2f %8.4f %8.4f %5d %5d %3d %5d %5d %3d %2d" %
(id1,id2, asigx, azscr, rslp[0], rslp[1], end1, y_end1, m_end1, beg2, y_beg2, m_beg2, iqtype))
# Store it!
valid_bps[bp] = stats
###############################
## Go back and see if we can get rid of some of the change points.
## If 2 or more of the chgpts are within MINLEN,
## a) if the chgpt estimates are the same sign, then test each
## singly with same endpoints and keep lowest BIC
## b) if not the same sign,
## retain earliest changepoint
# add the first, last to valid_bps
interior_bps = valid_bps.keys()
# Add first, last if not already in interior_bps
for bp in [first, last]:
if bp not in interior_bps:
interior_bps.append(bp)
sorted_bps = sorted(interior_bps)
for left in sorted_bps:
print sorted_bps, left
## We're looking for the next interim breakpoint that satisfies two
## conditions:
## 1) at least MINLEN valid data (non-missing to the right)
## 2) has at least one breakpoint between 'left' and it
right = 0
close_bps = []
for right in sorted_bps:
if right <= left: continue
if not close_bps:
close_bps.append(right)
else:
valid_between_bps = diff_data[close_bps[-1]:right]
valid_length = len(get_valid_data(valid_between_bps, MISS))
print imo2iym(close_bps[-1]),valid_length,imo2iym(right)
if valid_length > MINLEN:
break
close_bps.append(right)
# We could actually run out of things in sorted_bps, and wind up with
# right == close_bps[-1]. Detect that and break out of this analysis
# if that happens.
if close_bps[-1]==right: break
if left != first:
left = left + 1
close_bp_results = {}
for bp in close_bps:
# # recall that we only consider data after the first full year. we will be
# # computing regressions with the independent variable indexed from this
# # starting point, so we need to shift these indices. we also need to shift them
# # by +1 if this is any segment beyond the first one, so that we don't include
# # changepoints in more than one analysis.
# # TOTAL_SHIFT = -12 + 1 = -11
# #
# # However, this shift is only necessary while looking at the array indices that
# # we generate using range(). the data should already be aligned correctly.
total_shift = -12 + 1
left_shift, bp_shift, right_shift = left+total_shift, bp+total_shift, right+total_shift
y1, m1 = imo2iym(left)
yb, mb = imo2iym(bp)
y2, m2 = imo2iym(right)
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
print y1,m1,"-",yb,mb,"-",y2,m2
print "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
(seg_x, seg_data) = range(left_shift, right_shift+1), diff_data[left:right+1]
bp_index = bp-left
bp_analysis = minbic(seg_x, seg_data, bp_index, MISS, kthslr0_on=True)
cmodel=bp_analysis['cmodel']
iqtype= bp_analysis['iqtype']
offset= bp_analysis['offset']
rslp= bp_analysis['slopes']
crit_val = bp_analysis['crit_val']
test_stat = bp_analysis['test_stat']
bic = bp_analysis['bic']
print ("Interim chgpt: %s %4d %2d %4d %2d %4d %2d %8.2f %8.2f %8.2f %8.2f %7.3f %7.3f %2d" %
(pair_str, y1, m1, yb, mb, y2, m2, bic, test_stat, crit_val, offset, rslp[0], rslp[1], iqtype))
close_bp_results[bp] = bp_analysis
# Now we have a small problem... we might have more than one breakpoint,
# so we need to choose which one is best. We will check the sign of
# the breakpoint amplitude changes:
sign_of_amps = map(sign, [close_bp_results[bp]['offset'] for bp in close_bps])
positive = lambda x: sign(x) >= 0
negative = lambda x: sign(x) <= 0
zero = lambda x: sign(x) == 0
print "------------>",[close_bp_results[bp]['offset'] for bp in close_bps]
if (all(map(positive, sign_of_amps)) or
all(map(negative, sign_of_amps))):
# Pick the best (minimum BIC)
bics = [(bp, close_bp_results[bp]['bic']) for bp in close_bps]
sorted_bics = sorted(bics, key=operator.itemgetter(1))
smallest_bp = sorted_bics[0][0]
# Remove this smallest-bic bp from the in-interval bps
close_bps.remove(smallest_bp)
valid_bps[smallest_bp] = close_bp_results[smallest_bp]
#print "leftovers",close_bps
for bp in close_bps: # The remaining bps which we will reject
sorted_bps.remove(bp) # Remove them from this loop
del valid_bps[bp] # Remove them as valid
yb, mb = imo2iym(smallest_bp)
print ("Same domain - Lowest Interim: %s %4d %2d" %
(pair_str, yb, mb))
elif (all(map(zero, sign_of_amps))):
# Choose the earliest changepoint; the rest of these have
# amplitude changes which are 0.
first_bp, last_bp = close_bps[0], close_bps[-1]
# Remove the first interim bp and update valid_bps with this new
# computation.
close_bps.remove(first_bp)
valid_bps[first_bp] = close_bp_results[first_bp]
# Reject remaining interim bps
for bp in close_bps:
sorted_bps.remove(bp)
del valid_bps[bp]
yb, mb = imo2iym(first_bp)
print ("Null domain - Earliest Interim : %s %4d %2d" %
(pair_str, yb, mb))
else:
# We'll use the earliest interim changepoint, but we need
# to get rid of bad data. Replace all the data between the
# interim changepoints as missing and re-compute BIC.
first_bp, last_bp = close_bps[0], close_bps[-1]
first_bp_index = first_bp-left
last_bp_index = last_bp-left
print len(seg_x), len(seg_data)
print first_bp_index+1, last_bp_index+1
print left, bp, right
for i in range(first_bp_index+1, last_bp_index+1):
print i, imo2iym(i), i+left, imo2iym(i+left)
seg_x[i] = MISS
seg_data[i] = MISS
# Recall that seg_data[0] == diff_data[left]. ndelete records
# the *true month where there is unviable data*, so it needs to
# point back to the original element in diff_data we are
# worried about.
ndelete.append(i+left)
bp_analysis = minbic(seg_x, seg_data, first_bp_index, MISS, kthslr0_on=True)
# Remove the first interim bp and update valid_bps with this new
# computation.
close_bps.remove(first_bp)
valid_bps[first_bp] = bp_analysis
# Reject remaining interim bps
for bp in close_bps:
sorted_bps.remove(bp)
del valid_bps[bp]
yb, mb = imo2iym(first_bp)
print ("Diff domain - Earliest Interim : %s %4d %2d" %
(pair_str, yb, mb))
## Remove changepoints which are an SLR model.
nspan = [0]*num_months
bp_count = 1
for bp in sorted(valid_bps.keys()):
bp_analysis = valid_bps[bp]
if "SLR" in bp_analysis['cmodel']:
del valid_bps[bp]
continue
print " IN: ",bp
nspan[bp] = bp_count
## If adjacent months are missing next to this breakpoint, then
## assume that those could be a breakpoint as well and copy this
## breakpoint's analysis results for them.
for month in range(bp+1, last):
if (month in ndelete) or (diff_data[month] == MISS):
nspan[month] = bp_count
print " IN: ",month
valid_bps[month] = bp_analysis
else:
break
bp_count += 1
valid_bps['del'] = ndelete
valid_bps['nspan'] = nspan
pair_results[pair_str] = valid_bps
#print "ELAPSED TIMES = %3.2e %3.2e" % (elapsed1, elapsed2)
print "done"
##
import pickle
f = open("pair_results", 'w')
pickle.dump(pair_results, f)
return pair_results