def trend(year, dh): nt = len(year) _, ny, nx = dh.shape dhdt = np.zeros((ny, nx), 'f8') * np.nan for i in range(ny): for j in range(nx): ii, = np.where(~np.isnan(dh[:, i, j])) if len(ii) < 8: continue # TREND m, c = ap.linear_fit(year, dh[:, i, j], return_coef=True) dhdt[i, j] = m if m == 0: continue return dhdt
def trend(year, dh): nt = len(year) _, ny, nx = dh.shape dhdt = np.zeros((ny, nx), "f8") * np.nan for i in range(ny): for j in range(nx): ii, = np.where(~np.isnan(dh[:, i, j])) if len(ii) < 8: continue # TREND m, c = ap.linear_fit(year, dh[:, i, j], return_coef=True) dhdt[i, j] = m if m == 0: continue return dhdt
def plot_ts(time2, lon, lat, dh_mean_cor, dh_mean, dg_mean, R, S, diff=True): if np.alltrue(np.isnan(dh_mean[1:])): return None #time2 = y2dt(time2) R = np.mean(R) S = np.mean(S) # use only non-null and non-zero entries for correlation ind, = np.where((~np.isnan(dh_mean)) & (~np.isnan(dg_mean)) & \ (dh_mean!=0) & (dg_mean!=0)) t = np.arange(len(dh_mean)) if not diff: x, y = ap.linear_fit(dg_mean[ind], dh_mean[ind]) x2, y2 = ap.linear_fit_robust(dg_mean[ind], dh_mean[ind]) fig = plt.figure() ax = fig.add_subplot((111)) plt.plot(dg_mean[ind], dh_mean[ind], 'o') plt.plot(x, y, linewidth=2, label='lstsq fit') plt.plot(x2, y2, linewidth=2, label='robust fit') plt.legend(loc=2).draw_frame(False) plt.xlabel('dAGC (dB)') plt.ylabel('dh (m)') plt.title('Mixed-term sensitivity') else: dh_mean2 = np.diff(dh_mean) dg_mean2 = np.diff(dg_mean) dh_mean2 = np.append(dh_mean2, np.nan) dg_mean2 = np.append(dg_mean2, np.nan) x, y = ap.linear_fit(dg_mean2[ind], dh_mean2[ind]) x2, y2 = ap.linear_fit_robust(dg_mean2[ind], dh_mean2[ind]) fig = plt.figure() ax = fig.add_subplot((111)) plt.plot(dg_mean2[ind], dh_mean2[ind], 'o') plt.plot(x, y, linewidth=2, label='lstsq fit') plt.plot(x2, y2, linewidth=2, label='robust fit') plt.legend(loc=2).draw_frame(False) plt.xlabel('$\Delta$dAGC (dB)') plt.ylabel('$\Delta$dh (m)') plt.title('Short-term sensitivity') ax1 = viz.add_inner_title(ax, 'corrcoef: R = %.2f' % R, 3) ax1 = viz.add_inner_title(ax, 'slope: S = %.2f' % S, 4) plt.savefig('corr.png') #----------------- if not diff: fig = plt.figure() ax2 = plt.subplot((211)) plt.plot(time2, dh_mean, 'b', linewidth=2, label='dh') plt.plot(time2, dh_mean_cor, 'r', linewidth=2, label='dh$_{COR}$') #plt.legend().draw_frame(False) viz.add_inner_title(ax2, 'dh', 2) viz.add_inner_title(ax2, 'dh$_{COR}$', 3) plt.title('lon = %.2f, lat = %.2f' % (lon, lat)) plt.ylabel('m') #plt.xlim(1992, 2012.1) #plt.gca().xaxis.set_major_formatter(plt.FormatStrFormatter('%d')) ax3 = plt.subplot((212)) plt.plot(time2, dg_mean, 'g', linewidth=2, label='dAGC') #plt.legend().draw_frame(False) viz.add_inner_title(ax3, 'dAGC', 3) plt.ylabel('dB') #plt.xlim(1992, 2012.1) #plt.gca().xaxis.set_major_formatter(plt.FormatStrFormatter('%d')) else: fig = plt.figure() ax2 = plt.subplot((311)) plt.plot(time2, dh_mean, 'b', linewidth=2, label='dh') plt.plot(time2, dh_mean_cor, 'r', linewidth=2, label='dh$_{COR}$') #plt.legend().draw_frame(False) viz.add_inner_title(ax2, 'dh', 2) viz.add_inner_title(ax2, 'dh$_{COR}$', 3) plt.title('lon = %.2f, lat = %.2f' % (lon, lat)) plt.ylabel('m') #plt.xlim(1992, 2012.1) #plt.gca().xaxis.set_major_formatter(plt.FormatStrFormatter('%d')) ax3 = plt.subplot((312)) plt.plot(time2, dh_mean2, 'm', linewidth=2, label='$\Delta$dh') #plt.legend().draw_frame(False) viz.add_inner_title(ax3, '$\Delta$dh', 3) plt.ylabel('m') #plt.xlim(1992, 2012.1) #plt.gca().xaxis.set_major_formatter(plt.FormatStrFormatter('%d')) ax4 = plt.subplot((313)) plt.plot(time2, dg_mean2, 'c', linewidth=2, label='$\Delta$dAGC') #plt.legend().draw_frame(False) viz.add_inner_title(ax4, '$\Delta$dAGC', 3) plt.ylabel('dB') #plt.xlim(1992, 2012.1) #plt.gca().xaxis.set_major_formatter(plt.FormatStrFormatter('%d')) fig.autofmt_xdate() plt.savefig('ts.png') return fig
def main(): fname_in = sys.argv[1] din = GetData(fname_in, 'a') satname = din.satname time = change_day(din.time, 15) # change all days (e.g. 14,15,16,17) to 15 ts = getattr(din, VAR_TO_CALIBRATE) err = din.dh_error n_ad = din.n_ad n_da = din.n_da lon = din.lon lat = din.lat din.file.close() t = ap.num2year(time) if SUBSET: # get subset ts, lon2, lat2 = ap.get_subset(ap.amundsen, ts, lon, lat) err, lon2, lat2 = ap.get_subset(ap.amundsen, err, lon, lat) n_ad, lon2, lat2 = ap.get_subset(ap.amundsen, n_ad, lon, lat) n_da, lon2, lat2 = ap.get_subset(ap.amundsen, n_da, lon, lat) lon, lat = lon2, lat2 xx, yy = np.meshgrid(lon, lat) nt, ny, nx = ts.shape offset_12 = np.full((ny, nx), np.nan) offset_23 = np.full((ny, nx), np.nan) print 'cross-calibrating time series:', VAR_TO_CALIBRATE isfirst = True if SAT_NAMES is None: satnames = np.unique(din.satname) # iterate over every grid cell (all times) no_overlap_12 = 0 no_overlap_23 = 0 for i in xrange(ny): for j in xrange(nx): if 0: i, j = ap.find_nearest2(xx, yy, (LON, LAT)) i -= 0 j += 0 print 'grid-cell:', i, j ts_ij = ts[:, i, j] if np.isnan(ts_ij).all(): continue # get all time series (all sats) in one df (per grid-cell) var = create_df_with_sats(time, ts_ij, satname, SAT_NAMES) if DETREND: var = var.apply(detrend) if FILTER: var = var.apply(ap.hp_filt, lamb=7, nan=True) if PLOT_TS and (var.count().sum() > 10): print 'grid-cell:', i, j var.plot(linewidth=3, figsize=(9, 3), legend=False) plt.title('Elevation change, dh (lon=%.2f, lat=%.2f)' % (xx[i, j], yy[i, j])) plt.ylabel('m') plt.show() # compute offset (if ts overlap) #--------------------------------------------------- x = pd.notnull(var) overlap_12 = x['ers1'] & x['ers2'] overlap_23 = x['ers2'] & x['envi'] if np.sometrue(overlap_12): if SAT_BIAS: s1 = var['ers1'][overlap_12] s2 = var['ers2'][overlap_12] if LINEAR_FIT: # using linear fit s1 = s1[s1.notnull() & s2.notnull()] s2 = s2[s1.notnull() & s2.notnull()] if len(s1) > 1 and len(s2) > 1: s1.index, s1[:] = ap.linear_fit( ap.date2year(s1.index), s1.values) s2.index, s2[:] = ap.linear_fit( ap.date2year(s2.index), s2.values) offset = (s1.values[-1] - s1.values[0]) - ( s2.values[-1] - s2.values[0]) else: pass else: # using absolute values s1 = ap.referenced(s1, to='first') s2 = ap.referenced(s2, to='first') s1[0], s2[0] = np.nan, np.nan # remove first values offset = np.nanmean(s1 - s2) #pd.concat((s1, s2), axis=1).plot(marker='o') else: offset = np.nanmean(var['ers1'] - var['ers2']) offset_12[i, j] = offset else: no_overlap_12 += 1 if np.sometrue(overlap_23): if SAT_BIAS: s2 = var['ers2'][overlap_23] s3 = var['envi'][overlap_23] if LINEAR_FIT: s2 = s2[s2.notnull() & s3.notnull()] s3 = s3[s2.notnull() & s3.notnull()] if len(s2) > 1 and len(s3) > 1: s2.index, s2[:] = ap.linear_fit( ap.date2year(s2.index), s2.values) s3.index, s3[:] = ap.linear_fit( ap.date2year(s3.index), s3.values) offset = (s2.values[-1] - s2.values[0]) - ( s3.values[-1] - s3.values[0]) else: pass else: s2 = ap.referenced(s2, to='first') s3 = ap.referenced(s3, to='first') s2[0], s3[0] = np.nan, np.nan offset = np.nanmean(s2 - s3) #pd.concat((s2, s3), axis=1).plot(marker='o') #plt.show() else: offset = np.nanmean(var['ers2'] - var['envi']) offset_23[i, j] = offset else: no_overlap_23 += 1 #--------------------------------------------------- mean_offset_12 = np.nanmean(offset_12) median_offset_12 = np.nanmedian(offset_12) mean_offset_23 = np.nanmean(offset_23) median_offset_23 = np.nanmedian(offset_23) if SAVE_TO_FILE: fout = tb.open_file(FNAME_OUT, 'w') fout.create_array('/', 'lon', lon) fout.create_array('/', 'lat', lat) fout.create_array('/', 'offset_12', offset_12) fout.create_array('/', 'offset_23', offset_23) fout.close() if PLOT: plt.figure() plt.subplot(211) offset_12 = ap.median_filt(offset_12, 3, 3) plt.imshow(offset_12, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5) plt.title('ERS1-ERS2') plt.colorbar(shrink=0.8) plt.subplot(212) offset_23 = ap.median_filt(offset_23, 3, 3) plt.imshow(offset_23, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5) plt.title('ERS2-Envisat') #plt.colorbar(shrink=0.3, orientation='h') plt.colorbar(shrink=0.8) plt.figure() plt.subplot(121) o12 = offset_12[~np.isnan(offset_12)] plt.hist(o12, bins=100) plt.title('ERS1-ERS2') ax = plt.gca() ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_12, median_offset_12), ax=ax, loc=2) plt.xlim(-1, 1) plt.subplot(122) o23 = offset_23[~np.isnan(offset_23)] plt.hist(o23, bins=100) plt.title('ERS2-Envisat') ax = plt.gca() ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_23, median_offset_23), ax=ax, loc=2) plt.xlim(-1, 1) plt.show() print 'calibrated variable:', VAR_TO_CALIBRATE print 'no overlaps:', no_overlap_12, no_overlap_23 print 'mean offset:', mean_offset_12, mean_offset_23 print 'median offset:', median_offset_12, median_offset_23 print 'out file ->', FNAME_OUT
x = sin_data['x'].values y = sin_data['y'].values X = dmatrix('C(x, Poly)') N = 5 w = 1/noise out = ap.lstsq_cv(x, y, cv=10, max_deg=N, weight=w, randomise=True, return_coef=True) y_wls, coef, deg, mse, var = out y_ols = ap.lstsq_cv(x, y, cv=10, max_deg=N, weight=None, randomise=True) a2 = np.polyfit(x, y, 1, w=None)#w) y_line = np.polyval(a2, x) m, c = ap.linear_fit(x, y, return_coef=True) m2, c2 = ap.linear_fit_robust(x, y, return_coef=True) out = ap.lasso_cv(x, y, cv=10, max_deg=N, return_model=True) y_lasso, lasso = out a = np.append(lasso.intercept_, lasso.coef_) #y_lasso = np.dot(X[:,:N+1], a) #dy_lasso = a[1] * X[:,0] + 2 * a[2] * X[:,1] # + 3 * a[3] * X[:,2] dy_lasso = np.gradient(y_lasso, x[2] - x[1]) print a[1] print 'coef.:', a print 'slope:', y_lasso[-1] - y_lasso[0]
def main(): fname_in = sys.argv[1] din = GetData(fname_in, 'a') satname = din.satname time = change_day(din.time, 15) # change all days (e.g. 14,15,16,17) to 15 ts = getattr(din, VAR_TO_CALIBRATE) err = din.dh_error n_ad = din.n_ad n_da = din.n_da lon = din.lon lat = din.lat din.file.close() t = ap.num2year(time) if SUBSET: # get subset ts, lon2, lat2 = ap.get_subset(ap.amundsen, ts, lon, lat) err, lon2, lat2 = ap.get_subset(ap.amundsen, err, lon, lat) n_ad, lon2, lat2 = ap.get_subset(ap.amundsen, n_ad, lon, lat) n_da, lon2, lat2 = ap.get_subset(ap.amundsen, n_da, lon, lat) lon, lat = lon2, lat2 xx, yy = np.meshgrid(lon, lat) nt, ny, nx = ts.shape offset_12 = np.full((ny,nx), np.nan) offset_23 = np.full((ny,nx), np.nan) print 'cross-calibrating time series:', VAR_TO_CALIBRATE isfirst = True if SAT_NAMES is None: satnames = np.unique(din.satname) # iterate over every grid cell (all times) no_overlap_12 = 0 no_overlap_23 = 0 for i in xrange(ny): for j in xrange(nx): if 0: i, j = ap.find_nearest2(xx, yy, (LON,LAT)) i -= 0 j += 0 print 'grid-cell:', i, j ts_ij = ts[:,i,j] if np.isnan(ts_ij).all(): continue # get all time series (all sats) in one df (per grid-cell) var = create_df_with_sats(time, ts_ij, satname, SAT_NAMES) if DETREND: var = var.apply(detrend) if FILTER: var = var.apply(ap.hp_filt, lamb=7, nan=True) if PLOT_TS and (var.count().sum() > 10): print 'grid-cell:', i, j var.plot(linewidth=3, figsize=(9, 3), legend=False) plt.title('Elevation change, dh (lon=%.2f, lat=%.2f)' % (xx[i,j], yy[i,j])) plt.ylabel('m') plt.show() # compute offset (if ts overlap) #--------------------------------------------------- x = pd.notnull(var) overlap_12 = x['ers1'] & x['ers2'] overlap_23 = x['ers2'] & x['envi'] if np.sometrue(overlap_12): if SAT_BIAS: s1 = var['ers1'][overlap_12] s2 = var['ers2'][overlap_12] if LINEAR_FIT: # using linear fit s1 = s1[s1.notnull() & s2.notnull()] s2 = s2[s1.notnull() & s2.notnull()] if len(s1) > 1 and len(s2) > 1: s1.index, s1[:] = ap.linear_fit(ap.date2year(s1.index), s1.values) s2.index, s2[:] = ap.linear_fit(ap.date2year(s2.index), s2.values) offset = (s1.values[-1] - s1.values[0]) - (s2.values[-1] - s2.values[0]) else: pass else: # using absolute values s1 = ap.referenced(s1, to='first') s2 = ap.referenced(s2, to='first') s1[0], s2[0] = np.nan, np.nan # remove first values offset = np.nanmean(s1 - s2) #pd.concat((s1, s2), axis=1).plot(marker='o') else: offset = np.nanmean(var['ers1'] - var['ers2']) offset_12[i,j] = offset else: no_overlap_12 += 1 if np.sometrue(overlap_23): if SAT_BIAS: s2 = var['ers2'][overlap_23] s3 = var['envi'][overlap_23] if LINEAR_FIT: s2 = s2[s2.notnull() & s3.notnull()] s3 = s3[s2.notnull() & s3.notnull()] if len(s2) > 1 and len(s3) > 1: s2.index, s2[:] = ap.linear_fit(ap.date2year(s2.index), s2.values) s3.index, s3[:] = ap.linear_fit(ap.date2year(s3.index), s3.values) offset = (s2.values[-1] - s2.values[0]) - (s3.values[-1] - s3.values[0]) else: pass else: s2 = ap.referenced(s2, to='first') s3 = ap.referenced(s3, to='first') s2[0], s3[0] = np.nan, np.nan offset = np.nanmean(s2 - s3) #pd.concat((s2, s3), axis=1).plot(marker='o') #plt.show() else: offset = np.nanmean(var['ers2'] - var['envi']) offset_23[i,j] = offset else: no_overlap_23 += 1 #--------------------------------------------------- mean_offset_12 = np.nanmean(offset_12) median_offset_12 = np.nanmedian(offset_12) mean_offset_23 = np.nanmean(offset_23) median_offset_23 = np.nanmedian(offset_23) if SAVE_TO_FILE: fout = tb.open_file(FNAME_OUT, 'w') fout.create_array('/', 'lon', lon) fout.create_array('/', 'lat', lat) fout.create_array('/', 'offset_12', offset_12) fout.create_array('/', 'offset_23', offset_23) fout.close() if PLOT: plt.figure() plt.subplot(211) offset_12 = ap.median_filt(offset_12, 3, 3) plt.imshow(offset_12, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5) plt.title('ERS1-ERS2') plt.colorbar(shrink=0.8) plt.subplot(212) offset_23 = ap.median_filt(offset_23, 3, 3) plt.imshow(offset_23, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5) plt.title('ERS2-Envisat') #plt.colorbar(shrink=0.3, orientation='h') plt.colorbar(shrink=0.8) plt.figure() plt.subplot(121) o12 = offset_12[~np.isnan(offset_12)] plt.hist(o12, bins=100) plt.title('ERS1-ERS2') ax = plt.gca() ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_12, median_offset_12), ax=ax, loc=2) plt.xlim(-1, 1) plt.subplot(122) o23 = offset_23[~np.isnan(offset_23)] plt.hist(o23, bins=100) plt.title('ERS2-Envisat') ax = plt.gca() ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_23, median_offset_23), ax=ax, loc=2) plt.xlim(-1, 1) plt.show() print 'calibrated variable:', VAR_TO_CALIBRATE print 'no overlaps:', no_overlap_12, no_overlap_23 print 'mean offset:', mean_offset_12, mean_offset_23 print 'median offset:', median_offset_12, median_offset_23 print 'out file ->', FNAME_OUT