コード例 #1
0
ファイル: backscatter2.py プロジェクト: fspaolo/code
def main():

    fname_in = sys.argv[1]

    din = GetData(fname_in, 'a')
    time = ap.num2date(getattr(din, T_NAME)[:])
    lon = din.lon[:]
    lat = din.lat[:]
    nrows = len(time)
    nt, ny, nx =  getattr(din, H_NAME).shape    # i,j,k = t,y,x

    RR = np.empty((nt,ny,nx), 'f8') * np.nan
    SS = np.empty((nt,ny,nx), 'f8') * np.nan

    if TINT:
        intervals = [ap.year2date(tt) for tt in INTERVALS]

    if TINT:
        print 'using time-interval correlation'
    elif TVAR:
        print 'using time-variable correlation'
    else:
        print 'using constant correlation'
    print 'processing time series:'
    isfirst = True

    # iterate over every grid cell (all times): i,j = y,x
    #-----------------------------------------------------------------

    for i in xrange(ny):
        for j in xrange(nx):
            print 'time series of grid-cell:', i, j

            dh = getattr(din, H_NAME)[:nrows,i,j]
            dg = getattr(din, G_NAME)[:nrows,i,j]

            if np.alltrue(np.isnan(dh)): continue

            #---------------------------------------------------------

            if TINT:
                # satellite-dependent R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr3(dh, dg, time, intervals, 
                                         diff=DIFF, robust=True)
            elif TVAR:
                # time-varying R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr2(dh, dg, diff=DIFF, 
                                         robust=True, npts=NPTS)
            else:
                # constant R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr(dh, dg, diff=DIFF, robust=True)

            #---------------------------------------------------------

            # plot figures
            if PLOT_TS:
                dh_cor = ap.referenced(dh_cor, to='first')
                dh = ap.referenced(dh, to='first')
                dg = ap.referenced(dg, to='first')
                k, = np.where(~np.isnan(RR[:,i,j]))
                r = np.mean(RR[k,i,j])
                s = np.mean(SS[k,i,j])
                fig = plot_rs(time, RR[:,i,j], SS[:,i,j])
                fig = plot_ts(time, lon[j], lat[i], dh_cor, dh, dg, 
                              r, s, diff=DIFF)
                if fig is None: continue
                plt.show()

            # save one TS per grid cell at a time
            #---------------------------------------------------------

            if not SAVE_TO_FILE: continue

            if isfirst:
                # open or create output file
                isfirst = False
                atom = tb.Atom.from_type('float64', dflt=np.nan)
                filters = tb.Filters(complib='zlib', complevel=9)
                c1 = din.file.createCArray('/', SAVE_AS_NAME, atom, 
                                           (nt,ny,nx), '', filters)
                c2 = din.file.createCArray('/', R_NAME, atom, 
                                           (nt,ny,nx), '', filters)
                c3 = din.file.createCArray('/', S_NAME, atom, 
                                           (nt,ny,nx), '', filters)
            c1[:,i,j] = dh_cor

    if SAVE_TO_FILE:
        c2[:] = RR
        c3[:] = SS

    if PLOT_MAP:
        if TVAR:
            # 3D -> 2D
            RR = np.mean(RR[~np.isnan(RR)], axis=0)
            SS = np.mean(SS[~np.isnan(SS)], axis=0)
        plot_map(lon, lat, np.abs(RR), BBOX, MFILE, mres=1, vmin=0, vmax=1)
        plt.title('Correlation Coefficient, R')
        plt.savefig('map_r.png')
        plot_map(lon, lat, SS, BBOX, MFILE, mres=1, vmin=-0.2, vmax=0.7)
        plt.title('Correlation Gradient, S')
        plt.savefig('map_s.png')
        plt.show()

    din.file.close()

    if SAVE_TO_FILE:
        print 'out file -->', fname_in
コード例 #2
0
def main():

    fname_in = sys.argv[1]

    din = GetData(fname_in, 'a')
    satname = din.satname
    time = change_day(din.time, 15)  # change all days (e.g. 14,15,16,17) to 15
    ts = getattr(din, VAR_TO_CALIBRATE)
    err = din.dh_error
    n_ad = din.n_ad
    n_da = din.n_da
    lon = din.lon
    lat = din.lat
    din.file.close()
    t = ap.num2year(time)

    if SUBSET:  # get subset
        ts, lon2, lat2 = ap.get_subset(ap.amundsen, ts, lon, lat)
        err, lon2, lat2 = ap.get_subset(ap.amundsen, err, lon, lat)
        n_ad, lon2, lat2 = ap.get_subset(ap.amundsen, n_ad, lon, lat)
        n_da, lon2, lat2 = ap.get_subset(ap.amundsen, n_da, lon, lat)
        lon, lat = lon2, lat2

    xx, yy = np.meshgrid(lon, lat)
    nt, ny, nx = ts.shape
    offset_12 = np.full((ny, nx), np.nan)
    offset_23 = np.full((ny, nx), np.nan)

    print 'cross-calibrating time series:', VAR_TO_CALIBRATE

    isfirst = True

    if SAT_NAMES is None:
        satnames = np.unique(din.satname)

    # iterate over every grid cell (all times)
    no_overlap_12 = 0
    no_overlap_23 = 0
    for i in xrange(ny):
        for j in xrange(nx):

            if 0:
                i, j = ap.find_nearest2(xx, yy, (LON, LAT))
                i -= 0
                j += 0

            print 'grid-cell:', i, j

            ts_ij = ts[:, i, j]
            if np.isnan(ts_ij).all(): continue

            # get all time series (all sats) in one df (per grid-cell)
            var = create_df_with_sats(time, ts_ij, satname, SAT_NAMES)

            if DETREND:
                var = var.apply(detrend)

            if FILTER:
                var = var.apply(ap.hp_filt, lamb=7, nan=True)

            if PLOT_TS and (var.count().sum() > 10):
                print 'grid-cell:', i, j
                var.plot(linewidth=3, figsize=(9, 3), legend=False)
                plt.title('Elevation change, dh  (lon=%.2f, lat=%.2f)' %
                          (xx[i, j], yy[i, j]))
                plt.ylabel('m')
                plt.show()

            # compute offset (if ts overlap)
            #---------------------------------------------------
            x = pd.notnull(var)
            overlap_12 = x['ers1'] & x['ers2']
            overlap_23 = x['ers2'] & x['envi']

            if np.sometrue(overlap_12):
                if SAT_BIAS:
                    s1 = var['ers1'][overlap_12]
                    s2 = var['ers2'][overlap_12]
                    if LINEAR_FIT:
                        # using linear fit
                        s1 = s1[s1.notnull() & s2.notnull()]
                        s2 = s2[s1.notnull() & s2.notnull()]
                        if len(s1) > 1 and len(s2) > 1:
                            s1.index, s1[:] = ap.linear_fit(
                                ap.date2year(s1.index), s1.values)
                            s2.index, s2[:] = ap.linear_fit(
                                ap.date2year(s2.index), s2.values)
                            offset = (s1.values[-1] - s1.values[0]) - (
                                s2.values[-1] - s2.values[0])
                        else:
                            pass
                    else:
                        # using absolute values
                        s1 = ap.referenced(s1, to='first')
                        s2 = ap.referenced(s2, to='first')
                        s1[0], s2[0] = np.nan, np.nan  # remove first values
                        offset = np.nanmean(s1 - s2)

                    #pd.concat((s1, s2), axis=1).plot(marker='o')
                else:
                    offset = np.nanmean(var['ers1'] - var['ers2'])
                offset_12[i, j] = offset
            else:
                no_overlap_12 += 1

            if np.sometrue(overlap_23):
                if SAT_BIAS:
                    s2 = var['ers2'][overlap_23]
                    s3 = var['envi'][overlap_23]
                    if LINEAR_FIT:
                        s2 = s2[s2.notnull() & s3.notnull()]
                        s3 = s3[s2.notnull() & s3.notnull()]
                        if len(s2) > 1 and len(s3) > 1:
                            s2.index, s2[:] = ap.linear_fit(
                                ap.date2year(s2.index), s2.values)
                            s3.index, s3[:] = ap.linear_fit(
                                ap.date2year(s3.index), s3.values)
                            offset = (s2.values[-1] - s2.values[0]) - (
                                s3.values[-1] - s3.values[0])
                        else:
                            pass
                    else:
                        s2 = ap.referenced(s2, to='first')
                        s3 = ap.referenced(s3, to='first')
                        s2[0], s3[0] = np.nan, np.nan
                        offset = np.nanmean(s2 - s3)
                    #pd.concat((s2, s3), axis=1).plot(marker='o')
                    #plt.show()
                else:
                    offset = np.nanmean(var['ers2'] - var['envi'])
                offset_23[i, j] = offset
            else:
                no_overlap_23 += 1

            #---------------------------------------------------

    mean_offset_12 = np.nanmean(offset_12)
    median_offset_12 = np.nanmedian(offset_12)
    mean_offset_23 = np.nanmean(offset_23)
    median_offset_23 = np.nanmedian(offset_23)

    if SAVE_TO_FILE:
        fout = tb.open_file(FNAME_OUT, 'w')
        fout.create_array('/', 'lon', lon)
        fout.create_array('/', 'lat', lat)
        fout.create_array('/', 'offset_12', offset_12)
        fout.create_array('/', 'offset_23', offset_23)
        fout.close()

    if PLOT:
        plt.figure()
        plt.subplot(211)
        offset_12 = ap.median_filt(offset_12, 3, 3)
        plt.imshow(offset_12,
                   origin='lower',
                   interpolation='nearest',
                   vmin=-.5,
                   vmax=.5)
        plt.title('ERS1-ERS2')
        plt.colorbar(shrink=0.8)
        plt.subplot(212)
        offset_23 = ap.median_filt(offset_23, 3, 3)
        plt.imshow(offset_23,
                   origin='lower',
                   interpolation='nearest',
                   vmin=-.5,
                   vmax=.5)
        plt.title('ERS2-Envisat')
        #plt.colorbar(shrink=0.3, orientation='h')
        plt.colorbar(shrink=0.8)
        plt.figure()
        plt.subplot(121)
        o12 = offset_12[~np.isnan(offset_12)]
        plt.hist(o12, bins=100)
        plt.title('ERS1-ERS2')
        ax = plt.gca()
        ap.intitle('mean/median = %.2f/%.2f m' %
                   (mean_offset_12, median_offset_12),
                   ax=ax,
                   loc=2)
        plt.xlim(-1, 1)
        plt.subplot(122)
        o23 = offset_23[~np.isnan(offset_23)]
        plt.hist(o23, bins=100)
        plt.title('ERS2-Envisat')
        ax = plt.gca()
        ap.intitle('mean/median = %.2f/%.2f m' %
                   (mean_offset_23, median_offset_23),
                   ax=ax,
                   loc=2)
        plt.xlim(-1, 1)
        plt.show()

    print 'calibrated variable:', VAR_TO_CALIBRATE
    print 'no overlaps:', no_overlap_12, no_overlap_23
    print 'mean offset:', mean_offset_12, mean_offset_23
    print 'median offset:', median_offset_12, median_offset_23
    print 'out file ->', FNAME_OUT
コード例 #3
0
ファイル: trendfit.py プロジェクト: sjl421/code-2
nz, ny, nx = data.shape


if 0: # subset
    print 'subsetting...'
    region = ap.dotson
    data, _, _ = ap.get_subset(region, data, lon, lat)
    nt, ny, nx = data.shape                # i,j,k = t,y,x

#------------------------------------------------------

if 0: # plot alphas and MSEs (the prediction error curve)
    N = 3
    x = time
    y = data[:,5,2] # 6,2 -> PIG
    y = ap.referenced(y, to='mean')
    y_pred, lasso = ap.lasso_cv(x, y, cv=10, max_deg=N, max_iter=1e3, return_model=True)
    mse = lasso.mse_path_.mean(axis=1)
    std = lasso.mse_path_.std(axis=1, ddof=1) / np.sqrt(10)
    #plt.plot(np.log(lasso.alphas_), mse)
    plt.errorbar(np.log(lasso.alphas_), mse, yerr=std)
    plt.vlines(np.log(lasso.alpha_), ymin=mse.min(), ymax=mse.max(), color='r')
    plt.xlabel('log(alpha)')
    plt.ylabel('10-fold-average MSE')
    plt.show()
    exit()

#------------------------------------------------------

data = as_frame(data, time, lat, lon)
data = data.apply(ap.referenced, to='mean', raw=True)
コード例 #4
0
def main():

    fname_in = sys.argv[1]

    din = GetData(fname_in, 'a')
    time = ap.num2date(getattr(din, T_NAME)[:])
    lon = din.lon[:]
    lat = din.lat[:]
    sat = din.satname[:]
    nrows = len(time)
    nt, ny, nx =  getattr(din, H_NAME).shape    # i,j,k = t,y,x

    RR = np.empty((nt,ny,nx), 'f8') * np.nan
    SS = np.empty((nt,ny,nx), 'f8') * np.nan

    print 'processing time series:'
    isfirst = True

    # iterate over every grid cell (all times): i,j = y,x
    #-----------------------------------------------------------------

    for i in xrange(ny):
        for j in xrange(nx):
            print 'time series of grid-cell:', i, j

            dh = getattr(din, H_NAME)[:nrows,i,j]
            dg = getattr(din, G_NAME)[:nrows,i,j]
            dh_cor = np.zeros_like(dh)

            if np.alltrue(np.isnan(dh)): continue

            #---------------------------------------------------------

            # pull and correct a chunk of the array at a time
            for s in np.unique(sat):
                k = np.where(sat == s)
                dh_cor[k], R, S = ap.backscatter_corr(dh[k], dg[k], 
                                  diff=DIFF, robust=True)
                RR[k,i,j] = R
                SS[k,i,j] = S

            #---------------------------------------------------------

            if PLOT_TS:
                dh_cor = ap.referenced(dh_cor, to='first')
                dh = ap.referenced(dh, to='first')
                dg = ap.referenced(dg, to='first')
                fig = plot_rs(time, RR[:,i,j], SS[:,i,j])
                for s in np.unique(sat):
                    k = np.where(sat == s)
                    r, s = np.mean(RR[k,i,j]), np.mean(SS[k,i,j])
                    try:
                        fig = plot_ts(time[k], lon[j], lat[i], dh_cor[k], 
                                      dh[k], dg[k], r, s, diff=DIFF)
                    except:
                        print 'something wrong with ploting!'
                        print 'dh:', dh
                        print 'dg:', dg
                if fig is None: continue
                plt.show()

            # save one TS per grid cell at a time
            #---------------------------------------------------------

            if not SAVE_TO_FILE: continue

            if isfirst:
                # open or create output file
                isfirst = False
                atom = tb.Atom.from_type('float64', dflt=np.nan)
                filters = tb.Filters(complib='zlib', complevel=9)
                try:
                    c1 = din.file.create_carray('/', SAVE_AS_NAME, atom, 
                                               (nt,ny,nx), '', filters)
                except:
                    c1 = din.file.getNode('/', SAVE_AS_NAME)
                c2 = din.file.create_carray('/', R_NAME, atom, 
                                           (nt,ny,nx), '', filters)
                c3 = din.file.create_carray('/', S_NAME, atom, 
                                           (nt,ny,nx), '', filters)
            c1[:,i,j] = dh_cor

    if SAVE_TO_FILE:
        c2[:] = RR
        c3[:] = SS

    if PLOT_MAP:
        RR = RR[0]  # change accordingly
        SS = SS[0]
        plot_map(lon, lat, np.abs(RR), BBOX, MASK_FILE, mres=1, vmin=0, vmax=1)
        plt.title('Correlation Coefficient, R')
        plt.savefig('map_r.png')
        plot_map(lon, lat, SS, BBOX, MASK_FILE, mres=1, vmin=-0.2, vmax=0.7)
        plt.title('Correlation Gradient, S')
        plt.savefig('map_s.png')
        plt.show()

    din.file.close()

    if SAVE_TO_FILE:
        print 'out file -->', fname_in
コード例 #5
0
nz, ny, nx = data.shape


if 0:  # subset
    print "subsetting..."
    region = ap.dotson
    data, _, _ = ap.get_subset(region, data, lon, lat)
    nt, ny, nx = data.shape  # i,j,k = t,y,x

# ------------------------------------------------------

if 0:  # plot alphas and MSEs (the prediction error curve)
    N = 3
    x = time
    y = data[:, 5, 2]  # 6,2 -> PIG
    y = ap.referenced(y, to="mean")
    y_pred, lasso = ap.lasso_cv(x, y, cv=10, max_deg=N, max_iter=1e3, return_model=True)
    mse = lasso.mse_path_.mean(axis=1)
    std = lasso.mse_path_.std(axis=1, ddof=1) / np.sqrt(10)
    # plt.plot(np.log(lasso.alphas_), mse)
    plt.errorbar(np.log(lasso.alphas_), mse, yerr=std)
    plt.vlines(np.log(lasso.alpha_), ymin=mse.min(), ymax=mse.max(), color="r")
    plt.xlabel("log(alpha)")
    plt.ylabel("10-fold-average MSE")
    plt.show()
    exit()

# ------------------------------------------------------

data = as_frame(data, time, lat, lon)
data = data.apply(ap.referenced, to="mean", raw=True)
コード例 #6
0
ファイル: backscatter2.py プロジェクト: sjl421/code-2
def main():

    fname_in = sys.argv[1]

    din = GetData(fname_in, 'a')
    time = ap.num2date(getattr(din, T_NAME)[:])
    lon = din.lon[:]
    lat = din.lat[:]
    nrows = len(time)
    nt, ny, nx = getattr(din, H_NAME).shape  # i,j,k = t,y,x

    RR = np.empty((nt, ny, nx), 'f8') * np.nan
    SS = np.empty((nt, ny, nx), 'f8') * np.nan

    if TINT:
        intervals = [ap.year2date(tt) for tt in INTERVALS]

    if TINT:
        print 'using time-interval correlation'
    elif TVAR:
        print 'using time-variable correlation'
    else:
        print 'using constant correlation'
    print 'processing time series:'
    isfirst = True

    # iterate over every grid cell (all times): i,j = y,x
    #-----------------------------------------------------------------

    for i in xrange(ny):
        for j in xrange(nx):
            print 'time series of grid-cell:', i, j

            dh = getattr(din, H_NAME)[:nrows, i, j]
            dg = getattr(din, G_NAME)[:nrows, i, j]

            if np.alltrue(np.isnan(dh)): continue

            #---------------------------------------------------------

            if TINT:
                # satellite-dependent R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr3(dh, dg, time, intervals,
                                         diff=DIFF, robust=True)
            elif TVAR:
                # time-varying R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr2(dh, dg, diff=DIFF,
                                         robust=True, npts=NPTS)
            else:
                # constant R and S
                dh_cor, RR[:,i,j], SS[:,i,j] = \
                    ap.backscatter_corr(dh, dg, diff=DIFF, robust=True)

            #---------------------------------------------------------

            # plot figures
            if PLOT_TS:
                dh_cor = ap.referenced(dh_cor, to='first')
                dh = ap.referenced(dh, to='first')
                dg = ap.referenced(dg, to='first')
                k, = np.where(~np.isnan(RR[:, i, j]))
                r = np.mean(RR[k, i, j])
                s = np.mean(SS[k, i, j])
                fig = plot_rs(time, RR[:, i, j], SS[:, i, j])
                fig = plot_ts(time,
                              lon[j],
                              lat[i],
                              dh_cor,
                              dh,
                              dg,
                              r,
                              s,
                              diff=DIFF)
                if fig is None: continue
                plt.show()

            # save one TS per grid cell at a time
            #---------------------------------------------------------

            if not SAVE_TO_FILE: continue

            if isfirst:
                # open or create output file
                isfirst = False
                atom = tb.Atom.from_type('float64', dflt=np.nan)
                filters = tb.Filters(complib='zlib', complevel=9)
                c1 = din.file.createCArray('/', SAVE_AS_NAME, atom,
                                           (nt, ny, nx), '', filters)
                c2 = din.file.createCArray('/', R_NAME, atom, (nt, ny, nx), '',
                                           filters)
                c3 = din.file.createCArray('/', S_NAME, atom, (nt, ny, nx), '',
                                           filters)
            c1[:, i, j] = dh_cor

    if SAVE_TO_FILE:
        c2[:] = RR
        c3[:] = SS

    if PLOT_MAP:
        if TVAR:
            # 3D -> 2D
            RR = np.mean(RR[~np.isnan(RR)], axis=0)
            SS = np.mean(SS[~np.isnan(SS)], axis=0)
        plot_map(lon, lat, np.abs(RR), BBOX, MFILE, mres=1, vmin=0, vmax=1)
        plt.title('Correlation Coefficient, R')
        plt.savefig('map_r.png')
        plot_map(lon, lat, SS, BBOX, MFILE, mres=1, vmin=-0.2, vmax=0.7)
        plt.title('Correlation Gradient, S')
        plt.savefig('map_s.png')
        plt.show()

    din.file.close()

    if SAVE_TO_FILE:
        print 'out file -->', fname_in
コード例 #7
0
def main():

    fname_in = sys.argv[1] 

    din = GetData(fname_in, 'a')
    satname = din.satname
    time = change_day(din.time, 15)      # change all days (e.g. 14,15,16,17) to 15
    ts = getattr(din, VAR_TO_CALIBRATE)
    err = din.dh_error
    n_ad = din.n_ad
    n_da = din.n_da
    lon = din.lon
    lat = din.lat
    din.file.close()
    t = ap.num2year(time)

    if SUBSET: # get subset
        ts, lon2, lat2 = ap.get_subset(ap.amundsen, ts, lon, lat)
        err, lon2, lat2 = ap.get_subset(ap.amundsen, err, lon, lat)
        n_ad, lon2, lat2 = ap.get_subset(ap.amundsen, n_ad, lon, lat)
        n_da, lon2, lat2 = ap.get_subset(ap.amundsen, n_da, lon, lat)
        lon, lat = lon2, lat2

    xx, yy = np.meshgrid(lon, lat)
    nt, ny, nx = ts.shape
    offset_12 = np.full((ny,nx), np.nan)
    offset_23 = np.full((ny,nx), np.nan)

    print 'cross-calibrating time series:', VAR_TO_CALIBRATE

    isfirst = True

    if SAT_NAMES is None:
        satnames = np.unique(din.satname)

    # iterate over every grid cell (all times)
    no_overlap_12 = 0
    no_overlap_23 = 0
    for i in xrange(ny):
        for j in xrange(nx):

            if 0:
                i, j = ap.find_nearest2(xx, yy, (LON,LAT))
                i -= 0
                j += 0

            print 'grid-cell:', i, j

            ts_ij = ts[:,i,j]
            if np.isnan(ts_ij).all(): continue

            # get all time series (all sats) in one df (per grid-cell)
            var = create_df_with_sats(time, ts_ij, satname, SAT_NAMES)

            if DETREND:
                var = var.apply(detrend)

            if FILTER:
                var = var.apply(ap.hp_filt, lamb=7, nan=True)

            if PLOT_TS and (var.count().sum() > 10):
                print 'grid-cell:', i, j
                var.plot(linewidth=3, figsize=(9, 3), legend=False)
                plt.title('Elevation change, dh  (lon=%.2f, lat=%.2f)' % (xx[i,j], yy[i,j]))
                plt.ylabel('m')
                plt.show()

            # compute offset (if ts overlap)
            #---------------------------------------------------
            x = pd.notnull(var)
            overlap_12 = x['ers1'] & x['ers2']
            overlap_23 = x['ers2'] & x['envi']

            if np.sometrue(overlap_12):
                if SAT_BIAS:
                    s1 = var['ers1'][overlap_12]
                    s2 = var['ers2'][overlap_12]
                    if LINEAR_FIT:
                        # using linear fit
                        s1 = s1[s1.notnull() & s2.notnull()]
                        s2 = s2[s1.notnull() & s2.notnull()]
                        if len(s1) > 1 and len(s2) > 1:
                            s1.index, s1[:] = ap.linear_fit(ap.date2year(s1.index), s1.values)
                            s2.index, s2[:] = ap.linear_fit(ap.date2year(s2.index), s2.values)
                            offset = (s1.values[-1] - s1.values[0]) - (s2.values[-1] - s2.values[0])
                        else:
                            pass
                    else:
                        # using absolute values
                        s1 = ap.referenced(s1, to='first')
                        s2 = ap.referenced(s2, to='first')
                        s1[0], s2[0] = np.nan, np.nan # remove first values
                        offset = np.nanmean(s1 - s2)

                    #pd.concat((s1, s2), axis=1).plot(marker='o')
                else:
                    offset = np.nanmean(var['ers1'] - var['ers2'])
                offset_12[i,j] = offset
            else:
                no_overlap_12 += 1

            if np.sometrue(overlap_23):
                if SAT_BIAS:
                    s2 = var['ers2'][overlap_23]
                    s3 = var['envi'][overlap_23]
                    if LINEAR_FIT:
                        s2 = s2[s2.notnull() & s3.notnull()]
                        s3 = s3[s2.notnull() & s3.notnull()]
                        if len(s2) > 1 and len(s3) > 1:
                            s2.index, s2[:] = ap.linear_fit(ap.date2year(s2.index), s2.values)
                            s3.index, s3[:] = ap.linear_fit(ap.date2year(s3.index), s3.values)
                            offset = (s2.values[-1] - s2.values[0]) - (s3.values[-1] - s3.values[0])
                        else:
                            pass
                    else:
                        s2 = ap.referenced(s2, to='first')
                        s3 = ap.referenced(s3, to='first')
                        s2[0], s3[0] = np.nan, np.nan
                        offset = np.nanmean(s2 - s3)
                    #pd.concat((s2, s3), axis=1).plot(marker='o')
                    #plt.show()
                else:
                    offset = np.nanmean(var['ers2'] - var['envi'])
                offset_23[i,j] = offset 
            else:
                no_overlap_23 += 1

            #---------------------------------------------------

    mean_offset_12 = np.nanmean(offset_12)
    median_offset_12 = np.nanmedian(offset_12)
    mean_offset_23 = np.nanmean(offset_23)
    median_offset_23 = np.nanmedian(offset_23)

    if SAVE_TO_FILE:
        fout = tb.open_file(FNAME_OUT, 'w')
        fout.create_array('/', 'lon', lon)
        fout.create_array('/', 'lat', lat)
        fout.create_array('/', 'offset_12', offset_12)
        fout.create_array('/', 'offset_23', offset_23)
        fout.close()

    if PLOT:
        plt.figure()
        plt.subplot(211)
        offset_12 = ap.median_filt(offset_12, 3, 3)
        plt.imshow(offset_12, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5)
        plt.title('ERS1-ERS2')
        plt.colorbar(shrink=0.8)
        plt.subplot(212)
        offset_23 = ap.median_filt(offset_23, 3, 3)
        plt.imshow(offset_23, origin='lower', interpolation='nearest', vmin=-.5, vmax=.5)
        plt.title('ERS2-Envisat')
        #plt.colorbar(shrink=0.3, orientation='h')
        plt.colorbar(shrink=0.8)
        plt.figure()
        plt.subplot(121)
        o12 = offset_12[~np.isnan(offset_12)]
        plt.hist(o12, bins=100)
        plt.title('ERS1-ERS2')
        ax = plt.gca()
        ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_12, median_offset_12), 
                    ax=ax, loc=2)
        plt.xlim(-1, 1)
        plt.subplot(122)
        o23 = offset_23[~np.isnan(offset_23)]
        plt.hist(o23, bins=100)
        plt.title('ERS2-Envisat')
        ax = plt.gca()
        ap.intitle('mean/median = %.2f/%.2f m' % (mean_offset_23, median_offset_23),
                    ax=ax, loc=2)
        plt.xlim(-1, 1)
        plt.show()

    print 'calibrated variable:', VAR_TO_CALIBRATE
    print 'no overlaps:', no_overlap_12, no_overlap_23
    print 'mean offset:', mean_offset_12, mean_offset_23
    print 'median offset:', median_offset_12, median_offset_23
    print 'out file ->', FNAME_OUT