def consolidateNadir(svdat,autcln,outfile='SV_RESIDUALS.ND3') : ''' timestamp az zen lc(mm) prn Input: dphs a parsed dph structe obtained from resiudals.parseDPH(file) startDT a datetime object specify the start time of the first residual at epoch 1 Output: filename if it ends in gz it will be automatically compressed ''' lines = '' sep = ' ' dto = autcln['date'] with open(outfile,'a') as f: for i in range(0,32): prn = i+1 sv = svnav.findSV_DTO(svdat,prn,dto) ctr = 0 time = dto.strftime("%Y %j") res_line = '' for res in autcln['satNadir'][i]: if res < 99.9: res_line = res_line + ' ' + str(res) else : res_line = res_line + ' nan ' print(time,sv, res_line,file=f) return lines
def consolidateNadir(svdat, autcln, outfile='SV_RESIDUALS.ND3'): ''' timestamp az zen lc(mm) prn Input: dphs a parsed dph structe obtained from resiudals.parseDPH(file) startDT a datetime object specify the start time of the first residual at epoch 1 Output: filename if it ends in gz it will be automatically compressed ''' lines = '' sep = ' ' dto = autcln['date'] with open(outfile, 'a') as f: for i in range(0, 32): prn = i + 1 sv = svnav.findSV_DTO(svdat, prn, dto) ctr = 0 time = dto.strftime("%Y %j") res_line = '' for res in autcln['satNadir'][i]: if res < 99.9: res_line = res_line + ' ' + str(res) else: res_line = res_line + ' nan ' print(time, sv, res_line, file=f) return lines
def processAzSlice(model_residuals,svs,args,params,numDays,minVal_dt,az) : tSat = np.size(svs) * (int(14./args.nadir_grid) + 2) tSite = int(90./args.zen) + 1 numParams = tSat + tSite Neq = np.zeros((numParams,numParams)) AtWb = np.zeros(numParams) SiteFreq = np.zeros(tSite) # set up a lookup dictionary lookup_svs = {} lctr = 0 for sv in svs: lookup_svs[str(sv)] = lctr lctr+=1 site_geocentric_distance = np.linalg.norm(params['sitepos']) for d in range(0,numDays): minDTO = minVal_dt + dt.timedelta(days = d) maxDTO = minVal_dt + dt.timedelta(days = d+1) #print(d,"Stacking residuals on:",minDTO,maxDTO) criterion = ( ( model_residuals[:,0] >= calendar.timegm(minDTO.utctimetuple()) ) & ( model_residuals[:,0] < calendar.timegm(maxDTO.utctimetuple()) ) ) tind = np.array(np.where(criterion))[0] # if there are less than 300 obs, then skip to the next day if np.size(tind) < 300: continue #print("rejecting any residuals greater than 100mm",np.shape(site_residuals)) tdata = res.reject_absVal(model_residuals[tind,:],100.) #print("rejecting any residuals greater than 5 sigma",np.shape(tdata)) data = res.reject_outliers_elevation(tdata,5,0.5) #print("finished outlier detection",np.shape(data)) del tdata # determine the elevation dependent weighting a,b = res.gamitWeight(data) if(az - args.az/2. < 0) : criterion = (data[:,1] < (az + args.az/2.)) | (data[:,1] > (360. - args.az/2.) ) else: criterion = (data[:,1] < (az + args.az/2.)) & (data[:,1] > (az - args.az/2.) ) azind = np.array(np.where(criterion))[0] #print("Size of data before azimuth search",np.size(data)) data = data[azind,:] #print("Size of data after azimuth search",np.size(data)) # parse the broadcast navigation file for this day to get an accurate # nadir angle yy = minDTO.strftime("%y") doy = minDTO.strftime("%j") navfile = args.brdc_dir + 'brdc'+ doy +'0.'+ yy +'n' #print("Will read in the broadcast navigation file:",navfile) nav = rnxN.parseFile(navfile) # Get the total number of observations for this site numd = np.shape(data)[0] #print("Have:",numd,"observations") for i in range(0,numd): # work out the svn number svndto = gt.unix2dt(data[i,0]) svn = svnav.findSV_DTO(svdat,data[i,4],svndto) svn_search = 'G{:03d}'.format(svn) #print("Looking for:",svn_search,lookup_svs) ctr = lookup_svs[str(svn_search)] #print("Position is CTR:",ctr,data[i,4]) try: # get the satellite position svnpos = rnxN.satpos(data[i,4],svndto,nav) #print("SVNPOS:",svnpos[0]) satnorm = np.linalg.norm(svnpos[0]) #print("NORM:",np.linalg.norm(svnpos[0])) except: print("Error calculation satelite position for",svndto,data[i,:]) continue # work out the nadir angle nadir = NADIR.calcNadirAngle(data[i,2],site_geocentric_distance,satnorm) #print("Ele {:.2f} Old: {:.2f} New:{:.2f}".format(data[i,2],oldnadir,nadir)) #print("Ele {:.2f} New:{:.2f}".format(data[i,2],nadir)) w = a**2 + b**2/np.sin(np.radians(90.-data[i,2]))**2 w = 1./w # Work out the indices for the satellite parameters niz = int(np.floor(nadir/args.nadir_grid)) iz = int((numParamsPerSat * ctr) + niz) pco_iz = numParamsPerSat * (ctr+1) - 1 # work out the location of site parameters nsiz = int(np.floor(data[i,2]/args.zen)) #aiz = int(np.floor(data[i,1]/args.az)) #siz = int( tSat + (m*numParamsPerSite) + (aiz * nZen) + nsiz) siz = int( tSat + nsiz) # check that the indices are not overlapping if iz+1 >= pco_iz or iz >= pco_iz: #print("WARNING in indices iz+1 = pco_iz skipping obs",nadir,iz,pco_iz) continue #NadirFreq[ctr,niz] = NadirFreq[ctr,niz] +1 SiteFreq[nsiz] = SiteFreq[nsiz] +1 # # R = SITE_PCV_ERR + SAT_PCV_ERR + SAT_PCO_ERR * cos(nadir) # # dR/dSITE_PCV_ERR = 1 # dR/dSAT_PCV_ERR = 1 # dR/dSAT_PCO_ERR = cos(nadir) # # nice partial derivative tool: # http://www.symbolab.com/solver/partial-derivative-calculator # # Nadir partials.. Apart_1 = (1.-(nadir-niz*args.nadir_grid)/args.nadir_grid) Apart_2 = (nadir-niz*args.nadir_grid)/args.nadir_grid # # PCO partial ... Apart_3 = np.cos(np.radians(nadir)) # Site partials Apart_4 = (1.-(data[i,2]-nsiz*args.zen)/args.zen) Apart_5 = (data[i,2]-nsiz*args.zen)/args.zen #print("Finished forming Design matrix") #print("Starting AtWb",np.shape(AtWb),iz,pco_iz,siz) AtWb[iz] = AtWb[iz] + Apart_1 * data[i,3] * w AtWb[iz+1] = AtWb[iz+1] + Apart_2 * data[i,3] * w AtWb[pco_iz] = AtWb[pco_iz] + Apart_3 * data[i,3] * w AtWb[siz] = AtWb[siz] + Apart_4 * data[i,3] * w AtWb[siz+1] = AtWb[siz+1] + Apart_5 * data[i,3] * w #print("Finished forming b vector") Neq[iz,iz] = Neq[iz,iz] + (Apart_1 * Apart_1 * w) Neq[iz,iz+1] = Neq[iz,iz+1] + (Apart_1 * Apart_2 * w) Neq[iz,pco_iz] = Neq[iz,pco_iz] + (Apart_1 * Apart_3 * w) Neq[iz,siz] = Neq[iz,siz] + (Apart_1 * Apart_4 * w) Neq[iz,siz+1] = Neq[iz,siz+1] + (Apart_1 * Apart_5 * w) Neq[iz+1,iz] = Neq[iz+1,iz] + (Apart_2 * Apart_1 * w) Neq[iz+1,iz+1] = Neq[iz+1,iz+1] + (Apart_2 * Apart_2 * w) Neq[iz+1,pco_iz] = Neq[iz+1,pco_iz] + (Apart_2 * Apart_3 * w) Neq[iz+1,siz] = Neq[iz+1,siz] + (Apart_2 * Apart_4 * w) Neq[iz+1,siz+1] = Neq[iz+1,siz+1] + (Apart_2 * Apart_5 * w) #print("Finished NEQ Nadir estimates") Neq[pco_iz,iz] = Neq[pco_iz,iz] + (Apart_3 * Apart_1 * w) Neq[pco_iz,iz+1] = Neq[pco_iz,iz+1] + (Apart_3 * Apart_2 * w) Neq[pco_iz,pco_iz] = Neq[pco_iz,pco_iz] + (Apart_3 * Apart_3 * w) Neq[pco_iz,siz] = Neq[pco_iz,siz] + (Apart_3 * Apart_4 * w) Neq[pco_iz,siz+1] = Neq[pco_iz,siz+1] + (Apart_3 * Apart_5 * w) #print("Finished NEQ PCO estimates") Neq[siz,iz] = Neq[siz,iz] + (Apart_4 * Apart_1 * w) Neq[siz,iz+1] = Neq[siz,iz+1] + (Apart_4 * Apart_2 * w) Neq[siz,pco_iz] = Neq[siz,pco_iz] + (Apart_4 * Apart_3 * w) Neq[siz,siz] = Neq[siz,siz] + (Apart_4 * Apart_4 * w) Neq[siz,siz+1] = Neq[siz,siz+1] + (Apart_4 * Apart_5 * w) Neq[siz+1,iz] = Neq[siz+1,iz] + (Apart_5 * Apart_1 * w) Neq[siz+1,iz+1] = Neq[siz+1,iz+1] + (Apart_5 * Apart_2 * w) Neq[siz+1,pco_iz] = Neq[siz+1,pco_iz] + (Apart_5 * Apart_3 * w) Neq[siz+1,siz] = Neq[siz+1,siz] + (Apart_5 * Apart_4 * w) Neq[siz+1,siz+1] = Neq[siz+1,siz+1] + (Apart_5 * Apart_5 * w) #print("Finished NEQ Site estimates") if siz == pco_iz: print("ERROR in indices siz = pco_iz") # Add the parameter constraints to the Neq #Neq = np.add(Neq,C_inv) C_inv = formConstraints(args,tSat,tSite,1,numParams) Neq = np.add(Neq,C_inv) #print("Inverting") Cov = np.linalg.pinv(Neq) Sol = np.dot(Cov,AtWb) stdev = np.sqrt(np.diag(Cov)) return Sol, stdev, SiteFreq, az
def processAzSlice(model_residuals, svs, args, params, numDays, minVal_dt, az): tSat = np.size(svs) * (int(14. / args.nadir_grid) + 2) tSite = int(90. / args.zen) + 1 numParams = tSat + tSite Neq = np.zeros((numParams, numParams)) AtWb = np.zeros(numParams) SiteFreq = np.zeros(tSite) # set up a lookup dictionary lookup_svs = {} lctr = 0 for sv in svs: lookup_svs[str(sv)] = lctr lctr += 1 site_geocentric_distance = np.linalg.norm(params['sitepos']) for d in range(0, numDays): minDTO = minVal_dt + dt.timedelta(days=d) maxDTO = minVal_dt + dt.timedelta(days=d + 1) #print(d,"Stacking residuals on:",minDTO,maxDTO) criterion = ( (model_residuals[:, 0] >= calendar.timegm(minDTO.utctimetuple())) & (model_residuals[:, 0] < calendar.timegm(maxDTO.utctimetuple()))) tind = np.array(np.where(criterion))[0] # if there are less than 300 obs, then skip to the next day if np.size(tind) < 300: continue #print("rejecting any residuals greater than 100mm",np.shape(site_residuals)) tdata = res.reject_absVal(model_residuals[tind, :], 100.) #print("rejecting any residuals greater than 5 sigma",np.shape(tdata)) data = res.reject_outliers_elevation(tdata, 5, 0.5) #print("finished outlier detection",np.shape(data)) del tdata # determine the elevation dependent weighting a, b = res.gamitWeight(data) if (az - args.az / 2. < 0): criterion = (data[:, 1] < (az + args.az / 2.)) | (data[:, 1] > (360. - args.az / 2.)) else: criterion = (data[:, 1] < (az + args.az / 2.)) & (data[:, 1] > (az - args.az / 2.)) azind = np.array(np.where(criterion))[0] #print("Size of data before azimuth search",np.size(data)) data = data[azind, :] #print("Size of data after azimuth search",np.size(data)) # parse the broadcast navigation file for this day to get an accurate # nadir angle yy = minDTO.strftime("%y") doy = minDTO.strftime("%j") navfile = args.brdc_dir + 'brdc' + doy + '0.' + yy + 'n' #print("Will read in the broadcast navigation file:",navfile) nav = rnxN.parseFile(navfile) # Get the total number of observations for this site numd = np.shape(data)[0] #print("Have:",numd,"observations") for i in range(0, numd): # work out the svn number svndto = gt.unix2dt(data[i, 0]) svn = svnav.findSV_DTO(svdat, data[i, 4], svndto) svn_search = 'G{:03d}'.format(svn) #print("Looking for:",svn_search,lookup_svs) ctr = lookup_svs[str(svn_search)] #print("Position is CTR:",ctr,data[i,4]) try: # get the satellite position svnpos = rnxN.satpos(data[i, 4], svndto, nav) #print("SVNPOS:",svnpos[0]) satnorm = np.linalg.norm(svnpos[0]) #print("NORM:",np.linalg.norm(svnpos[0])) except: print("Error calculation satelite position for", svndto, data[i, :]) continue # work out the nadir angle nadir = NADIR.calcNadirAngle(data[i, 2], site_geocentric_distance, satnorm) #print("Ele {:.2f} Old: {:.2f} New:{:.2f}".format(data[i,2],oldnadir,nadir)) #print("Ele {:.2f} New:{:.2f}".format(data[i,2],nadir)) w = a**2 + b**2 / np.sin(np.radians(90. - data[i, 2]))**2 w = 1. / w # Work out the indices for the satellite parameters niz = int(np.floor(nadir / args.nadir_grid)) iz = int((numParamsPerSat * ctr) + niz) pco_iz = numParamsPerSat * (ctr + 1) - 1 # work out the location of site parameters nsiz = int(np.floor(data[i, 2] / args.zen)) #aiz = int(np.floor(data[i,1]/args.az)) #siz = int( tSat + (m*numParamsPerSite) + (aiz * nZen) + nsiz) siz = int(tSat + nsiz) # check that the indices are not overlapping if iz + 1 >= pco_iz or iz >= pco_iz: #print("WARNING in indices iz+1 = pco_iz skipping obs",nadir,iz,pco_iz) continue #NadirFreq[ctr,niz] = NadirFreq[ctr,niz] +1 SiteFreq[nsiz] = SiteFreq[nsiz] + 1 # # R = SITE_PCV_ERR + SAT_PCV_ERR + SAT_PCO_ERR * cos(nadir) # # dR/dSITE_PCV_ERR = 1 # dR/dSAT_PCV_ERR = 1 # dR/dSAT_PCO_ERR = cos(nadir) # # nice partial derivative tool: # http://www.symbolab.com/solver/partial-derivative-calculator # # Nadir partials.. Apart_1 = (1. - (nadir - niz * args.nadir_grid) / args.nadir_grid) Apart_2 = (nadir - niz * args.nadir_grid) / args.nadir_grid # # PCO partial ... Apart_3 = np.cos(np.radians(nadir)) # Site partials Apart_4 = (1. - (data[i, 2] - nsiz * args.zen) / args.zen) Apart_5 = (data[i, 2] - nsiz * args.zen) / args.zen #print("Finished forming Design matrix") #print("Starting AtWb",np.shape(AtWb),iz,pco_iz,siz) AtWb[iz] = AtWb[iz] + Apart_1 * data[i, 3] * w AtWb[iz + 1] = AtWb[iz + 1] + Apart_2 * data[i, 3] * w AtWb[pco_iz] = AtWb[pco_iz] + Apart_3 * data[i, 3] * w AtWb[siz] = AtWb[siz] + Apart_4 * data[i, 3] * w AtWb[siz + 1] = AtWb[siz + 1] + Apart_5 * data[i, 3] * w #print("Finished forming b vector") Neq[iz, iz] = Neq[iz, iz] + (Apart_1 * Apart_1 * w) Neq[iz, iz + 1] = Neq[iz, iz + 1] + (Apart_1 * Apart_2 * w) Neq[iz, pco_iz] = Neq[iz, pco_iz] + (Apart_1 * Apart_3 * w) Neq[iz, siz] = Neq[iz, siz] + (Apart_1 * Apart_4 * w) Neq[iz, siz + 1] = Neq[iz, siz + 1] + (Apart_1 * Apart_5 * w) Neq[iz + 1, iz] = Neq[iz + 1, iz] + (Apart_2 * Apart_1 * w) Neq[iz + 1, iz + 1] = Neq[iz + 1, iz + 1] + (Apart_2 * Apart_2 * w) Neq[iz + 1, pco_iz] = Neq[iz + 1, pco_iz] + (Apart_2 * Apart_3 * w) Neq[iz + 1, siz] = Neq[iz + 1, siz] + (Apart_2 * Apart_4 * w) Neq[iz + 1, siz + 1] = Neq[iz + 1, siz + 1] + (Apart_2 * Apart_5 * w) #print("Finished NEQ Nadir estimates") Neq[pco_iz, iz] = Neq[pco_iz, iz] + (Apart_3 * Apart_1 * w) Neq[pco_iz, iz + 1] = Neq[pco_iz, iz + 1] + (Apart_3 * Apart_2 * w) Neq[pco_iz, pco_iz] = Neq[pco_iz, pco_iz] + (Apart_3 * Apart_3 * w) Neq[pco_iz, siz] = Neq[pco_iz, siz] + (Apart_3 * Apart_4 * w) Neq[pco_iz, siz + 1] = Neq[pco_iz, siz + 1] + (Apart_3 * Apart_5 * w) #print("Finished NEQ PCO estimates") Neq[siz, iz] = Neq[siz, iz] + (Apart_4 * Apart_1 * w) Neq[siz, iz + 1] = Neq[siz, iz + 1] + (Apart_4 * Apart_2 * w) Neq[siz, pco_iz] = Neq[siz, pco_iz] + (Apart_4 * Apart_3 * w) Neq[siz, siz] = Neq[siz, siz] + (Apart_4 * Apart_4 * w) Neq[siz, siz + 1] = Neq[siz, siz + 1] + (Apart_4 * Apart_5 * w) Neq[siz + 1, iz] = Neq[siz + 1, iz] + (Apart_5 * Apart_1 * w) Neq[siz + 1, iz + 1] = Neq[siz + 1, iz + 1] + (Apart_5 * Apart_2 * w) Neq[siz + 1, pco_iz] = Neq[siz + 1, pco_iz] + (Apart_5 * Apart_3 * w) Neq[siz + 1, siz] = Neq[siz + 1, siz] + (Apart_5 * Apart_4 * w) Neq[siz + 1, siz + 1] = Neq[siz + 1, siz + 1] + (Apart_5 * Apart_5 * w) #print("Finished NEQ Site estimates") if siz == pco_iz: print("ERROR in indices siz = pco_iz") # Add the parameter constraints to the Neq #Neq = np.add(Neq,C_inv) C_inv = formConstraints(args, tSat, tSite, 1, numParams) Neq = np.add(Neq, C_inv) #print("Inverting") Cov = np.linalg.pinv(Neq) Sol = np.dot(Cov, AtWb) stdev = np.sqrt(np.diag(Cov)) return Sol, stdev, SiteFreq, az