def consolidateNadir(svdat,autcln,outfile='SV_RESIDUALS.ND3') :
'''
timestamp az zen lc(mm) prn
Input:
dphs a parsed dph structe obtained from resiudals.parseDPH(file)
startDT a datetime object specify the start time of the first residual at epoch 1
Output:
filename if it ends in gz it will be automatically compressed
'''
lines = ''
sep = ' '
dto = autcln['date']
with open(outfile,'a') as f:
for i in range(0,32):
prn = i+1
sv = svnav.findSV_DTO(svdat,prn,dto)
ctr = 0
time = dto.strftime("%Y %j")
res_line = ''
for res in autcln['satNadir'][i]:
if res < 99.9:
res_line = res_line + ' ' + str(res)
else :
res_line = res_line + ' nan '
print(time,sv, res_line,file=f)
return lines
def consolidateNadir(svdat, autcln, outfile='SV_RESIDUALS.ND3'):
'''
timestamp az zen lc(mm) prn
Input:
dphs a parsed dph structe obtained from resiudals.parseDPH(file)
startDT a datetime object specify the start time of the first residual at epoch 1
Output:
filename if it ends in gz it will be automatically compressed
'''
lines = ''
sep = ' '
dto = autcln['date']
with open(outfile, 'a') as f:
for i in range(0, 32):
prn = i + 1
sv = svnav.findSV_DTO(svdat, prn, dto)
ctr = 0
time = dto.strftime("%Y %j")
res_line = ''
for res in autcln['satNadir'][i]:
if res < 99.9:
res_line = res_line + ' ' + str(res)
else:
res_line = res_line + ' nan '
print(time, sv, res_line, file=f)
return lines
def processAzSlice(model_residuals,svs,args,params,numDays,minVal_dt,az) :
tSat = np.size(svs) * (int(14./args.nadir_grid) + 2)
tSite = int(90./args.zen) + 1
numParams = tSat + tSite
Neq = np.zeros((numParams,numParams))
AtWb = np.zeros(numParams)
SiteFreq = np.zeros(tSite)
# set up a lookup dictionary
lookup_svs = {}
lctr = 0
for sv in svs:
lookup_svs[str(sv)] = lctr
lctr+=1
site_geocentric_distance = np.linalg.norm(params['sitepos'])
for d in range(0,numDays):
minDTO = minVal_dt + dt.timedelta(days = d)
maxDTO = minVal_dt + dt.timedelta(days = d+1)
#print(d,"Stacking residuals on:",minDTO,maxDTO)
criterion = ( ( model_residuals[:,0] >= calendar.timegm(minDTO.utctimetuple()) ) &
( model_residuals[:,0] < calendar.timegm(maxDTO.utctimetuple()) ) )
tind = np.array(np.where(criterion))[0]
# if there are less than 300 obs, then skip to the next day
if np.size(tind) < 300:
continue
#print("rejecting any residuals greater than 100mm",np.shape(site_residuals))
tdata = res.reject_absVal(model_residuals[tind,:],100.)
#print("rejecting any residuals greater than 5 sigma",np.shape(tdata))
data = res.reject_outliers_elevation(tdata,5,0.5)
#print("finished outlier detection",np.shape(data))
del tdata
# determine the elevation dependent weighting
a,b = res.gamitWeight(data)
if(az - args.az/2. < 0) :
criterion = (data[:,1] < (az + args.az/2.)) | (data[:,1] > (360. - args.az/2.) )
else:
criterion = (data[:,1] < (az + args.az/2.)) & (data[:,1] > (az - args.az/2.) )
azind = np.array(np.where(criterion))[0]
#print("Size of data before azimuth search",np.size(data))
data = data[azind,:]
#print("Size of data after azimuth search",np.size(data))
# parse the broadcast navigation file for this day to get an accurate
# nadir angle
yy = minDTO.strftime("%y")
doy = minDTO.strftime("%j")
navfile = args.brdc_dir + 'brdc'+ doy +'0.'+ yy +'n'
#print("Will read in the broadcast navigation file:",navfile)
nav = rnxN.parseFile(navfile)
# Get the total number of observations for this site
numd = np.shape(data)[0]
#print("Have:",numd,"observations")
for i in range(0,numd):
# work out the svn number
svndto = gt.unix2dt(data[i,0])
svn = svnav.findSV_DTO(svdat,data[i,4],svndto)
svn_search = 'G{:03d}'.format(svn)
#print("Looking for:",svn_search,lookup_svs)
ctr = lookup_svs[str(svn_search)]
#print("Position is CTR:",ctr,data[i,4])
try:
# get the satellite position
svnpos = rnxN.satpos(data[i,4],svndto,nav)
#print("SVNPOS:",svnpos[0])
satnorm = np.linalg.norm(svnpos[0])
#print("NORM:",np.linalg.norm(svnpos[0]))
except:
print("Error calculation satelite position for",svndto,data[i,:])
continue
# work out the nadir angle
nadir = NADIR.calcNadirAngle(data[i,2],site_geocentric_distance,satnorm)
#print("Ele {:.2f} Old: {:.2f} New:{:.2f}".format(data[i,2],oldnadir,nadir))
#print("Ele {:.2f} New:{:.2f}".format(data[i,2],nadir))
w = a**2 + b**2/np.sin(np.radians(90.-data[i,2]))**2
w = 1./w
# Work out the indices for the satellite parameters
niz = int(np.floor(nadir/args.nadir_grid))
iz = int((numParamsPerSat * ctr) + niz)
pco_iz = numParamsPerSat * (ctr+1) - 1
# work out the location of site parameters
nsiz = int(np.floor(data[i,2]/args.zen))
#aiz = int(np.floor(data[i,1]/args.az))
#siz = int( tSat + (m*numParamsPerSite) + (aiz * nZen) + nsiz)
siz = int( tSat + nsiz)
# check that the indices are not overlapping
if iz+1 >= pco_iz or iz >= pco_iz:
#print("WARNING in indices iz+1 = pco_iz skipping obs",nadir,iz,pco_iz)
continue
#NadirFreq[ctr,niz] = NadirFreq[ctr,niz] +1
SiteFreq[nsiz] = SiteFreq[nsiz] +1
#
# R = SITE_PCV_ERR + SAT_PCV_ERR + SAT_PCO_ERR * cos(nadir)
#
# dR/dSITE_PCV_ERR = 1
# dR/dSAT_PCV_ERR = 1
# dR/dSAT_PCO_ERR = cos(nadir)
#
# nice partial derivative tool:
# http://www.symbolab.com/solver/partial-derivative-calculator
#
# Nadir partials..
Apart_1 = (1.-(nadir-niz*args.nadir_grid)/args.nadir_grid)
Apart_2 = (nadir-niz*args.nadir_grid)/args.nadir_grid
#
# PCO partial ...
Apart_3 = np.cos(np.radians(nadir))
# Site partials
Apart_4 = (1.-(data[i,2]-nsiz*args.zen)/args.zen)
Apart_5 = (data[i,2]-nsiz*args.zen)/args.zen
#print("Finished forming Design matrix")
#print("Starting AtWb",np.shape(AtWb),iz,pco_iz,siz)
AtWb[iz] = AtWb[iz] + Apart_1 * data[i,3] * w
AtWb[iz+1] = AtWb[iz+1] + Apart_2 * data[i,3] * w
AtWb[pco_iz] = AtWb[pco_iz] + Apart_3 * data[i,3] * w
AtWb[siz] = AtWb[siz] + Apart_4 * data[i,3] * w
AtWb[siz+1] = AtWb[siz+1] + Apart_5 * data[i,3] * w
#print("Finished forming b vector")
Neq[iz,iz] = Neq[iz,iz] + (Apart_1 * Apart_1 * w)
Neq[iz,iz+1] = Neq[iz,iz+1] + (Apart_1 * Apart_2 * w)
Neq[iz,pco_iz] = Neq[iz,pco_iz] + (Apart_1 * Apart_3 * w)
Neq[iz,siz] = Neq[iz,siz] + (Apart_1 * Apart_4 * w)
Neq[iz,siz+1] = Neq[iz,siz+1] + (Apart_1 * Apart_5 * w)
Neq[iz+1,iz] = Neq[iz+1,iz] + (Apart_2 * Apart_1 * w)
Neq[iz+1,iz+1] = Neq[iz+1,iz+1] + (Apart_2 * Apart_2 * w)
Neq[iz+1,pco_iz] = Neq[iz+1,pco_iz] + (Apart_2 * Apart_3 * w)
Neq[iz+1,siz] = Neq[iz+1,siz] + (Apart_2 * Apart_4 * w)
Neq[iz+1,siz+1] = Neq[iz+1,siz+1] + (Apart_2 * Apart_5 * w)
#print("Finished NEQ Nadir estimates")
Neq[pco_iz,iz] = Neq[pco_iz,iz] + (Apart_3 * Apart_1 * w)
Neq[pco_iz,iz+1] = Neq[pco_iz,iz+1] + (Apart_3 * Apart_2 * w)
Neq[pco_iz,pco_iz] = Neq[pco_iz,pco_iz] + (Apart_3 * Apart_3 * w)
Neq[pco_iz,siz] = Neq[pco_iz,siz] + (Apart_3 * Apart_4 * w)
Neq[pco_iz,siz+1] = Neq[pco_iz,siz+1] + (Apart_3 * Apart_5 * w)
#print("Finished NEQ PCO estimates")
Neq[siz,iz] = Neq[siz,iz] + (Apart_4 * Apart_1 * w)
Neq[siz,iz+1] = Neq[siz,iz+1] + (Apart_4 * Apart_2 * w)
Neq[siz,pco_iz] = Neq[siz,pco_iz] + (Apart_4 * Apart_3 * w)
Neq[siz,siz] = Neq[siz,siz] + (Apart_4 * Apart_4 * w)
Neq[siz,siz+1] = Neq[siz,siz+1] + (Apart_4 * Apart_5 * w)
Neq[siz+1,iz] = Neq[siz+1,iz] + (Apart_5 * Apart_1 * w)
Neq[siz+1,iz+1] = Neq[siz+1,iz+1] + (Apart_5 * Apart_2 * w)
Neq[siz+1,pco_iz] = Neq[siz+1,pco_iz] + (Apart_5 * Apart_3 * w)
Neq[siz+1,siz] = Neq[siz+1,siz] + (Apart_5 * Apart_4 * w)
Neq[siz+1,siz+1] = Neq[siz+1,siz+1] + (Apart_5 * Apart_5 * w)
#print("Finished NEQ Site estimates")
if siz == pco_iz:
print("ERROR in indices siz = pco_iz")
# Add the parameter constraints to the Neq
#Neq = np.add(Neq,C_inv)
C_inv = formConstraints(args,tSat,tSite,1,numParams)
Neq = np.add(Neq,C_inv)
#print("Inverting")
Cov = np.linalg.pinv(Neq)
Sol = np.dot(Cov,AtWb)
stdev = np.sqrt(np.diag(Cov))
return Sol, stdev, SiteFreq, az
def processAzSlice(model_residuals, svs, args, params, numDays, minVal_dt, az):
tSat = np.size(svs) * (int(14. / args.nadir_grid) + 2)
tSite = int(90. / args.zen) + 1
numParams = tSat + tSite
Neq = np.zeros((numParams, numParams))
AtWb = np.zeros(numParams)
SiteFreq = np.zeros(tSite)
# set up a lookup dictionary
lookup_svs = {}
lctr = 0
for sv in svs:
lookup_svs[str(sv)] = lctr
lctr += 1
site_geocentric_distance = np.linalg.norm(params['sitepos'])
for d in range(0, numDays):
minDTO = minVal_dt + dt.timedelta(days=d)
maxDTO = minVal_dt + dt.timedelta(days=d + 1)
#print(d,"Stacking residuals on:",minDTO,maxDTO)
criterion = (
(model_residuals[:, 0] >= calendar.timegm(minDTO.utctimetuple())) &
(model_residuals[:, 0] < calendar.timegm(maxDTO.utctimetuple())))
tind = np.array(np.where(criterion))[0]
# if there are less than 300 obs, then skip to the next day
if np.size(tind) < 300:
continue
#print("rejecting any residuals greater than 100mm",np.shape(site_residuals))
tdata = res.reject_absVal(model_residuals[tind, :], 100.)
#print("rejecting any residuals greater than 5 sigma",np.shape(tdata))
data = res.reject_outliers_elevation(tdata, 5, 0.5)
#print("finished outlier detection",np.shape(data))
del tdata
# determine the elevation dependent weighting
a, b = res.gamitWeight(data)
if (az - args.az / 2. < 0):
criterion = (data[:, 1] <
(az + args.az / 2.)) | (data[:, 1] >
(360. - args.az / 2.))
else:
criterion = (data[:, 1] <
(az + args.az / 2.)) & (data[:, 1] >
(az - args.az / 2.))
azind = np.array(np.where(criterion))[0]
#print("Size of data before azimuth search",np.size(data))
data = data[azind, :]
#print("Size of data after azimuth search",np.size(data))
# parse the broadcast navigation file for this day to get an accurate
# nadir angle
yy = minDTO.strftime("%y")
doy = minDTO.strftime("%j")
navfile = args.brdc_dir + 'brdc' + doy + '0.' + yy + 'n'
#print("Will read in the broadcast navigation file:",navfile)
nav = rnxN.parseFile(navfile)
# Get the total number of observations for this site
numd = np.shape(data)[0]
#print("Have:",numd,"observations")
for i in range(0, numd):
# work out the svn number
svndto = gt.unix2dt(data[i, 0])
svn = svnav.findSV_DTO(svdat, data[i, 4], svndto)
svn_search = 'G{:03d}'.format(svn)
#print("Looking for:",svn_search,lookup_svs)
ctr = lookup_svs[str(svn_search)]
#print("Position is CTR:",ctr,data[i,4])
try:
# get the satellite position
svnpos = rnxN.satpos(data[i, 4], svndto, nav)
#print("SVNPOS:",svnpos[0])
satnorm = np.linalg.norm(svnpos[0])
#print("NORM:",np.linalg.norm(svnpos[0]))
except:
print("Error calculation satelite position for", svndto,
data[i, :])
continue
# work out the nadir angle
nadir = NADIR.calcNadirAngle(data[i, 2], site_geocentric_distance,
satnorm)
#print("Ele {:.2f} Old: {:.2f} New:{:.2f}".format(data[i,2],oldnadir,nadir))
#print("Ele {:.2f} New:{:.2f}".format(data[i,2],nadir))
w = a**2 + b**2 / np.sin(np.radians(90. - data[i, 2]))**2
w = 1. / w
# Work out the indices for the satellite parameters
niz = int(np.floor(nadir / args.nadir_grid))
iz = int((numParamsPerSat * ctr) + niz)
pco_iz = numParamsPerSat * (ctr + 1) - 1
# work out the location of site parameters
nsiz = int(np.floor(data[i, 2] / args.zen))
#aiz = int(np.floor(data[i,1]/args.az))
#siz = int( tSat + (m*numParamsPerSite) + (aiz * nZen) + nsiz)
siz = int(tSat + nsiz)
# check that the indices are not overlapping
if iz + 1 >= pco_iz or iz >= pco_iz:
#print("WARNING in indices iz+1 = pco_iz skipping obs",nadir,iz,pco_iz)
continue
#NadirFreq[ctr,niz] = NadirFreq[ctr,niz] +1
SiteFreq[nsiz] = SiteFreq[nsiz] + 1
#
# R = SITE_PCV_ERR + SAT_PCV_ERR + SAT_PCO_ERR * cos(nadir)
#
# dR/dSITE_PCV_ERR = 1
# dR/dSAT_PCV_ERR = 1
# dR/dSAT_PCO_ERR = cos(nadir)
#
# nice partial derivative tool:
# http://www.symbolab.com/solver/partial-derivative-calculator
#
# Nadir partials..
Apart_1 = (1. - (nadir - niz * args.nadir_grid) / args.nadir_grid)
Apart_2 = (nadir - niz * args.nadir_grid) / args.nadir_grid
#
# PCO partial ...
Apart_3 = np.cos(np.radians(nadir))
# Site partials
Apart_4 = (1. - (data[i, 2] - nsiz * args.zen) / args.zen)
Apart_5 = (data[i, 2] - nsiz * args.zen) / args.zen
#print("Finished forming Design matrix")
#print("Starting AtWb",np.shape(AtWb),iz,pco_iz,siz)
AtWb[iz] = AtWb[iz] + Apart_1 * data[i, 3] * w
AtWb[iz + 1] = AtWb[iz + 1] + Apart_2 * data[i, 3] * w
AtWb[pco_iz] = AtWb[pco_iz] + Apart_3 * data[i, 3] * w
AtWb[siz] = AtWb[siz] + Apart_4 * data[i, 3] * w
AtWb[siz + 1] = AtWb[siz + 1] + Apart_5 * data[i, 3] * w
#print("Finished forming b vector")
Neq[iz, iz] = Neq[iz, iz] + (Apart_1 * Apart_1 * w)
Neq[iz, iz + 1] = Neq[iz, iz + 1] + (Apart_1 * Apart_2 * w)
Neq[iz, pco_iz] = Neq[iz, pco_iz] + (Apart_1 * Apart_3 * w)
Neq[iz, siz] = Neq[iz, siz] + (Apart_1 * Apart_4 * w)
Neq[iz, siz + 1] = Neq[iz, siz + 1] + (Apart_1 * Apart_5 * w)
Neq[iz + 1, iz] = Neq[iz + 1, iz] + (Apart_2 * Apart_1 * w)
Neq[iz + 1, iz + 1] = Neq[iz + 1, iz + 1] + (Apart_2 * Apart_2 * w)
Neq[iz + 1, pco_iz] = Neq[iz + 1, pco_iz] + (Apart_2 * Apart_3 * w)
Neq[iz + 1, siz] = Neq[iz + 1, siz] + (Apart_2 * Apart_4 * w)
Neq[iz + 1,
siz + 1] = Neq[iz + 1, siz + 1] + (Apart_2 * Apart_5 * w)
#print("Finished NEQ Nadir estimates")
Neq[pco_iz, iz] = Neq[pco_iz, iz] + (Apart_3 * Apart_1 * w)
Neq[pco_iz, iz + 1] = Neq[pco_iz, iz + 1] + (Apart_3 * Apart_2 * w)
Neq[pco_iz, pco_iz] = Neq[pco_iz, pco_iz] + (Apart_3 * Apart_3 * w)
Neq[pco_iz, siz] = Neq[pco_iz, siz] + (Apart_3 * Apart_4 * w)
Neq[pco_iz,
siz + 1] = Neq[pco_iz, siz + 1] + (Apart_3 * Apart_5 * w)
#print("Finished NEQ PCO estimates")
Neq[siz, iz] = Neq[siz, iz] + (Apart_4 * Apart_1 * w)
Neq[siz, iz + 1] = Neq[siz, iz + 1] + (Apart_4 * Apart_2 * w)
Neq[siz, pco_iz] = Neq[siz, pco_iz] + (Apart_4 * Apart_3 * w)
Neq[siz, siz] = Neq[siz, siz] + (Apart_4 * Apart_4 * w)
Neq[siz, siz + 1] = Neq[siz, siz + 1] + (Apart_4 * Apart_5 * w)
Neq[siz + 1, iz] = Neq[siz + 1, iz] + (Apart_5 * Apart_1 * w)
Neq[siz + 1,
iz + 1] = Neq[siz + 1, iz + 1] + (Apart_5 * Apart_2 * w)
Neq[siz + 1,
pco_iz] = Neq[siz + 1, pco_iz] + (Apart_5 * Apart_3 * w)
Neq[siz + 1, siz] = Neq[siz + 1, siz] + (Apart_5 * Apart_4 * w)
Neq[siz + 1,
siz + 1] = Neq[siz + 1, siz + 1] + (Apart_5 * Apart_5 * w)
#print("Finished NEQ Site estimates")
if siz == pco_iz:
print("ERROR in indices siz = pco_iz")
# Add the parameter constraints to the Neq
#Neq = np.add(Neq,C_inv)
C_inv = formConstraints(args, tSat, tSite, 1, numParams)
Neq = np.add(Neq, C_inv)
#print("Inverting")
Cov = np.linalg.pinv(Neq)
Sol = np.dot(Cov, AtWb)
stdev = np.sqrt(np.diag(Cov))
return Sol, stdev, SiteFreq, az
