def setUpCalcSiteModelPostFit(models,site_residuals, info, zen_grid, az_grid, cpus) :
nZen = int(90.0/zen_grid) + 1
nAz = int(360./az_grid)
print("nAz",nAz)
#numParamsPerSite = nZen * nAz
nModels = info['numModels']
#tSite = numParamsPerSite*params['numModels']
#numParams = tSat + tSite
prefit = np.zeros(nModels)
prefit_sums = np.zeros((nModels,nAz,nZen))
prefit_res = np.zeros((nModels,nAz,nZen))
postfit = np.zeros(nModels)
postfit_sums = np.zeros((nModels,nAz,nZen))
postfit_res = np.zeros((nModels,nAz,nZen))
numObs = np.zeros(nModels)
numObs_sums = np.zeros((nModels,nAz,nZen))
change = info['changes']
print("Changes for site",info['site'],change)
# keep track of how may observations are in each bin
#SiteFreq = np.zeros((nModels,nAz,nZen))
#Models = np.zeros((nModels,nAz,nZen))
#model_stdev = np.zeros((int(params['numModels']),nAz,nZen))
# create a new model everythime there has been a change of antenna
for m in range(0,nModels):
print(info['site'],"----> creating model",m+1,"of",info['numModels'])
# start_yyyy and start_ddd should always be defind, however stop_dd may be absent
#ie no changes have ocured since the last setup
minVal_dt = gt.ydhms2dt(change['start_yyyy'][m],change['start_ddd'][m],0,0,0)
if np.size(change['stop_ddd']) > m :
maxVal_dt = gt.ydhms2dt(change['stop_yyyy'][m],change['stop_ddd'][m],23,59,59)
print("Min:",minVal_dt,"Max:",maxVal_dt,m,np.size(change['stop_ddd']))
criterion = ( ( site_residuals[:,0] >= calendar.timegm(minVal_dt.utctimetuple()) ) &
( site_residuals[:,0] < calendar.timegm(maxVal_dt.utctimetuple()) ) )
else:
criterion = ( site_residuals[:,0] >= calendar.timegm(minVal_dt.utctimetuple()) )
maxVal_dt = gt.unix2dt(site_residuals[-1,0])
# get the residuals for this model time period
mind = np.array(np.where(criterion))[0]
model_residuals = site_residuals[mind,:]
diff_dt = maxVal_dt - minVal_dt
numDays = diff_dt.days + 1
print("Have a total of",numDays,"days",np.shape(models))
prefit[m], prefit_sums[m,:,:], prefit_res[m,:,:], postfit[m], postfit_sums[m,:,:], postfit_res[m,:,:],numObs[m],numObs_sums[m,:,:] = setUpPostFitTasks(models,model_residuals,args.cpu,zen_grid,az_grid,params,numDays,minVal_dt)
print("FINISHED Post fit RUN for model",m)
return prefit, prefit_sums, prefit_res, postfit, postfit_sums, postfit_res,numObs,numObs_sums
def parseConsolidatedNumpy(cfile):
'''
parseConsolidated Read in a consolidate phase residual file that contains all of the epochs
for a particular site
Usage: residuals = parseConsolidated('TOW2.2012.DPH.gz')
Input: file - TOW2.2012.DPH.gz, can take gzipp'd or plane txt files
Output: residuals - an array of dictionaries
'''
nlines = file_len(cfile)
residuals = np.zeros((nlines,5))
# work out if the file is compressed or not,
# and then get the correct file opener.
file_open = file_opener(cfile)
ctr = 0
with file_open(cfile) as f:
for line in f:
tmp = {}
yyyy, ddd, ts, az, zen, lc, prn = line.split( )
hh,mm,ss = ts.split(':')
dto = gt.ydhms2dt(yyyy,ddd,hh,mm,ss)
if float(lc) > 1000:
next
else:
residuals[ctr,0] = calendar.timegm(dto.utctimetuple())
residuals[ctr,1] = float(az)
residuals[ctr,2] = float(zen)
residuals[ctr,3] = float(lc)
residuals[ctr,4] = int(prn)
ctr += 1
# check to see if we are tie filtering the residuals
#if dt_start > 0.0001 :
# criterion = ( ( residuals[:,0] >= calendar.timegm(dt_start.utctimetuple()) ) &
# ( residuals[:,0] < calendar.timegm(dt_stop.utctimetuple()) ) )
# tind = np.array(np.where(criterion))[0]
# print("going from:",nlines,"to:",np.size(tind))
# res = np.zeros((np.size(tind,5)))
# res = residuals[tind,:]
#else:
#print("no time filtering")
res = np.zeros((ctr,5))
res = residuals[0:ctr,:]
return res
def parseConsolidatedNumpy(cfile):
'''
parseConsolidated Read in a consolidate phase residual file that contains all of the epochs
for a particular site
Usage: residuals = parseConsolidated('TOW2.2012.DPH.gz')
Input: file - TOW2.2012.DPH.gz, can take gzipp'd or plane txt files
Output: residuals - an array of dictionaries
'''
nlines = file_len(cfile)
residuals = np.zeros((nlines, 5))
# work out if the file is compressed or not,
# and then get the correct file opener.
file_open = file_opener(cfile)
ctr = 0
with file_open(cfile) as f:
for line in f:
tmp = {}
yyyy, ddd, ts, az, zen, lc, prn = line.split()
hh, mm, ss = ts.split(':')
dto = gt.ydhms2dt(yyyy, ddd, hh, mm, ss)
if float(lc) > 1000:
next
else:
residuals[ctr, 0] = calendar.timegm(dto.utctimetuple())
residuals[ctr, 1] = float(az)
residuals[ctr, 2] = float(zen)
residuals[ctr, 3] = float(lc)
residuals[ctr, 4] = int(prn)
ctr += 1
# check to see if we are tie filtering the residuals
#if dt_start > 0.0001 :
# criterion = ( ( residuals[:,0] >= calendar.timegm(dt_start.utctimetuple()) ) &
# ( residuals[:,0] < calendar.timegm(dt_stop.utctimetuple()) ) )
# tind = np.array(np.where(criterion))[0]
# print("going from:",nlines,"to:",np.size(tind))
# res = np.zeros((np.size(tind,5)))
# res = residuals[tind,:]
#else:
#print("no time filtering")
res = np.zeros((ctr, 5))
res = residuals[0:ctr, :]
return res
def solveSiteModel(site_residuals, svs, params, apr, nadSpacing=0.1, zenSpacing=0.5, azSpacing=0.5, brdc_dir="./"):
"""
Create a model for the satellites and sites at the same time.
PWL piece-wise-linear interpolation fit of phase residuals
-construct a PWL fit for each azimuth bin, and then paste them all together to get
the full model
-inversion is done within each bin
site_residuals = the one-way L3 post-fit, ambiguity fixed phase residuals
svs = an array of satellite SVN numbers that are spacebourne/operating
for the period of this residual stack
params = meta data about the solution bein attempted
['site'] = 4 char site id
['changes'] = dictionary of when model changes need to be applied
apr = satellite apriori data
"""
#prechi = 0
#NUMD = 0
# add one to make sure we have a linspace which includes 0.0 and 14.0
# add another parameter for the zenith PCO estimate
numNADS = int(14.0/nadSpacing) + 1
PCOEstimates = 1
numSVS = np.size(svs)
numParamsPerSat = numNADS + PCOEstimates
tSat = numParamsPerSat * numSVS
nZen = int(90.0/zenSpacing) + 1
nAz = int(360./azSpacing)
print("nAz",nAz)
numParamsPerSite = nZen * nAz
tSite = numParamsPerSite*params['numModels']
numParams = tSat + tSite
print("------------------------------------------------")
print("Processing Site: ",params['site'])
print("------------------------------------------------")
print("Sat Params:----------------",numParamsPerSat)
print("Number of Sats:------------",np.size(svs))
print("Total satellite parameters:-------------",tSat)
print("Site Params:---------------",numParamsPerSite)
print("Number of Models:----------",params['numModels'])
print("Total Site Params:----------------------",tSite)
print("------------------------------------------------")
print("Total Params:---------------------------",numParams)
print("------------------------------------------------")
# Creating matrices
#Neq = np.zeros((numParams,numParams))
#AtWb = np.zeros(numParams)
change = params['changes']
print("Changes for site",params['site'],change)
# keep track of how may observations are in each bin
#NadirFreq = np.zeros((numSVS,numNADS))
SiteFreq = np.zeros((int(params['numModels']),nAz,nZen))
Models = np.zeros((int(params['numModels']),nAz,nZen))
model_stdev = np.zeros((int(params['numModels']),nAz,nZen))
# create a new model everythime there has been a change of antenna
for m in range(0,int(params['numModels'])):
print(params['site'],"----> creating model",m+1,"of",params['numModels'])
# start_yyyy and start_ddd should always be defind, however stop_dd may be absent
#ie no changes have ocured since the last setup
minVal_dt = gt.ydhms2dt(change['start_yyyy'][m],change['start_ddd'][m],0,0,0)
if np.size(change['stop_ddd']) > m :
maxVal_dt = gt.ydhms2dt(change['stop_yyyy'][m],change['stop_ddd'][m],23,59,59)
print("Min:",minVal_dt,"Max:",maxVal_dt,m,np.size(change['stop_ddd']))
criterion = ( ( site_residuals[:,0] >= calendar.timegm(minVal_dt.utctimetuple()) ) &
( site_residuals[:,0] < calendar.timegm(maxVal_dt.utctimetuple()) ) )
else:
criterion = ( site_residuals[:,0] >= calendar.timegm(minVal_dt.utctimetuple()) )
maxVal_dt = gt.unix2dt(site_residuals[-1,0])
# get the residuals for this model time period
mind = np.array(np.where(criterion))[0]
model_residuals = site_residuals[mind,:]
diff_dt = maxVal_dt - minVal_dt
numDays = diff_dt.days + 1
print("Have a total of",numDays,"days")
Models[m,:,:], model_stdev[m,:,:],SiteFreq[m,:,:] = setUpAzTasks(model_residuals,svs,args,params,numDays,minVal_dt,nAz)
print("FINISHED AZ RUN for model",m)
print("Normal finish of pwl")
return Models, model_stdev, SiteFreq
type=int,
help='Day-of-Year of first epoch in DDD format')
parser.add_argument('--end_yyyy',
dest='end_yyyy',
type=int,
help='Year of last epoch in YYYY format')
parser.add_argument('--end_ddd',
dest='end_ddd',
type=int,
help='Day-of-Year of last epoch in DDD format')
args = parser.parse_args()
#=========================================
# find the station meta data for a particular site
station_metadata = parseSite(args.station_file, args.site)
#print("data",data)
#ind = antennaChange(data)
#for i in ind:
# print(data['start_yyyy'][i],data['start_ddd'][i],data['antenna_type'][i],data['dome_type'][i])
#print("ind:",ind)
if args.esm_report:
start = gt.ydhms2dt(args.start_yyyy, args.start_ddd, 0, 0, 0)
end = gt.ydhms2dt(args.end_yyyy, args.end_ddd, 0, 0, 0)
changes = determineESMChanges(start, end, station_metadata)
num_models = np.size(changes['ind']) + 1
print(args.site, "requires ", num_models)
def setUpCalcSiteModelPostFit(models, site_residuals, info, zen_grid, az_grid,
cpus):
nZen = int(90.0 / zen_grid) + 1
nAz = int(360. / az_grid)
print("nAz", nAz)
#numParamsPerSite = nZen * nAz
nModels = info['numModels']
#tSite = numParamsPerSite*params['numModels']
#numParams = tSat + tSite
prefit = np.zeros(nModels)
prefit_sums = np.zeros((nModels, nAz, nZen))
prefit_res = np.zeros((nModels, nAz, nZen))
postfit = np.zeros(nModels)
postfit_sums = np.zeros((nModels, nAz, nZen))
postfit_res = np.zeros((nModels, nAz, nZen))
numObs = np.zeros(nModels)
numObs_sums = np.zeros((nModels, nAz, nZen))
change = info['changes']
print("Changes for site", info['site'], change)
# keep track of how may observations are in each bin
#SiteFreq = np.zeros((nModels,nAz,nZen))
#Models = np.zeros((nModels,nAz,nZen))
#model_stdev = np.zeros((int(params['numModels']),nAz,nZen))
# create a new model everythime there has been a change of antenna
for m in range(0, nModels):
print(info['site'], "----> creating model", m + 1, "of",
info['numModels'])
# start_yyyy and start_ddd should always be defind, however stop_dd may be absent
#ie no changes have ocured since the last setup
minVal_dt = gt.ydhms2dt(change['start_yyyy'][m],
change['start_ddd'][m], 0, 0, 0)
if np.size(change['stop_ddd']) > m:
maxVal_dt = gt.ydhms2dt(change['stop_yyyy'][m],
change['stop_ddd'][m], 23, 59, 59)
print("Min:", minVal_dt, "Max:", maxVal_dt, m,
np.size(change['stop_ddd']))
criterion = ((site_residuals[:, 0] >= calendar.timegm(
minVal_dt.utctimetuple())) &
(site_residuals[:, 0] < calendar.timegm(
maxVal_dt.utctimetuple())))
else:
criterion = (site_residuals[:, 0] >= calendar.timegm(
minVal_dt.utctimetuple()))
maxVal_dt = gt.unix2dt(site_residuals[-1, 0])
# get the residuals for this model time period
mind = np.array(np.where(criterion))[0]
model_residuals = site_residuals[mind, :]
diff_dt = maxVal_dt - minVal_dt
numDays = diff_dt.days + 1
print("Have a total of", numDays, "days", np.shape(models))
prefit[m], prefit_sums[m, :, :], prefit_res[
m, :, :], postfit[m], postfit_sums[m, :, :], postfit_res[
m, :, :], numObs[m], numObs_sums[m, :, :] = setUpPostFitTasks(
models, model_residuals, args.cpu, zen_grid, az_grid,
params, numDays, minVal_dt)
print("FINISHED Post fit RUN for model", m)
return prefit, prefit_sums, prefit_res, postfit, postfit_sums, postfit_res, numObs, numObs_sums
def solveSiteModel(site_residuals,
svs,
params,
apr,
nadSpacing=0.1,
zenSpacing=0.5,
azSpacing=0.5,
brdc_dir="./"):
"""
Create a model for the satellites and sites at the same time.
PWL piece-wise-linear interpolation fit of phase residuals
-construct a PWL fit for each azimuth bin, and then paste them all together to get
the full model
-inversion is done within each bin
site_residuals = the one-way L3 post-fit, ambiguity fixed phase residuals
svs = an array of satellite SVN numbers that are spacebourne/operating
for the period of this residual stack
params = meta data about the solution bein attempted
['site'] = 4 char site id
['changes'] = dictionary of when model changes need to be applied
apr = satellite apriori data
"""
#prechi = 0
#NUMD = 0
# add one to make sure we have a linspace which includes 0.0 and 14.0
# add another parameter for the zenith PCO estimate
numNADS = int(14.0 / nadSpacing) + 1
PCOEstimates = 1
numSVS = np.size(svs)
numParamsPerSat = numNADS + PCOEstimates
tSat = numParamsPerSat * numSVS
nZen = int(90.0 / zenSpacing) + 1
nAz = int(360. / azSpacing)
print("nAz", nAz)
numParamsPerSite = nZen * nAz
tSite = numParamsPerSite * params['numModels']
numParams = tSat + tSite
print("------------------------------------------------")
print("Processing Site: ", params['site'])
print("------------------------------------------------")
print("Sat Params:----------------", numParamsPerSat)
print("Number of Sats:------------", np.size(svs))
print("Total satellite parameters:-------------", tSat)
print("Site Params:---------------", numParamsPerSite)
print("Number of Models:----------", params['numModels'])
print("Total Site Params:----------------------", tSite)
print("------------------------------------------------")
print("Total Params:---------------------------", numParams)
print("------------------------------------------------")
# Creating matrices
#Neq = np.zeros((numParams,numParams))
#AtWb = np.zeros(numParams)
change = params['changes']
print("Changes for site", params['site'], change)
# keep track of how may observations are in each bin
#NadirFreq = np.zeros((numSVS,numNADS))
SiteFreq = np.zeros((int(params['numModels']), nAz, nZen))
Models = np.zeros((int(params['numModels']), nAz, nZen))
model_stdev = np.zeros((int(params['numModels']), nAz, nZen))
# create a new model everythime there has been a change of antenna
for m in range(0, int(params['numModels'])):
print(params['site'], "----> creating model", m + 1, "of",
params['numModels'])
# start_yyyy and start_ddd should always be defind, however stop_dd may be absent
#ie no changes have ocured since the last setup
minVal_dt = gt.ydhms2dt(change['start_yyyy'][m],
change['start_ddd'][m], 0, 0, 0)
if np.size(change['stop_ddd']) > m:
maxVal_dt = gt.ydhms2dt(change['stop_yyyy'][m],
change['stop_ddd'][m], 23, 59, 59)
print("Min:", minVal_dt, "Max:", maxVal_dt, m,
np.size(change['stop_ddd']))
criterion = ((site_residuals[:, 0] >= calendar.timegm(
minVal_dt.utctimetuple())) &
(site_residuals[:, 0] < calendar.timegm(
maxVal_dt.utctimetuple())))
else:
criterion = (site_residuals[:, 0] >= calendar.timegm(
minVal_dt.utctimetuple()))
maxVal_dt = gt.unix2dt(site_residuals[-1, 0])
# get the residuals for this model time period
mind = np.array(np.where(criterion))[0]
model_residuals = site_residuals[mind, :]
diff_dt = maxVal_dt - minVal_dt
numDays = diff_dt.days + 1
print("Have a total of", numDays, "days")
Models[m, :, :], model_stdev[m, :, :], SiteFreq[
m, :, :] = setUpAzTasks(model_residuals, svs, args, params,
numDays, minVal_dt, nAz)
print("FINISHED AZ RUN for model", m)
print("Normal finish of pwl")
return Models, model_stdev, SiteFreq
parser.add_argument('--esm_report', dest='esm_report',default=False, action='store_true',
help='Determine when a station will need to create a new ESM, based on equipmet changes')
parser.add_argument('--start_yyyy',dest='start_yyyy',type=int,
help='Year of first epoch in YYYY format')
parser.add_argument('--start_ddd',dest='start_ddd',type=int,
help='Day-of-Year of first epoch in DDD format')
parser.add_argument('--end_yyyy',dest='end_yyyy',type=int,
help='Year of last epoch in YYYY format')
parser.add_argument('--end_ddd',dest='end_ddd',type=int,
help='Day-of-Year of last epoch in DDD format')
args = parser.parse_args()
#=========================================
# find the station meta data for a particular site
station_metadata = parseSite(args.station_file, args.site)
#print("data",data)
#ind = antennaChange(data)
#for i in ind:
# print(data['start_yyyy'][i],data['start_ddd'][i],data['antenna_type'][i],data['dome_type'][i])
#print("ind:",ind)
if args.esm_report:
start = gt.ydhms2dt(args.start_yyyy,args.start_ddd,0,0,0)
end = gt.ydhms2dt(args.end_yyyy,args.end_ddd,0,0,0)
changes = determineESMChanges(start,end,station_metadata)
num_models = np.size(changes['ind'])+1
print(args.site,"requires ",num_models)