def main(argv):
#----------------
# Initializations
#----------------
#------------
# Directories
#------------
wrkDir = os.getcwd() # Set current directory as the data directory
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:?')
except getopt.GetoptError as err:
print str(err)
usage(binDirVer)
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Data directory
if opt == '-i':
wrkDir = arg
dc.ckDir(wrkDir, exitFlg=True)
elif opt == '-?':
usage(binDirVer)
sys.exit()
else:
print 'Unhandled option: {}'.format(opt)
sys.exit()
if not (wrkDir.endswith('/')): wrkDir = wrkDir + '/'
#-----------------------------------------
# Assume that the sfit4.ctl file is in the
# working directory
#-----------------------------------------
ctlFile = wrkDir + 'sfit4.ctl'
#----------------------
# Initialize Plot Class
#----------------------
gas = dc.PlotData(wrkDir, ctlFile)
#--------------------------
# Call to plot spectral fit
#--------------------------
gas.pltSpectra()
#----------------------
# Call to plot profiles
#----------------------
gas.pltPrf(allGas=True)
#-----------------
# Call to plot AVK
#-----------------
if ('gas.profile.list' in gas.ctl) and gas.ctl['gas.profile.list']:
gas.pltAvk()
#-----------------------------
# Print summary file to screen
#-----------------------------
with open(wrkDir + 'summary', 'r') as fopen:
info = fopen.read()
print '\n****************SUMMARY FILE****************\n'
print(info)
print '\n****************END OF SUMMARY FILE****************\n'
#--------------------------------
# Pause so user can look at plots
#--------------------------------
user_input = raw_input('Press any key to exit >>> ')
sys.exit() # Exit program
def main():
#-----------------------------------------------------------------------------------------
# Initialization
#-----------------------------------------------------------------------------------------
loc = 'tab' # LOCATION
gas = 'pan' # GAS
alpha = [0.1, 1, 10, 100, 1000, 10000, 100000, 1000000] # ALPHA VALUES TO TEST
#alpha = [10000]#, 1000, 1e4] # ALPHA VALUES TO TEST
TikOut = '/data1/ebaumer/'+loc.lower()+'/'+gas.lower()+'/x.'+gas.lower()+'/' # PATH TO SAVE THE TIK MATRIX
binDir = '/data/ebaumer/Code/sfit-core-code/src/' # PATH FOR THE SFIT4 SOURCE CODE
errFlg = False # ERROR ANALYSIS?
saveFlg = True # SAVE PDF FILE?
TransFlg = True # OPTIONAL = TRANSFORMATION IN CASE IN CASE NON-CONSTANT RETRIEVAL GRID (e.g., http://www.atmos-meas-tech.net/4/1943/2011/)
if saveFlg: pltFile = TikOut + 'TikOpt.pdf'
#-----------------------------------------------------------------------------------------
# START
#-----------------------------------------------------------------------------------------
ckDir(TikOut, logFlg=False, exit=True)
if TransFlg:
fileLayers = '/data/Campaign/'+loc.upper()+'/local/station.layers'
ckFile(fileLayers, exit=True)
cols, indexToName = getColumns(fileLayers, headerrow=2, delim=' ', header=True)
midpnt = np.asarray(cols['midpnt'][0:-1]).astype(np.float)
thick = np.asarray(cols['thick'][0:-1]).astype(np.float)
level = np.asarray(cols['level']).astype(np.float)
else:
if loc == 'fl0': nlayer = 44
elif loc == 'mlo': nlayer = 41
elif loc == 'tab': nlayer = 47
else:
print "nlayer is not known for your location\nModify TikOpt to accomodate nlayer"
exit()
#-----------------------------------------------------------------------------------------
# Define variable to save
#-----------------------------------------------------------------------------------------
totCol = []
dof = []
rms = []
chi2y = []
tot_rnd = []
tot_sys = []
tot_std = []
tot_smt = []
tot_msr = []
#Find Current Directory
cwd = os.getcwd()
#-----------------------------------------------------------------------------------------
# Run Sfit
#-----------------------------------------------------------------------------------------
for ai in alpha:
if TransFlg:
tikCov_Trans(np.sort(level), ai, TikOut, transflag=True,normalizeflag=False)
else:
tikCov(nlayer, ai, TikOut)
print '************************************************************************'
print 'Running sfit4 - Tikhonov approach, alpha = {0:.3f}'.format(ai)
print '************************************************************************'
rtn = sc.subProcRun( [binDir + 'sfit4'] )
wrkDir = os.getcwd()
if not(wrkDir.endswith('/')): wrkDir = wrkDir + '/'
#-------------------
# Run error analysis
#-------------------
if errFlg:
print '**********************************************************************************'
print 'Running error analysis - Tikhonov approach, alpha = {0:.3f}\n'.format(ai)
print '**********************************************************************************'
ckFile('sfit4.ctl', exit=True)
ckFile('sb.ctl', exit = True)
ctlFile = sc.CtlInputFile('sfit4.ctl')
ctlFile.getInputs()
sbCtlFile = sc.CtlInputFile('sb.ctl')
sbCtlFile.getInputs()
rtn = errAnalysis(ctlFile,sbCtlFile,wrkDir)
gas = dc.PlotData(wrkDir,wrkDir+'sfit4.ctl')
gas.readsummary()
totCol.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_RetColmn']))
rms.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_FITRMS']))
dof.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_DOFS_TRG']))
chi2y.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_CHI_2_Y']))
if errFlg:
gas.readError(totFlg=True,sysFlg=False,randFlg=False,vmrFlg=False,avkFlg=False,KbFlg=False)
tot_rnd.append(np.array(gas.error['Total random uncertainty']))
tot_sys.append(np.array(gas.error['Total systematic uncertainty']))
tot_std.append(np.sqrt((np.array(gas.error['Total random uncertainty']))**2 + (np.array(gas.error['Total systematic uncertainty']))**2))
tot_smt.append(np.array(gas.error['Smoothing error (Ss)'])) #--> Eric's routine
#tot_smt.append(np.array(gas.error['Smoothing error (Ss, using sa)'])) #--> Bavo's routine
tot_msr.append(np.array(gas.error['Measurement error (Sm)']))
totCol = np.asarray(totCol)
dof = np.asarray(dof)
rms = np.asarray(rms)
chi2y = np.asarray(chi2y)
if errFlg:
tot_rnd = np.asarray(tot_rnd)
tot_sys = np.asarray(tot_sys)
tot_std = np.asarray(tot_std)
tot_smt = np.asarray(tot_smt)
tot_msr = np.asarray(tot_msr)
#-------------------
# Plots
#-------------------
print '**********************'
print ' Plots '
print '**********************'
if saveFlg: pdfsav = PdfPages(pltFile)
if errFlg: fig, ( (ax, ax2), (ax3, ax4) ) = plt.subplots(2, 2, sharex=True, figsize=(10, 8))
else: fig, (ax, ax2, ax3) = plt.subplots(3, 1, sharex=True)
#-------------------Total Column-------------------
ax.plot(alpha,totCol,label='totCol')
ax.scatter(alpha,totCol)
ax.grid(True,which='both')
ax.tick_params(which='both',labelsize=11)
ax.set_title('Total Column', multialignment='center')
ax.set_ylabel('Total Column [molec/cm$^2$]')
plt.xscale('log')
#-------------------rms-------------------
ax2.plot(alpha,rms,label='rms')
ax2.scatter(alpha,rms)
ax2.grid(True,which='both')
ax2.tick_params(which='both',labelsize=11)
ax2.set_title('RMS', multialignment='center')
ax2.set_ylabel('RMS [%]')
ax2.yaxis.set_major_formatter(FormatStrFormatter('%.4f'))
#-------------------dof-------------------
ax3.plot(alpha,dof,label='dof')
ax3.scatter(alpha,dof)
ax3.grid(True,which='both')
ax3.tick_params(which='both',labelsize=11)
ax3.set_title('DOF', multialignment='center')
ax3.set_ylabel('DOF')
ax3.set_xlabel('Regularization strength, alpha')
#-------------------Error-------------------
if errFlg:
ax4.plot(alpha,tot_smt, color='blue', label='smoothing')
ax4.scatter(alpha,tot_smt, color='blue')
ax4.plot(alpha,tot_msr, color='green', label='measurement')
ax4.scatter(alpha,tot_msr, color='green')
ax4.plot(alpha,np.sqrt(tot_msr**2 + tot_smt**2) , color='gray', label='Total (measurement + smoothing)')
ax4.scatter(alpha,np.sqrt(tot_msr**2 +tot_smt**2), color='gray')
ax4.grid(True,which='both')
ax4.tick_params(which='both',labelsize=11)
ax4.set_title('Error', multialignment='center')
ax4.set_ylabel('Error [%]')
ax4.set_xlabel('Regularization strength, alpha')
ax4.legend(prop={'size':10}, loc = 1)
fig.suptitle(cwd)
fig.tight_layout()
fig.subplots_adjust(left=0.1, right=0.95, top=0.9)
if saveFlg:
pdfsav.savefig(fig,dpi=200)
pdfsav.close()
else:
plt.show(block=False)
user_input = raw_input('Press any key to exit >>> ')
sys.exit()
def main(argv):
#------------------
# Set default flags
#------------------
logFile = False
lstFlg = False
pauseFlg = False
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:P:L:l?')
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Check input file flag and path
if opt == '-i':
# Input file instance
mainInF = sc.Layer1InputFile(arg)
# Pause after skip option
elif opt == '-P':
if not arg or arg.startswith('-'):
usage()
sys.exit()
pauseFlg = True
try:
nskips = int(arg) - 1
if nskips < 0: raise ValueError
except ValueError:
print 'Argument for -P flag: %s, needs to be an integer > 0' % arg
sys.exit()
# Show all command line flags
elif opt == '-?':
usage()
sys.exit()
# Option for Log File
elif opt == '-l':
logFile = True
# Option for List file
elif opt == '-L':
if not arg or arg.startswith('-'):
usage()
sys.exit()
lstFlg = True
lstFnameFlg = int(arg)
else:
print 'Unhandled option: ' + opt
sys.exit()
#----------------------------------------------
# Initialize main input variables as dicitonary
#----------------------------------------------
mainInF.getInputs()
#--------------------
# Initialize log file
#--------------------
# Write initial log data
if logFile:
log_fpath = mainInF.inputs['logDirOutput']
# check if '/' is included at end of path
if not (log_fpath.endswith('/')):
log_fpath = log_fpath + '/'
# check if path is valide
ckDir(log_fpath)
logFile = logging.getLogger('1')
logFile.setLevel(logging.INFO)
hdlr1 = logging.FileHandler(log_fpath +
mainInF.inputs['ctlList'][0][2] + '.log',
mode='w')
fmt1 = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s',
'%a, %d %b %Y %H:%M:%S')
hdlr1.setFormatter(fmt1)
logFile.addHandler(hdlr1)
logFile.info(
'**************** Starting Logging ***********************')
logFile.info('Input data file: ' + mainInF.fname)
logFile.info('Log file path: ' + log_fpath)
logFile.info('Station location: ' + mainInF.inputs['loc'])
#---------------------
# Initialize list file
#---------------------
if lstFlg:
lst_fpath = mainInF.inputs['logDirOutput']
# check if '/' is included at end of path
if not (lst_fpath.endswith('/')):
lst_fpath = lst_fpath + '/'
# check if path is valide
ckDir(lst_fpath)
lstFile = logging.getLogger('2')
lstFile.setLevel(logging.INFO)
if lstFnameFlg:
hdlr2 = logging.FileHandler(
lst_fpath + mainInF.inputs['ctlList'][0][2] + '.lst', mode='w')
else:
hdlr2 = logging.FileHandler(lst_fpath + 'testing.lst', mode='w')
fmt2 = logging.Formatter('')
hdlr2.setFormatter(fmt2)
lstFile.addHandler(hdlr2)
#-----------------------------
# Check the existance of files
#-----------------------------
# Spectral Database file
ckFile(mainInF.inputs['spcdbFile'], logFlg=logFile, exit=True)
# WACCM profile file
#ckFile(mainInF.inputs['WACCMfile'],logFlg=logFile,exit=True)
# ctl files
for ctlFile in mainInF.inputs['ctlList']:
ckFile(ctlFile[0], logFlg=logFile, exit=True)
#--------------------------------------------
# Program Looping structure. See Notes
# Level1 - LOC (Input/Output)
# Level2 - ctl file (Output)
# Level3 - Spectral db (Output)
# --Check I/O directory structure
#--------------------------------------------
# Establish Date Range
inDateRange = sc.DateRange(mainInF.inputs['iyear'],
mainInF.inputs['imnth'], mainInF.inputs['iday'],
mainInF.inputs['fyear'],
mainInF.inputs['fmnth'], mainInF.inputs['fday'])
#--------------------
# Level 1 -- LOC
#--------------------
if not (isinstance(mainInF.inputs['loc'], list)):
mainInF.inputs['loc'] = [mainInF.inputs['loc']]
for loc in mainInF.inputs['loc']:
#-------------------------------------------
# Check for existance of Input folder. Also,
# check if '/' is included at end of path
#-------------------------------------------
if not (mainInF.inputs['BaseDirInput'].endswith('/')):
wrkInputDir1 = mainInF.inputs['BaseDirInput'] + '/' + loc + '/'
else:
wrkInputDir1 = mainInF.inputs['BaseDirInput'] + loc + '/'
ckDir(wrkInputDir1, logFlg=logFile, exit=True)
#-----------------------------------------------------------
# Check for the existance of Output folder and create if DNE
# Also, check if '/' is included at end of path
#-----------------------------------------------------------
if not (mainInF.inputs['BaseDirOutput'].endswith('/')):
wrkOutputDir1 = mainInF.inputs['BaseDirOutput'] + '/'
else:
wrkOutputDir1 = mainInF.inputs['BaseDirOutput']
ckDirMk(wrkOutputDir1, logFile)
#--------------------------------------
# Find spectral db file and initialize
# instance and get inputs
#--------------------------------------
dbData = sc.DbInputFile(mainInF.inputs['spcdbFile'], logFile)
dbData.getInputs()
#-----------------------------
# Initial filter of data based
# on input date range
#-----------------------------
dbFltData_1 = dbData.dbFilterDate(inDateRange)
#---------------------------------
# Initialize error control file
# instance and get inputs (sb.ctl)
#---------------------------------
if mainInF.inputs['errFlg']:
ckFile(mainInF.inputs['sbCtlFile'], logFlg=logFile, exit=True)
SbctlFileVars = sc.CtlInputFile(mainInF.inputs['sbCtlFile'])
SbctlFileVars.getInputs()
#--------------------------------------
# Level 2 -- Loop through control files
#--------------------------------------
for ctl_ind, ctlFileList in enumerate(mainInF.inputs['ctlList']):
#-----------------------------
# Initialize ctl file instance
# and get inputs
#-----------------------------
ctlFile = ctlFileList[0]
ctlFileGlb = sc.CtlInputFile(ctlFile, logFile)
ctlFileGlb.getInputs()
#-----------------------------
# Write Meta-data to list file
#-----------------------------
if lstFlg:
lstFile.info('# Begin List File Meta-Data')
lstFile.info('Start Date = ' +
str(inDateRange.dateList[0]))
lstFile.info('End Date = ' +
str(inDateRange.dateList[-1]))
lstFile.info('WACCM_File = ' + mainInF.inputs['WACCMfile'])
lstFile.info('ctl_File = ' +
mainInF.inputs['ctlList'][ctl_ind][0])
lstFile.info('FilterID = ' +
mainInF.inputs['ctlList'][ctl_ind][1])
lstFile.info('VersionName = ' +
mainInF.inputs['ctlList'][ctl_ind][2])
lstFile.info('Site = ' + mainInF.inputs['loc'][0])
lstFile.info('statnLyrs_file = ' +
ctlFileGlb.inputs['file.in.stalayers'][0])
lstFile.info('primGas = ' + ctlFileGlb.primGas)
lstFile.info('specDBfile = ' + mainInF.inputs['spcdbFile'])
lstFile.info('Coadd flag = ' +
str(mainInF.inputs['coaddFlg']))
lstFile.info('nBNRfiles = ' +
str(mainInF.inputs['nBNRfiles']))
lstFile.info('ilsFlg = ' +
str(mainInF.inputs['ilsFlg']))
lstFile.info('pspecFlg = ' +
str(mainInF.inputs['pspecFlg']))
lstFile.info('refmkrFlg = ' +
str(mainInF.inputs['refmkrFlg']))
lstFile.info('sfitFlg = ' +
str(mainInF.inputs['sfitFlg']))
lstFile.info('lstFlg = ' +
str(mainInF.inputs['lstFlg']))
lstFile.info('errFlg = ' +
str(mainInF.inputs['errFlg']))
lstFile.info('zptFlg = ' +
str(mainInF.inputs['zptFlg']))
lstFile.info('refMkrLvl = ' +
str(mainInF.inputs['refMkrLvl']))
lstFile.info('wVer = ' + str(mainInF.inputs['wVer']))
lstFile.info('nbands = ' +
str(len(ctlFileGlb.inputs['band'])))
lstFile.info('# End List File Meta-Data')
lstFile.info('')
lstFile.info('Date TimeStamp Directory ')
#----------------------
# Filtering spectral DB
#----------------------
#-------------------------
# Filter spectral db based
# on wavenumber bounds in
# ctl file
#-------------------------
# Find the upper and lower bands from the ctl file
nu = []
for band in ctlFileGlb.inputs['band']:
bandstr = str(int(band))
nu.append(ctlFileGlb.inputs['band.' + bandstr +
'.nu_start'][0])
nu.append(ctlFileGlb.inputs['band.' + bandstr + '.nu_stop'][0])
nu.sort() # Sort wavenumbers
nuUpper = nu[-1] # Get upper wavenumber
nuLower = nu[0] # Get lower wavenumber
# Filter spectral DB based on wave number
dbFltData_2 = dbData.dbFilterNu(nuUpper, nuLower, dbFltData_1)
if not (dbFltData_2):
continue # Test for empty dicitonary (i.e. no data)
#------------------------------------------------------------------------------------------------
# In addition to filtering db based on wavenumbers in ctl file one can filter spectral db based
# on filter ID. Using this can help avoid the bug when pspec tries to apply a filter band outside
# spectral region of a bnr file.
#------------------------------------------------------------------------------------------------
if mainInF.inputs['ctlList'][ctl_ind][1]:
dbFltData_2 = dbData.dbFilterFltrID(
mainInF.inputs['ctlList'][ctl_ind][1], dbFltData_2)
if not (dbFltData_2):
continue # Test for empty dicitonary (i.e. no data)
#--------------------------------------------------------
# Filter database based on forward and backward scan flag
#--------------------------------------------------------
if mainInF.inputs['scnFlg'] > 0:
dbFltData_2 = dbData.dbFilterScn(mainInF.inputs['scnFlg'],
dbFltData_2)
#---------------------------------------------------------------------
# Check for the existance of Output folder <Version> and create if DNE
#---------------------------------------------------------------------
if mainInF.inputs['ctlList'][ctl_ind][2]:
wrkOutputDir2 = wrkOutputDir1 + mainInF.inputs['ctlList'][
ctl_ind][2] + '/'
ckDirMk(wrkOutputDir2, logFile)
else:
wrkOutputDir2 = wrkOutputDir1
#-----------------------------------------------
# Create a folder within the output directory to
# store various input files: ctl, hbin, isotope
#-----------------------------------------------
ctlPath, ctlFname = os.path.split(
mainInF.inputs['ctlList'][ctl_ind][0])
archDir = wrkOutputDir2 + 'inputFiles' + '/'
if ckDirMk(archDir, logFile):
for f in glob.glob(archDir + '*'):
os.remove(f)
shutil.copy(mainInF.inputs['ctlList'][ctl_ind][0],
archDir) # Copy ctl file
for file in glob.glob(ctlPath + '/*hbin*'): # Copy hbin files
shutil.copy(file, archDir)
for file in glob.glob(ctlPath + '/isotope*'): # Copy isotope file
shutil.copy(file, archDir)
#------------------------------------------
# Level 3 -- Loop through spectral db lines
#------------------------------------------
nobs = len(dbFltData_2['Date'])
for spcDBind in range(0, nobs):
#-----------------------------------------------------------
# Grab spectral data base information for specific retrieval
#-----------------------------------------------------------
# Get current date and time of spectral database entry
currntDayStr = str(int(dbFltData_2['Date'][spcDBind]))
currntDay = dt.datetime(
int(currntDayStr[0:4]), int(currntDayStr[4:6]),
int(currntDayStr[6:]),
int(dbFltData_2['Time'][spcDBind][0:2]),
int(dbFltData_2['Time'][spcDBind][3:5]),
int(dbFltData_2['Time'][spcDBind][6:]))
# Get dictionary with specific date
specDBone = dbData.dbFindDate(currntDay, fltDict=dbFltData_2)
brkFlg = True # Flag to break out of while statement
while True: # While statement is for the repeat function
print '\n\n\n'
print '*************************************************'
print '*************Begin New Retrieval*****************'
print '*************************************************'
#-------------------------------------------------------------
# If pause after skip flag is initialized, do several things:
# 1) Check if number of skips exceeds total number of filtered
# observations
# 2) Skip to specified starting point
# 3) Pause after first run
#-------------------------------------------------------------
if pauseFlg and (nskips > len(dbFltData_2['Date'])):
print 'Specified starting point in -P option (%d) is greater than number of observations in filtered database (%d)' % (
nskips, nobs)
if logFile:
logFile.critical(
'Specified starting point in -P option (%d) is greater than number of observations in filtered database (%d)'
% (nskips, nobs))
sys.exit()
if pauseFlg and (spcDBind < nskips): break
# Get date of observations
daystr = str(int(dbFltData_2['Date'][spcDBind]))
obsDay = dt.datetime(int(daystr[0:4]), int(daystr[4:6]),
int(daystr[6:]))
#----------------------------------------
# Check the existance of input and output
# directory structure
#----------------------------------------
# Find year month and day strings
yrstr = "{0:02d}".format(obsDay.year)
mnthstr = "{0:02d}".format(obsDay.month)
daystr = "{0:02d}".format(obsDay.day)
datestr = yrstr + mnthstr + daystr
# Check for existance of YYYYMMDD Input folder
# If this folder does not exist => there is no
# Data for this day
wrkInputDir2 = wrkInputDir1 + yrstr + mnthstr + daystr + '/'
ckDir(wrkInputDir2, logFlg=logFile, exit=True)
#-----------------------------------------
# Check for the existance of Output folder
# <Date>.<TimeStamp> and create if DNE
#-----------------------------------------
wrkOutputDir3 = wrkOutputDir2 + datestr + '.' + "{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind])) + '/'
if ckDirMk(wrkOutputDir3, logFile):
# Remove all files in Output directory if previously exists!!
for f in glob.glob(wrkOutputDir3 + '*'):
os.remove(f)
#-------------------------------
# Copy relavent files from input
# directory to output directoy
#-------------------------------
#-----------------------------------
# Copy control file to Output folder
# First check if location to copy ctl is
# the same location as original ctl file
#-----------------------------------
try:
shutil.copyfile(mainInF.inputs['ctlList'][ctl_ind][0],
wrkOutputDir3 + 'sfit4.ctl')
except IOError:
print 'Unable to copy template ctl file to working directory: %s' % wrkOutputDir3
if logFile:
logFile.critical(
'Unable to copy template ctl file to working directory: %s'
% wrkOutputDir3)
sys.exit()
#-------------------------------------
# Copy sb.ctl file to output directory
# if error analysis is chosen
#-------------------------------------
if mainInF.inputs['errFlg']:
try:
shutil.copyfile(mainInF.inputs['sbCtlFile'],
wrkOutputDir3 + 'sb.ctl')
except IOError:
print 'Unable to copy template sb.ctl file to working directory: %s' % wrkOutputDir3
if logFile:
logFile.critical(
'Unable to copy template sb.ctl file to working directory: %s'
% wrkOutputDir3)
sys.exit()
#----------------------------------
# Copy hbin details to output folder
# ** Assuming that the hbin.dtl and
# hbin.input files are in the same
# location as the global ctl file
#----------------------------------
try:
shutil.copyfile(ctlPath + '/hbin.dtl', wrkOutputDir3 +
'/hbin.dtl') # Copy hbin.dtl file
except IOError:
print 'Unable to copy file: %s' % (ctlPath +
'/hbin.dtl')
if logFile: logFile.error(IOError)
try:
shutil.copyfile(ctlPath + '/hbin.input',
wrkOutputDir3 +
'/hbin.input') # Copy hbin.input file
except IOError:
print 'Unable to copy file: %s' % (ctlPath +
'/hbin.input')
if logFile: logFile.error(IOError)
# Create instance of local control file (ctl file in working directory)
ctlFileLcl = sc.CtlInputFile(wrkOutputDir3 + 'sfit4.ctl',
logFile)
#-------------------------------------------------
# Determine whether to use ILS file. Empty string
# '' => no ILS file.
#-------------------------------------------------
if mainInF.inputs['ilsDir'] and mainInF.inputs['ilsFlg']:
#-------------------------------------------
# Determine if ilsDir is a file or directory
#-------------------------------------------
# If directory.....
if os.path.isdir(mainInF.inputs['ilsDir']):
# Determine which ILS file to use
ilsFileList = glob.glob(mainInF.inputs['ilsDir'] +
'ils*')
# Create a date list of ils files present
ilsYYYYMMDD = []
for ilsFile in ilsFileList:
ilsFileNpath = os.path.basename(ilsFile)
match = re.match(
r'\s*ils(\d\d\d\d)(\d\d)(\d\d).*',
ilsFileNpath)
ilsYYYYMMDD.append([
int(match.group(1)),
int(match.group(2)),
int(match.group(3))
])
ilsDateList = [
dt.date(ilsyear, ilsmonth, ilsday)
for ilsyear, ilsmonth, ilsday in ilsYYYYMMDD
]
# Find the ils date nearest to the current day
nearstDay = sc.nearestDate(ilsDateList,
obsDay.year,
obsDay.month,
obsDay.day)
nearstDayMnth = "{0:02d}".format(nearstDay.month)
nearstDayYr = "{0:02d}".format(nearstDay.year)
nearstDayDay = "{0:02d}".format(nearstDay.day)
nearstDaystr = nearstDayYr + nearstDayMnth + nearstDayDay
# Get File path and name for nearest ils file
for ilsFile in ilsFileList:
if nearstDaystr in os.path.basename(ilsFile):
ilsFname = ilsFile
# If file.....
elif os.path.isfile(mainInF.inputs['ilsDir']):
ilsFname = mainInF.inputs['ilsDir']
if logFile: logFile.info('Using ils file: ' + ilsFname)
# Replace ils file name in local ctl file (within working directory)
teststr = [
r'file.in.modulation_fcn', r'file.in.phase_fcn'
]
repVal = [ilsFname, ilsFname]
ctlFileLcl.replVar(teststr, repVal)
# Write FOV from spectral database file to ctl file (within working directory)
ctlFileLcl.replVar([r'band\.\d+\.omega'],
[str(specDBone['FOV'])])
#---------------------------
# Message strings for output
#---------------------------
msgstr1 = mainInF.inputs['ctlList'][ctl_ind][0]
msgstr2 = datestr + '.' + "{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind]))
#---------------------------
# Modification of Layer 1 (In case the ctl file should Use the OPD from the Spectra DataBase)
#---------------------------
# Write OPD from spectral database file to ctl file (within working directory)
#ctlFileLcl.replVar([r'band\.\d+\.max_opd'],[str(1.0/specDBone['Reso']*0.9)])
#----------------------------#
# #
# --- Run pspec--- #
# #
#----------------------------#
if mainInF.inputs['pspecFlg']:
print '*****************************************************'
print 'Running PSPEC for ctl file: %s' % msgstr1
print 'Processing spectral observation date: %s' % msgstr2
print '*****************************************************'
rtn = t15ascPrep(dbFltData_2, wrkInputDir2,
wrkOutputDir3, mainInF, spcDBind,
ctl_ind, logFile)
if logFile:
logFile.info('Ran PSPEC for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#----------------------------#
# #
# --- Run Refmaker--- #
# #
#----------------------------#
if mainInF.inputs['refmkrFlg']:
#-------------
# Run Refmaker
#-------------
if mainInF.inputs['waccmFlg']:
ckDir(mainInF.inputs['WACCMfolder'],
logFlg=logFile,
exit=True)
waccmFile = mainInF.inputs[
'WACCMfolder'] + 'WACCMref_V6-' + mnthstr + '.dat'
else:
waccmFile = mainInF.inputs['WACCMfile']
print '*****************************************************'
print 'Running REFMKRNCAR for ctl file: %s' % msgstr1
if mainInF.inputs['waccmFlg']:
print 'Using ' + waccmFile + ' WACCM Monthly Profile'
print 'Processing spectral observation date: %s' % msgstr2
print '*****************************************************'
rtn = refMkrNCAR(wrkInputDir2, waccmFile, wrkOutputDir3, \
mainInF.inputs['refMkrLvl'], mainInF.inputs['wVer'], mainInF.inputs['zptFlg'],\
dbFltData_2, spcDBind, logFile)
if logFile:
logFile.info('Ran REFMKRNCAR for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#----------------------------#
# #
# --- Run sfit4--- #
# #
#----------------------------#
#--------------
# Call to sfit4
#--------------
if mainInF.inputs['sfitFlg']:
print '*****************************************************'
print 'Running SFIT4 for ctl file: %s' % msgstr1
print 'Processing spectral observation date: %s' % msgstr2
print 'Ouput Directory: %s' % wrkOutputDir3
print '*****************************************************'
if logFile:
logFile.info('Ran SFIT4 for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#------------------------------
# Change working directory to
# output directory to run pspec
#------------------------------
try:
os.chdir(wrkOutputDir3)
except OSError as errmsg:
if logFile: logFile.error(errmsg)
sys.exit()
#---------------------
# Run sfit4 executable
#---------------------
sc.subProcRun([mainInF.inputs['binDir'] + 'sfit4'],
logFile)
#if ( stderr is None or not stderr):
#if log_flg:
#logFile.info('Finished running sfit4\n' + stdout)
#else:
#print 'Error running sfit4!!!'
#if log_flg:
#logFile.error('Error running sfit4 \n' + stdout)
#sys.exit()
#-----------------------------------
# Change permissions of all files in
# working directory
#-----------------------------------
for f in glob.glob(wrkOutputDir3 + '*'):
os.chmod(f, 0777)
#----------------------------------------------
# If succesfull run, write details to list file
#----------------------------------------------
if lstFlg:
fname = wrkOutputDir3 + 'sfit4.dtl'
cmpltFlg = False
with open(fname, 'r') as fopen:
for ind, line in enumerate(
reversed(fopen.readlines())):
if ind < 10:
if r'RDRV: DONE.' in line:
cmpltFlg = True
else:
break
if cmpltFlg and lstFile:
lstFile.info(
"{0:<13}".format(
int(dbFltData_2['Date'][spcDBind])) +
"{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind])) +
' ' + wrkOutputDir3)
#----------------------------#
# #
# --- Error Analysis --- #
# #
#----------------------------#
if mainInF.inputs['errFlg']:
if logFile:
logFile.info('Ran SFIT4 for ctl file: %s' %
msgstr1)
#-----------------------------------
# Enter into Error Analysis function
#-----------------------------------
rtn = errAnalysis(ctlFileGlb, SbctlFileVars,
wrkOutputDir3, logFile)
#---------------------------
# Continuation for Pause flg
#---------------------------
if pauseFlg:
while True:
user_input = raw_input(
'Paused processing....\n Enter: 0 to exit, -1 to repeat, 1 to continue to next, 2 to continue all, 3 plot retrieval results\n >>> '
)
plt.close('all')
try:
user_input = int(user_input)
if not any(user_input == val
for val in [-1, 0, 1, 2, 3]):
raise ValueError
except ValueError:
print 'Please enter -1, 0, 1, 2, or 3'
if user_input == 0: sys.exit() # Exit program
elif user_input == 1: # Exit while loop (Do not repeat)
brkFlg = True
break
elif user_input == 2: # Stop pause and exit while loop
pauseFlg = False
brkFlg = True
break
elif user_input == 3: # Plot retrieval results
#----------------------
# Initialize Plot Class
#----------------------
gas = dc.PlotData(
wrkOutputDir3,
wrkOutputDir3 + 'sfit4.ctl')
#--------------------------
# Call to plot spectral fit
#--------------------------
gas.pltSpectra()
#----------------------
# Call to plot profiles
#----------------------
try:
gas.pltPrf(allGas=True, errFlg=True)
except:
gas.pltPrf(allGas=True)
#-----------------
# Call to plot AVK
#-----------------
try:
if ('gas.profile.list' in gas.ctl
) and gas.ctl['gas.profile.list']:
gas.pltAvk()
except:
print "Unable to plot AVK!!"
#-----------------------------
# Print summary file to screen
#-----------------------------
if ckFile(wrkOutputDir3 + 'summary'):
with open(wrkOutputDir3 + 'summary',
'r') as fopen:
info = fopen.read()
print '****************SUMMARY FILE****************'
print(info)
print '****************END OF SUMMARY FILE****************'
elif user_input == -1: # Repeat loop
brkFlg = False
break
#-----------------------
# Exit out of while loop
#-----------------------
if brkFlg: break
def main(argv):
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:?')
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Check input file flag and path
if opt == '-i':
pltInputs = {}
ckFile(arg, exit=True)
try:
execfile(arg, pltInputs)
except IOError as errmsg:
print errmsg + ' : ' + arg
sys.exit()
if '__builtins__' in pltInputs:
del pltInputs['__builtins__']
# Show all command line flags
elif opt == '-?':
usage()
sys.exit()
else:
print 'Unhandled option: ' + opt
sys.exit()
#---------------------------------
# Check for the existance of files
# directories from input file
#---------------------------------
ckDir(pltInputs['retDir'], exit=True)
ckFile(pltInputs['ctlFile'], exit=True)
if pltInputs['saveFlg']:
ckDir(os.path.dirname(os.path.realpath(pltInputs['pltFile'])),
exit=True)
#-------------------------
# Create Instance of Class
#-------------------------
gas = dc.PlotData(pltInputs['retDir'],
pltInputs['ctlFile'],
iyear=pltInputs['iyear'],
imnth=pltInputs['imnth'],
iday=pltInputs['iday'],
fyear=pltInputs['fyear'],
fmnth=pltInputs['fmnth'],
fday=pltInputs['fday'],
outFname=pltInputs['pltFile'])
#----------------------
#Call to plot profiles
#----------------------
gas.pltPrf(fltr=pltInputs['fltrFlg'],
allGas=False,
sclfct=pltInputs['sclfct'],
sclname=pltInputs['sclfctName'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
errFlg=pltInputs['errorFlg'],
minSZA=pltInputs['minSZA'],
maxSZA=pltInputs['maxSZA'],
maxRMS=pltInputs['maxRMS'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
pcFlg=pltInputs['pcNegFlg'],
szaFlg=pltInputs['szaFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
chiFlg=pltInputs['chiFlg'],
tcMMflg=pltInputs['tcMMFlg'])
#-----------------
# Call to plot AVK
#-----------------
try:
gas.pltAvk(fltr=pltInputs['fltrFlg'],
errFlg=pltInputs['errorFlg'],
partialCols=pltInputs['pCols'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
minSZA=pltInputs['minSZA'],
maxSZA=pltInputs['maxSZA'],
maxRMS=pltInputs['maxRMS'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
pcFlg=pltInputs['pcNegFlg'],
szaFlg=pltInputs['szaFlg'],
chiFlg=pltInputs['chiFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
tcMMflg=pltInputs['tcMMFlg'])
except:
print "Unable to plot AVK!!"
#-------------------
# Plot total columns
#-------------------
gas.pltTotClmn(fltr=pltInputs['fltrFlg'],
sclfct=pltInputs['sclfct'],
sclname=pltInputs['sclfctName'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
partialCols=pltInputs['pCols'],
errFlg=pltInputs['errorFlg'],
minSZA=pltInputs['minSZA'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
maxSZA=pltInputs['maxSZA'],
maxRMS=pltInputs['maxRMS'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
pcFlg=pltInputs['pcNegFlg'],
szaFlg=pltInputs['szaFlg'],
chiFlg=pltInputs['chiFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
tcMMflg=pltInputs['tcMMFlg'])
#------------------
# Plot Spectral fit
#------------------
gas.pltSpectra(fltr=pltInputs['fltrFlg'],
minSZA=pltInputs['minSZA'],
maxSZA=pltInputs['maxSZA'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
maxRMS=pltInputs['maxRMS'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
pcFlg=pltInputs['pcNegFlg'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
szaFlg=pltInputs['szaFlg'],
chiFlg=pltInputs['chiFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
tcMMflg=pltInputs['tcMMFlg'])
##----
if pltInputs['saveFlg']: gas.closeFig()
#--------------------
# Create yearly plots
#--------------------
if pltInputs['byYrFlg']:
for year in gas.yearList():
if pltInputs['saveFlg']:
fname = pltInputs['pltFile'][:-4] + '_' + str(year) + '.pdf'
else:
fname = ''
gasYr = dc.PlotData(pltInputs['retDir'],
pltInputs['ctlFile'],
iyear=pltInputs['iyear'],
imnth=1,
iday=1,
fyear=year,
fmnth=12,
fday=31,
outFname=fname)
gas.pltPrf(fltr=pltInputs['fltrFlg'],
allGas=False,
sclfct=pltInputs['sclfct'],
sclname=pltInputs['sclfctName'],
errFlg=pltInputs['errorFlg'],
minSZA=pltInputs['minSZA'],
maxSZA=pltInputs['maxSZA'],
maxRMS=pltInputs['maxRMS'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
pcFlg=pltInputs['pcNegFlg'],
szaFlg=pltInputs['szaFlg'],
chiFlg=pltInputs['chiFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
tcMMflg=pltInputs['tcMMFlg'])
gas.pltTotClmn(fltr=pltInputs['fltrFlg'],
sclfct=pltInputs['sclfct'],
sclname=pltInputs['sclfctName'],
partialCols=pltInputs['pCols'],
errFlg=pltInputs['errorFlg'],
minSZA=pltInputs['minSZA'],
minTC=pltInputs['minTC'],
maxTC=pltInputs['maxTC'],
maxSZA=pltInputs['maxSZA'],
maxRMS=pltInputs['maxRMS'],
minDOF=pltInputs['minDOF'],
maxCHI=pltInputs['maxCHI'],
dofFlg=pltInputs['dofFlg'],
rmsFlg=pltInputs['rmsFlg'],
tcFlg=pltInputs['tcNegFlg'],
mnthFltr=pltInputs["mnths"],
mnthFltFlg=pltInputs["mnthFlg"],
pcFlg=pltInputs['pcNegFlg'],
szaFlg=pltInputs['szaFlg'],
chiFlg=pltInputs['chiFlg'],
cnvrgFlg=pltInputs['cnvrgFlg'],
tcMMflg=pltInputs['tcMMFlg'])
if pltInputs['saveFlg']: gasYr.closeFig()
print('\nFinished Plots.......\n')
#--------------------------------
# Pause so user can look at plots
#--------------------------------
if not pltInputs['saveFlg']:
user_input = raw_input('Press any key to exit >>> ')
sys.exit() # Exit program