def main(argv):
#----------------
# Initializations
#----------------
#------------
# Directories
#------------
wrkDir = os.getcwd() # Set current directory as the data directory
binDir = '/data/bin' # Default binary directory. Used of nothing is specified on command line
binDirVer = {
'v1':
'/data/ebaumer/Code/sfit-core-code/src/', # Version 1 for binary directory (Eric)
'v2':
'/data/tools/400/sfit-core/src/', # Version 2 for binary directory (Jim)
'v3':
'/Users/jamesw/FDP/sfit/400/sfit-core/src/', # Version 2 for binary directory (Jim)
'v4': '/home/ebaumer/Code/sfit4/src/',
'v5': '/Users/jamesw/FDP/sfit/400/src/src-irwg14-mp'
}
#----------
# Run flags
#----------
hbinFlg = False # Flag to run hbin
pspecFlg = False # Flag to run pspec
sfitFlg = False # Flag to run sfit4
errFlg = False # Flag to run error analysis
clnFlg = False # Flag to clean directory of output files listed in ctl file
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:b:f:?')
except getopt.GetoptError as err:
print str(err)
usage(binDirVer)
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Data directory
if opt == '-i':
wrkDir = arg
sc.ckDir(wrkDir, exitFlg=True)
# Binary directory
elif opt == '-b':
if not sc.ckDir(arg, exitFlg=False, quietFlg=True):
try:
binDir = binDirVer[arg.lower()]
except KeyError:
print '{} not a recognized version for -b option'.format(
arg)
sys.exit()
else:
binDir = arg
if not (binDir.endswith('/')): binDir = binDir + '/'
# Run flags
elif opt == '-f':
flgs = list(arg)
for f in flgs:
if f.lower() == 'h': hbinFlg = True
elif f.lower() == 'p': pspecFlg = True
elif f.lower() == 's': sfitFlg = True
elif f.lower() == 'e': errFlg = True
elif f.lower() == 'c': clnFile = True
else: print '{} not an option for -f ... ignored'.format(f)
elif opt == '-?':
usage(binDirVer)
sys.exit()
else:
print 'Unhandled option: {}'.format(opt)
sys.exit()
#--------------------------------------
# If necessary change working directory
# to directory with input data.
#--------------------------------------
if os.path.abspath(wrkDir) != os.getcwd(): os.chdir(wrkDir)
if not (wrkDir.endswith('/')): wrkDir = wrkDir + '/'
#--------------------------
# Initialize sfit ctl class
#--------------------------
if sc.ckFile(wrkDir + 'sfit4.ctl'): ctlFileName = wrkDir + 'sfit4.ctl'
else:
Tk().withdraw()
ctlFileName = askopenfilename(initialdir=wrkDir,
message='Please select sfit ctl file')
ctlFile = sc.CtlInputFile(ctlFileName)
ctlFile.getInputs()
#------------------------
# Initialize sb ctl class
#------------------------
if errFlg:
if sc.ckFile(wrkDir + 'sb.ctl'): sbCtlFileName = wrkDir + 'sb.ctl'
else:
TK().withdraw()
sbCtlFileName = askopenfilename(
initialdir=wrkDir, message='Please select sb ctl file')
sbCtlFile = sc.CtlInputFile(sbCtlFileName)
sbCtlFile.getInputs()
#---------------------------
# Clean up output from sfit4
#---------------------------
if clnFlg:
for k in ctlFile.inputs['file.out']:
if 'file.out' in k:
try:
os.remove(wrkDir + ctlFile.inputs[k])
except OSError:
pass
#----------
# Run pspec
#----------
if pspecFlg:
print '*************'
print 'Running pspec'
print '*************'
rtn = sc.subProcRun([binDir + 'pspec'])
#----------
# Run hbin
#----------
if hbinFlg:
print '************'
print 'Running hbin'
print '************'
rtn = sc.subProcRun([binDir + 'hbin'])
#----------
# Run sfit4
#----------
if sfitFlg:
print '************'
print 'Running sfit'
print '************'
rtn = sc.subProcRun([binDir + 'sfit4'])
#-------------------
# Run error analysis
#-------------------
if errFlg:
print '**********************'
print 'Running error analysis'
print '**********************'
rtn = errAnalysis(ctlFile, sbCtlFile, wrkDir)
def main(argv):
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:N:d:?')
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Layer1 input path and name
if opt == '-i':
inputFile = arg
# Output list file name and directory
elif opt == '-N':
outputFile = arg
# Base Directory for data
elif opt == '-d':
baseDir = arg
# Show all command line flags
elif opt == '-?':
usage()
sys.exit()
else:
print 'Unhandled option: ' + opt
sys.exit()
#---------------------------------
# Initialize list file as log file
#---------------------------------
lstFile = logging.getLogger('1')
lstFile.setLevel(logging.INFO)
hdlr1 = logging.FileHandler(outputFile, mode='w')
fmt1 = logging.Formatter('')
hdlr1.setFormatter(fmt1)
lstFile.addHandler(hdlr1)
#----------------------------------
# Check the existance of input file
#----------------------------------
ckFile(inputFile, exit=True)
#--------------------------------------
# Check the existance of base directory
#--------------------------------------
ckDir(baseDir, exit=True)
# check if '/' is included at end of path
if not (baseDir.endswith('/')):
baseDir = baseDir + '/'
#-------------------------
# Get data from input file
#-------------------------
inVars = sc.Layer1InputFile(inputFile)
inVars.getInputs()
#--------------------------------
# Check the existance of ctl file
#--------------------------------
ckFile(inVars.inputs['ctlList'][0][0], exit=True)
#------------------
# Get ctl file data
#------------------
ctlData = sc.CtlInputFile(inVars.inputs['ctlList'][0][0])
ctlData.getInputs()
#------------------------
# Write data to list file
#------------------------
lstFile.info('# Begin List File Meta-Data')
lstFile.info('Start Date = ')
lstFile.info('End Date = ')
lstFile.info('WACCM_File = ' + inVars.inputs['WACCMfile'])
lstFile.info('ctl_File = ' + inVars.inputs['ctlList'][0][0])
lstFile.info('FilterID = ' + inVars.inputs['ctlList'][0][1])
lstFile.info('VersionName = ' + inVars.inputs['ctlList'][0][2])
lstFile.info('Site = ' + inVars.inputs['loc'])
lstFile.info('statnLyrs_file = ' + ctlData.inputs['file.in.stalayers'][0])
lstFile.info('primGas = ' + ctlData.primGas)
lstFile.info('specDBfile = ' + inVars.inputs['spcdbFile'])
lstFile.info('Coadd flag = ' + str(inVars.inputs['coaddFlg']))
lstFile.info('nBNRfiles = ' + str(inVars.inputs['nBNRfiles']))
lstFile.info('ilsFlg = ' + str(inVars.inputs['ilsFlg']))
lstFile.info('pspecFlg = ' + str(inVars.inputs['pspecFlg']))
lstFile.info('refmkrFlg = ' + str(inVars.inputs['refmkrFlg']))
lstFile.info('sfitFlg = ' + str(inVars.inputs['sfitFlg']))
lstFile.info('lstFlg = ' + str(inVars.inputs['lstFlg']))
lstFile.info('errFlg = ' + str(inVars.inputs['errFlg']))
lstFile.info('zptFlg = ' + str(inVars.inputs['zptFlg']))
lstFile.info('refMkrLvl = ' + str(inVars.inputs['refMkrLvl']))
lstFile.info('wVer = ' + str(inVars.inputs['wVer']))
lstFile.info('# End List File Meta-Data')
lstFile.info('')
lstFile.info('Date TimeStamp Directory ')
#-----------------------------------------------------
# Loop through directory to find all valid retreivals.
# Retrieval is valid when summary file exists.
#-----------------------------------------------------
#----------------------------------------
# Walk through first level of directories
#----------------------------------------
lstDict = {}
for drs in os.walk(baseDir).next()[1]:
YYYYMMDD = drs[0:4] + drs[4:6] + drs[6:8]
hhmmss = drs[9:11] + drs[11:13] + drs[13:]
if os.path.isfile(baseDir + drs + '/summary'):
lstDict.setdefault('date', []).append(
dt.datetime(int(drs[0:4]), int(drs[4:6]), int(drs[6:8]),
int(drs[9:11]), int(drs[11:13]), int(drs[13:])))
lstDict.setdefault('YYYYMMDD', []).append(YYYYMMDD)
lstDict.setdefault('hhmmss', []).append(hhmmss)
lstDict.setdefault('directory', []).append(baseDir + drs)
lstDict = sortDict(lstDict, 'date')
for ind, val in enumerate(lstDict['date']):
lstFile.info("{0:<13}".format(lstDict['YYYYMMDD'][ind]) +
"{0:6}".format(lstDict['hhmmss'][ind]) + ' ' +
lstDict['directory'][ind] + '/')
def main(argv):
#------------------
# Set default flags
#------------------
logFile = False
lstFlg = False
pauseFlg = False
#--------------------------------
# Retrieve command line arguments
#--------------------------------
try:
opts, args = getopt.getopt(sys.argv[1:], 'i:P:L:l?')
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
#-----------------------------
# Parse command line arguments
#-----------------------------
for opt, arg in opts:
# Check input file flag and path
if opt == '-i':
# Input file instance
mainInF = sc.Layer1InputFile(arg)
# Pause after skip option
elif opt == '-P':
if not arg or arg.startswith('-'):
usage()
sys.exit()
pauseFlg = True
try:
nskips = int(arg) - 1
if nskips < 0: raise ValueError
except ValueError:
print 'Argument for -P flag: %s, needs to be an integer > 0' % arg
sys.exit()
# Show all command line flags
elif opt == '-?':
usage()
sys.exit()
# Option for Log File
elif opt == '-l':
logFile = True
# Option for List file
elif opt == '-L':
if not arg or arg.startswith('-'):
usage()
sys.exit()
lstFlg = True
lstFnameFlg = int(arg)
else:
print 'Unhandled option: ' + opt
sys.exit()
#----------------------------------------------
# Initialize main input variables as dicitonary
#----------------------------------------------
mainInF.getInputs()
#--------------------
# Initialize log file
#--------------------
# Write initial log data
if logFile:
log_fpath = mainInF.inputs['logDirOutput']
# check if '/' is included at end of path
if not (log_fpath.endswith('/')):
log_fpath = log_fpath + '/'
# check if path is valide
ckDir(log_fpath)
logFile = logging.getLogger('1')
logFile.setLevel(logging.INFO)
hdlr1 = logging.FileHandler(log_fpath +
mainInF.inputs['ctlList'][0][2] + '.log',
mode='w')
fmt1 = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s',
'%a, %d %b %Y %H:%M:%S')
hdlr1.setFormatter(fmt1)
logFile.addHandler(hdlr1)
logFile.info(
'**************** Starting Logging ***********************')
logFile.info('Input data file: ' + mainInF.fname)
logFile.info('Log file path: ' + log_fpath)
logFile.info('Station location: ' + mainInF.inputs['loc'])
#---------------------
# Initialize list file
#---------------------
if lstFlg:
lst_fpath = mainInF.inputs['logDirOutput']
# check if '/' is included at end of path
if not (lst_fpath.endswith('/')):
lst_fpath = lst_fpath + '/'
# check if path is valide
ckDir(lst_fpath)
lstFile = logging.getLogger('2')
lstFile.setLevel(logging.INFO)
if lstFnameFlg:
hdlr2 = logging.FileHandler(
lst_fpath + mainInF.inputs['ctlList'][0][2] + '.lst', mode='w')
else:
hdlr2 = logging.FileHandler(lst_fpath + 'testing.lst', mode='w')
fmt2 = logging.Formatter('')
hdlr2.setFormatter(fmt2)
lstFile.addHandler(hdlr2)
#-----------------------------
# Check the existance of files
#-----------------------------
# Spectral Database file
ckFile(mainInF.inputs['spcdbFile'], logFlg=logFile, exit=True)
# WACCM profile file
#ckFile(mainInF.inputs['WACCMfile'],logFlg=logFile,exit=True)
# ctl files
for ctlFile in mainInF.inputs['ctlList']:
ckFile(ctlFile[0], logFlg=logFile, exit=True)
#--------------------------------------------
# Program Looping structure. See Notes
# Level1 - LOC (Input/Output)
# Level2 - ctl file (Output)
# Level3 - Spectral db (Output)
# --Check I/O directory structure
#--------------------------------------------
# Establish Date Range
inDateRange = sc.DateRange(mainInF.inputs['iyear'],
mainInF.inputs['imnth'], mainInF.inputs['iday'],
mainInF.inputs['fyear'],
mainInF.inputs['fmnth'], mainInF.inputs['fday'])
#--------------------
# Level 1 -- LOC
#--------------------
if not (isinstance(mainInF.inputs['loc'], list)):
mainInF.inputs['loc'] = [mainInF.inputs['loc']]
for loc in mainInF.inputs['loc']:
#-------------------------------------------
# Check for existance of Input folder. Also,
# check if '/' is included at end of path
#-------------------------------------------
if not (mainInF.inputs['BaseDirInput'].endswith('/')):
wrkInputDir1 = mainInF.inputs['BaseDirInput'] + '/' + loc + '/'
else:
wrkInputDir1 = mainInF.inputs['BaseDirInput'] + loc + '/'
ckDir(wrkInputDir1, logFlg=logFile, exit=True)
#-----------------------------------------------------------
# Check for the existance of Output folder and create if DNE
# Also, check if '/' is included at end of path
#-----------------------------------------------------------
if not (mainInF.inputs['BaseDirOutput'].endswith('/')):
wrkOutputDir1 = mainInF.inputs['BaseDirOutput'] + '/'
else:
wrkOutputDir1 = mainInF.inputs['BaseDirOutput']
ckDirMk(wrkOutputDir1, logFile)
#--------------------------------------
# Find spectral db file and initialize
# instance and get inputs
#--------------------------------------
dbData = sc.DbInputFile(mainInF.inputs['spcdbFile'], logFile)
dbData.getInputs()
#-----------------------------
# Initial filter of data based
# on input date range
#-----------------------------
dbFltData_1 = dbData.dbFilterDate(inDateRange)
#---------------------------------
# Initialize error control file
# instance and get inputs (sb.ctl)
#---------------------------------
if mainInF.inputs['errFlg']:
ckFile(mainInF.inputs['sbCtlFile'], logFlg=logFile, exit=True)
SbctlFileVars = sc.CtlInputFile(mainInF.inputs['sbCtlFile'])
SbctlFileVars.getInputs()
#--------------------------------------
# Level 2 -- Loop through control files
#--------------------------------------
for ctl_ind, ctlFileList in enumerate(mainInF.inputs['ctlList']):
#-----------------------------
# Initialize ctl file instance
# and get inputs
#-----------------------------
ctlFile = ctlFileList[0]
ctlFileGlb = sc.CtlInputFile(ctlFile, logFile)
ctlFileGlb.getInputs()
#-----------------------------
# Write Meta-data to list file
#-----------------------------
if lstFlg:
lstFile.info('# Begin List File Meta-Data')
lstFile.info('Start Date = ' +
str(inDateRange.dateList[0]))
lstFile.info('End Date = ' +
str(inDateRange.dateList[-1]))
lstFile.info('WACCM_File = ' + mainInF.inputs['WACCMfile'])
lstFile.info('ctl_File = ' +
mainInF.inputs['ctlList'][ctl_ind][0])
lstFile.info('FilterID = ' +
mainInF.inputs['ctlList'][ctl_ind][1])
lstFile.info('VersionName = ' +
mainInF.inputs['ctlList'][ctl_ind][2])
lstFile.info('Site = ' + mainInF.inputs['loc'][0])
lstFile.info('statnLyrs_file = ' +
ctlFileGlb.inputs['file.in.stalayers'][0])
lstFile.info('primGas = ' + ctlFileGlb.primGas)
lstFile.info('specDBfile = ' + mainInF.inputs['spcdbFile'])
lstFile.info('Coadd flag = ' +
str(mainInF.inputs['coaddFlg']))
lstFile.info('nBNRfiles = ' +
str(mainInF.inputs['nBNRfiles']))
lstFile.info('ilsFlg = ' +
str(mainInF.inputs['ilsFlg']))
lstFile.info('pspecFlg = ' +
str(mainInF.inputs['pspecFlg']))
lstFile.info('refmkrFlg = ' +
str(mainInF.inputs['refmkrFlg']))
lstFile.info('sfitFlg = ' +
str(mainInF.inputs['sfitFlg']))
lstFile.info('lstFlg = ' +
str(mainInF.inputs['lstFlg']))
lstFile.info('errFlg = ' +
str(mainInF.inputs['errFlg']))
lstFile.info('zptFlg = ' +
str(mainInF.inputs['zptFlg']))
lstFile.info('refMkrLvl = ' +
str(mainInF.inputs['refMkrLvl']))
lstFile.info('wVer = ' + str(mainInF.inputs['wVer']))
lstFile.info('# End List File Meta-Data')
lstFile.info('')
lstFile.info('Date TimeStamp Directory ')
#-------------------------
# Filter spectral db based
# on wavenumber bounds in
# ctl file
#-------------------------
# Find the upper and lower bands from the ctl file
nu = []
for band in ctlFileGlb.inputs['band']:
bandstr = str(int(band))
nu.append(ctlFileGlb.inputs['band.' + bandstr +
'.nu_start'][0])
nu.append(ctlFileGlb.inputs['band.' + bandstr + '.nu_stop'][0])
nu.sort() # Sort wavenumbers
nuUpper = nu[-1] # Get upper wavenumber
nuLower = nu[0] # Get lower wavenumber
# Filter spectral DB based on wave number
dbFltData_2 = dbData.dbFilterNu(nuUpper, nuLower, dbFltData_1)
if not (dbFltData_2):
continue # Test for empty dicitonary (i.e. no data)
#------------------------------------------------------------------------------------------------
# In addition to filtering db based on wavenumbers in ctl file one can filter spectral db based
# on filter ID. Using this can help avoid the bug when pspec tries to apply a filter band outside
# spectral region of a bnr file.
#------------------------------------------------------------------------------------------------
if mainInF.inputs['ctlList'][ctl_ind][1]:
dbFltData_2 = dbData.dbFilterFltrID(
mainInF.inputs['ctlList'][ctl_ind][1], dbFltData_2)
if not (dbFltData_2):
continue # Test for empty dicitonary (i.e. no data)
#---------------------------------------------------------------------
# Check for the existance of Output folder <Version> and create if DNE
#---------------------------------------------------------------------
if mainInF.inputs['ctlList'][ctl_ind][2]:
wrkOutputDir2 = wrkOutputDir1 + mainInF.inputs['ctlList'][
ctl_ind][2] + '/'
ckDirMk(wrkOutputDir2, logFile)
else:
wrkOutputDir2 = wrkOutputDir1
#-----------------------------------------------
# Create a folder within the output directory to
# store various input files: ctl, hbin, isotope
#-----------------------------------------------
ctlPath, ctlFname = os.path.split(
mainInF.inputs['ctlList'][ctl_ind][0])
archDir = wrkOutputDir2 + 'inputFiles' + '/'
if ckDirMk(archDir, logFile):
for f in glob.glob(archDir + '*'):
os.remove(f)
shutil.copy(mainInF.inputs['ctlList'][ctl_ind][0],
archDir) # Copy ctl file
for file in glob.glob(ctlPath + '/*hbin*'): # Copy hbin files
shutil.copy(file, archDir)
for file in glob.glob(ctlPath + '/isotope*'): # Copy isotope file
shutil.copy(file, archDir)
#------------------------------------------
# Level 3 -- Loop through spectral db lines
#------------------------------------------
nobs = len(dbFltData_2['Date'])
for spcDBind in range(0, nobs):
#-----------------------------------------------------------
# Grab spectral data base information for specific retrieval
#-----------------------------------------------------------
# Get current date and time of spectral database entry
currntDayStr = str(int(dbFltData_2['Date'][spcDBind]))
currntDay = dt.datetime(
int(currntDayStr[0:4]), int(currntDayStr[4:6]),
int(currntDayStr[6:]),
int(dbFltData_2['Time'][spcDBind][0:2]),
int(dbFltData_2['Time'][spcDBind][3:5]),
int(dbFltData_2['Time'][spcDBind][6:]))
# Get dictionary with specific date
specDBone = dbData.dbFindDate(currntDay, fltDict=dbFltData_2)
brkFlg = True # Flag to break out of while statement
while True: # While statement is for the repeat function
#-------------------------------------------------------------
# If pause after skip flag is initialized, do several things:
# 1) Check if number of skips exceeds total number of filtered
# observations
# 2) Skip to specified starting point
# 3) Pause after first run
#-------------------------------------------------------------
if pauseFlg and (nskips > len(dbFltData_2['Date'])):
print 'Specified starting point in -P option (%d) is greater than number of observations in filtered database (%d)' % (
nskips, nobs)
if logFile:
logFile.critical(
'Specified starting point in -P option (%d) is greater than number of observations in filtered database (%d)'
% (nskips, nobs))
sys.exit()
if pauseFlg and (spcDBind < nskips): break
# Get date of observations
daystr = str(int(dbFltData_2['Date'][spcDBind]))
obsDay = dt.datetime(int(daystr[0:4]), int(daystr[4:6]),
int(daystr[6:]))
#----------------------------------------
# Check the existance of input and output
# directory structure
#----------------------------------------
# Find year month and day strings
yrstr = "{0:02d}".format(obsDay.year)
mnthstr = "{0:02d}".format(obsDay.month)
daystr = "{0:02d}".format(obsDay.day)
datestr = yrstr + mnthstr + daystr
# Check for existance of YYYYMMDD Input folder
# If this folder does not exist => there is no
# Data for this day
wrkInputDir2 = wrkInputDir1 + yrstr + mnthstr + daystr + '/'
ckDir(wrkInputDir2, logFlg=logFile, exit=True)
#-----------------------------------------
# Check for the existance of Output folder
# <Date>.<TimeStamp> and create if DNE
#-----------------------------------------
wrkOutputDir3 = wrkOutputDir2 + datestr + '.' + "{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind])) + '/'
if ckDirMk(wrkOutputDir3, logFile):
# Remove all files in Output directory if previously exists!!
for f in glob.glob(wrkOutputDir3 + '*'):
os.remove(f)
#-------------------------------
# Copy relavent files from input
# directory to output directoy
#-------------------------------
#-----------------------------------
# Copy control file to Output folder
# First check if location to copy ctl is
# the same location as original ctl file
#-----------------------------------
try:
shutil.copyfile(mainInF.inputs['ctlList'][ctl_ind][0],
wrkOutputDir3 + 'sfit4.ctl')
except IOError:
print 'Unable to copy template ctl file to working directory: %s' % wrkOutputDir3
if logFile:
logFile.critical(
'Unable to copy template ctl file to working directory: %s'
% wrkOutputDir3)
sys.exit()
#-------------------------------------
# Copy sb.ctl file to output directory
# if error analysis is chosen
#-------------------------------------
if mainInF.inputs['errFlg']:
try:
shutil.copyfile(mainInF.inputs['sbCtlFile'],
wrkOutputDir3 + 'sb.ctl')
except IOError:
print 'Unable to copy template sb.ctl file to working directory: %s' % wrkOutputDir3
if logFile:
logFile.critical(
'Unable to copy template sb.ctl file to working directory: %s'
% wrkOutputDir3)
sys.exit()
#----------------------------------
# Copy hbin details to output folder
# ** Assuming that the hbin.dtl and
# hbin.input files are in the same
# location as the global ctl file
#----------------------------------
try:
shutil.copyfile(ctlPath + '/hbin.dtl', wrkOutputDir3 +
'/hbin.dtl') # Copy hbin.dtl file
except IOError:
print 'Unable to copy file: %s' % (ctlPath +
'/hbin.dtl')
if logFile: logFile.error(IOError)
try:
shutil.copyfile(ctlPath + '/hbin.input',
wrkOutputDir3 +
'/hbin.input') # Copy hbin.input file
except IOError:
print 'Unable to copy file: %s' % (ctlPath +
'/hbin.input')
if logFile: logFile.error(IOError)
# Create instance of local control file (ctl file in working directory)
ctlFileLcl = sc.CtlInputFile(wrkOutputDir3 + 'sfit4.ctl',
logFile)
#-------------------------------------------------
# Determine whether to use ILS file. Empty string
# '' => no ILS file.
#-------------------------------------------------
if mainInF.inputs['ilsDir'] and mainInF.inputs['ilsFlg']:
#-------------------------------------------
# Determine if ilsDir is a file or directory
#-------------------------------------------
# If directory.....
if os.path.isdir(mainInF.inputs['ilsDir']):
# Determine which ILS file to use
ilsFileList = glob.glob(mainInF.inputs['ilsDir'] +
'ils*')
# Create a date list of ils files present
ilsYYYYMMDD = []
for ilsFile in ilsFileList:
ilsFileNpath = os.path.basename(ilsFile)
match = re.match(
r'\s*ils(\d\d\d\d)(\d\d)(\d\d).*',
ilsFileNpath)
ilsYYYYMMDD.append([
int(match.group(1)),
int(match.group(2)),
int(match.group(3))
])
ilsDateList = [
dt.date(ilsyear, ilsmonth, ilsday)
for ilsyear, ilsmonth, ilsday in ilsYYYYMMDD
]
# Find the ils date nearest to the current day
nearstDay = sc.nearestDate(ilsDateList,
obsDay.year,
obsDay.month,
obsDay.day)
nearstDayMnth = "{0:02d}".format(nearstDay.month)
nearstDayYr = "{0:02d}".format(nearstDay.year)
nearstDayDay = "{0:02d}".format(nearstDay.day)
nearstDaystr = nearstDayYr + nearstDayMnth + nearstDayDay
# Get File path and name for nearest ils file
for ilsFile in ilsFileList:
if nearstDaystr in os.path.basename(ilsFile):
ilsFname = ilsFile
# If file.....
elif os.path.isfile(mainInF.inputs['ilsDir']):
ilsFname = mainInF.inputs['ilsDir']
if logFile: logFile.info('Using ils file: ' + ilsFname)
# Replace ils file name in local ctl file (within working directory)
teststr = [
r'file.in.modulation_fcn', r'file.in.phase_fcn'
]
repVal = [ilsFname, ilsFname]
ctlFileLcl.replVar(teststr, repVal)
# Write FOV from spectral database file to ctl file (within working directory)
ctlFileLcl.replVar([r'band\.\d+\.omega'],
[str(specDBone['FOV'])])
#---------------------------
# Message strings for output
#---------------------------
msgstr1 = mainInF.inputs['ctlList'][ctl_ind][0]
msgstr2 = datestr + '.' + "{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind]))
#----------------------------#
# #
# --- Run pspec--- #
# #
#----------------------------#
if mainInF.inputs['pspecFlg']:
print '*****************************************************'
print 'Running PSPEC for ctl file: %s' % msgstr1
print 'Processing spectral observation date: %s' % msgstr2
print '*****************************************************'
rtn = t15ascPrep(dbFltData_2, wrkInputDir2,
wrkOutputDir3, mainInF, spcDBind,
ctl_ind, logFile)
if logFile:
logFile.info('Ran PSPEC for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#----------------------------#
# #
# --- Run Refmaker--- #
# #
#----------------------------#
if mainInF.inputs['refmkrFlg']:
#-------------
# Run Refmaker
#-------------
print '*****************************************************'
print 'Running REFMKRNCAR for ctl file: %s' % msgstr1
print 'Processing spectral observation date: %s' % msgstr2
print '*****************************************************'
rtn = refMkrNCAR(wrkInputDir2, mainInF.inputs['WACCMfile'], wrkOutputDir3, \
mainInF.inputs['refMkrLvl'], mainInF.inputs['wVer'], mainInF.inputs['zptFlg'],\
dbFltData_2, spcDBind, logFile)
if logFile:
logFile.info('Ran REFMKRNCAR for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#----------------------------#
# #
# --- Run sfit4--- #
# #
#----------------------------#
#--------------
# Call to sfit4
#--------------
if mainInF.inputs['sfitFlg']:
print '*****************************************************'
print 'Running SFIT4 for ctl file: %s' % msgstr1
print 'Processing spectral observation date: %s' % msgstr2
print 'Ouput Directory: %s' % wrkOutputDir3
print '*****************************************************'
if logFile:
logFile.info('Ran SFIT4 for ctl file: %s' %
msgstr1)
logFile.info(
'Processed spectral observation date: %s' %
msgstr2)
#------------------------------
# Change working directory to
# output directory to run pspec
#------------------------------
try:
os.chdir(wrkOutputDir3)
except OSError as errmsg:
if logFile: logFile.error(errmsg)
sys.exit()
#---------------------
# Run sfit4 executable
#---------------------
sc.subProcRun([mainInF.inputs['binDir'] + 'sfit4'],
logFile)
#if ( stderr is None or not stderr):
#if log_flg:
#logFile.info('Finished running sfit4\n' + stdout)
#else:
#print 'Error running sfit4!!!'
#if log_flg:
#logFile.error('Error running sfit4 \n' + stdout)
#sys.exit()
#-----------------------------------
# Change permissions of all files in
# working directory
#-----------------------------------
for f in glob.glob(wrkOutputDir3 + '*'):
os.chmod(f, 0777)
#----------------------------------------------
# If succesfull run, write details to list file
#----------------------------------------------
if lstFlg:
fname = wrkOutputDir3 + 'sfit4.dtl'
cmpltFlg = False
with open(fname, 'r') as fopen:
for ind, line in enumerate(
reversed(fopen.readlines())):
if ind < 10:
if r'RDRV: DONE.' in line:
cmpltFlg = True
else:
break
if cmpltFlg and lstFile:
lstFile.info(
"{0:<13}".format(
int(dbFltData_2['Date'][spcDBind])) +
"{0:06}".format(
int(dbFltData_2['TStamp'][spcDBind])) +
' ' + wrkOutputDir3)
#----------------------------#
# #
# --- Error Analysis --- #
# #
#----------------------------#
if mainInF.inputs['errFlg']:
if logFile:
logFile.info('Ran SFIT4 for ctl file: %s' %
msgstr1)
#-----------------------------------
# Enter into Error Analysis function
#-----------------------------------
rtn = errAnalysis(ctlFileGlb, SbctlFileVars,
wrkOutputDir3, logFile)
#---------------------------
# Continuation for Pause flg
#---------------------------
if pauseFlg:
while True:
user_input = raw_input(
'Paused processing....\n Enter: 0 to exit, -1 to repeat, 1 to continue to next, 2 to continue all\n >>> '
)
try:
user_input = int(user_input)
if not any(user_input == val
for val in [-1, 0, 1, 2]):
raise ValueError
break
except ValueError:
print 'Please enter -1, 0, 1, or 2'
if user_input == 0: sys.exit() # Exit program
elif user_input == 1:
brkFlg = True # Exit while loop (Do not repeat)
elif user_input == 2: # Stop pause and exit while loop
pauseFlg = False
brkFlg = True
elif user_input == -1: # Repeat loop
brkFlg = False
# Need to implement functionality to recopy ctl file, bnr file, etc
#-----------------------
# Exit out of while loop
#-----------------------
if brkFlg: break
def main():
#-----------------------------------------------------------------------------------------
# Initialization
#-----------------------------------------------------------------------------------------
loc = 'tab' # LOCATION
gas = 'pan' # GAS
alpha = [0.1, 1, 10, 100, 1000, 10000, 100000, 1000000] # ALPHA VALUES TO TEST
#alpha = [10000]#, 1000, 1e4] # ALPHA VALUES TO TEST
TikOut = '/data1/ebaumer/'+loc.lower()+'/'+gas.lower()+'/x.'+gas.lower()+'/' # PATH TO SAVE THE TIK MATRIX
binDir = '/data/ebaumer/Code/sfit-core-code/src/' # PATH FOR THE SFIT4 SOURCE CODE
errFlg = False # ERROR ANALYSIS?
saveFlg = True # SAVE PDF FILE?
TransFlg = True # OPTIONAL = TRANSFORMATION IN CASE IN CASE NON-CONSTANT RETRIEVAL GRID (e.g., http://www.atmos-meas-tech.net/4/1943/2011/)
if saveFlg: pltFile = TikOut + 'TikOpt.pdf'
#-----------------------------------------------------------------------------------------
# START
#-----------------------------------------------------------------------------------------
ckDir(TikOut, logFlg=False, exit=True)
if TransFlg:
fileLayers = '/data/Campaign/'+loc.upper()+'/local/station.layers'
ckFile(fileLayers, exit=True)
cols, indexToName = getColumns(fileLayers, headerrow=2, delim=' ', header=True)
midpnt = np.asarray(cols['midpnt'][0:-1]).astype(np.float)
thick = np.asarray(cols['thick'][0:-1]).astype(np.float)
level = np.asarray(cols['level']).astype(np.float)
else:
if loc == 'fl0': nlayer = 44
elif loc == 'mlo': nlayer = 41
elif loc == 'tab': nlayer = 47
else:
print "nlayer is not known for your location\nModify TikOpt to accomodate nlayer"
exit()
#-----------------------------------------------------------------------------------------
# Define variable to save
#-----------------------------------------------------------------------------------------
totCol = []
dof = []
rms = []
chi2y = []
tot_rnd = []
tot_sys = []
tot_std = []
tot_smt = []
tot_msr = []
#Find Current Directory
cwd = os.getcwd()
#-----------------------------------------------------------------------------------------
# Run Sfit
#-----------------------------------------------------------------------------------------
for ai in alpha:
if TransFlg:
tikCov_Trans(np.sort(level), ai, TikOut, transflag=True,normalizeflag=False)
else:
tikCov(nlayer, ai, TikOut)
print '************************************************************************'
print 'Running sfit4 - Tikhonov approach, alpha = {0:.3f}'.format(ai)
print '************************************************************************'
rtn = sc.subProcRun( [binDir + 'sfit4'] )
wrkDir = os.getcwd()
if not(wrkDir.endswith('/')): wrkDir = wrkDir + '/'
#-------------------
# Run error analysis
#-------------------
if errFlg:
print '**********************************************************************************'
print 'Running error analysis - Tikhonov approach, alpha = {0:.3f}\n'.format(ai)
print '**********************************************************************************'
ckFile('sfit4.ctl', exit=True)
ckFile('sb.ctl', exit = True)
ctlFile = sc.CtlInputFile('sfit4.ctl')
ctlFile.getInputs()
sbCtlFile = sc.CtlInputFile('sb.ctl')
sbCtlFile.getInputs()
rtn = errAnalysis(ctlFile,sbCtlFile,wrkDir)
gas = dc.PlotData(wrkDir,wrkDir+'sfit4.ctl')
gas.readsummary()
totCol.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_RetColmn']))
rms.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_FITRMS']))
dof.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_DOFS_TRG']))
chi2y.append(np.asarray(gas.summary[gas.PrimaryGas.upper()+'_CHI_2_Y']))
if errFlg:
gas.readError(totFlg=True,sysFlg=False,randFlg=False,vmrFlg=False,avkFlg=False,KbFlg=False)
tot_rnd.append(np.array(gas.error['Total random uncertainty']))
tot_sys.append(np.array(gas.error['Total systematic uncertainty']))
tot_std.append(np.sqrt((np.array(gas.error['Total random uncertainty']))**2 + (np.array(gas.error['Total systematic uncertainty']))**2))
tot_smt.append(np.array(gas.error['Smoothing error (Ss)'])) #--> Eric's routine
#tot_smt.append(np.array(gas.error['Smoothing error (Ss, using sa)'])) #--> Bavo's routine
tot_msr.append(np.array(gas.error['Measurement error (Sm)']))
totCol = np.asarray(totCol)
dof = np.asarray(dof)
rms = np.asarray(rms)
chi2y = np.asarray(chi2y)
if errFlg:
tot_rnd = np.asarray(tot_rnd)
tot_sys = np.asarray(tot_sys)
tot_std = np.asarray(tot_std)
tot_smt = np.asarray(tot_smt)
tot_msr = np.asarray(tot_msr)
#-------------------
# Plots
#-------------------
print '**********************'
print ' Plots '
print '**********************'
if saveFlg: pdfsav = PdfPages(pltFile)
if errFlg: fig, ( (ax, ax2), (ax3, ax4) ) = plt.subplots(2, 2, sharex=True, figsize=(10, 8))
else: fig, (ax, ax2, ax3) = plt.subplots(3, 1, sharex=True)
#-------------------Total Column-------------------
ax.plot(alpha,totCol,label='totCol')
ax.scatter(alpha,totCol)
ax.grid(True,which='both')
ax.tick_params(which='both',labelsize=11)
ax.set_title('Total Column', multialignment='center')
ax.set_ylabel('Total Column [molec/cm$^2$]')
plt.xscale('log')
#-------------------rms-------------------
ax2.plot(alpha,rms,label='rms')
ax2.scatter(alpha,rms)
ax2.grid(True,which='both')
ax2.tick_params(which='both',labelsize=11)
ax2.set_title('RMS', multialignment='center')
ax2.set_ylabel('RMS [%]')
ax2.yaxis.set_major_formatter(FormatStrFormatter('%.4f'))
#-------------------dof-------------------
ax3.plot(alpha,dof,label='dof')
ax3.scatter(alpha,dof)
ax3.grid(True,which='both')
ax3.tick_params(which='both',labelsize=11)
ax3.set_title('DOF', multialignment='center')
ax3.set_ylabel('DOF')
ax3.set_xlabel('Regularization strength, alpha')
#-------------------Error-------------------
if errFlg:
ax4.plot(alpha,tot_smt, color='blue', label='smoothing')
ax4.scatter(alpha,tot_smt, color='blue')
ax4.plot(alpha,tot_msr, color='green', label='measurement')
ax4.scatter(alpha,tot_msr, color='green')
ax4.plot(alpha,np.sqrt(tot_msr**2 + tot_smt**2) , color='gray', label='Total (measurement + smoothing)')
ax4.scatter(alpha,np.sqrt(tot_msr**2 +tot_smt**2), color='gray')
ax4.grid(True,which='both')
ax4.tick_params(which='both',labelsize=11)
ax4.set_title('Error', multialignment='center')
ax4.set_ylabel('Error [%]')
ax4.set_xlabel('Regularization strength, alpha')
ax4.legend(prop={'size':10}, loc = 1)
fig.suptitle(cwd)
fig.tight_layout()
fig.subplots_adjust(left=0.1, right=0.95, top=0.9)
if saveFlg:
pdfsav.savefig(fig,dpi=200)
pdfsav.close()
else:
plt.show(block=False)
user_input = raw_input('Press any key to exit >>> ')
sys.exit()