def __init__(self, inDir, rrtmgp=False, sw=False):
"""
Read in timing.*.*.* files that were generated in either the
RRTMG or RRTMGP drivers, then calculate the moments (over the
number of iterations) for a given block size
We assume a naming convention of timing.TRIAL.BLOCK_SIZE.MODEL
underneath inDir, where TRIAL is the trial number, BLOCK_SIZE is
the block size used in the computation, and MODEL is either
RRTMG or RRTMGP
"""
utils.file_check(inDir)
self.modelStr = 'RRTMGP' if rrtmgp else 'RRTMG'
self.inFiles = sorted(glob.glob('%s/timing.*.*.%s' % \
(inDir, self.modelStr)))
self.allBlockSizes = \
np.array([os.path.basename(inFile).split('.')[2] \
for inFile in self.inFiles]).astype(int)
self.blockSizes = np.unique(self.allBlockSizes)
# wall clock search strings for the timing logs
domain = 'sw' if sw else 'lw'
if rrtmgp:
self.clockStrings = \
[' gas_optics (%s)' % domain.upper(), 'rte_%s' % domain]
else:
self.clockStrings = [' RRTMG (%s)' % domain.upper()]
def __init__(self, inArgs):
"""
- Time the difference between netCDF4 and xarray libraries
- find pressure, temperature, spectral point, and h2o (if
necessary) corresponding to user specifications
- print absorption coefficient at user-specified array coordinate
"""
utils.file_check(inArgs['ncFile'])
self.ncFile = str(inArgs['ncFile'])
self.userP = float(inArgs['in_pressure'])
self.userT = float(inArgs['in_temp'])
self.userH2O = float(inArgs['in_h2o'])
self.userO2 = float(inArgs['in_o2'])
self.molName = os.path.basename(self.ncFile).split('_')[0]
self.h2o = True if self.molName in ['O2', 'CO2', 'N2'] else False
self.o2 = True if self.molName == 'O2' else False
# ironically, this cannot be frequency
freq = float(inArgs['in_spectral'][0])
units = inArgs['in_spectral'][1]
if units not in ['cm-1', 'um', 'nm']:
sys.exit('Please provide valid spectral unit [cm-1, um, nm]')
if units == 'um': wnConvert = 1e4
if units == 'nm': wnConvert = 1e7
self.userWN = wnConvert / freq if units in ['um', 'nm'] else \
float(freq)
# tolerance for float equality check in valueLocate() method
tol = inArgs['tolerance']
self.tol = 1e-5 if tol is None else float(tol)
def __init__(self, inDict):
"""
Not a whole lot happens in this constructor because we need to
extract information from the configuration file
We are inheriting the wrapper combineBandmerge class only for
the bandmerge() method
"""
self.iniFile = inDict['config_file']
utils.file_check(self.iniFile)
self.nBands = 16
self.nGpt = 256
self.nProf = 42
self.broadOnly = inDict['broadband_only']
# for heating rate calculations
self.heatFactor = 8.4391
# each g-point has a weight. this is in the literature but
# was also provided by Eli in
# https://rrtmgp2.slack.com/archives/D942AU7QE/p1550264049021200
# these will only be used if we're doing the by-band calculations
self.gWeights = np.array([0.1527534276, 0.1491729617, \
0.1420961469, 0.1316886544, 0.1181945205, 0.1019300893, \
0.0832767040, 0.0626720116, 0.0424925000,0.0046269894, \
0.0038279891, 0.0030260086, 0.0022199750, 0.0014140010, \
0.0005330000, 0.0000750000])
def __init__(self, inDict):
"""
Replace 1-angle flux and HR calculations with the fluxes computed
angle_optimize.py
Input
inDict -- dictionary with the following keys:
reference_nc -- string, netCDF file with fluxes. Used in
RRTMGP runs. It is overwritten with new up and down
broadband fluxes.
optmized_nc -- string, netCDF generated with
angle_optimize.py module. It contains downwelling and
upwelling g-point spectral fluxes.
mv -- boolean, move the new netCDF to the Jupyter notebook
directory for comparison with LBLRTM
notebook_path -- string, move netCDF to this path if mv is set
"""
self.refNC = inDict['reference_nc']
utils.file_check(self.refNC)
self.optNC = inDict['optimized_nc']
utils.file_check(self.optNC)
self.moveNC = inDict['mv']
self.pathNB = inDict['notebook_path']
# for heating rate calculations
self.heatFactor = 8.4391
def __init__(self, inFile):
"""
Parse the input .ini file (inFile) and return as a dictionary for
use in the rest of this module
Inputs
inFile -- string, full path to .ini file that specifies ...
Keywords
doSW -- boolean, process shortwave instead of longwave
"""
utils.file_check(inFile)
# standard library, but name depends on Python version
if sys.version_info.major < 3:
import ConfigParser
else:
import configparser as ConfigParser
# endif Python version
cParse = ConfigParser.ConfigParser()
cParse.read(inFile)
cpSections = cParse.sections()
# loop over each field (of all sections) and keep the field and
# associated value in returned object (self)
for iCPS, cps in enumerate(cpSections):
cItems = cParse.items(cps)
for item in cItems:
setattr(self, item[0], item[1])
def makeSymLinks(sources, targets):
"""
Loop over input files and make symbolic links for them
"""
for source, target in zip(sources, targets):
utils.file_check(source)
if os.path.exists(target): os.unlink(target)
os.symlink(source, target)
# end link loop
return None
def __init__(self, lineFileDir=DEFBUILDDIR, \
lnflPath=DEFLNFL, wnBounds=[0.0, 200000.0], \
tape5Dir='%s/LNFL_input_files' % DEFBUILDDIR, \
tape3Dir='%s/lnfl/TAPE3_files' % DEFBUILDDIR, \
wvIso=False):
"""
Constructor for genTAPE3 class -- where are ASCII line files,
what are the executable paths, where should output files go, etc.
Inputs
Outputs
Keywords
lineFileDir -- string, full path to line file build directory
(where construction of a new line file is done
before releasing it); it is assumed that line_file/,
line_files_By_Molecule/, and lnfl/ subdirectories exist
underneath it
lnflPath -- string, full path to LNFL executable to use
wnBounds -- float array, 2-element array of starting and ending
wavenumbers for the TAPE3 files (25 cm-1 will be added to
each bound to include "relevant" line contributions)
tape5Dir -- string, path to which TAPE5 files that will be
generated are written
tape3Dir -- string, path to which TAPE3 files that will be
generated are written
wvIso -- boolean, only process the water vapor isotopologues
(each of which has a separate line file)
"""
lfMolDir = '%s/line_files_By_Molecule' % lineFileDir
pathCheck = [lineFileDir, lfMolDir, lnflPath]
for path in pathCheck:
utils.file_check(path)
# set some attributes necessary for the rest of the processing
self.lineFileDir = str(lineFileDir)
self.pathLNFL = str(lnflPath)
self.mols = sorted(glob.glob('%s/*' % lfMolDir))
self.nMols = len(self.mols)
self.dirT5 = str(tape5Dir)
self.dirT3 = str(tape3Dir)
self.wnLims = list(wnBounds)
self.isoH2O = bool(wvIso)
# do the output dirs exist?
for path in [tape5Dir, tape3Dir]:
self.makeDirs(path)
def __init__(self, inDict):
"""
`zenodo_request.py -h`
"""
self.url = inDict['url']
self.tokenFile = inDict['access_token']
utils.file_check(self.tokenFile)
self.localDir = inDict['local_dir']
self.title = inDict['dataset_title']
self.sandbox = inDict['sandbox']
# these two attributes are always the same for uploads and are
# linked with the user account associated with the access token
# https://developers.zenodo.org/?python#requests
self.headers = {"Content-Type": "application/json"}
self.depID = inDict['id_dep']
# HTTP status codes for successful requests
self.success = [200, 201, 202]
# make sure we have data to upload
utils.file_check(self.localDir)
# upload URL will always be the same
domain = 'sandbox.zenodo' if self.sandbox else 'zenodo'
self.url = 'https://%s.org/api/deposit/depositions' % domain
# create list of dictionaries for authors/institutions
authors = inDict['creators']
affils = inDict['affiliations']
if len(authors) != len(affils):
print('Number of creators and affiliations must be equal.')
sys.exit(1)
# endif authors
# one dictionary per author
self.creators = []
for c, a in zip(authors, affils):
self.creators.append({'name': c, 'affiliation': a})
# all uploads will probably be datasets, but eventually we can
# add more flexibility
self.type = 'dataset'
# communities can be changed in the web interface
# for now, let's just throw it into the AER archive
self.communities = [{'identifier': 'aer'}]
def __init__(self, inVars, trial=1, blockSize=100):
"""
Run a driver got a given block size and rename its GPTL timing
output. Block size by default is the number of RFMIP profiles
"""
utils.file_check(inVars['exe'])
self.driverExe = str(inVars['exe'])
self.runDir = os.path.dirname(self.driverExe)
self.iter = int(trial)
self.blockSize = int(blockSize)
self.topDir = os.getcwd()
# making some assumptions here regarding driver name
self.model = 'RRTMGP' if 'rrtmgp' in self.driverExe else 'RRTMG'
def __init__(self, inDict):
"""
Read atmospheric specifications, write pressure ASCII files for
each profile, select standard atmosphere, then write a CSV with
O2 profiles in them for the profile pressures
"""
self.ncFile = inDict['nc_file']
self.molCSV = inDict['csvMol']
self.xsCSV = inDict['csvXS']
inFiles = [self.ncFile, self.molCSV, self.xsCSV]
for inFile in inFiles:
utils.file_check(inFile)
self.outDir = inDict['out_dir']
if not os.path.exists(self.outDir): os.makedirs(self.outDir)
def __init__(self, inArgs, lnfl=False, lbl=False, lines=False):
"""
Build models as needed (one at a time) and replace paths in
ABSCO_tables.py configuration file if specified
"""
errOne = 'Please specify one of lnfl, lbl, or lines'
modBool = np.array([lnfl, lbl, lines])
if np.all(modBool): sys.exit(errOne)
if np.where(modBool)[0].size > 1: sys.exit(errOne)
# preprocessing
compiler = inArgs['compiler']
compiler = compiler.lower()
if compiler not in ['ifort', 'gfortran', 'pgf90']:
sys.exit('%s is not a supported compiler.' % compiler)
iniFile = args.config_file
if iniFile is not None: utils.file_check(iniFile)
if lnfl:
lnflDir = args.lnfl_path; utils.file_check(lnflDir)
if lbl:
lblDir = args.lblrtm_path; utils.file_check(lblDir)
if lines:
linesDir = args.lines_path; utils.file_check(linesDir)
self.compiler = str(compiler)
self.iniFile = None if iniFile is None else str(iniFile)
self.lines = False
self.topDir = str(inArgs['top_dir'])
# LNFL: always single precision, LBLRTM: always double
if lnfl:
self.modelDir = str(lnflDir)
self.modelStr = 'LNFL'
self.doLNFL = True
self.precision = 'sgl'
self.makeStr = 'make_lnfl'
self.pathStr = 'lnfl_path'
elif lbl:
self.modelDir = str(lblDir)
self.modelStr = 'LBLRTM'
self.doLBL = True
self.precision = 'dbl'
self.makeStr = 'make_lblrtm'
self.pathStr = 'lbl_path'
elif lines:
self.modelDir = str(linesDir)
self.pathStr = ['tape1_path', 'tape2_path', 'extra_params', \
'xs_path', 'fscdxs']
self.lines = True
else:
sys.exit('No model build chosen')
def srfAIRS(fileSRF='/nas/ASR/RC_Release_Benchmark_Tests/AIRS/' + \
'data/srftables_051118v4.h5'):
"""
Read in and return the AIRS spectral response function from HDF5
file (amongst other parameters -- outDict output)
Input
None
Output
outDict -- dictionary with following keys:
wavenumber -- float array, wavenumbers of spectrum (NOT the
line centers; rather, the wavenumber associated with each
point in the SRF)
FWHM -- float array, FWHM at each wavenum
SRF -- float array (nWN x nFGrid)
center_line -- float array, line centers (cm-1)
Keywords
fileSRF -- string, full path to static spectral response function
(SRF) HDF5 file. this was converted from HDF4 with h4toh5
(https://ftp.hdfgroup.org/h4toh5/download.html)
"""
import h5py
utils.file_check(fileSRF)
h5Obj = h5py.File(fileSRF, 'r')
center = h5Obj['freq'][:]
srf = h5Obj['srfval'][:]
fwGrid = h5Obj['fwgrid'][:] # not entirely sure what this is...
width = h5Obj['width'][:]
h5Obj.close()
# make arrays of repeated vectors such that (nWN x 1) and
# (1 x nFGrid) can be added and multiplied to produce a full grid
# of wavenumbers associated with the SRF
nWN, nFGrid = srf.shape
centerArr = np.tile(center, (1, nFGrid))
widthArr = np.tile(width, (1, nFGrid))
fwArr = np.tile(fwGrid, (nWN, 1))
waveNum = centerArr + fwArr * widthArr
return {'wavenumber': waveNum, 'FWHM': width, 'SRF': srf, \
'center_line': center}
def __init__(self, odDir, subset, profiles, bands, totalOD=False):
"""
Generate a netCDF for a given a subset of molecules and a set of
ODInt files from LBLRTM that were generated with RTRefOD. The
file will contain an (nLay x nWN x nProf) array of optical
depths as well as pressure levels and spectral points
(wavenumbers) used
'all_molecule' subset ODs are the column-integrated ODs
other, single-molecule subsets ODs are layer ODs
Input
odDir -- string, path to OD files generated with RTRefOD
subset -- string, indicates what subset of molecules for which
to create an OD netCDF
profiles -- dictionary, output from
profile_extraction.readProfiles()
bands -- nBands x 2 array of starting and ending wavenumbers
for each band used in the RTRefOD.runLBL()
Keywords
totalOD -- boolean, compute column-integrated optical depth
at each layer (default is to just store OD for a given layer)
"""
self.odDir = str(odDir)
utils.file_check(odDir)
self.subStr = str(subset)
self.profiles = dict(profiles)
self.bands = np.array(bands)
self.totalOD = bool(totalOD)
self.nBands = bands.shape[0]
self.chunkSize = 5000
self.compressLev = 4
# put the netCDF files in a separate subdirectory
self.ncDir = 'OD_netCDF'
if not os.path.exists(self.ncDir): os.makedirs(self.ncDir)
def profO2(self, iProfile):
"""
It is assumed that O2_profiles/* were generated with the default
configuration of o2_profiles.py and thus follow a specific naming
convention. these should be in version control.
NOTE: we are using the altitudes from the O2 profiles
(standard atmosphere calculation). calcAlt() altitudes are not
trusted.
"""
import pandas as PD
fileO2 = os.path.join('O2_profiles',
'UV_O2_profile_{}.csv'.format(iProfile))
utils.file_check(fileO2)
dfO2 = PD.read_csv(fileO2)
self.levelZ = dfO2['ALT']
self.profileO2 = dfO2['O2'] * 1e6
self.profile['molecules'] = \
np.append((self.profile['molecules']), 'O2')
default=np.exp(4.6), \
help='Pressure threshold that separates troposphere and ' + \
'stratosphere.')
parser.add_argument('--band', type=int, nargs='+', \
help='Number of band to plot. Default (band=None) is all bands.')
parser.add_argument('--log_y', action='store_true', \
help='Generate a semilog-y plot.')
parser.add_argument('--broadband', action='store_true', \
help='Generate a broadband plot for profiles (stats process ' + \
'every band and broadband relatively quickly). WARNING: ' + \
'broadband supercedes band, so if this is set, the only ' + \
'result will be a broadband figure.')
parser.add_argument('--out_dir', type=str, default=os.getcwd(), \
help='Path to output directory in which PDF files are saved.')
args = parser.parse_args()
conFile = args.config_file
utils.file_check(conFile)
figOutDir = args.out_dir
utils.file_check(figOutDir)
configDict = sw_compare.parseConfig(conFile)
plotObj = refTestFluxCompare(configDict, outDir=figOutDir, \
tPauseP=args.tropopause_pressure, bands=args.band, \
yLog=args.log_y, broadband=args.broadband)
if args.plot_stats: plotObj.statPDF()
if args.plot_profiles: plotObj.profPDF()
# end main()
from FaceDetectionSSD import FaceDetectionSSD
from augmentation import augmentImage
from framsdb import FRAMSDatabase
import utils as u
import os, shutil
import cv2, numpy as np
BASE_DIR = "original"
OUTPUT_DIR = "faces"
BACKUP_DIR = "backup"
DATA_DIR = "data"
CONFIG_FILE = os.path.join(DATA_DIR, "config.txt")
u.file_check(CONFIG_FILE, "add_multiple_users.py", "Config file not found...")
configs = eval(u.read_txtfile(CONFIG_FILE)[0])
dbConfig = configs["db"]
host, user, passwd, dbname = dbConfig["host"], dbConfig["user"], dbConfig[
"passwd"], dbConfig["db"]
print("[INFO] [recognize_multi.py] DB configs file loaded...")
users = os.listdir(BASE_DIR)
db = FRAMSDatabase(host, user, passwd, dbname)
try:
detector = FaceDetectionSSD()
for user in users:
sid, sclass, sname = user.split("_")
source = os.path.join(BASE_DIR, user)
parser.add_argument('-nc', '--profile_nc', action='store_true', \
help='Write a netCDF that contains reference and test flux ' + \
'and heating rate arrays (nBand x nCol x nLevel or nLayer).')
parser.add_argument('--band', type=int, nargs='+', \
help='Number of band to plot. Default (band=None) is all bands.')
parser.add_argument('--log_y', action='store_true', \
help='Generate a semilog-y plot.')
parser.add_argument('--broad_only', action='store_true', \
help='Only generate a broadband plot.')
args = parser.parse_args()
conFile = args.config_file
pTrop = args.tropopause_pressure
if conFile:
utils.file_check(conFile)
cParse = ConfigParser.ConfigParser()
cParse.read(conFile)
cRefName = cParse.get('Plot Params', 'reference_model')
cRefMD = cParse.get('Plot Params', 'reference_description')
cTestMD = cParse.get('Plot Params', 'test_description')
cTestName = cParse.get('Plot Params', 'test_model')
aType = cParse.get('Plot Params', 'atmosphere')
refFile = cParse.get('Filename Params', 'reference_path')
testFile = cParse.get('Filename Params', 'test_path')
profPrefix = cParse.get('Filename Params', 'profiles_prefix')
statPrefix = cParse.get('Filename Params', 'stats_prefix')
#fluxPrefix = cParse.get('Filename Params', 'fluxes_prefix')
help='Reduced k-distribution netCDF file.')
parser.add_argument('--baseline',
'-b',
type=str,
default='rrtmgp-data-lw-g256-2018-12-04.nc',
help='netCDF with correct encoding.')
parser.add_argument('--shortwave', '-sw', action='store_true', \
help='Add some scalars from full SW k-distribution')
parser.add_argument('--outfile',
'-o',
default='encoded_rrtmgp-data-LW-g-red.nc',
help='Name of modified k-dist netCDF')
args = parser.parse_args()
inFile = args.infile
utils.file_check(inFile)
baseFile = args.baseline
utils.file_check(baseFile)
outFile = args.outfile
strVars = [
'gas_minor', 'gas_names', 'identifier_minor', 'minor_gases_lower',
'minor_gases_upper', 'scaling_gas_lower', 'scaling_gas_upper'
]
encode = {}
for sv in strVars:
encode[sv] = {
'zlib': True,
'complevel': 5,
'char_dim_name': 'string_len'
}
def parseConfig(inFile):
"""
Read configuration file input into main() and extract necessary
information from it
Input
inFile -- string, full path to input configuration file
Output
dictionary with the following keys:
ref -- string, full path to reference model netCDF (LBLRTM)
test -- string, full path to test model netCDF (RRTMGP)
stat -- string, prefix for stats PDF file
prof -- string, prefix for profiles PDF file
x -- string, x-axis label
y -- string, y-axis label
atm -- string, atmosphere type (e.g., Garand)
forcing -- boolean, are forcing files being compared?
"""
# extract metadata from configuration file
cParse = configparser.ConfigParser()
cParse.read(inFile)
cRefName = cParse.get('Plot Params', 'reference_model')
cRefMD = cParse.get('Plot Params', 'reference_description')
cTestMD = cParse.get('Plot Params', 'test_description')
cTestName = cParse.get('Plot Params', 'test_model')
aType = cParse.get('Plot Params', 'atmosphere')
# flux files
refFile = cParse.get('Filename Params', 'reference_path')
testFile = cParse.get('Filename Params', 'test_path')
utils.file_check(refFile); utils.file_check(testFile)
# forcing files
rForceFile = \
cParse.get('Filename Params', 'reference_force_path')
tForceFile = cParse.get('Filename Params', 'test_force_path')
if len(rForceFile) > 0 and len(tForceFile) > 0:
forcing = True
else:
forcing = False
if forcing:
utils.file_check(rForceFile); utils.file_check(tForceFile)
refFile, testFile = compare.forcingDiff(refFile, testFile, \
rForceFile, tForceFile, \
repVars=['flux_dir_dn', 'band_flux_dir_dn'])
cRefForceName = \
cParse.get('Plot Params', 'reference_forcing_model')
cRefForceMD = cParse.get('Plot Params', 'reference_description')
cTestForceName = cParse.get('Plot Params', 'test_forcing_model')
cTestForceMD = cParse.get('Plot Params', 'test_description')
xt = cRefForceName
yt = '%s - %s' % (cTestForceName, cRefForceName)
else:
xt = cRefName
yt = '%s - %s' % (cTestName, cRefName)
# endif forcing
statPrefix = cParse.get('Filename Params', 'stats_prefix')
profPrefix = cParse.get('Filename Params', 'profiles_prefix')
return {'ref': refFile, 'test': testFile, 'x': xt, 'y': yt, \
'atm': aType, 'stat': statPrefix, 'prof': profPrefix, \
'forcing': forcing}
parser.add_argument('--band', type=int, nargs='+', \
help='Number of band to plot. Default (band=None) is all bands.')
parser.add_argument('--log_y', action='store_true', \
help='Generate a semilog-y plot.')
parser.add_argument('--diffuse', action='store_true', \
help='Plot diffuse downwelling flux array instead of flux ' + \
'from direct beam.')
parser.add_argument('--broad_only', action='store_true', \
help='Only generate a broadband plot.')
parser.add_argument('--charts', action='store_true', \
help='Plot panels relevant for CHARTS study (PROFILES ONLY).')
parser.add_argument('--out_dir', type=str, default='.', \
help='Path to output directory in which PDF files are saved.')
args = parser.parse_args()
conFile = args.config_file; utils.file_check(conFile)
pTrop = args.tropopause_pressure
doDif = args.diffuse
figOutDir = args.out_dir
utils.file_check(figOutDir)
configDict = parseConfig(conFile)
# plot the test and reference upward and downward flux together
# with the heating rates (and differences for each)
inBand = None if args.band is None else np.array(args.band)-1
if args.plot_profiles:
if inBand is None:
if args.charts:
profCHARTS(configDict['ref'], configDict['test'], \
configDict['y'], tPauseP=pTrop, \
def mark_attendance(db, attendance_dict):
if len(attendance_dict.keys()) > 0:
values = []
for key in attendance_dict.keys():
for val in attendance_dict[key]:
dt = val["dt"].strftime("%Y/%m/%d")
tm = val["dt"].strftime("%H:%M:%S")
dist = val["dist"]
values.append((key, 1,dt, tm, dist))
rows = db.addAttendanceMulti(values)
print(f"{rows} inserted....")
u.file_check(DISTANCE_FILE, "recognize_multi.py", "No User exists. Add a user...")
u.file_check(LABELS_FILE, "recognize_multi.py", "User names not found...")
u.file_check(CAM_FILE, "recognize_multi.py", "Cam file not found...")
u.file_check(CONFIG_FILE, "recognize_multi.py", "Config file not found...")
annoy_object = u.load_index(DISTANCE_FILE)
print("[INFO] [recognize_multi.py] Distance file loaded...")
labels = u.read_data(LABELS_FILE)["labels"]
print("[INFO] [recognize_multi.py] Labels file loaded...")
cam_links = u.read_txtfile(CAM_FILE)
print("[INFO] [recognize_multi.py] cam file loaded...")
configs = eval(u.read_txtfile(CONFIG_FILE)[0])
import time
start_time = time.time()
# settings
dataset = "salinasA"
feature = 'raw'
classifiers = ["KNN","GaussNB","LDA","LR","KSVM","DT","RF","GB","MLR"]
train_size = 0.01
repeat_num = 1
model_selection = True
isdraw = True
if isdraw==True:
file_check(dataset)
# run
Cla_Acc_Mean,Cla_Acc_Std,Seg_Acc_Mean,Seg_Acc_Std,df_result = SHSIC(dataset,feature,classifiers,\
train_size,repeat_num,\
model_selection,isdraw)
print("---------------------------Results Summary-----------------------------")
print("Dataset: "+ dataset)
print("Feature: "+ feature)
print("CLassifier: "+ str(classifiers))
print("The classification result is:")
print(df_result)
print('The best classifier for ' + feature + ' feature is ' + str(classifiers[Seg_Acc_Mean.argmax()]) + '.')
print("Time cost is %.3f" %(time.time()-start_time))
import os, sys, argparse, glob
import subprocess as sub
sys.path.append('common')
import utils
parser = argparse.ArgumentParser(\
description='Generate a subdirectory structure for the PDFs ' + \
'written by parallel_e2e.py and extract_mean_profile.py.')
parser.add_argument('--indir', '-d', type=str, \
default='profile_plots/linear', \
help='Directory to restructure.')
args = parser.parse_args()
# grab all of the PDFs to use in restructuring
inDir = args.indir; utils.file_check(inDir)
inFiles = sorted(glob.glob('%s/*.pdf' % inDir))
nFiles = len(inFiles)
# grab the substrings from each file that indicate the coefficients
# that we used for the flux calculations
# we're making some assumptions about file naming convention here,
# namely what we use in make_config_files.py
# e.g., for Band 15: lin_coeff_pos0.50_1.50_15.pdf
coeffs = []
for pdf in inFiles:
split = pdf.split('_')
combCoeff = ''.join(coeffs)
for iSub, substr in enumerate(split):
if ('pos' in substr) or ('neg' in substr):
coeff = '%s_%s' % (substr, split[iSub+1])
def readConfig(self):
"""
Read in the configuration file parameters that will be used with
the COMPARE module
Kinda works like a second constructor
"""
# all of the valid affirmative entries for booleans in .ini file
yes = ['y', 'ye', 'yes', 't', 'tr', 'tru', 'true']
print('Reading %s' % self.iniFile)
cParse = ConfigParser.ConfigParser()
cParse.read(self.iniFile)
self.bandAvg = cParse.get('Computation', 'band_average')
self.doPWV = cParse.get('Computation', 'pwv')
self.refName = cParse.get('Plot Params', 'reference_model')
self.refDesc = cParse.get('Plot Params', 'reference_description')
self.testName = cParse.get('Plot Params', 'test_model')
self.TestDesc = cParse.get('Plot Params', 'test_description')
self.atmType = cParse.get('Plot Params', 'atmosphere')
self.yLog = cParse.get('Plot Params', 'log')
self.bands = cParse.get('Plot Params', 'bands')
self.doStats = cParse.get('Plot Params', 'stats_plots')
self.doProfs = cParse.get('Plot Params', 'prof_plots')
self.refFile = cParse.get('Filename Params', 'reference_path')
self.testFile = cParse.get('Filename Params', 'test_path')
self.profPrefix = cParse.get('Filename Params', 'profiles_prefix')
self.statPrefix = cParse.get('Filename Params', 'stats_prefix')
self.csvFile = cParse.get('Filename Params', 'coefficients_file')
self.outDir = cParse.get('Filename Params', 'output_dir')
self.exe = cParse.get('Filename Params', 'solver')
self.ncDir = cParse.get('Filename Params', 'netcdf_dir')
# handle forcing parameters
self.forcing = False
self.rForceFile = \
cParse.get('Filename Params', 'reference_force_path')
self.tForceFile = cParse.get('Filename Params', 'test_force_path')
self.rForceName = \
cParse.get('Plot Params', 'reference_forcing_model')
self.rForceDesc = \
cParse.get('Plot Params', 'reference_description')
self.tForceName = \
cParse.get('Plot Params', 'test_forcing_model')
self.tForceDesc = \
cParse.get('Plot Params', 'test_description')
if os.path.exists(self.rForceFile) and \
os.path.exists(self.tForceFile):
self.forcing = True
self.xTitle = self.rForceName
yt = '%s - %s' % (cTestForceName, cRefForceName)
self.refFile, self.testFile = \
COMPARE.forcingDiff(self.refFile, self.testFile, \
self.rForceFile, self.tForceFile)
else:
forceFile = self.rForceFile if not \
os.path.exists(self.rForceFile) else self.tForceFile
if forceFile != '':
print('Could not find %s, forcing not done' % forceFile)
# end forcing
# check that everything that is required exists
paths = [self.refFile, self.testFile, self.csvFile, \
self.exe, self.ncDir]
for path in paths:
utils.file_check(path)
if not os.path.exists(self.outDir): os.makedirs(self.outDir)
# for consistency with bandmerge() method -- the bandmerged
# file is the same as the test netCDF file
self.mergeNC = self.testFile
# standardize boolean inputs
self.doPWV = True if self.doPWV in yes else False
self.bandAvg = True if self.bandAvg in yes else False
self.yLog = True if self.yLog.lower() in yes else False
self.doStats = True if self.doStats in yes else False
self.doProfs = True if self.doProfs in yes else False
# by default, plot all bands
self.bands = np.arange(self.nBands) if self.bands == '' else \
np.array(self.bands.split()).astype(int)-1
self.xTitle = self.refName
self.yTitle = '%s - %s' % (self.testName, self.refName)
# coefficients from RRTMG, which used PWV instead of transmittance
# in the equation a0(ibnd) + a1(ibnd)*exp(a2(ibnd)*pwvcm)
if self.doPWV:
a0 = [1.66, 1.55, 1.58, 1.66, 1.54, 1.454, 1.89, 1.33, \
1.668, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66]
a1 = [0.00, 0.25, 0.22, 0.00, 0.13, 0.446, -0.10, 0.40, \
-0.006, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]
a2 = [0.00, -12.0, -11.7, 0.00, -0.72,-0.243, 0.19,-0.062, \
0.414, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]
self.coeffs = np.array([a0, a1, a2]).T
# RRTMG hard codes the secant range to be between 1.5 and 1.8
self.secMM = np.array([1.5, 1.8])
parser.add_argument('--broadband', action='store_true', \
help='Generate a broadband PDF.')
parser.add_argument('--log_y', action='store_true', \
help='Generate a semilog-y plot.')
parser.add_argument('--cores', type=int, default=4, \
help='Number of cores to use in parallelization.')
args = parser.parse_args()
stats = args.stats
profs = args.profiles
if not stats and not profs:
sys.exit('Please use --stats or --profs in arguments.')
# grab all configuration (.ini) files
inDir = args.indir
utils.file_check(inDir)
iniFiles = sorted(glob.glob('%s/*.ini' % inDir))
pTrop = args.tropopause_pressure
difFlux = args.diffuse
# for each ini file, generate a dictionary to be used in
# pool functions
inDicts = []
for iniFile in iniFiles:
if stats:
cDict = configToDict(iniFile, tPauseP=pTrop, diffuse=difFlux)
if profs:
doBands = True if args.bands else False
if doBands:
# generate a zero-offset band array from the band list
def configToDict(inFile, tPauseP=np.exp(4.6), diffuse=False, \
bands=None, broadband=False, yLog=False):
"""
Convert configuration file into a dictionary for input into either
poolStats() or poolProfs()
Input
inFile -- string, .ini file to parse and save into outDict
Output
outDict -- dictionary with the following keys:
ref_name -- string, reference model name
test_name -- string, test model name
ref_desc -- string, reference model description
test_desc -- string, test model description
ref_file -- string, path to reference model netCDF
test_file -- string, path to test model netCDF
ref_force_file -- string, path to reference model forcing netCDF
test_force_file -- string, path to test model forcing netCDF
stat_prefix -- string, prefix for statistics PDF filename
profile_prefix --string, prefix for profile PDF filenames
x_label -- string, label for x axes in plots (stats only)
y_label -- string, label for y axes in profile and stats plots
atm_type -- string, type of atmosphere provided by user into
LBLRTM to produce ref_file and test_file
tpause_pres -- float, pressure at tropopause (tPauseP value)
ini_file -- string, inFile
diffuse -- boolean, generate plots for diffuse flux rather than
of direct beam flux (diffuse keyword value)
forcing -- boolean, plot differences in ref and test forcing
The dictionary will also contain the following fields, but they
will probably only be used with the profile plots (poolProfs())
bands -- int array, zero-offset list of bands to process (only
included if profile plotting is being done)
broadband -- boolean, for profiles, make a page with broadband
parameter differences
log_y -- boolean, plot flux and HR profiles on a semilog plot
(pressure is the log axis)
Keywords
tPauseP -- float, pressure at tropopause
diffuse -- boolean, generate plots for diffuse flux rather than
of direct beam flux
bands -- int array, zero-offset list of bands to process (only
necessary with profile plotting)
broadband -- plot profiles for SW broadband
yLog -- plot profiles on a log scale on the y axis (pressure)
"""
# extract metadata from configuration file
cParse = ConfigParser.ConfigParser()
cParse.read(inFile)
cRefName = cParse.get('Plot Params', 'reference_model')
cRefMD = cParse.get('Plot Params', 'reference_description')
cTestMD = cParse.get('Plot Params', 'test_description')
cTestName = cParse.get('Plot Params', 'test_model')
aType = cParse.get('Plot Params', 'atmosphere')
# flux files
refFile = cParse.get('Filename Params', 'reference_path')
testFile = cParse.get('Filename Params', 'test_path')
utils.file_check(refFile)
utils.file_check(testFile)
# forcing files
rForceFile = \
cParse.get('Filename Params', 'reference_force_path')
tForceFile = cParse.get('Filename Params', 'test_force_path')
utils.file_check(rForceFile)
utils.file_check(tForceFile)
# forcing keyword is True if both ref and test forcing files are
# provided (still a lot of work to do here)
forcing = True if rForceFile and tForceFile else False
# if forcing:
# cRefName, cTestName = sw_compare.compare.forcingDiff(\
# cRefName, cTestName, \
# inDict['ref_force_file'], inDict['test_force_file'])
# # endif forcing
statPrefix = cParse.get('Filename Params', 'stats_prefix')
profPrefix = cParse.get('Filename Params', 'profiles_prefix')
xt = cRefName
yt = '%s - %s' % (cTestName, cRefName)
outDict = {'ref_name': cRefName, 'test_name': cTestName, \
'ref_desc': cRefMD, 'test_desc': cTestMD, \
'ref_file': refFile, 'test_file': testFile, \
'ref_force_file': rForceFile, 'test_force_file': tForceFile, \
'stat_prefix': statPrefix, 'profile_prefix': profPrefix, \
'x_label': xt, 'y_label': yt, 'atm_type': aType, \
'tpause_pres': tPauseP, 'ini_file': inFile, \
'diffuse': diffuse, 'forcing': forcing}
outDict['bands'] = bands
outDict['broadband'] = broadband
outDict['log_y'] = yLog
return outDict
parser.add_argument('--lw_template', type=str, default=REFNCLW, \
help='Full path to netCDF that will be used as a template ' + \
'on which the output LW netCDF will be based.')
parser.add_argument('--sw_template', type=str, default=REFNCSW, \
help='Full path to netCDF that will be used as a template ' + \
'on which the output SW netCDF will be based.')
parser.add_argument('-n', '--nccopy_path', type=str, \
default='/nas/project/p1770/dependencies/bin/nccopy', \
help='Full path to the nccopy executable in the C netCDF ' + \
'library (must be version 4.3.0 or newer).')
parser.add_argument('--upwelling', action='store_true', \
help='For RFMIP only -- process upwelling fluxes and not ' + \
'downwelling.')
args = parser.parse_args()
lwDir = args.lw_dir; utils.file_check(lwDir)
swDir = args.sw_dir; utils.file_check(swDir)
ncMode = args.mode.lower()
if ncMode not in MODES: sys.exit('Set mode to any of %s' % MODES)
ncTempLW = args.lw_template; utils.file_check(ncTempLW)
ncTempSW = args.sw_template; utils.file_check(ncTempSW)
ncCopy = args.nccopy_path; utils.file_check(ncCopy)
if ncMode == 'garand':
# LW
# rrtmgObj = rrtmg(lwDir, searchStr=args.search, profiles=ncMode, \
# ncTemplate=ncTempLW, suffix=args.suffix, ncCopyPath=ncCopy)
# rrtmgObj.writeNC()
# print('LW done')
def __init__(self, inFile, molActiveCheck=True, prompt_user=True):
"""
Parse the input .ini file (inFile) and return as an object for
use in makeABSCO class. Also do some error checking and other
processes that are not directly related to running LNFL and LBLRTM
Inputs
inFile -- string, full path to .ini file that specifies paths
and filenames for...
Keywords
molActiveCheck -- boolean, run a test to determine for which
bands LNFL and LBLRTM should be run based on the user-input
ranges and whether the given molecules are active in the bands
"""
# Allow turning of interactive prompts for batch processing
self.prompt_user = prompt_user
# extract everything from config (.ini) file (inFile)
self.configFile = str(inFile)
self.readConfig()
self.checkAtts()
# these intermediate subdirs should be underneath intdir,
# which can be an another file system
self.lnfl_run_dir = os.path.join(self.intdir, self.lnfl_run_dir)
self.lbl_run_dir = os.path.join(self.intdir, self.lbl_run_dir)
self.tape3_dir = os.path.join(self.intdir, self.tape3_dir)
self.outdir = os.path.join(self.intdir, self.outdir)
# these guys should always be in the "source" directory -- the
# directory into which the ABSCO repo is cloned, which is assumed
# to be the working dir
gitDir = os.path.dirname(__file__)
self.pfile = os.path.join(gitDir, self.pfile)
self.ptfile = os.path.join(gitDir, self.ptfile)
self.vmrfile = os.path.join(gitDir, self.vmrfile)
self.xs_lines = os.path.join(gitDir, self.xs_lines)
# let's pack all of the files into a single list
self.paths = [self.pfile, self.ptfile, self.vmrfile, \
self.extra_params, self.lnfl_path, \
self.lbl_path, self.xs_path, self.fscdxs, self.xs_lines]
self.outDirs = [self.lnfl_run_dir, self.lbl_run_dir, \
self.tape3_dir, self.tape5_dir, self.outdir]
# make sure paths exist before proceeding
for path in self.paths: utils.file_check(path)
# cross section molecules will have to be handled differently
# from HITRAN molecules
self.xsNames = ['CCL4', 'F11', 'F12', 'F22', 'ISOP', 'PAN', 'BRO']
# these also have line parameters, but HITRAN recommends to use
# their XS coefficients in certain bands (see xsCheck() method).
self.xsLines = ['CF4', 'SO2', 'NO2', 'HNO3']
# these guys have continua that are affected by WV amount
self.molH2O = ['CO2', 'N2', 'O2', 'H2O']
# and these guys are neither HITRAN or XS molecules
self.dunno = ['HDO']
# O2 is a special case where we'll have to fix the VMR
self.vmrO2 = np.array([0.19, 0.23]) * 1e6
# read in pressures and do some quality control
self.processP()
# XS or line parameter processing?
self.xsCheck()
# verification of molecules and associated bands to process
# this should be called to limit unnecessary runs of LNFL and LBL
self.molProcess()
# spectral degradation weight calculation
self.degradeWeighting()
# was the line parameter source HITRAN or AER Line Parameter Data?
self.determineSources()
self.calcRAM()
singleAtm['level_T'] = singleAtm['level_T'][iPosAlt]
singleAtm['VMR'] = singleAtm['VMR'][:, iPosAlt]
# end surface
return singleAtm
# end singleProfile
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(\
description='Read in JPL-provided netCDF with atmospheric ' + \
'profiles')
parser.add_argument('--profiles_nc', '-p', type=str, \
default='uv_benchmark_scenes.nc', \
help='netCDF file with atmospheric conditions for a number ' + \
'of profiles.')
parser.add_argument('--index', '-i', type=int, \
help='Zero-offset index for profile of interest.')
args = parser.parse_args()
ncFile = args.profiles_nc
utils.file_check(ncFile)
atm = readProfiles(ncFile)
iProf = args.index
if iProf is not None: atm = singleProfile(atm, iProf)
# end main()
default=os.getcwd(), help='Top-level directory (used in build)')
parser.add_argument('--molecules', '-m', type=int, nargs='*', \
default=[27, 2, 6, 22, 24, 26], \
help='List of zero-offset indices that correspond to molecule ' + \
'numbers inferred from nc_file Atmosphere.Absorber.name array. ' + \
'All molecule index options can be displayed with --mol_list')
parser.add_argument('--mol_list', '-ml', action='store_true', \
help='List all available molecules and their indices to be ' + \
'used with --molecules argument, then exit. Zero-offset.')
parser.add_argument('--profiles', '-p', type=int, nargs='*', \
default=[0,1,2,3], \
help='Zero-offset indices of profiles to model.')
args = parser.parse_args()
ncFile = args.nc_file
utils.file_check(ncFile)
profiles = readProfiles(ncFile, ppmv=True)
nProf = profiles['VMR'].shape[0]
if args.mol_list:
# it is assumed all molecules have the same molecule names array
print('{0:<10s}{1:<10s}'.format('Index', 'Molecule'))
for iMol, mol in enumerate(profiles['molecules'][0]):
print('{0:<10d}{1:<10s}'.format(iMol, mol))
print('{0:<10d}{1:<10s}'.format(iMol + 1, 'All Molecules'))
print('Exiting after listing molecule indices')
sys.exit(0)
# end mol_list
# build models if necessary; start with dictionary that is necessary
# for BUILD objects (see common/build_models.py)
