def __init__(self, inDir, rrtmgp=False, sw=False): """ Read in timing.*.*.* files that were generated in either the RRTMG or RRTMGP drivers, then calculate the moments (over the number of iterations) for a given block size We assume a naming convention of timing.TRIAL.BLOCK_SIZE.MODEL underneath inDir, where TRIAL is the trial number, BLOCK_SIZE is the block size used in the computation, and MODEL is either RRTMG or RRTMGP """ utils.file_check(inDir) self.modelStr = 'RRTMGP' if rrtmgp else 'RRTMG' self.inFiles = sorted(glob.glob('%s/timing.*.*.%s' % \ (inDir, self.modelStr))) self.allBlockSizes = \ np.array([os.path.basename(inFile).split('.')[2] \ for inFile in self.inFiles]).astype(int) self.blockSizes = np.unique(self.allBlockSizes) # wall clock search strings for the timing logs domain = 'sw' if sw else 'lw' if rrtmgp: self.clockStrings = \ [' gas_optics (%s)' % domain.upper(), 'rte_%s' % domain] else: self.clockStrings = [' RRTMG (%s)' % domain.upper()]
def __init__(self, inArgs): """ - Time the difference between netCDF4 and xarray libraries - find pressure, temperature, spectral point, and h2o (if necessary) corresponding to user specifications - print absorption coefficient at user-specified array coordinate """ utils.file_check(inArgs['ncFile']) self.ncFile = str(inArgs['ncFile']) self.userP = float(inArgs['in_pressure']) self.userT = float(inArgs['in_temp']) self.userH2O = float(inArgs['in_h2o']) self.userO2 = float(inArgs['in_o2']) self.molName = os.path.basename(self.ncFile).split('_')[0] self.h2o = True if self.molName in ['O2', 'CO2', 'N2'] else False self.o2 = True if self.molName == 'O2' else False # ironically, this cannot be frequency freq = float(inArgs['in_spectral'][0]) units = inArgs['in_spectral'][1] if units not in ['cm-1', 'um', 'nm']: sys.exit('Please provide valid spectral unit [cm-1, um, nm]') if units == 'um': wnConvert = 1e4 if units == 'nm': wnConvert = 1e7 self.userWN = wnConvert / freq if units in ['um', 'nm'] else \ float(freq) # tolerance for float equality check in valueLocate() method tol = inArgs['tolerance'] self.tol = 1e-5 if tol is None else float(tol)
def __init__(self, inDict): """ Not a whole lot happens in this constructor because we need to extract information from the configuration file We are inheriting the wrapper combineBandmerge class only for the bandmerge() method """ self.iniFile = inDict['config_file'] utils.file_check(self.iniFile) self.nBands = 16 self.nGpt = 256 self.nProf = 42 self.broadOnly = inDict['broadband_only'] # for heating rate calculations self.heatFactor = 8.4391 # each g-point has a weight. this is in the literature but # was also provided by Eli in # https://rrtmgp2.slack.com/archives/D942AU7QE/p1550264049021200 # these will only be used if we're doing the by-band calculations self.gWeights = np.array([0.1527534276, 0.1491729617, \ 0.1420961469, 0.1316886544, 0.1181945205, 0.1019300893, \ 0.0832767040, 0.0626720116, 0.0424925000,0.0046269894, \ 0.0038279891, 0.0030260086, 0.0022199750, 0.0014140010, \ 0.0005330000, 0.0000750000])
def __init__(self, inDict): """ Replace 1-angle flux and HR calculations with the fluxes computed angle_optimize.py Input inDict -- dictionary with the following keys: reference_nc -- string, netCDF file with fluxes. Used in RRTMGP runs. It is overwritten with new up and down broadband fluxes. optmized_nc -- string, netCDF generated with angle_optimize.py module. It contains downwelling and upwelling g-point spectral fluxes. mv -- boolean, move the new netCDF to the Jupyter notebook directory for comparison with LBLRTM notebook_path -- string, move netCDF to this path if mv is set """ self.refNC = inDict['reference_nc'] utils.file_check(self.refNC) self.optNC = inDict['optimized_nc'] utils.file_check(self.optNC) self.moveNC = inDict['mv'] self.pathNB = inDict['notebook_path'] # for heating rate calculations self.heatFactor = 8.4391
def __init__(self, inFile): """ Parse the input .ini file (inFile) and return as a dictionary for use in the rest of this module Inputs inFile -- string, full path to .ini file that specifies ... Keywords doSW -- boolean, process shortwave instead of longwave """ utils.file_check(inFile) # standard library, but name depends on Python version if sys.version_info.major < 3: import ConfigParser else: import configparser as ConfigParser # endif Python version cParse = ConfigParser.ConfigParser() cParse.read(inFile) cpSections = cParse.sections() # loop over each field (of all sections) and keep the field and # associated value in returned object (self) for iCPS, cps in enumerate(cpSections): cItems = cParse.items(cps) for item in cItems: setattr(self, item[0], item[1])
def makeSymLinks(sources, targets): """ Loop over input files and make symbolic links for them """ for source, target in zip(sources, targets): utils.file_check(source) if os.path.exists(target): os.unlink(target) os.symlink(source, target) # end link loop return None
def __init__(self, lineFileDir=DEFBUILDDIR, \ lnflPath=DEFLNFL, wnBounds=[0.0, 200000.0], \ tape5Dir='%s/LNFL_input_files' % DEFBUILDDIR, \ tape3Dir='%s/lnfl/TAPE3_files' % DEFBUILDDIR, \ wvIso=False): """ Constructor for genTAPE3 class -- where are ASCII line files, what are the executable paths, where should output files go, etc. Inputs Outputs Keywords lineFileDir -- string, full path to line file build directory (where construction of a new line file is done before releasing it); it is assumed that line_file/, line_files_By_Molecule/, and lnfl/ subdirectories exist underneath it lnflPath -- string, full path to LNFL executable to use wnBounds -- float array, 2-element array of starting and ending wavenumbers for the TAPE3 files (25 cm-1 will be added to each bound to include "relevant" line contributions) tape5Dir -- string, path to which TAPE5 files that will be generated are written tape3Dir -- string, path to which TAPE3 files that will be generated are written wvIso -- boolean, only process the water vapor isotopologues (each of which has a separate line file) """ lfMolDir = '%s/line_files_By_Molecule' % lineFileDir pathCheck = [lineFileDir, lfMolDir, lnflPath] for path in pathCheck: utils.file_check(path) # set some attributes necessary for the rest of the processing self.lineFileDir = str(lineFileDir) self.pathLNFL = str(lnflPath) self.mols = sorted(glob.glob('%s/*' % lfMolDir)) self.nMols = len(self.mols) self.dirT5 = str(tape5Dir) self.dirT3 = str(tape3Dir) self.wnLims = list(wnBounds) self.isoH2O = bool(wvIso) # do the output dirs exist? for path in [tape5Dir, tape3Dir]: self.makeDirs(path)
def __init__(self, inDict): """ `zenodo_request.py -h` """ self.url = inDict['url'] self.tokenFile = inDict['access_token'] utils.file_check(self.tokenFile) self.localDir = inDict['local_dir'] self.title = inDict['dataset_title'] self.sandbox = inDict['sandbox'] # these two attributes are always the same for uploads and are # linked with the user account associated with the access token # https://developers.zenodo.org/?python#requests self.headers = {"Content-Type": "application/json"} self.depID = inDict['id_dep'] # HTTP status codes for successful requests self.success = [200, 201, 202] # make sure we have data to upload utils.file_check(self.localDir) # upload URL will always be the same domain = 'sandbox.zenodo' if self.sandbox else 'zenodo' self.url = 'https://%s.org/api/deposit/depositions' % domain # create list of dictionaries for authors/institutions authors = inDict['creators'] affils = inDict['affiliations'] if len(authors) != len(affils): print('Number of creators and affiliations must be equal.') sys.exit(1) # endif authors # one dictionary per author self.creators = [] for c, a in zip(authors, affils): self.creators.append({'name': c, 'affiliation': a}) # all uploads will probably be datasets, but eventually we can # add more flexibility self.type = 'dataset' # communities can be changed in the web interface # for now, let's just throw it into the AER archive self.communities = [{'identifier': 'aer'}]
def __init__(self, inVars, trial=1, blockSize=100): """ Run a driver got a given block size and rename its GPTL timing output. Block size by default is the number of RFMIP profiles """ utils.file_check(inVars['exe']) self.driverExe = str(inVars['exe']) self.runDir = os.path.dirname(self.driverExe) self.iter = int(trial) self.blockSize = int(blockSize) self.topDir = os.getcwd() # making some assumptions here regarding driver name self.model = 'RRTMGP' if 'rrtmgp' in self.driverExe else 'RRTMG'
def __init__(self, inDict): """ Read atmospheric specifications, write pressure ASCII files for each profile, select standard atmosphere, then write a CSV with O2 profiles in them for the profile pressures """ self.ncFile = inDict['nc_file'] self.molCSV = inDict['csvMol'] self.xsCSV = inDict['csvXS'] inFiles = [self.ncFile, self.molCSV, self.xsCSV] for inFile in inFiles: utils.file_check(inFile) self.outDir = inDict['out_dir'] if not os.path.exists(self.outDir): os.makedirs(self.outDir)
def __init__(self, inArgs, lnfl=False, lbl=False, lines=False): """ Build models as needed (one at a time) and replace paths in ABSCO_tables.py configuration file if specified """ errOne = 'Please specify one of lnfl, lbl, or lines' modBool = np.array([lnfl, lbl, lines]) if np.all(modBool): sys.exit(errOne) if np.where(modBool)[0].size > 1: sys.exit(errOne) # preprocessing compiler = inArgs['compiler'] compiler = compiler.lower() if compiler not in ['ifort', 'gfortran', 'pgf90']: sys.exit('%s is not a supported compiler.' % compiler) iniFile = args.config_file if iniFile is not None: utils.file_check(iniFile) if lnfl: lnflDir = args.lnfl_path; utils.file_check(lnflDir) if lbl: lblDir = args.lblrtm_path; utils.file_check(lblDir) if lines: linesDir = args.lines_path; utils.file_check(linesDir) self.compiler = str(compiler) self.iniFile = None if iniFile is None else str(iniFile) self.lines = False self.topDir = str(inArgs['top_dir']) # LNFL: always single precision, LBLRTM: always double if lnfl: self.modelDir = str(lnflDir) self.modelStr = 'LNFL' self.doLNFL = True self.precision = 'sgl' self.makeStr = 'make_lnfl' self.pathStr = 'lnfl_path' elif lbl: self.modelDir = str(lblDir) self.modelStr = 'LBLRTM' self.doLBL = True self.precision = 'dbl' self.makeStr = 'make_lblrtm' self.pathStr = 'lbl_path' elif lines: self.modelDir = str(linesDir) self.pathStr = ['tape1_path', 'tape2_path', 'extra_params', \ 'xs_path', 'fscdxs'] self.lines = True else: sys.exit('No model build chosen')
def srfAIRS(fileSRF='/nas/ASR/RC_Release_Benchmark_Tests/AIRS/' + \ 'data/srftables_051118v4.h5'): """ Read in and return the AIRS spectral response function from HDF5 file (amongst other parameters -- outDict output) Input None Output outDict -- dictionary with following keys: wavenumber -- float array, wavenumbers of spectrum (NOT the line centers; rather, the wavenumber associated with each point in the SRF) FWHM -- float array, FWHM at each wavenum SRF -- float array (nWN x nFGrid) center_line -- float array, line centers (cm-1) Keywords fileSRF -- string, full path to static spectral response function (SRF) HDF5 file. this was converted from HDF4 with h4toh5 (https://ftp.hdfgroup.org/h4toh5/download.html) """ import h5py utils.file_check(fileSRF) h5Obj = h5py.File(fileSRF, 'r') center = h5Obj['freq'][:] srf = h5Obj['srfval'][:] fwGrid = h5Obj['fwgrid'][:] # not entirely sure what this is... width = h5Obj['width'][:] h5Obj.close() # make arrays of repeated vectors such that (nWN x 1) and # (1 x nFGrid) can be added and multiplied to produce a full grid # of wavenumbers associated with the SRF nWN, nFGrid = srf.shape centerArr = np.tile(center, (1, nFGrid)) widthArr = np.tile(width, (1, nFGrid)) fwArr = np.tile(fwGrid, (nWN, 1)) waveNum = centerArr + fwArr * widthArr return {'wavenumber': waveNum, 'FWHM': width, 'SRF': srf, \ 'center_line': center}
def __init__(self, odDir, subset, profiles, bands, totalOD=False): """ Generate a netCDF for a given a subset of molecules and a set of ODInt files from LBLRTM that were generated with RTRefOD. The file will contain an (nLay x nWN x nProf) array of optical depths as well as pressure levels and spectral points (wavenumbers) used 'all_molecule' subset ODs are the column-integrated ODs other, single-molecule subsets ODs are layer ODs Input odDir -- string, path to OD files generated with RTRefOD subset -- string, indicates what subset of molecules for which to create an OD netCDF profiles -- dictionary, output from profile_extraction.readProfiles() bands -- nBands x 2 array of starting and ending wavenumbers for each band used in the RTRefOD.runLBL() Keywords totalOD -- boolean, compute column-integrated optical depth at each layer (default is to just store OD for a given layer) """ self.odDir = str(odDir) utils.file_check(odDir) self.subStr = str(subset) self.profiles = dict(profiles) self.bands = np.array(bands) self.totalOD = bool(totalOD) self.nBands = bands.shape[0] self.chunkSize = 5000 self.compressLev = 4 # put the netCDF files in a separate subdirectory self.ncDir = 'OD_netCDF' if not os.path.exists(self.ncDir): os.makedirs(self.ncDir)
def profO2(self, iProfile): """ It is assumed that O2_profiles/* were generated with the default configuration of o2_profiles.py and thus follow a specific naming convention. these should be in version control. NOTE: we are using the altitudes from the O2 profiles (standard atmosphere calculation). calcAlt() altitudes are not trusted. """ import pandas as PD fileO2 = os.path.join('O2_profiles', 'UV_O2_profile_{}.csv'.format(iProfile)) utils.file_check(fileO2) dfO2 = PD.read_csv(fileO2) self.levelZ = dfO2['ALT'] self.profileO2 = dfO2['O2'] * 1e6 self.profile['molecules'] = \ np.append((self.profile['molecules']), 'O2')
default=np.exp(4.6), \ help='Pressure threshold that separates troposphere and ' + \ 'stratosphere.') parser.add_argument('--band', type=int, nargs='+', \ help='Number of band to plot. Default (band=None) is all bands.') parser.add_argument('--log_y', action='store_true', \ help='Generate a semilog-y plot.') parser.add_argument('--broadband', action='store_true', \ help='Generate a broadband plot for profiles (stats process ' + \ 'every band and broadband relatively quickly). WARNING: ' + \ 'broadband supercedes band, so if this is set, the only ' + \ 'result will be a broadband figure.') parser.add_argument('--out_dir', type=str, default=os.getcwd(), \ help='Path to output directory in which PDF files are saved.') args = parser.parse_args() conFile = args.config_file utils.file_check(conFile) figOutDir = args.out_dir utils.file_check(figOutDir) configDict = sw_compare.parseConfig(conFile) plotObj = refTestFluxCompare(configDict, outDir=figOutDir, \ tPauseP=args.tropopause_pressure, bands=args.band, \ yLog=args.log_y, broadband=args.broadband) if args.plot_stats: plotObj.statPDF() if args.plot_profiles: plotObj.profPDF() # end main()
from FaceDetectionSSD import FaceDetectionSSD from augmentation import augmentImage from framsdb import FRAMSDatabase import utils as u import os, shutil import cv2, numpy as np BASE_DIR = "original" OUTPUT_DIR = "faces" BACKUP_DIR = "backup" DATA_DIR = "data" CONFIG_FILE = os.path.join(DATA_DIR, "config.txt") u.file_check(CONFIG_FILE, "add_multiple_users.py", "Config file not found...") configs = eval(u.read_txtfile(CONFIG_FILE)[0]) dbConfig = configs["db"] host, user, passwd, dbname = dbConfig["host"], dbConfig["user"], dbConfig[ "passwd"], dbConfig["db"] print("[INFO] [recognize_multi.py] DB configs file loaded...") users = os.listdir(BASE_DIR) db = FRAMSDatabase(host, user, passwd, dbname) try: detector = FaceDetectionSSD() for user in users: sid, sclass, sname = user.split("_") source = os.path.join(BASE_DIR, user)
parser.add_argument('-nc', '--profile_nc', action='store_true', \ help='Write a netCDF that contains reference and test flux ' + \ 'and heating rate arrays (nBand x nCol x nLevel or nLayer).') parser.add_argument('--band', type=int, nargs='+', \ help='Number of band to plot. Default (band=None) is all bands.') parser.add_argument('--log_y', action='store_true', \ help='Generate a semilog-y plot.') parser.add_argument('--broad_only', action='store_true', \ help='Only generate a broadband plot.') args = parser.parse_args() conFile = args.config_file pTrop = args.tropopause_pressure if conFile: utils.file_check(conFile) cParse = ConfigParser.ConfigParser() cParse.read(conFile) cRefName = cParse.get('Plot Params', 'reference_model') cRefMD = cParse.get('Plot Params', 'reference_description') cTestMD = cParse.get('Plot Params', 'test_description') cTestName = cParse.get('Plot Params', 'test_model') aType = cParse.get('Plot Params', 'atmosphere') refFile = cParse.get('Filename Params', 'reference_path') testFile = cParse.get('Filename Params', 'test_path') profPrefix = cParse.get('Filename Params', 'profiles_prefix') statPrefix = cParse.get('Filename Params', 'stats_prefix') #fluxPrefix = cParse.get('Filename Params', 'fluxes_prefix')
help='Reduced k-distribution netCDF file.') parser.add_argument('--baseline', '-b', type=str, default='rrtmgp-data-lw-g256-2018-12-04.nc', help='netCDF with correct encoding.') parser.add_argument('--shortwave', '-sw', action='store_true', \ help='Add some scalars from full SW k-distribution') parser.add_argument('--outfile', '-o', default='encoded_rrtmgp-data-LW-g-red.nc', help='Name of modified k-dist netCDF') args = parser.parse_args() inFile = args.infile utils.file_check(inFile) baseFile = args.baseline utils.file_check(baseFile) outFile = args.outfile strVars = [ 'gas_minor', 'gas_names', 'identifier_minor', 'minor_gases_lower', 'minor_gases_upper', 'scaling_gas_lower', 'scaling_gas_upper' ] encode = {} for sv in strVars: encode[sv] = { 'zlib': True, 'complevel': 5, 'char_dim_name': 'string_len' }
def parseConfig(inFile): """ Read configuration file input into main() and extract necessary information from it Input inFile -- string, full path to input configuration file Output dictionary with the following keys: ref -- string, full path to reference model netCDF (LBLRTM) test -- string, full path to test model netCDF (RRTMGP) stat -- string, prefix for stats PDF file prof -- string, prefix for profiles PDF file x -- string, x-axis label y -- string, y-axis label atm -- string, atmosphere type (e.g., Garand) forcing -- boolean, are forcing files being compared? """ # extract metadata from configuration file cParse = configparser.ConfigParser() cParse.read(inFile) cRefName = cParse.get('Plot Params', 'reference_model') cRefMD = cParse.get('Plot Params', 'reference_description') cTestMD = cParse.get('Plot Params', 'test_description') cTestName = cParse.get('Plot Params', 'test_model') aType = cParse.get('Plot Params', 'atmosphere') # flux files refFile = cParse.get('Filename Params', 'reference_path') testFile = cParse.get('Filename Params', 'test_path') utils.file_check(refFile); utils.file_check(testFile) # forcing files rForceFile = \ cParse.get('Filename Params', 'reference_force_path') tForceFile = cParse.get('Filename Params', 'test_force_path') if len(rForceFile) > 0 and len(tForceFile) > 0: forcing = True else: forcing = False if forcing: utils.file_check(rForceFile); utils.file_check(tForceFile) refFile, testFile = compare.forcingDiff(refFile, testFile, \ rForceFile, tForceFile, \ repVars=['flux_dir_dn', 'band_flux_dir_dn']) cRefForceName = \ cParse.get('Plot Params', 'reference_forcing_model') cRefForceMD = cParse.get('Plot Params', 'reference_description') cTestForceName = cParse.get('Plot Params', 'test_forcing_model') cTestForceMD = cParse.get('Plot Params', 'test_description') xt = cRefForceName yt = '%s - %s' % (cTestForceName, cRefForceName) else: xt = cRefName yt = '%s - %s' % (cTestName, cRefName) # endif forcing statPrefix = cParse.get('Filename Params', 'stats_prefix') profPrefix = cParse.get('Filename Params', 'profiles_prefix') return {'ref': refFile, 'test': testFile, 'x': xt, 'y': yt, \ 'atm': aType, 'stat': statPrefix, 'prof': profPrefix, \ 'forcing': forcing}
parser.add_argument('--band', type=int, nargs='+', \ help='Number of band to plot. Default (band=None) is all bands.') parser.add_argument('--log_y', action='store_true', \ help='Generate a semilog-y plot.') parser.add_argument('--diffuse', action='store_true', \ help='Plot diffuse downwelling flux array instead of flux ' + \ 'from direct beam.') parser.add_argument('--broad_only', action='store_true', \ help='Only generate a broadband plot.') parser.add_argument('--charts', action='store_true', \ help='Plot panels relevant for CHARTS study (PROFILES ONLY).') parser.add_argument('--out_dir', type=str, default='.', \ help='Path to output directory in which PDF files are saved.') args = parser.parse_args() conFile = args.config_file; utils.file_check(conFile) pTrop = args.tropopause_pressure doDif = args.diffuse figOutDir = args.out_dir utils.file_check(figOutDir) configDict = parseConfig(conFile) # plot the test and reference upward and downward flux together # with the heating rates (and differences for each) inBand = None if args.band is None else np.array(args.band)-1 if args.plot_profiles: if inBand is None: if args.charts: profCHARTS(configDict['ref'], configDict['test'], \ configDict['y'], tPauseP=pTrop, \
def mark_attendance(db, attendance_dict): if len(attendance_dict.keys()) > 0: values = [] for key in attendance_dict.keys(): for val in attendance_dict[key]: dt = val["dt"].strftime("%Y/%m/%d") tm = val["dt"].strftime("%H:%M:%S") dist = val["dist"] values.append((key, 1,dt, tm, dist)) rows = db.addAttendanceMulti(values) print(f"{rows} inserted....") u.file_check(DISTANCE_FILE, "recognize_multi.py", "No User exists. Add a user...") u.file_check(LABELS_FILE, "recognize_multi.py", "User names not found...") u.file_check(CAM_FILE, "recognize_multi.py", "Cam file not found...") u.file_check(CONFIG_FILE, "recognize_multi.py", "Config file not found...") annoy_object = u.load_index(DISTANCE_FILE) print("[INFO] [recognize_multi.py] Distance file loaded...") labels = u.read_data(LABELS_FILE)["labels"] print("[INFO] [recognize_multi.py] Labels file loaded...") cam_links = u.read_txtfile(CAM_FILE) print("[INFO] [recognize_multi.py] cam file loaded...") configs = eval(u.read_txtfile(CONFIG_FILE)[0])
import time start_time = time.time() # settings dataset = "salinasA" feature = 'raw' classifiers = ["KNN","GaussNB","LDA","LR","KSVM","DT","RF","GB","MLR"] train_size = 0.01 repeat_num = 1 model_selection = True isdraw = True if isdraw==True: file_check(dataset) # run Cla_Acc_Mean,Cla_Acc_Std,Seg_Acc_Mean,Seg_Acc_Std,df_result = SHSIC(dataset,feature,classifiers,\ train_size,repeat_num,\ model_selection,isdraw) print("---------------------------Results Summary-----------------------------") print("Dataset: "+ dataset) print("Feature: "+ feature) print("CLassifier: "+ str(classifiers)) print("The classification result is:") print(df_result) print('The best classifier for ' + feature + ' feature is ' + str(classifiers[Seg_Acc_Mean.argmax()]) + '.') print("Time cost is %.3f" %(time.time()-start_time))
import os, sys, argparse, glob import subprocess as sub sys.path.append('common') import utils parser = argparse.ArgumentParser(\ description='Generate a subdirectory structure for the PDFs ' + \ 'written by parallel_e2e.py and extract_mean_profile.py.') parser.add_argument('--indir', '-d', type=str, \ default='profile_plots/linear', \ help='Directory to restructure.') args = parser.parse_args() # grab all of the PDFs to use in restructuring inDir = args.indir; utils.file_check(inDir) inFiles = sorted(glob.glob('%s/*.pdf' % inDir)) nFiles = len(inFiles) # grab the substrings from each file that indicate the coefficients # that we used for the flux calculations # we're making some assumptions about file naming convention here, # namely what we use in make_config_files.py # e.g., for Band 15: lin_coeff_pos0.50_1.50_15.pdf coeffs = [] for pdf in inFiles: split = pdf.split('_') combCoeff = ''.join(coeffs) for iSub, substr in enumerate(split): if ('pos' in substr) or ('neg' in substr): coeff = '%s_%s' % (substr, split[iSub+1])
def readConfig(self): """ Read in the configuration file parameters that will be used with the COMPARE module Kinda works like a second constructor """ # all of the valid affirmative entries for booleans in .ini file yes = ['y', 'ye', 'yes', 't', 'tr', 'tru', 'true'] print('Reading %s' % self.iniFile) cParse = ConfigParser.ConfigParser() cParse.read(self.iniFile) self.bandAvg = cParse.get('Computation', 'band_average') self.doPWV = cParse.get('Computation', 'pwv') self.refName = cParse.get('Plot Params', 'reference_model') self.refDesc = cParse.get('Plot Params', 'reference_description') self.testName = cParse.get('Plot Params', 'test_model') self.TestDesc = cParse.get('Plot Params', 'test_description') self.atmType = cParse.get('Plot Params', 'atmosphere') self.yLog = cParse.get('Plot Params', 'log') self.bands = cParse.get('Plot Params', 'bands') self.doStats = cParse.get('Plot Params', 'stats_plots') self.doProfs = cParse.get('Plot Params', 'prof_plots') self.refFile = cParse.get('Filename Params', 'reference_path') self.testFile = cParse.get('Filename Params', 'test_path') self.profPrefix = cParse.get('Filename Params', 'profiles_prefix') self.statPrefix = cParse.get('Filename Params', 'stats_prefix') self.csvFile = cParse.get('Filename Params', 'coefficients_file') self.outDir = cParse.get('Filename Params', 'output_dir') self.exe = cParse.get('Filename Params', 'solver') self.ncDir = cParse.get('Filename Params', 'netcdf_dir') # handle forcing parameters self.forcing = False self.rForceFile = \ cParse.get('Filename Params', 'reference_force_path') self.tForceFile = cParse.get('Filename Params', 'test_force_path') self.rForceName = \ cParse.get('Plot Params', 'reference_forcing_model') self.rForceDesc = \ cParse.get('Plot Params', 'reference_description') self.tForceName = \ cParse.get('Plot Params', 'test_forcing_model') self.tForceDesc = \ cParse.get('Plot Params', 'test_description') if os.path.exists(self.rForceFile) and \ os.path.exists(self.tForceFile): self.forcing = True self.xTitle = self.rForceName yt = '%s - %s' % (cTestForceName, cRefForceName) self.refFile, self.testFile = \ COMPARE.forcingDiff(self.refFile, self.testFile, \ self.rForceFile, self.tForceFile) else: forceFile = self.rForceFile if not \ os.path.exists(self.rForceFile) else self.tForceFile if forceFile != '': print('Could not find %s, forcing not done' % forceFile) # end forcing # check that everything that is required exists paths = [self.refFile, self.testFile, self.csvFile, \ self.exe, self.ncDir] for path in paths: utils.file_check(path) if not os.path.exists(self.outDir): os.makedirs(self.outDir) # for consistency with bandmerge() method -- the bandmerged # file is the same as the test netCDF file self.mergeNC = self.testFile # standardize boolean inputs self.doPWV = True if self.doPWV in yes else False self.bandAvg = True if self.bandAvg in yes else False self.yLog = True if self.yLog.lower() in yes else False self.doStats = True if self.doStats in yes else False self.doProfs = True if self.doProfs in yes else False # by default, plot all bands self.bands = np.arange(self.nBands) if self.bands == '' else \ np.array(self.bands.split()).astype(int)-1 self.xTitle = self.refName self.yTitle = '%s - %s' % (self.testName, self.refName) # coefficients from RRTMG, which used PWV instead of transmittance # in the equation a0(ibnd) + a1(ibnd)*exp(a2(ibnd)*pwvcm) if self.doPWV: a0 = [1.66, 1.55, 1.58, 1.66, 1.54, 1.454, 1.89, 1.33, \ 1.668, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66] a1 = [0.00, 0.25, 0.22, 0.00, 0.13, 0.446, -0.10, 0.40, \ -0.006, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00] a2 = [0.00, -12.0, -11.7, 0.00, -0.72,-0.243, 0.19,-0.062, \ 0.414, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00] self.coeffs = np.array([a0, a1, a2]).T # RRTMG hard codes the secant range to be between 1.5 and 1.8 self.secMM = np.array([1.5, 1.8])
parser.add_argument('--broadband', action='store_true', \ help='Generate a broadband PDF.') parser.add_argument('--log_y', action='store_true', \ help='Generate a semilog-y plot.') parser.add_argument('--cores', type=int, default=4, \ help='Number of cores to use in parallelization.') args = parser.parse_args() stats = args.stats profs = args.profiles if not stats and not profs: sys.exit('Please use --stats or --profs in arguments.') # grab all configuration (.ini) files inDir = args.indir utils.file_check(inDir) iniFiles = sorted(glob.glob('%s/*.ini' % inDir)) pTrop = args.tropopause_pressure difFlux = args.diffuse # for each ini file, generate a dictionary to be used in # pool functions inDicts = [] for iniFile in iniFiles: if stats: cDict = configToDict(iniFile, tPauseP=pTrop, diffuse=difFlux) if profs: doBands = True if args.bands else False if doBands: # generate a zero-offset band array from the band list
def configToDict(inFile, tPauseP=np.exp(4.6), diffuse=False, \ bands=None, broadband=False, yLog=False): """ Convert configuration file into a dictionary for input into either poolStats() or poolProfs() Input inFile -- string, .ini file to parse and save into outDict Output outDict -- dictionary with the following keys: ref_name -- string, reference model name test_name -- string, test model name ref_desc -- string, reference model description test_desc -- string, test model description ref_file -- string, path to reference model netCDF test_file -- string, path to test model netCDF ref_force_file -- string, path to reference model forcing netCDF test_force_file -- string, path to test model forcing netCDF stat_prefix -- string, prefix for statistics PDF filename profile_prefix --string, prefix for profile PDF filenames x_label -- string, label for x axes in plots (stats only) y_label -- string, label for y axes in profile and stats plots atm_type -- string, type of atmosphere provided by user into LBLRTM to produce ref_file and test_file tpause_pres -- float, pressure at tropopause (tPauseP value) ini_file -- string, inFile diffuse -- boolean, generate plots for diffuse flux rather than of direct beam flux (diffuse keyword value) forcing -- boolean, plot differences in ref and test forcing The dictionary will also contain the following fields, but they will probably only be used with the profile plots (poolProfs()) bands -- int array, zero-offset list of bands to process (only included if profile plotting is being done) broadband -- boolean, for profiles, make a page with broadband parameter differences log_y -- boolean, plot flux and HR profiles on a semilog plot (pressure is the log axis) Keywords tPauseP -- float, pressure at tropopause diffuse -- boolean, generate plots for diffuse flux rather than of direct beam flux bands -- int array, zero-offset list of bands to process (only necessary with profile plotting) broadband -- plot profiles for SW broadband yLog -- plot profiles on a log scale on the y axis (pressure) """ # extract metadata from configuration file cParse = ConfigParser.ConfigParser() cParse.read(inFile) cRefName = cParse.get('Plot Params', 'reference_model') cRefMD = cParse.get('Plot Params', 'reference_description') cTestMD = cParse.get('Plot Params', 'test_description') cTestName = cParse.get('Plot Params', 'test_model') aType = cParse.get('Plot Params', 'atmosphere') # flux files refFile = cParse.get('Filename Params', 'reference_path') testFile = cParse.get('Filename Params', 'test_path') utils.file_check(refFile) utils.file_check(testFile) # forcing files rForceFile = \ cParse.get('Filename Params', 'reference_force_path') tForceFile = cParse.get('Filename Params', 'test_force_path') utils.file_check(rForceFile) utils.file_check(tForceFile) # forcing keyword is True if both ref and test forcing files are # provided (still a lot of work to do here) forcing = True if rForceFile and tForceFile else False # if forcing: # cRefName, cTestName = sw_compare.compare.forcingDiff(\ # cRefName, cTestName, \ # inDict['ref_force_file'], inDict['test_force_file']) # # endif forcing statPrefix = cParse.get('Filename Params', 'stats_prefix') profPrefix = cParse.get('Filename Params', 'profiles_prefix') xt = cRefName yt = '%s - %s' % (cTestName, cRefName) outDict = {'ref_name': cRefName, 'test_name': cTestName, \ 'ref_desc': cRefMD, 'test_desc': cTestMD, \ 'ref_file': refFile, 'test_file': testFile, \ 'ref_force_file': rForceFile, 'test_force_file': tForceFile, \ 'stat_prefix': statPrefix, 'profile_prefix': profPrefix, \ 'x_label': xt, 'y_label': yt, 'atm_type': aType, \ 'tpause_pres': tPauseP, 'ini_file': inFile, \ 'diffuse': diffuse, 'forcing': forcing} outDict['bands'] = bands outDict['broadband'] = broadband outDict['log_y'] = yLog return outDict
parser.add_argument('--lw_template', type=str, default=REFNCLW, \ help='Full path to netCDF that will be used as a template ' + \ 'on which the output LW netCDF will be based.') parser.add_argument('--sw_template', type=str, default=REFNCSW, \ help='Full path to netCDF that will be used as a template ' + \ 'on which the output SW netCDF will be based.') parser.add_argument('-n', '--nccopy_path', type=str, \ default='/nas/project/p1770/dependencies/bin/nccopy', \ help='Full path to the nccopy executable in the C netCDF ' + \ 'library (must be version 4.3.0 or newer).') parser.add_argument('--upwelling', action='store_true', \ help='For RFMIP only -- process upwelling fluxes and not ' + \ 'downwelling.') args = parser.parse_args() lwDir = args.lw_dir; utils.file_check(lwDir) swDir = args.sw_dir; utils.file_check(swDir) ncMode = args.mode.lower() if ncMode not in MODES: sys.exit('Set mode to any of %s' % MODES) ncTempLW = args.lw_template; utils.file_check(ncTempLW) ncTempSW = args.sw_template; utils.file_check(ncTempSW) ncCopy = args.nccopy_path; utils.file_check(ncCopy) if ncMode == 'garand': # LW # rrtmgObj = rrtmg(lwDir, searchStr=args.search, profiles=ncMode, \ # ncTemplate=ncTempLW, suffix=args.suffix, ncCopyPath=ncCopy) # rrtmgObj.writeNC() # print('LW done')
def __init__(self, inFile, molActiveCheck=True, prompt_user=True): """ Parse the input .ini file (inFile) and return as an object for use in makeABSCO class. Also do some error checking and other processes that are not directly related to running LNFL and LBLRTM Inputs inFile -- string, full path to .ini file that specifies paths and filenames for... Keywords molActiveCheck -- boolean, run a test to determine for which bands LNFL and LBLRTM should be run based on the user-input ranges and whether the given molecules are active in the bands """ # Allow turning of interactive prompts for batch processing self.prompt_user = prompt_user # extract everything from config (.ini) file (inFile) self.configFile = str(inFile) self.readConfig() self.checkAtts() # these intermediate subdirs should be underneath intdir, # which can be an another file system self.lnfl_run_dir = os.path.join(self.intdir, self.lnfl_run_dir) self.lbl_run_dir = os.path.join(self.intdir, self.lbl_run_dir) self.tape3_dir = os.path.join(self.intdir, self.tape3_dir) self.outdir = os.path.join(self.intdir, self.outdir) # these guys should always be in the "source" directory -- the # directory into which the ABSCO repo is cloned, which is assumed # to be the working dir gitDir = os.path.dirname(__file__) self.pfile = os.path.join(gitDir, self.pfile) self.ptfile = os.path.join(gitDir, self.ptfile) self.vmrfile = os.path.join(gitDir, self.vmrfile) self.xs_lines = os.path.join(gitDir, self.xs_lines) # let's pack all of the files into a single list self.paths = [self.pfile, self.ptfile, self.vmrfile, \ self.extra_params, self.lnfl_path, \ self.lbl_path, self.xs_path, self.fscdxs, self.xs_lines] self.outDirs = [self.lnfl_run_dir, self.lbl_run_dir, \ self.tape3_dir, self.tape5_dir, self.outdir] # make sure paths exist before proceeding for path in self.paths: utils.file_check(path) # cross section molecules will have to be handled differently # from HITRAN molecules self.xsNames = ['CCL4', 'F11', 'F12', 'F22', 'ISOP', 'PAN', 'BRO'] # these also have line parameters, but HITRAN recommends to use # their XS coefficients in certain bands (see xsCheck() method). self.xsLines = ['CF4', 'SO2', 'NO2', 'HNO3'] # these guys have continua that are affected by WV amount self.molH2O = ['CO2', 'N2', 'O2', 'H2O'] # and these guys are neither HITRAN or XS molecules self.dunno = ['HDO'] # O2 is a special case where we'll have to fix the VMR self.vmrO2 = np.array([0.19, 0.23]) * 1e6 # read in pressures and do some quality control self.processP() # XS or line parameter processing? self.xsCheck() # verification of molecules and associated bands to process # this should be called to limit unnecessary runs of LNFL and LBL self.molProcess() # spectral degradation weight calculation self.degradeWeighting() # was the line parameter source HITRAN or AER Line Parameter Data? self.determineSources() self.calcRAM()
singleAtm['level_T'] = singleAtm['level_T'][iPosAlt] singleAtm['VMR'] = singleAtm['VMR'][:, iPosAlt] # end surface return singleAtm # end singleProfile if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(\ description='Read in JPL-provided netCDF with atmospheric ' + \ 'profiles') parser.add_argument('--profiles_nc', '-p', type=str, \ default='uv_benchmark_scenes.nc', \ help='netCDF file with atmospheric conditions for a number ' + \ 'of profiles.') parser.add_argument('--index', '-i', type=int, \ help='Zero-offset index for profile of interest.') args = parser.parse_args() ncFile = args.profiles_nc utils.file_check(ncFile) atm = readProfiles(ncFile) iProf = args.index if iProf is not None: atm = singleProfile(atm, iProf) # end main()
default=os.getcwd(), help='Top-level directory (used in build)') parser.add_argument('--molecules', '-m', type=int, nargs='*', \ default=[27, 2, 6, 22, 24, 26], \ help='List of zero-offset indices that correspond to molecule ' + \ 'numbers inferred from nc_file Atmosphere.Absorber.name array. ' + \ 'All molecule index options can be displayed with --mol_list') parser.add_argument('--mol_list', '-ml', action='store_true', \ help='List all available molecules and their indices to be ' + \ 'used with --molecules argument, then exit. Zero-offset.') parser.add_argument('--profiles', '-p', type=int, nargs='*', \ default=[0,1,2,3], \ help='Zero-offset indices of profiles to model.') args = parser.parse_args() ncFile = args.nc_file utils.file_check(ncFile) profiles = readProfiles(ncFile, ppmv=True) nProf = profiles['VMR'].shape[0] if args.mol_list: # it is assumed all molecules have the same molecule names array print('{0:<10s}{1:<10s}'.format('Index', 'Molecule')) for iMol, mol in enumerate(profiles['molecules'][0]): print('{0:<10d}{1:<10s}'.format(iMol, mol)) print('{0:<10d}{1:<10s}'.format(iMol + 1, 'All Molecules')) print('Exiting after listing molecule indices') sys.exit(0) # end mol_list # build models if necessary; start with dictionary that is necessary # for BUILD objects (see common/build_models.py)