def check_args_cc4cl(args): """Ensure ORAC suite wrapper parser arguments are valid.""" from os import makedirs from os.path import isdir, join from pyorac.local_defaults import log_dir, extra_lines, retrieval_settings # Add extra lines files extra_lines.update({key + '_extra': val for key, val in args.extra_lines}) args.__dict__.update(extra_lines) log_path = join(args.out_dir, log_dir) if args.batch and not isdir(log_path): makedirs(log_path, 0o774) if args.preset_settings is not None: try: args.settings = retrieval_settings[args.preset_settings] except KeyError: raise BadValue("preset settings", "not defined in local_defaults") elif args.settings_file is not None: try: with open(args.settings_file) as settings_file: args.settings = settings_file.read().splitlines() except IOError: raise FileMissing('Description of settings', args.settings_file) elif args.settings is None: args.settings = retrieval_settings[args.File.sensor]
def check_args_common(args): """Ensure common parser arguments are valid.""" from os import makedirs from os.path import dirname, basename from pyorac.definitions import FileName from pyorac.local_defaults import channels # If not explicitly given, assume input folder is in target definition if args.in_dir is None: args.in_dir = [ dirname(args.target), ] args.target = basename(args.target) else: if "/" in args.target: raise BadValue("file target", "contains a /") if args.out_dir is None: args.out_dir = args.in_dir[0] args.File = FileName(args.in_dir, args.target) if args.available_channels is None: args.available_channels = channels[args.File.sensor] for d in args.in_dir: if not isdir(d): raise FileMissing('in_dir', d) if not isdir(args.out_dir): makedirs(args.out_dir, 0o774) if not isdir(args.orac_dir): raise FileMissing('ORAC repository directory', args.orac_dir) if not isfile(args.orac_lib): raise FileMissing('ORAC library file', args.orac_lib)
def check_args_cc4cl(args): """Ensure ORAC suite wrapper parser arguments are valid.""" from pyorac.local_defaults import log_dir, extra_lines, retrieval_settings # Add extra lines files extra_lines.update({key + '_extra': val for key, val in args.extra_lines}) args.__dict__.update(extra_lines) log_path = os.path.join(args.out_dir, log_dir) if args.batch and not os.path.isdir(log_path): os.makedirs(log_path, 0o774) if args.settings_file is not None: # A procedure outlined in a file try: with open(args.settings_file) as settings_file: args.settings = settings_file.read().splitlines() except IOError: raise FileMissing('Description of settings', args.settings_file) elif args.preset_settings is not None: # A procedure named in local_defaults try: args.settings = retrieval_settings[args.preset_settings] except KeyError: raise BadValue("preset settings", "not defined in local_defaults") elif args.settings is None: if args.phase is None: # Default procedure for this sensor from local_defaults args.settings = retrieval_settings[args.File.sensor] else: # Process a single type args.settings = (" ", ) return args
def build_preproc_driver(args): """Prepare a driver file for the preprocessor.""" from itertools import product from re import search from subprocess import CalledProcessError, check_output, STDOUT from uuid import uuid4 from pyorac.definitions import BadValue from pyorac.util import (build_orac_library_path, extract_orac_libraries, read_orac_library_file) l1b = _glob_dirs(args.in_dir, args.File.l1b, 'L1B file') geo = _glob_dirs(args.in_dir, args.File.geo, 'geolocation file') # Select NISE file if args.use_ecmwf_snow or args.no_snow_corr: nise = '' else: # There are usually too many files in this directory to glob quickly. # Instead, guess where it is. If your search is failing here, but the # appropriate file is present, you need to add a format to one of # these loops that finds your file. nise_locations = ( 'NISE.005/%Y.%m.%d', 'NISE.004/%Y.%m.%d', 'NISE.002/%Y.%m.%d', 'NISE.001/%Y.%m.%d', '%Y', '%Y.%m.%d', '%Y_%m_d', '%Y-%m-%d', '' ) nise_formats = ( 'NISE_SSMISF18_%Y%m%d.HDFEOS', 'NISE_SSMISF17_%Y%m%d.HDFEOS', 'NISE_SSMIF13_%Y%m%d.HDFEOS', ) for nise_location, nise_format in product(nise_locations, nise_formats): nise = args.File.time.strftime(os.path.join( args.nise_dir, nise_location, nise_format )) if os.path.isfile(nise): break else: raise FileMissing('NISE', args.nise_dir) # Select previous surface reflectance and emissivity files if args.swansea: alb = _date_back_search(args.swansea_dir, args.File.time, 'SW_SFC_PRMS_%m.nc', 'years') brdf = None else: for ver in (61, 6, 5): try: alb = _date_back_search(args.mcd43c3_dir, args.File.time, f'MCD43C3.A%Y%j.{ver:03d}.*.hdf', 'days') brdf = None if args.lambertian else _date_back_search( args.mcd43c1_dir, args.File.time, f'MCD43C1.A%Y%j.{ver:03d}.*.hdf', 'days' ) break except FileMissing: pass else: raise FileMissing('MODIS albedo', args.mcd43c3_dir) if args.use_modis_emis: emis = None elif args.use_camel_emis: emis = _date_back_search( args.camel_dir, args.File.time, 'CAM5K30EM_emis_%Y%m_V???.nc', 'years' ) else: emis = _date_back_search( args.emis_dir, args.File.time, 'global_emis_inf10_monthFilled_MYD11C3.A%Y%j.*nc', 'days' ) # Select ECMWF files bounds = _bound_time(args.File.time + args.File.dur // 2) if args.nwp_flag == 0: ecmwf_nlevels = 91 raise NotImplementedError('Filename syntax for --nwp_flag 0 unknown') elif args.nwp_flag == 4: ecmwf_nlevels = 60 ggam = _form_bound_filenames(bounds, args.ggam_dir, 'ggam%Y%m%d%H%M.grb') ggas = _form_bound_filenames(bounds, args.ggas_dir, 'ggas%Y%m%d%H%M.nc') spam = _form_bound_filenames(bounds, args.spam_dir, 'spam%Y%m%d%H%M.grb') elif args.nwp_flag == 3: ecmwf_nlevels = 60 raise NotImplementedError('Filename syntax for --nwp_flag 3 unknown') elif args.nwp_flag == 1: ecmwf_nlevels = 137 for form, ec_hour in (('C3D*%m%d%H*.nc', 3), ('ECMWF_OPER_%Y%m%d_%H+00.nc', 6), ('ECMWF_ERA5_%Y%m%d_%H_0.5.nc', 6), ('ECMWF_ERA_%Y%m%d_%H_0.5.nc', 6), ('ECMWF_ERA_%Y%m%d_%H+00_0.5.nc', 6)): try: bounds = _bound_time(args.File.time + args.File.dur // 2, ec_hour) ggam = _form_bound_filenames(bounds, args.ecmwf_dir, form) break except FileMissing as tmp_err: err = tmp_err else: raise err ggas = ["", ""] spam = ["", ""] elif args.nwp_flag == 2: ecmwf_nlevels = 137 # Interpolation is done in the code ggam = [args.ecmwf_dir, args.ecmwf_dir] ggas = ["", ""] spam = ["", ""] else: raise BadValue('nwp_flag', args.nwp_flag) if args.use_oc: for oc_version in (5.0, 4.2, 4.1, 4.0, 3.1, 3.0, 2.0, 1.0): occci = args.File.time.strftime(os.path.join( args.occci_dir, 'ESACCI-OC-L3S-IOP-MERGED-1M_MONTHLY' f'_4km_GEO_PML_OCx_QAA-%Y%m-fv{oc_version:.1f}.nc' )) if os.path.isfile(occci): break else: raise FileMissing('Ocean Colour CCI', occci) else: occci = '' # ------------------------------------------------------------------------ if args.uuid: uid = str(uuid4()) else: uid = 'n/a' libs = read_orac_library_file(args.orac_lib) lib_list = extract_orac_libraries(libs) os.environ["LD_LIBRARY_PATH"] = build_orac_library_path(lib_list=lib_list) # Determine current time production_time = datetime.now().strftime("%Y%m%d%H%M%S") # Determine NCDF version from command line for fdr in lib_list: ncdf_exe = os.path.join(fdr, "..", "bin", "ncdump") try: tmp0 = check_output(ncdf_exe, stderr=STDOUT, universal_newlines=True) except FileNotFoundError: continue except CalledProcessError: raise OracError('ncdump is non-functional.') mat0 = search(r'netcdf library version (.+?) of', tmp0) if mat0: ncdf_version = mat0.group(1) else: ncdf_version = 'n/a' warnings.warn('Output formatting of ncdump may have changed.', OracWarning, stacklevel=2) break else: raise OracError('NetCDF lib improperly built as ncdump not present. ' 'LD_LIBRARY_PATH=' + os.environ["LD_LIBRARY_PATH"]) # Fetch ECMWF version from header of NCDF file if 3 <= args.nwp_flag <= 4: try: ecmwf_check_file = ggam[0] if ggam[0].endswith('nc') else ggas[0] tmp1 = check_output([ncdf_exe, "-h", ecmwf_check_file], universal_newlines=True) except OSError: raise FileMissing('ECMWF ggas file', ggas[0]) mat1 = search(r':history = "(.+?)" ;', tmp1) if mat1: ecmwf_version = mat1.group(1) else: ecmwf_version = 'n/a' warnings.warn('Header of ECMWF file may have changed.', OracWarning, stacklevel=2) elif args.nwp_flag == 2: ecmwf_version = 'ERA5' else: # TODO: Fetch version information from GFS files ecmwf_version = 'n/a' # RTTOV version number from small executable try: rttov_version_exe = os.path.join(args.orac_dir, "common", "rttov_version") if not os.path.isfile(rttov_version_exe): rttov_version_exe = os.path.join(args.orac_dir, "rttov_version") rttov_version = check_output( rttov_version_exe, universal_newlines=True ).strip() except CalledProcessError: rttov_version = 'n/a' warnings.warn('RTTOV library version number unavailable.', OracWarning, stacklevel=2) # Fetch GIT version cwd = os.getcwd() try: os.chdir(os.path.join(args.orac_dir, 'pre_processing')) tmp3 = check_output(["git", "--version"], universal_newlines=True) mat3 = search('git version (.+?)\n', tmp3) git_version = mat3.group(1) except (FileNotFoundError, CalledProcessError, AttributeError): git_version = 'n/a' warnings.warn('Unable to call git.', OracWarning, stacklevel=2) finally: os.chdir(cwd) file_version = f'R{args.File.revision}' chunk_flag = False # File chunking no longer required assume_full_paths = True # We pass absolute paths cldtype = not args.skip_cloud_type include_full_brdf = not args.lambertian # ------------------------------------------------------------------------ # Write driver file driver = f"""{args.File.sensor} {l1b} {geo} {args.usgs_file} {ggam[0]} {args.coef_dir} {args.atlas_dir} {nise} {alb} {brdf} {emis} {args.dellon} {args.dellat} {args.out_dir} {args.limit[0]} {args.limit[1]} {args.limit[2]} {args.limit[3]} {ncdf_version} {args.cfconvention} {args.institute} {args.processor} {args.email} {args.url} {file_version} {args.references} {args.history} {args.summary} {args.keywords} {args.comments} {args.project} {args.license} {uid} {production_time} {args.calib_file} {args.nwp_flag} {ggas[0]} {spam[0]} {chunk_flag} {args.day_flag} {args.verbose} - {assume_full_paths} {include_full_brdf} {rttov_version} {ecmwf_version} {git_version} ECMWF_TIME_INT_METHOD={args.single_ecmwf} ECMWF_PATH_2={ggam[1]} ECMWF_PATH2_2={ggas[1]} ECMWF_PATH3_2={spam[1]} USE_ECMWF_SNOW_AND_ICE={args.use_ecmwf_snow} USE_MODIS_EMIS_IN_RTTOV={args.use_modis_emis} ECMWF_NLEVELS={ecmwf_nlevels} USE_L1_LAND_MASK={args.l1_land_mask} USE_OCCCI={args.use_oc} OCCCI_PATH={occci} DISABLE_SNOW_ICE_CORR={args.no_snow_corr} DO_CLOUD_EMIS={args.cloud_emis} DO_IRONLY={args.ir_only} DO_CLDTYPE={cldtype} USE_CAMEL_EMIS={args.use_camel_emis} USE_SWANSEA_CLIMATOLOGY={args.swansea}""" if args.available_channels is not None: driver += "\nN_CHANNELS={}".format(len(args.available_channels)) driver += "\nCHANNEL_IDS={}".format( ','.join(str(k) for k in args.available_channels) ) for part, filename in args.extra_lines: if part == "pre" and filename != "": try: with open(filename, "r") as extra: driver += "\n" + extra.read() except IOError: raise FileMissing('extra_lines_file', filename) for sec, key, val in args.additional: if sec == "pre": driver += f"\n{key}={val}" if args.File.predef and not args.no_predef: driver += f""" USE_PREDEF_LSM=False EXT_LSM_PATH={args.prelsm_file} USE_PREDEF_GEO=False EXT_GEO_PATH={args.pregeo_file}""" if args.product_name is not None: driver += f"\nPRODUCT_NAME={args.product_name}" return driver
def build_preproc_driver(args): """Prepare a driver file for the preprocessor.""" from pyorac.definitions import FileName, BadValue from pyorac.util import build_orac_library_path, read_orac_libraries from re import search from subprocess import check_output, STDOUT from uuid import uuid4 file = _glob_dirs(args.in_dir, args.File.l1b, 'L1B file') geo = _glob_dirs(args.in_dir, args.File.geo, 'geolocation file') # Select NISE file if args.use_ecmwf_snow or args.no_snow_corr: nise = '' else: for form in ('NISE.004/%Y.%m.%d/NISE_SSMISF17_%Y%m%d.HDFEOS', 'NISE.002/%Y.%m.%d/NISE_SSMIF13_%Y%m%d.HDFEOS', '%Y/NISE_SSMIF13_%Y%m%d.HDFEOS', '%Y/NISE_SSMIF17_%Y%m%d.HDFEOS'): nise = args.File.time.strftime(os.path.join(args.nise_dir, form)) if os.path.isfile(nise): break else: raise FileMissing('NISE', nise) # Select previous surface reflectance and emissivity files if args.swansea: alb = _date_back_search(args.swansea_dir, args.File.time, 'SW_SFC_PRMS_%m.nc') brdf = None else: alb = _date_back_search(args.mcd43c3_dir, args.File.time, 'MCD43C3.A%Y%j.*.hdf') brdf = None if args.lambertian else _date_back_search( args.mcd43c1_dir, args.File.time, 'MCD43C1.A%Y%j.*.hdf') emis = None if args.use_modis_emis else _date_back_search( args.emis_dir, args.File.time, 'global_emis_inf10_monthFilled_MYD11C3.A%Y%j.041.nc') # Select ECMWF files bounds = _bound_time(args.File.time + args.File.dur // 2) if args.ecmwf_flag == 0: ggam = _form_bound_filenames(bounds, args.ggam_dir, 'ERA_Interim_an_%Y%m%d_%H+00.nc') elif args.ecmwf_flag == 1: ggam = _form_bound_filenames(bounds, args.ggam_dir, 'ggam%Y%m%d%H%M.nc') ggas = _form_bound_filenames(bounds, args.ggas_dir, 'ggas%Y%m%d%H%M.nc') spam = _form_bound_filenames(bounds, args.spam_dir, 'gpam%Y%m%d%H%M.nc') elif args.ecmwf_flag == 2: ggam = _form_bound_filenames(bounds, args.ggam_dir, 'ggam%Y%m%d%H%M.grb') ggas = _form_bound_filenames(bounds, args.ggas_dir, 'ggas%Y%m%d%H%M.nc') spam = _form_bound_filenames(bounds, args.spam_dir, 'spam%Y%m%d%H%M.grb') elif args.ecmwf_flag == 3: raise NotImplementedError('Filename syntax for --ecmwf_flag 3 unknown') elif args.ecmwf_flag == 4: for form, hr in (('C3D*%m%d%H*.nc', 3), ('ECMWF_OPER_%Y%m%d_%H+00.nc', 6), ('ECMWF_ERA_%Y%m%d_%H+00_0.5.nc', 6)): try: bounds = _bound_time(args.File.time + args.File.dur // 2, timedelta(hours=hr)) ggam = _form_bound_filenames(bounds, args.ggam_dir, form) break except FileMissing as e: err = e else: raise err ggas = ggam spam = ggam else: raise BadValue('ecmwf_flag', args.ecmwf_flag) if not args.skip_ecmwf_hr: #hr_ecmwf = _form_bound_filenames(bounds, args.hr_dir, # 'ERA_Interim_an_%Y%m%d_%H+00_HR.grb') # These files don't zero-pad the hour for some reason bounds = _bound_time(args.File.time + args.File.dur // 2, timedelta(hours=6)) hr_ecmwf = [ time.strftime( os.path.join(args.hr_dir, 'ERA_Interim_an_%Y%m%d_') + '{:d}+00_HR.grb'.format(time.hour * 100)) for time in bounds ] if not os.path.isfile(hr_ecmwf[0]): hr_ecmwf = [ time.strftime( os.path.join(args.hr_dir, 'ERA_Interim_an_%Y%m%d_') + '{:d}+00_HR.grb'.format(time.hour * 100)) for time in bounds ] for f in hr_ecmwf: if not os.path.isfile(f): raise FileMissing('HR ECMWF file', f) else: hr_ecmwf = ['', ''] occci = args.File.time.strftime( os.path.join( args.occci_dir, 'ESACCI-OC-L3S-IOP-MERGED-1M_MONTHLY' '_4km_GEO_PML_OCx_QAA-%Y%m-fv3.0.nc')) #------------------------------------------------------------------------ if args.uuid: uid = str(uuid4()) else: uid = 'n/a' # Add NetCDF library to path so following calls works libs = read_orac_libraries(args.orac_lib) try: os.environ["PATH"] = os.path.join(libs["NCDFLIB"][:-4], 'bin:') + \ os.environ["PATH"] except KeyError: pass os.environ["LD_LIBRARY_PATH"] = build_orac_library_path() # Determine current time production_time = datetime.now().strftime("%Y%m%d%H%M%S") # Determine NCDF version from command line try: tmp0 = check_output("ncdump", stderr=STDOUT, universal_newlines=True) except OSError: raise OracError('NetCDF lib improperly built as ncdump not present.') m0 = search(r'netcdf library version (.+?) of', tmp0) if m0: ncdf_version = m0.group(1) else: ncdf_version = 'n/a' warnings.warn('Output formatting of ncdump may have changed.', OracWarning, stacklevel=2) # Fetch ECMWF version from header of NCDF file try: ecmwf_check_file = ggam[0] if ggam[0][-2:] == 'nc' else ggas[0] tmp1 = check_output(["ncdump", "-h", ecmwf_check_file], universal_newlines=True) except OSError: raise FileMissing('ECMWF ggas file', ggas[0]) m1 = search(r':history = "(.+?)" ;', tmp1) if m1: ecmwf_version = m1.group(1) else: ecmwf_version = 'n/a' warnings.warn('Header of ECMWF file may have changed.', OracWarning, stacklevel=2) # Strip RTTOV version from library definition try: rttov_lib = glob(os.path.join(libs['RTTOVLIB'], 'librttov?*_main.a')) except KeyError: rttov_lib = glob( os.path.join(libs['CONDA_PREFIX'] + '/lib', 'librttov?*_main.a')) for rttov_file in rttov_lib: try: m2 = search(r'librttov([\d\.]+)_main.a', rttov_file) rttov_version = m2.group(1) break except: pass else: rttov_version = 'n/a' warnings.warn('Naming of RTTOV library directory may have changed.', OracWarning, stacklevel=2) # Fetch GIT version cwd = os.getcwd() try: os.chdir(os.path.join(args.orac_dir, 'pre_processing')) tmp3 = check_output(["git", "--version"], universal_newlines=True) m3 = search('git version (.+?)\n', tmp3) git_version = m3.group(1) except: git_version = 'n/a' warnings.warn('Unable to call git.', OracWarning, stacklevel=2) finally: os.chdir(cwd) # Fetch repository commit number if not args.revision: args.revision = get_repository_revision() file_version = 'R{}'.format(args.revision) #------------------------------------------------------------------------ # Write driver file driver = """{sensor} {l1b} {geo} {usgs} {ggam[0]} {coef} {atlas} {nise} {alb} {brdf} {emis} {dellon} {dellat} {out_dir} {limit[0]} {limit[1]} {limit[2]} {limit[3]} {ncdf_version} {conventions} {institution} {l2_processor} {creator_email} {creator_url} {file_version} {references} {history} {summary} {keywords} {comment} {project} {license} {uuid} {production_time} {atsr_calib} {ecmwf_flag} {ggas[0]} {spam[0]} {chunk_flag} {day_flag} {verbose} - {assume_full_paths} {include_full_brdf} {rttov_version} {ecmwf_version} {git_version} ECMWF_TIME_INT_METHOD={ecmwf_int_method} ECMWF_PATH_2={ggam[1]} ECMWF_PATH2_2={ggas[1]} ECMWF_PATH3_2={spam[1]} USE_HR_ECMWF={use_ecmwf_hr} ECMWF_PATH_HR={ecmwf_hr[0]} ECMWF_PATH_HR_2={ecmwf_hr[1]} USE_ECMWF_SNOW_AND_ICE={ecmwf_nise} USE_MODIS_EMIS_IN_RTTOV={modis_emis} ECMWF_NLEVELS={ecmwf_nlevels} USE_L1_LAND_MASK={l1_land_mask} USE_OCCCI={use_occci} OCCCI_PATH={occci_file} DISABLE_SNOW_ICE_CORR={no_snow} DO_CLOUD_EMIS={cld_emis} DO_IRONLY={ir_only} DO_CLDTYPE={cldtype} USE_CAMEL_EMIS={camel} USE_SWANSEA_CLIMATOLOGY={swansea}""".format( alb=alb, assume_full_paths=True, # Above file searching returns paths nor dirs atlas=args.atlas_dir, atsr_calib=args.calib_file, brdf=brdf, camel=args.camel_emis, chunk_flag=False, # File chunking no longer required cldtype=not args.skip_cloud_type, cld_emis=args.cloud_emis, coef=args.coef_dir, comment=args.comments, conventions=args.cfconvention, creator_email=args.email, creator_url=args.url, day_flag=args.day_flag, # 0=1=Day, 2=Night dellat=args.dellat, dellon=args.dellon, ecmwf_flag=args.ecmwf_flag, ecmwf_hr=hr_ecmwf, ecmwf_int_method=args.single_ecmwf, ecmwf_nise=args.use_ecmwf_snow, ecmwf_nlevels=args.ecmwf_nlevels, ecmwf_version=ecmwf_version, emis=emis, file_version=file_version, geo=geo, ggam=ggam, ggas=ggas, history=args.history, include_full_brdf=not args.lambertian, institution=args.institute, ir_only=args.ir_only, keywords=args.keywords, l1_land_mask=args.l1_land_mask, l1b=file, l2_processor=args.processor, license=args.license, limit=args.limit, modis_emis=args.use_modis_emis, ncdf_version=ncdf_version, nise=nise, no_snow=args.no_snow_corr, occci_file=occci, out_dir=args.out_dir, usgs=args.usgs_file, production_time=production_time, project=args.project, references=args.references, rttov_version=rttov_version, sensor=args.File.sensor, spam=spam, summary=args.summary, swansea=args.swansea, git_version=git_version, uuid=uid, use_ecmwf_hr=not args.skip_ecmwf_hr, use_occci=args.use_oc, verbose=args.verbose, ) if args.available_channels is not None: driver += "\nN_CHANNELS={}".format(len(args.available_channels)) driver += "\nCHANNEL_IDS={}".format(','.join( str(k) for k in args.available_channels)) for part, f in args.extra_lines: if part == "pre" and f != "": try: with open(f, "r") as e: driver += "\n" + e.read() except IOError: raise FileMissing('extra_lines_file', f) for sec, key, val in args.additional: if sec == "pre": driver += "\n{}={}".format(key, val) if args.File.predef and not args.no_predef: driver += """ USE_PREDEF_LSM=True EXT_LSM_PATH={lsm} USE_PREDEF_GEO=True EXT_GEO_PATH={geo}""".format(lsm=args.prelsm_file, geo=args.pregeo_file) if args.product_name is not None: driver += "\nPRODUCT_NAME={}".format(args.product_name) return driver