def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): if ref_dir == "" and hasattr(tracer, "dir"): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == "" and hasattr(tracer, "file"): input_file = tracer.file else: input_file = ref_file list_files = { datei: datei.strftime("{}/{}".format(input_dir, input_file)) for datei in input_dates } list_dates = {datei: [] for datei in input_dates} # Fetching for datei in input_dates: f = list_files[datei] target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) list_files[datei] = os.path.basename(target_file) return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): if ref_dir == "" and hasattr(tracer, "dir"): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == "" and hasattr(tracer, "file"): input_file = tracer.file else: input_file = ref_file list_files = { datei: datei.strftime("{}/{}".format(input_dir, input_file)) for datei in input_dates } list_dates = {datei: [] for datei in input_dates} # Fetching or creating if necessary for datei in input_dates: f = list_files[datei] target_file = "{}/{}".format(target_dir, os.path.basename(f)) if os.path.isfile(f): path.link(f, target_file) else: flx = make(tracer, target_file, tracer.flx_text) tracer.write("", target_file, flx[0, 0]) list_files[datei] = target_file return list_files, list_dates
def fetch( ref_dir, ref_file, input_dates, target_dir, tracer=None, filetypes=["defstoke", "fluxstoke", "fluxstokev", "phystoke"], **kwargs ): """Reads meteorology and links to the working directory Args: meteo (dictionary): dictionary defining the domain. Should include dirmeteo to be able to read the meteorology datei (datetime.datetime): initial date for the inversion window datef (datetime.datetime): end date for the inversion window workdir (str): path to the working directory where meteo files should be copied logfile (str): path to the log file filetypes ([str]): list of file radicals to copy in the working directory **kwargs (dictionary): extra arguments Return: ???????? Notes: At some point, include option to compute mass fluxes for LMDz, with different physics What is needed to do that? Possible only on CCRT? Flexibility to define new domains Can be very heavy and not necessarily relevant """ info("Copying meteo files from {} to {}".format(ref_dir, target_dir)) # Create the sub-directory to store meteo files path.init_dir(target_dir) # Loop over dates and file types for date in input_dates: for filetype in filetypes: meteo_file = "{}.an{}.m{:02d}.nc".format( filetype, date.year, date.month ) if filetype == "defstoke" and not os.path.isfile( ref_dir + meteo_file ): meteo_file = filetype + ".nc" target = "{}/{}".format(target_dir, meteo_file) source = "{}/{}".format(ref_dir, meteo_file) path.link(source, target) list_files = {datei: [] for datei in input_dates} list_dates = {datei: [] for datei in input_dates} return list_files, list_dates
def make_chemfields(self, data_in, input_type, ddi, ddf, runsubdir, mode): # Deals only with species whose prodloss is activated if not hasattr(getattr(self, "chemistry", None), input_type): return # Binary and nc file name depending on input_type bin_name = "prodscale" if input_type == "prodloss3d" else "scale" nc_name = "prodloss" if input_type == "prodloss3d" else "prescr" datastore = data_in.datastore for spec in getattr(self.chemistry, input_type).attributes: trid = (input_type, spec) if trid not in datastore: continue data = datastore[trid] # Links reference netCDF files that are needed anyway by LMDZ try: dirorig = datastore[trid]["dirorig"] fileorig = datastore[trid]["fileorig"] if fileorig is None or dirorig is None: raise KeyError except KeyError: tracer = getattr(getattr(self.chemistry, input_type), spec) dirorig = tracer.dir fileorig = tracer.file origin = ddi.strftime("{}/{}".format(dirorig, fileorig)) target = "{}/{}_{}.nc".format(runsubdir, nc_name, spec) path.link(origin, target) # Skip if that species is not in the control vector if "spec" not in data: continue # If tangent-linear mode, include tl increments if "incr" in data and mode == "tl": incr = data["incr"] else: incr = 0.0 * data["spec"] # Write to FORTRAN binary prod_file = "{}/mod_{}_{}.bin".format(runsubdir, bin_name, spec) with FortranFile(prod_file, "w") as f: # Looping over all values and writing to binary prod = data["spec"].values incr = data["incr"].values for d0, d1 in zip(np.ravel(prod), np.ravel(incr)): f.write_record(np.array([d0, d1], dtype=float))
def ini_data(plugin, **kwargs): """Initializes the observation vector from information in the Yaml file Args: """ # Set dump type if not defined; default is nc if not hasattr(plugin, "dump_type"): plugin.dump_type = "nc" # Set default file_obsvect file_default = "{}/obsvect/monitor.{}".format(plugin.workdir, plugin.dump_type) plugin.file_obsvect = getattr(plugin, "file_obsvect", file_default) # Keeping check_monitor as a class method plugin.check_monitor = MethodType(check_monitor, plugin) # Initializing y0 measurements = plugin.measurements plugin.dump_type = getattr(plugin, "dump_type", "nc") init_y0(plugin, measurements, **kwargs) # Initialize R if any observation if plugin.datastore.size > 0: init_rinvprod(plugin, measurements, **kwargs) print("Link satellite files to the working directory") print(hasattr(plugin, "dir_satellites"), plugin.workdir) if hasattr(plugin, "dir_satellites"): path.init_dir("{}/obsvect/satellites/".format(plugin.workdir)) for dd in plugin.model.subsimu_dates[:-1]: sat_files = glob.glob( dd.strftime("{}/infos_*%Y%m%d%H%M.nc").format( plugin.dir_satellites)) for obs_file in sat_files: target = "{}/obsvect/satellites/{}".format( plugin.workdir, os.path.basename(obs_file)) path.link(obs_file, target) plugin.has_satellites = False if np.where((plugin.datastore["level"] < 0))[0].size > 0: plugin.has_satellites = True
def default_fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): # Picking info from tracer if not in inputs if ref_dir == "" and hasattr(tracer, "dir"): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == "" and hasattr(tracer, "file"): input_file = tracer.file else: input_file = ref_file info("Fetching input files using directory and file format") info("{}/{}".format(input_dir, input_file)) list_files = {} list_dates = {} for datei in input_dates: tmp_files = [] tmp_dates = [] for dd in input_dates[datei]: dir_dd = dd.strftime(input_dir) file_dd = dd.strftime(input_file) tmp_files.append("{}/{}".format(dir_dd, file_dd)) # Fetching local_files = [] for f in tmp_files: target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) local_files.append(target_file) list_files[datei] = list(set(local_files)) list_dates[datei] = list(set(tmp_dates)) return list_files, list_dates
def make_meteo(self, data, ddi, ddf, runsubdir, mode): datei = min(ddi, ddf) dir_meteo = data.datastore[("meteo", "")]["dirorig"] # Links meteorological mass fluxes for ftype in ["defstoke", "fluxstoke", "fluxstokev", "phystoke"]: meteo_file = "{}.an{}.m{:02d}.nc".format( ftype, datei.year, datei.month ) source = "{}/{}".format(dir_meteo, meteo_file) target = "{}/{}.nc".format(runsubdir, ftype) if ftype == "defstoke" and not os.path.isfile(source): meteo_file = "{}.nc".format(ftype) source = "{}/{}".format(dir_meteo, meteo_file) path.link(source, target) # Links vertical coordinates from previous simulations source = self.file_vcoord target = "{}/vcoord.nc".format(runsubdir) path.link(source, target)
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): list_files = {} list_dates = {} for datei in input_dates: tmp_files = [] tmp_dates = [] for dd in input_dates[datei]: file_flx = dd.strftime(ref_file) dir_flx = dd.strftime(ref_dir) date_flx = dd if not os.path.isfile( "{}/{}".format(dir_flx, file_flx)) and getattr( tracer, "closest_year", False): info("Warning: could not find correct year for EDGAR; " "using closest available one") list_dates_avail = [ datetime.datetime.strptime(os.path.basename(f), ref_file) for f in glob.glob("{}/v50_*nc".format(dir_flx)) ] delta_dates = np.abs(dd - np.array(list_dates_avail)) date_flx = list_dates_avail[np.argmin(delta_dates)] file_flx = date_flx.strftime(ref_file) tmp_files.append("{}/{}".format(dir_flx, file_flx)) tmp_dates.append(date_flx) # Fetching local_files = [] for f in tmp_files: target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) local_files.append(target_file) list_files[datei] = list(set(local_files)) list_dates[datei] = list(set(tmp_dates)) return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): if ref_dir == '' and hasattr(tracer, 'dir'): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == '' and hasattr(tracer, 'file'): input_file = tracer.file else: input_file = ref_file list_files = { datei: datei.strftime('{}/{}'.format(input_dir, input_file)) for datei in input_dates } if hasattr(tracer, "file_glob"): list_files_glo = \ {datei: datei.strftime('{}/{}'.format(input_dir, tracer.file_glob)) for datei in input_dates} list_dates = {datei: input_dates[datei] for datei in input_dates} # Fetching for datei in input_dates: f = list_files[datei] target_file = '{}/{}'.format(target_dir, os.path.basename(f)) path.link(f, target_file) list_files[datei] = target_file # Initialize global file for later if hasattr(tracer, "file_glob"): f = list_files_glo[datei] target_file = '{}/{}'.format(target_dir, os.path.basename(f)) path.link(f, target_file) return list_files, list_dates
def fetch( ref_dir, ref_file, input_dates, target_dir, tracer=None, component=None ): if ref_dir == "" and hasattr(tracer, "dir"): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == "" and hasattr(tracer, "file"): input_file = tracer.file else: input_file = ref_file # Two cases: initial conditions or LBC if getattr(component, "orig_name", "") == "inicond": datei = np.min(list(input_dates.keys())) list_files = { datei: datei.strftime("{}/{}".format(input_dir, input_file)) } list_dates = {datei: []} else: list_files = { datei: datei.strftime("{}/{}".format(input_dir, input_file)) for datei in input_dates } list_dates = {datei: [] for datei in input_dates} # Fetching for datei in input_dates: f = list_files[datei] target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) list_files[datei] = os.path.basename(target_file) return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): list_files = {} list_dates = {} for datei in input_dates: tmp_files = [] tmp_dates = [] for dd in input_dates[datei]: dir_dd = dd.strftime(ref_dir) files_3d, dates_3d = find_valid_file(dir_dd, tracer.file_3d, dd) tmp_files.extend(files_3d) tmp_dates.extend(dates_3d) # Fetching local_files = [] for f in tmp_files: target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) local_files.append(target_file) list_files[datei] = list(set(local_files)) list_dates[datei] = list(set(tmp_dates)) return list_files, list_dates
def make_fluxes(self, data, runsubdir, datei, mode): """Make AEMISSIONS.nc and BEMISSIONS.nc files for CHIMERE. Use chemical scheme to check which species is needed and either take it from the datastore (i.e. when defined in the control vector), or take it from prescribed emissions Args: self (pycif.utils.classes.Fluxes.fluxes): Flux plugin with all attributes datastore (dict): information on flux species runsubdir (str): directory to the current run nho (int): number of hours in the run mode (str): running mode: 'fwd', 'tl' or 'adj' """ datastore = { trid: data.datastore[trid] for trid in data.datastore if trid[0] in ["fluxes", "biofluxes"] } # List of dates for which emissions are needed list_dates = pd.date_range(datei, periods=self.nhours + 1, freq="H") # Getting the right emissions # Loop on all anthropogenic and biogenic species # If in datastore, take data, otherwise, link to original A/B EMISSIONS list_trid = [("fluxes", spec) for spec in self.chemistry.emis_species.attributes] \ + [("biofluxes", spec) for spec in self.chemistry.bio_species.attributes] for trid in list_trid: spec = trid[1] emis_type = trid[0] flx_plg = self.fluxes if emis_type == "anthro" else self.biofluxes if trid in datastore: pass # If spec not explicitly defined in datastore, # fetch general component information if available elif trid not in datastore and (emis_type, "") in datastore: trid = (emis_type, "") else: continue # Bio or anthro file if emis_type == "fluxes": file_emisout = "{}/AEMISSIONS.nc".format(runsubdir) file_emisincrout = "{}/AEMISSIONS.increment.nc".format( runsubdir) else: file_emisout = "{}/BEMISSIONS.nc".format(runsubdir) file_emisincrout = "{}/BEMISSIONS.increment.nc".format(runsubdir) tracer = datastore[trid] dirorig = tracer["dirorig"] fileorig = tracer["fileorig"] fileemis = datei.strftime("{}/{}".format(dirorig, fileorig)) # If no data is provided, just copy from original file if "spec" not in tracer: linked = False # If does not exist, just link if not os.path.isfile(file_emisout): path.link(fileemis, file_emisout) linked = True # Otherwise, check for difference if not linked: if not filecmp.cmp(fileemis, file_emisout): with Dataset(fileemis, "r") as fin: emisin = fin.variables[spec][:] emisin = xr.DataArray( emisin, coords={"time": list_dates}, dims=("time", "lev", "lat", "lon"), ) flx_plg.write(spec, file_emisout, emisin) # Repeat operations for tangent linear if mode != "tl": continue if "spec" not in tracer: # If does not exist, just link if not os.path.isfile(file_emisincrout): shutil.copy(fileemis, file_emisincrout) flx_incr = xr.DataArray( np.zeros( ( len(list_dates), self.nlevemis if emis_type == "fluxes" else self.nlevemis_bio, self.domain.nlat, self.domain.nlon, ) ), coords={"time": list_dates}, dims=("time", "lev", "lat", "lon"), ) flx_plg.write(spec, file_emisincrout, flx_incr) else: # Replace existing link by copy of original file to modify it path.copyfromlink(file_emisout) # Put in dataset and write to input flx_fwd = datastore[trid]["spec"] flx_plg.write(spec, file_emisout, flx_fwd) if mode == "tl": path.copyfromlink(file_emisincrout) flx_tl = datastore[trid].get("incr", 0.0 * flx_fwd) flx_plg.write(spec, file_emisincrout, flx_tl)
def native2inputs(self, datastore, input_type, datei, datef, runsubdir, mode="fwd"): """Converts data at the model data resolution to model compatible input files. Args: self: the model Plugin input_type (str): one of 'fluxes' datastore: data to convert if input_type == 'fluxes', datei, datef: date interval of the sub-simulation mode (str): running mode: one of 'fwd', 'adj' and 'tl' runsubdir (str): sub-directory for the current simulation workdir (str): the directory of the whole pyCIF simulation Notes: - CHIMERE expects hourly inputs; """ ddi = min(datei, datef) ddf = max(datei, datef) # Hour steps of the sub-run hour_dates = pd.date_range(ddi, ddf, freq="1H") if datastore is None: datastore = {} sdc = ddi.strftime("%Y%m%d%H") # Choose the right executable if input_type == "exe": if mode == "fwd": source = "fwdchimere.e" elif mode == "tl": source = "tlchimere.e" elif mode == "adj": source = "achimere.e" else: raise Exception( "Unknown mode '{}' for executable with CHIMERE".format(mode)) path.link( "{}/model/{}".format(self.workdir, source), "{}/chimere.e".format(runsubdir), ) # Deals with the nml file if input_type == "nml": make_nml(self, runsubdir, sdc, mode) # Deals with fluxes # WARNING: Not specified emissions are set to zero?? if input_type in ["fluxes", "biofluxes"]: make_fluxes(self, datastore, runsubdir, ddi, mode) if input_type in ["latcond", "topcond"]: make_boundcond(self, datastore, runsubdir, ddi, mode, input_type) if input_type == "inicond": make_inicond(self, datastore, runsubdir, mode, ddi) if input_type == "meteo": make_meteo(self, runsubdir, sdc) # Deals with observations if input_type == "obs": make_obs(self, datastore, runsubdir, mode) return
def make_inicond(self, data, runsubdir, mode, datei): datastore = { trid: data.datastore[trid] for trid in data.datastore if trid[0] == "inicond" } # Fixed name for INI_CONCS files fileout = "{}/INI_CONCS.0.nc".format(runsubdir) fileoutincr = "{}/INI_CONCS.0.increment.nc".format(runsubdir) nho = self.nho # Getting the forward initial concentrations (needed even for adjoint) # if chained period if hasattr(self, "chain"): if mode in ["tl", "fwd"]: filein = self.chain.strftime( "{}/../chain/end.%Y%m%d%H.{}.nc".format(runsubdir, nho)) else: subsimu_dates = self.subsimu_dates date_index = np.where(subsimu_dates == datei)[0][0] refdir = self.adj_refdir filein = subsimu_dates[date_index - 1].strftime( "{}/chain/end.%Y%m%d%H.{}.nc".format(refdir, nho)) path.link(filein, fileout) # Adjoint needs to chain sensitivities as well if mode == "adj": filein = self.chain.strftime( "{}/../chain/aend.%Y%m%d%H.{}.nc".format(runsubdir, nho)) fileout = self.chain.strftime("{}/aini.nc".format(runsubdir)) path.link(filein, fileout) # Exit if not first period if datei != self.datei: return # Loop on all active species # If in datastore, take data, otherwise, link to original INI_CONCS for spec in self.chemistry.acspecies.attributes: trid = ("inicond", spec) if trid in datastore: pass # If spec not explicitly defined in datastore, # fetch general component information if available elif trid not in datastore and ("inicond", "") in datastore: trid = ("inicond", "") else: continue tracer = datastore[trid] dirorig = tracer["dirorig"] fileorig = tracer["fileorig"] fileini = "{}/{}".format(dirorig, fileorig) # If no data is provided, just copy from original file if "spec" not in tracer: # If does not exist, just link linked = False if not os.path.isfile(fileout): path.link(fileini, fileout) linked = True # Otherwise, check for difference if not linked: if not filecmp.cmp(fileini, fileout): raise Exception("I need to transfer inicond " "from one file to the other one") # Repeat operations for tangent linear if mode != "tl": continue # If does not exist, just link if not os.path.isfile(fileoutincr): shutil.copy(fileini, fileoutincr) with Dataset(fileoutincr, "a") as fout: if spec in fout.variables: fout.variables[spec][:] = 0.0 else: # Replace existing link by copy # of original file to modify it path.copyfromlink(fileout) # Write initial conditions ini_fwd = datastore[trid]["spec"] self.inicond.write(spec, fileout, ini_fwd, comp_type="inicond") if mode == "tl": path.copyfromlink(fileoutincr) ini_tl = datastore[trid].get("incr", 0.0 * ini_fwd) self.inicond.write(spec, fileoutincr, ini_tl, comp_type="inicond")
def make_input(self, data, mod_input, di, df, runsubdir, mode, lmdz_yearref=1979, **kwargs): """Prepares inputs for the current LMDZ simulation. It includes: - meteo files (defstoke.nc, fluxstoke.nc fluxstokev.nc and phystoke.nc) - def files (run.def, totinput, ACTIVE_SPECIES) - initial conditions (start.nc) - fields for SACS (inca.nc, incaMCF.nc) Args: - self (Plugin): LMDZ plugin - mod_input (str): one of: 'meteo', 'def', 'inicond', 'chem_fields' - di (datetime): beginning of the simulation window (along the fwd or bckwd axis) - df (datetime): end of simulation - workdir (str): path to the pycif simulation - runsubdir (str): path to the LMDZ sub-simulation - mode (str): running mode; fwd or adj - lmdz_yearref (int): reference year for LMDZ dates. Default is 1979 """ datei = min(di, df) if mod_input == "meteo": make_meteo(self, data, di, df, runsubdir, mode) elif mod_input == "def": # run.def # make_rundef(self.filedef, datei, self.physic, runsubdir, lmdz_yearref) # totinput make_totinput(self, runsubdir, datei, mode) # Remove restart TL if mode != "tl": shutil.rmtree("{}/*start_tl.bin".format(runsubdir), ignore_errors=True) # TODO: at the moment, no satellite observation os.system("echo 0 > {}/infousedsat.txt".format(runsubdir)) elif mod_input == "chem_fields": if hasattr(self, "chemistry"): # Linking to pre-computed INCA fields for kinetics source = datei.strftime("{}/{}".format( self.chemistry.kinetic.dir, self.chemistry.kinetic.file)) target = "{}/kinetic.nc".format(runsubdir) path.link(source, target) # Linking to pre-computed INCA fields for deposition if hasattr(self.chemistry, "deposition"): for spec in self.chemistry.deposition.attributes: tracer = getattr(self.chemistry.deposition, spec) source = datei.strftime("{}/{}".format( tracer.dir, tracer.file)) target = "{}/dep_{}.nc".format(runsubdir, spec) path.link(source, target) finf = "{}/chemical_scheme.nml".format(self.chemistry.dirchem_ref) target = "{}/chemical_scheme.nml".format(runsubdir) path.link(finf, target) elif mod_input == "traj": if mode == "adj": if not hasattr(self, "adj_refdir"): info("Adjoint LMDZ couldn't be initialized " "with forward trajq.bin files") raise Exception else: for spec in self.chemistry.acspecies.attributes: traj_file = "trajq_{}_%Y%m%d%H%M.bin".format(spec) source = datei.strftime("{}/chain/{}".format( self.adj_refdir, traj_file)) target = "{}/trajq_{}.bin".format(runsubdir, spec) path.link(source, target) elif mod_input == "exe": # Linking executable source = "{}/model/dispersion.e".format(self.workdir) target = "{}/dispersion.e".format(runsubdir) path.link(source, target)
def make_meteo(self, runsubdir, sdc): # use ready-made METEO.nc files # Getting the right one nho = self.nho filemet = self.meteo.dir + "METEO." + sdc + "." + str(nho) + ".nc" path.link(filemet, runsubdir + "/METEO.nc")
def make_boundcond(self, data, runsubdir, ddi, mode, input_type): """ Generates boundary conditions files for CHIMERE :param self: :param datastore: :type datastore: dict :param runsubdir: :type runsubdir: str :param sdc: :type sdc: str :param hour_dates: :param mode: :param input_type: :return: """ datastore = { trid: data.datastore[trid] for trid in data.datastore if trid[0] == input_type } # Name of variable in netCDF nc_varname = "top_conc" if input_type == "topcond" else "lat_conc" # Fixed name for BC files fileout = "{}/BOUN_CONCS.nc".format(runsubdir) fileoutincr = "{}/BOUN_CONCS.increment.nc".format(runsubdir) # Loop on all active species # If in datastore, take data, otherwise, link to original INI_CONCS for spec in self.chemistry.acspecies.attributes: trid = (input_type, spec) if trid in datastore: pass # If spec not explicitly defined in datastore, # fetch general component information if available elif trid not in datastore and (input_type, "") in datastore: trid = (input_type, "") else: continue tracer = datastore[trid] dirorig = tracer["dirorig"] fileorig = tracer["fileorig"] fileini = ddi.strftime("{}/{}".format(dirorig, fileorig)) # If no data is provided, just copy from original file if "spec" not in tracer: # If does not exist, just link if not os.path.isfile(fileout): path.link(fileini, fileout) # # Otherwise, check for difference # elif not filecmp.cmp(fileini, fileout): # # print(__file__) # import code # code.interact(local=dict(locals(), **globals())) # raise Exception('I need to transfer lbc ' # 'from one file to the other one') else: # Replace existing link by copy # of original file to modify it path.copyfromlink(fileout) # Write initial conditions lbc_fwd = datastore[trid]["spec"] self.latcond.write(spec, fileout, lbc_fwd, comp_type=input_type) if mode == "tl": path.copyfromlink(fileoutincr) lbc_tl = datastore[trid].get("incr", 0.0 * lbc_fwd) self.latcond.write(spec, fileoutincr, lbc_tl, comp_type=input_type) # Repeat operations for tangent linear if mode != "tl": continue if "spec" not in tracer: # If does not exist, just link if not os.path.isfile(fileoutincr): shutil.copy(fileini, fileoutincr) with Dataset(fileoutincr, "a") as fout: nc_var_out = fout.variables[nc_varname][:] nc_names = [ str(b"".join(s).strip().lower(), "utf-8") for s in fout.variables["species"][:] ] if spec.lower() not in nc_names: continue # Apply to original data nc_var_out[..., nc_names.index(spec.lower())][:] = 0.0 fout.variables[nc_varname][:] = nc_var_out
def run(self, runsubdir, mode, workdir, do_simu=True, **kwargs): """Run dummy_txt Gaussian model in forward or adjoint mode Args: runsubdir (str): working directory for the current run mode (str): forward or backward workdir (str): pycif working directory do_simu (bool): if False, considers that the simulation was already run """ # Ignore warnings in the toy model warnings.simplefilter("ignore") # Do the simulation if do_simu: # Linking Pasquill-Gifford classifileation source = "{}/model/Pasquill-Gifford.txt".format(workdir) target = "{}/Pasquill-Gifford.txt".format(runsubdir) path.link(source, target) # If H was already computed, take it from previous simulation info("Running the model Dummy in {}".format(runsubdir)) dref = self.meteo.data.index.min() if dref in self.H_matrix: H = self.H_matrix[dref] else: H = fortran_exe(self, target, mode) # Saving H for later computations if required if self.save_H: self.H_matrix[dref] = H # Compute H.x or H^T.dy depending on the running mode meteo_index = self.dataobs["tstep"] if mode in ["fwd", "tl"]: # In fwd and tl, simple product of flux and footprints self.dataobs["sim"] = np.nansum( self.dataflx.values[meteo_index, 0] * H, axis=(1, 2) ) self.dataobs["sim_tl"] = np.nansum( self.dataflx_tl.values[meteo_index, 0] * H, axis=(1, 2) ) else: # In adjoint, aggregation of footprints, to model resolution dflx = self.dataobs["obs_incr"][:, np.newaxis, np.newaxis] * H dflx[np.isnan(dflx)] = 0.0 dflx = xr.DataArray( dflx[:, np.newaxis], coords={ "time": meteo_index, "lev": [0], "lat": list(range(self.domain.nlat)), "lon": list(range(self.domain.nlon)), }, dims=("time", "lev", "lat", "lon"), ) # Summing observations at the same hour and reprojecting to meteo # resolution if missing observations dflx = ( dflx.to_dataframe(name="flx") .groupby(["time", "lev", "lat", "lon"]) .sum() ) dflx = dflx.unstack(level=["lev", "lat", "lon"]) dflx.index = self.meteo.data.index[dflx.index] dflx = dflx.reindex(self.meteo.data.index).stack( level=["lev", "lat", "lon"], dropna=False ) dflx = xr.Dataset.from_dataframe(dflx)["flx"] dflx = dflx.rename({"date": "time"}) # Filling NaNs with 0, as it corresponds to hours with no obs dflx = dflx.fillna(0.0) # Saving sensitivity to the model object # It corresponds to outputs in classical numerical models self.dflx = dflx
def init_y0(obsvect, measurements, **kwargs): """Initializes the observation vector. In most cases the observation vector is similar to the measurement vector but there is no reason for it to be the same, especially when super-observations are used (e.g. daily or afternoon averages, gradients, etc.) Args: obsvect (Plugin): the observation vector with all its attributes measurements (Plugin): the pre-loaded measurements Returns: obsvect, updated with horizontal and vertical coordinates, as well as model time steps """ # Saves initialized observation vector to workdir/obsvect/monitor # Try linking file_obsvect to workdir/obsvect if existing # Otherwise, define it and generate it later on path.init_dir("{}/obsvect".format(obsvect.workdir)) file_monitor = "{}/obsvect/monitor.{}".format(obsvect.workdir, obsvect.dump_type) shutil.rmtree(file_monitor, ignore_errors=True) if os.path.isfile(obsvect.file_obsvect): path.link(obsvect.file_obsvect, file_monitor) # From here, no interaction with files outside the working directory obsvect.file_obsvect = file_monitor # Try reading file allcorrec, ok_hcoord, ok_vcoord, do_tstep = False, False, False, True exclude_stations = getattr(obsvect, "exclude_stat", []) try: # Reading a monitor file if available info("Try opening {}".format(file_monitor)) obsvect.datastore = read_datastore(file_monitor, dump_type=obsvect.dump_type) # Check that the monitor is consistent with the simulation to be run # If True, returns directly the observation as is allcorrect, ok_hcoord, ok_vcoord, do_tstep = obsvect.check_monitor() if allcorrect and len(exclude_stations) == 0: return obsvect # Otherwise, recompute necessary items (i, j, lev, tstep) # of the observation error raise PluginError # If the monitor file could not be opened, start back from measurements except IOError: info("Couldn't open the observation file") info("Generating observation vector from measurements") # Copying attributes from measurements for key in measurements.attributes: setattr(obsvect, key, getattr(measurements, key)) # Copying datastore from measurements if existing and not empty obsvect.datastore = getattr(measurements, "datastore", init_empty()) except PluginError: info("Warning! The prescribed monitor file is not consistent with " "the current simulation. Re-analysing it") # Remove unwanted stations if len(exclude_stations) > 0: mask = ~(obsvect.datastore["station"].isin(exclude_stations)) obsvect.datastore = obsvect.datastore.loc[mask] # Crops y0 to the simulation period obsvect.datastore = crop_monitor(obsvect.datastore, obsvect.datei, obsvect.datef, **kwargs) # Computes grid cells where observations fall if not ok_hcoord: obsvect = hcoord(obsvect, **kwargs) # Computes model layer where observations fall if not ok_vcoord: obsvect = vcoord(obsvect, **kwargs) # Compute time steps corresponding to the observations if do_tstep: obsvect = tstep(obsvect, **kwargs) # Dumps the datastore # Attributes to keep in the monitor if NetCDF format (recommanded!) nc_attributes = { "datei": obsvect.datei.strftime("%d-%m-%Y %H:%M:%S"), "datef": obsvect.datef.strftime("%d-%m-%Y %H:%M:%S"), "model name": obsvect.model.plugin.name, "model version": obsvect.model.plugin.version, "domain nlon": str(obsvect.model.domain.nlon), "domain nlat": str(obsvect.model.domain.nlat), } if getattr(obsvect, "dump", True): dump_datastore( obsvect.datastore, file_monit=obsvect.file_obsvect, dump_type=obsvect.dump_type, nc_attributes=nc_attributes, mode="w", ) return obsvect
def make_inicond(self, data_in, datei, datef, runsubdir, mode): ddi = min(datei, datef) ddf = max(datei, datef) # Reference initial condition file for looping sub-simulations if hasattr(self, "chain"): source = self.chain.strftime( "{}/../chain/restart_%Y%m%d%H%M.nc".format(runsubdir)) target = "{}/start.nc".format(runsubdir) path.link(source, target) if mode == "tl": source = self.chain.strftime( "{}/../chain/restart_tl_%Y%m%d%H%M.bin".format(runsubdir)) target = "{}/start_tl.bin".format(runsubdir) path.link(source, target) return # Generating reference initial conditions if first sub-simulation datastore = data_in.datastore for spec in self.chemistry.acspecies.attributes: restartID = getattr(self.chemistry.acspecies, spec).restart_id if ("inicond", spec) not in datastore: continue ini = datastore[("inicond", spec)] ref_dir = ini["dirorig"] ref_file = ini["fileorig"] # Copy if does not already exists source = ddi.strftime("{}/{}".format(ref_dir, ref_file)) target = "{}/start.nc".format(runsubdir) if not os.path.isfile(target): shutil.copy(source, target) if mode == "tl": target_tl = "{}/start_tl.nc".format(runsubdir) if not os.path.isfile(target_tl): shutil.copy(source, target_tl) # Updates data if mode in ["fwd", "tl"] and ddi == self.datei: var = "q{:02d}".format(restartID) if "spec" in ini: data = ini["spec"] with Dataset(target, "a") as f: if var in f.variables: f.variables[var][:] = data.values else: self.inicond.write(var, target, data) if mode == "tl": if "incr" in ini: data = ini["incr"] with Dataset(target_tl, "a") as f: if var in f.variables: f.variables[var][:] = data.values else: self.inicond.write(var, target_tl, data) else: with Dataset(target_tl, "a") as f: if var in f.variables: f.variables[var][:] = 0.0 elif mode == "adj" and ddf == self.datef: for spec in self.chemistry.acspecies.attributes: restartID = getattr(self.chemistry.acspecies, spec).restart_id with Dataset(target, "a") as f: var = "q{:02d}".format(restartID) if var in f.variables: f.variables[var][:] = 0.0