def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs):
if ref_dir == "" and hasattr(tracer, "dir"):
input_dir = tracer.dir
else:
input_dir = ref_dir
if ref_file == "" and hasattr(tracer, "file"):
input_file = tracer.file
else:
input_file = ref_file
list_files = {
datei: datei.strftime("{}/{}".format(input_dir, input_file))
for datei in input_dates
}
list_dates = {datei: [] for datei in input_dates}
# Fetching
for datei in input_dates:
f = list_files[datei]
target_file = "{}/{}".format(target_dir, os.path.basename(f))
path.link(f, target_file)
list_files[datei] = os.path.basename(target_file)
return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs):
if ref_dir == "" and hasattr(tracer, "dir"):
input_dir = tracer.dir
else:
input_dir = ref_dir
if ref_file == "" and hasattr(tracer, "file"):
input_file = tracer.file
else:
input_file = ref_file
list_files = {
datei: datei.strftime("{}/{}".format(input_dir, input_file))
for datei in input_dates
}
list_dates = {datei: [] for datei in input_dates}
# Fetching or creating if necessary
for datei in input_dates:
f = list_files[datei]
target_file = "{}/{}".format(target_dir, os.path.basename(f))
if os.path.isfile(f):
path.link(f, target_file)
else:
flx = make(tracer, target_file, tracer.flx_text)
tracer.write("", target_file, flx[0, 0])
list_files[datei] = target_file
return list_files, list_dates
def fetch(
ref_dir,
ref_file,
input_dates,
target_dir,
tracer=None,
filetypes=["defstoke", "fluxstoke", "fluxstokev", "phystoke"],
**kwargs
):
"""Reads meteorology and links to the working directory
Args:
meteo (dictionary): dictionary defining the domain. Should include
dirmeteo to be able to read the meteorology
datei (datetime.datetime): initial date for the inversion window
datef (datetime.datetime): end date for the inversion window
workdir (str): path to the working directory where meteo files
should be copied
logfile (str): path to the log file
filetypes ([str]): list of file radicals to copy in the working
directory
**kwargs (dictionary): extra arguments
Return:
????????
Notes: At some point, include option to compute mass fluxes for LMDz,
with different physics What is needed to do that? Possible only on CCRT?
Flexibility to define new domains Can be very heavy and not necessarily
relevant
"""
info("Copying meteo files from {} to {}".format(ref_dir, target_dir))
# Create the sub-directory to store meteo files
path.init_dir(target_dir)
# Loop over dates and file types
for date in input_dates:
for filetype in filetypes:
meteo_file = "{}.an{}.m{:02d}.nc".format(
filetype, date.year, date.month
)
if filetype == "defstoke" and not os.path.isfile(
ref_dir + meteo_file
):
meteo_file = filetype + ".nc"
target = "{}/{}".format(target_dir, meteo_file)
source = "{}/{}".format(ref_dir, meteo_file)
path.link(source, target)
list_files = {datei: [] for datei in input_dates}
list_dates = {datei: [] for datei in input_dates}
return list_files, list_dates
def make_chemfields(self, data_in, input_type, ddi, ddf, runsubdir, mode):
# Deals only with species whose prodloss is activated
if not hasattr(getattr(self, "chemistry", None), input_type):
return
# Binary and nc file name depending on input_type
bin_name = "prodscale" if input_type == "prodloss3d" else "scale"
nc_name = "prodloss" if input_type == "prodloss3d" else "prescr"
datastore = data_in.datastore
for spec in getattr(self.chemistry, input_type).attributes:
trid = (input_type, spec)
if trid not in datastore:
continue
data = datastore[trid]
# Links reference netCDF files that are needed anyway by LMDZ
try:
dirorig = datastore[trid]["dirorig"]
fileorig = datastore[trid]["fileorig"]
if fileorig is None or dirorig is None:
raise KeyError
except KeyError:
tracer = getattr(getattr(self.chemistry, input_type), spec)
dirorig = tracer.dir
fileorig = tracer.file
origin = ddi.strftime("{}/{}".format(dirorig, fileorig))
target = "{}/{}_{}.nc".format(runsubdir, nc_name, spec)
path.link(origin, target)
# Skip if that species is not in the control vector
if "spec" not in data:
continue
# If tangent-linear mode, include tl increments
if "incr" in data and mode == "tl":
incr = data["incr"]
else:
incr = 0.0 * data["spec"]
# Write to FORTRAN binary
prod_file = "{}/mod_{}_{}.bin".format(runsubdir, bin_name, spec)
with FortranFile(prod_file, "w") as f:
# Looping over all values and writing to binary
prod = data["spec"].values
incr = data["incr"].values
for d0, d1 in zip(np.ravel(prod), np.ravel(incr)):
f.write_record(np.array([d0, d1], dtype=float))
def ini_data(plugin, **kwargs):
"""Initializes the observation vector from information in the Yaml file
Args:
"""
# Set dump type if not defined; default is nc
if not hasattr(plugin, "dump_type"):
plugin.dump_type = "nc"
# Set default file_obsvect
file_default = "{}/obsvect/monitor.{}".format(plugin.workdir,
plugin.dump_type)
plugin.file_obsvect = getattr(plugin, "file_obsvect", file_default)
# Keeping check_monitor as a class method
plugin.check_monitor = MethodType(check_monitor, plugin)
# Initializing y0
measurements = plugin.measurements
plugin.dump_type = getattr(plugin, "dump_type", "nc")
init_y0(plugin, measurements, **kwargs)
# Initialize R if any observation
if plugin.datastore.size > 0:
init_rinvprod(plugin, measurements, **kwargs)
print("Link satellite files to the working directory")
print(hasattr(plugin, "dir_satellites"), plugin.workdir)
if hasattr(plugin, "dir_satellites"):
path.init_dir("{}/obsvect/satellites/".format(plugin.workdir))
for dd in plugin.model.subsimu_dates[:-1]:
sat_files = glob.glob(
dd.strftime("{}/infos_*%Y%m%d%H%M.nc").format(
plugin.dir_satellites))
for obs_file in sat_files:
target = "{}/obsvect/satellites/{}".format(
plugin.workdir, os.path.basename(obs_file))
path.link(obs_file, target)
plugin.has_satellites = False
if np.where((plugin.datastore["level"] < 0))[0].size > 0:
plugin.has_satellites = True
def default_fetch(ref_dir,
ref_file,
input_dates,
target_dir,
tracer=None,
**kwargs):
# Picking info from tracer if not in inputs
if ref_dir == "" and hasattr(tracer, "dir"):
input_dir = tracer.dir
else:
input_dir = ref_dir
if ref_file == "" and hasattr(tracer, "file"):
input_file = tracer.file
else:
input_file = ref_file
info("Fetching input files using directory and file format")
info("{}/{}".format(input_dir, input_file))
list_files = {}
list_dates = {}
for datei in input_dates:
tmp_files = []
tmp_dates = []
for dd in input_dates[datei]:
dir_dd = dd.strftime(input_dir)
file_dd = dd.strftime(input_file)
tmp_files.append("{}/{}".format(dir_dd, file_dd))
# Fetching
local_files = []
for f in tmp_files:
target_file = "{}/{}".format(target_dir, os.path.basename(f))
path.link(f, target_file)
local_files.append(target_file)
list_files[datei] = list(set(local_files))
list_dates[datei] = list(set(tmp_dates))
return list_files, list_dates
def make_meteo(self, data, ddi, ddf, runsubdir, mode):
datei = min(ddi, ddf)
dir_meteo = data.datastore[("meteo", "")]["dirorig"]
# Links meteorological mass fluxes
for ftype in ["defstoke", "fluxstoke", "fluxstokev", "phystoke"]:
meteo_file = "{}.an{}.m{:02d}.nc".format(
ftype, datei.year, datei.month
)
source = "{}/{}".format(dir_meteo, meteo_file)
target = "{}/{}.nc".format(runsubdir, ftype)
if ftype == "defstoke" and not os.path.isfile(source):
meteo_file = "{}.nc".format(ftype)
source = "{}/{}".format(dir_meteo, meteo_file)
path.link(source, target)
# Links vertical coordinates from previous simulations
source = self.file_vcoord
target = "{}/vcoord.nc".format(runsubdir)
path.link(source, target)
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs):
list_files = {}
list_dates = {}
for datei in input_dates:
tmp_files = []
tmp_dates = []
for dd in input_dates[datei]:
file_flx = dd.strftime(ref_file)
dir_flx = dd.strftime(ref_dir)
date_flx = dd
if not os.path.isfile(
"{}/{}".format(dir_flx, file_flx)) and getattr(
tracer, "closest_year", False):
info("Warning: could not find correct year for EDGAR; "
"using closest available one")
list_dates_avail = [
datetime.datetime.strptime(os.path.basename(f), ref_file)
for f in glob.glob("{}/v50_*nc".format(dir_flx))
]
delta_dates = np.abs(dd - np.array(list_dates_avail))
date_flx = list_dates_avail[np.argmin(delta_dates)]
file_flx = date_flx.strftime(ref_file)
tmp_files.append("{}/{}".format(dir_flx, file_flx))
tmp_dates.append(date_flx)
# Fetching
local_files = []
for f in tmp_files:
target_file = "{}/{}".format(target_dir, os.path.basename(f))
path.link(f, target_file)
local_files.append(target_file)
list_files[datei] = list(set(local_files))
list_dates[datei] = list(set(tmp_dates))
return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs):
if ref_dir == '' and hasattr(tracer, 'dir'):
input_dir = tracer.dir
else:
input_dir = ref_dir
if ref_file == '' and hasattr(tracer, 'file'):
input_file = tracer.file
else:
input_file = ref_file
list_files = {
datei: datei.strftime('{}/{}'.format(input_dir, input_file))
for datei in input_dates
}
if hasattr(tracer, "file_glob"):
list_files_glo = \
{datei: datei.strftime('{}/{}'.format(input_dir, tracer.file_glob))
for datei in input_dates}
list_dates = {datei: input_dates[datei] for datei in input_dates}
# Fetching
for datei in input_dates:
f = list_files[datei]
target_file = '{}/{}'.format(target_dir, os.path.basename(f))
path.link(f, target_file)
list_files[datei] = target_file
# Initialize global file for later
if hasattr(tracer, "file_glob"):
f = list_files_glo[datei]
target_file = '{}/{}'.format(target_dir, os.path.basename(f))
path.link(f, target_file)
return list_files, list_dates
def fetch(
ref_dir, ref_file, input_dates, target_dir, tracer=None, component=None
):
if ref_dir == "" and hasattr(tracer, "dir"):
input_dir = tracer.dir
else:
input_dir = ref_dir
if ref_file == "" and hasattr(tracer, "file"):
input_file = tracer.file
else:
input_file = ref_file
# Two cases: initial conditions or LBC
if getattr(component, "orig_name", "") == "inicond":
datei = np.min(list(input_dates.keys()))
list_files = {
datei: datei.strftime("{}/{}".format(input_dir, input_file))
}
list_dates = {datei: []}
else:
list_files = {
datei: datei.strftime("{}/{}".format(input_dir, input_file))
for datei in input_dates
}
list_dates = {datei: [] for datei in input_dates}
# Fetching
for datei in input_dates:
f = list_files[datei]
target_file = "{}/{}".format(target_dir, os.path.basename(f))
path.link(f, target_file)
list_files[datei] = os.path.basename(target_file)
return list_files, list_dates
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs):
list_files = {}
list_dates = {}
for datei in input_dates:
tmp_files = []
tmp_dates = []
for dd in input_dates[datei]:
dir_dd = dd.strftime(ref_dir)
files_3d, dates_3d = find_valid_file(dir_dd, tracer.file_3d, dd)
tmp_files.extend(files_3d)
tmp_dates.extend(dates_3d)
# Fetching
local_files = []
for f in tmp_files:
target_file = "{}/{}".format(target_dir, os.path.basename(f))
path.link(f, target_file)
local_files.append(target_file)
list_files[datei] = list(set(local_files))
list_dates[datei] = list(set(tmp_dates))
return list_files, list_dates
def make_fluxes(self, data, runsubdir, datei, mode):
"""Make AEMISSIONS.nc and BEMISSIONS.nc files for CHIMERE.
Use chemical scheme to check which species is needed and either take it
from the datastore (i.e. when defined in the control vector), or take it
from prescribed emissions
Args:
self (pycif.utils.classes.Fluxes.fluxes): Flux plugin with all
attributes
datastore (dict): information on flux species
runsubdir (str): directory to the current run
nho (int): number of hours in the run
mode (str): running mode: 'fwd', 'tl' or 'adj'
"""
datastore = {
trid: data.datastore[trid]
for trid in data.datastore
if trid[0] in ["fluxes", "biofluxes"]
}
# List of dates for which emissions are needed
list_dates = pd.date_range(datei, periods=self.nhours + 1, freq="H")
# Getting the right emissions
# Loop on all anthropogenic and biogenic species
# If in datastore, take data, otherwise, link to original A/B EMISSIONS
list_trid = [("fluxes", spec)
for spec in self.chemistry.emis_species.attributes] \
+ [("biofluxes", spec)
for spec in self.chemistry.bio_species.attributes]
for trid in list_trid:
spec = trid[1]
emis_type = trid[0]
flx_plg = self.fluxes if emis_type == "anthro" else self.biofluxes
if trid in datastore:
pass
# If spec not explicitly defined in datastore,
# fetch general component information if available
elif trid not in datastore and (emis_type, "") in datastore:
trid = (emis_type, "")
else:
continue
# Bio or anthro file
if emis_type == "fluxes":
file_emisout = "{}/AEMISSIONS.nc".format(runsubdir)
file_emisincrout = "{}/AEMISSIONS.increment.nc".format(
runsubdir)
else:
file_emisout = "{}/BEMISSIONS.nc".format(runsubdir)
file_emisincrout = "{}/BEMISSIONS.increment.nc".format(runsubdir)
tracer = datastore[trid]
dirorig = tracer["dirorig"]
fileorig = tracer["fileorig"]
fileemis = datei.strftime("{}/{}".format(dirorig, fileorig))
# If no data is provided, just copy from original file
if "spec" not in tracer:
linked = False
# If does not exist, just link
if not os.path.isfile(file_emisout):
path.link(fileemis, file_emisout)
linked = True
# Otherwise, check for difference
if not linked:
if not filecmp.cmp(fileemis, file_emisout):
with Dataset(fileemis, "r") as fin:
emisin = fin.variables[spec][:]
emisin = xr.DataArray(
emisin,
coords={"time": list_dates},
dims=("time", "lev", "lat", "lon"),
)
flx_plg.write(spec, file_emisout, emisin)
# Repeat operations for tangent linear
if mode != "tl":
continue
if "spec" not in tracer:
# If does not exist, just link
if not os.path.isfile(file_emisincrout):
shutil.copy(fileemis, file_emisincrout)
flx_incr = xr.DataArray(
np.zeros(
(
len(list_dates),
self.nlevemis if emis_type == "fluxes"
else self.nlevemis_bio,
self.domain.nlat,
self.domain.nlon,
)
),
coords={"time": list_dates},
dims=("time", "lev", "lat", "lon"),
)
flx_plg.write(spec, file_emisincrout, flx_incr)
else:
# Replace existing link by copy of original file to modify it
path.copyfromlink(file_emisout)
# Put in dataset and write to input
flx_fwd = datastore[trid]["spec"]
flx_plg.write(spec, file_emisout, flx_fwd)
if mode == "tl":
path.copyfromlink(file_emisincrout)
flx_tl = datastore[trid].get("incr", 0.0 * flx_fwd)
flx_plg.write(spec, file_emisincrout, flx_tl)
def native2inputs(self,
datastore,
input_type,
datei,
datef,
runsubdir,
mode="fwd"):
"""Converts data at the model data resolution to model compatible input
files.
Args:
self: the model Plugin
input_type (str): one of 'fluxes'
datastore: data to convert
if input_type == 'fluxes',
datei, datef: date interval of the sub-simulation
mode (str): running mode: one of 'fwd', 'adj' and 'tl'
runsubdir (str): sub-directory for the current simulation
workdir (str): the directory of the whole pyCIF simulation
Notes:
- CHIMERE expects hourly inputs;
"""
ddi = min(datei, datef)
ddf = max(datei, datef)
# Hour steps of the sub-run
hour_dates = pd.date_range(ddi, ddf, freq="1H")
if datastore is None:
datastore = {}
sdc = ddi.strftime("%Y%m%d%H")
# Choose the right executable
if input_type == "exe":
if mode == "fwd":
source = "fwdchimere.e"
elif mode == "tl":
source = "tlchimere.e"
elif mode == "adj":
source = "achimere.e"
else:
raise Exception(
"Unknown mode '{}' for executable with CHIMERE".format(mode))
path.link(
"{}/model/{}".format(self.workdir, source),
"{}/chimere.e".format(runsubdir),
)
# Deals with the nml file
if input_type == "nml":
make_nml(self, runsubdir, sdc, mode)
# Deals with fluxes
# WARNING: Not specified emissions are set to zero??
if input_type in ["fluxes", "biofluxes"]:
make_fluxes(self, datastore, runsubdir, ddi, mode)
if input_type in ["latcond", "topcond"]:
make_boundcond(self, datastore, runsubdir, ddi, mode, input_type)
if input_type == "inicond":
make_inicond(self, datastore, runsubdir, mode, ddi)
if input_type == "meteo":
make_meteo(self, runsubdir, sdc)
# Deals with observations
if input_type == "obs":
make_obs(self, datastore, runsubdir, mode)
return
def make_inicond(self, data, runsubdir, mode, datei):
datastore = {
trid: data.datastore[trid]
for trid in data.datastore if trid[0] == "inicond"
}
# Fixed name for INI_CONCS files
fileout = "{}/INI_CONCS.0.nc".format(runsubdir)
fileoutincr = "{}/INI_CONCS.0.increment.nc".format(runsubdir)
nho = self.nho
# Getting the forward initial concentrations (needed even for adjoint)
# if chained period
if hasattr(self, "chain"):
if mode in ["tl", "fwd"]:
filein = self.chain.strftime(
"{}/../chain/end.%Y%m%d%H.{}.nc".format(runsubdir, nho))
else:
subsimu_dates = self.subsimu_dates
date_index = np.where(subsimu_dates == datei)[0][0]
refdir = self.adj_refdir
filein = subsimu_dates[date_index - 1].strftime(
"{}/chain/end.%Y%m%d%H.{}.nc".format(refdir, nho))
path.link(filein, fileout)
# Adjoint needs to chain sensitivities as well
if mode == "adj":
filein = self.chain.strftime(
"{}/../chain/aend.%Y%m%d%H.{}.nc".format(runsubdir, nho))
fileout = self.chain.strftime("{}/aini.nc".format(runsubdir))
path.link(filein, fileout)
# Exit if not first period
if datei != self.datei:
return
# Loop on all active species
# If in datastore, take data, otherwise, link to original INI_CONCS
for spec in self.chemistry.acspecies.attributes:
trid = ("inicond", spec)
if trid in datastore:
pass
# If spec not explicitly defined in datastore,
# fetch general component information if available
elif trid not in datastore and ("inicond", "") in datastore:
trid = ("inicond", "")
else:
continue
tracer = datastore[trid]
dirorig = tracer["dirorig"]
fileorig = tracer["fileorig"]
fileini = "{}/{}".format(dirorig, fileorig)
# If no data is provided, just copy from original file
if "spec" not in tracer:
# If does not exist, just link
linked = False
if not os.path.isfile(fileout):
path.link(fileini, fileout)
linked = True
# Otherwise, check for difference
if not linked:
if not filecmp.cmp(fileini, fileout):
raise Exception("I need to transfer inicond "
"from one file to the other one")
# Repeat operations for tangent linear
if mode != "tl":
continue
# If does not exist, just link
if not os.path.isfile(fileoutincr):
shutil.copy(fileini, fileoutincr)
with Dataset(fileoutincr, "a") as fout:
if spec in fout.variables:
fout.variables[spec][:] = 0.0
else:
# Replace existing link by copy
# of original file to modify it
path.copyfromlink(fileout)
# Write initial conditions
ini_fwd = datastore[trid]["spec"]
self.inicond.write(spec, fileout, ini_fwd, comp_type="inicond")
if mode == "tl":
path.copyfromlink(fileoutincr)
ini_tl = datastore[trid].get("incr", 0.0 * ini_fwd)
self.inicond.write(spec,
fileoutincr,
ini_tl,
comp_type="inicond")
def make_input(self,
data,
mod_input,
di,
df,
runsubdir,
mode,
lmdz_yearref=1979,
**kwargs):
"""Prepares inputs for the current LMDZ simulation. It includes:
- meteo files (defstoke.nc, fluxstoke.nc fluxstokev.nc and phystoke.nc)
- def files (run.def, totinput, ACTIVE_SPECIES)
- initial conditions (start.nc)
- fields for SACS (inca.nc, incaMCF.nc)
Args:
- self (Plugin): LMDZ plugin
- mod_input (str): one of: 'meteo', 'def', 'inicond', 'chem_fields'
- di (datetime): beginning of the simulation window (along the fwd
or bckwd axis)
- df (datetime): end of simulation
- workdir (str): path to the pycif simulation
- runsubdir (str): path to the LMDZ sub-simulation
- mode (str): running mode; fwd or adj
- lmdz_yearref (int): reference year for LMDZ dates. Default is 1979
"""
datei = min(di, df)
if mod_input == "meteo":
make_meteo(self, data, di, df, runsubdir, mode)
elif mod_input == "def":
# run.def
# make_rundef(self.filedef, datei, self.physic, runsubdir, lmdz_yearref)
# totinput
make_totinput(self, runsubdir, datei, mode)
# Remove restart TL
if mode != "tl":
shutil.rmtree("{}/*start_tl.bin".format(runsubdir),
ignore_errors=True)
# TODO: at the moment, no satellite observation
os.system("echo 0 > {}/infousedsat.txt".format(runsubdir))
elif mod_input == "chem_fields":
if hasattr(self, "chemistry"):
# Linking to pre-computed INCA fields for kinetics
source = datei.strftime("{}/{}".format(
self.chemistry.kinetic.dir, self.chemistry.kinetic.file))
target = "{}/kinetic.nc".format(runsubdir)
path.link(source, target)
# Linking to pre-computed INCA fields for deposition
if hasattr(self.chemistry, "deposition"):
for spec in self.chemistry.deposition.attributes:
tracer = getattr(self.chemistry.deposition, spec)
source = datei.strftime("{}/{}".format(
tracer.dir, tracer.file))
target = "{}/dep_{}.nc".format(runsubdir, spec)
path.link(source, target)
finf = "{}/chemical_scheme.nml".format(self.chemistry.dirchem_ref)
target = "{}/chemical_scheme.nml".format(runsubdir)
path.link(finf, target)
elif mod_input == "traj":
if mode == "adj":
if not hasattr(self, "adj_refdir"):
info("Adjoint LMDZ couldn't be initialized "
"with forward trajq.bin files")
raise Exception
else:
for spec in self.chemistry.acspecies.attributes:
traj_file = "trajq_{}_%Y%m%d%H%M.bin".format(spec)
source = datei.strftime("{}/chain/{}".format(
self.adj_refdir, traj_file))
target = "{}/trajq_{}.bin".format(runsubdir, spec)
path.link(source, target)
elif mod_input == "exe":
# Linking executable
source = "{}/model/dispersion.e".format(self.workdir)
target = "{}/dispersion.e".format(runsubdir)
path.link(source, target)
def make_meteo(self, runsubdir, sdc):
# use ready-made METEO.nc files
# Getting the right one
nho = self.nho
filemet = self.meteo.dir + "METEO." + sdc + "." + str(nho) + ".nc"
path.link(filemet, runsubdir + "/METEO.nc")
def make_boundcond(self, data, runsubdir, ddi, mode, input_type):
"""
Generates boundary conditions files for CHIMERE
:param self:
:param datastore:
:type datastore: dict
:param runsubdir:
:type runsubdir: str
:param sdc:
:type sdc: str
:param hour_dates:
:param mode:
:param input_type:
:return:
"""
datastore = {
trid: data.datastore[trid]
for trid in data.datastore if trid[0] == input_type
}
# Name of variable in netCDF
nc_varname = "top_conc" if input_type == "topcond" else "lat_conc"
# Fixed name for BC files
fileout = "{}/BOUN_CONCS.nc".format(runsubdir)
fileoutincr = "{}/BOUN_CONCS.increment.nc".format(runsubdir)
# Loop on all active species
# If in datastore, take data, otherwise, link to original INI_CONCS
for spec in self.chemistry.acspecies.attributes:
trid = (input_type, spec)
if trid in datastore:
pass
# If spec not explicitly defined in datastore,
# fetch general component information if available
elif trid not in datastore and (input_type, "") in datastore:
trid = (input_type, "")
else:
continue
tracer = datastore[trid]
dirorig = tracer["dirorig"]
fileorig = tracer["fileorig"]
fileini = ddi.strftime("{}/{}".format(dirorig, fileorig))
# If no data is provided, just copy from original file
if "spec" not in tracer:
# If does not exist, just link
if not os.path.isfile(fileout):
path.link(fileini, fileout)
# # Otherwise, check for difference
# elif not filecmp.cmp(fileini, fileout):
#
# print(__file__)
# import code
# code.interact(local=dict(locals(), **globals()))
# raise Exception('I need to transfer lbc '
# 'from one file to the other one')
else:
# Replace existing link by copy
# of original file to modify it
path.copyfromlink(fileout)
# Write initial conditions
lbc_fwd = datastore[trid]["spec"]
self.latcond.write(spec, fileout, lbc_fwd, comp_type=input_type)
if mode == "tl":
path.copyfromlink(fileoutincr)
lbc_tl = datastore[trid].get("incr", 0.0 * lbc_fwd)
self.latcond.write(spec,
fileoutincr,
lbc_tl,
comp_type=input_type)
# Repeat operations for tangent linear
if mode != "tl":
continue
if "spec" not in tracer:
# If does not exist, just link
if not os.path.isfile(fileoutincr):
shutil.copy(fileini, fileoutincr)
with Dataset(fileoutincr, "a") as fout:
nc_var_out = fout.variables[nc_varname][:]
nc_names = [
str(b"".join(s).strip().lower(), "utf-8")
for s in fout.variables["species"][:]
]
if spec.lower() not in nc_names:
continue
# Apply to original data
nc_var_out[..., nc_names.index(spec.lower())][:] = 0.0
fout.variables[nc_varname][:] = nc_var_out
def run(self, runsubdir, mode, workdir, do_simu=True, **kwargs):
"""Run dummy_txt Gaussian model in forward or adjoint mode
Args:
runsubdir (str): working directory for the current run
mode (str): forward or backward
workdir (str): pycif working directory
do_simu (bool): if False, considers that the simulation was
already run
"""
# Ignore warnings in the toy model
warnings.simplefilter("ignore")
# Do the simulation
if do_simu:
# Linking Pasquill-Gifford classifileation
source = "{}/model/Pasquill-Gifford.txt".format(workdir)
target = "{}/Pasquill-Gifford.txt".format(runsubdir)
path.link(source, target)
# If H was already computed, take it from previous simulation
info("Running the model Dummy in {}".format(runsubdir))
dref = self.meteo.data.index.min()
if dref in self.H_matrix:
H = self.H_matrix[dref]
else:
H = fortran_exe(self, target, mode)
# Saving H for later computations if required
if self.save_H:
self.H_matrix[dref] = H
# Compute H.x or H^T.dy depending on the running mode
meteo_index = self.dataobs["tstep"]
if mode in ["fwd", "tl"]:
# In fwd and tl, simple product of flux and footprints
self.dataobs["sim"] = np.nansum(
self.dataflx.values[meteo_index, 0] * H, axis=(1, 2)
)
self.dataobs["sim_tl"] = np.nansum(
self.dataflx_tl.values[meteo_index, 0] * H, axis=(1, 2)
)
else:
# In adjoint, aggregation of footprints, to model resolution
dflx = self.dataobs["obs_incr"][:, np.newaxis, np.newaxis] * H
dflx[np.isnan(dflx)] = 0.0
dflx = xr.DataArray(
dflx[:, np.newaxis],
coords={
"time": meteo_index,
"lev": [0],
"lat": list(range(self.domain.nlat)),
"lon": list(range(self.domain.nlon)),
},
dims=("time", "lev", "lat", "lon"),
)
# Summing observations at the same hour and reprojecting to meteo
# resolution if missing observations
dflx = (
dflx.to_dataframe(name="flx")
.groupby(["time", "lev", "lat", "lon"])
.sum()
)
dflx = dflx.unstack(level=["lev", "lat", "lon"])
dflx.index = self.meteo.data.index[dflx.index]
dflx = dflx.reindex(self.meteo.data.index).stack(
level=["lev", "lat", "lon"], dropna=False
)
dflx = xr.Dataset.from_dataframe(dflx)["flx"]
dflx = dflx.rename({"date": "time"})
# Filling NaNs with 0, as it corresponds to hours with no obs
dflx = dflx.fillna(0.0)
# Saving sensitivity to the model object
# It corresponds to outputs in classical numerical models
self.dflx = dflx
def init_y0(obsvect, measurements, **kwargs):
"""Initializes the observation vector. In most cases the observation
vector is similar to the measurement vector but there is no reason
for it to be the same, especially when super-observations are used
(e.g. daily or afternoon averages, gradients, etc.)
Args:
obsvect (Plugin): the observation vector with all its attributes
measurements (Plugin): the pre-loaded measurements
Returns:
obsvect, updated with horizontal and vertical coordinates,
as well as model time steps
"""
# Saves initialized observation vector to workdir/obsvect/monitor
# Try linking file_obsvect to workdir/obsvect if existing
# Otherwise, define it and generate it later on
path.init_dir("{}/obsvect".format(obsvect.workdir))
file_monitor = "{}/obsvect/monitor.{}".format(obsvect.workdir,
obsvect.dump_type)
shutil.rmtree(file_monitor, ignore_errors=True)
if os.path.isfile(obsvect.file_obsvect):
path.link(obsvect.file_obsvect, file_monitor)
# From here, no interaction with files outside the working directory
obsvect.file_obsvect = file_monitor
# Try reading file
allcorrec, ok_hcoord, ok_vcoord, do_tstep = False, False, False, True
exclude_stations = getattr(obsvect, "exclude_stat", [])
try:
# Reading a monitor file if available
info("Try opening {}".format(file_monitor))
obsvect.datastore = read_datastore(file_monitor,
dump_type=obsvect.dump_type)
# Check that the monitor is consistent with the simulation to be run
# If True, returns directly the observation as is
allcorrect, ok_hcoord, ok_vcoord, do_tstep = obsvect.check_monitor()
if allcorrect and len(exclude_stations) == 0:
return obsvect
# Otherwise, recompute necessary items (i, j, lev, tstep)
# of the observation error
raise PluginError
# If the monitor file could not be opened, start back from measurements
except IOError:
info("Couldn't open the observation file")
info("Generating observation vector from measurements")
# Copying attributes from measurements
for key in measurements.attributes:
setattr(obsvect, key, getattr(measurements, key))
# Copying datastore from measurements if existing and not empty
obsvect.datastore = getattr(measurements, "datastore", init_empty())
except PluginError:
info("Warning! The prescribed monitor file is not consistent with "
"the current simulation. Re-analysing it")
# Remove unwanted stations
if len(exclude_stations) > 0:
mask = ~(obsvect.datastore["station"].isin(exclude_stations))
obsvect.datastore = obsvect.datastore.loc[mask]
# Crops y0 to the simulation period
obsvect.datastore = crop_monitor(obsvect.datastore, obsvect.datei,
obsvect.datef, **kwargs)
# Computes grid cells where observations fall
if not ok_hcoord:
obsvect = hcoord(obsvect, **kwargs)
# Computes model layer where observations fall
if not ok_vcoord:
obsvect = vcoord(obsvect, **kwargs)
# Compute time steps corresponding to the observations
if do_tstep:
obsvect = tstep(obsvect, **kwargs)
# Dumps the datastore
# Attributes to keep in the monitor if NetCDF format (recommanded!)
nc_attributes = {
"datei": obsvect.datei.strftime("%d-%m-%Y %H:%M:%S"),
"datef": obsvect.datef.strftime("%d-%m-%Y %H:%M:%S"),
"model name": obsvect.model.plugin.name,
"model version": obsvect.model.plugin.version,
"domain nlon": str(obsvect.model.domain.nlon),
"domain nlat": str(obsvect.model.domain.nlat),
}
if getattr(obsvect, "dump", True):
dump_datastore(
obsvect.datastore,
file_monit=obsvect.file_obsvect,
dump_type=obsvect.dump_type,
nc_attributes=nc_attributes,
mode="w",
)
return obsvect
def make_inicond(self, data_in, datei, datef, runsubdir, mode):
ddi = min(datei, datef)
ddf = max(datei, datef)
# Reference initial condition file for looping sub-simulations
if hasattr(self, "chain"):
source = self.chain.strftime(
"{}/../chain/restart_%Y%m%d%H%M.nc".format(runsubdir))
target = "{}/start.nc".format(runsubdir)
path.link(source, target)
if mode == "tl":
source = self.chain.strftime(
"{}/../chain/restart_tl_%Y%m%d%H%M.bin".format(runsubdir))
target = "{}/start_tl.bin".format(runsubdir)
path.link(source, target)
return
# Generating reference initial conditions if first sub-simulation
datastore = data_in.datastore
for spec in self.chemistry.acspecies.attributes:
restartID = getattr(self.chemistry.acspecies, spec).restart_id
if ("inicond", spec) not in datastore:
continue
ini = datastore[("inicond", spec)]
ref_dir = ini["dirorig"]
ref_file = ini["fileorig"]
# Copy if does not already exists
source = ddi.strftime("{}/{}".format(ref_dir, ref_file))
target = "{}/start.nc".format(runsubdir)
if not os.path.isfile(target):
shutil.copy(source, target)
if mode == "tl":
target_tl = "{}/start_tl.nc".format(runsubdir)
if not os.path.isfile(target_tl):
shutil.copy(source, target_tl)
# Updates data
if mode in ["fwd", "tl"] and ddi == self.datei:
var = "q{:02d}".format(restartID)
if "spec" in ini:
data = ini["spec"]
with Dataset(target, "a") as f:
if var in f.variables:
f.variables[var][:] = data.values
else:
self.inicond.write(var, target, data)
if mode == "tl":
if "incr" in ini:
data = ini["incr"]
with Dataset(target_tl, "a") as f:
if var in f.variables:
f.variables[var][:] = data.values
else:
self.inicond.write(var, target_tl, data)
else:
with Dataset(target_tl, "a") as f:
if var in f.variables:
f.variables[var][:] = 0.0
elif mode == "adj" and ddf == self.datef:
for spec in self.chemistry.acspecies.attributes:
restartID = getattr(self.chemistry.acspecies, spec).restart_id
with Dataset(target, "a") as f:
var = "q{:02d}".format(restartID)
if var in f.variables:
f.variables[var][:] = 0.0