def vcoord(obsvect, **kwargs): """Computes the vertical layer in which fall the observations """ info("Finding model levels corresponding to observations") # Don't do anything if the datastore is empty if len(obsvect.datastore) == 0: return obsvect # If a file with fixed vertical coordinates is specified, use it if hasattr(obsvect, "file_statlev"): file_statlev = obsvect.file_statlev info( "Using pre-defined vertical coordinates for stations: {}".format( file_statlev ) ) obsvect.datastore = vcoordfromfile( obsvect.datastore, file_statlev, **kwargs ) # Else compute vertical coordinates from meteo files # To be coded from pyCIF-CHIMERE # To be generalized by using meteo plugin else: obsvect.datastore = vcoordfrommeteo( obsvect.workdir, obsvect.datastore, **kwargs ) return obsvect
def loop_zonal_stats(lyr, raster, resol=10, option="mean", return_weight=False): featList = range(lyr.GetFeatureCount()) statList = [] for FID in featList: try: feat = lyr.GetFeature(FID) meanValue = zonal_stats( feat, raster, resol=resol, option=option, return_weight=return_weight, ) except Exception as e: info(e) meanValue = raster.RasterCount * [np.nan] raise e if FID % (len(featList) / 10) == 0: info(FID, len(featList)) statList.append(meanValue) if return_weight: return statList else: return np.array(statList)
def ini_data(plugin, **kwargs): """Initializes the observation operator Args: plugin (dict): dictionary defining the plugin **kwargs (dictionary): possible extra parameters """ info("Initializing the observation operator") workdir = plugin.workdir # Initializes the directory path.init_dir("{}/obsoperator".format(workdir)) # Initializes transforms init_transform(plugin, plugin.statevect) init_transform(plugin, plugin.obsvect, transform_type="obs") # Re-compile model if necessary if hasattr(plugin.model, "compile"): plugin.model.compile() return plugin
def ini_data(self, **kwargs): """Initializes the chemistry depending on the model used for the inversion. Args: plugin (ChemistryPlugin): chemistry definition Returns: Updates on the fly the chemistry """ info("Initializing the Chemistry") # Copying the chemical scheme to the working directory workdir = self.workdir dirchem_ref = "{}/chemical_scheme/{}/".format(workdir, self.schemeid) self.dirchem_ref = dirchem_ref shutil.rmtree(dirchem_ref, ignore_errors=True) init_dir(dirchem_ref) # If pre-computed scheme is specified if hasattr(self, "dir_precomp"): copy_tree("{}/{}/".format(self.dir_precomp, self.schemeid), dirchem_ref) # Read chemistry self.read_chemicalscheme(**kwargs) # Otherwise, initialize files from the yaml else: self.create_chemicalscheme()
def ini_data(self, **kwargs): """Initializes the domain depending on the model used for the inversion. Defines a domain grid from a grid file or a set of parameters if the domain was not already defined. Domains are model dependant, so the outputs can be different depending on the model. Args: plugin (DomainPlugin): domain definition Returns: Updates on the fly the domain """ # Read or create a domain try: # Read domain self.read_grid(**kwargs) except (IOError, AttributeError): # Generate a domain info("Couldn't read the domain. Generating it.") self.create_domain(**kwargs) # Compute areas that can be needed for emissions or diagnostics if not hasattr(self, "areas") and getattr(self, "compute_areas", False): info("Computing areas") self.calc_areas(**kwargs)
def check_monitor(self): """Check the consistency between the observation datastore and the model configuration set up""" datastore = self.datastore # For old monitor with no nc_attributes, do nothing if not hasattr(datastore, "nc_attributes"): info("Cannot check the datastore from a previous version." "Please be careful with the use of it") return True, True, True, False # Otherwise, check what part of the monitor is to be re-computed nc_attributes = datastore.nc_attributes ok_hcoord = (nc_attributes.get("domain nlat", None) == str(self.model.domain.nlat)) and (nc_attributes.get( "domain nlon", None) == str(self.model.domain.nlon)) ok_vcoord = True ok_tstep = ( nc_attributes.get("datei", None) == self.datei.strftime("%d-%m-%Y %H:%M:%S")) and (nc_attributes.get( "datef", None) == self.datef.strftime("%d-%m-%Y %H:%M:%S")) allcorrec = ok_hcoord and ok_tstep and ok_vcoord return allcorrec, ok_hcoord, ok_vcoord, not ok_tstep
def run_simu(cls, args): # Dealing with relative and variable path def_file = os.path.abspath(os.path.expanduser(args["def_file"])) # Loading Yaml setup = cls.yaml_to_setup(def_file) setup = cls.load_config(setup) # Copying Yaml file for traceability of simulations os.system("cp " + setup.def_file + " " + setup.workdir + "/") # Saving the loaded configuration if getattr(setup, "dump_config", False): cls.to_yaml( setup, "{}/loaded.{}".format(setup.workdir, os.path.basename(setup.def_file)), ) # Run the mode if getattr(getattr(setup, "mode", None), "loaded_requirements", False): return setup.mode.execute(**args) else: info("pycif has correctly been initialized " "but no execution mode was specified")
def fetch( ref_dir, ref_file, input_dates, target_dir, tracer=None, filetypes=["defstoke", "fluxstoke", "fluxstokev", "phystoke"], **kwargs ): """Reads meteorology and links to the working directory Args: meteo (dictionary): dictionary defining the domain. Should include dirmeteo to be able to read the meteorology datei (datetime.datetime): initial date for the inversion window datef (datetime.datetime): end date for the inversion window workdir (str): path to the working directory where meteo files should be copied logfile (str): path to the log file filetypes ([str]): list of file radicals to copy in the working directory **kwargs (dictionary): extra arguments Return: ???????? Notes: At some point, include option to compute mass fluxes for LMDz, with different physics What is needed to do that? Possible only on CCRT? Flexibility to define new domains Can be very heavy and not necessarily relevant """ info("Copying meteo files from {} to {}".format(ref_dir, target_dir)) # Create the sub-directory to store meteo files path.init_dir(target_dir) # Loop over dates and file types for date in input_dates: for filetype in filetypes: meteo_file = "{}.an{}.m{:02d}.nc".format( filetype, date.year, date.month ) if filetype == "defstoke" and not os.path.isfile( ref_dir + meteo_file ): meteo_file = filetype + ".nc" target = "{}/{}".format(target_dir, meteo_file) source = "{}/{}".format(ref_dir, meteo_file) path.link(source, target) list_files = {datei: [] for datei in input_dates} list_dates = {datei: [] for datei in input_dates} return list_files, list_dates
def read(self, name, tracdir, tracfile, varnames, dates, interpol_flx=False, **kwargs): """Get fluxes from pre-computed fluxes and load them into a pycif variables Args: self: the model Plugin name: the name of the component tracdir, tracfile: flux directory and file format dates: list of dates to extract interpol_flx (bool): if True, interpolates fluxes at time t from values of surrounding available files """ list_file_flx = [dd.strftime(tracfile) for dd in dates] trcr_dates = [] trcr_flx = [] trcr_flx_tl = [] for dd, file_flx in zip(dates, list_file_flx): # Read binary file data = [] with FortranFile("{}/{}".format(tracdir, file_flx)) as f: while True: try: data.append(f.read_reals()) except BaseException: info("End of file {}".format(file_flx)) break # Reshape file nlon = self.domain.nlon nlat = self.domain.nlat data = np.array(data) flx = data[:, 0].reshape((-1, nlat, nlon)) flx_tl = data[:, 1].reshape((-1, nlat, nlon)) trcr_flx.append(flx) trcr_flx_tl.append(flx_tl) trcr_dates.extend(list(pd.date_range(dd, freq="D", periods=len(flx)))) xmod = xr.DataArray(trcr_flx[0], coords={"time": trcr_dates}, dims=("time", "lat", "lon")) return xmod
def execute(self, **kwargs): """Runs the model in forward mode Args: setup (Plugin): definition of the full set-up """ # Working directory workdir = self.workdir # Control vector statevect = self.statevect # Observation operator obsoper = self.obsoperator # Simulation window datei = self.datei datef = self.datef # Some verbose info("Running a direct run") # Putting x at xb value if available if hasattr(statevect, "xb"): statevect.x = statevect.xb # Running the observation operator obsvect = obsoper.obsoper(statevect, "fwd", datei=datei, datef=datef, workdir=workdir, **kwargs) # Perturbs the output monitor if required in the Yaml if getattr(self, "perturb_obsvect", False): # Altering obsvect and save data obserror = self.obserror * obsvect.datastore["sim"].mean() obsvect.datastore["obs"] = (np.random.normal( loc=0, scale=obserror, size=obsvect.datastore.index.size) + obsvect.datastore["sim"]) obsvect.datastore["obserror"] = obserror # Dumping the datastore with reference data dump_datastore( obsvect.datastore, file_monit=obsvect.file_obsvect, dump_type="nc", mode="w", ) return obsvect
def read_grid(domain, **kwargs): """Reads a grid from an existing file Args: domain (Plugin): dictionary defining the domain. Should include filegrid to be able to read the grid from a file Return: Grid dictionary with meshgrids for center lon/lat and corner lon/lat Notes: Coordinates are in meters from a reference point """ # Tries open filelon, filelat try: zlon = np.loadtxt(domain.filelon) zlat = np.loadtxt(domain.filelat) nlon = zlon.size nlat = zlat.size # Corner coordinates dlon = np.ptp(zlon) / (nlon - 1) / 2.0 zlonc = zlon - dlon zlonc = np.append(zlonc, zlonc[-1] + 2 * dlon) dlat = np.ptp(zlat) / (nlat - 1) / 2.0 zlatc = zlat - dlat zlatc = np.append(zlatc, zlatc[-1] + 2 * dlat) # Meshgrids zlon, zlat = np.meshgrid(zlon, zlat) zlonc, zlatc = np.meshgrid(zlonc, zlatc) # Saving information to domain attributes domain.nlon = nlon domain.nlat = nlat domain.zlon = zlon domain.zlat = zlat domain.zlonc = zlonc domain.zlatc = zlatc except (IOError, AttributeError): info("Couldn't read longitudes and latitudes.\n" "Make them from given coordinates") domain.create_domain() # Compute areas in m2 domain.areas = (np.diff(domain.zlatc, axis=1)[:-1] * np.diff(domain.zlonc, axis=0)[:, :-1]) # Projection not as GPS domain.projection = "xy"
def read(self, name, tracdir, tracfile, varnames, dates, interpol_flx=False, tracer=None, model=None, **kwargs): # tracfile can be a list of same length as dates try: if len(tracfile) != len(dates): raise Exception( "Try read EDGAR files from a list of dates and a " "list of files, but not of same length:\n{}\n{}".format( tracfile, dates)) list_files = tracfile[:] except TypeError: list_files = len(dates) * [tracfile] # Reading fluxes for periods within the simulation window trcr_flx = [] for dd, dd_file in zip(dates, list_files): file_flx = dd.strftime(dd_file) dir_flx = dd.strftime(tracdir) if not os.path.isfile("{}/{}".format(dir_flx, file_flx)) and getattr( self, "closest_year", False): info("Warning: could not find correct year for EDGAR; " "using closest available one") list_dates = [ datetime.datetime.strptime(os.path.basename(f), tracfile) for f in glob.glob("{}/v50_*nc".format(dir_flx)) ] delta_dates = np.abs(dd - np.array(list_dates)) file_flx = list_dates[np.argmin(delta_dates)].strftime(tracfile) nc = xr.open_dataset("{}/{}".format(dir_flx, file_flx), decode_times=False) trcr_flx.append(nc[varnames].values) xmod = xr.DataArray( np.array(trcr_flx)[:, np.newaxis, ...], coords={"time": dates}, dims=("time", "lev", "lat", "lon"), ) return xmod
def ini_data(plugin, **kwargs): """Initializes CHIMERE Args: plugin (dict): dictionary defining the plugin **kwargs (dictionary): possible extra parameters Returns: loaded plugin and directory with executable """ info("Initializing the model") workdir = getattr(plugin, "workdir", "./") # Initializes the directory path.init_dir("{}/model".format(workdir)) # Default values: # period: '1D' plugin.periods = getattr(plugin, "periods", "1D") # Number of hours per period plugin.nhours = int( pd.to_timedelta(plugin.periods).total_seconds() // 3600) plugin.nho = "{:.0f}".format(plugin.nhours) # Replacing nsaveconcs if not specified # Forces the end.nc file to contain concentration every N hours # By default, saves only at the end if not hasattr(plugin, "nsaveconcs"): plugin.nsaveconcs = plugin.nhours # Replace name for METEO files plugin.meteo.file = plugin.meteo.file.format(nho=plugin.nho) # Replace name for AEMISSION files and BEMISSIONS files plugin.fluxes.file = plugin.fluxes.file.format(nho=plugin.nho) plugin.fluxes.nlevemis = plugin.nlevemis plugin.biofluxes.file = plugin.biofluxes.file.format(nho=plugin.nho) plugin.biofluxes.nlevemis = plugin.nlevemis_bio # Replace name for BOUN_CONCS files plugin.latcond.file = plugin.latcond.file.format(nho=plugin.nho) plugin.topcond.file = plugin.topcond.file.format(nho=plugin.nho) return plugin
def grib_file_reader(filepath, varname, attribute=None): """ Filepath is the absolute file path Dimension_i is the name of dimension i, e.i. given between cotes '' if there is not third dimension, Dimension_3's value = None Variable is the variable's name, e.i. given between cotes '' """ # Forcing import of attributes if not in attribute key list if (attribute is not None and attribute not in cfgrib.dataset.DATA_ATTRIBUTES_KEYS): cfgrib.dataset.DATA_ATTRIBUTES_KEYS.append(attribute) cfgrib.dataset.ALL_KEYS = sorted(cfgrib.dataset.ALL_KEYS + [attribute]) if not os.path.exists(filepath): info("{} was not found".format(filepath)) raise IOError info("Reading {}".format(filepath)) df = cfgrib.open_file(filepath) if len(np.shape(varname)) == 0: varnames = [varname] else: varnames = varname[:] varout = [] for name in varnames: var = df.variables[name].data if hasattr(var, "build_array"): var = var.build_array() varout.append(var) # Fetching attributes if needed if attribute is not None: for name in df.variables: attr = df.variables[name].attributes.get( "GRIB_{}".format(attribute), None) if attr is not None: return attr raise Exception("Could not find attribute {} in {}".format( attribute, filepath)) if len(np.shape(varname)) == 0: return varout[0] else: return varout
def create_domain(domain, **kwargs): """Creates a grid if needed Args: domain (dictionary): dictionary defining the domain. Returns: Error as LMDZ shouldn't be used with unknown grids """ logfile = kwargs.get("logfile", None) info("TO DO", logfile) raise Exception
def simul(self, chi, grad=True, run_id=-1, **kwargs): """Computes the cost function J (and its gradient) based on the Gaussian formulation of the inversion framework: J(x) = chi^T chi + (Hx-y)^T R^-1 (Hx-y) = j_b + j_r gradJ(x) = 2 * chi + 2 * H^T R^(-1) (Hx-y) Args: chi (np.array): a flat vector defining the current state of the control vector grad (bool, optional): if True, returns both the function value and its gradient run_id (int): ID for the current run (determines the folder names) Returns: J(x), gradJ(x) """ # Various variables datei = self.datei datef = self.datef workdir = self.workdir # Get the observation operator from extra arguments if not hasattr(self, "obsoperator"): raise Exception("Observation operator is missing to compute the " "simulator. Please check your setup files") obsoper = self.obsoperator statevect = self.statevect # Saving chi to the control vector for later statevect.chi = chi zcost = np.sum((chi - np.arange(len(chi)))**2) zgrad = 2 * (chi - np.arange(len(chi))) # Verbose the norms znorm_grad_b = np.dot(zgrad, zgrad)**0.5 info("In Simulator:\n" " grad(Jb) = {}\n" " Jb = {}\n".format(znorm_grad_b, zcost)) return zcost, zgrad
def make_fluxes(self, data, ddi, ddf, runsubdir, mode): """Prepare a binary file per emitted species containing flux data. :param self: :param datastore: :param ddi: :param ddf: :param runsubdir: :param mode: :return: """ datastore = data.datastore for spec in self.chemistry.emis_species.attributes: tracer = getattr(self.chemistry.emis_species, spec) if not ("fluxes", spec) in datastore: info("{} not available for being emitted in LMDZ".format(spec)) continue info("LMDZ is generating flux inputs for {}".format(spec)) data = datastore[("fluxes", spec)] # If not determined by the control vector if "spec" not in data: data["spec"] = self.fluxes.read( spec, data["dirorig"], data["fileorig"], data["varname"], self.input_dates[ddi], ) # Adds empty increments if not available if "incr" not in data: data["incr"] = 0.0 * data["spec"] # Put in dataset for writing by 'write' ds = xr.Dataset({"fwd": data["spec"], "tl": data["incr"]}) # Write to FORTRAN binary flx_file = "{}/mod_{}.bin".format(runsubdir, spec) self.emis_species.write(spec, flx_file, ds)
def crop_monitor(datastore, datei, datef, **kwargs): """Crops observation datasets to keep observations whose duration fits entirely during the simulation period Args: datastore (pd.DataFrame): observation dataset datei (datetime.datetime): start date datef (datetime.datetime): end date Returns: pd.DataFrame: Cropped dataframe """ info("Cropping obsvect.datastore to simulation window") info("{} to {}".format(datei, datef)) mask = (datastore.index >= datei) & (datastore.index + pd.to_timedelta( datastore["duration"], unit="h") <= datef) return datastore.loc[mask]
def default_fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): # Picking info from tracer if not in inputs if ref_dir == "" and hasattr(tracer, "dir"): input_dir = tracer.dir else: input_dir = ref_dir if ref_file == "" and hasattr(tracer, "file"): input_file = tracer.file else: input_file = ref_file info("Fetching input files using directory and file format") info("{}/{}".format(input_dir, input_file)) list_files = {} list_dates = {} for datei in input_dates: tmp_files = [] tmp_dates = [] for dd in input_dates[datei]: dir_dd = dd.strftime(input_dir) file_dd = dd.strftime(input_file) tmp_files.append("{}/{}".format(dir_dd, file_dd)) # Fetching local_files = [] for f in tmp_files: target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) local_files.append(target_file) list_files[datei] = list(set(local_files)) list_dates[datei] = list(set(tmp_dates)) return list_files, list_dates
def create_domain(domain, **kwargs): """Creates a grid if needed Args: domain (dictionary): dictionary defining the domain. Returns: Error as LMDZ shouldn't be used with unknown grids """ logfile = kwargs.get("logfile", None) info( "Cannot create a LMDZ grid as LMDZ should be used with " "pre-defined grids only", logfile, ) raise Exception
def fetch(ref_dir, ref_file, input_dates, target_dir, tracer=None, **kwargs): list_files = {} list_dates = {} for datei in input_dates: tmp_files = [] tmp_dates = [] for dd in input_dates[datei]: file_flx = dd.strftime(ref_file) dir_flx = dd.strftime(ref_dir) date_flx = dd if not os.path.isfile( "{}/{}".format(dir_flx, file_flx)) and getattr( tracer, "closest_year", False): info("Warning: could not find correct year for EDGAR; " "using closest available one") list_dates_avail = [ datetime.datetime.strptime(os.path.basename(f), ref_file) for f in glob.glob("{}/v50_*nc".format(dir_flx)) ] delta_dates = np.abs(dd - np.array(list_dates_avail)) date_flx = list_dates_avail[np.argmin(delta_dates)] file_flx = date_flx.strftime(ref_file) tmp_files.append("{}/{}".format(dir_flx, file_flx)) tmp_dates.append(date_flx) # Fetching local_files = [] for f in tmp_files: target_file = "{}/{}".format(target_dir, os.path.basename(f)) path.link(f, target_file) local_files.append(target_file) list_files[datei] = list(set(local_files)) list_dates[datei] = list(set(tmp_dates)) return list_files, list_dates
def minimize(self, finit, gradinit, chi0, **kwargs): # x, f, g, auxil, io, niter, nsim, iz, df1, m=5, dxmin=1.e-20, # epsg=1.e-20, impres=1, mode=0, **kwargs """Entry point for M1QN3 algorithm. Args: finit (float): initial value for the function to minimize gradinit (np.array): gradient at the starting point chi (np.array): initial state for the unknown to optimize simulator (module): simulator module to evaluate the function and its gradient minimizer (module): minimizer module, used to define minimizer options Returns: (np.array, float): a tuple with the optimized vector and the corresponding function maximum """ # Initializing options (and filling missing values with default) self = self.check_options(chi0, finit, **kwargs) # Running M1QN3 xopt, fopt, gradopt, niter, nsim, epsg, mode = self.m1qn3( finit, gradinit, chi0, **kwargs ) # Final verbose and output towrite = """ M1QN3: output mode is {} number of iterations: {} number of simulations: {} realized relative precision on g: {} """.format( mode, niter, nsim, epsg ) info(towrite) r1 = np.sqrt(np.dot(xopt, xopt)) r2 = np.sqrt(np.dot(gradopt, gradopt)) info("norm of x = " + str(r1)) info("f = " + str(fopt)) info("norm of g = " + str(r2)) return xopt
def ini_data(plugin, **kwargs): """Initializes the dummy_txt Gaussian model Args: plugin (Plugin): the model plugin to initialize **kwargs (dictionary): possible extra parameters Returns: loaded plugin and directory with executable """ info("Initializing the model") workdir = getattr(plugin, "workdir", "./") # Initializes the directory path.init_dir("{}/model".format(workdir)) # copying the model Pasquill Gifford matrix target = "{}/model/".format(workdir) + os.path.basename(plugin.file_pg) source = plugin.file_pg shutil.copy(source, target) # Required inputs for running a LMDz simulations plugin.required_inputs = ["fluxes", "meteo", "param"] # Initializes default values: # - sub-simulations of 1day # - time steps of 1 hour plugin.periods = getattr(plugin, "periods", "1D") plugin.tstep = getattr(plugin, "tstep", "1H") plugin.save_H = getattr(plugin, "save_H", False) plugin.H_matrix = {} return plugin
def config_info(cls, setup): """Prints out main input parameters for pyCIF """ verbose_txt = [ "pyCIF has been initialized with the following parameters:", "Yaml configuration file: {}".format(setup.def_file), "Log file: {}".format(setup.logfile), "Start date: {}".format(setup.datei), "End date: {}".format(setup.datef), "Working directory: {}".format(setup.workdir), ] list(map(lambda v: info(v), verbose_txt))
def native2state(transform, data, mapper, mod_input, di, df, mode, runsubdir, workdir, trans_mode, **kwargs): datastore = data.datastore for trid in mapper["inputs"]: input_type = trid[0] # If trid in datastore, dumps this one if trid in datastore: todump = [trid] # If input parameter is '', # dumps all available parameters of this component elif trid[1] == "": todump = [t for t in datastore if t[0] == trid[0]] # Otherwise check whether there is a component # encompassing all parameters (i.e., with '' as parameter) else: todump = [(trid[0], "")] # Create new data to extract data2dump = {t: datastore[t] for t in todump} data2dump = transform.model.outputs2native(data2dump, input_type, di, df, runsubdir, mode) for tr in data2dump: if tr in datastore: datastore[tr].update(data2dump[tr]) else: info("{} was simulated by the model, " "but could not be transferred to the control vector". format(tr)) return data
def read(self, name, tracdir, tracfile, varnames, dates, interpol_flx=False, **kwargs): """Get fluxes from pre-computed fluxes and load them into a pycif variables Args: self: the model Plugin name: the name of the component tracdir, tracfile: flux directory and file format dates: list of dates to extract interpol_flx (bool): if True, interpolates fluxes at time t from values of surrounding available files """ info("READING NetCDF dummy fluxes") raise Exception
def check_inputs(inputs, mode): """Check the consistency of inputs given to the observation operator. """ if mode not in ["tl", "fwd", "adj"]: info("The following running mode is not accepted by the " "observation operator: {}".format(mode)) raise Exception if mode == "tl" and not (hasattr(inputs, "x") and hasattr(inputs, "dx")): info("The observation operator was operated in tangent-linear mode " "but not with both increments and control vector") raise Exception if mode == "fwd" and not hasattr(inputs, "x"): info("The observation operator was operated in forward mode " "with no control vector") info("All inputs will be dealt as fixed") return True
def parse_multiple_files(self, **kwargs): """Parses multiple files specified by a glob pattern and stores the content into a datastore Args: provider_name (str): provider of the input file file_format_id (str): name of the type of file with a given format glob_pattern (str): glob pattern: /**/ for recursive matching in subdirectories Keyword Args: encoding (str): Encoding of input files freq (str): frequency after re-sampling see `Offset Aliases`_ for valid strings src_freq (str): explicit setting of the frequency in the input file shouldn't be necessary Notes: - Additional kwargs for a parser are possible or even required. See the respective documentation Returns: dict: {obs_file} = df[obssite_id, parameter] """ # parser = cls.get_parser(provider_name, file_format_id) dfs = {} info("Reading files in " + self.dir_obs) for obs_file in sorted(glob.glob(self.dir_obs + "*")): try: dfs[os.path.basename(obs_file)] = self.parse_file( obs_file, **kwargs ) except error.PluginError as e: info( "{} was not loaded for the following reason".format( obs_file ) ) info(e.message) if dfs != {}: return pd.concat(list(dfs.values())) else: return pd.DataFrame({})
def minimize(self, finit, gradinit, chi0, **kwargs): # x, f, g, auxil, io, niter, nsim, iz, df1, m=5, dxmin=1.e-20, # epsg=1.e-20, impres=1, mode=0, **kwargs """Entry point for CONGRAD algorithm. Args: finit (float): initial value for the function to minimize gradinit (np.array): gradient at the starting point chi (np.array): initial state for the unknown to optimize simulator (module): simulator module to evaluate the function and its gradient minimizer (module): minimizer module, used to define minimizer options Returns: (np.array, float): a tuple with the optimized vector and the corresponding function maximum """ # Initializing options (and filling missing values with default) self = self.check_options(chi0, **kwargs) # Running CONGRAD lanczvect0 = copy.deepcopy(gradinit) xopt, gradopt, preduc, pevecs, iiter = self.congrad( chi0, gradinit, lanczvect0, **kwargs) # Final verbose and output towrite = """ CONGRAD: number of iterations: {} achieved relative reduction of the gradient: {} """.format(iiter, preduc) info(towrite) r1 = np.sqrt(np.dot(xopt, xopt)) r2 = np.sqrt(np.dot(gradopt, gradopt)) info("norm of x = " + str(r1)) info("norm of g = " + str(r2)) return xopt
def from_yaml(cls, def_file): """Generates a dictionary including all pyCIF parameters Args: def_file (string) : Path to the definition file Handles both absolute and relative paths Returns: config_dict (dictionary): Dictionary populated with all pyCIF parameters """ yml_file = os.path.abspath(os.path.expanduser(def_file)) try: with open(yml_file, "r") as f: config_dict = ordered_load(f) config_dict["def_file"] = yml_file if "datei" in config_dict: # Converting dates to datetime if necessary config_dict["datei"] = dates.date2datetime( config_dict["datei"]) config_dict["datef"] = dates.date2datetime( config_dict["datef"]) return config_dict except IOError as e: info("Couldn't find config file: {}".format(yml_file)) info("Please check directories") raise e except yaml.scanner.ScannerError as e: info("Error in the syntax of config file: {}".format(yml_file)) raise e