class SizedistributionBins(Sizedistribution): """ """ #nr_of_levels = 3 savepath_pressure_coordinates = constants.get_outdata_path( 'pressure_coords') def __init__(self, *vars, diameters=constants_sizedist.diameter_obs_df, **kwargs): self.diameters = diameters return super().__init__(*vars, **kwargs) def compute_Nd_vars(self, diameters=None, overwrite=False): """ :param diameters: dataframe with index: var_name and rows 'from_diameter' and 'to_diameter' :param overwrite: :return: """ if diameters is None: diameters = self.diameters # TODO: check if needed to load dataset # Get variable list needed: varl = self.get_varlist_input() # Get input data: input_ds = get_pressure_coord_fields(self.case_name, varl, self.from_time, self.to_time, self.history_field, model=self.model_name) print(diameters) print(type(diameters)) for key in diameters.index: fromd = float(diameters.loc[key]['from_diameter']) tod = float(diameters.loc[key]['to_diameter']) out_varn = key # get_varname_Nd(fromd, tod) log.ger.info('Calculating %s' % out_varn) da_Nd = calc_Nd_interval_NorESM(input_ds, fromd, tod, out_varn) fn = self.get_Nd_output_name(out_varn) if os.path.isfile(fn) and not overwrite: continue #da_Nd.attrs['nice_name'] = get_N_nice_name_Nd(out_varn) #da_Nd.attrs['fancy_name'] = get_N_nice_name_Nd(out_varn) #self.to_netcdf(da_Nd.to_dataset(), fn) self.to_netcdf(da_Nd, fn) def get_Nd_output_name(self, out_varn): fn = get_filename_pressure_coordinate_field(out_varn, self.model_name, self.case_name, self.from_time, self.to_time) return fn
def __init__(self, case_name, from_time, to_time, isSectional, time_res, space_res='full', model_name='NorESM', history_field='.h0.', raw_data_path=constants.get_input_datapath(), locations=constants.collocate_locations, read_from_file=True, chunks=None, use_pressure_coords=False, dataset=None, savepath_root=constants.get_outdata_path('collocated')): """ :param case_name: :param from_time: :param to_time: :param raw_data_path: :param isSectional: :param time_res: 'month', 'year', 'hour' :param space_res: 'full', 'locations' :param model_name: """ self.chunks = chunks self.read_from_file = read_from_file self.model_name = model_name # self.case_plotting_name = model_name self.dataset = None self.use_pressure_coords = use_pressure_coords self.case_name_nice = find_model_case_name.find_name( model_name, case_name) self.case_name = case_name self.raw_data_path = raw_data_path self.from_time = from_time self.to_time = to_time self.time_resolution = time_res self.space_resolution = space_res self.history_field = history_field self.locations = locations self.isSectional = isSectional self.locations = constants.collocate_locations self.dataset = dataset self.savepath_root = savepath_root self.attrs_ds = dict(raw_data_path=self.raw_data_path, model=self.model_name, model_name=self.model_name, case_name=self.case_name, case=self.case_name, case_name_nice=self.case_name_nice, isSectional=str(self.isSectional), from_time=self.from_time, to_time=self.to_time)
# %autoreload 2 from sectional_v2.util.Nd.sizedist_class_v2.SizedistributionBins import SizedistributionStationBins from sectional_v2.util.collocate.collocateLONLAToutput import CollocateLONLATout from sectional_v2.constants import sized_varListNorESM #from useful_scit.util import log import useful_scit.util.log as log log.ger.setLevel(log.log.INFO) # %% [markdown] # ## Savepath: # %% from sectional_v2.constants import get_outdata_path path_out = get_outdata_path('eusaar') version ='_noresmv21_dd'#_noresm2'#_fbvoc' file_out = path_out + 'Nd_cat_sources_timeseries%s.csv'%version # %% file_out # %% [markdown] # ### Model data: # %% nr_of_bins = 5 maxDiameter = 39.6 # 23.6 #e-9 minDiameter = 5.0 # e-9 history_field='.h1.' #cases_sec = ['SECTv11_ctrl', 'SECTv11_ctrl_fbvoc']
import numpy as np import xarray as xr # import analysis_tools.area_pkg_sara from oas_dev.util.imports.get_fld_fixed import get_field_fixed from oas_dev.util.imports.import_fields_xr_v2 import import_constants from sectional_v2.util.slice_average import area_mod #from sectional_v2.util.slice_average.avg_pkg import maps # import analysis_tools.var_overview_sql # from analysis_tools import area_pkg_sara, practical_functions from sectional_v2 import constants from useful_scit.util import log as log import sys path_to_global_avg = constants.get_outdata_path('area_means') # 'Data/area_means/' path_to_map_avg = constants.get_outdata_path('map_means') path_to_profile_avg = constants.get_outdata_path('profile_means') # Fields that should be weighted: fields4weighted_avg = {'AREL_incld': ['AREL', 'FREQL'], 'AWNC_incld': ['AWNC', 'FREQL'], 'ACTNL_incld': ['ACTNL', 'FCTL'], 'ACTREL_incld': ['ACTREL', 'FCTL']} """ Recomended: only use average_model_var """ def masked_average(xa: xr.DataArray, dim=None, weights: xr.DataArray = None, mask: xr.DataArray = None):
import numpy as np import xarray as xr from dask.diagnostics import ProgressBar from sectional_v2.util.practical_functions import make_folders import useful_scit.util.log as log from sectional_v2.constants import get_outdata_path from sectional_v2.util.eusaar_data import subs_codes as subset_codes_eusaar from sectional_v2.util.eusaar_data.distc_var import percs_in_eusaar_files from sectional_v2.util.eusaar_data.flags import make_data_flags # path_eusaar_outdata log.ger.setLevel(log.log.INFO) savepath_distc_model_ds = get_outdata_path( 'eusaar') + '/noresm/' # distc_ds_noresm.nc' print(savepath_distc_model_ds) # %% def compute_percentile_flag(ds, flag, quants=None): if quants is None: quants = np.array(percs_in_eusaar_files) / 100. # get's flags: # %% flags = make_data_flags() ds_w = ds.where(flags[flag]) log.ger.info(f'Computing percentiles for {flag}') with ProgressBar(): ds_w = ds_w.compute() da_percs = ds_w.quantile(quants, dim='time')
# %% # load and autoreload from IPython import get_ipython # noinspection PyBroadException try: _ipython = get_ipython() _magic = _ipython.magic _magic('load_ext autoreload') _magic('autoreload 2') except: pass # %% from sectional_v2.constants import get_outdata_path path_in = get_outdata_path('eusaar') version ='_noresmv21_dd' file_in = path_in + 'Nd_cat_sources_timeseries%s.csv'%version plot_path = get_plotpath('eusaar') version ='_noresmv21dd_both' # %% # %% # case_ns = 'noSECTv11_ctrl_fbvoc' # case_sec = 'SECTv11_ctrl_fbvoc' case_sec='SECTv21_ctrl_koagD' #'SECTv11_ctrl_fbvoc']#'SECTv11_ctrl']#,'SECTv11_ctrl_fbvoc']#'SECTv11_ctrl'] case_ns ='noSECTv21_ox_ricc' #'noSECTv11_ctrl_fbvoc'] #/no SECTv11_ctrl cases_ns = ['noSECTv21_default_dd','noSECTv21_ox_ricc_dd'] cases_s = [case_sec] cases = cases_ns + cases_s
from sectional_v2.constants import get_outdata_path, collocate_locations import xarray as xr from pathlib import Path import shutil # %% from sectional_v2.util.practical_functions import make_folders dir_collocated = get_outdata_path('collocated') locs_t = collocate_locations.transpose() # %% # %% # %% def fix_coord_station(ds): # %% both_coords = get_ds_old_stationc2new() both_coords # %% ds['nstation'] = both_coords['nstation'] ds_n = ds.rename({'station': 'station_tab', 'nstation': 'station'}) ds_n = ds_n.swap_dims({'station_tab': 'station'}) return ds_n #ds_n['station'].values # %% def get_ds_old_stationc2new(): alt_c = 'Alternative_code'
import Ngl # import Nio # import os import xarray as xr import numpy as np # import matplotlib as mpl from sectional_v2 import constants # from useful_scit.util import log import useful_scit.util.log as log from sectional_v2.util.filenames import get_filename_pressure_coordinate_field from sectional_v2.util.practical_functions import extract_path_from_filepath, make_folders default_save_pressure_coordinates = constants.get_outdata_path('pressure_coords') # 'Data/Fields_pressure_coordinates' def hybsig2pres(ds, var, save_field=False): pnew = ds['lev'].values # [:] # [1013, 850.] if 'time' in ds['hyam'].dims: hyam = ds["hyam"].isel(time=0).values # [:] else: hyam = ds["hyam"].values # [:] if 'time' in ds['hybm'].dims: hybm = ds["hybm"].isel(time=0).values # [:] else: hybm = ds["hybm"].values vals_in = ds[var].values da_in = ds[var] psrf = (ds["PS"][:, :, :]) if 'time' in ds["P0"].dims: P0mb = 0.01 * ds["P0"].isel(time=0).values
class Sizedistribution: """ Class to calculate and read sizedistribution dataset """ default_savepath_root = constants.get_outdata_path('sizedistrib_files') # noinspection PyTypeChecker def __init__(self, case_name, from_time, to_time, dlim_sec, isSectional, time_res, raw_data_path=constants.get_input_datapath(), space_res='full', nr_bins=5, print_stat=False, model_name='NorESM', history_field='.h0.', locations=constants.locations, chunks={'diameter': 20}, use_pressure_coords=True, use_eusaar_diam=True): """ :param case_name: :param from_time: :param to_time: :param raw_data_path: :param dlim_sec: :param isSectional: :param time_res: 'month', 'year', 'hour' :param space_res: 'full', 'locations' :param print_stat: :param model_name: """ self.chunks = chunks self.dmin_sec = dlim_sec[0] self.dmax_sec = dlim_sec[1] self.nr_bins = nr_bins bin_diam, bin_diam_int = self.get_sectional_params() self.bin_diameter_int = bin_diam_int self.bin_diameter = bin_diam if use_eusaar_diam: d_arr = distc_var.get_diameter_sized() else: d_arr = np.logspace(np.log10(3), 4, 50) # np.logspace(0, 4, 50) self.diameter = xr.DataArray(d_arr, name='diameter', coords=[d_arr], dims='diameter', attrs={'units': 'nm'}) # self.read_from_file = read_from_file self.model_name = model_name # self.case_plotting_name = model_name self.dataset = None self.use_pressure_coords = use_pressure_coords self.case_name_nice = find_model_case_name.find_name( model_name, case_name) self.case_name = case_name self.raw_data_path = raw_data_path self.from_time = from_time self.to_time = to_time self.time_resolution = time_res self.space_resolution = space_res self.history_field = history_field self.locations = locations self.isSectional = isSectional self.final_sizedist_vars = self.varl_sizedist_final() # self.savepath_sizedist = self.dataset_savepath(case_name, model_name) self.print = print_stat self.attrs_ds = dict(raw_data_path=self.raw_data_path, model=self.model_name, model_name=self.model_name, case_name=self.case_name, case=self.case_name, case_name_nice=self.case_name_nice, isSectional=str(self.isSectional), from_time=self.from_time, to_time=self.to_time, time_resolution=self.time_resolution, history_field=self.history_field, pressure_coords=str(self.use_pressure_coords)) # self.size_dtset = self.get_sizedistrib_dataset() # self.attrs = vars(self self.dmin_sec = dlim_sec[0] return def varl_sizedist_final(self): if self.isSectional: return [D_NDLOG_D_SEC, D_NDLOG_D_MOD] else: return [D_NDLOG_D_MOD] def get_sectional_params(self): """ Set sectional parameters. :return: """ max_diameter = self.dmax_sec min_diameter = self.dmin_sec nr_bins = self.nr_bins bin_diam, bin_diam_int = get_bin_diameter(nr_bins, min_diameter, max_diameter) return bin_diam, bin_diam_int def to_netcdf(self, ds, fn): """ Saves dataset to netcdf with attributes :param ds: :param fn: :return: """ atts = self.attrs_ds attrs_to = ds.attrs update_dic(atts, attrs_to) ds = ds.assign_attrs(attrs_to) delayed_obj = ds.to_netcdf(fn, compute=False) # , chunks={'diameter':1}) with ProgressBar(): results = delayed_obj.compute() def get_sizedist_var(self, var_names=None, CHUNKS=None): """ :param var_names: :param CHUNKS: :return: """ if var_names is None: if not self.isSectional: var_names = [D_NDLOG_D_MOD] else: var_names = [D_NDLOG_D_MOD, D_NDLOG_D_SEC] if CHUNKS is None: CHUNKS = self.chunks fn_list = [] for var_name in var_names: # fn = self.dataset_savepath_var(var_name, self.case_name, self.model_name) fn = self.make_sure_sizedist_varfile_exists(var_name) fn_list.append(fn) log.ger.info('Opening: [' + 'j'.join(fn_list) + ']') ds = xr.open_mfdataset(fn_list, combine='by_coords', chunks=CHUNKS) make_tot = True for var in TOTAL_VARS: if var not in ds.data_vars: make_tot = False if make_tot: self.make_total_sizedist(ds) elif D_NDLOG_D_MOD in ds.data_vars and not self.isSectional: ds['dNdlogD'] = ds[D_NDLOG_D_MOD].copy() ds['dNdlogD'].attrs = ds[D_NDLOG_D_MOD].attrs # ds['dNdlogD'].attrs['long_name'] = 'dNdlogD' return ds def make_total_sizedist(self, ds): if self.isSectional: ds['dNdlogD'] = ds[D_NDLOG_D_SEC] + ds[D_NDLOG_D_MOD] else: ds['dNdlogD'] = ds[D_NDLOG_D_MOD].copy() ds['dNdlogD'].attrs = ds[D_NDLOG_D_MOD].attrs ds['dNdlogD'].attrs['long_name'] = 'dNdlogD' #def get_var(self, var_name, CHUNKS={'diameter': 20}): # fn = self.dataset_savepath_var(var_name, self.case_name, self.model_name) # return xr.open_dataset(fn, chunks=CHUNKS) # def dataset_savepath(self, case_name, model_name): # """ # Returns filename of dataset # :param case_name: # :param model_name: # :return: # """ # # case_name = case_name.replace(' ', '_') # _savep = self.default_savepath_root # st = '%s/%s/%s/%s' % (_savep, model_name, case_name, case_name) # st = st + '_%s_%s' % (self.from_time, self.to_time) # st = st + '_%s_%s' % (self.time_resolution, self.space_resolution) # fn = st + '.nc' # make_folders(fn) # return fn def dataset_savepath_var(self, var_name, case_name, model_name): """ Returns filename of dataset :param var_name: :param case_name: :param model_name: :return: """ case_name = case_name.replace(' ', '_') _savep = self.default_savepath_root st = '%s/%s/%s/%s_%s' % (_savep, model_name, case_name, var_name, case_name) st = st + '_%s_%s' % (self.from_time, self.to_time) st = st + '_%s_%s' % (self.time_resolution, self.space_resolution) fn = st + '.nc' make_folders(fn) return fn def compute_sizedist_var(self, var_name): """ Compute sizedistribution variable. :param var_name: :return: """ if 'sec' in var_name: ds = self.compute_sizedist_sec_var(var_name) return ds else: ds = self.compute_sizedist_mod_var(var_name) return ds def compute_sizedist_mod_var(self, var_name): """ Compute modal file :param var_name: The variable to be computed :return: """ num = var_name[-2:] if num.isdigit(): return self._calc_sizedist_mod_var_nr(var_name, num=num) else: return self.compute_sizedist_mod_tot() def make_sure_sizedist_varfile_exists(self, var_name): """ Check that var is computed and if not compute the var :param var_name: the variable in question :return: filename of file """ fn = self.dataset_savepath_var(var_name, self.case_name, self.model_name) if not os.path.isfile(fn): log.ger.debug('computing file for %s: \n %s' % (var_name, fn)) t1 = time.time() ds = self.compute_sizedist_var(var_name) if not os.path.isfile(fn): self.to_netcdf(ds, fn) ds.close() del ds t2 = time.time() log.ger.info('computed %s in time : %s m' % (fn, (t2 - t1) / 60.)) return fn def compute_sizedist_tot(self): if self.isSectional: self.compute_sizedist_sec_tot() self.compute_sizedist_mod_tot() def compute_sizedist_mod_tot(self): """ Compute total of modal, i.e. the sum of all modes. :return: """ dNdlogD_var = D_NDLOG_D_MOD fn_final = self.dataset_savepath_var(dNdlogD_var, self.case_name, self.model_name) if os.path.isfile(fn_final): log.ger.info('Modal tot file found %s' % fn_final) return xr.open_dataset(fn_final) else: log.ger.info('Computing file %s' % fn_final) vs_NCONC = varListNorESM['NCONC'] fl = [] l_vars_dNdlogD = [] for var in vs_NCONC: dNdlogD_var_nr = _varname_mod_nr(var) l_vars_dNdlogD.append(dNdlogD_var_nr) _f = self.make_sure_sizedist_varfile_exists(dNdlogD_var_nr) fl.append(_f) log.ger.debug('Modal tot file found %s' % var) log.ger.debug(fl) start = time.time() CHUNKS = {'diameter': 20} ds = xr.open_mfdataset( fl, combine='by_coords', parallel=True, chunks=CHUNKS) # .isel(diameter=slice(0,30))#.squeeze() da = sum_vars(ds, l_vars_dNdlogD, dNdlogD_var, long_name='dN/dlogD (modal)') self.to_netcdf( da, fn_final) # , compute=False) # , chunks={'diameter':1}) # delayed_obj = self.to_netcdf(da, fn_final)#, compute=False) # , chunks={'diameter':1}) # with ProgressBar(): # results = delayed_obj.compute() end = time.time() log.ger.debug('Time elapsed: %f' % (end - start)) ds.close() da.close() del da del ds return xr.open_dataset(fn_final) def _calc_sizedist_mod_var_nr(self, var_name, num=None): if num is None: num = var_name[-2:] varN = _get_nconc_varname(num) varNMR = _get_nmr_varname(num) varSIG = _get_sig_varname(num) varl = [varN, varNMR, varSIG] input_ds = get_pressure_coord_fields(self.case_name, varl, self.from_time, self.to_time, self.history_field, model=self.model_name) ds_dNdlogD = self._compute_dNdlogD_mod(var_name, self.diameter, input_ds, varN, varNMR, varSIG) input_ds.close() del input_ds return ds_dNdlogD def _compute_dNdlogD_mod(self, dNdlogD_var, diameter, input_ds, varN, varNMR, varSIG): size_dtset = xr.Dataset(coords={ **input_ds.coords, 'diameter': self.diameter }) log.ger.debug(varN) # varListNorESM['NCONC'][i]) # varNMR = varListNorESM['NMR'][i] NCONC = input_ds[varN] # [::]*10**(-6) #m-3 --> cm-3 SIGMA = input_ds[varSIG] # [::]#*10**6 NMR = input_ds[varNMR] * 2. # radius --> diameter # number: size_dtset[dNdlogD_var] = dNdlogD_modal(NCONC, NMR, SIGMA, diameter) size_dtset[dNdlogD_var].attrs['units'] = 'cm-3' size_dtset[dNdlogD_var].attrs[ 'long_name'] = 'dN/dlogD (mode' + dNdlogD_var[-2:] + ')' return size_dtset def compute_sizedist_sec_tot(self, chunks=None): """ Compute total of sizedistribution sectional total :param chunks: :return: :return: """ if chunks is None: chunks = self.chunks dNdlogD_var = D_NDLOG_D_SEC fn_final = self.dataset_savepath_var(dNdlogD_var, self.case_name, self.model_name) if os.path.isfile(fn_final): log.ger.info('opening :%s' % fn_final) return xr.open_dataset(fn_final) # vs_NCONC = varListNorESM['NCONC'] if not self.isSectional: return input_vars = self.get_varlist_input_sec() # start = time.time() l_vars_dNdlogD = [] fl = [] for var in input_vars: dNdlogD_var_nr = _varname_sec_nr(var) fl.append(self.make_sure_sizedist_varfile_exists(dNdlogD_var_nr)) l_vars_dNdlogD.append(dNdlogD_var_nr) fl = list(dict.fromkeys(fl)) ds = xr.open_mfdataset(fl, combine='by_coords', chunks=chunks, parallel=True) da = sum_vars(ds, l_vars_dNdlogD, dNdlogD_var, long_name='dN/dlogD (sectional)') # ex_var = l_vars_dNdlogD[0] # keep_coords = list(ds[ex_var].dims) # drop_l = list(set(ds.variables) - set(l_vars_dNdlogD + keep_coords)) # ds = ds.drop(drop_l) # log.ger.warning(ds) # da = ds.to_array(dim='variable', name=dNdlogD_var) # 'dNdlogD_mod') # da = da.sum('variable') self.to_netcdf( da, fn_final ) # da.to_netcdf(fn_final, compute=False) # , chunks={'diameter':1}) # delayed_obj =self.to_netcdf(da,fn_final)# da.to_netcdf(fn_final, compute=False) # , chunks={'diameter':1}) # with ProgressBar(): # results = delayed_obj.compute() end = time.time() log.ger.debug('Time elapsed: %f' % (end - start)) ds.close() da.close() del da del ds return xr.open_dataset(fn_final) # return #ds def compute_sizedist_sec_var(self, var_name): """ Compute sectional dNdlogD variable :param var_name: :return: """ num = var_name[-2:] if num.isdigit(): return self._calc_sizedist_sec_var_nr(var_name, num=num) else: return self.compute_sizedist_sec_tot() def _calc_sizedist_sec_var_nr(self, var_name, num): if num is None: num = var_name[-2:] varl = get_nrSEC_varname(num) input_ds = get_pressure_coord_fields(self.case_name, varl, self.from_time, self.to_time, self.history_field, model=self.model_name) ds_dNdlogD = self._compute_dNdlogD_sec(var_name, self.diameter, input_ds, num) input_ds.close() del input_ds return ds_dNdlogD def get_input_data(self, varlist): if self.isSectional: get_pressure_coord_fields(self.case_name, varlist, self.from_time, self.to_time, self.history_field, model=self.model_name) else: log.ger.warning('NOT IMPLEMENTED FOR NATIVE LEV COORD') def get_varlist_input_sec(self): """ returns necessary input vars for sectional :return: """ nr_bins = self.nr_bins vl = [] for i in range(1, nr_bins + 1): vl = vl + get_nrSEC_varname(i) return vl def get_varlist_input(self): vl = [] if self.isSectional: vl = self.get_varlist_input_sec() for key in sized_varListNorESM: vl = vl + sized_varListNorESM[key] return vl def _compute_dNdlogD_sec(self, dNdlogD_var, diameter, input_ds, num): varnSOA = get_nrSEC_varname(num)[0] varnSO4 = get_nrSEC_varname(num)[1] size_dtset = xr.Dataset(coords={ **input_ds.coords, 'diameter': self.diameter }) SOA = input_ds[varnSOA] # [::]*10**(-6) #m-3 --> cm-3 SO4 = input_ds[varnSO4] # [::]#*10**6 # number: bin_diameter_int = self.bin_diameter_int size_dtset[dNdlogD_var] = dNdlogD_sec(diameter, SOA, SO4, num, bin_diameter_int) size_dtset[dNdlogD_var].attrs['units'] = 'cm-3' size_dtset[dNdlogD_var].attrs[ 'long_name'] = 'dNdlogD (sectional' + dNdlogD_var[-2:] + ')' return size_dtset
from sectional_v2.util.practical_functions import make_folders from sectional_v2.util.imports.get_pressure_coord_fields import get_pressure_coord_fields from sectional_v2.util.naming_conventions import find_model_case_name from sectional_v2 import constants import useful_scit.util.log as log import matplotlib.pyplot as plt import time # TODO: comment code D_NDLOG_D_SEC = 'dNdlogD_sec' D_NDLOG_D_MOD = 'dNdlogD_mod' TOTAL_VARS = [D_NDLOG_D_MOD, D_NDLOG_D_SEC] varListNorESM = constants.sized_varListNorESM default_savepath = constants.get_outdata_path('sizedistrib_files') def update_dic(dic_from, dic_to): for key in dic_from: dic_to[key] = dic_from[key] def append2dic(ds_append, ds_add): """ :param ds_append: :param ds_add: """ for key in ds_add.attrs.keys(): if key not in ds_append.attrs: