def main(project_info): """Diagnostics and plotting script for Southern Hemisphere radiation.""" # ESMValProject provides some easy methods to access information in # project_info but you can also access it directly (as in main.py) # First we get some general configurations (ESMValProject) # and then we read in the model configuration file E = ESMValProject(project_info) config_file = E.get_configfile() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() # A-laue_ax+ diag_script = E.get_diag_script_name() res = E.write_references( diag_script, # diag script name ["A_maek_ja"], # authors ["A_eval_ma", "A_jone_co"], # contributors [""], # diag_references [""], # obs_references ["P_embrace"], # proj_references project_info, verbosity, False) # A-laue_ax- modelconfig = ConfigParser.ConfigParser() modelconfig.read(config_file) E.ensure_directory(plot_dir) # Check which parts of the code to run if (modelconfig.getboolean('general', 'plot_scatter')): info("Starting scatter plot", verbosity, 2) process_scatter(E, modelconfig)
def endElement(self, name): """ @brief default SAX endElement event handler @param name default SAX endElement argument """ def replaceAttag(tbr): pat = re.compile('@\\{..*\\}') if pat.search(tbr): patlist = pat.findall(tbr) patlist = [item[2:-1] for item in patlist] tmp = list(set(patlist) & set(self.conf.pathCollection.keys())) for t in tmp: tbr = re.sub("@{"+t+"}", self.conf.getPathByID(t), tbr) return tbr info("endElement: " + name, verbosity, required_verbosity=12) if self.include is not None: if name == self.current_tag.__class__.__name__: self.project_info[name] = getattr(self.current_tag, 'closing_tag')(replaceAttag(self.str), self.attributes) elif name == 'include': pass elif name == 'namelist': pass else: self.current_tag.add_nml_entry(name, replaceAttag(self.str), self.attributes) else: if name == self.current_tag.__class__.__name__: self.project_info[name] = getattr(self.current_tag, 'closing_tag')(self.str, self.attributes) elif name == 'include': pass elif name == 'namelist': pass else: self.current_tag.add_nml_entry(name, self.str, self.attributes)
def write_env_projinfo(self, project_info): """ @brief Writes XML-file information to env. variables @param project_info Current namelist in dictionary format Information between Python and NCL is (partially) exchanged through environment variables. This function write the project_info-dictionary content to environment variables prefixed with "ESMValTool_". """ verbosity = project_info['GLOBAL']['verbosity'] for section_key in ['GLOBAL', 'RUNTIME', 'TEMPORARY']: if section_key in project_info.keys(): for key in project_info[section_key]: info("writing key to env. variable, key=" + key, verbosity, required_verbosity=11) # Check and fail on duplicate entries if "ESMValTool_" + key in os.environ: raise writeProjinfoError("Environment variable " + "'ESMValTool_" + key + "' already defined") os.environ["ESMValTool_" + key] = \ str(project_info[section_key][key])
def startElement(self, name, attr): """ @brief default SAX startElement event handler @param name default SAX startElement argument @param attr default SAX startElement attribute """ info("startElement: " + name, verbosity, required_verbosity=12) if name == 'include': self.include = attr['href'] self.conf = configPrivateHandler() p = xml.sax.make_parser() p.setContentHandler(self.conf) p.parse(self.include) if name in vars(nml_tags): self.current_tag = vars(nml_tags)[name]() else: info("clearing str '" + self.str + "'", verbosity, required_verbosity=12) self.attributes = {} self.str = "" if attr: for attr_key in attr.keys(): self.attributes[attr_key.lower()] = attr[attr_key]
def select_base_vars(self, variables, model, currProject, project_info): """ @brief Determine base variables to be read from input file(s) @param variables - base variables provided by variable_defs script @param model - current model @param currProject - current project @param project_info - project_info-dictionary @return base_vars - variables to be read from input file(s) """ verbosity = project_info['GLOBAL']['verbosity'] for base_var in variables: # Check if this variable should be excluded for this model-id if self.id_is_explicitly_excluded(base_var, model): continue # first try: use base variables provided by variable_defs script os.environ['__ESMValTool_base_var'] = base_var.var infile = reformat.infile(currProject, project_info, base_var, model) precomputed = currProject.get_cf_fullpath(project_info, model, base_var.fld, base_var.var, base_var.mip, base_var.exp) if (len(glob.glob(infile) + glob.glob(precomputed)) == 0): info( " No input files found for " + base_var.var + " (" + base_var.fld + ") as " + infile, verbosity, 1) base_var.var = base_var.var0 base_var.fld = base_var.fld0 # try again with input variable = base variable (non derived) infile = reformat.infile(currProject, project_info, base_var, model) precomputed = currProject.get_cf_fullpath( project_info, model, base_var.fld, base_var.var, base_var.mip, base_var.exp) if (len(glob.glob(infile) + glob.glob(precomputed)) == 0): raise exceptions.IOError(2, "No input files found in ", infile) else: info( " Using " + base_var.var + " (" + base_var.fld + ") as " + infile, verbosity, 1) base_vars = [base_var] break # discard other base vars else: base_vars = variables return base_vars
def climate(currProject, currDiag, project_info, model, field, base_variable, variable_attributes): """ @brief Wrapper to call ncl script that calculates climatology @param currProject An instance of the current project @param project_info Dictionary with all info from the namelist @param currDiag Current diagnostic """ infilename = currProject.get_cf_fullpath(project_info, model, field, base_variable, variable_attributes) cfield_m, cfield_s, cfield_a = currDiag.get_climate_field_type(field) file_monthly_clim = currProject.get_cf_fullpath(project_info, model, cfield_m, base_variable, variable_attributes) file_season_clim = currProject.get_cf_fullpath(project_info, model, cfield_s, base_variable, variable_attributes) file_annual_clim = currProject.get_cf_fullpath(project_info, model, cfield_a, base_variable, variable_attributes) if ((not os.path.isfile(file_monthly_clim) or not os.path.isfile(file_season_clim) or not os.path.isfile(file_annual_clim)) or project_info['GLOBAL']['force_processing']): verbosity = project_info['GLOBAL']['verbosity'] project_info['TEMPORARY'] = {} project_info['TEMPORARY']['infilename'] = infilename project_info['TEMPORARY']['mfile'] = file_monthly_clim project_info['TEMPORARY']['sfile'] = file_season_clim project_info['TEMPORARY']['afile'] = file_annual_clim project_info['TEMPORARY']['base_variable'] = base_variable info(" Calling climate.py", verbosity, required_verbosity=1) info(" INFILE = " + infilename, 1, verbosity) info(" OUTFILES = " + file_monthly_clim, 1, verbosity) info(" " + file_season_clim, 1, verbosity) info(" " + file_annual_clim, 1, verbosity) exit_on_warning = project_info['GLOBAL']['exit_on_warning'] executable = "./interface_scripts/climate.ncl" projects.run_executable(executable, project_info, verbosity, exit_on_warning) del (project_info['TEMPORARY'])
def endElement(self, element_name): """ @brief default SAX endElement event handler @param name default SAX endElement argument """ info("endElement: " + element_name, verbosity, required_verbosity=12) if element_name == 'ESGF': pass elif element_name in ESGFConfig.valid_node_names\ or element_name == "USER_CACHE": self.node = None else: self.current_tag.add_ESGF_entry( element_name, self.string.strip(), self.attributes, self.node )
def main(project_info): """Diagnostics and plotting script for Tropical Variability Equatorial. We use ts as a proxy for Sea Surface Temperature. """ # ESMValProject provides some easy methods to access information in # project_info but you can also access it directly (as in main.py) # We need to ensure that needed files for precipitation and temperature # are present before we can proceed E = ESMValProject(project_info) config_file = E.get_configfile() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() # A-laue_ax+ diag_script = E.get_diag_script_name() res = E.write_references( diag_script, # diag script name ["A_maek_ja"], # authors ["A_eval_ma", "A_jone_co"], # contributors ["D_li14jclim"], # diag_references [""], # obs_references ["P_embrace"], # proj_references project_info, verbosity, False) # A-laue_ax- modelconfig = ConfigParser.ConfigParser() modelconfig.read(config_file) E.ensure_directory(plot_dir) # Here we check and process only desired parts of the diagnostics if (modelconfig.getboolean('general', 'plot_equatorial')): info("Starting to plot equatorial means", verbosity, 2) process_equatorial_means(E, modelconfig) post_process(E)
def main(project_info): """Preprocessing script for Tropical Variability equatorial divergence winds. """ # ESMValProject provides some easy methods to access information in # project_info but you can also access it directly (as in main.py) # First we get some general configurations (ESMValProject) # and then we read in the model configuration file E = ESMValProject(project_info) config_file = E.get_configfile() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() # A-laue_ax+ diag_script = E.get_diag_script_name() res = E.write_references( diag_script, # diag script name ["A_maek_ja"], # authors ["A_eval_ma", "A_jone_co"], # contributors ["D_li14jclim"], # diag_references [""], # obs_references ["P_embrace"], # proj_references project_info, verbosity, False) # A-laue_ax- modelconfig = ConfigParser.ConfigParser() modelconfig.read(config_file) E.ensure_directory(plot_dir) # Here we check and process only desired parts of the diagnostics if (modelconfig.getboolean('general', 'plot_equatorial')): info("Starting to gather values for equatorial divergence plots", verbosity, 2) process_divergence(E, modelconfig)
def main(project_info): """Diagnostics and plotting script for Southern Hemisphere radiation.""" # ESMValProject provides some easy methods to access information in # project_info but you can also access it directly (as in main.py) # First we get some general configurations (ESMValProject) # and then we read in the model configuration file E = ESMValProject(project_info) config_file = E.get_configfile() datakeys = E.get_currVars() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() # A-laue_ax+ diag_script = E.get_diag_script_name() res = E.write_references(diag_script, # diag script name ["A_maek_ja"], # authors ["A_eval_ma", "A_jone_co"], # contributors [""], # diag_references [""], # obs_references ["P_embrace"], # proj_references project_info, verbosity, False) # A-laue_ax- modelconfig = ConfigParser.ConfigParser() modelconfig.read(config_file) E.ensure_directory(plot_dir) # Check at which stage of the program we are clouds = False fluxes = False radiation = False if ('clt' in datakeys or 'clivi' in datakeys or 'clwvi' in datakeys): clouds = True elif ('hfls' in datakeys or 'hfss' in datakeys): fluxes = True else: radiation = True # Check which parts of the code to run if (modelconfig.getboolean('general', 'plot_clouds') and clouds is True): info("Starting to plot clouds", verbosity, 2) process_clouds(E, modelconfig) if (modelconfig.getboolean('general', 'plot_fluxes') and fluxes is True): info("Starting to plot turbulent fluxes", verbosity, 2) process_fluxes(E, modelconfig) if (modelconfig.getboolean('general', 'plot_radiation') and radiation is True): info("Starting to plot radiation graphs", verbosity, 2) process_radiation(E, modelconfig)
def process_zonal_means(E, modelconfig): """Main script for plotting zonal means. Outputs variable specific plots containing all models for that variable (including observations). """ experiment = 'zonal_means' datakeys = E.get_currVars() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() areas = modelconfig.get(experiment, 'areas').split() plot_grid = modelconfig.getboolean('general', 'plot_grid') # We don't do the zonal means for winds (equatorial elsewhere) if 'ua' in datakeys: datakeys.remove('ua') for area in areas: plt.clf() fig, axs = plt.subplots(2, 2, figsize=(12, 8)) fig.subplots_adjust(top=0.87) fig.subplots_adjust(right=0.67) fig.subplots_adjust(hspace=0.2) fig.subplots_adjust(wspace=0.4) for datakey in datakeys: col = datakeys.index(datakey) # We output a file for each variable model_filenames = E.get_clim_model_filenames(variable=datakey) # Start looping and generating subplots ymin, ymax = False, False lon_min = modelconfig.getint(experiment + '_' + area, 'lon_min') lon_max = modelconfig.getint(experiment + '_' + area, 'lon_max') lon_area = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons') ymin = -9999 ymax = -9999 sd = np.zeros(len(model_filenames)) mindex = 0 for model in model_filenames: # read in specified data and concatenate datafile = nc.Dataset(model_filenames[model], 'r') # A-laue_ax+ E.add_to_filelist(model_filenames[model]) # A-laue_ax- data_units = datafile.variables[datakey].units lats, data = E.get_model_data(modelconfig, experiment, area, datakey, datafile) data = E.average_data(data, 1) datafile.close() norm = data.mean() # get y min and max iteratively if (ymin == -9999): ymin = data.min() ymax = data.max() else: ymin = np.min([data.min(), ymin]) ymax = np.max([data.max(), ymax]) sd[mindex] = (data / norm).std() mindex += 1 # read model specific plotting style color, dashes, width = E.get_model_plot_style(model) # plotting custom dashes requires some extra effort (else) # with empty dashes the format is default if (len(dashes) == 0): axs[0, col].plot(lats, data, color=color, linewidth=width, label=model) axs[1, col].plot(lats, data / norm, color=color, linewidth=width, label=model) else: line1, = axs[0, col].plot(lats, data, '--', color=color, linewidth=width, label=model) line1.set_dashes(dashes) line2, = axs[1, col].plot(lats, data / norm, color=color, linewidth=width, label=model) line2.set_dashes(dashes) # Now to make the plot pretty i.e. headers, ticks etc. # axs[-1, col].yaxis.set_label_position("right") axs[0, col].grid(plot_grid) axs[1, col].grid(plot_grid) area_key = experiment + '_' + area xmin = modelconfig.getint(area_key, 'lat_min') xmax = modelconfig.getint(area_key, 'lat_max') axs[0, col].set_xlim(xmin, xmax) axs[1, col].set_xlim(xmin, xmax) xticks = E.get_ticks(5, np.array([xmin, xmax])) labels = E.get_ticks_labels(xticks, 'lats') axs[0, col].xaxis.set_ticks(xticks) axs[0, col].set_xticklabels(labels) axs[1, col].xaxis.set_ticks(xticks) axs[1, col].set_xticklabels(labels) yticks = E.get_ticks(8, np.array([ymin, ymax])) axs[0, col].yaxis.set_ticks(yticks) axs[1, col].yaxis.set_ticks([1.0]) axs[1, col].set_yticklabels(['$\mu$']) #axs[1, col].yaxis.set_ticks([1.0-sd.mean(), 1.0, 1.0+sd.mean()]) #axs[1, col].set_yticklabels(['$\mu$-$\sigma$', '$\mu$', '$\mu$+$\sigma$']) if (datakey == 'pr' or datakey == 'pr-mmday'): axs[0, col].set_title("Precipitation") axs[0, col].set_ylabel("[" + data_units + "]") axs[1, col].set_ylabel("normalized") elif (datakey == 'ts'): axs[0, col].set_title("Temperature") axs[0, 0].set_ylabel("[" + data_units + "]") axs[1, 0].set_ylabel("normalized") plt.suptitle(area + " ocean [" + lon_area[0] + ":" + lon_area[1] + "] seasonal mean", fontsize=16) # Adding the legend handles0, labels0 = axs[0, 0].get_legend_handles_labels() handles1, labels1 = axs[0, 1].get_legend_handles_labels() handles_all = handles0 + handles1 labels_all = labels0 + labels1 labels_sorted = zip(*sorted(zip(labels_all, handles_all)))[0] handles_sorted = zip(*sorted(zip(labels_all, handles_all)))[1] handles = [] labels = [] for item in xrange(len(labels_sorted)): if (labels_sorted[item] not in labels): labels.append(labels_sorted[item]) handles.append(handles_sorted[item]) plt.legend(handles, labels, loc='center left', bbox_to_anchor=(0.7, 0.5), bbox_transform=plt.gcf().transFigure) # And save the plot diag_name = E.get_diag_script_name() output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) specifier = area + '-seasonal-mean' variable = "".join(E.get_currVars()) output_file = E.get_plot_output_filename(diag_name=diag_name, variable=variable, specifier=specifier) plt.savefig(os.path.join(output_dir, output_file)) info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1)
def read_pr_sm_topo(project_info, model): """ ;; Arguments ;; project_info: dictionary ;; all info from namelist ;; ;; Return ;; pr: iris cube [time, lat, lon] ;; precipitation time series ;; sm: iris cube [time, lat, lon] ;; soil moisture time series ;; topo: array [lat, lon] ;; topography ;; lon: array [lon] ;; longitude ;; lat: array [lat] ;; latitude ;; time: iris cube coords ;; time info of cube ;; time_bnds_1: float ;; first time_bnd of time series ;; ;; ;; Description ;; Read cmip5 input data for computing the diagnostic ;; """ import projects E = ESMValProject(project_info) verbosity = E.get_verbosity() #------------------------- # Read model info #------------------------- currProject = getattr(vars()['projects'], model.split_entries()[0])() model_info = model.split_entries() mip = currProject.get_model_mip(model) exp = currProject.get_model_exp(model) start_year = currProject.get_model_start_year(model) end_year = currProject.get_model_end_year(model) years = range(int(start_year), int(end_year) + 1) ''' #------------------------- # Read model info #------------------------- model_name = model_info[1] time_step = model_info[2] exp_fam = model_info[3] model_run = model_info[4] year_start = model_info[5] year_end = model_info[6] filedir = model_info[7] years = range(int(year_start), int(year_end)+1) ''' #------------------------- # Input data directories #------------------------- currDiag = project_info['RUNTIME']['currDiag'] pr_index = currDiag.get_variables().index('pr') pr_field = currDiag.get_field_types()[pr_index] sm_index = currDiag.get_variables().index('mrsos') sm_field = currDiag.get_field_types()[sm_index] indir = currProject.get_cf_outpath(project_info, model) in_file = currProject.get_cf_outfile(project_info, model, pr_field, 'pr', mip, exp) pr_files = [os.path.join(indir, in_file)] in_file = currProject.get_cf_outfile(project_info, model, sm_field, 'mrsos', mip, exp) sm_files = [os.path.join(indir, in_file)] ''' #------------------------- # Input data directories #------------------------- pr_files = [] sm_files = [] for yy in years: Patt = filedir+'pr_'+time_step+'_'+model_name+'_'+exp_fam+'_'+\ model_run+'_'+str(yy)+'*.nc' pr_files.append(glob.glob(Patt)) Patt = filedir+'mrsos_'+time_step+'_'+model_name+'_'+exp_fam+'_'+\ model_run+'_'+str(yy)+'*.nc' sm_files.append(glob.glob(Patt)) pr_files = [l[0] for l in pr_files if len(l)>0] pr_files = sorted(pr_files) sm_files = [l[0] for l in sm_files if len(l)>0] sm_files = sorted(sm_files) ''' #---------------------- # Read in precipitation #---------------------- pr_list = [] for pr_file in pr_files: info('Reading precipitation from ' + pr_file, verbosity, required_verbosity=1) pr = iris.load(pr_file)[0] for at_k in pr.attributes.keys(): pr.attributes.pop(at_k) pr_list.append(pr) pr = iris.cube.CubeList(pr_list) pr = pr.concatenate()[0] # Convert longitude from 0_360 to -180_180 pr = coord_change([pr])[0] # Add metadata: day, month, year add_month(pr, 'time') add_day_of_month(pr, 'time', name='dom') add_year(pr, 'time') # Convert units to kg m-2 hr-1 pr.convert_units('kg m-2 hr-1') #----------------------- # Read in soil moisture #----------------------- sm_list = [] for sm_file in sm_files: info('Reading soil moisture from ' + sm_file, verbosity, required_verbosity=1) sm = iris.load(sm_file)[0] for at_k in sm.attributes.keys(): sm.attributes.pop(at_k) sm_list.append(sm) sm = iris.cube.CubeList(sm_list) sm = sm.concatenate()[0] # Convert longitude from 0_360 to -180_180 sm = coord_change([sm])[0] # Add metadata: day, month, year add_month(sm, 'time') add_day_of_month(sm, 'time', name='dom') add_year(sm, 'time') #---------------------------------------------- # Constrain pr and sm data to latitude 60S_60N #---------------------------------------------- latconstraint = iris.Constraint(latitude=lambda cell: -59.0 <= cell <= 59.0) pr = pr.extract(latconstraint) sm = sm.extract(latconstraint) #--------------------------------------------------- # Read in grid info: latitude, longitude, timestamp #--------------------------------------------------- lon = sm.coords('longitude')[0].points lat = sm.coords('latitude')[0].points time = sm.coords('time') # -------------------------------------- # Convert missing data (if any) to -999. # -------------------------------------- try: sm.data.set_fill_value(-999) sm.data.data[sm.data.mask] = -999. except: info('no missing data conversion', verbosity, required_verbosity=1) #---------------------- # Read in topography #---------------------- # Topography map specs: # latitude 60S_60N # longitude 180W_180E # model resolution #ftopo = currProject.get_cf_fx_file(project_info, model) #dt = '>f4' #topo = (np.fromfile(ftopo, dtype=dt)).reshape(len(lat), len(lon)) topo = get_topo(project_info, lon, lat, model) #---------------------- # Read in time bounds #---------------------- indir, infiles = currProject.get_cf_infile(project_info, model, pr_field, 'pr', mip, exp) Patt = os.path.join(indir, infiles) pr_files = sorted(glob.glob(Patt)) ncf = nc4.Dataset(pr_files[0]) time_bnds_1 = ncf.variables['time_bnds'][0][0] time_bnds_1 = time_bnds_1 - int(time_bnds_1) ncf.close() #----------------------------------------------- # Return input data to compute sm_pr diagnostic #----------------------------------------------- return pr, sm, topo, lon, lat, time, time_bnds_1
def main(project_info): """ ;; Description ;; Main fuction ;; Call all callable fuctions to ;; read CMIP5 data, compute and plot diagnostic """ E = ESMValProject(project_info) plot_dir = E.get_plot_dir() work_dir = E.get_work_dir() verbosity = E.get_verbosity() fileout = work_dir if not os.path.exists(plot_dir): os.makedirs(plot_dir) for model in project_info['MODELS']: info(model, verbosity, required_verbosity=1) if not os.path.exists(work_dir+'/sample_events'): os.makedirs(work_dir+'/sample_events') if not os.path.exists(work_dir+'/event_output'): os.makedirs(work_dir+'/event_output') # -------------------------------------- # Get input data to compute diagnostic # -------------------------------------- # Read cmip5 model data pr, sm, topo, lon, lat, time, time_bnds_1 = read_pr_sm_topo(project_info, model) # -------------------------------------- # Get sm monthly climatology # at selected local time: 6:00 am # -------------------------------------- smclim = get_smclim(sm, lon, time) # ------------------------------- # Compute diagnostic per month # ------------------------------- samplefileout = fileout + 'sample_events/' for mn in np.arange(1, 13): # ------------------------------------------------- # Create montly arrays required by fortran routines # ------------------------------------------------- prbef, smbef, praft, smaft, \ monthlypr, monthlysm, days_per_year = get_monthly_input(project_info, mn, time, lon, lat, time_bnds_1, pr, sm, fileout, samplefileout, model, verbosity) # ----------------------- # Run fortran routines # ----------------------- info('Executing global_rain_sm for month ' + str(mn), verbosity, required_verbosity=1) grs.global_rain_sm(np.asfortranarray(monthlypr), np.asfortranarray(prbef), np.asfortranarray(praft), np.asfortranarray(monthlysm), np.asfortranarray(smbef), np.asfortranarray(smaft), np.asfortranarray(smclim[mn - 1, :, :]), np.asfortranarray(topo), np.asfortranarray(lon), np.asfortranarray(mn), fileout, days_per_year) info('Executing sample_events for month ' + str(mn), verbosity, required_verbosity=1) se.sample_events(np.asfortranarray(monthlysm), np.asfortranarray(smbef), np.asfortranarray(smaft), np.asfortranarray(lon), np.asfortranarray(lat), np.asfortranarray(mn), fileout, days_per_year, samplefileout) # --------------------------------------------------- # Compute p_values (as in Fig. 3, Taylor et al 2012) # -------------------------------------------------- info('Computing diagnostic', verbosity, required_verbosity=1) xs, ys, p_vals = get_p_val(samplefileout) # -------------------------------------------------- # Save diagnostic to netCDF file and plot # -------------------------------------------------- write_nc(fileout, xs, ys, p_vals, project_info, model) plot_diagnostic(fileout, plot_dir, project_info, model) # -------------------------------------------------- # Remove temporary folders # -------------------------------------------------- shutil.rmtree(str(fileout) + 'event_output') shutil.rmtree(str(fileout) + 'sample_events')
def get_topo(project_info, longi, lati, model): """ ;; Arguments ;; in_dir: dir ;; directory with input file "topo_var_5x5.gra" ;; longi: array [lon] ;; longitude in degrees east ;; lati: array [lat] ;; latitude ;; Return ;; topo: array [lat, lon] ;; topography ranges in model grid ;; ;; Description ;; Computes topography information ;; """ import projects E = ESMValProject(project_info) verbosity = E.get_verbosity() #------------------------- # Read model info #------------------------- currProject = getattr(vars()['projects'], model.split_entries()[0])() model_info = model.split_entries() # load topo max-min topography(m) # within 1.25x1.25 area nx = 1440 ny = 480 ftopo = currProject.get_cf_fx_file(project_info, model) info('topo file: ' + ftopo , verbosity, required_verbosity=1) dt = '>f4' topo = (np.fromfile(ftopo, dtype=dt)).reshape(4, ny, nx) topo_range = np.transpose(topo[3,:,:]) # Average onto model grid nxo = len(longi) nyo = len(lati) lon1 = longi[0] lat1 = lati[0] dlon = np.abs(longi[0]-longi[1]) dlat = np.abs(lati[0]-lati[1]) topo_model = np.zeros((nxo, nyo), dtype = 'f4') n = np.zeros((nxo, nyo), dtype = 'f4') for j in range(0, ny): for i in range(0, nx): lon = i*0.25-179.875 lat = j*0.25-59.875 jj = int(round((lat-lat1)/dlat)) ii = int(round((lon-lon1)/dlon)) if(ii==nxo): ii = 0 if(jj==nyo): jj = nyo -1 if(topo_range[i,j]!=-999.): topo_model[ii,jj] = topo_model[ii,jj] + topo_range[i,j] n[ii,jj] = n[ii,jj] + 1. topo_model[n>0] = topo_model[n>0]/n[n>0] topo_model[n==0] = -999. topo_model[topo_model==-999] = 0 return np.transpose(topo_model)
def get_monthly_input(project_info, mn, time, lon, lat, time_bnds_1, pr, sm, fileout, samplefileout, model, verbosity): """ ;; Arguments ;; mn: int ;; month, values from 1 to 12 ;; time: iris cube coords ;; time info of cube ;; lon: array [lon] ;; longitude ;; lat: array [lat] ;; latitude ;; time_bnds_1: float ;; first time_bnd of time series ;; pr: iris cube [time, lat, lon] ;; 3-hourly precipitation time series ;; sm: iris cube [time, lat, lon] ;; 3-hourly soil moisture time series ;; fileout: dir ;; output directory ;; samplefileout: dir ;; temporary outpout directory used by fortran routines ;; ;; Return ;; prbef: array [year, day time steps (=8), lat, lon] ;; 3-hourly precipitation in the previous day ;; smbef: array [year, day time steps (=8), lat, lon] ;; 3-hourly soil moisture in the previous day ;; praf: array [year, day time steps (=8), lat, lon] ;; 3-hourly precipitation in the following day ;; smaft: array [year, day time steps (=8), lat, lon] ;; 3-hourly soil moisture in the following day ;; monthlypr: array [year, days in month * day time steps, lat, lon] ;; 3-hourly precipitation in month mn for the whole analysis period ;; monthlysm: array [year, days in month * day time steps, lat, lon] ;; 3-hourly soil moisture in month mn for the whole analysis period ;; days_per_year: int ;; number of days per year ;; ;; Description ;; Prepare monthly input data for fortran routines ;; """ import projects E = ESMValProject(project_info) verbosity = E.get_verbosity() #------------------------- # Read model info #------------------------- model = project_info['MODELS'][0] currProject = getattr(vars()['projects'], model.split_entries()[0])() model_info = model.split_entries() # -------------------------------------- # Get grid and calendar info # -------------------------------------- utimedate = time[0].units first_year = int(model_info[5]) last_year = int(model_info[6]) nyr = last_year - first_year + 1 years = np.arange(nyr) + first_year months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] calendar = utimedate.calendar nx = len(lon) ny = len(lat) days_permonth_360 = [30 for i in range(0, 12)] days_permonth_noleap = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] days_permonth_leap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] if calendar == '360_day': days_per_year = 360 days_permonth = days_permonth_360 nts = [8 * dpm for dpm in days_permonth_360] elif any([calendar == '365_day', calendar == 'noleap']): days_per_year = 365 days_permonth = days_permonth_noleap nts = [8 * dpm for dpm in days_permonth_noleap] elif any([calendar == 'gregorian', calendar == 'standard']): days_per_year = 365 days_permonth = days_permonth_noleap nts = [8 * dpm for dpm in days_permonth_noleap] # Leap days are considered by get_monthly_input() else: error('Missing calendar info') # -------------------------------------- # Create pr, sm before and after arrays # -------------------------------------- prbef = np.zeros((nyr, 8, ny, nx), dtype='f4') praft = np.zeros((nyr, 8, ny, nx), dtype='f4') smbef = np.zeros((nyr, 8, ny, nx), dtype='f4') smaft = np.zeros((nyr, 8, ny, nx), dtype='f4') try: os.mkdir(os.path.join(samplefileout, "5x5G_mon" + str(mn).zfill(2)), stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) except OSError as exc: if exc.errno != errno.EEXIST: raise exc pass try: os.mkdir(os.path.join(fileout, "event_output", "mon" + str(mn).zfill(2)), stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) except OSError as exc: if exc.errno != errno.EEXIST: raise exc pass nt = nts[mn - 1] monthlypr_list = [np.zeros((nt, ny, nx), dtype='f4') for y in years] monthlysm_list = [np.zeros((nt, ny, nx), dtype='f4') for y in years] for yr, year in enumerate(years): info('month, year: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1) if all([mn == 2, any([calendar == 'gregorian', calendar == 'standard'])]): monthlysm_list[yr][:,:,:] = ( sm.extract(iris.Constraint(month=months[mn-1],year=year, dom = range(1,29) )).data) monthlypr_list[yr][:,:,:] = ( pr.extract(iris.Constraint(month=months[mn-1],year=year, dom = range(1,29) )).data) else: try: monthlypr_list[yr][:,:,:] = ( pr.extract(iris.Constraint(month=months[mn-1],year=year)).data) except: try: monthlypr_list[yr][0,:,:] = -999. monthlypr_list[yr][1::,:,:] = ( pr.extract(iris.Constraint(month=months[mn-1],year=year)).data) except: info('omitted pr: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1) try: monthlysm_list[yr][:,:,:] = ( sm.extract(iris.Constraint(month=months[mn-1],year=year)).data) except: try: monthlysm_list[yr][0,:,:] = -999. monthlysm_list[yr][1::,:,:] = ( sm.extract(iris.Constraint(month=months[mn-1],year=year)).data) except: info('omitted sm: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1) # last day of previous month if all([mn == 1, year == first_year]): prbef[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999. smbef[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999. elif (mn == 1): prbef[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year-1,month='Dec',dom=days_permonth[-1])).data) smbef[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year-1,month='Dec',dom=days_permonth[-1])).data) else: if any([calendar == '360_day', calendar == '365_day', calendar == 'noleap']): prbef[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data) smbef[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data) elif any([calendar == 'gregorian', calendar == 'standard']): if cal.isleap(year): prbef[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth_leap[mn-2])).data) smbef[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth_leap[mn-2])).data) else: prbef[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data) smbef[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data) # first day of following month if (mn == 12 and year == last_year): praft[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999. smaft[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999. elif (mn == 12): praft[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year+1,month='Jan',dom=1)).data) smaft[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year+1,month='Jan',dom=1)).data) else: if any([calendar == '360_day', calendar == '365_day', calendar == 'noleap']): praft[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data) smaft[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data) elif any([calendar == 'gregorian', calendar == 'standard']): if all([cal.isleap(year),mn == 2]): praft[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[1],dom=29)).data) smaft[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[1],dom=29)).data) else: praft[yr,:,:,:] = ( pr.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data) smaft[yr,:,:,:] = ( sm.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data) monthlypr = np.vstack(tuple(monthlypr_list)) monthlysm = np.vstack(tuple(monthlysm_list)) monthlypr = monthlypr.reshape(nyr, nt, ny, nx) monthlysm = monthlysm.reshape(nyr, nt, ny, nx) if time_bnds_1 == 0.0625: monthlypr[:, 0:-1, :, :] = monthlypr[:, 1::, :, :] monthlypr[:, -1, :, :] = -9999. prbef[:, 0:-1, :, :] = prbef[:, 1::, :, :] praft[:, 0:-1, :, :] = praft[:, 1::, :, :] prbef[:, -1, :, :] = -9999. praft[:, -1, :, :] = -9999. return prbef, smbef, praft, smaft, monthlypr, monthlysm, days_per_year
input_xml_full_path = args[0] # Parse input namelist into project_info-dictionary. Project = xml_parsers.namelistHandler() parser = xml.sax.make_parser() parser.setContentHandler(Project) parser.parse(input_xml_full_path) # Project_info is a dictionary with all info from the namelist. project_info = Project.project_info if options.reformat: if 'REFORMAT' not in project_info.keys(): error('No REFORMAT tag specified in {0}'.format(input_xml_full_path)) if len(project_info['REFORMAT']) == 0: info('No reformat script specified', 1, 1) print_header({}, options.reformat) for k, v in project_info['REFORMAT'].iteritems(): if not os.path.exists(v): error('Path {0} does not exist'.format(v)) projects.run_executable(v, project_info, 1, False, write_di=False) sys.exit(0) verbosity = project_info['GLOBAL']['verbosity'] climo_dir = project_info['GLOBAL']['climo_dir'] exit_on_warning = project_info['GLOBAL'].get('exit_on_warning', False) # Additional entries to 'project_info'. The 'project_info' construct # is one way by which Python passes on information to the NCL-routines. project_info['RUNTIME'] = {}
def process_equatorial_means(E, modelconfig): """Main script for plotting equatorial means. Outputs one image with four subplots (precipitation, temperature, equatorial winds and divergence). TODO: This script is not too flexible - since we're using ncl for preprocessing we cant exclude multiple obs files - so we can use only two different observation models (so they can exclude one another). """ experiment = 'equatorial' datakeys = E.get_currVars() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() work_dir = E.get_work_dir() areas = modelconfig.get(experiment, 'areas').split() plot_grid = modelconfig.getboolean('general', 'plot_grid') ua_key = datakeys[0] # Get the model paths etc. including the obs file for winds models = E.get_clim_model_filenames(ua_key) # Get other variables and check data existance datakeys.extend(check_data_existance(E, modelconfig)) # Looping over areas etc. for area in areas: area_key = experiment + '_' + area lat_min = modelconfig.getint(area_key, 'lat_min') lat_max = modelconfig.getint(area_key, 'lat_max') lon_min = modelconfig.getint(area_key, 'lon_min') lon_max = modelconfig.getint(area_key, 'lon_max') lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats') # Initial figure config fig, axs = plt.subplots(4, 1, figsize=(12, 11)) fig.subplots_adjust(top=0.88) fig.subplots_adjust(right=0.67) fig.subplots_adjust(hspace=0.4) for datakey in datakeys: # Output precipitation / temperature and winds # Adjust titles accordingly suptitle = area + " ocean equatorial [" suptitle += lat_area[0] + ":" + lat_area[1] + "] mean" plt.suptitle(suptitle, fontsize=16) if (datakey == 'pr' or datakey == 'pr-mmday'): row = 0 title = "Precipitation" elif (datakey == 'ts'): row = 1 title = "Temperature" elif (datakey == 'ua' or datakey == 'ua-1000'): row = 2 title = "Eastward wind" elif (datakey == 'divergence'): row = 3 title = "Wind divergence" # The data is handled differently for winds and the rest # For winds we have to extract the data for each model # For the rest we only have one file / datakey scaling = '' if (datakey == 'ua' or datakey == 'ua-1000'): for model in models: # extract the model specific data datafile = nc.Dataset(models[model], 'r') # A-laue_ax+ E.add_to_filelist(models[model]) # A-laue_ax- data_units = datafile.variables[datakey].units lats, lons, data = E.get_model_data( modelconfig, experiment, area, datakey, datafile) datafile.close() lons = E.ensure_looping(lons) # Take the means and get the plot style data = E.average_data(data, 2) color, dashes, width = E.get_model_plot_style(model) if (len(dashes) == 0): axs[row].plot(lons, data, color=color, linewidth=width, label=model) else: line, = axs[row].plot(lons, data, '--', color=color, linewidth=width, label=model) line.set_dashes(dashes) ymin = modelconfig.getint(area_key, 'wind_min') ymax = modelconfig.getint(area_key, 'wind_max') yticks = E.get_ticks(8, np.array([ymin, ymax])) # Next the datakeys with all data in one file (per datakey) # These will be read from external files previously written by # TropicalVariability.py elif (datakey == 'pr' or datakey == 'pr-mmday' or datakey == 'ts'): base_name = work_dir + 'TropicalVariability_'\ + area_key + '_' + datakey + '_model_' datafiles = glob(base_name + '*') key_models = [] for dfile in datafiles: start = dfile.find('_model_') + 7 key_models.append(str(dfile)[start:-4]) # now we can do the loop over the models for model in key_models: # extract model specific data alldata = np.load(base_name + model + '.npy') lons = alldata[0] data = alldata[1] color, dashes, width = E.get_model_plot_style(model) lons = E.ensure_looping(lons) # get the style and plot it out if (len(dashes) == 0): axs[row].plot(lons, data, color=color, linewidth=width, label=model) else: line, = axs[row].plot(lons, data, '--', color=color, linewidth=width, label=model) line.set_dashes(dashes) # Some additional figure config if (datakey == 'pr' or datakey == 'pr-mmday'): ymin = modelconfig.getint(area_key, 'prec_min') ymax = modelconfig.getint(area_key, 'prec_max') yticks = E.get_ticks(6, np.array([ymin, ymax])) if (datakey == 'pr'): data_units = 'kg/m2s' else: data_units = 'mm/day' elif (datakey == 'ts'): ymin = modelconfig.getint(area_key, 'temp_min') ymax = modelconfig.getint(area_key, 'temp_max') yticks = E.get_ticks(6, np.array([ymin, ymax])) data_units = 'K' # Next the wind divergence that is also read from external files # previously written by TropicalVariability_wind.py elif (datakey == 'divergence'): base_name = work_dir + 'TropicalVariability_'\ + area_key + '_' + datakey + '_model_' datafiles = glob(base_name + '*') key_models = [] for dfile in datafiles: start = dfile.find('_model_') + 7 key_models.append(str(dfile)[start:-4]) # now we can do the loop over the models for model in key_models: # extract model specific data alldata = np.load(base_name + model + '.npy') lons = alldata[0] data = 1E6 * alldata[1] color, dashes, width = E.get_model_plot_style(model) lons = E.ensure_looping(lons) # get the style and plot it out if (len(dashes) == 0): axs[row].plot(lons, data, color=color, linewidth=width, label=model) else: line, = axs[row].plot(lons, data, '--', color=color, linewidth=width, label=model) line.set_dashes(dashes) ymin = modelconfig.getint(area_key, 'div_min') ymax = modelconfig.getint(area_key, 'div_max') yticks = E.get_ticks(6, np.array([ymin, ymax])) data_units = '1/s' scaling = '1E6' # This part is the same for all subplots axs[row].set_ylim(ymin, ymax) axs[row].yaxis.set_ticks(yticks) xticks = E.get_ticks(9, np.array([lon_min, lon_max])) labels = E.get_ticks_labels(xticks, 'lons') axs[row].xaxis.set_ticks(xticks) axs[row].set_xticklabels(labels) if (lon_min < lon_max): axs[row].set_xlim(lon_min, lon_max) else: axs[row].set_xlim(lon_min, 360 + lon_max) if (len(scaling) > 0): axs[row].set_title(title + ' ' + scaling + ' * [' + data_units + ']') else: axs[row].set_title(title + ' [' + data_units + ']') axs[row].grid(plot_grid) # Adding the legend handles0, labels0 = axs[0].get_legend_handles_labels() handles1, labels1 = axs[1].get_legend_handles_labels() handles2, labels2 = axs[2].get_legend_handles_labels() handles3, labels3 = axs[3].get_legend_handles_labels() handles_all = handles0 + handles1 + handles2 + handles3 labels_all = labels0 + labels1 + labels2 + labels3 # Sort both lists based on labels labels_sorted = zip(*sorted(zip(labels_all, handles_all)))[0] handles_sorted = zip(*sorted(zip(labels_all, handles_all)))[1] # Now we remove duplicates handles = [] labels = [] for item in xrange(len(labels_sorted)): if (labels_sorted[item] not in labels): labels.append(labels_sorted[item]) handles.append(handles_sorted[item]) # Plotting the legend at its proper place plt.legend(handles, labels, loc='center left', bbox_to_anchor=(0.7, 0.5), bbox_transform=plt.gcf().transFigure) # Saving the plot and printing the image location diag_name = E.get_diag_script_name() output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.suptitle(area + " ocean mean wind direction and strength", fontsize=16) specifier = '-'.join([area, experiment, 'mean']) variable = "".join(E.get_currVars()) output_file = E.get_plot_output_filename(diag_name=diag_name, variable=variable, specifier=specifier) plt.savefig(os.path.join(output_dir, output_file)) info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1)
def process_scatter(E, modelconfig): """ Main script for gathering cloud vs radiation values and plotting them. """ config_file = E.get_configfile() experiment = 'SouthernHemisphere' areas = modelconfig.get(experiment, 'scatter_areas').split() seasons = modelconfig.get(experiment, 'seasons').split() plot_grid = modelconfig.getboolean('general', 'plot_background_grid') plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() work_dir = E.get_work_dir() datakeys = E.get_currVars() # Check which cloud diagnostic we are working with if ('clt' in datakeys): cl_key = 'clt' elif ('clivi' in datakeys): cl_key = 'clivi' elif ('clwvi' in datakeys): cl_key = 'clwvi' # Extract cloud observations datakeys.remove(cl_key) cl_obs, cl_loc, cl_models = E.get_clim_model_and_obs_filenames(cl_key) cl_obsfile = nc.Dataset(cl_loc, 'r') # A-laue_ax+ E.add_to_filelist(cl_loc) # A-laue_ax- # Basic structure of the figure is same for all # plots - we clear it at intervals for area in areas: # Get area specific configuration and observations coords = E.get_area_coordinates(modelconfig, experiment, area) area_key = experiment + '_scatter_' + area cl_lats, cl_lons, cl_data = E.get_model_data(modelconfig, experiment, area, cl_key, cl_obsfile) cl_data = E.average_data(cl_data, 'annual') for datakey in datakeys: # Extract radiation observations rd_obs, rd_loc, rd_models = E.get_clim_model_and_obs_filenames( datakey) rd_obsfile = nc.Dataset(rd_loc, 'r') # A-laue_ax+ E.add_to_filelist(rd_loc) # A-laue_ax- rd_lats, rd_lons, rd_data = E.get_model_data(modelconfig, experiment, area, datakey, rd_obsfile, extend='lons') # Check that we have similar model sets E.check_model_instances(cl_models, rd_models) # Average and interpolate to cloud obsevation grid rd_data = E.average_data(rd_data, 'annual') rd_data = interpolate_3d(rd_data, rd_lats, rd_lons, cl_lats, cl_lons) for model in rd_models: # Get model and area specific values cl_datafile = nc.Dataset(cl_models[model], 'r') rd_datafile = nc.Dataset(rd_models[model], 'r') # A-laue_ax+ E.add_to_filelist(cl_models[model]) E.add_to_filelist(rd_models[model]) # A-laue_ax- mcl_lats, mcl_lons, mcl_data = E.get_model_data( modelconfig, experiment, area, cl_key, cl_datafile) mrd_lats, mrd_lons, mrd_data = E.get_model_data(modelconfig, experiment, area, datakey, rd_datafile, extend='lons') cl_datafile.close() rd_datafile.close() # Average and interpolate model data to cloud data grid mcl_data = E.average_data(mcl_data, 'annual') mrd_data = E.average_data(mrd_data, 'annual') mrd_data = interpolate_3d(mrd_data, mrd_lats, mrd_lons, mcl_lats, mcl_lons) # One plot for all seasons plt.clf() fig, axs = plt.subplots(2, (len(seasons)), figsize=(4 * len(seasons), 8)) fig.subplots_adjust(top=0.8) fig.subplots_adjust(bottom=0.15) fig.subplots_adjust(right=0.75) fig.subplots_adjust(hspace=0.4) fig.subplots_adjust(wspace=0.4) for season in seasons: col = seasons.index(season) # Mask unwanted seasonal values obs_cl = E.extract_seasonal_mean_values(modelconfig, cl_data, experiment, season, monthly=True) obs_rd = E.extract_seasonal_mean_values(modelconfig, rd_data, experiment, season, monthly=True) data_cl = E.extract_seasonal_mean_values(modelconfig, mcl_data, experiment, season, monthly=True) data_rd = E.extract_seasonal_mean_values(modelconfig, mrd_data, experiment, season, monthly=True) # Calculate scatterplot values obs_out = calculate_scatterplot_values( modelconfig, area, cl_key, obs_cl, obs_rd) model_out = calculate_scatterplot_values(modelconfig, area, cl_key, data_cl, data_rd, bins=obs_out[0]) centers = np.zeros(len(obs_out[0]) - 1) for i in xrange(len(centers)): centers[i] = (obs_out[0][i + 1] + obs_out[0][i]) / 2. # Plot and make it pretty modelcolor, dashes, width = E.get_model_plot_style(model) obs_label = cl_obs + ', \n' + rd_obs axs[0, col].scatter(model_out[2], model_out[3], facecolors=modelcolor, edgecolors=modelcolor, label=model) axs[0, col].scatter(obs_out[2], obs_out[3], facecolors='none', edgecolors='black', label=obs_label) if (len(dashes) == 0): axs[1, col].plot(centers, obs_out[1], color=modelcolor, linewidth=width, label=model) else: line1, = axs[1, col].plot(centers, model_out[1], '--', color=modelcolor, linewidth=width, label=model) line1.set_dashes(dashes) axs[1, col].plot(centers, obs_out[1], color='black', label=obs_label) # If we get nan as values we convert it to zero obs_x = np.ma.masked_array(obs_out[2]) obs_y = np.ma.masked_array(obs_out[3]) model_x = np.ma.masked_array(model_out[2]) model_y = np.ma.masked_array(model_out[3]) rmse = (np.mean((obs_x - model_x)**2) + np.mean( (obs_y - model_y)**2))**0.5 rmse = round(rmse, 1) xmin = int(obs_out[0][0]) xmax = int(np.ceil(obs_out[0][-1])) # Make it prettier axs[0, col].set_title(season + " (RMSE: " + str(rmse) + ")", fontsize=12) axs[0, col].set_xlim(xmin, xmax) axs[0, col].locator_params(axis='x', nbins=5) axs[0, col].locator_params(axis='y', nbins=5) axs[0, col].grid(plot_grid) axs[1, col].set_xlim(xmin, xmax) axs[1, col].locator_params(axis='x', nbins=5) axs[1, col].locator_params(axis='y', nbins=5) axs[1, col].grid(plot_grid) xlabel = E.get_title_basename(cl_key) ylabel = E.get_title_basename(datakey) axs[0, 0].set_xlabel(xlabel) axs[0, 0].set_ylabel('Radiation') axs[1, 0].set_xlabel('Cloud cover percentage') axs[1, 0].set_ylabel('Proportion of values') # Title and legend suptitle = ylabel + " sensitivity to " + xlabel plt.suptitle(suptitle, fontsize=20) handles0, labels0 = axs[0, 0].get_legend_handles_labels() handles1, labels1 = axs[1, 0].get_legend_handles_labels() handles = handles0 + handles1 labels = labels0 + labels1 plt.legend(handles, labels, loc='center left', bbox_to_anchor=(0.78, 0.5), bbox_transform=plt.gcf().transFigure) # Define ouput name and save variable = cl_key + '-' + datakey diag_name = E.get_diag_script_name() output_file = E.get_plot_output_filename(diag_name=diag_name, variable=variable, model=model) output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.savefig(os.path.join(output_dir, output_file)) plt.clf() info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1) cl_obsfile.close() rd_obsfile.close()
def process_divergence(E, modelconfig): """This script preprocessess equatorial divergence values. The data is extracted and saved to a npy-file. Plotting is done in TropicalVariability_EQ.py. """ experiment = 'equatorial' datakeys = E.get_currVars() verbosity = E.get_verbosity() plot_dir = E.get_plot_dir() work_dir = E.get_work_dir() areas = modelconfig.get(experiment, 'areas').split() if ('ua' in datakeys[0]): ua_key = datakeys[0] va_key = datakeys[1] elif ('ua' in datakeys[1]): ua_key = datakeys[1] va_key = datakeys[0] else: print("PY ERROR: Unexpected variables for divergence.") print("PY ERROR: I was expecting only 'ua' and 'va' (or similar).") print("PY ERROR: What I got was: " + datakeys[0] + ", " + datakeys[1]) print("PY ERROR: Stopping the script and exiting") sys.exit() # Starting to extract required areas for area in areas: area_key = experiment + '_' + area lat_min = modelconfig.getint(area_key, 'lat_min') lat_max = modelconfig.getint(area_key, 'lat_max') lon_min = modelconfig.getint(area_key, 'lon_min') lon_max = modelconfig.getint(area_key, 'lon_max') # create a python array file for each area, model and datakey # overwrite if exists base_name = work_dir\ + 'TropicalVariability_'\ + area_key\ + '_divergence_model_' ua_models = E.get_clim_model_filenames(variable=ua_key) va_models = E.get_clim_model_filenames(variable=va_key) E.check_model_instances(ua_models, va_models) # Get obs model name and extract these first for model in ua_models: if (E.get_model_id(model) == 'obs'): obs = model out_file = base_name + model + '.npy' ua_datafile = nc.Dataset(ua_models[model], 'r') va_datafile = nc.Dataset(va_models[model], 'r') # A-laue_ax+ E.add_to_filelist(ua_models[model]) E.add_to_filelist(va_models[model]) # A-laue_ax- # Add ghost layers for ua longtitudes # va doesn't need them. Lats and lons are same for both ulats, ulons, ua_data = E.get_model_data(modelconfig, experiment, area, ua_key, ua_datafile, extend='lons') vlats, vlons, va_data = E.get_model_data(modelconfig, experiment, area, va_key, va_datafile, extend='lats') ulons = E.ensure_looping(ulons) vlons = E.ensure_looping(vlons) ua_datafile.close() va_datafile.close() ua_obs, ua_derv = calculate_derivatives( E, ulats, ulons, ua_data, 'ua') va_obs, va_derv = calculate_derivatives( E, vlats, vlons, va_data, 'va') diverg = np.zeros(len(vlons)) for lon in xrange(diverg.shape[0]): diverg[lon] = np.mean(ua_derv[:, lon] + va_derv[:, lon]) # Now we have everything so only output needed outdata = vlons, diverg np.save(out_file, outdata) obscolor, obsdashes, obswidth = E.get_model_plot_style(model) obslats, obslons = ulats, vlons del ua_models[obs] # Now we do the same for all models and plot the wind vector field for model in ua_models: out_file = base_name + model + '.npy' ua_datafile = nc.Dataset(ua_models[model], 'r') va_datafile = nc.Dataset(va_models[model], 'r') # A-laue_ax+ E.add_to_filelist(ua_models[model]) E.add_to_filelist(va_models[model]) # A-laue_ax- # Add ghost layers for ua longtitudes # va doesn't need them. Lats and lons are same for both ulats, ulons, ua_data = E.get_model_data(modelconfig, experiment, area, ua_key, ua_datafile, extend='lons') vlats, vlons, va_data = E.get_model_data(modelconfig, experiment, area, va_key, va_datafile, extend='lats') ulons = E.ensure_looping(ulons) vlons = E.ensure_looping(vlons) ua_datafile.close() va_datafile.close() ua_mean, ua_derv = calculate_derivatives(E, ulats, ulons, ua_data, 'ua') va_mean, va_derv = calculate_derivatives(E, vlats, vlons, va_data, 'va') diverg = np.zeros(len(vlons)) for lon in xrange(diverg.shape[0]): diverg[lon] = np.mean(ua_derv[:, lon] + va_derv[:, lon]) # Now we have everything so only output needed outdata = vlons, diverg np.save(out_file, outdata) # Interpolate model data to obs grid ua_mean = interpolate_data_grid(ua_mean, ulats, vlons, obslats, obslons) va_mean = interpolate_data_grid(va_mean, ulats, vlons, obslats, obslons) color, dashes, width = E.get_model_plot_style(model) # Ensure proper handling for lons if (lon_max < lon_min): lon_max += 360 # Start the plots plt.clf() fig, axs = plt.subplots(2, 1, figsize=(5 + (lon_max - lon_min) / 10, 10)) fig.subplots_adjust(hspace=0.3) for ax in axs.flat: map_ax = Basemap(ax=ax, fix_aspect=False, llcrnrlat=obslats[0], urcrnrlat=obslats[-1], llcrnrlon=obslons[0], urcrnrlon=obslons[-1]) map_ax.drawcoastlines() Q0 = axs[0].quiver(obslons, obslats, va_mean, ua_mean, color=color, pivot='middle', units='xy', scale=5.0, width=0.1, headwidth=4, headlength=3, headaxislength=4) Q1 = axs[1].quiver(obslons, obslats, va_obs, ua_obs, color=obscolor, pivot='middle', units='xy', scale=5.0, width=0.1, headwidth=4, headlength=3, headaxislength=4) qk0 = plt.quiverkey(Q0, 1.05, 0.5, 5, r'$5 \frac{m}{s}$') qk1 = plt.quiverkey(Q1, 1.05, 0.5, 5, r'$5 \frac{m}{s}$') # Ticks, labels, header etc. xticks = E.get_ticks(10, np.array([lon_min, lon_max])) yticks = E.get_ticks(5, np.array([lat_min, lat_max])) xlabels = E.get_ticks_labels(xticks, 'lons') ylabels = E.get_ticks_labels(yticks, 'lats') axs[0].set_title(model) axs[0].set_xlim(lon_min, lon_max) axs[0].set_ylim(lat_min, lat_max) axs[0].xaxis.set_ticks(xticks) axs[0].yaxis.set_ticks(yticks) axs[0].set_xticklabels(xlabels) axs[0].set_yticklabels(ylabels) axs[1].set_title(obs) axs[1].set_xlim(lon_min, lon_max) axs[1].set_ylim(lat_min, lat_max) axs[1].xaxis.set_ticks(xticks) axs[1].yaxis.set_ticks(yticks) axs[1].set_xticklabels(xlabels) axs[1].set_yticklabels(ylabels) # Get output filename and save the figure diag_name = E.get_diag_script_name() output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.suptitle(area + " ocean mean wind direction and strength", fontsize=16) variable_string = ''.join([ua_key, va_key]) output_file = E.get_plot_output_filename(diag_name=diag_name, model=model, variable=variable_string, specifier=area) plt.savefig(os.path.join(output_dir, output_file)) info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1)
def process_radiation_maps(E, modelconfig, datakey, specifier): """ Main script for gathering radiation seasonal map values and plotting them. """ config_file = E.get_configfile() experiment = 'SouthernHemisphere' areas = modelconfig.get(experiment, 'areas').split() seasons = modelconfig.get(experiment, 'seasons').split() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() work_dir = E.get_work_dir() datakeycs = datakey + 'cs' # Check that we have similar model sets (obs should be the same) obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey) obsmodel, obscs_loc, modelscs = E.get_clim_model_and_obs_filenames(datakeycs) E.check_model_instances(models, modelscs) # Loop over areas and seasons - generate season specific maps obsfile = nc.Dataset(obs_loc, 'r') obscsfile = nc.Dataset(obscs_loc, 'r') # A-laue_ax+ E.add_to_filelist(obs_loc) E.add_to_filelist(obscs_loc) # A-laue_ax- # Basic structure of the figure is same for all plots, # we clear it at intervals for area in areas: # Get area specific configuration and observations coords = E.get_area_coordinates(modelconfig, experiment, area) xticks = E.get_ticks(5, coords[2:]) yticks = E.get_ticks(3, coords[:2]) xlabels = E.get_ticks_labels(xticks, 'lons') ylabels = E.get_ticks_labels(yticks, 'lats') area_key = experiment + '_' + area obslats, obslons, obsdata = E.get_model_data(modelconfig, experiment, area, datakey, obsfile) obscslats, obscslons, obscsdata = E.get_model_data(modelconfig, experiment, area, datakeycs, obscsfile) for model in models: # Get model and area specific values datafile = nc.Dataset(models[model], 'r') datafilecs = nc.Dataset(modelscs[model], 'r') # A-laue_ax+ E.add_to_filelist(models[model]) E.add_to_filelist(modelscs[model]) # A-laue_ax- mlats, mlons, mdata = E.get_model_data(modelconfig, experiment, area, datakey, datafile) mcslats, mcslons, mcsdata = E.get_model_data(modelconfig, experiment, area, datakeycs, datafilecs) units = datafile.variables[datakey].units for season in seasons: # Extract seasonal specific values from area values data = E.extract_seasonal_mean_values(modelconfig, mdata, experiment, season) datacs = E.extract_seasonal_mean_values(modelconfig, mcsdata, experiment, season) obs = E.extract_seasonal_mean_values(modelconfig, obsdata, experiment, season) obscs = E.extract_seasonal_mean_values(modelconfig, obscsdata, experiment, season) # Interpolate to obs grid data = interpolate_data_grid(data, mlats, mlons, obslats, obslons) datacs = interpolate_data_grid(datacs, mcslats, mcslons, obslats, obslons) obscs = interpolate_data_grid(obscs, obscslats, obscslons, obslats, obslons) # Get contour specific plotting limits a, b, c, d, e, f = get_contour_config(modelconfig, config_file, area_key, datakey) lev, diff, diff2, cm_model, cm_diff, cm_dev = a, b, c, d, e, f # Plot the figures fig, axs = plt.subplots(3, 2, figsize=(21, 15)) fig.subplots_adjust(hspace=1.0) # Same Basemap (coastlines) for all subplots # This is what usually takes time to process for ax in axs.flat: map_ax = Basemap(ax=ax, fix_aspect=False, llcrnrlat=coords[0], urcrnrlat=coords[1], llcrnrlon=coords[2], urcrnrlon=coords[3]) map_ax.drawcoastlines() # Plot the contours and colourmaps dash = ' - ' # Could also use u"\u2014" for long dash c00 = axs[0, 0].contourf(obslons, obslats, data, levels=lev, cmap=cm_model, extend='both', latlon=True) cax00 = fig.add_axes([0.15, 0.69, 0.3, 0.01]) cbar00 = plt.colorbar(c00, cax=cax00, orientation='horizontal') cbar00.set_label(units) axs[0, 0].set_title(model) c01 = axs[0, 1].contourf(obslons, obslats, datacs, levels=lev, cmap=cm_model, extend='both', latlon=True) cax01 = fig.add_axes([0.575, 0.69, 0.3, 0.01]) cbar01 = plt.colorbar(c01, cax=cax01, orientation='horizontal') cbar01.set_label(units) axs[0, 1].set_title(model + ' (clear sky)') c10 = axs[1, 0].contourf(obslons, obslats, data - obs, levels=diff, cmap=cm_diff, extend='both', latlon=True) cax10 = fig.add_axes([0.15, 0.37, 0.3, 0.01]) cbar10 = plt.colorbar(c10, cax=cax10, orientation='horizontal') cbar10.set_label(units) axs[1, 0].set_title(model + dash + obsmodel) c11 = axs[1, 1].contourf(obslons, obslats, datacs - obscs, levels=diff, cmap=cm_diff, extend='both', latlon=True) cax11 = fig.add_axes([0.575, 0.37, 0.3, 0.01]) cbar11 = plt.colorbar(c11, cax=cax11, orientation='horizontal') cbar11.set_label(units) axs[1, 1].set_title(model + ' (cs)' + dash + obsmodel + ' (cs)') c20 = axs[2, 0].contourf(obslons, obslats, data - datacs, levels=diff2, cmap=cm_dev, extend='both', latlon=True) cax20 = fig.add_axes([0.15, 0.05, 0.3, 0.01]) cbar20 = plt.colorbar(c20, cax=cax20, orientation='horizontal') cbar20.set_label(units) axs[2, 0].set_title(model + dash + model + ' (cs)') c21 = axs[2, 1].contourf(obslons, obslats, obs - obscs, levels=diff2, cmap=cm_dev, extend='both', latlon=True) cax21 = fig.add_axes([0.575, 0.05, 0.3, 0.01]) cbar21 = plt.colorbar(c21, cax=cax21, orientation='horizontal') cbar21.set_label(units) axs[2, 1].set_title(obsmodel + dash + obsmodel + ' (cs)') # Labels for row in xrange(3): for col in xrange(2): axs[row, col].xaxis.set_ticks(xticks) axs[row, col].set_xticklabels(xlabels) axs[row, col].yaxis.set_ticks(yticks) axs[row, col].set_yticklabels(ylabels) # Title and filename based on variables suptitle = E.get_title_basename(datakey) + " for months: " + season plt.suptitle(suptitle, fontsize=20) variable = datakey specifier = specifier + "-" + season diag_name = E.get_diag_script_name() output_file = E.get_plot_output_filename(variable=variable, specifier=specifier, model=model) output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.savefig(os.path.join(output_dir, output_file)) plt.clf() info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1) datafile.close() datafilecs.close() obsfile.close() obscsfile.close()
def process_scatterplot(E, modelconfig): """Main script for scatterplots. Outputs model specific scatterplots with specified observations. Also prints out comparable statistics. """ experiment = 'scatter' config_file = E.get_configfile() datakeys = E.get_currVars() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() plot_grid = modelconfig.getboolean('general', 'plot_grid') areas = modelconfig.get(experiment, 'areas').split() seasons = modelconfig.get(experiment, 'seasons').split() # Scatterplots are only for temperature/precipitation if 'ua' in datakeys: datakeys = datakeys.remove('ua') # We extract the explicit precipitation datakey since precip can be in # mmday or kg/m2s - this affects the datakey (pr or pr-mmday) # For consistency we do this also for ts for datakey in datakeys: if (datakey == 'ts'): ts_key = datakey elif (datakey == 'pr' or datakey == 'pr-mmday'): pr_key = datakey # Get the required filenames and check consistency of model filenames # The model keys should be same - the key values (paths) are different pr_model, pr_obs_loc, pr_models = E.get_clim_model_and_obs_filenames( pr_key) ts_model, ts_obs_loc, ts_models = E.get_clim_model_and_obs_filenames( ts_key) E.check_model_instances(pr_models, ts_models) # Initialize observations pr_obs_file = nc.Dataset(pr_obs_loc, 'r') ts_obs_file = nc.Dataset(ts_obs_loc, 'r') # A-laue_ax+ E.add_to_filelist(pr_obs_loc) E.add_to_filelist(ts_obs_loc) # A-laue_ax- # We are outputting model specific scatterplots with areas and seasons # So some looping to do. Remember that keys in pr_models and ts_models # are the same (we checked this) - the paths differ. for model in pr_models: # Initializing model specific files pr_file = nc.Dataset(pr_models[model], 'r') ts_file = nc.Dataset(ts_models[model], 'r') # A-laue_ax+ E.add_to_filelist(pr_models[model]) E.add_to_filelist(ts_models[model]) # A-laue_ax- pr_units = pr_file.variables[pr_key].units ts_units = ts_file.variables[ts_key].units # Extract model specific plotting style (we only need the colour) modelcolor, dashes, width = E.get_model_plot_style(model) plt.clf() fig, axs = plt.subplots(len(areas), len(seasons), figsize=(3 * len(seasons), 1 + 3 * (len(areas)))) fig.subplots_adjust(top=0.83) fig.subplots_adjust(right=0.85) fig.subplots_adjust(hspace=0.4) fig.subplots_adjust(wspace=0.3) # Next we extract the required values for each area for area in areas: pr_data = E.get_model_data(modelconfig, experiment, area, pr_key, pr_file) ts_data = E.get_model_data(modelconfig, experiment, area, ts_key, ts_file) pr_obs = E.get_model_data(modelconfig, experiment, area, pr_key, pr_obs_file) ts_obs = E.get_model_data(modelconfig, experiment, area, ts_key, ts_obs_file) row = areas.index(area) area_key = experiment + '_' + area lat_min = modelconfig.getint(area_key, 'lat_min') lat_max = modelconfig.getint(area_key, 'lat_max') lon_min = modelconfig.getint(area_key, 'lon_min') lon_max = modelconfig.getint(area_key, 'lon_max') lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats') lon_area = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons') c_area = "[" + lat_area[0] + ":" + lat_area[1] + ", "\ + lon_area[0] + ":" + lon_area[1] + "]" # And one more loop for different seasons for season in seasons: col = seasons.index(season) pr_season = get_scatterplot_values(modelconfig, season, pr_data) ts_season = get_scatterplot_values(modelconfig, season, ts_data) pr_obs_se = get_scatterplot_values(modelconfig, season, pr_obs) ts_obs_se = get_scatterplot_values(modelconfig, season, ts_obs) # plot axs[row, col].scatter(ts_season, pr_season, facecolors=modelcolor, edgecolors=modelcolor, label='modelled') axs[row, col].scatter(ts_obs_se, pr_obs_se, facecolors='none', edgecolors='black', label='observed') # statitics: errors in precip/temp and both mse_pr = np.mean((pr_season - pr_obs_se)**2) mse_ts = np.mean((ts_season - ts_obs_se)**2) mse = (mse_pr + mse_ts)**0.5 rmse = np.round(mse, 2) if (row == 0): axs[row, col].set_title(season + "\n rmse: " + str(rmse)) else: axs[row, col].set_title("rmse: " + str(rmse)) # Get specific plotting limits if modelconfig.getboolean(experiment, 'seasonal_limits'): # If we use user defined limits limits = get_scatterplot_limits(modelconfig, config_file, area, season) if (len(limits) == 4): axs[row, col].set_xlim(limits[0], limits[1]) axs[row, col].set_ylim(limits[2], limits[3]) xticks = E.get_ticks(5, limits[:2]) yticks = E.get_ticks(5, limits[2:]) else: xticks = E.get_ticks(5, ts_season, ts_obs_se) yticks = E.get_ticks(5, pr_season, pr_obs_se) # Setting the ticks (where to place the numbers axs[row, col].xaxis.set_ticks(xticks) axs[row, col].yaxis.set_ticks(yticks) else: # Let the Gods decide ts_min = int(np.min(ts_obs_se - 0.7)) ts_max = int(np.max(ts_obs_se + 1.7)) pr_min = int(np.min(pr_obs_se - 1.7)) pr_max = int(np.max(pr_obs_se + 2.7)) axs[row, col].set_xlim(ts_min, ts_max) axs[row, col].set_ylim(pr_min, pr_max) if (ts_max - ts_min > 4): axs[row, col].locator_params(axis='x', nbins=5) else: axs[row, col].locator_params(axis='x', nbins=ts_max - ts_min) if (pr_max - pr_min > 6): axs[row, col].locator_params(axis='y', nbins=7) else: axs[row, col].locator_params(axis='y', nbins=pr_max - pr_min) axs[row, -1].yaxis.set_label_position("right") axs[row, -1].set_ylabel(area + "\n" + c_area, rotation=0, horizontalalignment='left') axs[row, col].grid(plot_grid) # Closing the model datafiles pr_file.close() ts_file.close() axs[-1, 0].set_ylabel("Precipitation [" + pr_units + "]") axs[-1, 0].set_xlabel("SST [" + ts_units + "]") plt.suptitle("Seasonal (annual/monthly) mean values of " + "SST and precipitation for \n" + model + " (coloured) vs " + ts_model + " (sst obs) and " + pr_model + " (pr obs)", fontsize=20) # Saving the plot and letting the user know what just happened variable = ''.join([ts_key, pr_key]) diag_name = E.get_diag_script_name() output_file = E.get_plot_output_filename(diag_name=diag_name, variable=variable, model=model, specifier=experiment) output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.savefig(os.path.join(output_dir, output_file)) info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1) # After all the loops remember to close the observation files aswell pr_obs_file.close() ts_obs_file.close()
def process_mean_plots(E, modelconfig, datakey, orientation, mean_name): """Generates latitudal / lontitudal / monthly mean plots. Seasonal values are extracted for lat/lon plots. """ experiment = 'SouthernHemisphere' plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() areas = [] plot_grid = modelconfig.getboolean('general', 'plot_background_grid') # Extract required keys based on datakey if (orientation == 'monthly'): if (modelconfig.getboolean('general', 'plot_monthly_averages')): areas += modelconfig.get(experiment, 'areas').split() if (modelconfig.getboolean('general', 'plot_sub_areas')): areas += modelconfig.get(experiment, 'sub_areas').split() seasons = ['monthly'] else: areas += modelconfig.get(experiment, 'areas').split() seasons = modelconfig.get(experiment, 'seasons').split() # Observations location obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey) obsfile = nc.Dataset(obs_loc, 'r') # A-laue_ax+ E.add_to_filelist(obs_loc) # A-laue_ax- scale_cloud = False # Ugly fix for scaling cloud ice/liquid water path values if (datakey == 'clivi' or datakey == 'clwvi'): scale_cloud = True scale = 1E3 scale_str = ' * 1E3' for area in areas: # Value configurations area_key = experiment + '_' + area lat_min = modelconfig.getint(area_key, 'lat_min') lat_max = modelconfig.getint(area_key, 'lat_max') lon_min = modelconfig.getint(area_key, 'lon_min') lon_max = modelconfig.getint(area_key, 'lon_max') lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats') # First extract the observations and then start looping over seasons obslats, obslons, obsdata = E.get_model_data(modelconfig, experiment, area, datakey, obsfile) for season in seasons: # Plot layout configuration plt.clf() fig, axs = plt.subplots(2, 1, figsize=(15, 10)) fig.subplots_adjust(top=0.9) fig.subplots_adjust(right=0.67) fig.subplots_adjust(hspace=0.35) if (orientation == 'lats'): odata = E.extract_seasonal_mean_values(modelconfig, obsdata, experiment, season, monthly=True) odata = E.average_data(odata, 1) xobs = obslats elif (orientation == 'lons'): odata = E.extract_seasonal_mean_values(modelconfig, obsdata, experiment, season, monthly=True) odata = E.average_data(odata, 2) xobs = obslons elif (orientation == 'monthly'): odata = E.average_data(obsdata, 'monthly') xobs = np.arange(0, 12, 1) # Bad cloud values fix if (scale_cloud): odata = odata * scale # Plot model values to first graph ocolor, odashes, owidth = E.get_model_plot_style(obsmodel) if (len(odashes) == 0): axs[0].plot(xobs, odata, color=ocolor, linewidth=owidth, label=obsmodel + ' (obs)') else: line1, = axs[0].plot(xobs, odata, '--', color=ocolor, linewidth=owidth, label=obsmodel + ' (obs)') line1.set_dashes(odashes) multimodelmean_initialized = False for model in models: # Read in model specific data datafile = nc.Dataset(models[model], 'r') # A-laue_ax+ E.add_to_filelist(models[model]) # A-laue_ax- data_units = datafile.variables[datakey].units if (orientation == 'monthly'): lats, lons, data = E.get_model_data(modelconfig, experiment, area, datakey, datafile) else: lats, lons, data = E.get_model_data(modelconfig, experiment, area, datakey, datafile, extend=orientation) datafile.close() # Bad cloud values fix if (scale_cloud): data = data * scale if (data_units == 'kg m-2'): data_units = r'$\mu$m' else: data_units += scale_str # Process depending on orientation if (orientation == 'lats'): data = E.extract_seasonal_mean_values(modelconfig, data, experiment, season, monthly=True) data = E.average_data(data, 1) xdata = lats interpolate = interp1d(xdata, data, kind='cubic', bounds_error=False) idata = interpolate(xobs) elif (orientation == 'lons'): data = E.extract_seasonal_mean_values(modelconfig, data, experiment, season, monthly=True) data = E.average_data(data, 2) xdata = lons interpolate = interp1d(xdata, data, kind='cubic', bounds_error=False) idata = interpolate(xobs) elif (orientation == 'monthly'): data = E.average_data(data, 'monthly') xdata = xobs idata = data # plotting custom dashes requires some extra effort (else) # with empty dashes the format is default rmse = round(np.mean((idata - odata) ** 2) ** 0.5, 1) mlabel = model + " (RMSE: " + str(rmse) + ")" color, dashes, width = E.get_model_plot_style(model) if (len(dashes) == 0): axs[0].plot(xdata, data, color=color, linewidth=width, label=mlabel) axs[1].plot(xobs, idata - odata, color=color, linewidth=width, label=mlabel) else: line1, = axs[0].plot(xdata, data, '--', color=color, linewidth=width, label=mlabel) line1.set_dashes(dashes) line2, = axs[1].plot(xobs, idata - odata, color=color, linewidth=width, label=mlabel) line2.set_dashes(dashes) # Next the multimodel mean. We can just sum it up and divide if not (multimodelmean_initialized): multimodelmean_initialized = True mmmean = idata else: mmmean += idata # Plot the multimodel mean out if required if (len(models) > 1): mmmean = mmmean / len(models) rmse = round(np.mean((mmmean - odata) ** 2) ** 0.5, 1) mlabel = "Multimodel mean (RMSE: " + str(rmse) + ")" color, dashes, width = E.get_model_plot_style('model_mean') if (len(dashes) == 0): axs[0].plot(xobs, mmmean, color=color, linewidth=width, label=mlabel) axs[1].plot(xobs, mmmean - odata, color=color, linewidth=width, label=mlabel) else: line1, = axs[0].plot(xobs, mmmean, '--', color=color, linewidth=width, label=mlabel) line1.set_dashes(dashes) line2, = axs[1].plot(xobs, mmmean - odata, color=color, linewidth=width, label=mlabel) line2.set_dashes(dashes) # Now to make the plot pretty i.e. headers, ticks, title etc. suptitle = E.get_title_basename(datakey) specifier = mean_name if (orientation == 'lats'): title = "Latitudinal mean" specifier += '-latitudinal-mean' x_min = lat_min x_max = lat_max xticks = E.get_ticks(8, np.array([x_min, x_max])) lon_info = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons') if (int(lon_max) == 360): suptitle += " (lons [" + lon_info[0] + ":360" + "])" else: suptitle += " (lons [" + lon_info[0] + ":" + lon_info[1] + "])" elif (orientation == 'lons'): title = "Longitudinal mean" specifier += '-longitudinal-mean' x_min = lon_min x_max = lon_max xticks = E.get_ticks(8, np.array([x_min, x_max])) lat_info = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats') suptitle += " (lats [" + lat_info[0] + ":" + lat_info[1] + "])" elif (orientation == 'monthly'): title = "Monthly mean" specifier += '-yearly-cycle-' + area x_min = 0 x_max = 11 xticks = xobs lat_info = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats') lon_info = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons') suptitle += " (lats [" + lat_info[0] + ":" + lat_info[1] + "], " if (lon_max == 360): suptitle += " lons [" + lon_info[0] + ":360" + "])" else: suptitle += " lons [" + lon_info[0] + ":" + lon_info[1] + "])" if (season == 'monthly'): pass else: suptitle += " for " + season specifier = specifier + "-" + season plt.suptitle(suptitle, fontsize=20) labels = E.get_ticks_labels(xticks, orientation) axs[0].grid(plot_grid) axs[0].set_xlim(x_min, x_max) axs[0].xaxis.set_ticks(xticks) axs[0].set_xticklabels(labels) axs[0].locator_params(axis='y', nbins=5) axs[0].set_title(title + " [" + data_units + "]") axs[1].grid(plot_grid) axs[1].set_xlim(x_min, x_max) axs[1].xaxis.set_ticks(xticks) axs[1].set_xticklabels(labels) axs[1].locator_params(axis='y', nbins=5) axs[1].set_title(title + " (model - obs) [" + data_units + "]") # Saving the plot and letting the user know what just happened diag_name = E.get_diag_script_name() output_file = E.get_plot_output_filename(diag_name=diag_name, variable=datakey, specifier=specifier) output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) handles, labels = axs[0].get_legend_handles_labels() plt.legend(handles, labels, loc='center left', bbox_to_anchor=(0.7, 0.5), bbox_transform=plt.gcf().transFigure) plt.savefig(os.path.join(output_dir, output_file)) info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1) obsfile.close()
def main(_argv): print("Start") info("OpenCV version: " + CV_VERSION) # Default parameters: args = Struct( ImageFile= "/Users/shahargino/Documents/ImageProcessing/Bracelet_decoder/Database/180717/24.png", PreprocessCvcSel="V", PreprocessMode="Legacy", PreprocessGaussKernel=(5, 5), PreprocessThreshBlockSize=19, PreprocessThreshweight=7, PreprocessMorphKernel=(3, 3), PreprocessMedianBlurKernel=13, PreprocessCannyThr=80, imgEnhancementEn=False, MinPixelWidth=7, MaxPixelWidth=30, MinPixelHeight=7, MaxPixelHeight=30, MinAspectRatio=0.6, MaxAspectRatio=2.5, MinPixelArea=150, MaxPixelArea=600, MinExtent=0.4, MaxExtent=0.9, MaxDrift=2.5, MarksRows=3, MarksCols=10, ROI=(0, 0), PerspectiveMode=0, FindContoursMode="Legacy", HoughParams=(-1, -1, -1, -1, -1, -1), debugMode=False) # -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. # User-Arguments parameters (overrides Defaults): try: opts, user_args = getopt(_argv, "hvi:", [ "PreprocessCvcSel=", "PreprocessMode=", "PreprocessGaussKernel=", "PreprocessThreshBlockSize=", "PreprocessThreshweight=", "PreprocessMorphKernel=", "PreprocessMedianBlurKernel=", "PreprocessCannyThr=", "ROI=", "MinPixelWidth=", "MaxPixelWidth=", "MinPixelHeight=", "MaxPixelHeight=", "MinAspectRatio=", "MaxAspectRatio=", "MinPixelArea=", "MaxPixelArea=", "MinExtent=", "MaxExtent=", "MaxDrift=", "MarksRows=", "MarksCols=", "imgEnhancementEn", "PerspectiveMode=", "FindContoursMode=", "HoughParams=", "debug", "version" ]) for opt, user_arg in opts: if opt == '-h': usage() exit() elif opt in "-i": args.ImageFile = user_arg elif opt in "--PreprocessCvcSel": args.PreprocessCvcSel = user_arg elif opt in "--PreprocessMode": args.PreprocessMode = user_arg elif opt in "--PreprocessGaussKernel": args.PreprocessGaussKernel = literal_eval(user_arg) elif opt in "--PreprocessThreshBlockSize": args.PreprocessThreshBlockSize = int(user_arg) elif opt in "--PreprocessThreshweight": args.PreprocessThreshweight = int(user_arg) elif opt in "--PreprocessMorphKernel": args.PreprocessMorphKernel = literal_eval(user_arg) elif opt in "--PreprocessMedianBlurKernel": args.PreprocessMedianBlurKernel = int(user_arg) elif opt in "--PreprocessCannyThr": args.PreprocessCannyThr = int(user_arg) elif opt in "--ROI": args.ROI = literal_eval(user_arg) elif opt in "--MinPixelWidth": args.MinPixelWidth = float(user_arg) elif opt in "--MaxPixelWidth": args.MaxPixelWidth = float(user_arg) elif opt in "--MinPixelHeight": args.MinPixelHeight = float(user_arg) elif opt in "--MaxPixelHeight": args.MaxPixelHeight = float(user_arg) elif opt in "--MinAspectRatio": args.MinAspectRatio = float(user_arg) elif opt in "--MaxAspectRatio": args.MaxAspectRatio = float(user_arg) elif opt in "--MinPixelArea": args.MinPixelArea = float(user_arg) elif opt in "--MaxPixelArea": args.MaxPixelArea = float(user_arg) elif opt in "--MinExtent": args.MinExtent = float(user_arg) elif opt in "--MaxExtent": args.MaxExtent = float(user_arg) elif opt in "--MaxDrift": args.MaxDrift = float(user_arg) elif opt in "--MarksRows": args.MarksRows = int(user_arg) elif opt in "--MarksCols": args.MarksCols = int(user_arg) elif opt in "--imgEnhancementEn": args.imgEnhancementEn = True elif opt in "--PerspectiveMode": args.PerspectiveMode = int(user_arg) elif opt in "--FindContoursMode": args.FindContoursMode = user_arg elif opt in "--HoughParams": args.HoughParams = literal_eval(user_arg) elif opt in "--debug": args.debugMode = True elif opt in "--version" or opt == '-v': info("BraceletDecoder version: %s" % __version__) exit() except GetoptError as e: error(str(e)) usage() exit(2) # -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. # Load input image: frame_orig = imread(args.ImageFile) height, width, _ = frame_orig.shape # Create a working environment (debugMode): if args.debugMode: envpath = datetime.now().strftime( "results_%d%m%y_%H%M%S_") + args.ImageFile.split('/')[-1].replace( '.', '_') if not path.exists(envpath): makedirs(envpath) cwd = getcwd() chdir(envpath) # Resizing preparations: if args.PreprocessMode == "Legacy": resizingVec = (1600, 1200) if (width < height): resizingVec = resizingVec[::-1] elif args.PreprocessMode == "BlurAndCanny": resizingVec = (1120, 840) else: error("Unsupported PreprocessMode mode: %s" % args.PreprocessMode) # Resizing and ROI cropping: if args.PreprocessMode == "Legacy": imgResized = resize(frame_orig, resizingVec) imgCropped = crop_roi_from_image(imgResized, args.ROI) image = imgCropped elif args.PreprocessMode == "BlurAndCanny": imgCropped = crop_roi_from_image(frame_orig, args.ROI) imgResized = resize(imgCropped, resizingVec) image = imgResized # Image enhancement: if (args.imgEnhancementEn): imgEnhanced = imageEnhancement(image, 2, (8, 8), 3, args.debugMode) else: imgEnhanced = image # Preprocess: frame_gray, frame_thresh = preprocess( imgEnhanced, args.PreprocessCvcSel, args.PreprocessMode, args.PreprocessGaussKernel, args.PreprocessThreshBlockSize, args.PreprocessThreshweight, args.PreprocessMorphKernel, args.PreprocessMedianBlurKernel, args.PreprocessCannyThr) # Find bracelet marks: possible_marks, rotation_angle = find_possible_marks( frame_thresh, args.MinPixelWidth, args.MaxPixelWidth, args.MinPixelHeight, args.MaxPixelHeight, args.MinAspectRatio, args.MaxAspectRatio, args.MinPixelArea, args.MaxPixelArea, args.MinExtent, args.MaxExtent, args.MaxDrift, args.PerspectiveMode, args.FindContoursMode, args.HoughParams, args.debugMode) # Encode marks: code = decode_marks(possible_marks, args.MarksRows, args.MarksCols, frame_thresh.shape, rotation_angle, args.debugMode) info("Code = %s" % code) if args.debugMode: imwrite("frame_orig.jpg", draw_roi(frame_orig, args.ROI)) imwrite("frame_gray.jpg", frame_gray) imwrite("frame_thresh.jpg", frame_thresh) # Restore path: chdir(cwd)
def process_simple_maps(E, modelconfig, datakey, specifier): """ Main script for gathering seasonal map values and plotting them. """ config_file = E.get_configfile() experiment = 'SouthernHemisphere' areas = modelconfig.get(experiment, 'areas').split() seasons = modelconfig.get(experiment, 'seasons').split() plot_dir = E.get_plot_dir() verbosity = E.get_verbosity() work_dir = E.get_work_dir() # Get the observations obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey) obsfile = nc.Dataset(obs_loc, 'r') # A-laue_ax+ E.add_to_filelist(obs_loc) # A-laue_ax- scale_cloud = False # Ugly fix for scaling cloud ice/liquid water path values if (datakey == 'clivi' or datakey == 'clwvi'): scale_cloud = True scale = 1E3 scale_str = ' * 1E3' # Basic structure of the figure is same for all plots, # we clear it at intervals for area in areas: # Get area specific configuration and observations coords = E.get_area_coordinates(modelconfig, experiment, area) xticks = E.get_ticks(5, coords[2:]) yticks = E.get_ticks(3, coords[:2]) xlabels = E.get_ticks_labels(xticks, 'lons') ylabels = E.get_ticks_labels(yticks, 'lats') area_key = experiment + '_' + area obslats, obslons, obsdata = E.get_model_data(modelconfig, experiment, area, datakey, obsfile) if (scale_cloud): obsdata = obsdata * scale for model in models: # Get model and area specific values datafile = nc.Dataset(models[model], 'r') # A-laue_ax+ E.add_to_filelist(models[model]) # A-laue_ax- mlats, mlons, mdata = E.get_model_data(modelconfig, experiment, area, datakey, datafile) units = datafile.variables[datakey].units if (scale_cloud): mdata = mdata * scale if (units == 'kg m-2'): units = r'$\mu$m' else: units += scale_str for season in seasons: # Extract seasonal specific values from area values data = E.extract_seasonal_mean_values(modelconfig, mdata, experiment, season) obs = E.extract_seasonal_mean_values(modelconfig, obsdata, experiment, season) # Interpolate to obs grid data = interpolate_data_grid(data, mlats, mlons, obslats, obslons) # Get contour specific plotting limits lev, diff, cm_model, cm_diff = get_contour_config(modelconfig, config_file, area_key, datakey) # Plot the figures fig, axs = plt.subplots(2, 1, figsize=(15, 10)) fig.subplots_adjust(top=0.9) fig.subplots_adjust(hspace=0.5) # Same Basemap (coastlines) for all subplots # This is what usually takes time to process for ax in axs.flat: map_ax = Basemap(ax=ax, fix_aspect=False, llcrnrlat=coords[0], urcrnrlat=coords[1], llcrnrlon=coords[2], urcrnrlon=coords[3]) map_ax.drawcoastlines() # Plot the contours and colourmaps dash = ' - ' # Could also use u"\u2014" for long dash c0 = axs[0].contourf(obslons, obslats, data, levels=lev, cmap=cm_model, extend='both', latlon=True) cax0 = fig.add_axes([0.36, 0.525, 0.3, 0.01]) cbar0 = plt.colorbar(c0, cax=cax0, orientation='horizontal') cbar0.set_label(units) axs[0].set_title(model) c1 = axs[1].contourf(obslons, obslats, data - obs, levels=diff, cmap=cm_diff, extend='both', latlon=True) cax1 = fig.add_axes([0.36, 0.05, 0.3, 0.01]) cbar1 = plt.colorbar(c1, cax=cax1, orientation='horizontal') cbar1.set_label(units) axs[1].set_title(model + dash + obsmodel) # Labels for row in xrange(2): axs[row].xaxis.set_ticks(xticks) axs[row].set_xticklabels(xlabels) axs[row].yaxis.set_ticks(yticks) axs[row].set_yticklabels(ylabels) # Title and filename based on variables suptitle = E.get_title_basename(datakey) + " for months: " + season plt.suptitle(suptitle, fontsize=20) variable = datakey specifier = specifier + "-" + season diag_name = E.get_diag_script_name() output_file = E.get_plot_output_filename(diag_name=diag_name, variable=variable, specifier=specifier, model=model) # Save the image and let the user know what happened output_dir = os.path.join(plot_dir, diag_name) E.ensure_directory(output_dir) plt.savefig(os.path.join(output_dir, output_file)) plt.clf() info("", verbosity, 1) info("Created image: ", verbosity, 1) info(output_file, verbosity, 1) datafile.close() obsfile.close()
def cmor_reformat(currProject, project_info, variable, model): model_name = currProject.get_model_name(model) project_name = currProject.get_project_name(model) project_basename = currProject.get_project_basename() project_info['RUNTIME']['model'] = model_name project_info['RUNTIME']['project'] = project_name project_info['RUNTIME']['project_basename'] = project_basename verbosity = project_info["GLOBAL"]["verbosity"] exit_on_warning = project_info['GLOBAL'].get('exit_on_warning', False) # Variable put in environment to be used for the (optional) # wildcard syntax in the model path, ".../${VARIABLE}/..." # in the namelist os.environ['__ESMValTool_base_var'] = variable.var # Build input and output file names indir, infile = currProject.get_cf_infile(project_info, model, variable.fld, variable.var, variable.mip, variable.exp) fullpath = currProject.get_cf_fullpath(project_info, model, variable.fld, variable.var, variable.mip, variable.exp) # print "indir = %s" % indir # print "infile = %s" % infile # print "fullpath = %s" % fullpath if (not os.path.isdir(os.path.dirname(fullpath))): os.makedirs(os.path.dirname(fullpath)) # Area file name for ocean grids areafile_path = currProject.get_cf_areafile(project_info, model) # Land-mask file name for land variables lmaskfile_path = currProject.get_cf_lmaskfile(project_info, model) omaskfile_path = currProject.get_cf_omaskfile(project_info, model) # Porosity file name for land variables porofile_path = currProject.get_cf_porofile(project_info, model) # Additional grid file names for ocean grids, if available (ECEARTH) hgridfile_path = False zgridfile_path = False lsmfile_path = False if hasattr(currProject, "get_cf_hgridfile"): hgridfile_path = currProject.get_cf_hgridfile(project_info, model) if hasattr(currProject, "get_cf_zgridfile"): zgridfile_path = currProject.get_cf_zgridfile(project_info, model) if hasattr(currProject, "get_cf_lsmfile"): lsmfile_path = \ currProject.get_cf_lsmfile(project_info, model, variable.fld) # General fx file name entry fx_file_path = False if hasattr(currProject, "get_cf_fx_file"): fx_file_path = currProject.get_cf_fx_file(project_info, model) project, name, ensemble, start_year, end_year, dir\ = currProject.get_cf_sections(model) info("project is " + project, verbosity, required_verbosity=4) info("ensemble is " + ensemble, verbosity, required_verbosity=4) info("dir is " + dir, verbosity, required_verbosity=4) # Check if the current project has a specific reformat routine, # otherwise use default if (os.path.isdir("reformat_scripts/" + project)): which_reformat = project else: which_reformat = 'default' reformat_script = os.path.join("reformat_scripts", which_reformat, "reformat_" + which_reformat + "_main.ncl") # Set enviroment variables project_info['TEMPORARY'] = {} project_info['TEMPORARY']['indir_path'] = indir project_info['TEMPORARY']['outfile_fullpath'] = fullpath project_info['TEMPORARY']['infile_path'] = os.path.join(indir, infile) project_info['TEMPORARY']['areafile_path'] = areafile_path project_info['TEMPORARY']['lmaskfile_path'] = lmaskfile_path project_info['TEMPORARY']['omaskfile_path'] = omaskfile_path project_info['TEMPORARY']['porofile_path'] = porofile_path project_info['TEMPORARY']['start_year'] = start_year project_info['TEMPORARY']['end_year'] = end_year project_info['TEMPORARY']['ensemble'] = ensemble project_info['TEMPORARY']['variable'] = variable.var project_info['TEMPORARY']['field'] = variable.fld # FX file path if fx_file_path: project_info['TEMPORARY']['fx_file_path'] = fx_file_path # Special cases if 'realm' in currProject.get_model_sections(model): project_info['TEMPORARY']['realm'] = \ currProject.get_model_sections(model)["realm"] if 'shift_year' in currProject.get_model_sections(model): project_info['TEMPORARY']['shift_year'] = \ currProject.get_model_sections(model)["shift_year"] if 'case_name' in currProject.get_model_sections(model): project_info['TEMPORARY']['case_name'] = \ currProject.get_model_sections(model)["case_name"] if hgridfile_path and zgridfile_path: project_info['TEMPORARY']['hgridfile_path'] = hgridfile_path project_info['TEMPORARY']['zgridfile_path'] = zgridfile_path if lsmfile_path: project_info['TEMPORARY']['lsmfile_path'] = lsmfile_path # Execute the ncl reformat script if ((not os.path.isfile(project_info['TEMPORARY']['outfile_fullpath'])) or project_info['GLOBAL']['force_processing']): info(" Calling " + reformat_script + " to check/reformat model data", verbosity, required_verbosity=1) projects.run_executable(reformat_script, project_info, verbosity, exit_on_warning) if 'NO_REFORMAT' in reformat_script: pass else: if (not os.path.isfile(project_info['TEMPORARY']['outfile_fullpath'])): raise exceptions.IOError( 2, "Expected reformatted file isn't available: ", project_info['TEMPORARY']['outfile_fullpath']) del (project_info['TEMPORARY'])