def execute_netcdf_task(dataset, task): global log, grid_ids_, depth_axes_, time_axes_ task.status = cmor_task.status_cmorizing grid_axes = [] if not hasattr(task, "grid_id") else [getattr(task, "grid_id")] z_axes = getattr(task, "z_axes", []) t_axes = [] if not hasattr(task, "time_axis") else [getattr(task, "time_axis")] axes = grid_axes + z_axes + t_axes for type_axis in type_axes_: if type_axis in getattr(task.target, cmor_target.dims_key): axes.append(type_axes_[type_axis]) varid = create_cmor_variable(task, dataset, axes) ncvar = dataset.variables[task.source.variable()] missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan)) if not any(grid_axes): # Fix for global averages vals = numpy.copy(ncvar[:, :, :]) vals[vals == missval] = numpy.nan ncvar = numpy.mean(vals[:, :, :], axis=(1, 2)) factor = get_conversion_factor(getattr(task, cmor_task.conversion_key, None)) log.info("CMORizing variable %s in table %s form %s in " "file %s..." % (task.target.variable, task.target.table, task.source.variable(), getattr(task, cmor_task.output_path_key))) cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, missval=getattr(task.target, cmor_target.missval_key, missval)) closed_file = cmor.close(varid, file_name=True) log.info("CMOR closed file %s" % closed_file) task.status = cmor_task.status_cmorized
def execute_netcdf_task(dataset, task): global log task.status = cmor_task.status_cmorizing grid_axes = [] if not hasattr(task, "grid_id") else [ getattr(task, "grid_id") ] z_axes = getattr(task, "z_axes", []) t_axes = [] if not hasattr(task, "time_axis") else [ getattr(task, "time_axis") ] type_axes = [ getattr(task, dim + "_axis") for dim in type_axes_.get(task.target.table, {}).keys() if hasattr(task, dim + "_axis") ] # TODO: Read axes order from netcdf file! axes = grid_axes + z_axes + type_axes + t_axes srcvar = task.source.variable() if task.target.variable == "basin": ncvar, dimensions, missval = create_basins(task.target, dataset) else: ncvar = dataset.variables[srcvar] dimensions = ncvar.dimensions missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan)) varid = create_cmor_variable(task, srcvar, ncvar, axes) time_dim, index, time_sel = -1, 0, None for d in dimensions: if d.startswith("time"): time_dim = index break index += 1 time_sel = None if len(t_axes) > 0 > time_dim: for d in dataset.dimensions: if d.startswith("time"): time_sel = range(len(d)) # ensure copying of constant fields break if len(grid_axes) == 0: # Fix for global averages/sums vals = numpy.ma.masked_equal(ncvar[...], missval) ncvar = numpy.mean(vals, axis=(1, 2)) factor, term = get_conversion_constants( getattr(task, cmor_task.conversion_key, None)) log.info('Cmorizing variable {:20} in table {:7} in file {}'.format( srcvar, task.target.table, getattr(task, cmor_task.output_path_key))) mask = getattr(task.target, cmor_target.mask_key, None) if mask is not None: mask = nemo_masks_.get(mask, None) cmor_utils.netcdf2cmor(varid, ncvar, time_dim, factor, term, missval=getattr(task.target, cmor_target.missval_key, missval), time_selection=time_sel, mask=mask) cmor.close(varid, file_name=True) task.status = cmor_task.status_cmorized
def execute_netcdf_task(task, dataset, tableid): global log, grid_ids_, depth_axes_, time_axes_ task.status = cmor_task.status_cmorizing dims = task.target.dims globvar = (task.source.grid() == cmor_source.nemo_grid[ cmor_source.nemo_grid.scalar]) if (globvar): axes = [] else: if (not task.source.grid() in grid_ids_): log.error( "Grid axis for %s has not been created; skipping variable." % task.source.grid()) return axes = [grid_ids_[task.source.grid()]] if ((globvar and dims == 1) or (not globvar and dims == 3)): grid_index = cmor_source.nemo_grid.index(task.source.grid()) if (not grid_index in cmor_source.nemo_depth_axes): log.error( "Depth axis for grid %s has not been created; skipping variable." % task.source.grid()) return zaxid = depth_axes_[tableid][grid_index] axes.append(zaxid) axes.append(time_axes_[tableid]) for type in type_axes_: if type in getattr(task.target, cmor_target.dims_key): axes.append(type_axes_[type]) varid = create_cmor_variable(task, dataset, axes) ncvar = dataset.variables[task.source.var()] missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan)) if (globvar): # Fix for global averages vals = numpy.copy(ncvar[:, :, :]) vals[vals == missval] = numpy.nan ncvar = numpy.nanmean(vals[:, :, :], axis=(1, 2)) factor = get_conversion_factor( getattr(task, cmor_task.conversion_key, None)) cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, missval=getattr(task.target, cmor_target.missval_key, missval)) cmor.close(varid) task.status = cmor_task.status_cmorized
def execute_netcdf_task(task,tableid): """excute task for netcdf data Args: task (cmor.task): task which will be handled tableid (cmor.table): table which will have this task Returns: boolean: success of writing a variable """ global log,dim_ids_,depth_axes_,time_axes_,areacella_ interpolate_to_pressure=False task.status = cmor_task.status_cmorizing filepath = getattr(task, cmor_task.output_path_key, None) if not filepath: log.error("ERR -15: Could not find file containing data for variable %s in table %s" % (task.target.variable,task.target.table)) task.set_failed() return store_var = getattr(task, "store_with", None) if( task.target.dims >= 3): if ('lon' in dim_ids_ and 'lat' in dim_ids_): axes = [dim_ids_['lat'],dim_ids_['lon']] else: dim_ids_['lat']=create_lat() dim_ids_['lon']=create_lon() axes=[dim_ids_['lat'],dim_ids_['lon']] if hasattr(task, "z_axis_id"): axes.append(getattr(task, "z_axis_id")) checkaxes=getattr(task.target, cmor_target.dims_key).split() if 'plev19' in checkaxes: interpolate_to_pressure=True elif'plev39' in checkaxes: interpolate_to_pressure=True # make explicit if just to check that all axes are there elif 'alevel' in checkaxes: interpolate_to_pressure=False elif 'alevhalf' in checkaxes: interpolate_to_pressure=False else: log.error('ERR -16: unknown dimension in z_axis_id') else: log.error('ERR -17: No z_axis_id found.') elif ( task.target.dims == 2): if task.target.table=='AERmonZ': ''' 2D Zonal lat+lev ''' #cmor.load_table(table_root_ + "_coordinate.json") if 'lat' in dim_ids_: axes=[dim_ids_['lat']] else: dim_ids_['lat']=create_lat() axes=[dim_ids_['lat']] # zonal variables... #needs lat only, no grid.... if hasattr(task, "z_axis_id"): axes.append(getattr(task, "z_axis_id")) if 'plev19' in getattr(task.target, cmor_target.dims_key).split(): interpolate_to_pressure=True elif'plev39' in getattr(task.target, cmor_target.dims_key).split(): interpolate_to_pressure=True elif not hasattr(task, "z_axis_id"): ''' 2D variables lon+lat ''' if not ('lon' in dim_ids_ and 'lat' in dim_ids_): dim_ids_['lat']=create_lat() dim_ids_['lon']=create_lon() axes=[dim_ids_['lat'],dim_ids_['lon']] else: axes = [dim_ids_['lat'],dim_ids_['lon']] else: log.error('ERR -18: unsupported 2D dimensions %s'%task.target.dims) exit('Exiting!') elif task.target.dims==0: axes=[] else: log.error('ERR -19: unsupported dimensions %s for variable'%(task.target.dims,task.target.variable)) exit() time_id = getattr(task, "time_axis", 0) if time_id != 0: axes.append(time_id) for key in type_axes_: if key[0]==task.target.table and key[1] in getattr(task.target, cmor_target.dims_key): axes.append(type_axes_[key]) try: dataset = netCDF4.Dataset(filepath, 'r') except Exception as e: log.error("ERR -20: Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % ( filepath, task.target.variable, task.target.table, e.message)) return varid = create_cmor_variable(task,dataset,axes) if varid <0: return False ## for pressure level variables we need to do interpolation, for which we need ## pyngl module if interpolate_to_pressure: psdata=get_ps_var(getattr(getattr(task,'ps_task',None),cmor_task.output_path_key,None)) pressure_levels=getattr(task,'pressure_levels') ncvar=interpolate_plev(pressure_levels,dataset,psdata,task.source.variable()) else: ncvar = dataset.variables[task.source.variable()] # handle zonal vars if task.target.table=='AERmonZ': # assumption: data is shape [time,lat,lon] (roll longitude dimensio vals=numpy.copy(ncvar[:]) # zonal mean so mean over longitudes vals=numpy.mean(vals,axis=-1) # change shape, swap lat<->lev vals=numpy.swapaxes(vals,1,2) missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan)) ncvar=vals.copy() #handle global means elif task.target.dims==0: # global means missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan)) vals=numpy.copy(ncvar[:]) vals=numpy.mean(vals,axis=(1)) # calculate area-weighted mean vals=numpy.sum((vals*areacella_[numpy.newaxis,:,:]),axis=(1,2))/numpy.sum(areacella_) #handle normal case else:# assumption: data is shape [time,lat,lon] (we need to roll longitude dimension so that # the data corresponds to the dimension definition of tables (from [-180 , 180] to [0,360] deg).) # so by half the longitude dimension missval = getattr(ncvar,"missing_value",getattr(ncvar,"_FillValue",numpy.nan)) vals=numpy.copy(ncvar[:]) dims = numpy.shape(vals) nroll=dims[-1]/2 ncvar = numpy.roll(vals,nroll,len(dims)-1) vals=numpy.copy(ncvar[:,:,:]) # Default values factor = 1.0 term=0.0 timdim=0 # 3D variables need the surface pressure for calculating the pressure at model levels if store_var: #get the ps-data associated with this data psdata=get_ps_var(getattr(getattr(task,'ps_task',None),cmor_task.output_path_key,None)) # roll psdata like the original psdata=numpy.roll(psdata[:],nroll,len(numpy.shape(psdata[:]))-1) cmor_utils.netcdf2cmor(varid, ncvar, timdim, factor, term, store_var, psdata, swaplatlon=False, fliplat=True, mask=None,missval=missval) else: cmor_utils.netcdf2cmor(varid, ncvar, timdim, factor, term, store_var, None, swaplatlon=False, fliplat=True, mask=None,missval=missval) cmor.close(varid) if store_var: cmor.close(store_var) task.status = cmor_task.status_cmorized
def execute_netcdf_task(task, tableid): global log, grid_ids_, depth_axes_, time_axes_ #if not tableid in depth_axes_: # depth_axes_[tableid]=create_hybrid_level_axis(task) task.status = cmor_task.status_cmorizing filepath = getattr(task, cmor_task.output_path_key, None) if not filepath: log.error( "Could not find file containing data for variable %s in table %s" % (task.target.variable, task.target.table)) return store_var = getattr(task, "store_with", None) dims = task.target.dims if (not task.source.grid() in grid_ids_): log.error("Grid axis for %s has not been created; skipping variable." % task.source.grid()) return axes = [grid_ids_[task.source.grid()]] if (dims == 3): grid_index = cmor_source.tm5_grid.index(task.source.grid()) print cmor_source.tm5_depth_axes, grid_index, tableid, depth_axes_ if (not grid_index in cmor_source.tm5_depth_axes): log.error( "Depth axis for grid %s has not been created; skipping variable." % task.source.grid()) return #zaxid = depth_axes_[tableid][grid_index] if hasattr(task, "z_axis_id"): axes.append(getattr(task, "z_axis_id")) #axes.append(zaxid) time_id = getattr(task, "time_axis", 0) if time_id != 0: axes.append(time_id) #axes.append(time_axes_[tableid]) for type in type_axes_: if type in getattr(task.target, cmor_target.dims_key): axes.append(type_axes_[type]) try: dataset = netCDF4.Dataset(filepath, 'r') except Exception as e: log.error( "Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % (filepath, task.target.variable, task.target.table, e.message)) return varid = create_cmor_variable(task, dataset, axes) ncvar = dataset.variables[task.source.var()] vals = numpy.copy(ncvar[:]) dims = numpy.shape(vals) nroll = dims[-1] / 2 ncvar = numpy.roll(vals, nroll, len(dims) - 1) missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan)) vals = numpy.copy(ncvar[:, :, :]) #factor 1. keep it for time being factor = 1.0 #get_conversion_factor(getattr(task,cmor_task.conversion_key,None)) cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, store_var, get_ps_var( getattr(ps_tasks[task.target.frequency], cmor_task.output_path_key, None)), swaplatlon=False, fliplat=False, mask=None, missval=missval) cmor.close(varid) if store_var: cmor.close(store_var) task.status = cmor_task.status_cmorized