def map_to_region(fin, fout, xfirst, xinc, xsize, yfirst, yinc, ysize):
gfname = tempfile.mktemp()
with open(gfname, 'w') as o:
o.write(GRID_DESC.format(xsize, ysize, xfirst, xinc, yfirst, yinc))
c = cdo.Cdo()
c.remapbil(gfname, input=fin, output=fout)
os.unlink(gfname)
def main(args):
nc_paths = get_paths(args.sim_dir)
c = cdo.Cdo()
for date_str, nc_list in nc_paths.items():
print("%s IN:" % date_str)
start_list, stop_list = [], []
for nc in nc_list:
start, stop = get_dt_range(c, nc)
print(" %s (%s to %s)" % (basename(nc), start, stop))
start_list.append(start)
stop_list.append(stop)
start, stop = min(start_list), max(stop_list)
out_subd = join(args.out_dir, date_str)
print("%s OUT:" % date_str)
print(" %s (%s to %s)" % (date_str, start, stop))
sys.stdout.flush()
try:
os.makedirs(out_subd)
except FileExistsError:
pass
pairs = utils.get_images(args.radar_dir, after=start, before=stop)
for dt, src in pairs:
out_name = "%s.png" % strftime(dt, utils.FMT)
dst = join(out_subd, out_name)
with open(src, "rb") as fi, open(dst, "wb") as fo:
fo.write(fi.read())
print()
def zmnam_preproc(ifile):
"""Preprocessing of the input dataset files."""
cdo = cd.Cdo()
# Delete leap day, if any.
full_da_nl = cdo.delete('month=2,day=29', input=ifile)
# Fill missing values with bilinear method.
full_da_nl_nn = cdo.fillmiss(input=full_da_nl)
# Detrend with memory-efficient method.
full_da_nl_a, full_da_nl_b = cdo.trend(input=full_da_nl_nn)
full_da_nl_detr = cdo.subtrend(input=full_da_nl_nn + ' ' + full_da_nl_a +
' ' + full_da_nl_b)
# Compute anomalies from the daily/monthly means.
gh_da_dm = cdo.ydaymean(input=full_da_nl_detr)
gh_da_an = cdo.sub(input=full_da_nl_detr + ' ' + gh_da_dm)
gh_da_an_zm = cdo.zonmean(input=gh_da_an)
# Compute monthly mean anomalies.
gh_mo = cdo.monmean(input=full_da_nl_detr)
gh_mo_mm = cdo.ymonmean(input=gh_mo)
gh_mo_an = cdo.sub(input=gh_mo + ' ' + gh_mo_mm)
return (gh_da_an_zm, gh_mo_an)
def build_cdo_object():
try:
return cdo.Cdo()
except Exception as e:
print("Python exception:", e)
print("Unable to use CDO, this won't work, sorry...")
print("You may consider running >> module load cdo << and trying to run again")
sys.exit(1)
def setUp(self):
"""Class constructor. Create a Dynamic_HD_Production_Run_Driver object."""
self.driver = Dynamic_HD_Production_Run_Drivers()
self.cdo_instance = cdo.Cdo()
self.temp_dir = os.path.join(data_dir, 'temp', 'temp_workdir')
try:
os.stat(self.temp_dir)
except:
os.mkdir(self.temp_dir)
def setUp(self):
"""Class constructor. Create a Dynamic_HD_Production_Run_Driver object."""
self.driver = Dynamic_Lake_Production_Run_Drivers()
self.cdo_instance = cdo.Cdo()
self.temp_dirs = [
os.path.join(data_dir, 'temp', 'temp_workdir_lake_deglac'),
os.path.join(data_dir, 'temp', 'temp_workdir_pd')
]
self.clean_up()
for temp_dir in self.temp_dirs:
try:
os.stat(temp_dir)
except:
os.mkdir(temp_dir)
def zmnam_preproc(ifile):
"""Preprocessing of the input dataset files."""
cdo = cd.Cdo()
# Delete leap day, if any.
full_da_nl = cdo.delete('month=2,day=29', input=ifile)
# Compute anomalies from the daily/monthly means.
gh_da_dm = cdo.ydaymean(input=full_da_nl)
gh_da_an = cdo.sub(input=full_da_nl + ' ' + gh_da_dm)
gh_da_an_zm = cdo.zonmean(input=gh_da_an)
gh_mo = cdo.monmean(input=full_da_nl)
gh_mo_mm = cdo.ymonmean(input=gh_mo)
gh_mo_an = cdo.sub(input=gh_mo + ' ' + gh_mo_mm)
return (gh_da_an_zm, gh_mo_an)
def setup(self):
"""Setup process."""
self.ifile = './tests/testdata/sresa1b_ncar_ccsm3-example.nc'
cdomethods = cdo.Cdo()
self.tmp_iter = cdomethods.mermean(input=self.ifile,
output=self.ifile[:-3]+'-mer.nc',
options="-O -f nc")
self.final = cdomethods.zonmean(input=self.tmp_iter,
output=self.ifile[:-3]+'-mer-zon.nc',
options="-O -f nc")
self.data_iter = ncd.Dataset(self.final)
tmp_at_one_go = "-mermean "+self.ifile
self.final_str = cdomethods.zonmean(input=tmp_at_one_go,
output=self.ifile[:-3] +
'-mer-zon-str.nc')
self.datas = ncd.Dataset(self.final_str)
self.sellev = cdomethods.sellevidx(12, input=self.ifile,
output=self.ifile[:-3]+'-lvl12.nc',
options="-O -f nc")
def find_sp_variable(task):
global ifs_gridpoint_file_, ifs_spectral_file_, surface_pressure, ln_surface_pressure
log.info("Looking for surface pressure variable in input files...")
command = cdo.Cdo()
code_string = command.showcode(input=ifs_spectral_file_)
codes = [cmor_source.grib_code(int(c)) for c in code_string[0].split()]
if surface_pressure in codes:
log.info("Found surface pressure in spectral file")
setattr(task, "path", ifs_spectral_file_)
task.source.grid_ = 1
return
if ln_surface_pressure in codes:
log.info("Found lnsp in spectral file")
setattr(task, "path", ifs_spectral_file_)
task.source = cmor_source.ifs_source.read("var134=exp(var152)")
return
log.info(
"Did not find sp or lnsp in spectral file: assuming gridpoint file contains sp"
)
setattr(task, cmor_task.output_path_key, ifs_gridpoint_file_)
task.source.grid_ = 0
def __init__(self, other_parent_dirs=None, **ComponentComputeArgs):
super(ComponentCouple, self).__init__(**ComponentComputeArgs)
try:
assert isinstance(self.calendar, CouplingEsmCalendar)
except AssertionError:
raise TypeError("You must supply a calendar with coupling functionality: CouplingEsmCalendar, and not %s" % type(self.calendar))
self.files["couple"] = FileDict()
self._register_directory("couple", use_name="generic")
self.couple_attrs = {}
for couple_type in self.COMPATIBLE_COUPLE_TYPES:
if couple_type in other_parent_dirs:
self.couple_attrs[couple_type] = {"parent_dir": other_parent_dirs[couple_type]}
else:
self.couple_attrs[couple_type] = {"parent_dir": self._parent_dir}
self._grid_string = None
self._cleanup_list = []
self._cdo_stderr = open(self.couple_dir+"/"+self.NAME+"Couple_cdo_log", "w")
self.CDO = cdo.Cdo()
self.NCO = nco.Nco()
def interpolate_icon_grib_to_latlon(path, grib_filename, latlon_filename,
model):
if model == 'icon-global-det':
targetgridfile = path['base'] + path[
'grid'] + 'target_grid_global_latlon_0.1.txt'
weightsfile = path['base'] + path[
'grid'] + 'weights_dis_{}_icosahedral_to_latlon_0.1.nc'.format(
model)
if model == 'icon-eu-eps':
targetgridfile = path['base'] + path[
'grid'] + 'target_grid_eu_latlon_0.2.txt'
weightsfile = path['base'] + path[
'grid'] + 'weights_dis_{}_icosahedral_to_latlon_0.2.nc'.format(
model)
cdo_module = cdo.Cdo()
cdo_module.remap(targetgridfile + ',' + weightsfile,
input=path['base'] + path['subdir'] + grib_filename,
output=path['base'] + path['subdir'] + latlon_filename,
options='-f nc')
return
def write_fields(self, filename, fields,griddescfile=None,fieldnames=None):
"""Write a field to a given target NetCDF4 file
Arguments:
filename: full path of the netcdf file to write to
fields: A list of Field (or Field subclass object); fields to write
griddescfile (optional): string; full path to the grid description metadata
to add to file written out. Nothing is added if this is
set to None
fieldnames: A list of strings; name of the output fields to create and write to
Returns:nothing
"""
nlat,nlong = fields[0].get_grid().get_grid_dimensions()
if fieldnames is None:
fieldnames = ['field_value']*len(fields)
print("Writing output to {0}".format(filename))
if griddescfile is not None:
output_filename=filename
filename=path.splitext(filename)[0] + '_temp' + path.splitext(filename)[1]
with netCDF4.Dataset(filename,mode='w',format='NETCDF4') as dataset:
dataset.createDimension("latitude",nlat)
dataset.createDimension("longitude",nlong)
for field,fieldname in zip(fields,fieldnames):
data_was_bool = False
if field.get_data().dtype == np.bool_:
field.set_data(field.get_data().astype(np.int32))
data_was_bool=True
field_values = dataset.createVariable(fieldname,field.get_data().dtype,
('latitude','longitude'))
field_values[:,:] = field.get_data()
if data_was_bool:
field.set_data(field.get_data().astype(np.bool_))
if griddescfile is not None:
cdo_instance = cdo.Cdo()
cdo_instance.setgrid(griddescfile,input=filename,output=output_filename)
os.remove(filename)
def create_time_axis(freq, path, name, has_bounds):
global log, start_date_, ref_date_
command = cdo.Cdo()
times = command.showtimestamp(input=path)[0].split()
datetimes = sorted(
set(
map(lambda s: datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S"),
times)))
if len(datetimes) == 0:
log.error(
"Empty time step list encountered at time axis creation for files %s"
% str(path))
return
refdate = cmor_utils.make_datetime(ref_date_)
if has_bounds:
n = len(datetimes)
bounds = numpy.empty([n, 2])
rounded_times = map(
lambda time:
(cmor_utils.get_rounded_time(freq, time) - refdate).total_seconds(
) / 3600., datetimes)
bounds[:, 0] = rounded_times[:]
bounds[0:n - 1, 1] = rounded_times[1:n]
bounds[n - 1,
1] = (cmor_utils.get_rounded_time(freq, datetimes[n - 1], 1) -
refdate).total_seconds() / 3600.
times[:] = bounds[:, 0] + (bounds[:, 1] - bounds[:, 0]) / 2
return cmor.axis(table_entry=str(name),
units="hours since " + str(ref_date_),
coord_vals=times,
cell_bounds=bounds)
times = numpy.array([(d - refdate).total_seconds() / 3600
for d in datetimes])
return cmor.axis(table_entry=str(name),
units="hours since " + str(ref_date_),
coord_vals=times)
def __init__(self, **EchamComputeArgs):
# FIXME: This all belongs in a generalized class, not here...
super(Echam6Couple_Ice, self).__init__(**EchamComputeArgs)
try:
assert isinstance(self.calendar, CouplingEsmCalendar)
except AssertionError:
raise TypeError(
"You must supply a calendar with coupling functionality: CouplingEsmCalendar, and not %s"
% type(self.calendar))
self.files["couple"] = FileDict()
self._register_directory("couple", use_Name="generic")
self.__cleanup_list = []
self._cdo_stderr = open(self.couple_dir + "/EchamCouple_Ice_cdo_log",
"w")
self.CDO = cdo.Cdo(logging=True, logFile=self._cdo_stderr)
# Get relevant environmental variables
self.ECHAM_TO_ISM_multiyear_mean = load_environmental_variable_1_0(
"ECHAM_TO_ISM_multiyear_mean")
self.ECHAM_TO_ISM_time_mean = load_environmental_variable_1_0(
"ECHAM_TO_ISM_time_mean")
def advanced_basin_evaluation_driver(
input_minima_file,
input_minima_fieldname,
input_raw_orography_file,
input_raw_orography_fieldname,
input_corrected_orography_file,
input_corrected_orography_fieldname,
input_cell_areas_file,
input_cell_areas_fieldname,
input_prior_fine_rdirs_file,
input_prior_fine_rdirs_fieldname,
input_prior_fine_catchments_file,
input_prior_fine_catchments_fieldname,
input_coarse_catchment_nums_file,
input_coarse_catchment_nums_fieldname,
input_coarse_rdirs_file,
input_coarse_rdirs_fieldname,
combined_output_filename,
output_filepath,
output_filelabel,
output_basin_catchment_nums_filepath=None):
input_minima = iodriver.advanced_field_loader(
input_minima_file,
field_type='Generic',
fieldname=input_minima_fieldname)
input_raw_orography = iodriver.advanced_field_loader(
input_raw_orography_file,
field_type='Orography',
fieldname=input_raw_orography_fieldname)
input_corrected_orography = iodriver.advanced_field_loader(
input_corrected_orography_file,
field_type='Orography',
fieldname=input_corrected_orography_fieldname)
input_cell_areas = iodriver.advanced_field_loader(
input_cell_areas_file,
field_type='Generic',
fieldname=input_cell_areas_fieldname)
input_prior_fine_rdirs = iodriver.advanced_field_loader(
input_prior_fine_rdirs_file,
field_type='RiverDirections',
fieldname=input_prior_fine_rdirs_fieldname)
input_prior_fine_catchments = iodriver.advanced_field_loader(
input_prior_fine_catchments_file,
field_type='Generic',
fieldname=input_prior_fine_catchments_fieldname)
input_coarse_catchment_nums = iodriver.advanced_field_loader(
input_coarse_catchment_nums_file,
field_type='Generic',
fieldname=input_coarse_catchment_nums_fieldname)
input_coarse_rdirs = iodriver.advanced_field_loader(
input_coarse_rdirs_file,
field_type='Generic',
fieldname=input_coarse_rdirs_fieldname)
fine_grid = input_raw_orography.get_grid()
fine_shape = input_raw_orography.get_data().shape
connection_volume_thresholds = field.Field(
np.zeros(fine_shape, dtype=np.float64, order='C'), fine_grid)
flood_volume_thresholds = field.Field(
np.zeros(fine_shape, dtype=np.float64, order='C'), fine_grid)
flood_next_cell_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_next_cell_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_next_cell_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_next_cell_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_force_merge_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_force_merge_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_force_merge_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_force_merge_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_redirect_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_redirect_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_redirect_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_redirect_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_flood_redirect_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_flood_redirect_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_connect_redirect_lat_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_connect_redirect_lon_index = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
flood_local_redirect = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
connect_local_redirect = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_flood_local_redirect = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
additional_connect_local_redirect = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
merge_points = field.Field(np.zeros(fine_shape, dtype=np.int32, order='C'),
fine_grid)
if output_basin_catchment_nums_filepath is not None:
basin_catchment_numbers = field.Field(
np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid)
else:
basin_catchment_numbers = None
evaluate_basins_wrapper.evaluate_basins(
minima_in_int=np.ascontiguousarray(input_minima.get_data(),
dtype=np.int32),
raw_orography_in=np.ascontiguousarray(input_raw_orography.get_data(),
dtype=np.float64),
corrected_orography_in=np.ascontiguousarray(
input_corrected_orography.get_data(), dtype=np.float64),
cell_areas_in=np.ascontiguousarray(input_cell_areas.get_data(),
dtype=np.float64),
connection_volume_thresholds_in=connection_volume_thresholds.get_data(
),
flood_volume_thresholds_in=flood_volume_thresholds.get_data(),
prior_fine_rdirs_in=np.ascontiguousarray(
input_prior_fine_rdirs.get_data(), dtype=np.float64),
prior_coarse_rdirs_in=np.ascontiguousarray(
input_coarse_rdirs.get_data(), dtype=np.float64),
prior_fine_catchments_in=np.ascontiguousarray(
input_prior_fine_catchments.get_data(), dtype=np.int32),
coarse_catchment_nums_in=np.ascontiguousarray(
input_coarse_catchment_nums.get_data(), dtype=np.int32),
flood_next_cell_lat_index_in=flood_next_cell_lat_index.get_data(),
flood_next_cell_lon_index_in=flood_next_cell_lon_index.get_data(),
connect_next_cell_lat_index_in=connect_next_cell_lat_index.get_data(),
connect_next_cell_lon_index_in=connect_next_cell_lon_index.get_data(),
flood_force_merge_lat_index_in=flood_force_merge_lat_index.get_data(),
flood_force_merge_lon_index_in=flood_force_merge_lon_index.get_data(),
connect_force_merge_lat_index_in=connect_force_merge_lat_index.
get_data(),
connect_force_merge_lon_index_in=connect_force_merge_lon_index.
get_data(),
flood_redirect_lat_index_in=flood_redirect_lat_index.get_data(),
flood_redirect_lon_index_in=flood_redirect_lon_index.get_data(),
connect_redirect_lat_index_in=connect_redirect_lat_index.get_data(),
connect_redirect_lon_index_in=connect_redirect_lon_index.get_data(),
additional_flood_redirect_lat_index_in=
additional_flood_redirect_lat_index.get_data(),
additional_flood_redirect_lon_index_in=
additional_flood_redirect_lon_index.get_data(),
additional_connect_redirect_lat_index_in=
additional_connect_redirect_lat_index.get_data(),
additional_connect_redirect_lon_index_in=
additional_connect_redirect_lon_index.get_data(),
flood_local_redirect_out_int=flood_local_redirect.get_data(),
connect_local_redirect_out_int=connect_local_redirect.get_data(),
additional_flood_local_redirect_out_int=additional_flood_local_redirect
.get_data(),
additional_connect_local_redirect_out_int=
additional_connect_local_redirect.get_data(),
merge_points_out_int=merge_points.get_data(),
basin_catchment_numbers_in=basin_catchment_numbers.get_data())
connection_volume_thresholds_filename = path.join(
output_filepath, "connect_vts_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connection_volume_thresholds_filename,
connection_volume_thresholds,
fieldname='connection_volume_thresholds')
flood_volume_thresholds_filename = path.join(
output_filepath, "flood_vts_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_volume_thresholds_filename,
flood_volume_thresholds,
fieldname='flood_volume_thresholds')
flood_next_cell_lat_index_filename = path.join(
output_filepath, "flood_nci_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_next_cell_lat_index_filename,
flood_next_cell_lat_index,
fieldname='flood_next_cell_lat_index')
flood_next_cell_lon_index_filename = path.join(
output_filepath, "flood_nci_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_next_cell_lon_index_filename,
flood_next_cell_lon_index,
fieldname='flood_next_cell_lon_index')
connect_next_cell_lat_index_filename = path.join(
output_filepath, "connect_nci_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_next_cell_lat_index_filename,
connect_next_cell_lat_index,
fieldname='connect_next_cell_lat_index')
connect_next_cell_lon_index_filename = path.join(
output_filepath, "connect_nci_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_next_cell_lon_index_filename,
connect_next_cell_lon_index,
fieldname='connect_next_cell_lon_index')
flood_force_merge_lat_index_filename = path.join(
output_filepath, "flood_fmi_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_force_merge_lat_index_filename,
flood_force_merge_lat_index,
fieldname='flood_force_merge_lat_index')
flood_force_merge_lon_index_filename = path.join(
output_filepath, "flood_fmi_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_force_merge_lon_index_filename,
flood_force_merge_lon_index,
fieldname='flood_force_merge_lon_index')
connect_force_merge_lat_index_filename = path.join(
output_filepath, "connect_fmi_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_force_merge_lat_index_filename,
connect_force_merge_lat_index,
fieldname='connect_force_merge_lat_index')
connect_force_merge_lon_index_filename = path.join(
output_filepath, "connect_fmi_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_force_merge_lon_index_filename,
connect_force_merge_lon_index,
fieldname='connect_force_merge_lon_index')
flood_redirect_lat_index_filename = path.join(
output_filepath, "flood_ri_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_redirect_lat_index_filename,
flood_redirect_lat_index,
fieldname='flood_redirect_lat_index')
flood_redirect_lon_index_filename = path.join(
output_filepath, "flood_ri_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_redirect_lon_index_filename,
flood_redirect_lon_index,
fieldname='flood_redirect_lon_index')
connect_redirect_lat_index_filename = path.join(
output_filepath, "connect_ri_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_redirect_lat_index_filename,
connect_redirect_lat_index,
fieldname='connect_redirect_lat_index')
connect_redirect_lon_index_filename = path.join(
output_filepath, "connect_ri_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_redirect_lon_index_filename,
connect_redirect_lon_index,
fieldname='connect_redirect_lon_index')
additional_flood_redirect_lat_index_filename = path.join(
output_filepath, "additional_flood_ri_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(
additional_flood_redirect_lat_index_filename,
additional_flood_redirect_lat_index,
fieldname='additional_flood_redirect_lat_index')
additional_flood_redirect_lon_index_filename = path.join(
output_filepath, "additional_flood_ri_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(
additional_flood_redirect_lon_index_filename,
additional_flood_redirect_lon_index,
fieldname='additional_flood_redirect_lon_index')
additional_connect_redirect_lat_index_filename = path.join(
output_filepath,
"additional_connect_ri_lat_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(
additional_connect_redirect_lat_index_filename,
additional_connect_redirect_lat_index,
fieldname='additional_connect_redirect_lat_index')
additional_connect_redirect_lon_index_filename = path.join(
output_filepath,
"additional_connect_ri_lon_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(
additional_connect_redirect_lon_index_filename,
additional_connect_redirect_lon_index,
fieldname='additional_connect_redirect_lon_index')
flood_local_redirect_filename = path.join(
output_filepath, "flood_local_r_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(flood_local_redirect_filename,
flood_local_redirect,
fieldname='flood_local_redirect')
connect_local_redirect_filename = path.join(
output_filepath, "connect_local_r_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(connect_local_redirect_filename,
connect_local_redirect,
fieldname='connect_local_redirect')
additional_flood_local_redirect_filename = path.join(
output_filepath,
"additional_flood_local_r_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(additional_flood_local_redirect_filename,
additional_flood_local_redirect,
fieldname='additional_flood_local_redirect')
additional_connect_local_redirect_filename = path.join(
output_filepath,
"additional_connect_local_r_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(
additional_connect_local_redirect_filename,
additional_connect_local_redirect,
fieldname='additional_connect_local_redirect')
merge_points_filename = path.join(
output_filepath, "merge_points_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(merge_points_filename,
merge_points,
fieldname='merge_points')
lake_centers_filename = path.join(
output_filepath, "lake_centers_" + output_filelabel + ".nc")
iodriver.advanced_field_writer(lake_centers_filename,
input_minima,
fieldname="lake_centers")
individual_field_filenames = [
connection_volume_thresholds_filename,
flood_volume_thresholds_filename, flood_next_cell_lat_index_filename,
flood_next_cell_lon_index_filename,
connect_next_cell_lat_index_filename,
connect_next_cell_lon_index_filename,
flood_force_merge_lat_index_filename,
flood_force_merge_lon_index_filename,
connect_force_merge_lat_index_filename,
connect_force_merge_lon_index_filename,
flood_redirect_lat_index_filename, flood_redirect_lon_index_filename,
connect_redirect_lat_index_filename,
connect_redirect_lon_index_filename,
additional_flood_redirect_lat_index_filename,
additional_flood_redirect_lon_index_filename,
additional_connect_redirect_lat_index_filename,
additional_connect_redirect_lon_index_filename,
flood_local_redirect_filename, connect_local_redirect_filename,
additional_flood_local_redirect_filename,
additional_connect_local_redirect_filename, merge_points_filename,
lake_centers_filename
]
cdo_inst = cdo.Cdo()
cdo_inst.merge(input=" ".join(individual_field_filenames),
output=combined_output_filename)
for individual_field_filename in individual_field_filenames:
os.remove(individual_field_filename)
if output_basin_catchment_nums_filepath is not None:
iodriver.advanced_field_writer(output_basin_catchment_nums_filepath,
basin_catchment_numbers,
fieldname="basin_catchment_numbers")
"""loading_tools
======================
The loading_tools module of cmipdata is a set of functions which use
the cdo python bindings and NetCDF4 to load data from input NetCDF
files listed in a cmipdata ensemble object into python numpy arrays.
Some processing can optionally be done during the loading, specifically
remapping, time-slicing, time-averaging and zonal-averaging.
.. moduleauthor:: Neil Swart <*****@*****.**>
"""
import cdo as cdo
cdo = cdo.Cdo() # recommended import
import os
import numpy as np
from netCDF4 import Dataset, num2date, date2num
import datetime
# clean out tmp to make space for CDO processing.
os.system('rm -rf /tmp/cdo*')
def loadvar(ifile, varname, cdostr=None, **kwargs):
"""
Load variables from a NetCDF file with optional pre-processing.
Load a CMIP5 netcdf variable "varname" from "ifile" and an optional
cdo string for preprocessing the data from the netCDF files.
Requires netCDF4, CDO and CDO python bindings.
Returns a masked array, var.
"""
def split_years(fname, target):
cdoapp = cdo.Cdo()
# cdoapp.debug = True
return cdoapp.splityear(input=fname, output=target)
def load_remap(ifile_fp, varname, file_type=None):
path, ifile = os.path.split(ifile_fp)
if not os.path.isfile('remapped-data/intlev_{}'.format(ifile)):
try:
import cdo
cdo = cdo.Cdo()
cdo.env = {'REMAP EXTRAPOLATE': 'off'}
except ImportError:
print 'CDO must be installed to preproces files.'
raise
print 'preprocessing', ifile
with open('parameters/default_depths') as f:
default_depths = f.read().replace('\n', ',')
try:
with Dataset('parameters/masks.nc', 'r') as mask:
with Dataset(ifile_fp, 'r') as data:
if np.all(np.array(mask['nav_lat'][:]) == np.array(data['nav_lat'][:])) and \
np.all(np.array(mask['nav_lon'][:]) == np.array(data['nav_lon'][:])):
cdo.yearmean(input=ifile_fp, output='temp_' + ifile)
else:
raise AttributeError
with Dataset('temp_' + ifile, 'a') as to_mask:
if file_type:
with open(
'parameters/scale_factors.json') as json_file:
var_defs = json.load(json_file)
to_mask.set_auto_maskandscale(True)
for var in to_mask.variables:
if var in var_defs[file_type].keys():
to_mask[var].setncattr(
'scale', var_defs[file_type][var][0])
to_mask[var].units = var_defs[file_type][
var][1]
to_mask[var].long_name = var_defs[
file_type][var][2]
for var in to_mask.variables:
if len(to_mask[var].shape) == 4:
to_mask[var][:] = np.ma.masked_invalid(
np.where(mask['tmask'][0, ], to_mask[var][:],
np.NaN))
cdo.intlevel(default_depths,
input='-remapdis,parameters/default_grid temp_' +
ifile,
output='remapped-data/intlev_' + ifile,
options='-L',
force=False)
os.remove('temp_' + ifile)
except (AttributeError, IndexError):
print 'Data from {} is not on the NAA grid, no land mask applied.'.format(
ifile)
cdo.intlevel(default_depths,
input='-remapdis,parameters/default_grid -yearmean ' +
ifile_fp,
output='remapped/intlev_' + ifile,
options='-L',
force=False)
print ifile, 'preprocessed.'
remapped_file = 'remapped-data/intlev_' + ifile
with Dataset(remapped_file, 'r') as nc:
ncvar = nc.variables[varname]
data = ncvar[:].squeeze()
try:
units = ncvar.units
except AttributeError:
print 'Units not given for {} in {}, leaving empty.'.format(
varname, ifile)
units = ''
dimensions = ncvar.dimensions
try:
for dimension in dimensions:
if 'depth' in dimension.lower():
depth = nc.variables[dimension][:]
break
else:
raise IndexError
except IndexError:
raise SystemExit('\nDepths not given for {} in {}.'.format(
varname, ifile))
try:
dates = num2date(nc['time_counter'][:],
units=nc['time_counter'].units,
calendar=nc['time_counter'].calendar)
years = [x.year for x in dates]
except IndexError:
years = [0]
lon = np.linspace(0, 360, 721)
lat = np.linspace(40, 90, 101)
return data, units, lon, lat, depth, dimensions, years
"""
Does remapping of the reanalyses and observations onto a 1x1 grid and computes
zonal means.
.. moduleauthor:: Neil Swart <*****@*****.**>
"""
import os
import glob
import cdo; cdo = cdo.Cdo()
def preprocess_observations(datapath='./'):
# where we are starting from
cwd = os.getcwd()
# move to where the data is
os.chdir(datapath)
# Get the reanalysis monthly mean files
rean = ['R1', 'R2', '20CR', 'ERA-Int', 'CFSR', 'MERRA']
var = ['slp', 'u10m', 'uflx']
files = []
for r in rean:
for v in var:
files.extend([r + '_' + v + '.mon.mean.nc'])
files = [f for f in files if not f.startswith('remap') and not
f.startswith('zonal-mean')]
# Add in CCMp and HadSLP2r files
files.extend(['CCMP_198701-201112.nc', 'HadSLP2r_slp.mon.mean.nc'])
files.extend(['20CR_ens_slp.mon.mean.nc', '20CR_ens_u10m.mon.mean.nc'])
for f in files:
def __init__(self, code=0):
self.operators = {}
self.app = cdo.Cdo()
if code > 0:
self.add_operator(cdo_command.select_code_operator, code)
def __init__(self, target_grid: str, outfile: str, sources: list, **kwargs):
"""
Contains universal methods
:param target_grid: grid_routines to be interpolated onto
:param sources: Information about which variables should be taken from which files using
which interpolation method onto which type of grid_routines and whether to interpolate them vertically
:param scrip_grid: Scrip file on rho grid_routines to interpolate with. Will be created if not provided
:param verbose: whether text should be printed as the program is running
Can have any of the following optional arguments:
theta_s - S-coordinate surface control parameter - default 7.0
theta_b - S-coordinate bottom control parameter - default 0.0
layers - Number of S-coordinate layers - default 32
hc - S-coordinate critical depth - default 150
tcline - S-coordinate surface/bottom layer width - default 150
sigma_type - default 3
zeta_source - dict with keys 'name' 'file'. Sets zeta to the first timestep of 'name' in 'file'.
file_type - output filetype - default nc4c
processes - number of processes cdo should use - default 8
scrip_grid - SCRIP version of target_grid - optional, will be created if not passed
verbose - whether to print runtime information - default false
time_underscored - whether 'time' variables should be replaced with '#_time' - default False
keep_weights - whether to keep calculated weights - default False
keep_z_clim - whether to keep zclim - default False
"""
# cdo
self.cdo = cdo.Cdo()
test_cdo(self.cdo)
# Sources
verify_sources(sources, kwargs.get('verbose', False))
self.sources, self.target_grid = sources, target_grid
# Vertical interpolation information
# Replace anything not passed in with default values
self.has_vertical = has_vertical(self.sources)
if self.has_vertical:
self.theta_s, self.theta_b, self.layers, self.hc, self.tcline, self.sigma_type = (
kwargs.get('theta_s', 7.0), kwargs.get('theta_b', 0.0),
kwargs.get("layers", 32), kwargs.get("hc", 150),
kwargs.get('tcline', 150), kwargs.get("sigma_type", 3)
)
self.sc = sigma_stretch_sc(self.layers, True)
self.cs = sigma_stretch_cs(self.theta_s, self.theta_b, self.sc, self.sigma_type)
# Get z_levels
with netCDF4.Dataset(target_grid, mode='r') as my_grid:
self.h = my_grid.variables['h'][:]
self.zeta = self.get_zeta(kwargs['zeta_source']) if 'zeta_source' in kwargs else np.zeros_like(self.h)
self.z_level_rho, self.z_level_u, self.z_level_v = get_z_levels(self.h, self.sc, self.cs, self.hc,
self.zeta)
# CDO options
self.file_type, self.processes = kwargs.get('file_type', 'nc4c'), kwargs.get('processes', 8)
# Other Options
self.verbose, self.time_underscored, self.keep_weights, self.keep_z_clim = (
kwargs.get('verbose', False), kwargs.get('time_underscored', False),
kwargs.get('keep_weights', False), kwargs.get('keep_z_clim', False)
)
self._adjustments = None
self.outfile = outfile
# Interpolator
self.scrip_grid = kwargs.get('scrip_grid', scrip_grid_from_nc(target_grid))
self.shift_pairs = ShiftPairCollection()
if self.verbose:
print("Finished setup")
if not os.path.exists(tmp):
os.makedirs(tmp)
# setup folder for working via sshfs_mistral in ~/mistral_work
elif my_system is 'local':
mistral_work = f'~/mistral_work/'
work = f'{mistral_work}{group}/{user}/'
cdo_mistral = True
if cdo_mistral:
tmp = os.path.expanduser('~/tmp')
else:
tmp = work + 'tmp'
if not os.path.exists(tmp):
os.makedirs(tmp)
# start
cdo = cdo.Cdo(tempdir=tmp)
cmip6_folder = mistral_work+'ik1017/CMIP6/data/CMIP6'
cmip5_folder = mistral_work+'kd0956/CMIP5/data/cmip5/output1'
GE_folder = mistral_work+'mh1007'
def remap_cdo(da):
if not isinstance(da, xr.core.dataset.Dataset):
da = da.to_dataset()
remap = cdo.remapbil(
'r360x180', input=da, returnXDataset=True, options='-P 8')
return remap
def concatenate(ncs_dir, output_nc):
ncs = sorted(get_files(ncs_dir, '.nc'))
c = cdo.Cdo()
c.cat(input=' '.join(ncs), output=output_nc, options='-r -f nc')
"""
Calculate the SAM index from a netCDF file containing sea-level pressure.
.. moduleauthor:: Neil Swart <*****@*****.**>
"""
import numpy as np
import pandas as pd
import cdo as cdo; cdo = cdo.Cdo() # recommended import
import os
os.system( 'rm -rf /tmp/cdo*') # clean out tmp to make space for CDO processing.
import cmipdata as cd
from netCDF4 import Dataset
def calc_sam(psl_file, varname, start_date='1800-01-01', end_date='2013-12-31'):
"""
Compute the SAM index as the pressure difference between 40 and 65S
Parameters:
-----------
psl_file : str
The name of the **zonal meaned** SLP netcdf file to compute the SAM
from. Can be a full path.
varname : str
The name of the Sea Level Pressure variable in psl_file.
Returns:
-------
sam : array
The calculated SAM index
"""
def read_time_stamps(path):
command = cdo.Cdo()
times = command.showtimestamp(input=path)[0].split()
return map(lambda s: datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S"), times)
#!/usr/bin/env python
# coding: utf-8
import matplotlib.pyplot as plt
from plot_tools import _decorate_x_axes_for_ymonmean
from simulation_tools import cosmos_standard_analysis
import cdo
from custom_io import get_remote_data
from scipy.io import netcdf
CDO = cdo.Cdo(cdfMod="scipy")
user = "******"
host = "stan1"
prepath = "pgierz@stan1:/ace/user/pgierz/cosmos-aso-wiso"
Eem130 = cosmos_standard_analysis(user, host,
prepath.split(":")[-1],
"Eem130-S2").ymonmean("SICOMO")
Eem125 = cosmos_standard_analysis(user, host,
prepath.split(":")[-1],
"Eem125-S2").ymonmean("SICOMO")
Eem120 = cosmos_standard_analysis(user, host,
prepath.split(":")[-1],
"Eem120-S2").ymonmean("SICOMO")
PI = cosmos_standard_analysis(user, host,
prepath.split(":")[-1],
"EXP003").ymonmean("SICOMO")
# Runs with future CO2
prepath = "pgierz@stan1:/ace/user/pgierz/cosmos-ao"
RCP6 = cosmos_standard_analysis(user, host,
def main(args):
'''
The main function
'''
climatedb.connect()
variable_name, data_source, frequency = get_args(args)
print(data_source, variable_name)
config = load_config()
if data_source in config:
organisation, author, year, article_url = get_citation_from_config(
config, data_source)
file_url = get_file_url_from_config(config, data_source, variable_name)
file_urls = [file_url]
else:
model, scenario = data_source.split('.')
search_url = config['esgf']['search']['url']
project = config['esgf']['search']['project']
variant = config['esgf']['search']['variant']
datasets = search_esgf(search_url, project, variable_name, model,
scenario, frequency, variant)
dataset_info = datasets[0]
organisation = dataset_info['institution_id'][0]
author, year, article_url = get_citation_from_url(
dataset_info['citation_url'][0])
thredds_url, thredds_id = get_thredds_url(dataset_info)
file_urls = get_file_urls_from_thredds(thredds_url, thredds_id)
file_paths = fetch_urls(file_urls)
if len(file_paths) == 0:
print('No datasets found', file=sys.stderr)
return 1
file_base, ext = file_paths[0].rsplit(os.path.extsep)
output_file_path = os.path.join(
DATASET_DIR, '%s-%s.%s' % (data_source, variable_name, ext))
os.path.exists(output_file_path) and os.remove(output_file_path)
if len(file_paths) == 1:
file_path = file_paths[0]
os.symlink(os.path.basename(file_path), output_file_path)
else:
if ext == 'nc':
cdo.Cdo().selall(input=sorted(file_paths),
output=output_file_path,
options='-f nc4')
baseline = (config['baseline'] == data_source)
create_data_source(data_source, organisation, author, year, article_url,
baseline)
return 0
import os
import tempfile
# External Imports
import cdo
import nco
import numpy as np
import xarray as xr
# This Library Imports:
from pyesm.core.component.component_coupling import ComponentCouple, cleanup_after_send, write_couple_vars_to_json
from pyesm.components.pism.pism_compute import PismCompute
from pyesm.core.helpers import load_environmental_variable_1_0, ComponentFile, FileDict
from pyesm.core.errors import CouplingError
CDO = cdo.Cdo()
NCO = nco.Nco()
class PismCouple(PismCompute, ComponentCouple):
""" Functionality to couple PISM with other models """
COMPATIBLE_COUPLE_TYPES = ["atmosphere", "solid_earth"]
def __init__(self, **PismComputeArgs):
super(PismCouple, self).__init__(**PismComputeArgs)
self.files['couple']['ice_grid'] = ComponentFile(
src=self.POOL_DIR + "grids/" + self.Domain + "/pismr_" + self.Domain + "_" + self.LateralResolution + ".griddes",
dest=self.couple_dir)
################################################################################
################################################################################
===============
This module contains several functions associated with traversing
directories and gathering information from files.
.. moduleauthor:: David Fallis
"""
import os
from netCDF4 import Dataset, num2date, date2num
import datetime
import itertools
import tarfile
import cmipdata as cd
import validate
import cdo
cdo = cdo.Cdo()
MEANDIR = None
def _variable_dictionary(plots):
""" Creates a dictionary with the variable names as keys
mapped to empty lists
Parameters
----------
plots : list of dictionaries with 'variable' key
Returns
-------
dictionary
def load(ifile_fp, varname, file_type=None):
path, ifile = os.path.split(ifile_fp)
if not os.path.isfile('remapped-data/yearmean_{}'.format(ifile)):
try:
import cdo
cdo = cdo.Cdo()
cdo.env = {'REMAP EXTRAPOLATE': 'off'}
except ImportError:
raise SystemExit('CDO must be installed to preproces files.')
cdo.yearmean(input=ifile_fp, output='remapped-data/yearmean_' + ifile)
with Dataset('parameters/masks.nc',
'r') as mask, Dataset('remapped-data/yearmean_' + ifile,
'a') as to_mask:
if file_type:
with open('parameters/scale_factors.json') as json_file:
var_defs = json.load(json_file)
for var in to_mask.variables:
if var in var_defs[file_type].keys():
to_mask[var].setncattr('scale',
var_defs[file_type][var][0])
to_mask[var].units = var_defs[file_type][var][1]
to_mask[var].long_name = var_defs[file_type][var][
2]
for var in to_mask.variables:
if len(to_mask[var].shape) == 4:
to_mask[var][:] = np.ma.masked_invalid(
np.where(mask['tmask'][0, ], to_mask[var][:], np.NaN))
with Dataset('remapped-data/yearmean_' + ifile, 'r') as nc:
ncvar = nc.variables[varname]
data = ncvar[:].squeeze()
try:
units = ncvar.units
except AttributeError:
print 'Units not given for {} in {}, leaving empty.'.format(
varname, ifile)
units = ''
dimensions = ncvar.dimensions
try:
for dimension in dimensions:
if 'depth' in dimension.lower():
depth = nc.variables[dimension][:]
break
else:
raise IndexError
except IndexError:
raise SystemExit('\nDepths not given for {} in {}.'.format(
varname, ifile))
try:
dates = num2date(nc['time_counter'][:],
units=nc['time_counter'].units,
calendar=nc['time_counter'].calendar)
years = [x.year for x in dates]
except IndexError:
years = [0]
lon = nc.variables['nav_lon']
lon = lon[:].squeeze()
lat = nc.variables['nav_lat']
lat = lat[:].squeeze()
return data, units, lon, lat, depth, dimensions, years
