def cmd(args):
in_ds = Dataset(args.input_file, 'r')
create_dir_for_file(args.output_file)
out_ds = Dataset(args.output_file, 'w', format='NETCDF4')
copy_nc_attributes(in_ds, out_ds)
for in_dim in in_ds.dimensions.values():
out_ds.createDimension(in_dim.name,
None if in_dim.isunlimited() else in_dim.size)
all_args = vars(args)
compression_args = {}
for key in ['zlib', 'complevel', 'shuffle', 'fletcher32']:
compression_args[key] = all_args[key]
for in_var in in_ds.variables.values():
print in_var.name
out_var = create_nc_var_like_other(out_ds, in_var, **compression_args)
iter_mask = np.ones((len(in_var.shape, )), dtype=bool)
iter_mask[-MAX_COPY_DIM_COUNT:] = False
dim_iterator = DimIterator(in_var.shape, None, iter_mask)
write_op_count = len(dim_iterator)
for write_op_num, slc in enumerate(dim_iterator.slice_tuples()):
_progress(write_op_num, write_op_count)
out_var[slc] = in_var[slc]
add_or_append_history(out_ds)
in_ds.close()
out_ds.close()
def cmd(args):
create_dir_for_file(args.output_file)
shutil.copyfile(args.input_file, args.output_file)
output_ds = Dataset(args.output_file, 'r+')
appended_ds = Dataset(args.appended_file, 'r')
for appended_var_name in args.appended_var_names:
appended_var = appended_ds.variables[appended_var_name]
output_var = output_ds.createVariable(
appended_var_name,
appended_var.dtype,
dimensions=appended_var.dimensions)
output_var[:] = appended_var[:]
copy_nc_attributes(appended_var, output_var)
add_or_append_history(output_ds)
output_ds.close()
def cmd(args):
in_ds = Dataset(args.input_file, 'r')
in_grid, in_grid_dim_names = \
init_grid_from_vars(in_ds.variables[args.lon_name],
in_ds.variables[args.lat_name])
scalar_vars, vector_vars = split_scalar_and_vector_vars(args.var_names)
create_dir_for_file(args.output_file)
out_ds = Dataset(args.output_file, 'w')
add_missing_dim_vars(in_ds, out_ds, in_grid_dim_names)
grid_ds = Dataset(args.grid_file, 'r')
grid_proj_var = grid_ds.variables[names.VAR_PROJECTION]
converter = init_converter_from_proj_var(grid_proj_var)
print 'Calculating coordinates of grid points:'
xx, yy = [], []
for i in xrange(in_grid.shape[0]):
_progress(i, in_grid.shape[0])
row_xx, row_yy = converter.convert_points(in_grid[i, :, 1],
in_grid[i, :, 0])
xx.append(row_xx)
yy.append(row_yy)
xx = np.concatenate(xx)
yy = np.concatenate(yy)
_progress(in_grid.shape[0], in_grid.shape[0])
out_proj_var = out_ds.createVariable(names.VAR_PROJECTION,
grid_proj_var.dtype)
copy_nc_attributes(grid_proj_var, out_proj_var)
out_x_var = out_ds.createVariable(args.x_name,
xx.dtype,
dimensions=in_grid_dim_names)
copy_nc_attributes(grid_ds.variables[names.DIMVAR_X], out_x_var)
out_x_var[:, :] = xx
out_y_var = out_ds.createVariable(args.y_name,
yy.dtype,
dimensions=in_grid_dim_names)
copy_nc_attributes(grid_ds.variables[names.DIMVAR_Y], out_y_var)
out_y_var[:, :] = yy
grid_ds.close()
if len(scalar_vars) > 0:
print 'Processing scalar fields:'
for var_name in scalar_vars:
print var_name
in_var = in_ds.variables[var_name]
add_missing_dim_vars(in_ds, out_ds, in_var.dimensions)
out_var = out_ds.createVariable(var_name,
in_var.dtype,
dimensions=in_var.dimensions)
copy_nc_attributes(in_var, out_var)
iter_mask = np.ones((len(in_var.shape, )), dtype=bool)
iter_mask[-MAX_COPY_DIM_COUNT:] = False
dim_iterator = DimIterator(in_var.shape, None, iter_mask)
write_op_count = len(dim_iterator)
for write_op_num, slc in enumerate(dim_iterator.slice_tuples()):
_progress(write_op_num, write_op_count)
out_var[slc] = in_var[slc]
_progress(write_op_count, write_op_count)
if len(vector_vars) > 0:
print 'Processing vector fields:'
for var_name_pair in vector_vars:
print var_name_pair
in_u_var = in_ds.variables[var_name_pair[0]]
in_v_var = in_ds.variables[var_name_pair[1]]
if in_u_var.dimensions != in_v_var.dimensions:
raise Exception()
grid_dim_indices = find_dim_indices(in_grid_dim_names,
in_u_var.dimensions)
if any(idx is None for idx in grid_dim_indices):
raise Exception()
add_missing_dim_vars(in_ds, out_ds, in_u_var.dimensions)
out_x_var = out_ds.createVariable('_'.join(var_name_pair) + '_' +
args.x_name,
in_u_var.dtype,
dimensions=in_u_var.dimensions)
out_x_var.projection = out_proj_var.grid_mapping_name
out_y_var = out_ds.createVariable('_'.join(var_name_pair) + '_' +
args.y_name,
in_v_var.dtype,
dimensions=in_u_var.dimensions)
out_y_var.projection = out_proj_var.grid_mapping_name
swap_axes = grid_dim_indices[0] > grid_dim_indices[1]
iter_mask = np.ones((len(in_u_var.shape, )), dtype=bool)
iter_mask[grid_dim_indices] = False
dim_iterator = DimIterator(in_u_var.shape, None, iter_mask)
write_op_count = len(dim_iterator)
for write_op_num, slc in enumerate(dim_iterator.slice_tuples()):
_progress(write_op_num, write_op_count)
in_u_field = in_u_var[slc]
in_v_field = in_v_var[slc]
if swap_axes:
in_u_field = np.swapaxes(in_u_field, grid_dim_indices[0],
grid_dim_indices[1])
in_v_field = np.swapaxes(in_v_field, grid_dim_indices[0],
grid_dim_indices[1])
out_x_field, out_y_field, = \
converter.convert_vectors(in_u_field, in_v_field,
in_grid[..., 1], in_grid[..., 0])
if swap_axes:
out_x_field = np.swapaxes(out_x_field, grid_dim_indices[0],
grid_dim_indices[1])
out_y_field = np.swapaxes(out_y_field, grid_dim_indices[0],
grid_dim_indices[1])
out_x_var[slc] = out_x_field
out_y_var[slc] = out_y_field
_progress(write_op_count, write_op_count)
add_history(out_ds, get_history(in_ds))
in_ds.close()
out_ds.close()
def cmd(args):
in_ds = Dataset(args.input_file, 'r')
# Latitude variable is mandatory.
in_lat_var = in_ds.variables[args.lat_name]
if 1 != len(in_lat_var.dimensions):
raise Exception('\'%s\' is not 1D variable.' % args.lat_name)
in_lat_dim_name = in_lat_var.dimensions[0]
# Longitude variable is optional (but only for scalar fields).
if args.lon_name is not None:
in_lon_var = in_ds.variables[args.lon_name]
if 1 != len(in_lon_var.dimensions):
raise Exception('\'%s\' is not 1D variable.' % args.lon_name)
in_lon_dim_name = in_lon_var.dimensions[0]
if in_lat_dim_name == in_lon_dim_name:
raise Exception('Latitude and longitude dimension variables '
'can not be specified along the same dimension.')
scalar_vars, vector_vars = split_scalar_and_vector_vars(args.var_names)
if len(vector_vars) > 0 and args.lon_name is None:
raise Exception('Vector fields cannot be processed without longitude '
'variable.')
add_north_pole = (args.add == 'north' or args.add == 'both')
add_south_pole = (args.add == 'south' or args.add == 'both')
lat_list = in_lat_var[:]
pole_tol = lat_list.dtype.type(POLE_TOLERANCE)
np_lat = lat_list.dtype.type(90)
min_lat_idx, max_lat_idx = 0, -1
lat_list_ascending = True
if lat_list.shape[0] > 1:
if np.all(lat_list[1:] > lat_list[:-1]):
pass
elif np.all(lat_list[1:] < lat_list[:-1]):
min_lat_idx, max_lat_idx = max_lat_idx, min_lat_idx
lat_list_ascending = False
else:
raise Exception('Latitudes must be sorted and not be duplicated.')
elif lat_list.shape[0] != 1:
raise Exception('Array of latitudes must not be empty.')
append_row = prepend_row = False
if add_north_pole:
if np.abs(lat_list[max_lat_idx] - np_lat) <= pole_tol:
raise Exception('Input grid already contains grid points for the '
'North Pole.')
if lat_list_ascending:
append_row = True
else:
prepend_row = True
lat_list = _extend_axis(lat_list, np_lat, not lat_list_ascending)
if add_south_pole:
if np.abs(lat_list[min_lat_idx] + np_lat) <= pole_tol:
raise Exception('Input grid already contains grid points for the '
'South Pole.')
if lat_list_ascending:
prepend_row = True
else:
append_row = True
lat_list = _extend_axis(lat_list, -np_lat, lat_list_ascending)
create_dir_for_file(args.output_file)
out_ds = Dataset(args.output_file, 'w')
out_ds.createDimension(
in_lat_dim_name, None if
in_ds.dimensions[in_lat_dim_name].isunlimited() else lat_list.shape[0])
out_lat_var = out_ds.createVariable(args.lat_name,
in_lat_var.dtype,
dimensions=(in_lat_dim_name, ))
copy_nc_attributes(in_lat_var, out_lat_var)
out_lat_var[:] = lat_list
if args.lon_name is not None:
lon_list = in_lon_var[:]
out_ds.createDimension(
in_lon_dim_name,
None if in_ds.dimensions[in_lon_dim_name].isunlimited() else
lon_list.shape[0])
out_lon_var = out_ds.createVariable(args.lon_name,
in_lon_var.dtype,
dimensions=(in_lon_dim_name, ))
copy_nc_attributes(in_lon_var, out_lon_var)
out_lon_var[:] = lon_list
if len(scalar_vars) > 0:
print 'Processing scalar fields:'
for var_name in scalar_vars:
print var_name
in_var = in_ds.variables[var_name]
add_missing_dim_vars(in_ds, out_ds, in_var.dimensions)
lat_idx, lon_idx = find_dim_indices([args.lat_name, args.lon_name],
in_var.dimensions)
out_var = out_ds.createVariable(var_name,
in_var.dtype,
dimensions=in_var.dimensions)
iter_mask = np.ones((len(in_var.shape, )), dtype=bool)
if lat_idx is not None:
iter_mask[lat_idx] = False
swap_axes = False
if lon_idx is not None:
iter_mask[lon_idx] = False
swap_axes = lon_idx < lat_idx
read_iter = DimIterator(in_var.shape, None, iter_mask)
write_iter = DimIterator(out_var.shape, None, iter_mask)
write_op_count = len(read_iter)
for write_op_num, (read_slc, write_slc) in enumerate(
izip(read_iter.slice_tuples(), write_iter.slice_tuples())):
_progress(write_op_num, write_op_count)
in_field = in_var[read_slc]
if swap_axes:
in_field = np.swapaxes(in_field, lat_idx, lon_idx)
out_field = in_field
if prepend_row:
out_field = _extend_scalar_field(out_field, True)
if append_row:
out_field = _extend_scalar_field(out_field, False)
if swap_axes:
out_field = np.swapaxes(out_field, lat_idx, lon_idx)
out_var[write_slc] = out_field
_progress(write_op_count, write_op_count)
if len(vector_vars) > 0:
print 'Processing vector fields:'
to_zero, from_zero = gen_rot_matrices_deg(lon_list, True)
for var_name_pair in vector_vars:
print var_name_pair
in_u_var = in_ds.variables[var_name_pair[0]]
in_v_var = in_ds.variables[var_name_pair[1]]
if in_u_var.dimensions != in_v_var.dimensions:
raise Exception()
lat_idx, lon_idx = find_dim_indices([args.lat_name, args.lon_name],
in_u_var.dimensions)
if lat_idx is None or lon_idx is None:
raise Exception()
add_missing_dim_vars(in_ds, out_ds, in_u_var.dimensions)
out_u_var = out_ds.createVariable(var_name_pair[0],
in_u_var.dtype,
dimensions=in_u_var.dimensions)
out_v_var = out_ds.createVariable(var_name_pair[1],
in_v_var.dtype,
dimensions=in_v_var.dimensions)
swap_axes = lon_idx < lat_idx
iter_mask = np.ones((len(in_u_var.shape, )), dtype=bool)
iter_mask[lat_idx] = iter_mask[lon_idx] = False
read_iter = DimIterator(in_u_var.shape, None, iter_mask)
write_iter = DimIterator(out_u_var.shape, None, iter_mask)
write_op_count = len(read_iter)
for write_op_num, (read_slc, write_slc) in enumerate(
izip(read_iter.slice_tuples(), write_iter.slice_tuples())):
_progress(write_op_num, write_op_count)
in_u_field = in_u_var[read_slc]
in_v_field = in_v_var[read_slc]
if swap_axes:
in_u_field = np.swapaxes(in_u_field, lat_idx, lon_idx)
in_v_field = np.swapaxes(in_v_field, lat_idx, lon_idx)
out_u_field, out_v_field = in_u_field, in_v_field
if prepend_row:
out_u_field, out_v_field = \
_extend_vector_field(out_u_field, out_v_field,
to_zero, from_zero,
True)
if append_row:
out_u_field, out_v_field = \
_extend_vector_field(out_u_field, out_v_field,
to_zero, from_zero,
False)
if swap_axes:
out_u_field = np.swapaxes(out_u_field, lat_idx, lon_idx)
out_v_field = np.swapaxes(out_v_field, lat_idx, lon_idx)
out_u_var[write_slc] = out_u_field
out_v_var[write_slc] = out_v_field
_progress(write_op_count, write_op_count)
add_history(out_ds, get_history(in_ds))
in_ds.close()
out_ds.close()
def cmd(args):
scalar_vars, vector_vars = split_scalar_and_vector_vars(args.var_names)
if len(vector_vars) < 1:
raise Exception('No input vector fields specified.')
if args.angle_file is None:
args.angle_file = args.input_file
angle_ds = Dataset(args.angle_file, 'r')
if args.angle_name not in angle_ds.variables:
raise Exception('Angle file \'%s\' does not contain angle variable '
'\'%s\'' % (args.angle_file, args.angle_name))
angle_var = angle_ds.variables[args.angle_name]
angle_shape = angle_var.shape
in_ds = Dataset(args.input_file, 'r')
if args.dim_names is None:
args.dim_names = angle_var.dimensions
else:
if len(args.dim_names) != len(angle_var.dimensions):
raise Exception('Number of dimensions specified by the parameter '
'--dim-names is not equal to the number of '
'dimensions of the --angle-name variable.')
for dim_idx, dim_name in enumerate(args.dim_names):
if dim_name not in in_ds.dimensions:
raise Exception('Input file does not contain dimension '
'\'%s\'.' % dim_name)
if in_ds.dimensions[dim_name].size != angle_shape[dim_idx]:
raise Exception('Size of the dimension \'%s\' of the input '
'file is not equal to the size of the '
'dimension \'%s\' of the angle file.'
% (dim_name, angle_var.dimensions[dim_idx]))
rot_matrices = gen_rot_matrices_rad(angle_var[:]) \
if args.angle_units == 'rad' \
else gen_rot_matrices_deg(angle_var[:])
angle_ds.close()
create_dir_for_file(args.output_file)
out_ds = Dataset(args.output_file, 'w')
if len(scalar_vars) > 0:
print 'Processing scalar fields:'
for var_name in scalar_vars:
print var_name
in_var = in_ds.variables[var_name]
add_missing_dim_vars(in_ds, out_ds, in_var.dimensions)
out_var = out_ds.createVariable(var_name,
in_var.dtype,
dimensions=in_var.dimensions)
copy_nc_attributes(in_var, out_var)
iter_mask = np.ones((len(in_var.shape, )), dtype=bool)
iter_mask[-MAX_COPY_DIM_COUNT:] = False
dim_iterator = DimIterator(in_var.shape, None, iter_mask)
write_op_count = len(dim_iterator)
for write_op_num, slc in enumerate(dim_iterator.slice_tuples()):
_progress(write_op_num, write_op_count)
out_var[slc] = in_var[slc]
_progress(write_op_count, write_op_count)
if len(vector_vars) > 0:
print 'Processing vector fields:'
angle_iter_mask = np.ones((len(args.dim_names),), dtype=bool)
dims_per_operation = len(args.dim_names) \
if len(args.dim_names) < MAX_COPY_DIM_COUNT \
else MAX_COPY_DIM_COUNT
angle_iter_mask[-dims_per_operation:] = False
angle_dim_iterator = DimIterator(angle_shape, None, angle_iter_mask)
angle_iter_count = len(angle_dim_iterator)
for var_name_pair in vector_vars:
print var_name_pair
in_u_var = in_ds.variables[var_name_pair[0]]
in_v_var = in_ds.variables[var_name_pair[1]]
if in_u_var.dimensions != in_v_var.dimensions:
raise Exception()
angle_dim_indices = find_dim_indices(args.dim_names,
in_u_var.dimensions)
for idx, dim_name in enumerate(args.dim_names):
if angle_dim_indices[idx] is None:
raise Exception('Variable \'%s\' is not specified along '
'dimension \'%s\'.'
% (var_name_pair[0], dim_name))
add_missing_dim_vars(in_ds, out_ds, in_u_var.dimensions)
out_u_var = out_ds.createVariable(
var_name_pair[0],
in_u_var.dtype,
dimensions=in_u_var.dimensions)
out_v_var = out_ds.createVariable(
var_name_pair[1],
in_v_var.dtype,
dimensions=in_u_var.dimensions)
fixed_indices = angle_dim_indices[-dims_per_operation:]
dim_var_order = np.arange(len(fixed_indices))
dim_angle_order = np.argsort(fixed_indices)
var_iter_mask = np.ones((len(in_u_var.shape, )), dtype=bool)
var_iter_mask[fixed_indices] = False
for angle_iter_num, angle_slc in \
enumerate(angle_dim_iterator.slice_tuples()):
var_fixed_slices = np.repeat(None, len(in_u_var.shape))
var_fixed_slices[angle_dim_indices] = angle_slc
var_iter = DimIterator(in_u_var.shape, var_fixed_slices,
var_iter_mask)
var_iter_count = len(var_iter)
total_iter_count = angle_iter_count * var_iter_count
for var_iter_num, var_slc in enumerate(var_iter.slice_tuples()):
_progress(var_iter_num + angle_iter_num * var_iter_count,
total_iter_count)
in_u_field = in_u_var[var_slc]
in_v_field = in_v_var[var_slc]
in_u_field = np.moveaxis(in_u_field, dim_var_order,
dim_angle_order)
in_v_field = np.moveaxis(in_v_field, dim_var_order,
dim_angle_order)
out_u_field, out_v_field = \
apply_rot_matrices(in_u_field, in_v_field,
rot_matrices[(slice(None), slice(None))
+ angle_slc])
out_u_field = np.moveaxis(out_u_field, dim_angle_order,
dim_var_order)
out_v_field = np.moveaxis(out_v_field, dim_angle_order,
dim_var_order)
out_u_var[var_slc] = out_u_field
out_v_var[var_slc] = out_v_field
_progress(total_iter_count, total_iter_count)
add_history(out_ds, get_history(in_ds))
in_ds.close()
out_ds.close()