def variables_at_depth(
input_dir,
output_dir,
model_name,
i_times=[0],
i_depths=[0],
variables=['u', 'v', 'mld', 'salinity', 'temp', 'sea_ice_cover'],
filename_format='%Y%m%d',
log_file='edt/extract_depth.log'):
filenames = get_ncfiles_in_dir(input_dir)
for filename in filenames:
netcdf = Dataset(input_dir + filename)
modeldata = modeldata_from_downloaded(netcdf,
variables,
model_name,
i_times=i_times,
i_depths=i_depths)
output_path = modeldata.get_output_path(
output_dir, filename_format=filename_format)
if os.path.exists(output_path):
log.info(log_file, f'File already exists, skipping: {output_path}')
netcdf.close()
continue
log.info(
log_file,
f'Extracting depth layers {str(i_depths)} and saving to file: {output_path}'
)
_ = modeldata.write_to_netcdf(output_dir,
filename_format=filename_format)
netcdf.close()
def get_total_mean_from_local_files(input_dir,output_path,i_depths=None,i_lats=None,i_lons=None,
log_file='get_total_mean.log'):
ncfiles = get_ncfiles_in_dir(input_dir)
netcdf = Dataset(input_dir+ncfiles[0])
time = Dimension('time',np.array([0.0]),'days since 1900-01-01')
depth = netcdf_to_dimension(netcdf,'depth',new_name='depth',i_use=i_depths)
lon = netcdf_to_dimension(netcdf,'lon',new_name='lon',i_use=i_lons)
lat = netcdf_to_dimension(netcdf,'lat',new_name='lat',i_use=i_lats)
variables = list(set(netcdf.variables.keys())-set(['time','depth','lat','lon']))
netcdf.close()
modeldata = ModelData(time,depth,lat,lon)
for variable in variables:
n = 0
for ncfile in ncfiles:
netcdf = Dataset(input_dir+ncfile)
if n == 0:
quantity = netcdf_to_quantity(netcdf,variable,new_name=variable,i_depths=i_depths)
else:
next_quantity = netcdf_to_quantity(netcdf,variable,new_name=variable,i_depths=i_depths)
quantity.values = np.concatenate((quantity.values,next_quantity.values),axis=0)
netcdf.close()
next_quantity = None
n += 1
quantity.values = np.nanmean(quantity.values,axis=0)
modeldata.fill_variable(variable,quantity)
modeldata.write_to_netcdf(None,output_path=output_path)
def get_monthly_means_from_local_daily_files(input_dir : str, output_dir : str,
input_filenameformat = '%Y%m%d',
output_filenameformat = '%Y%m',
log_file='merge_daily_to_monthly.log'):
ncfiles = get_ncfiles_in_dir(input_dir)
for ncfile in ncfiles:
ncfile_datestr,_ = os.path.splitext(ncfile)
ncfile_datetime = datetime.strptime(ncfile_datestr,input_filenameformat)
output_path = output_dir+ncfile_datetime.strftime(output_filenameformat)+'.nc'
if os.path.exists(output_path):
log.info(log_file,f'File already exists, skipping: {output_path}')
continue
log.info(log_file,f'Getting monthly means for {ncfile_datetime.month} {ncfile_datetime.year}')
start_date = datetime(ncfile_datetime.year,ncfile_datetime.month,1)
if ncfile_datetime.month < 12:
end_date = datetime(ncfile_datetime.year,ncfile_datetime.month+1,1)
ncfiles_merge = get_ncfiles_in_time_range(input_dir,start_date,end_date,including_end=0)
else:
end_date = datetime(ncfile_datetime.year,ncfile_datetime.month,31)
ncfiles_merge = get_ncfiles_in_time_range(input_dir,start_date,end_date,including_end=1)
log.info(log_file,f'Loading data from {input_dir+ncfiles_merge[0]}')
modeldata = from_local_file(input_dir+ncfiles_merge[0])
for i in range(1,len(ncfiles_merge)):
netcdf = Dataset(input_dir+ncfiles_merge[i])
variables = list(set(netcdf.variables.keys())-set(['time','depth','lat','lon']))
for variable in variables:
log.info(log_file,f'Appending data from {input_dir+ncfiles_merge[i]}')
variable_values = netcdf[variable][:].filled(fill_value=np.nan)
modeldata.append_to_variable(variable,variable_values)
netcdf.close()
log.info(log_file,f'Getting time mean values')
modeldata.take_time_mean()
modeldata.get_output_path(output_dir,filename_format=output_filenameformat)
log.info(log_file,f'Writing to {output_path}')
modeldata.write_to_netcdf(output_dir,filename_format=output_filenameformat)
def get_total_monthly_mean_from_local_files(input_dir : str, output_dir : str,
filename_format = '%Y%m', log_file='get_total_monthly_mean.log',
i_depths=None, i_lats=None, i_lons=None) -> ModelData:
ncfiles = get_ncfiles_in_dir(input_dir)
netcdf = Dataset(input_dir+ncfiles[0])
depth = netcdf_to_dimension(netcdf,'depth',new_name='depth',i_use=i_depths)
lon = netcdf_to_dimension(netcdf,'lon',new_name='lon',i_use=i_lons)
lat = netcdf_to_dimension(netcdf,'lat',new_name='lat',i_use=i_lats)
variables = list(set(netcdf.variables.keys())-set(['time','depth','lat','lon']))
netcdf.close()
for month in np.arange(1,13):
log.info(log_file, f'Getting total mean for month: {month}')
output_path = f'{output_dir}climatology_{datetime(1900,month,1).strftime("%b").lower()}.nc'
# initialise modeldata
time = Dimension('time',np.array([(datetime(1900,month,1)-datetime(1900,1,1)).days]),'days since 1900-01-01')
modeldata = ModelData(time,depth,lat,lon)
for variable in variables:
n = 0
for ncfile in ncfiles:
ncfile_datestr,_ = os.path.splitext(ncfile)
if datetime.strptime(ncfile_datestr,filename_format).month == month:
netcdf = Dataset(input_dir+ncfile)
if n == 0:
quantity = netcdf_to_quantity(netcdf,variable,new_name=variable,i_depths=i_depths)
else:
next_quantity = netcdf_to_quantity(netcdf,variable,new_name=variable,i_depths=i_depths)
quantity.values = np.concatenate((quantity.values,next_quantity.values),axis=0)
netcdf.close()
next_quantity = None
n += 1
quantity.values = np.nanmean(quantity.values,axis=0)
modeldata.fill_variable(variable,quantity)
log.info(log_file, f'Writing to file: {output_path}')
modeldata.write_to_netcdf(None, output_path=output_path)
def all_files_in_dir_horizontally(input_dir: str,
output_dir: str,
dx=1,
x0=0.5,
filename_format='%Y%m',
method='linear',
log_file='edt/interpolate_horizontally.log'):
lon_interp = np.arange(-180. + x0, 180 + x0, dx)
lat_interp = np.arange(-80 + x0, 90 + x0, dx)
filenames = get_ncfiles_in_dir(input_dir)
for filename in filenames:
modeldata = modeldata_from_local_file(input_dir + filename)
modeldata.sort_lat_ascending()
output_path = modeldata.get_output_path(output_dir)
if os.path.exists(output_path):
log.info(log_file, f'File already exists, skipping: {output_path}')
continue
log.info(
log_file,
f'Horizontally interpolating to {str(dx)} resolution, saving file: {output_path}'
)
modeldata_interp = horizontally(modeldata,
lat_interp,
lon_interp,
method=method)
modeldata_interp.write_to_netcdf(output_dir,
filename_format=filename_format)
def merge_netcdfs_in_folder_to_single_file(input_dir,output_path,variables=['u','v'],velocities=None,log_file=None):
ncfiles = get_ncfiles_in_dir(input_dir)
log.info(log_file,f'Loading data {input_dir}{ncfiles[0]}')
modeldata = from_local_file(input_dir+ncfiles[0],variables=variables)
for ncfile in ncfiles[1:]:
netcdf = Dataset(input_dir+ncfile)
for variable_name in variables:
log.info(log_file,f'Appending {variable_name} data from {input_dir+ncfile}')
modeldata.append_to_variable(variable_name,netcdf[variable_name][:].filled(fill_value=np.nan))
log.info(log_file,f'Appending time data from {input_dir+ncfile}')
modeldata.append_to_time(netcdf['time'][:].filled(fill_value=np.nan))
netcdf.close()
if velocities == 'currents':
log.info(log_file,f'Adding absolute current velocity')
modeldata.add_absolute_current_velocity()
elif velocities == 'wind':
log.info(log_file,f'Adding absolute wind velocity')
modeldata.add_absolute_wind_velocity()
modeldata.write_to_netcdf(None,output_path=output_path)
def merge_variables_into_netcdf(input_dirs,output_dir,timeformat='%Y%m',log_file='edt/merge_netcdfs.log'):
main_input_dir = input_dirs[0]
input_dirs = input_dirs[1:]
nc_files = get_ncfiles_in_dir(main_input_dir)
for nc_file in nc_files:
log.info(log_file,f'Loading data {main_input_dir+nc_file}')
modeldata = from_local_file(main_input_dir+nc_file)
date0 = modeldata.time.datetime[0].date()
for input_dir in input_dirs:
nc_file_add = get_ncfiles_in_time_range(input_dir,date0,date0,timeformat=timeformat)
if len(nc_file_add) == 0:
continue
if len(nc_file_add) > 1:
raise ValueError(f'More than one netcdf file found in date range {date0.strftime(timeformat)}')
netcdf = Dataset(input_dir+nc_file_add[0])
variable_names = list(set(netcdf.variables.keys())-set(['time','depth','lat','lon']))
for variable_name in variable_names:
log.info(log_file,f'Adding variable {variable_name} to netcdf')
variable = netcdf_to_quantity(netcdf,variable_name)
modeldata.fill_variable(variable_name,variable)
netcdf.close()
output_path = modeldata.get_output_path(output_dir,filename_format=timeformat)
log.info(log_file,f'Saving merged netcdf to {output_path}')
_ = modeldata.write_to_netcdf(output_dir,filename_format=timeformat)
def get_ozroms_daily_ncfiles_from_irds(irds_mount, start_date, end_date):
all_ncfiles = get_ncfiles_in_dir(irds_mount)
ncfiles = get_ozroms_daily_ncfiles(all_ncfiles, start_date, end_date)
return ncfiles