def hourly_date_range_from_irds_file(input_path,
output_dir,
date_range,
lon_range=None,
lat_range=None,
variables=['u', 'v'],
i_depths=[0],
log_file='dl/ozroms_hourly.log'):
log.info(log_file, f'Accessing: {input_path}')
i_lons, i_lats = _get_i_lons_and_i_lats(input_path, lon_range, lat_range)
netcdf = Dataset(input_path)
time_description = get_variable_name('ozroms', 'time')
times_org = netcdf[time_description][:].filled(fill_value=np.nan)
times = convert_time_to_datetime(times_org, netcdf[time_description].units)
n_days = (date_range[1] - date_range[0]).days
for i in range(n_days):
time = date_range[0] + timedelta(days=i)
output_path = f'{output_dir}{time.strftime("%Y%m%d")}.nc'
if os.path.exists(output_path):
log.info(log_file, f'File already exists, skipping: {output_path}')
continue
log.info(log_file, f'Downloading {time.strftime("%d-%m-%Y")}')
i_times = get_time_indices(times, time)
ozromsdata = modeldata_from_downloaded(netcdf,
variables,
'ozroms',
i_times=i_times,
i_depths=i_depths,
i_lons=i_lons,
i_lats=i_lats)
ozromsdata.write_to_netcdf(output_dir)
netcdf.close()
def monthly_files(input_path,
output_dir,
model_name,
log_file='edt/extract_monthly.log'):
netcdf = Dataset(input_path)
# keep original variables: find general name from model specific name
model_variables = list(
set(netcdf.variables.keys()) - set(netcdf.dimensions.keys()))
variables = []
for model_variable in model_variables:
variables.append(get_variable_name_reverse(model_name, model_variable))
time_var = get_variable_name(model_name, 'time')
time = convert_time_to_datetime(netcdf[time_var][:],
netcdf[time_var].units)
n_months = get_n_months(time[0], time[-1])
date0 = datetime(time[0].year, time[0].month, 1)
for i in range(n_months):
start_date = add_month_to_timestamp(date0, i)
end_date = add_month_to_timestamp(date0, i + 1) - timedelta(days=1)
l_times = get_l_time_range(time, start_date, end_date)
i_times = np.where(l_times)[0]
modeldata = modeldata_from_downloaded(netcdf,
variables,
model_name,
i_times=i_times)
output_path = modeldata.get_output_path(output_dir,
filename_format='%Y%m')
if os.path.exists(output_path):
log.info(log_file, f'File already exists, skipping: {output_path}')
continue
log.info(log_file, f'Writing monthly data to file: {output_path}')
_ = modeldata.write_to_netcdf(output_dir, filename_format='%Y%m')
netcdf.close()
def __init__(self, name: str, values: np.ndarray, units: str):
self.name = name
self.values = values
self.units = units
self.length = len(values)
if name == 'time':
self.datetime = convert_time_to_datetime(self.values,self.units)
def file_from_opendap_server(output_dir,
input_url,
lon_range=None,
lat_range=None,
variables=['u', 'v'],
i_depths=[0],
log_file='dl/ozroms.log'):
i_lons, i_lats = _get_i_lons_and_i_lats(input_url, lon_range, lat_range)
netcdf = Dataset(input_url)
time_description = get_variable_name('ozroms', 'time')
times_org = netcdf[time_description][:].filled(fill_value=np.nan)
times = convert_time_to_datetime(times_org, netcdf[time_description].units)
for time in times:
output_path = output_dir + time.strftime('%Y%m%d') + '.nc'
if os.path.exists(output_path):
log.info(log_file,
f'File {output_path} already exists, skipping download.')
continue
# download daily output
log.info(log_file, f'Downloading {time.strftime("%d-%m-%Y")}')
i_times = get_time_indices(times, time)
ozromsdata = modeldata_from_downloaded(netcdf,
variables,
'ozroms',
i_times=i_times,
i_depths=i_depths,
i_lons=i_lons,
i_lats=i_lats)
ozromsdata.write_to_netcdf(output_dir)
netcdf.close()
def read_from_netcdf(input_path,time_start=None,time_end=None,t_index=None):
data = Dataset(input_path)
time_org = data['time'][:].filled(fill_value=np.nan)
time_units = data['time'].units
time = convert_time_to_datetime(time_org,time_units)
if time_start is not None and time_end is not None:
t_start = get_closest_time_index(time,time_start)
t_end = get_closest_time_index(time,time_end)
t = np.arange(t_start,t_end+1,1)
density = data['density'][t,:,:].filled(fill_value=np.nan)
total_particles = data['total_particles'][t].filled(fill_value=np.nan)
time = time[t]
elif t_index is not None:
density = data['density'][t_index,:,:].filled(fill_value=np.nan)
total_particles = data['total_particles'][t_index].filled(fill_value=np.nan)
time = time[t_index]
else:
density = data['density'][:].filled(fill_value=np.nan)
total_particles = data['total_particles'][:].filled(fill_value=np.nan)
lon = data['lon'][:].filled(fill_value=np.nan)
lat = data['lat'][:].filled(fill_value=np.nan)
data.close()
dx = np.unique(np.diff(lon))[0]
dy = np.unique(np.diff(lat))[0]
lon_range = [lon.min(),lon.max()]
lat_range = [lat.min(),lat.max()]
grid = Grid(dx,lon_range,lat_range,dy=dy)
return Density(grid,time,total_particles,density)
def __init__(self, input_path):
netcdf = Dataset(input_path)
pid = netcdf['trajectory'][:, 0].filled(fill_value=np.nan)
time_all = netcdf['time'][:].filled(fill_value=np.nan)
time_units = netcdf['time'].units
lon_org = netcdf['lon'][:].filled(fill_value=np.nan)
lat_org = netcdf['lat'][:].filled(fill_value=np.nan)
netcdf.close()
# construct time array for all particles with dt as
# maximum output frequency. this is needed because
# when particles are deleted from the simulation,
# their final time is written to file. this time
# can be smaller than the specified output frequency.
t0 = np.nanmin(time_all)
tend = np.nanmax(time_all)
dt = np.nanmax(np.diff(time_all))
time_org = np.arange(t0, tend + dt, dt)
lon = np.empty(lon_org.shape) * np.nan
lat = np.empty(lat_org.shape) * np.nan
total_particles = []
for t, time in enumerate(time_org):
i, j = np.where(time_all == time)
lon[i, np.repeat(t, len(i))] = lon_org[i, j]
lat[i, np.repeat(t, len(i))] = lat_org[i, j]
total_particles.append(len(i))
time0 = convert_time_to_datetime([time_org[-1]], time_units)
# get particle locations from final time to restart
self.lon0 = lon[:, -1]
self.lat0 = lat[:, -1]
self.time0 = np.repeat(time0, len(self.lon0))
def get_data_from_parcels_netcdf(input_path,t_interval=5):
data = Dataset(input_path)
pid = data['trajectory'][:,0].filled(fill_value=np.nan)
time_all = data['time'][:].filled(fill_value=np.nan)
time_units = data['time'].units
lon_org = data['lon'][:].filled(fill_value=np.nan)
lat_org = data['lat'][:].filled(fill_value=np.nan)
data.close()
# construct time array for all particles with dt as
# maximum output frequency. this is needed because
# when particles are deleted from the simulation,
# their final time is written to file. this time
# can be smaller than the specified output frequency.
t0 = np.nanmin(time_all)
tend = np.nanmax(time_all)
dt = np.nanmax(np.diff(time_all))
time_org = np.arange(t0,tend+dt,dt*t_interval)
lon = np.empty((len(pid),len(time_org)))*np.nan
lat = np.empty((len(pid),len(time_org)))*np.nan
for t,time in enumerate(time_org):
i,j = np.where(time_all == time)
lon[i,np.repeat(t,len(i))] = lon_org[i,j]
lat[i,np.repeat(t,len(i))] = lat_org[i,j]
time = convert_time_to_datetime(time_org,time_units)
beached = np.zeros(lon.shape)
return (pid,time,lon,lat,beached)
def daily_opendap_server(output_dir,
main_data_url,
dl_date,
log_file='dl/hycom.log',
variables=['u', 'v'],
i_depths=[0]):
output_path = output_dir + dl_date.strftime('%Y%m%d') + '.nc'
if os.path.exists(output_path):
log.info(log_file,
f'File {output_path} already exists, skipping download.')
else:
input_path = main_data_url + str(dl_date.year)
netcdf = Dataset(input_path)
times_org = netcdf['time'][:]
time_units = netcdf['time'].units
times = convert_time_to_datetime(times_org, time_units)
i_times = get_time_indices(times, dl_date)
log.info(log_file, f'Downloading {dl_date.strftime("%d-%m-%Y")}')
hycomdata = modeldata_from_downloaded(netcdf,
variables,
'hycom',
i_times=i_times,
i_depths=i_depths)
hycomdata.write_to_netcdf(output_dir)
netcdf.close()
def opendap_server(output_dir,
main_data_url,
start_date,
end_date,
log_file='dl/hycom.log',
variables=['u', 'v'],
i_depths=[0]):
'''Downloads 3-hourly HYCOM data from OpenDAP server and saves data to daily netcdf file.
Input:
- output_dir [string]: directory where output netcdf files are saved
- main_data_url [string]: url to relevant HYCOM OpenDAP server
- start_date [datetime.date]: date to start download
- end_date [datetime.date]: date to end download
- log_file (optional) [string]: file in which log messages are saved (default: dl/hycom.log)
- variables (optional) [list of strings]: list of strings specifying variables to download, these
should match the variable names in "input/variables.json" (default: ['u','v'])
- i_depths (optional) [list of integers]: indices of depth layers to download (default: [0], surface)'''
n_years = end_date.year - start_date.year
for n_year in range(n_years):
year = start_date.year + n_year
input_path = main_data_url + str(year)
log.info(log_file, 'Loading OpenDap data...')
netcdf = Dataset(input_path)
times_org = netcdf['time'][:]
time_units = netcdf['time'].units
times = convert_time_to_datetime(times_org, time_units)
for time in times:
output_path = output_dir + time.strftime('%Y%m%d') + '.nc'
if os.path.exists(output_path):
log.info(
log_file,
f'File {output_path} already exists, skipping download.')
continue
# download daily output
log.info(log_file, f'Downloading {time.strftime("%d-%m-%Y")}')
i_times = get_time_indices(times, time)
hycomdata = modeldata_from_downloaded(netcdf,
variables,
'hycom',
i_times=i_times,
i_depths=i_depths)
hycomdata.write_to_netcdf(output_dir)
netcdf.close()
def read_from_netcdf(input_path):
log.info(
None,
f'Reading Indian Ocean particle development timeseries from netcdf: {input_path}'
)
netcdf = Dataset(input_path)
time_org = netcdf['time'][:].filled(fill_value=np.nan)
time = convert_time_to_datetime(time_org, netcdf['time'].units)
total_particles = netcdf['total_particles'][:].filled(
fill_value=np.nan)
ocean_nio = netcdf['ocean_nio'][:].filled(fill_value=np.nan)
ocean_sio = netcdf['ocean_sio'][:].filled(fill_value=np.nan)
beached_nio = netcdf['beached_nio'][:].filled(fill_value=np.nan)
beached_sio = netcdf['beached_sio'][:].filled(fill_value=np.nan)
escaped_io = netcdf['escaped_io'][:].filled(fill_value=np.nan)
return IoParticleDevelopmentTimeseries(time, total_particles,
ocean_nio, ocean_sio,
beached_nio, beached_sio,
escaped_io)
def read_from_netcdf(input_path,time_start=None,time_end=None):
log.info(None,f'Reading particles from: {input_path}')
data = Dataset(input_path)
pid = data['pid'][:].filled(fill_value=np.nan)
time_org = data['time'][:].filled(fill_value=np.nan)
time_units = data['time'].units
time = convert_time_to_datetime(time_org,time_units)
if time_start is not None and time_end is not None:
t_start = get_closest_time_index(time,time_start)
t_end = get_closest_time_index(time,time_end)
t = np.arange(t_start,t_end+1,1)
lon = data['lon'][:,t].filled(fill_value=np.nan)
lat = data['lat'][:,t].filled(fill_value=np.nan)
beached = data['beached'][:,t].filled(fill_value=np.nan)
time = time[t]
else:
lon = data['lon'][:].filled(fill_value=np.nan)
lat = data['lat'][:].filled(fill_value=np.nan)
beached = data['beached'][:].filled(fill_value=np.nan)
data.close()
return BeachingParticles(pid,time,lon,lat,beached,t_interval=5)