def find_paths(E,date,file_type='diag',hostname='taurus',debug=False):
import DART as dart
"""
This subroutine takes a DART experiment dictionary and returns the file path for the
needed diagnostic.
The optional input, `file_type`, can have one of these values:
+ 'covariance' -- then we load pre-computed data of covariances between state variables and a given obs
+ 'obs_epoch' -- load obs_epoch_XXXX.nc files
+ 'diag' -- load standard DART Posterior_Diag or Prior_Diag files
+ 'truth' -- load true state files from a perfect-model simulation
"""
path_found = False
if E['run_category'] == 'NCAR':
data_dir_list,truth_dir_list = exp_paths_NCAR(hostname,E['exp_name'])
path_found = True
if 'ERA' in E['exp_name']:
data_dir_list,truth_dir_list = exp_paths_era(date,hostname,diagnostic=E['diagn'])
path_found = True
if not path_found:
data_dir_list,truth_dir_list = exp_paths(hostname,E['exp_name'])
#------------COVARIANCE FILES
if file_type == 'covariance':
fname = E['exp_name']+'_'+'covariance_'+E['obs_name']+'_'+E['variable']+'_'+date.strftime('%Y-%m-%d')+'.nc'
#------------OBS EPOCH FILES
if file_type == 'obs_epoch':
DR = get_experiment_date_ranges(E['exp_name'])
delta_time = date-DR[0]
obs_epoch_no = delta_time.days+1
if obs_epoch_no < 10:
obs_epoch_name = 'obs_epoch_00'+str(obs_epoch_no)+'.nc'
if (obs_epoch_no >= 10) and (obs_epoch_no < 100):
obs_epoch_name = 'obs_epoch_0'+str(obs_epoch_no)+'.nc'
if (obs_epoch_no >= 100):
obs_epoch_name = 'obs_epoch_'+str(obs_epoch_no)+'.nc'
if E['run_category'] is None:
fname = '/dart/hist/'+obs_epoch_name
if E['run_category'] == 'ERPDA':
fname = '/../obs_epoch/'+obs_epoch_name
#------------regular DART output files or true state files
if (file_type == 'diag') or (file_type == 'truth'):
if E['diagn']=='Truth':
file_type='truth'
# either load a given date, or a time mean
if isinstance(date,str):
endstring=date
else:
datestr = date.strftime("%Y-%m-%d")
seconds = date.hour*60*60
if seconds == 0:
timestr = '00000'
else:
timestr = str(seconds)
endstring =datestr+'-'+timestr
if E['run_category'] is None:
diagstring = 'Diag'
# additional diagnostics files have the 'Diag' string replaced with something else.
TIL_variables = ['theta','ptrop','Nsq','P','brunt','ztrop']
if E['variable'] in TIL_variables:
diagstring='TIL'
fname = '/dart/hist/cam_'+E['diagn']+'_'+diagstring+'.'+endstring+'.nc'
#fname = '/dart/hist/cam_'+E['diagn']+'_'+diagstring+'.'+datestr+'-'+timestr+'.nc'
fname_truth = '/dart/hist/cam_'+'True_State'+'.'+endstring+'.nc'
if E['run_category'] == 'ERPDA':
gday = dart.date_to_gday(date)
# for all my (Lisa's) old experiments, obs sequence 1 is 1 Jan 2009
gday1 = dart.date_to_gday(datetime.datetime(2009,1,1,0,0,0))
obs_seq_no = int(gday-gday1+1)
if (obs_seq_no < 10):
mid = 'obs_000'+str(obs_seq_no)
else:
mid = 'obs_00'+str(obs_seq_no)
fname_truth = mid+'/'+'True_State.nc'
fname = mid+'/'+E['diagn']+'_Diag.nc'
if E['run_category']=='NCAR':
if E['exp_name'] == 'NCAR_LAONLY':
suffix = '_LAONLY'
else:
suffix = ''
fname_truth = '/'+'True_State'+'_'+datestr+'-'+timestr+'.nc'+suffix
fname = '/'+E['diagn']+'_Diag.'+datestr+'-'+timestr+'.nc'+suffix
if file_type == 'truth':
fname = fname_truth
data_dir_list = truth_dir_list
# if data_dir_list was not found, throw an error
if data_dir_list is None:
print('experiment_settings.py cannot find settings for the following experiment dict:')
print(E)
return None
#-----search for the right files
correct_filepath_found = False
for data_dir in data_dir_list:
filename = data_dir+fname
if debug:
print('Looking for file '+filename)
if os.path.exists(filename):
correct_filepath_found = True
break
# return the file filename with path
return filename
def find_paths(E, date, file_type='diag', hostname='taurus', debug=False):
import DART as dart
"""
This subroutine takes a DART experiment dictionary and returns the file path for the
needed diagnostic.
The optional input, `file_type`, can have one of these values:
+ 'covariance' -- then we load pre-computed data of covariances between state variables and a given obs
+ 'obs_epoch' -- load obs_epoch_XXXX.nc files
+ 'diag' -- load standard DART Posterior_Diag or Prior_Diag files
+ 'truth' -- load true state files from a perfect-model simulation
Note that if E has an additional entry called "extra_string", that string is added
to the name of the file that we retrieve -- that makes it easy to retrieve
unusual files that were created later but in the same style as other files.
"""
path_found = False
if E['run_category'] == 'NCAR':
data_dir_list, truth_dir_list = exp_paths_NCAR(hostname, E['exp_name'])
path_found = True
if 'ERA' in E['exp_name']:
data_dir_list, truth_dir_list = exp_paths_era(date,
hostname,
diagnostic=E['diagn'])
path_found = True
if not path_found:
data_dir_list, truth_dir_list = exp_paths(hostname, E['exp_name'])
#------------COVARIANCE FILES
if file_type == 'covariance':
fname = E['exp_name'] + '_' + 'covariance_' + E['obs_name'] + '_' + E[
'variable'] + '_' + date.strftime('%Y-%m-%d') + '.nc'
#------------OBS EPOCH FILES
if file_type == 'obs_epoch':
DR = get_experiment_date_ranges(E['exp_name'])
delta_time = date - DR[0]
obs_epoch_no = delta_time.days + 1
if obs_epoch_no < 10:
obs_epoch_name = 'obs_epoch_00' + str(obs_epoch_no) + '.nc'
if (obs_epoch_no >= 10) and (obs_epoch_no < 100):
obs_epoch_name = 'obs_epoch_0' + str(obs_epoch_no) + '.nc'
if (obs_epoch_no >= 100):
obs_epoch_name = 'obs_epoch_' + str(obs_epoch_no) + '.nc'
if E['run_category'] is None:
fname = '/dart/hist/' + obs_epoch_name
if E['run_category'] == 'ERPDA':
fname = '/../obs_epoch/' + obs_epoch_name
#------------regular DART output files or true state files
if (file_type == 'diag') or (file_type == 'truth'):
if E['diagn'] == 'Truth':
file_type = 'truth'
# either load a given date, or a time mean
if isinstance(date, str):
endstring = date
else:
datestr = date.strftime("%Y-%m-%d")
seconds = date.hour * 60 * 60
if seconds == 0:
timestr = '00000'
else:
timestr = str(seconds)
endstring = datestr + '-' + timestr
if E['run_category'] is None:
diagstring = 'Diag'
# additional diagnostics files have the 'Diag' string replaced with something else.
TIL_variables = ['theta', 'ptrop', 'Nsq', 'P', 'brunt', 'ztrop']
# the following list returns list of the above variables that appear in the requested variable type
import re
matches = [
string for string in TIL_variables if string in E['variable']
]
if len(matches) > 0:
diagstring = 'TIL'
if 'extrastring' not in E:
E['extrastring'] = ''
if E['extrastring'] == '':
fname = '/dart/hist/cam_' + E[
'diagn'] + '_' + diagstring + '.' + endstring + '.nc'
fname_truth = '/dart/hist/cam_' + 'True_State' + '.' + endstring + '.nc'
else:
fname = '/dart/hist/cam_' + E[
'diagn'] + '_' + diagstring + '.' + E[
'extrastring'] + '.' + endstring + '.nc'
fname_truth = '/dart/hist/cam_' + 'True_State' + '.' + E[
'extrastring'] + '.' + endstring + '.nc'
if E['run_category'] == 'ERPDA':
gday = dart.date_to_gday(date)
# for all my (Lisa's) old experiments, obs sequence 1 is 1 Jan 2009
gday1 = dart.date_to_gday(datetime.datetime(2009, 1, 1, 0, 0, 0))
obs_seq_no = int(gday - gday1 + 1)
if (obs_seq_no < 10):
mid = 'obs_000' + str(obs_seq_no)
else:
mid = 'obs_00' + str(obs_seq_no)
fname_truth = mid + '/' + 'True_State.nc'
fname = mid + '/' + E['diagn'] + '_Diag.nc'
if E['run_category'] == 'NCAR':
if E['exp_name'] == 'NCAR_LAONLY':
suffix = '_LAONLY'
else:
suffix = ''
fname_truth = '/' + 'True_State' + '_' + datestr + '-' + timestr + '.nc' + suffix
fname = '/' + E[
'diagn'] + '_Diag.' + datestr + '-' + timestr + '.nc' + suffix
if file_type == 'truth':
fname = fname_truth
data_dir_list = truth_dir_list
# if data_dir_list was not found, throw an error
if data_dir_list is None:
print(
'experiment_settings.py cannot find settings for the following experiment dict:'
)
print(E)
return None
#-----search for the right files
correct_filepath_found = False
for data_dir in data_dir_list:
filename = data_dir + fname
if debug:
print('Looking for file ' + filename)
if os.path.exists(filename):
correct_filepath_found = True
break
# return the file filename with path
return filename
