def _load_dataset(dataset_config_data):
""""""
if dataset_config_data['data_source'] == 'local':
if dataset_config_data['file_count'] > 1:
logger.error(
'Multi-file datasets are currently not supported. Cancelling load '
'of the following dataset: {}'.format(dataset_config_data))
return None
return local.load_file(dataset_config_data['path'],
dataset_config_data['variable'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'rcmed':
return rcmed.parameter_dataset(
dataset_config_data['dataset_id'],
dataset_config_data['parameter_id'],
dataset_config_data['min_lat'], dataset_config_data['max_lat'],
dataset_config_data['min_lon'], dataset_config_data['min_lon'],
dataset_config_data['start_time'], dataset_config_data['end_time'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'esgf':
return esgf.load_dataset(
dataset_config_data['dataset_id'], dataset_config_data['variable'],
dataset_config_data['esgf_username'],
dataset_config_data['esgf_password'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'dap':
return dap.load(dataset_config_data['url'],
dataset_config_data['variable'],
**dataset_config_data('optional_args', {}))
def _load_dataset(dataset_config_data):
""""""
if dataset_config_data['data_source'] == 'local':
if dataset_config_data['file_count'] > 1:
logger.error(
'Multi-file datasets are currently not supported. Cancelling load '
'of the following dataset: {}'.format(dataset_config_data)
)
return None
return local.load_file(dataset_config_data['path'],
dataset_config_data['variable'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'rcmed':
return rcmed.parameter_dataset(dataset_config_data['dataset_id'],
dataset_config_data['parameter_id'],
dataset_config_data['min_lat'],
dataset_config_data['max_lat'],
dataset_config_data['min_lon'],
dataset_config_data['min_lon'],
dataset_config_data['start_time'],
dataset_config_data['end_time'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'esgf':
return esgf.load_dataset(dataset_config_data['dataset_id'],
dataset_config_data['variable'],
dataset_config_data['esgf_username'],
dataset_config_data['esgf_password'],
**dataset_config_data.get('optional_args', {}))
elif dataset_config_data['data_source'] == 'dap':
return dap.load(dataset_config_data['url'],
dataset_config_data['variable'],
**dataset_config_data('optional_args', {}))
def main():
"""
An example of using the OCW ESGF library. Connects to an ESGF
server and downloads a dataset.
"""
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
dataset_id = 'obs4mips.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov'
variable = 'zosStderr'
if sys.version_info[0] >= 3:
username = input('Enter your ESGF OpenID:\n')
else:
username = raw_input('Enter your ESGF OpenID:\n')
password = getpass(prompt='Enter your ESGF Password:\n')
# Multiple datasets are returned in a list if the ESGF dataset is
# divided into multiple files.
datasets = esgf.load_dataset(dataset_id, variable, username, password)
# For this example, our dataset is only stored in a single file so
# we only need to look at the 0-th value in the returned list.
dataset = datasets[0]
print('\n--------\n')
print('Variable: ', dataset.variable)
print('Shape: ', dataset.values.shape)
print('A Value: ', dataset.values[100][100][100])
# under the License.
import ocw.data_source.esgf as esgf
from getpass import getpass
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
dataset_id = 'obs4MIPs.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov'
variable = 'zosStderr'
username = raw_input('Enter your ESGF OpenID:\n')
password = getpass(prompt='Enter your ESGF Password:\n')
# Multiple datasets are returned in a list if the ESGF dataset is
# divided into multiple files.
datasets = esgf.load_dataset(dataset_id,
variable,
username,
password)
# For this example, our dataset is only stored in a single file so
# we only need to look at the 0-th value in the returned list.
ds = datasets[0]
print '\n--------\n'
print 'Variable: ', ds.variable
print 'Shape: ', ds.values.shape
print 'A Value: ', ds.values[100][100][100]
print 'Loading observation dataset:\n',ref_data_info
ref_name = ref_data_info['data_name']
if ref_data_info['data_source'] == 'local':
ref_dataset = local.load_file(ref_data_info['path'],
ref_data_info['variable'], name=ref_name,
lat_name=ref_lat_name, lon_name=ref_lon_name)
elif ref_data_info['data_source'] == 'rcmed':
ref_dataset = rcmed.parameter_dataset(ref_data_info['dataset_id'],
ref_data_info['parameter_id'],
min_lat, max_lat, min_lon, max_lon,
start_time, end_time)
elif ref_data_info['data_source'] == 'ESGF':
username=raw_input('Enter your ESGF OpenID:\n')
password=raw_input('Enter your ESGF password:\n')
ds = esgf.load_dataset(dataset_id = ref_data_info['dataset_id'],
variable = ref_data_info['variable'],
esgf_username=username,
esgf_password=password)
ref_dataset = ds[0]
else:
print ' '
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, temporal_resolution)
if 'multiplying_factor' in ref_data_info.keys():
ref_dataset.values = ref_dataset.values*ref_data_info['multiplying_factor']
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """
model_data_info = config['datasets']['targets']
model_lat_name = None
model_lon_name = None
if 'latitude_name' in model_data_info.keys():
model_lat_name = model_data_info['latitude_name']
import ocw.data_source.esgf as esgf
from getpass import getpass
import ssl
import sys
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
dataset_id = 'obs4mips.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov'
variable = 'zosStderr'
if sys.version_info[0] >= 3:
username = input('Enter your ESGF OpenID:\n')
else:
username = raw_input('Enter your ESGF OpenID:\n')
password = getpass(prompt='Enter your ESGF Password:\n')
# Multiple datasets are returned in a list if the ESGF dataset is
# divided into multiple files.
datasets = esgf.load_dataset(dataset_id, variable, username, password)
# For this example, our dataset is only stored in a single file so
# we only need to look at the 0-th value in the returned list.
ds = datasets[0]
print('\n--------\n')
print('Variable: ', ds.variable)
print('Shape: ', ds.values.shape)
print('A Value: ', ds.values[100][100][100])
if 'longitude_name' in ref_data_info.keys():
ref_lon_name = ref_data_info['longitude_name']
print 'Loading observation dataset:\n',ref_data_info
ref_name = ref_data_info['data_name']
if ref_data_info['data_source'] == 'local':
ref_dataset = local.load_file(ref_data_info['path'],
ref_data_info['variable'], name=ref_name,
lat_name=ref_lat_name, lon_name=ref_lon_name)
elif ref_data_info['data_source'] == 'rcmed':
ref_dataset = rcmed.parameter_dataset(ref_data_info['dataset_id'],
ref_data_info['parameter_id'],
min_lat, max_lat, min_lon, max_lon,
start_time, end_time)
elif ref_data_info['data_source'] == 'ESGF':
ds = esgf.load_dataset(dataset_id = ref_data_info['dataset_id'],
variable = ref_data_info['variable'],
esgf_username=username,
esgf_password=password)
ref_dataset = ds[0]
else:
print ' '
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, temporal_resolution)
if 'multiplying_factor' in ref_data_info.keys():
ref_dataset.values = ref_dataset.values*ref_data_info['multiplying_factor']
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """
model_lat_name = None
model_lon_name = None
if 'latitude_name' in model_data_info.keys():
model_lat_name = model_data_info['latitude_name']
if 'longitude_name' in model_data_info.keys():