def test_different_dataset_temporal_overlap(self):
new_times = np.array(
[datetime.datetime(2002, x, 1) for x in range(1, 13)])
another_dataset = Dataset(self.lat, self.lon, new_times,
self.value, self.variable)
self.dataset_array = [self.test_dataset, another_dataset]
maximum, minimum = utils.get_temporal_overlap(self.dataset_array)
self.assertEqual(maximum, datetime.datetime(2002, 1, 1))
self.assertEqual(minimum, datetime.datetime(2000, 12, 1))
obs_datasets[i] = dsp.normalize_dataset_datetimes(
dataset, temporal_resolution)
if multiplying_factor[i] != 1:
obs_dataset.values *= multiplying_factor[i]
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """
model_datasets = load_datasets_from_config(extra_opts, *model_data_info)
model_names = [dataset.name for dataset in model_datasets]
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
for i, dataset in enumerate(model_datasets):
model_datasets[i] = dsp.normalize_dataset_datetimes(
dataset, temporal_resolution)
""" Step 3: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if time_info['maximum_overlap_period']:
start_time, end_time = utils.get_temporal_overlap(obs_datasets +
model_datasets)
print 'Maximum overlap period'
print 'start_time:', start_time
print 'end_time:', end_time
if temporal_resolution == 'monthly' and end_time.day != 1:
end_time = end_time.replace(day=1)
for i, dataset in enumerate(obs_datasets):
min_lat = np.max([min_lat, dataset.lats.min()])
max_lat = np.min([max_lat, dataset.lats.max()])
min_lon = np.max([min_lon, dataset.lons.min()])
max_lon = np.min([max_lon, dataset.lons.max()])
if not 'boundary_type' in space_info:
bounds = Bounds(lat_min=min_lat,
md = esgf.load_dataset(dataset_id=model_data_info['dataset_id'],
variable=model_data_info['variable'],
esgf_username=username,
esgf_password=password)
model_datasets, model_names = md[0]
else:
print ' '
# TO DO: support RCMED
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
for idata,dataset in enumerate(model_datasets):
model_datasets[idata] = dsp.normalize_dataset_datetimes(dataset, temporal_resolution)
""" Step 3: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if time_info['maximum_overlap_period']:
start_time, end_time = utils.get_temporal_overlap([ref_dataset]+model_datasets)
print 'Maximum overlap period'
print 'start_time:', start_time
print 'end_time:', end_time
if temporal_resolution == 'monthly' and end_time.day !=1:
end_time = end_time.replace(day=1)
if ref_data_info['data_source'] == 'rcmed':
min_lat = np.max([min_lat, ref_dataset.lats.min()])
max_lat = np.min([max_lat, ref_dataset.lats.max()])
min_lon = np.max([min_lon, ref_dataset.lons.min()])
max_lon = np.min([max_lon, ref_dataset.lons.max()])
bounds = Bounds(min_lat, max_lat, min_lon, max_lon, start_time, end_time)
ref_dataset = dsp.subset(bounds,ref_dataset)
if ref_dataset.temporal_resolution() != temporal_resolution:
print('Loading datasets:\n{}'.format(data_info))
datasets = load_datasets_from_config(extra_opts, *data_info)
multiplying_factor = np.ones(len(datasets))
multiplying_factor[0] = fact
names = [dataset.name for dataset in datasets]
for i, dataset in enumerate(datasets):
res = dataset.temporal_resolution()
if res == 'daily' or res == 'monthly':
datasets[i] = dsp.normalize_dataset_datetimes(dataset, res)
if multiplying_factor[i] != 1:
datasets[i].values *= multiplying_factor[i]
""" Step 2: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if maximum_overlap_period:
start_time, end_time = utils.get_temporal_overlap(datasets)
print('Maximum overlap period')
print('start_time: {}'.format(start_time))
print('end_time: {}'.format(end_time))
if temporal_resolution == 'monthly' and end_time.day !=1:
end_time = end_time.replace(day=1)
for i, dataset in enumerate(datasets):
min_lat = np.max([min_lat, dataset.lats.min()])
max_lat = np.min([max_lat, dataset.lats.max()])
min_lon = np.max([min_lon, dataset.lons.min()])
max_lon = np.min([max_lon, dataset.lons.max()])
if not 'boundary_type' in space_info:
bounds = Bounds(lat_min=min_lat,
def test_same_dataset_temporal_overlap(self):
maximum, minimum = utils.get_temporal_overlap(self.dataset_array)
self.assertEqual(maximum, datetime.datetime(2000, 1, 1))
self.assertEqual(minimum, datetime.datetime(2000, 12, 1))
model_data_info = config['datasets']['targets']
print 'Loading model datasets:\n', model_data_info
if model_data_info['data_source'] == 'local':
model_datasets, model_names = local.load_multiple_files(
file_path=model_data_info['path'],
variable_name=model_data_info['variable'])
else:
print ' '
# TO DO: support RCMED and ESGF
for idata, dataset in enumerate(model_datasets):
model_datasets[idata] = dsp.normalize_dataset_datetimes(
dataset, temporal_resolution)
""" Step 3: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if time_info['maximum_overlap_period']:
start_time, end_time = utils.get_temporal_overlap([ref_dataset] +
model_datasets)
print 'Maximum overlap period'
print 'start_time:', start_time
print 'end_time:', end_time
if temporal_resolution == 'monthly' and end_time.day != 1:
end_time = end_time.replace(day=1)
if ref_data_info['data_source'] == 'rcmed':
min_lat = np.max([min_lat, ref_dataset.lats.min()])
max_lat = np.min([max_lat, ref_dataset.lats.max()])
min_lon = np.max([min_lon, ref_dataset.lons.min()])
max_lon = np.min([max_lon, ref_dataset.lons.max()])
bounds = Bounds(min_lat, max_lat, min_lon, max_lon, start_time, end_time)
if ref_dataset.lats.ndim != 2 and ref_dataset.lons.ndim != 2:
ref_dataset = dsp.subset(bounds, ref_dataset)
""" Step 1: Load the datasets """
print('Loading datasets:\n{}'.format(data_info))
datasets = load_datasets_from_config(extra_opts, *data_info)
multiplying_factor = np.ones(len(datasets))
multiplying_factor[0] = fact
names = [dataset.name for dataset in datasets]
for i, dataset in enumerate(datasets):
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
datasets[i] = dsp.normalize_dataset_datetimes(dataset,
temporal_resolution)
if multiplying_factor[i] != 1:
datasets[i].values *= multiplying_factor[i]
""" Step 2: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if maximum_overlap_period:
start_time, end_time = utils.get_temporal_overlap(datasets)
print('Maximum overlap period')
print('start_time: {}'.format(start_time))
print('end_time: {}'.format(end_time))
if temporal_resolution == 'monthly' and end_time.day != 1:
end_time = end_time.replace(day=1)
for i, dataset in enumerate(datasets):
min_lat = np.max([min_lat, dataset.lats.min()])
max_lat = np.min([max_lat, dataset.lats.max()])
min_lon = np.max([min_lon, dataset.lons.min()])
max_lon = np.min([max_lon, dataset.lons.max()])
if not 'boundary_type' in space_info:
bounds = Bounds(lat_min=min_lat,