def test_unlim_loc_file_writing(self):
with nc.IndexedRaggedTs(self.testfilename, mode='w') as dataset:
for n_data in [2, 5, 6]:
for location in [1, 2, 3]:
data = {'test': np.arange(n_data)}
base = datetime(2007, 1, n_data)
dates = np.array(
[base + timedelta(hours=i) for i in range(n_data)])
dataset.write_ts(location,
data,
dates,
loc_descr='first station',
lon=location,
lat=location,
alt=location)
with nc.IndexedRaggedTs(self.testfilename) as dataset:
data = dataset.read_all_ts(1)
nptest.assert_array_equal(
data['test'],
np.concatenate([np.arange(2),
np.arange(5),
np.arange(6)]))
test_dates = []
for n_data in [2, 5, 6]:
base = datetime(2007, 1, n_data)
test_dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(test_dates)
nptest.assert_array_equal(data['time'], dates)
def test_file_writing_multiple_points_at_once(self):
"""
Write multiple points at once. This means that we can have multiple
locations more than once. Dates and data must be in the same order as
locations. This mean we only need to translate from locations to index
and can write the data as is.
"""
with nc.IndexedRaggedTs(self.testfilename, n_loc=3,
mode='w') as dataset:
locations = np.array([1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3])
data = {
'test':
np.concatenate([np.arange(2),
np.arange(5),
np.arange(6)])
}
dates = []
for n_data in [2, 5, 6]:
base = datetime(2007, 1, n_data)
dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(dates)
dataset.write_ts(locations,
data,
dates,
loc_descr=['first station'] * 13,
lon=locations,
lat=locations,
alt=locations)
with nc.IndexedRaggedTs(self.testfilename) as dataset:
for gpis, n_data in zip([1, 2, 3], [2, 5, 6]):
data = dataset.read_all_ts(gpis)
nptest.assert_array_equal(data['test'], np.arange(n_data))
test_dates = []
base = datetime(2007, 1, n_data)
test_dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(test_dates)
nptest.assert_array_equal(data['time'], dates)
def test_file_writing_with_attributes(self):
with nc.IndexedRaggedTs(self.testfilename, n_loc=3,
mode='w') as dataset:
for n_data in [2, 5, 6]:
for location in [1, 2, 3]:
data = {'test': np.arange(n_data)}
base = datetime(2007, 1, n_data)
dates = np.array(
[base + timedelta(hours=i) for i in range(n_data)])
dataset.write_ts(
location,
data,
dates,
loc_descr='first station',
lon=0,
lat=0,
alt=5,
attributes={'testattribute': 'teststring'})
with nc.IndexedRaggedTs(self.testfilename) as dataset:
data = dataset.read_all_ts(1)
assert dataset.dataset.variables[
'test'].testattribute == 'teststring'
nptest.assert_array_equal(
data['test'],
np.concatenate([np.arange(2),
np.arange(5),
np.arange(6)]))
test_dates = []
for n_data in [2, 5, 6]:
base = datetime(2007, 1, n_data)
test_dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(test_dates)
nptest.assert_array_equal(data['time'], dates)
def test_file_writing_multiple_points_at_once_two_steps_recarray_input(
self):
"""
Write multiple points at once. Add more during an append. Use record arrays as input.
"""
with nc.IndexedRaggedTs(self.testfilename, n_loc=4,
mode='w') as dataset:
locations = np.array([1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3])
data = {
'test':
np.concatenate([np.arange(2),
np.arange(5),
np.arange(6)])
}
dates = []
for n_data in [2, 5, 6]:
base = datetime(2007, 1, n_data)
dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(dates)
data = np.array(data['test'],
dtype={
'names': ['test'],
'formats': ['i8']
})
dataset.write_ts(locations,
data,
dates,
loc_descr=['first station'] * 13,
lon=locations,
lat=locations,
alt=locations)
with nc.IndexedRaggedTs(self.testfilename, n_loc=4,
mode='a') as dataset:
locations = np.array([1, 1, 4, 4])
data = {'test': np.concatenate([np.arange(2), np.arange(2)])}
dates = []
for n_data in [2, 2]:
base = datetime(2007, 2, n_data)
dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
dates = np.concatenate(dates)
data = np.array(data['test'],
dtype={
'names': ['test'],
'formats': ['i8']
})
dataset.write_ts(locations,
data,
dates,
loc_descr=['first station'] * 4,
lon=locations,
lat=locations,
alt=locations)
with nc.IndexedRaggedTs(self.testfilename) as dataset:
for gpis, n_data, base_month in zip([1, 2, 3, 4], [2, 5, 6, 2],
[1, 1, 1, 2]):
data = dataset.read_all_ts(gpis)
if gpis == 1:
nptest.assert_array_equal(
data['test'],
np.concatenate([np.arange(n_data),
np.arange(n_data)]))
else:
nptest.assert_array_equal(data['test'], np.arange(n_data))
test_dates = []
base = datetime(2007, base_month, n_data)
test_dates.append(
np.array(
[base + timedelta(hours=i) for i in range(n_data)]))
if gpis == 1:
base = datetime(2007, 2, n_data)
test_dates.append(
np.array([
base + timedelta(hours=i) for i in range(n_data)
]))
dates = np.concatenate(test_dates)
nptest.assert_array_equal(data['time'], dates)
def calc(self):
"""
go through all images and retrieve a stack of them
then go through all grid points in cell order and write to netCDF file
"""
# save grid information in file
grid2nc.save_grid(os.path.join(self.outputpath, self.gridname),
self.target_grid)
for img_stack_dict, start, end, dates, jd_stack in self.img_bulk():
#==================================================================
start_time = datetime.now()
for cell in self.target_grid.get_cells():
cell_gpis, cell_lons, cell_lats = self.target_grid.grid_points_for_cell(
cell)
# look where in the subset the data is
cell_index = np.where(
cell == self.target_grid.activearrcell)[0]
if cell_index.size == 0:
raise Img2TsError('cell not found in grid subset')
data = {}
for key in img_stack_dict:
# rename variable in output dataset
if self.variable_rename is None:
var_new_name = str(key)
else:
var_new_name = self.variable_rename[key]
output_array = np.swapaxes(
img_stack_dict[key][:, cell_index], 0, 1)
# change dtypes of output time series
if self.ts_dtypes is not None:
if type(self.ts_dtypes) == dict:
output_dtype = self.ts_dtypes[key]
else:
output_dtype = self.ts_dtypes
output_array = output_array.astype(output_dtype)
data[var_new_name] = output_array
if self.orthogonal:
with nc.OrthoMultiTs(
os.path.join(self.outputpath,
self.filename_templ % cell),
n_loc=cell_gpis.size,
mode='a',
zlib=self.zlib,
unlim_chunksize=self.unlim_chunksize,
time_units=self.time_units) as dataout:
# add global attributes to file
if self.global_attr is not None:
for attr in self.global_attr:
dataout.add_global_attr(
attr, self.global_attr[attr])
dataout.add_global_attr('geospatial_lat_min',
np.min(cell_lats))
dataout.add_global_attr('geospatial_lat_max',
np.max(cell_lats))
dataout.add_global_attr('geospatial_lon_min',
np.min(cell_lons))
dataout.add_global_attr('geospatial_lon_max',
np.max(cell_lons))
dataout.write_ts_all_loc(cell_gpis,
data,
dates,
lons=cell_lons,
lats=cell_lats,
attributes=self.ts_attributes)
elif not self.orthogonal:
with nc.IndexedRaggedTs(
os.path.join(self.outputpath,
self.filename_templ % cell),
n_loc=cell_gpis.size,
mode='a',
zlib=self.zlib,
unlim_chunksize=self.unlim_chunksize,
time_units=self.non_ortho_time_units) as dataout:
# add global attributes to file
if self.global_attr is not None:
for attr in self.global_attr:
dataout.add_global_attr(
attr, self.global_attr[attr])
dataout.add_global_attr('geospatial_lat_min',
np.min(cell_lats))
dataout.add_global_attr('geospatial_lat_max',
np.max(cell_lats))
dataout.add_global_attr('geospatial_lon_min',
np.min(cell_lons))
dataout.add_global_attr('geospatial_lon_max',
np.max(cell_lons))
# for this dataset we have to loop through the gpis since each time series
# can be different in length
for i, (gpi, gpi_lon, gpi_lat) in enumerate(
zip(cell_gpis, cell_lons, cell_lats)):
gpi_data = {}
# convert to modified julian date
gpi_jd = jd_stack[:, cell_index[i]] - 2400000.5
# remove measurements that were filled with the fill value
# during resampling
# doing this on the basis of the time variable should
# be enought since without time -> no valid
# observations
if self.resample:
if self.r_fill_values is not None:
if type(self.r_fill_values) == dict:
time_fill_value = self.r_fill_values[
self.time_var]
else:
time_fill_value = self.r_fill_values
valid_mask = gpi_jd != time_fill_value
else:
valid_mask = np.invert(gpi_jd.mask)
gpi_jd = gpi_jd[valid_mask]
else:
# all are valid if no resampling took place
valid_mask = slice(None, None, None)
for key in data:
gpi_data[key] = data[key][i, valid_mask]
if gpi_jd.data.size > 0:
dataout.write_ts(gpi,
gpi_data,
gpi_jd,
lon=gpi_lon,
lat=gpi_lat,
attributes=self.ts_attributes,
dates_direct=True)
data = {}
output_array = None
logging.log(logging.INFO, datetime.now() - start_time)