def test_imt_multi_not_string(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-multiple-nonstring.nc')
CFDataset.load(filepath).close()
with IncompleteMultidimensionalTrajectory(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False)
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile) as result_ncd:
assert 'trajectory' in result_ncd.dimensions
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True) as result_ncd:
# Could not reduce dims since there was more than one trajectory
assert 'trajectory' not in result_ncd.dimensions
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, unlimited=True) as result_ncd:
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd:
# Could not reduce dims since there was more than one trajectory
assert 'trajectory' not in result_ncd.dimensions
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_serialize_and_reload_data(self):
ncfile = os.path.join(os.path.dirname(__file__), "resources/qc-month.nc")
with CFDataset(ncfile) as cfncd:
# Data from netCDF variable
ncdata = cfncd.variables['data1'][:]
# Not filled
meta = cfncd.json(return_data=True, fill_data=False)
jsdata = meta['variables']['data1']['data']
npt.assert_array_equal(ncdata, jsdata)
with tempfile.NamedTemporaryFile() as f:
with CFDataset(f.name, 'w') as newcf:
newcf.apply_json(meta)
with CFDataset(f.name, 'r') as rcf:
newncdata = rcf.variables['data1'][:]
npt.assert_array_equal(ncdata, newncdata)
# Filled
meta = cfncd.json(return_data=True, fill_data=True)
jsdata = meta['variables']['data1']['data']
npt.assert_array_equal(ncdata, jsdata)
with tempfile.NamedTemporaryFile() as f:
with CFDataset(f.name, 'w') as newcf:
newcf.apply_json(meta)
with CFDataset(f.name, 'r') as rcf:
newncdata = rcf.variables['data1'][:]
npt.assert_array_equal(ncdata, newncdata)
def test_omp_dataframe_multi_unique_dims(self):
CFDataset.load(self.multi)
fid, multi_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalProfile(self.multi) as ncd:
df = ncd.to_dataframe()
with self.assertRaises(NotImplementedError):
with OrthogonalMultidimensionalProfile.from_dataframe(df, multi_tmp, unique_dims=True) as result_ncd:
assert 'profile_dim' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalProfile, multi_tmp) # Try to load it again
os.close(fid)
os.remove(multi_tmp)
def test_is_mine(klass, fp):
with CFDataset.load(fp) as dsg:
assert dsg.__class__ == klass
allsubs = list(all_subclasses(CFDataset))
subs = [s for s in allsubs if s != klass]
with CFDataset(fp) as dsg:
logger.debug('\nTesting {}'.format(klass.__name__))
assert klass.is_mine(dsg) is True
for s in subs:
if hasattr(s, 'is_mine'):
logger.debug(' * Trying {}...'.format(s.__name__))
assert s.is_mine(dsg) is False
def load_netcdf(ipt: Path) -> pd.DataFrame:
ds = CFDataset.load(str(ipt))
axes = dict(t='time',
z='z',
x='lon',
y='lat',
profile='profile',
trajectory='trajectory')
meta = ds.meta()['attributes']
valid_meta = {
'uuid': meta.get('uuid')['data'],
'ptt': meta.get('ptt', '')['data']
}
df = ds.to_dataframe(axes=axes, clean_cols=False, clean_rows=False)
try:
df.time.astype(int)
except TypeError:
if isinstance(df.time.iloc[0], cftime.datetime):
df['time'] = df.time.apply(lambda x: x._to_real_datetime())
else:
raise ValueError(
"Could not find a pandas compatible 'time' column")
return (df, valid_meta)
def test_load_strict(self):
ncfile = os.path.join(os.path.dirname(__file__), 'dsg', 'profile', 'resources', 'om-single.nc')
ncd = CFDataset.load(ncfile)
assert omt.is_mine(ncd) is False
with self.assertRaises(BaseException):
omt.is_mine(ncd, strict=True)
ncd.close()
def test_serialize_and_reload_data(self):
ncfile = os.path.join(os.path.dirname(__file__),
"resources/qc-month.nc")
with CFDataset(ncfile) as cfncd:
# Data from netCDF variable
ncdata = cfncd.variables['data1'][:]
# Not filled
meta = cfncd.json(return_data=True, fill_data=False)
jsdata = meta['variables']['data1']['data']
npt.assert_array_equal(ncdata, jsdata)
fhandle1, fname1 = tempfile.mkstemp()
with CFDataset(fname1, 'w') as newcf:
newcf.apply_json(meta)
with CFDataset(fname1, 'r') as rcf:
newncdata = rcf.variables['data1'][:]
npt.assert_array_equal(ncdata, newncdata)
os.close(fhandle1)
os.remove(fname1)
# Filled
meta = cfncd.json(return_data=True, fill_data=True)
jsdata = meta['variables']['data1']['data']
npt.assert_array_equal(ncdata, jsdata)
fhandle2, fname2 = tempfile.mkstemp()
with CFDataset(fname2, 'w') as newcf:
newcf.apply_json(meta)
with CFDataset(fname2, 'r') as rcf:
newncdata = rcf.variables['data1'][:]
npt.assert_array_equal(ncdata, newncdata)
os.close(fhandle2)
os.remove(fname2)
def test_wrap_dateline(self):
ncfile = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"resources/wrapping_dateline.nc")
with CFDataset.load(ncfile) as ncd:
axes = {
't': 'time',
'z': 'z',
'x': 'lon',
'y': 'lat',
}
df = ncd.to_dataframe(axes=axes)
meta = utils.get_geographic_attributes(df, axes=axes)
assert meta == {
"variables": {
"lat": {
"attributes": {
"actual_min": 61.777,
"actual_max": 67.068
}
},
"lon": {
"attributes": {
"actual_min": -179.966,
"actual_max": 179.858
}
}
},
"attributes": {
"geospatial_lat_min": 61.777,
"geospatial_lat_max": 67.068,
"geospatial_lon_min": -179.966,
"geospatial_lon_max": 179.858,
"geospatial_bbox":
"POLYGON ((198.669 61.777, 198.669 67.068, 174.792 67.068, 174.792 61.777, 198.669 61.777))",
'geospatial_bounds':
"POLYGON ((174.792 61.777, 174.926 62.206, 178.838667737 64.055605136, 178.916 64.084, 179.858 64.311, 192.86 67.029, 196.86 67.068, 198.669 66.861, 187.753767857 64.334204193, 179.195 62.395, 176.169 61.862, 174.792 61.777))",
"geospatial_bounds_crs": "EPSG:4326"
}
}
def test_load_url(self):
ncd = CFDataset.load('http://geoport.whoi.edu/thredds/dodsC/usgs/data2/emontgomery/stellwagen/CF-1.6/ARGO_MERCHANT/1211-AA.cdf')
assert omt.is_mine(ncd) is True
ncd.close()