def test_appending_variables(self):
datasets_path = resource('erddap', 'datasets.xml')
netcdf_files = [
resource('erddap', 'fakedeployment', 'rt', 'netcdf', '1.nc'), # Creates datasets.xml
resource('erddap', 'fakedeployment', 'rt', 'netcdf', '2.nc'), # Adds additional variables
resource('erddap', 'fakedeployment', 'rt', 'netcdf', '3.nc') # Should not remove any variables
]
for n in netcdf_files:
netcdf_to_erddap_dataset(
resource('erddap'),
datasets_path,
n,
None
)
xmltree = etree.parse(datasets_path).getroot()
find_dataset = etree.XPath("//erddapDatasets/dataset")
ds = find_dataset(xmltree)[0]
vs = [ d.findtext('sourceName') for d in ds.iter('dataVariable') ]
# Temperature is only in 2.nc and not 1.nc or 3.nc. Make sure
# it was carried through correctly
assert 'temperature' in vs
assert 'conductivity' in vs
assert 'salinity' in vs
assert 'density' in vs
def test_large_merge(self):
folder = resource('slocum', 'merge', 'large')
output = resource('slocum', 'merge', 'output', 'large.nc')
merge_profile_netcdf_files(folder, output)
with ContiguousRaggedTrajectoryProfile(output) as ncd:
assert ncd.is_valid()
def setUp(self):
super().setUp()
self.binary_path = resource('slocum', 'modena-20150625T0000',
'delayed', 'binary')
self.ascii_path = resource('slocum', 'modena-20150625T0000', 'delayed',
'ascii')
self.cache_path = resource('slocum', 'modena-20150625T0000', 'config')
def test_gutils_netcdf_to_erddap_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Netcdf2ErddapProcessor(
deployments_path=resource('slocum'),
erddap_content_path=erddap_content_path,
erddap_flag_path=erddap_flag_path
)
notifier = ThreadedNotifier(wm, processor, read_freq=5)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
netcdf_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
orig_netcdf = resource('profile.nc')
dummy_netcdf = os.path.join(netcdf_path, 'profile.nc')
shutil.copy(orig_netcdf, dummy_netcdf)
wait_for_files(erddap_content_path, 1)
wait_for_files(erddap_flag_path, 1)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def setUp(self):
super(TestSlocumMerger, self).setUp()
self.binary_path = resource('slocum', 'real', 'binary',
'bass-20150407T1300')
self.ascii_path = resource('slocum', 'real', 'ascii',
'bass-20150407T1300')
def test_real_deployments(deployment):
setup_testing_logger(level=logging.WARNING)
binary_path = resource('slocum', deployment, 'rt', 'binary')
ascii_path = resource('slocum', deployment, 'rt', 'ascii')
netcdf_path = resource('slocum', deployment, 'rt', 'netcdf')
config_path = resource('slocum', deployment, 'config')
# Static args
args = dict(reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='ioos_ngdac',
profile_id_type=2,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
try:
merger = SlocumMerger(
binary_path,
ascii_path,
cache_directory=config_path,
)
for p in merger.convert():
args['file'] = p['ascii']
create_dataset(**args)
finally:
# Cleanup
shutil.rmtree(ascii_path, ignore_errors=True) # Remove generated ASCII
shutil.rmtree(netcdf_path,
ignore_errors=True) # Remove generated netCDF
# Remove any cached .cac files
for cac in glob(os.path.join(binary_path, '*.cac')):
os.remove(cac)
def setUp(self):
super().setUp()
self.binary_path = resource('slocum', 'unit_507_one', 'delayed',
'binary')
self.ascii_path = resource('slocum', 'unit_507_one', 'delayed',
'ascii')
self.netcdf_path = resource('slocum', 'unit_507_one', 'delayed',
'netcdf')
self.cache_path = resource('slocum', 'unit_507_one', 'config')
def test_gutils_ascii_to_netcdf_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'),
subset=False,
template='trajectory',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True)
# Wait 5 seconds for the watch to start
time.sleep(5)
# Make the ASCII we are watching for
merger = SlocumMerger(original_binary,
ascii_path,
globs=['*.tbd', '*.sbd'])
merger.convert()
wait_for_files(netcdf_path, 230)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def test_gutils_binary_to_ascii_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert binary data to ASCII
processor = Slocum2AsciiProcessor(
deployments_path=resource('slocum'),
)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
binary_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
gpath = os.path.join(original_binary, '*.*bd')
# Sort the files so the .cac files are generated in the right order
for g in sorted(glob(gpath)):
shutil.copy2(g, binary_path)
wait_for_files(ascii_path, 32)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def tearDown(self):
outputs = [
resource('slocum', 'real', 'netcdf')
]
for d in outputs:
try:
shutil.rmtree(d)
except (IOError, OSError):
pass
def test_all_ascii(self):
out_base = resource('slocum', 'bass-test-ascii', 'rt', 'netcdf')
try:
for f in glob(resource('slocum', 'bass-test-ascii', 'rt', 'ascii', 'usf_bass*.dat')):
args = dict(
file=f,
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=False,
template='ioos_ngdac',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1
)
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [ os.path.join(out_base, o) for o in output_files ]
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 0
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == len(output_files) - 1
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
finally:
# Cleanup
shutil.rmtree(out_base)
def test_defaults(self):
out_base = resource('slocum', 'bass-test-ascii', 'rt', 'netcdf')
try:
args = dict(
file=resource('slocum', 'bass-test-ascii', 'rt', 'ascii', 'usf_bass_2016_253_0_6_sbd.dat'),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=False,
template='trajectory',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1
)
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [ os.path.join(out_base, o) for o in output_files ]
assert len(output_files) == 32
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1473499526
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1473509128
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
finally:
# Cleanup
shutil.rmtree(out_base)
def test_all_ascii(self):
out_base = resource('slocum', 'real', 'netcdf', 'bass-20160909T1733')
safe_makedirs(out_base)
for f in glob(resource('slocum', 'usf_bass*.dat')):
args = dict(
file=f,
reader_class=SlocumReader,
config_path=resource('slocum', 'config', 'bass-20160909T1733'),
output_path=out_base,
subset=False,
template='ioos_ngdac',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1
)
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [ os.path.join(out_base, o) for o in output_files ]
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 0
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == len(output_files) - 1
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
def test_parameter_filters_override_config(self):
out_base = resource('slocum', 'bass-test-filters-override', 'rt',
'netcdf')
try:
args = dict(file=resource('slocum', 'bass-test-filters-override',
'rt', 'ascii',
'usf_bass_2016_253_0_6_sbd.dat'),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='ioos_ngdac',
profile_id_type=1,
tsint=None,
filter_distance=None,
filter_points=None,
filter_time=None,
filter_z=32)
# This filters to a single profile
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [os.path.join(out_base, o) for o in output_files]
assert len(output_files) == 1
# Only profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1473507417
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
finally:
# Cleanup
shutil.rmtree(out_base)
def test_real_deployments(deployment):
setup_testing_logger(level=logging.WARNING)
binary_path = resource('slocum', 'real', 'binary', deployment)
ascii_path = resource('slocum', 'real', 'ascii', deployment)
netcdf_path = resource('slocum', 'real', 'netcdf', deployment)
default_configs = resource('slocum', 'real', 'config', deployment)
# Config path is usually an env variable pointing to a configuration setup
all_config_path = os.environ.get('GUTILS_TEST_CONFIG_DIRECTORY',
default_configs)
config_path = os.path.join(all_config_path, deployment)
# Static args
args = dict(reader_class=SlocumReader,
config_path=config_path,
output_path=netcdf_path,
subset=True,
template='ioos_ngdac',
profile_id_type=2,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
try:
merger = SlocumMerger(binary_path, ascii_path)
for p in merger.convert():
args['file'] = p['ascii']
create_dataset(**args)
finally:
# Cleanup
shutil.rmtree(ascii_path, ignore_errors=True) # Remove generated ASCII
shutil.rmtree(netcdf_path,
ignore_errors=True) # Remove generated netCDF
# Remove any cached .cac files
for cac in glob(os.path.join(binary_path, '*.cac')):
os.remove(cac)
def test_delayed(self):
out_base = resource('slocum', 'real', 'netcdf', 'modena-2015')
args = dict(
file=resource('slocum', 'modena_2015_175_0_9_dbd.dat'),
reader_class=SlocumReader,
config_path=resource('slocum', 'config', 'modena-2015'),
output_path=out_base,
subset=False,
template='trajectory',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1
)
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [ os.path.join(out_base, o) for o in output_files ]
assert len(output_files) == 6
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1435257435
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1435264145
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
def test_defaults(self):
out_base = resource('slocum', 'real', 'netcdf', 'bass-20160909T1733')
args = dict(
file=resource('slocum', 'usf_bass_2016_253_0_6_sbd.dat'),
reader_class=SlocumReader,
config_path=resource('slocum', 'config', 'bass-20160909T1733'),
output_path=out_base,
subset=False,
template='trajectory',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1
)
create_dataset(**args)
output_files = sorted(os.listdir(out_base))
output_files = [ os.path.join(out_base, o) for o in output_files ]
assert len(output_files) == 32
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1473499507
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1473509118
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
def test_z_axis_method(self):
merger = SlocumMerger(self.binary_path,
self.ascii_path,
cache_directory=self.cache_path,
globs=['unit_507-2021-308*'])
_ = merger.convert()
dat_files = [
p for p in os.listdir(self.ascii_path) if p.endswith('.dat')
]
for ascii_file in dat_files:
args = dict(file=os.path.join(self.ascii_path, ascii_file),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='slocum_dac',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1,
z_axis_method=2)
create_dataset(**args)
assert os.path.exists(self.netcdf_path)
output_files = sorted(os.listdir(self.netcdf_path))
output_files = [
os.path.join(self.netcdf_path, o) for o in output_files
]
assert len(output_files) == 28
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1636072703
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1636146248
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
def test_pseudogram(self):
merger = SlocumMerger(self.binary_path,
self.ascii_path,
cache_directory=self.cache_path,
globs=['*'])
_ = merger.convert()
dat_files = [
p for p in os.listdir(self.ascii_path) if p.endswith('.dat')
]
for ascii_file in dat_files:
args = dict(file=os.path.join(self.ascii_path, ascii_file),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='slocum_dac',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1,
z_axis_method=1)
create_dataset(**args)
assert os.path.exists(self.netcdf_path)
output_files = sorted(os.listdir(self.netcdf_path))
output_files = [
os.path.join(self.netcdf_path, o) for o in output_files
]
assert len(output_files) == 17
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1639020410
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1639069272
def setUp(self):
super(TestSlocumExportDelayed, self).setUp()
self.binary_path = resource('slocum', 'real', 'binary', 'modena-2015')
self.ascii_path = resource('slocum', 'real', 'ascii', 'modena-2015')
self.cache_path = os.path.join(self.binary_path, 'cac')
WatchManager
)
from gutils import safe_makedirs
from gutils.slocum import SlocumMerger
from gutils.watch.binary import Slocum2AsciiProcessor
from gutils.watch.ascii import Slocum2NetcdfProcessor
from gutils.watch.netcdf import Netcdf2ErddapProcessor
from gutils.tests import resource, output, GutilsTestClass
import logging
L = logging.getLogger(__name__) # noqa
deployment = 'bass-test-watch'
binary_path = resource('slocum', deployment, 'rt', 'binary')
ascii_path = resource('slocum', deployment, 'rt', 'ascii')
netcdf_path = resource('slocum', deployment, 'rt', 'netcdf')
original_binary = resource('slocum', 'bass-20160909T1733', 'rt', 'binary')
config_path = resource('slocum', deployment, 'config')
erddap_content_path = output('erddap', 'content')
erddap_flag_path = output('erddap', 'flag')
ftp_path = output('ftp')
def wait_for_files(path, number, loops=20, sleep=6):
# Wait for NetCDF to be created
count = 0
while True:
try:
def is_continuous(profiled_dataset):
last_profile_id = 0
for i, row in enumerate(profiled_dataset.itertuples()):
profile_diff = abs(last_profile_id - row.profile)
if profile_diff == 1:
last_profile_id = row.profile
elif profile_diff > 1:
print("Inconsistency @: %d, Last Profile: %d, Current: %d" %
(i, last_profile_id, row.profile))
return False
return True
ctd_filepath = resource('slocum', 'bass-test-ascii', 'rt', 'ascii',
'usf_bass_2016_253_0_6_sbd.dat')
class TestFindProfile(GutilsTestClass):
def setUp(self):
super(TestFindProfile, self).setUp()
sr = SlocumReader(ctd_filepath)
self.df = sr.standardize()
self.profiled_dataset = assign_profiles(self.df, tsint=10)
def test_find_profile(self):
assert len(self.profiled_dataset) != 0
assert len(self.profiled_dataset) == len(self.df)
assert len(self.profiled_dataset.profile.dropna().unique()) == 32
from glob import glob
from pyinotify import (IN_CLOSE_WRITE, IN_MOVED_TO, ThreadedNotifier,
WatchManager)
from gutils import safe_makedirs
from gutils.slocum import SlocumMerger
from gutils.watch.binary import Slocum2AsciiProcessor
from gutils.watch.ascii import Slocum2NetcdfProcessor
from gutils.watch.netcdf import Netcdf2ErddapProcessor
from gutils.tests import resource, output, GutilsTestClass
import logging
L = logging.getLogger(__name__) # noqa
config_path = resource('slocum', 'config', 'bass-20160909T1733')
original_binary = resource('slocum', 'real', 'binary', 'bass-20160909T1733')
binary_path = output('binary', 'bass-20160909T1733')
ascii_path = output('ascii', 'bass-20160909T1733')
netcdf_path = output('netcdf', 'bass-20160909T1733')
erddap_content_path = output('erddap', 'content')
erddap_flag_path = output('erddap', 'flag')
ftp_path = output('ftp')
def wait_for_files(path, number):
# Wait for NetCDF to be created
count = 0
loops = 20
while True:
try:
def setUp(self):
super().setUp()
self.binary_path = resource('slocum', 'bass-20150407T1300', 'rt',
'binary')
self.ascii_path = resource('slocum', 'bass-20150407T1300', 'rt',
'ascii')
def setUp(self):
super().setUp()
self.binary_path = resource('slocum', 'ecometrics3', 'rt', 'binary')
self.ascii_path = resource('slocum', 'ecometrics3', 'rt', 'ascii')
self.netcdf_path = resource('slocum', 'ecometrics3', 'rt', 'netcdf')
self.cache_path = resource('slocum', 'ecometrics3', 'config')
def setUp(self):
super().setUp()
self.binary_path = resource('slocum', 'bass-20160909T1733', 'rt',
'binary')
self.ascii_path = resource('slocum', 'bass-20160909T1733', 'rt',
'ascii')
def test_failing_testing_compliance(self):
args = self.args(file=resource('should_fail.nc'))
assert check_dataset(args) == 1
def setUp(self):
super(TestSlocumReaderWithGPS, self).setUp()
self.binary_path = resource('slocum', 'real', 'binary',
'bass-20160909T1733')
self.ascii_path = resource('slocum', 'real', 'ascii',
'bass-20160909T1733')
