def test_gutils_ascii_to_netcdf_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'),
subset=False,
template='trajectory',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True)
# Wait 5 seconds for the watch to start
time.sleep(5)
# Make the ASCII we are watching for
merger = SlocumMerger(original_binary,
ascii_path,
globs=['*.tbd', '*.sbd'])
merger.convert()
wait_for_files(netcdf_path, 230)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def convert_to_ascii(self, event):
file_pairs = self.check_for_pair(event)
# Create a folder inside of the output directory for this glider folder name.
glider_folder_name = os.path.basename(event.path)
outputs_folder = os.path.join(self.outputs_path, glider_folder_name)
merger = SlocumMerger(
event.path,
outputs_folder,
cache_directory=event.path, # Default the cache directory to the data folder
globs=file_pairs
)
merger.convert()
def convert_to_ascii(self, event):
file_pairs = self.check_for_pair(event)
# Create a folder inside of the output directory for this glider folder name.
# Assuming the binary file is in [rt|delayed]/binary, we just go back and add ascii
binary_folder = Path(event.path)
outputs_folder = binary_folder.parent / 'ascii'
merger = SlocumMerger(
event.path,
outputs_folder,
cache_directory=event.
path, # Default the cache directory to the data folder
globs=file_pairs)
merger.convert()
def test_real_deployments(deployment):
setup_testing_logger(level=logging.WARNING)
binary_path = resource('slocum', deployment, 'rt', 'binary')
ascii_path = resource('slocum', deployment, 'rt', 'ascii')
netcdf_path = resource('slocum', deployment, 'rt', 'netcdf')
config_path = resource('slocum', deployment, 'config')
# Static args
args = dict(reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='ioos_ngdac',
profile_id_type=2,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
try:
merger = SlocumMerger(
binary_path,
ascii_path,
cache_directory=config_path,
)
for p in merger.convert():
args['file'] = p['ascii']
create_dataset(**args)
finally:
# Cleanup
shutil.rmtree(ascii_path, ignore_errors=True) # Remove generated ASCII
shutil.rmtree(netcdf_path,
ignore_errors=True) # Remove generated netCDF
# Remove any cached .cac files
for cac in glob(os.path.join(binary_path, '*.cac')):
os.remove(cac)
def test_convert_single_pair(self):
merger = SlocumMerger(
self.binary_path,
self.ascii_path,
globs=['usf-bass-2014-048-0-0.tbd', 'usf-bass-2014-048-0-0.sbd'])
p = merger.convert()
assert p == [{
'ascii':
os.path.join(self.ascii_path, 'usf_bass_2014_048_0_0_sbd.dat'),
'binary':
sorted([
os.path.join(self.binary_path, 'usf-bass-2014-048-0-0.sbd'),
os.path.join(self.binary_path, 'usf-bass-2014-048-0-0.tbd')
]),
}]
assert len(glob(os.path.join(self.ascii_path, '*.dat'))) == 1
af = p[0]['ascii']
sr = SlocumReader(af)
raw = sr.data
assert 'density' not in raw.columns
assert 'salinity' not in raw.columns
assert 't' not in raw.columns
assert 'x' not in raw.columns
assert 'y' not in raw.columns
assert 'z' not in raw.columns
enh = sr.standardize()
assert not enh['density'].any(
) # No GPS data so we can't compute density
assert 'salinity' in enh.columns
assert 't' in enh.columns
assert 'x' in enh.columns
assert 'y' in enh.columns
assert 'z' in enh.columns
def test_convert_default_cache_directory(self):
merger = SlocumMerger(self.binary_path,
self.ascii_path,
globs=['*.tbd', '*.sbd'])
p = merger.convert()
assert len(p) > 0
assert len(glob(os.path.join(self.ascii_path, '*.dat'))) > 0
def test_single_pair_existing_cac_files(self):
# The 0 files are there to produce the required .cac files
merger = SlocumMerger(
self.binary_path,
self.ascii_path,
cache_directory=self.cache_path,
globs=['modena-2015-175-0-9.dbd', 'modena-2015-175-0-9.ebd'])
p = merger.convert()
af = p[-1]['ascii']
sr = SlocumReader(af)
raw = sr.data
assert 'density' not in raw.columns
assert 'salinity' not in raw.columns
assert 't' not in raw.columns
assert 'x' not in raw.columns
assert 'y' not in raw.columns
assert 'z' not in raw.columns
enh = sr.standardize()
assert 'density' in enh.columns
assert 'salinity' in enh.columns
assert 't' in enh.columns
assert 'x' in enh.columns
assert 'y' in enh.columns
assert 'z' in enh.columns
def test_read_all_pairs_gps(self):
merger = SlocumMerger(
self.binary_path,
self.ascii_path,
globs=['*.tbd', '*.sbd']
)
p = merger.convert()
af = p[0]['ascii']
sr = SlocumReader(af)
raw = sr.data
assert 'density' not in raw.columns
assert 'salinity' not in raw.columns
assert 't' not in raw.columns
assert 'x' not in raw.columns
assert 'y' not in raw.columns
assert 'z' not in raw.columns
enh = sr.standardize()
assert 'density' in enh.columns
assert 'salinity' in enh.columns
assert 't' in enh.columns
assert 'x' in enh.columns
assert 'y' in enh.columns
assert 'z' in enh.columns
def test_convert_single_pair(self):
merger = SlocumMerger(
self.binary_path,
self.ascii_path,
globs=['usf-bass-2014-048-0-0.tbd', 'usf-bass-2014-048-0-0.sbd'])
p = merger.convert()
assert p == [{
'ascii':
os.path.join(self.ascii_path, 'usf_bass_2014_048_0_0_sbd.dat'),
'binary':
sorted([
os.path.join(self.binary_path, 'usf-bass-2014-048-0-0.sbd'),
os.path.join(self.binary_path, 'usf-bass-2014-048-0-0.tbd')
]),
}]
assert len(glob(os.path.join(self.ascii_path, '*.dat'))) == 1
def test_z_axis_method(self):
merger = SlocumMerger(self.binary_path,
self.ascii_path,
cache_directory=self.cache_path,
globs=['unit_507-2021-308*'])
_ = merger.convert()
dat_files = [
p for p in os.listdir(self.ascii_path) if p.endswith('.dat')
]
for ascii_file in dat_files:
args = dict(file=os.path.join(self.ascii_path, ascii_file),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='slocum_dac',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1,
z_axis_method=2)
create_dataset(**args)
assert os.path.exists(self.netcdf_path)
output_files = sorted(os.listdir(self.netcdf_path))
output_files = [
os.path.join(self.netcdf_path, o) for o in output_files
]
assert len(output_files) == 28
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1636072703
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1636146248
# Check netCDF file for compliance
ds = namedtuple('Arguments', ['file'])
for o in output_files:
assert check_dataset(ds(file=o)) == 0
def test_pseudogram(self):
merger = SlocumMerger(self.binary_path,
self.ascii_path,
cache_directory=self.cache_path,
globs=['*'])
_ = merger.convert()
dat_files = [
p for p in os.listdir(self.ascii_path) if p.endswith('.dat')
]
for ascii_file in dat_files:
args = dict(file=os.path.join(self.ascii_path, ascii_file),
reader_class=SlocumReader,
deployments_path=resource('slocum'),
subset=True,
template='slocum_dac',
profile_id_type=1,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1,
z_axis_method=1)
create_dataset(**args)
assert os.path.exists(self.netcdf_path)
output_files = sorted(os.listdir(self.netcdf_path))
output_files = [
os.path.join(self.netcdf_path, o) for o in output_files
]
assert len(output_files) == 17
# First profile
with nc4.Dataset(output_files[0]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1639020410
# Last profile
with nc4.Dataset(output_files[-1]) as ncd:
assert ncd.variables['profile_id'].ndim == 0
assert ncd.variables['profile_id'][0] == 1639069272
def test_real_deployments(deployment):
setup_testing_logger(level=logging.WARNING)
binary_path = resource('slocum', 'real', 'binary', deployment)
ascii_path = resource('slocum', 'real', 'ascii', deployment)
netcdf_path = resource('slocum', 'real', 'netcdf', deployment)
default_configs = resource('slocum', 'real', 'config', deployment)
# Config path is usually an env variable pointing to a configuration setup
all_config_path = os.environ.get('GUTILS_TEST_CONFIG_DIRECTORY',
default_configs)
config_path = os.path.join(all_config_path, deployment)
# Static args
args = dict(reader_class=SlocumReader,
config_path=config_path,
output_path=netcdf_path,
subset=True,
template='ioos_ngdac',
profile_id_type=2,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
try:
merger = SlocumMerger(binary_path, ascii_path)
for p in merger.convert():
args['file'] = p['ascii']
create_dataset(**args)
finally:
# Cleanup
shutil.rmtree(ascii_path, ignore_errors=True) # Remove generated ASCII
shutil.rmtree(netcdf_path,
ignore_errors=True) # Remove generated netCDF
# Remove any cached .cac files
for cac in glob(os.path.join(binary_path, '*.cac')):
os.remove(cac)
def test_read_single_pair(self):
merger = SlocumMerger(
self.binary_path,
self.ascii_path,
globs=['usf-bass-2014-048-0-0.tbd', 'usf-bass-2014-048-0-0.sbd'])
p = merger.convert()
af = p[0]['ascii']
sr = SlocumReader(af)
raw = sr.data
assert 'density' not in raw.columns
assert 'salinity' not in raw.columns
assert 't' not in raw.columns
assert 'x' not in raw.columns
assert 'y' not in raw.columns
assert 'z' not in raw.columns
enh = sr.standardize()
assert 'density' not in enh.columns # No GPS data so we can't compute density
assert 'salinity' in enh.columns
assert 't' in enh.columns
assert 'x' in enh.columns
assert 'y' in enh.columns
assert 'z' in enh.columns