def init_netcdf(file_path, attrs, segment_id, profile_id):
# Check if the output path already exists, remove old file
mode = 'w'
if os.path.isfile(file_path):
os.remove(file_path)
with open_glider_netcdf(file_path, mode) as glider_nc:
# Set global attributes
glider_nc.set_global_attributes(attrs['global'])
# Set Trajectory
glider_nc.set_trajectory_id(attrs['deployment']['glider'],
attrs['deployment']['trajectory_date'])
# Set Platform
glider_nc.set_platform(attrs['deployment']['platform'])
# Set Instruments
glider_nc.set_instruments(attrs['instruments'])
# Set Segment ID
glider_nc.set_segment_id(segment_id)
# Set Profile ID
glider_nc.set_profile_id(profile_id)
def init_netcdf(file_path, attrs, segment_id, profile_id):
# Check if the output path already exists, remove old file
mode = 'w'
if os.path.isfile(file_path):
os.remove(file_path)
with open_glider_netcdf(file_path, mode) as glider_nc:
# Set global attributes
glider_nc.set_global_attributes(attrs['global'])
# Set Trajectory
glider_nc.set_trajectory_id(
attrs['deployment']['glider'],
attrs['deployment']['trajectory_date']
)
# Set Platform
glider_nc.set_platform(attrs['deployment']['platform'])
# Set Instruments
glider_nc.set_instruments(attrs['instruments'])
# Set Segment ID
glider_nc.set_segment_id(segment_id)
# Set Profile ID
glider_nc.set_profile_id(profile_id)
def process_dataset(args, attrs):
flight_path = args.flight
science_path = args.science
# Find profile breaks
profiles = find_profiles(flight_path, science_path, args.time, args.depth)
# Interpolate GPS
interp_gps = get_file_set_gps(flight_path, science_path, args.time,
args.gps_prefix)
# Create NetCDF Files for Each Profile
profile_id = 0
profile_end = 0
file_path = None
uv_values = None
empty_uv_processed_paths = []
reader = create_reader(flight_path, science_path)
for line in reader:
if profile_end < line['timestamp']:
# Open new NetCDF
begin_time = datetime.fromtimestamp(line['timestamp'])
filename = "%s_%s_%s.nc" % (args.glider_name,
begin_time.isoformat(), args.mode)
file_path = os.path.join(args.output_path, filename)
profile = profiles[profiles[:, 2] == profile_id]
# NOTE: Store 1 based profile id
init_netcdf(file_path, attrs, args.segment_id, profile_id + 1)
profile = profiles[profiles[:, 2] == profile_id]
profile_end = max(profile[:, 0])
with open_glider_netcdf(file_path, 'a') as glider_nc:
while line['timestamp'] <= profile_end:
line = fill_gps(line, interp_gps, args.time, args.gps_prefix)
glider_nc.stream_dict_insert(line)
try:
line = reader.next()
except StopIteration:
break
# Handle UV Variables
if glider_nc.contains('time_uv'):
uv_values = backfill_uv_variables(glider_nc,
empty_uv_processed_paths)
elif uv_values is not None:
fill_uv_variables(glider_nc, uv_values)
del empty_uv_processed_paths[:]
else:
empty_uv_processed_paths.append(file_path)
glider_nc.update_profile_vars()
try:
glider_nc.calculate_salinity()
glider_nc.calculate_density()
except Exception, ex:
print "(%s)- %s" % (file_path, ex)
profile_id += 1
def test_set_platform(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_platform(self.deployment['platform'])
nc = glider_nc.nc
self.assertEqual(
nc.variables['platform'].getncattr('wmo_id'),
4801516
)
def backfill_uv_variables(src_glider_nc, empty_uv_processed_paths):
uv_values = {}
for key_name in GLIDER_UV_DATATYPE_KEYS:
uv_values[key_name] = src_glider_nc.get_scalar(key_name)
for file_path in empty_uv_processed_paths:
with open_glider_netcdf(file_path, 'a') as dst_glider_nc:
fill_uv_variables(dst_glider_nc, uv_values)
return uv_values
def test_data_insert(self):
flightReader = GliderBDReader(
['./test_data/usf-bass/usf-bass-2014-061-1-0.sbd'])
scienceReader = GliderBDReader(
['./test_data/usf-bass/usf-bass-2014-061-1-0.tbd'])
reader = MergedGliderBDReader(flightReader, scienceReader)
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
for line in reader:
glider_nc.stream_dict_insert(line)
def test_set_trajectory_id(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_trajectory_id(self.deployment['glider'],
self.deployment['trajectory_date'])
nc = glider_nc.nc
traj_str = "%s-%s" % (self.deployment['glider'],
self.deployment['trajectory_date'])
self.assertEqual(nc.variables['trajectory'][:].tostring(),
traj_str)
def backfill_uv_variables(src_glider_nc, empty_uv_processed_paths):
uv_values = {}
for key_name in GLIDER_UV_DATATYPE_KEYS:
uv_values[key_name] = src_glider_nc.get_scalar(key_name)
for file_path in empty_uv_processed_paths:
with open_glider_netcdf(file_path, 'a') as dst_glider_nc:
fill_uv_variables(dst_glider_nc, uv_values)
return uv_values
def test_data_insert(self):
flightReader = GliderBDReader(
['./test_data/usf-bass/usf-bass-2014-061-1-0.sbd']
)
scienceReader = GliderBDReader(
['./test_data/usf-bass/usf-bass-2014-061-1-0.tbd']
)
reader = MergedGliderBDReader(flightReader, scienceReader)
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
for line in reader:
glider_nc.stream_dict_insert(line)
def test_set_trajectory_id(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_trajectory_id(
self.deployment['glider'],
self.deployment['trajectory_date']
)
nc = glider_nc.nc
traj_str = "%s-%s" % (
self.deployment['glider'],
self.deployment['trajectory_date']
)
self.assertEqual(
nc.variables['trajectory'][:].tostring(), traj_str
)
def write_segment_netcdf(configs, message):
dataset = GliderDataset(message)
# No longer need the dataset stored by handlers
del sets[set_key]
global_attributes = (
generate_global_attributes(configs, dataset)
)
filename = generate_filename(configs, dataset)
tmp_path = '/tmp/' + filename
with open_glider_netcdf(tmp_path, 'w') as glider_nc:
glider_nc.set_global_attributes(global_attributes)
glider_nc.set_platform(
configs[dataset.glider]['deployment']['platform']
)
glider_nc.set_trajectory_id(1)
glider_nc.set_segment_id(dataset.segment)
glider_nc.set_datatypes(configs['datatypes'])
glider_nc.set_instruments(configs[dataset.glider]['instruments'])
glider_nc.set_times(dataset.times)
# Insert time_uv parameters
glider_nc.set_time_uv(dataset.time_uv)
glider_nc.set_profile_ids(dataset.calculate_profiles())
for datatype, data in dataset.data_by_type.items():
glider_nc.insert_data(datatype, data)
deployment_path = (
configs['output_directory'] + '/'
+ configs[dataset.glider]['deployment']['directory']
)
if not os.path.exists(deployment_path):
os.mkdir(deployment_path)
file_path = deployment_path + '/' + filename
shutil.move(tmp_path, file_path)
logger.info("Datafile written to %s" % file_path)
def test_segment_id(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_segment_id(3)
nc = glider_nc.nc
self.assertEqual(nc.variables['segment_id'].getValue(), 3)
def test_set_instruments(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_instruments(self.instruments)
nc = glider_nc.nc
self.assertIn('instrument_ctd', nc.variables)
def test_set_instruments(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_instruments(self.instruments)
nc = glider_nc.nc
self.assertIn('instrument_ctd', nc.variables)
def test_set_platform(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_platform(self.deployment['platform'])
nc = glider_nc.nc
self.assertEqual(nc.variables['platform'].getncattr('wmo_id'),
4801516)
def test_profile_ids(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_profile_id(4)
nc = glider_nc.nc
self.assertEqual(nc.variables['profile_id'].getValue(), 4)
def test_segment_id(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_segment_id(3)
nc = glider_nc.nc
self.assertEqual(nc.variables['segment_id'].getValue(), 3)
def process_dataset(args, attrs):
flight_path = args.flight
science_path = args.science
# Find profile breaks
profiles = find_profiles(flight_path, science_path, args.time, args.depth)
# Interpolate GPS
interp_gps = get_file_set_gps(
flight_path, science_path, args.time, args.gps_prefix
)
# Create NetCDF Files for Each Profile
profile_id = 0
profile_end = 0
file_path = None
uv_values = None
empty_uv_processed_paths = []
reader = create_reader(flight_path, science_path)
for line in reader:
if profile_end < line['timestamp']:
# Open new NetCDF
begin_time = datetime.fromtimestamp(line['timestamp'])
filename = "%s_%s_%s.nc" % (
args.glider_name,
begin_time.isoformat(),
args.mode
)
file_path = os.path.join(
args.output_path,
filename
)
profile = profiles[profiles[:, 2] == profile_id]
# NOTE: Store 1 based profile id
init_netcdf(file_path, attrs, args.segment_id, profile_id + 1)
profile = profiles[profiles[:, 2] == profile_id]
profile_end = max(profile[:, 0])
with open_glider_netcdf(file_path, 'a') as glider_nc:
while line['timestamp'] <= profile_end:
line = fill_gps(line, interp_gps, args.time, args.gps_prefix)
glider_nc.stream_dict_insert(line)
try:
line = reader.next()
except StopIteration:
break
# Handle UV Variables
if glider_nc.contains('time_uv'):
uv_values = backfill_uv_variables(
glider_nc, empty_uv_processed_paths
)
elif uv_values is not None:
fill_uv_variables(glider_nc, uv_values)
del empty_uv_processed_paths[:]
else:
empty_uv_processed_paths.append(file_path)
glider_nc.update_profile_vars()
try:
glider_nc.calculate_salinity()
glider_nc.calculate_density()
except Exception, ex:
print "(%s)- %s" % (file_path, ex)
profile_id += 1
def test_profile_ids(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_profile_id(4)
nc = glider_nc.nc
self.assertEqual(nc.variables['profile_id'].getValue(), 4)
def test_with(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_global_attributes(self.global_attributes)
self.assertTrue(os.path.isfile(self.test_path))
def test_with(self):
with open_glider_netcdf(self.test_path, self.mode) as glider_nc:
glider_nc.set_global_attributes(self.global_attributes)
self.assertTrue(os.path.isfile(self.test_path))