def test_glider_rename_netcdf_lat_lon(self):
ctd_sk = StreamKey('CP05MOAS', 'GL388', '03-CTDGVM000',
'recovered_host',
'ctdgv_m_glider_instrument_recovered')
ctd_fn = 'deployment0008_CP05MOAS-GL388-03-CTDGVM000-recovered_host-ctdgv_m_glider_instrument_recovered.nc'
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, ctd_fn),
decode_times=False)
self.assertIn('glider_gps_position-m_gps_lat', ctd_ds)
self.assertIn('glider_gps_position-m_gps_lon', ctd_ds)
self.assertIn('glider_gps_position-m_lat', ctd_ds)
self.assertIn('glider_gps_position-m_lon', ctd_ds)
self.assertIn('glider_gps_position-interp_lat', ctd_ds)
self.assertIn('glider_gps_position-interp_lon', ctd_ds)
self.assertNotIn('m_gps_lat', ctd_ds)
self.assertNotIn('m_gps_lon', ctd_ds)
self.assertNotIn('m_lat', ctd_ds)
self.assertNotIn('m_lon', ctd_ds)
self.assertNotIn('lat', ctd_ds)
self.assertNotIn('lon', ctd_ds)
modified = rename_glider_lat_lon(ctd_sk, ctd_ds)
self.assertNotIn('glider_gps_position-m_gps_lat', modified)
self.assertNotIn('glider_gps_position-m_gps_lon', modified)
self.assertNotIn('glider_gps_position-m_lat', modified)
self.assertNotIn('glider_gps_position-m_lon', modified)
self.assertNotIn('glider_gps_position-interp_lat', modified)
self.assertNotIn('glider_gps_position-interp_lon', modified)
self.assertIn('m_gps_lat', modified)
self.assertIn('m_gps_lon', modified)
self.assertIn('m_lat', modified)
self.assertIn('m_lon', modified)
self.assertIn('lat', modified)
self.assertIn('lon', modified)
def test_glider_rename_netcdf_lat_lon(self):
ctd_sk = StreamKey('CE05MOAS', 'GL319', '05-CTDGVM000', 'recovered_host', 'ctdgv_m_glider_instrument_recovered')
ctd_fn = 'deployment0003_CE05MOAS-GL319-05-CTDGVM000-recovered_host-ctdgv_m_glider_instrument_recovered.nc'
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, ctd_fn), decode_times=False)
self.assertIn('glider_gps_position-m_gps_lat', ctd_ds)
self.assertIn('glider_gps_position-m_gps_lon', ctd_ds)
modified = rename_glider_lat_lon(ctd_sk, ctd_ds)
self.assertNotIn('glider_gps_position-m_gps_lat', modified)
self.assertNotIn('glider_gps_position-m_gps_lon', modified)
self.assertIn('lat', modified)
self.assertIn('lon', modified)
def _create_files(self, base_path):
file_paths = []
for stream_key, stream_dataset in self.stream_request.datasets.iteritems():
for deployment, ds in stream_dataset.datasets.iteritems():
add_dynamic_attributes(ds)
start = ds.attrs['time_coverage_start'].translate(None, '-:')
end = ds.attrs['time_coverage_end'].translate(None, '-:')
# provenance types will be written to JSON files
prov_fname = 'deployment%04d_%s_provenance_%s-%s.json' % (deployment,
stream_key.as_dashed_refdes(), start, end)
prov_json = os.path.join(base_path, prov_fname)
file_paths.append(prov_json)
stream_dataset.provenance_metadata.dump_json(prov_json)
file_name = 'deployment%04d_%s_%s-%s.nc' % (deployment, stream_key.as_dashed_refdes(), start, end)
file_path = os.path.join(base_path, file_name)
ds = rename_glider_lat_lon(stream_key, ds)
write_netcdf(ds, file_path, classic=self.classic)
file_paths.append(file_path)
return file_paths
def _create_files(self, base_path):
file_paths = []
# annotation data will be written to a JSON file
if self.stream_request.include_annotations:
time_range_string = str(self.stream_request.time_range).replace(
" ", "")
anno_fname = 'annotations_%s.json' % time_range_string
anno_json = os.path.join(base_path, anno_fname)
file_paths.append(anno_json)
self.stream_request.annotation_store.dump_json(anno_json)
for stream_key, stream_dataset in self.stream_request.datasets.iteritems(
):
for deployment, ds in stream_dataset.datasets.iteritems():
add_dynamic_attributes(ds)
start = ds.attrs['time_coverage_start'].translate(None, '-:')
end = ds.attrs['time_coverage_end'].translate(None, '-:')
# provenance types will be written to JSON files
if self.stream_request.include_provenance:
prov_fname = 'deployment%04d_%s_provenance_%s-%s.json' % (
deployment, stream_key.as_dashed_refdes(), start, end)
prov_json = os.path.join(base_path, prov_fname)
file_paths.append(prov_json)
stream_dataset.provenance_metadata.dump_json(prov_json)
file_name = 'deployment%04d_%s_%s-%s.nc' % (
deployment, stream_key.as_dashed_refdes(), start, end)
file_path = os.path.join(base_path, file_name)
ds = rename_glider_lat_lon(stream_key, ds)
# include all directly requested_parameters
params_to_include = [
p.name for p in self.stream_request.requested_parameters
]
# also include any indirectly derived pressure parameter (9328)
pressure_params = [
(sk, param) for sk in self.stream_request.external_includes
for param in self.stream_request.external_includes[sk]
if param.data_product_identifier == PRESSURE_DPI
]
if pressure_params:
params_to_include.append(INT_PRESSURE_NAME)
# include all external parameters associated with the directly requested parameters (12886)
for external_stream_key in self.stream_request.external_includes:
for parameter in self.stream_request.external_includes[
external_stream_key]:
params_to_include.append(parameter.name)
long_parameter_name = external_stream_key.stream_name + "-" + parameter.name
if long_parameter_name in ds:
# rename the parameter without the stream_name prefix (12544 AC1)
ds = ds.rename(
{long_parameter_name: parameter.name})
# record the instrument and stream (12544 AC2)
ds[parameter.name].attrs[
'instrument'] = external_stream_key.as_three_part_refdes(
)
ds[parameter.name].attrs[
'stream'] = external_stream_key.stream_name
# associated variables with their contributors (12544 AC3)
for requested_parameter in self.stream_request.requested_parameters:
if requested_parameter.needs and requested_parameter.name in ds:
for k, need_list in requested_parameter.needs:
for need in need_list:
if need.name in params_to_include:
if 'ancillary_variables' in ds[
requested_parameter.name].attrs:
ds[requested_parameter.name].attrs[
'ancillary_variables'] += "," + need.name
else:
ds[requested_parameter.name].attrs[
'ancillary_variables'] = need.name
break
# setup coordinate variables (10745)
ds = self._setup_coordinate_variables(ds)
if params_to_include:
ds = self._filter_params(ds, params_to_include)
write_netcdf(ds, file_path, classic=self.classic)
file_paths.append(file_path)
return file_paths