def define_attributes(nc_file, expocode, sect_id, data_type, stnnbr, castno,
bottom_depth):
nc_file.EXPOCODE = expocode
nc_file.Conventions = 'COARDS/WOCE'
nc_file.WOCE_VERSION = '3.0'
nc_file.WOCE_ID = sect_id
nc_file.DATA_TYPE = data_type
nc_file.STATION_NUMBER = stnnbr
nc_file.CAST_NUMBER = castno
nc_file.BOTTOM_DEPTH_METERS = bottom_depth
nc_file.Creation_Time = fns.strftime_iso(datetime.datetime.utcnow())
def write(self, handle):
"""How to write a CTD NetCDF file."""
UNKNOWN = 'UNKNOWN'
UNSPECIFIED_UNITS = 'unspecified'
STRLEN = 40
# Prepare file handles and datetime information
tmp = NamedTemporaryFile()
strdate = str(self.globals['DATE'])
strtime = str(self.globals['TIME'])
isowocedate = datetime.strptime(strdate + strtime, '%Y%m%d%H%M')
nc_file = nc.Dataset(tmp.name, 'w', format='NETCDF3_CLASSIC')
# Define dimensions variables
makeDim = nc_file.createDimension
makeDim('time', 1)
makeDim('depth', len(self))
makeDim('latitude', 1)
makeDim('longitude', 1)
makeDim('string_dimension', STRLEN)
# Define dataset attributes
nc_file.EXPOCODE = self.globals['EXPOCODE']
nc_file.Conventions = 'COARDS-DEV/WOCE'
nc_file.WOCE_VERSION = '3.0'
nc_file.WOCE_ID = self.globals['SECT_ID'] if 'SECT_ID' in self.globals \
else UNKNOWN
nc_file.DATA_TYPE = 'WOCE CTD'
nc_file.STATION_NUMBER = self.globals['STNNBR'] or UNKNOWN
nc_file.CAST_NUMBER = self.globals['CASTNO'] or UNKNOWN
nc_file.BOTTOM_DEPTH_METERS = nc.simplest_str(float(self.globals['DEPTH']))
nc_file.Creation_Time = strftime_iso(datetime.utcnow())
nc_file.ORIGINAL_HEADER = self.globals['header']
nc_file.WOCE_CTD_FLAG_DESCRIPTION = woce.CTD_FLAG_DESCRIPTION
def MISSING_COORD_VAR (param):
return ("expected global coordinate variable %s "
"but not found (XXX)") % param
# Coordinate variables
if 'TIME' not in self.globals:
raise AttributeError(MISSING_COORD_VAR('TIME'))
var_time = nc_file.createVariable('time', 'i', ('time', ))
var_time.long_name = 'time'
var_time.units = 'minutes since %s' % strftime_iso(nc.EPOCH)
var_time.data_min = nc.minutes_since_epoch(isowocedate)
var_time.data_max = var_time.data_min
var_time.C_format = '%10d'
var_time[:] = var_time.data_min
if 'LATITUDE' not in self.globals:
raise AttributeError(MISSING_COORD_VAR('LATITUDE'))
var_latitude = nc_file.createVariable('latitude', 'f', ('latitude',))
var_latitude.long_name = 'latitude'
var_latitude.units = 'degrees_N'
var_latitude.data_min = float(self.globals['LATITUDE'])
var_latitude.data_max = var_latitude.data_min
var_latitude.C_format = '%9.4f'
var_latitude[:] = var_latitude.data_min
if 'LONGITUDE' not in self.globals:
raise AttributeError(MISSING_COORD_VAR('LONGITUDE'))
var_longitude = nc_file.createVariable('longitude', 'f', ('longitude',))
var_longitude.long_name = 'longitude'
var_longitude.units = 'degrees_E'
var_longitude.data_min = float(self.globals['LONGITUDE'])
var_longitude.data_max = var_longitude.data_min
var_longitude.C_format = '%9.4f'
var_longitude[:] = var_longitude.data_min
woce_datetime = woce.strftime_woce_date_time(isowocedate)
if 'DATE' not in self.globals:
raise AttributeError(MISSING_COORD_VAR('DATE'))
var_woce_date = nc_file.createVariable('woce_date', 'i', ('time',))
var_woce_date.long_name = 'WOCE date'
var_woce_date.units = 'yyyymmdd UTC'
var_woce_date.data_min = int(woce_datetime[0] or -9)
var_woce_date.data_max = var_woce_date.data_min
var_woce_date.C_format = '%8d'
var_woce_date[:] = var_woce_date.data_min
var_woce_time = nc_file.createVariable('woce_time', 'i2', ('time',))
var_woce_time.long_name = 'WOCE time'
var_woce_time.units = 'hhmm UTC'
var_woce_time.data_min = int(woce_datetime[1] or -9)
var_woce_time.data_max = var_woce_time.data_min
var_woce_time.C_format = '%4d'
var_woce_time[:] = var_woce_time.data_min
var_station = nc_file.createVariable('station', 'c', ('string_dimension', ))
var_station.long_name = 'STATION'
var_station.units = UNSPECIFIED_UNITS
var_station.C_format = '%s'
var_station[:] = nc.simplest_str(self.globals['STNNBR']).ljust(len(var_station))
var_cast = nc_file.createVariable('cast', 'c', ('string_dimension', ))
var_cast.long_name = 'CAST'
var_cast.units = UNSPECIFIED_UNITS
var_cast.C_format = '%s'
var_cast[:] = nc.simplest_str(self.globals['CASTNO']).ljust(len(var_cast))
# Create data variables and fill them
for param, column in self.columns.iteritems():
parameter = column.parameter
parameter_name = parameter.mnemonic_woce()
if parameter_name in nc.STATIC_PARAMETERS_PER_CAST:
continue
var = nc_file.createVariable(
parameter.full_name.encode('ascii', 'replace'), 'f8',
('time', 'depth', 'latitude', 'longitude', ))
var.long_name = parameter.full_name.encode('ascii', 'replace')
var.units = parameter.units.name.encode('ascii', 'replace') if \
parameter.units else UNSPECIFIED_UNITS
compact_column = filter(None, column)
if compact_column:
var.data_min = min(compact_column)
var.data_max = max(compact_column)
else:
var.data_min = float('-inf')
var.data_max = float('inf')
var.C_format = parameter.format.encode('ascii', 'replace')
print parameter_name, len(var), len(column.values)
var[:] = column.values
if column.is_flagged_woce():
vfw = nc_file.createVariable(parameter.name + nc.QC_SUFFIX, 'i2',
('time', 'depth', 'latitude', 'longitude', ))
vfw.long_name = parameter.name + nc.QC_SUFFIX
vfw[:] = column.flags_woce
# Transfer finished NetCDF file to provided handle
nc_file.close()
handle.write(tmp.read())
tmp.close()
def test_strftime_iso(self):
self.assertEqual('2010-05-04T13:42:39Z',
fns.strftime_iso(datetime(2010, 5, 4, 13, 42, 39)))
def create_oceansites_nc(df, filename, data_type, version=None):
from libcchdo.formats import netcdf as nc
info = {
'date_start': df.globals['_DATETIME'],
'lat': df.globals['LATITUDE'],
'lon': df.globals['LONGITUDE'],
'depths': df.globals['DEPTH'],
'data_length': len(df),
}
version = _sanitize_os_version(version)
nc_file = nc.Dataset(filename, 'w', format='NETCDF3_CLASSIC')
nc_file.data_type = 'OceanSITES time-series {data_type} data'.format(
data_type=data_type)
nc_file.format_version = version
# 2011-11-28 Jing Zhou says platform code may be left blank as NA myshen
nc_file.wmo_platform_code = ''
if version == '1.2':
nc_file.platform_code = ''
now = datetime.utcnow()
nc_file.date_update = strftime_iso(now)
nc_file.source = 'Shipborne observation'
nc_file.history = ''.join([
info['date_start'].isoformat(), "Z data collected\n",
now.isoformat(), "Z date file translated/written"
])
nc_file.data_mode = 'D'
nc_file.quality_control_indicator = '1'
nc_file.quality_index = 'B'
if version == '1.1':
nc_file.conventions = 'OceanSITES Manual 1.1, CF-1.1'
elif version == '1.2':
nc_file.Conventions = 'CF-1.4, OceanSITES 1.1'
nc_file.netcdf_version = '3.x'
nc_file.naming_authority = 'OceanSITES'
nc_file.cdm_data_type = 'Station'
nc_file.geospatial_lat_min = str(info['lat'])
nc_file.geospatial_lat_max = str(info['lat'])
nc_file.geospatial_lon_min = str(info['lon'])
nc_file.geospatial_lon_max = str(info['lon'])
nc_file.geospatial_vertical_min = 0
nc_file.geospatial_vertical_max = str(info['depths'])
nc_file.geospatial_vertical_positive = 'down'
nc_file.author = 'Shen:Diggs (Scripps)'
nc_file.data_assembly_center = data_assembly_center(version)
nc_file.distribution_statement = OS_TEXT['DISTRIBUTION_STATEMENT']
nc_file.citation = OS_TEXT['CITATION']
nc_file.update_interval = 'void'
nc_file.qc_manual = qc_manual(version)
nc_file.time_coverage_start = strftime_iso(info['date_start'])
nc_file.time_coverage_end = strftime_iso(info['date_start'])
nc_file.createDimension('TIME')
try:
nc_file.createDimension('DEPTH', info['data_length'])
except RuntimeError:
raise AttributeError("There is no data to be written.")
nc_file.createDimension('LATITUDE', 1)
nc_file.createDimension('LONGITUDE', 1)
nc_file.createDimension('POSITION', 1)
# OceanSITES coordinate variables
var_time = nc_file.createVariable('TIME',
'd', ('TIME', ),
fill_value=999999.0)
var_time.long_name = 'time'
var_time.standard_name = 'time'
var_time.units = 'days since 1950-01-01T00:00:00Z'
var_time.valid_min = 0.0
var_time.valid_max = 90000.0
var_time.QC_indicator = 7 # Matthias Lankhorst
var_time.QC_procedure = 5 # Matthias Lankhorst
# 1/24 assuming a typical cast lasts one hour Matthias Lankhorst
var_time.uncertainty = 0.0417
var_time.axis = 'T'
var_time[:] = [fractional_days_since(info['date_start'])]
var_latitude = nc_file.createVariable('LATITUDE',
'f', ('LATITUDE', ),
fill_value=99999.0)
var_latitude.long_name = 'Latitude of each location'
var_latitude.standard_name = 'latitude'
var_latitude.units = 'degrees_north'
var_latitude.valid_min = -90.0
var_latitude.valid_max = 90.0
var_latitude.QC_indicator = 7 # Matthias Lankhorst
var_latitude.QC_procedure = 5 # Matthias Lankhorst
var_latitude.uncertainty = 0.0045 # Matthias Lankhorst
var_latitude.axis = 'Y'
if version == '1.1':
pass
elif version == '1.2':
var_latitude.reference = 'WGS84'
var_latitude.coordinate_reference_frame = 'urn:ogc:crs:EPSG::4326'
var_latitude[:] = [info['lat']]
var_longitude = nc_file.createVariable('LONGITUDE',
'f', ('LONGITUDE', ),
fill_value=99999.0)
var_longitude.long_name = 'Longitude of each location'
var_longitude.standard_name = 'longitude'
var_longitude.units = 'degrees_east'
var_longitude.valid_min = -180.0
var_longitude.valid_max = 180.0
var_longitude.QC_indicator = 7 # Matthias Lankhorst
var_longitude.QC_procedure = 5 # Matthias Lankhorst
# Matthias Lankhorst
var_longitude.uncertainty = 0.0045 / cos(float(info['lat']))
var_longitude.axis = 'X'
if version == '1.1':
pass
elif version == '1.2':
var_longitude.reference = 'WGS84'
var_longitude.coordinate_reference_frame = 'urn:ogc:crs:EPSG::4326'
var_longitude[:] = [info['lon']]
var_depth = nc_file.createVariable('DEPTH',
'f', ('DEPTH', ),
fill_value=-99999.0)
var_depth.long_name = 'Depth of each measurement'
var_depth.standard_name = 'depth'
var_depth.units = 'meters'
var_depth.valid_min = 0.0
var_depth.valid_max = 12000.0
# Subject: OceanSITES: more on QC flags, uncertainty, depth
# Interpolated from latitude and pressure.
var_depth.QC_indicator = 8
var_depth.QC_procedure = 2 # See above
var_depth.uncertainty = 1.0 # A decibar
if version == '1.1':
var_depth.axis = 'down' # oceanic
elif version == '1.2':
var_depth.positive = 'down'
var_depth.axis = 'Z'
var_depth.reference = 'sea_level' # TODO is this right?
var_depth.coordinate_reference_frame = 'urn:ogc:crs:EPSG::5113'
return nc_file
def _write_dfile(dfile, fileobj, cfg=DEFAULT_CFG, cvt=None):
"""Write microstructure COARDS compliant file."""
log.debug(dfile.globals)
with nc.nc_dataset_to_stream(fileobj, format='NETCDF3_CLASSIC') as nc_file:
nc_file.Conventions = 'CF-1.6'
nc_file.netcdf_version = '3'
nc_file.history = ''.join([
"data collected\n",
datetime.utcnow().isoformat(), "Z date file translated/written"
])
nc_file.source = _real_data_type(cfg)
# TODO README file inclusion or reference? details quality, instruments
# used, TS used pumped instruments etc
# TODO will supply DOIs
nc_file.references = 'references'
nc_file.data_type = _real_data_type(cfg)
nc_file.format_version = '0.1-beta'
nc_file.date_update = fns.strftime_iso(datetime.utcnow())
(minlng, minlat), (maxlng, maxlat) = _get_coordinate_bounds(dfile)
nc_file.geospatial_lat_min = str(minlat)
nc_file.geospatial_lat_max = str(maxlat)
nc_file.geospatial_lon_min = str(minlng)
nc_file.geospatial_lon_max = str(maxlng)
nc_file.geospatial_vertical_min = 0
nc_file.geospatial_vertical_max = int(dfile.globals['DEPTH'])
nc_file.geospatial_vertical_positive = 'down'
nc_file.author = cfg['originator']
nc_file.pi_name = cfg['pi']
# non-OceanSITES standard
nc_file.data_originator = cfg['originator']
nc_file.data_assembly_center = 'CCHDO'
# TODO
nc_file.distribution_statement = ('HRP distribution statement. TBD.')
# TODO
nc_file.citation = ('Citation statement. TBD.')
nc_file.update_interval = 'void'
nc_file.time_coverage_start = fns.strftime_iso(
dfile.globals['_DATETIME'])
nc_file.time_coverage_end = fns.strftime_iso(
dfile.globals['_DATETIME'])
nc_file.dive_number = int(dfile.globals['STNNBR'])
nc_file.createDimension('TIME')
nc_file.createDimension('BOTTOM_DEPTH', 1)
try:
nc_file.createDimension('DEPTH', len(dfile))
except RuntimeError:
raise AttributeError("There is no data to be written.")
nc_file.createDimension('LATITUDE', 1)
nc_file.createDimension('LONGITUDE', 1)
var_time = nc_file.createVariable('TIME',
'd', ('TIME', ),
fill_value=999999.0)
var_time.long_name = 'time'
var_time.standard_name = 'time'
var_time.units = 'days since 1950-01-01T00:00:00Z'
var_time.valid_min = 0.0
var_time.valid_max = 90000.0
var_time.axis = 'T'
var_latitude = nc_file.createVariable('LATITUDE',
'f', ('LATITUDE', ),
fill_value=99999.0)
var_latitude.long_name = 'Latitude of each location'
var_latitude.standard_name = 'latitude'
var_latitude.units = 'degrees_north'
var_latitude.valid_min = -90.0
var_latitude.valid_max = 90.0
var_latitude.axis = 'Y'
var_longitude = nc_file.createVariable('LONGITUDE',
'f', ('LONGITUDE', ),
fill_value=99999.0)
var_longitude.long_name = 'Longitude of each location'
var_longitude.standard_name = 'longitude'
var_longitude.units = 'degrees_east'
var_longitude.valid_min = -180.0
var_longitude.valid_max = 180.0
var_longitude.axis = 'X'
if dfile.globals.get('DEPTH'):
var_bot_depth = nc_file.createVariable('BOT_DEPTH',
'f', ('BOTTOM_DEPTH', ),
fill_value=-99999.0)
var_bot_depth.long_name = 'Sea floor depth below sea level'
var_bot_depth.standard_name = 'sea_floor_depth_below_sea_level'
var_bot_depth.units = 'meters'
var_bot_depth.valid_min = 0.0
var_bot_depth.valid_max = 12000.0
var_bot_depth.axis = 'Z'
var_bot_depth = float(dfile.globals['DEPTH'])
var_depth = nc_file.createVariable('DEPTH',
'f', ('DEPTH', ),
fill_value=-99999.0)
var_depth.long_name = 'Depth of each measurement'
var_depth.standard_name = 'depth'
var_depth.units = 'meters'
var_depth.valid_min = 0.0
var_depth.valid_max = 12000.0
var_depth.axis = 'Z'
# Write variables
var_time[:] = [decimal_days_since(dfile.globals['_DATETIME'])]
var_latitude[:] = [dfile.globals['LATITUDE']]
var_longitude[:] = [dfile.globals['LONGITUDE']]
var_depth[:] = _write_depths(var_depth, dfile, cvt)
write_columns(dfile, nc_file, cvt)
nc_file.title = '{0} ExpoCode={1} Dive={2}'.format(
cfg['data_type'], dfile.globals['EXPOCODE'],
dfile.globals['STNNBR'])
nc_file.id = '{0} {1} {2}'.format(cfg['data_type'],
dfile.globals['EXPOCODE'],
dfile.globals['STNNBR'])
nc.check_variable_ranges(nc_file)