def test_from_bad_string(self):
u = IoosUrn.from_string('urn:ioos:sensor:whatami')
assert u.urn is None
u = IoosUrn.from_string('urn:ioos:nothinghere')
assert u.urn is None
u = IoosUrn.from_string('urn:totesbroken')
assert u.urn is None
def test_from_bad_string(self):
u = IoosUrn.from_string('urn:ioos:sensor:whatami')
assert u.urn is None
u = IoosUrn.from_string('urn:ioos:nothinghere')
assert u.urn is None
u = IoosUrn.from_string('urn:totesbroken')
assert u.urn is None
def add_instrument_variable(self, variable_name):
if variable_name not in self._nc.variables:
logger.error("Variable {} not found in file, cannot create instrument metadata variable")
return
elif 'id' not in self._nc.ncattrs() or 'naming_authority' not in self._nc.ncattrs():
logger.error("Global attributes 'id' and 'naming_authority' are required to create an instrument variable")
return
instr_var_name = "{}_instrument".format(variable_name)
instrument = self._nc.createVariable(instr_var_name, "i4")
datavar = self._nc.variables[variable_name]
vats = { k: getattr(datavar, k) for k in datavar.ncattrs() }
instrument_urn = urnify(self._nc.naming_authority, self._nc.id, vats)
inst_urn = IoosUrn.from_string(instrument_urn)
instrument.long_name = 'Instrument measuring {} from {}'.format(inst_urn.component, inst_urn.label)
instrument.ioos_code = instrument_urn
instrument.short_name = inst_urn.component
instrument.definition = "http://mmisw.org/ont/ioos/definition/sensorID"
datavar.instrument = instr_var_name
# Append the instrument to the ancilary variables
av = getattr(datavar, 'ancillary_variables', '')
av += ' {}'.format(instr_var_name)
datavar.ancillary_variables = av.strip()
self._nc.sync()
def test_messy_urn(self):
u = IoosUrn.from_string(
'urn:ioos:sensor:myauthority:mylabel:standard_name#key=key1:value1,key2:value2;some_other_key=some_other_value'
)
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'standard_name#key=key1:value1,key2:value2;some_other_key=some_other_value'
def test_cdiac_urn(self):
u = IoosUrn.from_string(
'urn:ioos:sensor:gov.ornl.cdiac:cheeca_80w_25n:sea_water_temperature'
)
assert u.asset_type == 'sensor'
assert u.authority == 'gov.ornl.cdiac'
assert u.label == 'cheeca_80w_25n'
assert u.component == 'sea_water_temperature'
def test_from_string(self):
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel:mycomponent')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel:mycomponent:myversion')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
assert u.version == 'myversion'
def test_change_sensor_to_station(self):
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel:mycomponent')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
u.asset_type = 'station'
u.component = None
assert u.urn == 'urn:ioos:station:myauthority:mylabel'
def test_change_sensor_to_station(self):
u = IoosUrn.from_string(
'urn:ioos:sensor:myauthority:mylabel:mycomponent')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
u.asset_type = 'station'
u.component = None
assert u.urn == 'urn:ioos:station:myauthority:mylabel'
def test_from_string(self):
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
u = IoosUrn.from_string(
'urn:ioos:sensor:myauthority:mylabel:mycomponent')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
u = IoosUrn.from_string(
'urn:ioos:sensor:myauthority:mylabel:mycomponent:myversion')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'mycomponent'
assert u.version == 'myversion'
def test_messy_urn(self):
u = IoosUrn.from_string('urn:ioos:sensor:myauthority:mylabel:standard_name#key=key1:value1,key2:value2;some_other_key=some_other_value')
assert u.asset_type == 'sensor'
assert u.authority == 'myauthority'
assert u.label == 'mylabel'
assert u.component == 'standard_name#key=key1:value1,key2:value2;some_other_key=some_other_value'
def test_from_long_string(self):
u = IoosUrn.from_string('urn:ioos:sensor:whatami:wow:i:have:lots:of:things')
assert u.urn == 'urn:ioos:sensor:whatami:wow:i:have'
def test_cdiac_urn(self):
u = IoosUrn.from_string('urn:ioos:sensor:gov.ornl.cdiac:cheeca_80w_25n:sea_water_temperature')
assert u.asset_type == 'sensor'
assert u.authority == 'gov.ornl.cdiac'
assert u.label == 'cheeca_80w_25n'
assert u.component == 'sea_water_temperature'
def test_from_long_string(self):
u = IoosUrn.from_string(
'urn:ioos:sensor:whatami:wow:i:have:lots:of:things')
assert u.urn == 'urn:ioos:sensor:whatami:wow:i:have'
def dictify_urn(urn, combine_interval=True):
"""
By default, this will put the `interval` as part of the `cell_methods`
attribute (NetCDF CF style). To return `interval` as its own key, use
the `combine_interval=False` parameter.
"""
ioos_urn = IoosUrn.from_string(urn)
if ioos_urn.valid() is False:
return dict()
if ioos_urn.asset_type != 'sensor':
logger.error("This function only works on 'sensor' URNs.")
return dict()
if '#' in ioos_urn.component:
standard_name, extras = ioos_urn.component.split('#')
else:
standard_name = ioos_urn.component
extras = ''
d = dict(standard_name=standard_name)
# Discriminant
if '-' in ioos_urn.component:
d['discriminant'] = ioos_urn.component.split('-')[-1]
d['standard_name'] = ioos_urn.component.split('-')[0]
intervals = []
cell_methods = []
if extras:
for section in extras.split(';'):
key, values = section.split('=')
if key == 'interval':
# special case, intervals should be appended to the cell_methods
for v in values.split(','):
intervals.append(v)
else:
if key == 'cell_methods':
value = [ x.replace('_', ' ').replace(':', ': ') for x in values.split(',') ]
cell_methods = value
else:
value = ' '.join([x.replace('_', ' ').replace(':', ': ') for x in values.split(',')])
d[key] = value
if combine_interval is True:
if cell_methods and intervals:
if len(cell_methods) == len(intervals):
d['cell_methods'] = ' '.join([ '{} (interval: {})'.format(x[0], x[1].upper()) for x in zip(cell_methods, intervals) ])
else:
d['cell_methods'] = ' '.join(cell_methods)
for i in intervals:
d['cell_methods'] += ' (interval: {})'.format(i.upper())
elif cell_methods:
d['cell_methods'] = ' '.join(cell_methods)
for i in intervals:
d['cell_methods'] += ' (interval: {})'.format(i.upper())
elif intervals:
raise ValueError("An interval without a cell_method is not allowed! Not possible!")
else:
d['cell_methods'] = ' '.join(cell_methods)
d['interval'] = ','.join(intervals).upper()
if 'vertical_datum' in d:
d['vertical_datum'] = d['vertical_datum'].upper()
return d
def dictify_urn(urn, combine_interval=True):
"""
By default, this will put the `interval` as part of the `cell_methods`
attribute (NetCDF CF style). To return `interval` as its own key, use
the `combine_interval=False` parameter.
"""
ioos_urn = IoosUrn.from_string(urn)
if ioos_urn.valid() is False:
return dict()
if ioos_urn.asset_type != 'sensor':
logger.error("This function only works on 'sensor' URNs.")
return dict()
if '#' in ioos_urn.component:
standard_name, extras = ioos_urn.component.split('#')
else:
standard_name = ioos_urn.component
extras = ''
d = dict(standard_name=standard_name)
# Discriminant
if '-' in ioos_urn.component:
d['discriminant'] = standard_name.split('-')[-1]
d['standard_name'] = standard_name.split('-')[0]
intervals = []
cell_methods = []
if extras:
for section in extras.split(';'):
key, values = section.split('=')
if key == 'interval':
# special case, intervals should be appended to the cell_methods
for v in values.split(','):
intervals.append(v)
else:
if key == 'cell_methods':
value = [ x.replace('_', ' ').replace(':', ': ') for x in values.split(',') ]
cell_methods = value
else:
value = ' '.join([x.replace('_', ' ').replace(':', ': ') for x in values.split(',')])
d[key] = value
if combine_interval is True:
if cell_methods and intervals:
if len(cell_methods) == len(intervals):
d['cell_methods'] = ' '.join([ '{} (interval: {})'.format(x[0], x[1].upper()) for x in zip(cell_methods, intervals) ])
else:
d['cell_methods'] = ' '.join(cell_methods)
for i in intervals:
d['cell_methods'] += ' (interval: {})'.format(i.upper())
elif cell_methods:
d['cell_methods'] = ' '.join(cell_methods)
for i in intervals:
d['cell_methods'] += ' (interval: {})'.format(i.upper())
elif intervals:
raise ValueError("An interval without a cell_method is not allowed! Not possible!")
else:
d['cell_methods'] = ' '.join(cell_methods)
d['interval'] = ','.join(intervals).upper()
if 'vertical_datum' in d:
d['vertical_datum'] = d['vertical_datum'].upper()
return d
def __init__(self, output_directory, latitude, longitude, station_name, global_attributes, times=None, verticals=None, vertical_fill=None, output_filename=None, vertical_axis_name=None, vertical_positive=None):
if output_filename is None:
output_filename = '{}_{}.nc'.format(station_name, int(random.random() * 100000))
logger.info("No output filename specified, saving as {}".format(output_filename))
self.vertical_positive = vertical_positive or 'down'
self.vertical_axis_name = vertical_axis_name or 'z'
self.time_axis_name = 'time'
# Make directory
if not os.path.exists(output_directory):
os.makedirs(output_directory)
self.time = None
self.out_file = os.path.abspath(os.path.join(output_directory, output_filename))
if os.path.isfile(self.out_file):
os.remove(self.out_file)
with EnhancedDataset(self.out_file, 'w') as nc:
# Global attributes
# These are set by this script, we don't someone to be able to set them manually
global_skips = ["time_coverage_start", "time_coverage_end", "time_coverage_duration", "time_coverage_resolution",
"featureType", "geospatial_vertical_positive", "geospatial_vertical_min", "geospatial_vertical_max",
"geospatial_lat_min", "geospatial_lon_min", "geospatial_lat_max", "geospatial_lon_max", "geospatial_bounds"
"geospatial_vertical_resolution", "geospatial_lat_resolution", "geospatial_lon_resolution",
"Conventions", "date_created", "date_modified", "date_issued"]
for k, v in global_attributes.items():
if v is None:
v = "None"
if k not in global_skips:
nc.setncattr(k, v)
now_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:00Z")
nc.setncattr("Conventions", "CF-1.6,ACDD-1.3")
nc.setncattr("date_created", now_date)
nc.setncattr("date_modified", now_date)
nc.setncattr("date_issued", now_date)
if not hasattr(nc, "date_metadata_modified"):
nc.setncattr("date_metadata_modified", now_date)
# Allow the customization of this attribute
if 'cdm_data_type' not in global_attributes:
nc.setncattr('cdm_data_type', 'Station')
old_history = getattr(nc, 'history', '')
new_history = '{} - {} - {}'.format(now_date, 'pyaxiom', 'File created using pyaxiom')
if old_history:
nc.setncattr('history', '{}\n{}'.format(old_history, new_history))
else:
nc.setncattr('history', new_history)
# Station name
nc.createDimension("feature_type_instance", len(station_name))
name = nc.createVariable("feature_type_instance", "S1", ("feature_type_instance",))
name.cf_role = "timeseries_id"
name.long_name = "Identifier for each feature type instance"
name[:] = list(station_name)
# Location
lat = nc.createVariable("latitude", get_type(latitude))
lat.units = "degrees_north"
lat.standard_name = "latitude"
lat.long_name = "sensor latitude"
lat.axis = "Y"
lat.valid_min = latitude
lat.valid_max = latitude
lat[:] = latitude
nc.setncattr("geospatial_lat_min", latitude)
nc.setncattr("geospatial_lat_max", latitude)
nc.setncattr("geospatial_lat_resolution", 0)
nc.setncattr("geospatial_lat_units", "degrees_north")
lon = nc.createVariable("longitude", get_type(longitude))
lon.units = "degrees_east"
lon.standard_name = "longitude"
lon.long_name = "sensor longitude"
lon.axis = "X"
lon.valid_min = longitude
lon.valid_max = longitude
lon[:] = longitude
nc.setncattr("geospatial_lon_min", longitude)
nc.setncattr("geospatial_lon_max", longitude)
nc.setncattr("geospatial_lon_resolution", 0)
nc.setncattr("geospatial_lon_units", "degrees_east")
nc.setncattr("geospatial_bounds", "POINT({} {})".format(longitude, latitude))
if not hasattr(nc, "geospatial_bounds_crs"):
nc.setncattr("geospatial_bounds_crs", "EPSG:4326")
# Metadata variables
self.crs = nc.createVariable("crs", "i4")
self.crs.long_name = "http://www.opengis.net/def/crs/EPSG/0/4326"
self.crs.grid_mapping_name = "latitude_longitude"
self.crs.epsg_code = "EPSG:4326"
self.crs.semi_major_axis = float(6378137.0)
self.crs.inverse_flattening = float(298.257223563)
platform = nc.createVariable("platform", "i4")
platform.definition = "http://mmisw.org/ont/ioos/definition/stationID"
urn = IoosUrn.from_string(station_name)
if urn.valid() is True:
platform.short_name = global_attributes.get("title", urn.label)
platform.long_name = global_attributes.get('summary', 'Station {}'.format(urn.label))
platform.ioos_code = urn.urn
else:
platform.short_name = global_attributes.get("title", station_name)
platform.long_name = global_attributes.get("summary", station_name)
platform.ioos_code = station_name
if vertical_fill is None:
vertical_fill = -9999.9
self.vertical_fill = vertical_fill
self._nc = EnhancedDataset(self.out_file, 'a')
self.setup_times_and_verticals(times, verticals)
logger.info("Created file at '{}'".format(self.out_file))