def test_get_dataset_to_xml(self):
def init(self):
super(RegistrationProcess, self).__init__()
self.CFG = CFG
RegistrationProcess.__init__ = init
self.rp = RegistrationProcess()
self.rp.on_start()
dataset_id = self._make_dataset()
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
cov = SimplexCoverage.load(coverage_path)
xml_str = self.rp.get_dataset_xml(coverage_path, 'product_id', 'product_name')
dom = parseString(xml_str)
node = dom.getElementsByTagName('addAttributes')
metadata = node[0]
for n in metadata.childNodes:
if n.nodeType != 3:
if n.attributes["name"].value == "title":
self.assertEquals('product_name', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "institution":
self.assertEquals('OOI', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "infoUrl":
self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue)
parameters = []
node = dom.getElementsByTagName('sourceName')
for n in node:
if n.nodeType != 3:
parameters.append(str(n.childNodes[0].nodeValue))
cov_params = [key for key in cov.list_parameters()]
for p in parameters:
self.assertIn(p, cov_params)
cov.close()
def test_plot_1():
from coverage_model.test.examples import SimplexCoverage
import matplotlib.pyplot as plt
cov=SimplexCoverage.load('test_data/usgs.cov')
log.debug('Plot the \'water_temperature\' and \'streamflow\' for all times')
wtemp = cov.get_parameter_values('water_temperature')
wtemp_pc = cov.get_parameter_context('water_temperature')
sflow = cov.get_parameter_values('streamflow')
sflow_pc = cov.get_parameter_context('streamflow')
times = cov.get_parameter_values('time')
time_pc = cov.get_parameter_context('time')
fig = plt.figure()
ax1 = fig.add_subplot(2,1,1)
ax1.plot(times,wtemp)
ax1.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom))
ax1.set_ylabel('{0} ({1})'.format(wtemp_pc.name, wtemp_pc.uom))
ax2 = fig.add_subplot(2,1,2)
ax2.plot(times,sflow)
ax2.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom))
ax2.set_ylabel('{0} ({1})'.format(sflow_pc.name, sflow_pc.uom))
plt.show(0)
def test_get_dataset_to_xml(self):
dataset_id = self._make_dataset()
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
cov = SimplexCoverage.load(coverage_path)
xml_str = self.rp.get_dataset_xml(coverage_path)
dom = parseString(xml_str)
node = dom.getElementsByTagName('addAttributes')
metadata = node[0]
for n in metadata.childNodes:
if n.nodeType != 3:
if n.attributes["name"].value == "title":
self.assertEquals(cov.name, n.childNodes[0].nodeValue)
if n.attributes["name"].value == "institution":
self.assertEquals('OOI', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "infoUrl":
self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue)
parameters = []
node = dom.getElementsByTagName('sourceName')
for n in node:
if n.nodeType != 3:
parameters.append(str(n.childNodes[0].nodeValue))
cov_params = [key for key in cov.list_parameters()]
self.assertEquals(parameters, cov_params)
cov.close()
def test_get_dataset_to_xml(self):
dataset_id = self._make_dataset()
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
cov = SimplexCoverage.load(coverage_path)
xml_str = self.rp.get_dataset_xml(coverage_path)
dom = parseString(xml_str)
node = dom.getElementsByTagName('addAttributes')
metadata = node[0]
for n in metadata.childNodes:
if n.nodeType != 3:
if n.attributes["name"].value == "title":
self.assertEquals(cov.name, n.childNodes[0].nodeValue)
if n.attributes["name"].value == "institution":
self.assertEquals('OOI', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "infoUrl":
self.assertEquals(self.rp.pydap_url + cov.name,
n.childNodes[0].nodeValue)
parameters = []
node = dom.getElementsByTagName('sourceName')
for n in node:
if n.nodeType != 3:
parameters.append(str(n.childNodes[0].nodeValue))
cov_params = [key for key in cov.list_parameters()]
self.assertEquals(parameters, cov_params)
cov.close()
def methodized_read():
from coverage_model import SimplexCoverage
from coverage_model.basic_types import DomainOfApplication
import numpy as np
import os
log.debug('============ Station ============')
pth = 'test_data/usgs.cov'
if not os.path.exists(pth):
raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth))
cov=SimplexCoverage.load(pth)
ra=np.zeros([0])
log.debug('\n>> All data for first timestep\n')
log.debug(cov.get_parameter_values('water_temperature',0,None,ra))
log.debug('\n>> All data\n')
log.debug(cov.get_parameter_values('water_temperature',None,None,None))
log.debug('\n>> All data for second, fifth and sixth timesteps\n')
log.debug(cov.get_parameter_values('water_temperature',[[1,4,5]],None,None))
log.debug('\n>> First datapoint (in x) for every 5th timestep\n')
log.debug(cov.get_parameter_values('water_temperature',slice(0,None,5),0,None))
log.debug('\n>> First datapoint for first 10 timesteps, passing DOA objects\n')
tdoa = DomainOfApplication(slice(0,10))
sdoa = DomainOfApplication(0)
log.debug(cov.get_parameter_values('water_temperature',tdoa,sdoa,None))
log.debug('\n============ Grid ============')
pth = 'test_data/ncom.cov'
if not os.path.exists(pth):
raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth))
cov=SimplexCoverage.load(pth)
ra=np.zeros([0])
log.debug('\n>> All data for first timestep\n')
log.debug(cov.get_parameter_values('water_temp',0,None,ra))
log.debug('\n>> All data\n')
log.debug(cov.get_parameter_values('water_temp',None,None,None))
log.debug('\n>> All data for first, fourth, and fifth timesteps\n')
log.debug(cov.get_parameter_values('water_temp',[[0,3,4]],None,None))
log.debug('\n>> Data from z=0, y=10, x=10 for every 2nd timestep\n')
log.debug(cov.get_parameter_values('water_temp',slice(0,None,2),[0,10,10],None))
log.debug('\n>> Data from z=0-10, y=10, x=10 for the first 2 timesteps, passing DOA objects\n')
tdoa = DomainOfApplication(slice(0,2))
sdoa = DomainOfApplication([slice(0,10),10,10])
log.debug(cov.get_parameter_values('water_temp',tdoa,sdoa,None))
def execute(self, quals, req_columns):
#WARNING: qualField:time qualOperator:>= qualValue:2011-02-11 00:00:00
#WARNING: qualField:time qualOperator:<= qualValue:2011-02-12 23:59:59
log_to_postgres("dir:"+os.getcwd())
os.chdir("/Users/rpsdev/externalization")
log_to_postgres("dir:"+os.getcwd())
#cov_path = self.cov_path[:len(self.cov_path)-len(self.cov_id)]
log_to_postgres("LOADING Coverage At Path: "+self.cov_path, WARNING)
log_to_postgres("LOADING Coverage ID: "+self.cov_id, WARNING)
cov_available = False
try:
log_to_postgres("LOADING Coverage", WARNING)
cov = SimplexCoverage.load(self.cov_path)
cov_available = True
log_to_postgres("Cov Type:"+type(cov))
except Exception, e:
log_to_postgres("failed to load coverage...:" + str(e),WARNING)
def get_dataset_xml(self, coverage_path, product_name=''):
#http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html
result = ''
paths = os.path.split(coverage_path)
cov = SimplexCoverage.load(coverage_path)
doc = xml.dom.minidom.Document()
#erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'}
#Get lists of variables with unique sets of dimensions.
#Datasets can only have variables with the same sets of dimensions
if not cov.list_parameters():
raise BadRequest('Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' %(coverage_path, cov))
datasets = {}
for key in cov.list_parameters():
pc = cov.get_parameter_context(key)
#if not isinstance(pc.param_type, QuantityType):
# continue
param = cov.get_parameter(key)
dims = (cov.temporal_parameter_name,)
if len(param.shape) == 2:
dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name)
if not dims in datasets.keys():
datasets[dims] = []
datasets[dims].append(key)
index = 0
if not datasets:
raise BadRequest('Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' %( coverage_path, cov))
for dims, vars in datasets.iteritems():
erd_name_map = self.get_errdap_name_map(vars)
if len(vars)==1:
raise BadRequest('A dataset needs a proper range, not just the temporal dimension. %s\n%s' %( coverage_path, cov))
if not (len(dims) == 1 and dims[0] == vars[0]):
dataset_element = doc.createElement('dataset')
dataset_element.setAttribute('type', 'EDDGridFromDap')
dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index))
dataset_element.setAttribute('active', 'True')
source_element = doc.createElement('sourceUrl')
text_node = doc.createTextNode(self.pydap_url + paths[1])
source_element.appendChild(text_node)
dataset_element.appendChild(source_element)
reload_element = doc.createElement('reloadEveryNMinutes')
text_node = doc.createTextNode('5')
reload_element.appendChild(text_node)
dataset_element.appendChild(reload_element)
add_attributes_element = doc.createElement('addAttributes')
atts = {}
atts['title'] = product_name or urllib.unquote(cov.name)
atts['infoUrl'] = self.pydap_url + paths[1]
atts['institution'] = 'OOI'
atts['Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"
atts['license'] = '[standard]'
atts['summary'] = cov.name
atts['cdm_data_type'] = 'Grid'
atts['subsetVariables'] = ','.join([erd_name_map[v] for v in vars])
atts['standard_name_vocabulary'] = 'CF-12'
try:
lat_min,lat_max = cov.get_data_bounds("lat")
atts['geospatial_lat_max'] = str(lat_max)
atts['geospatial_lat_min'] = str(lat_min)
pc = cov.get_parameter_context("lat")
atts['geospatial_lat_units'] = str(pc.uom)
lon_min,lon_max = cov.get_data_bounds("lon")
atts['geospatial_lon_max'] = str(lon_max)
atts['geospatial_lon_min'] = str(lon_min)
pc = cov.get_parameter_context("lon")
atts['geospatial_lon_units'] = str(pc.uom)
except:
#silently fail and just don't fill attributes
pass
for key, val in atts.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
if len(add_attributes_element.childNodes) > 0:
dataset_element.appendChild(add_attributes_element)
for var_name in vars:
var = cov.get_parameter_context(var_name)
units = "unknown"
if hasattr(var,'uom') and var.uom:
units = var.uom
#if len(param.shape) >=1 and not param.is_coordinate: #dataVariable
data_element = doc.createElement('dataVariable')
source_name_element = doc.createElement('sourceName')
text_node = doc.createTextNode(var.name)
source_name_element.appendChild(text_node)
data_element.appendChild(source_name_element)
destination_name_element = doc.createElement('destinationName')
text_node = doc.createTextNode(erd_name_map[var.name])
destination_name_element.appendChild(text_node)
data_element.appendChild(destination_name_element)
add_attributes_element = doc.createElement('addAttributes')
if var.ATTRS is not None:
for key in var.ATTRS:
if not hasattr(var,key):
continue
val = getattr(var,key)
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'ioos_category')
text_node = doc.createTextNode(self.get_ioos_category(var.name, units))
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'long_name')
long_name = ""
if hasattr(var,'display_name') and var.display_name is not None:
long_name = var.display_name
text_node = doc.createTextNode(long_name)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
standard_name = ""
if hasattr(var,'standard_name') and var.standard_name is not None:
standard_name = var.standard_name
att_element.setAttribute('name', 'standard_name')
text_node = doc.createTextNode(standard_name)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'units')
text_node = doc.createTextNode(units)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
data_element.appendChild(add_attributes_element)
dataset_element.appendChild(data_element)
index += 1
#bug with prettyxml
#http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/
#result += dataset_element.toprettyxml() + '\n'
result += dataset_element.toxml() + '\n'
cov.close()
return result
def get_dataset_xml(self, coverage_path, product_name=''):
#http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html
result = ''
paths = os.path.split(coverage_path)
cov = SimplexCoverage.load(coverage_path)
doc = xml.dom.minidom.Document()
#erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'}
#Get lists of variables with unique sets of dimensions.
#Datasets can only have variables with the same sets of dimensions
if not cov.list_parameters():
raise BadRequest(
'Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s'
% (coverage_path, cov))
datasets = {}
for key in cov.list_parameters():
pc = cov.get_parameter_context(key)
#if not isinstance(pc.param_type, QuantityType):
# continue
param = cov.get_parameter(key)
dims = (cov.temporal_parameter_name, )
if len(param.shape) == 2:
dims = (cov.temporal_parameter_name,
cov.spatial_domain.shape.name)
if not dims in datasets.keys():
datasets[dims] = []
datasets[dims].append(key)
index = 0
if not datasets:
raise BadRequest(
'Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s'
% (coverage_path, cov))
for dims, vars in datasets.iteritems():
erd_name_map = self.get_errdap_name_map(vars)
if len(vars) == 1:
raise BadRequest(
'A dataset needs a proper range, not just the temporal dimension. %s\n%s'
% (coverage_path, cov))
if not (len(dims) == 1 and dims[0] == vars[0]):
dataset_element = doc.createElement('dataset')
dataset_element.setAttribute('type', 'EDDGridFromDap')
dataset_element.setAttribute('datasetID',
'{0}_{1}'.format(paths[1], index))
dataset_element.setAttribute('active', 'True')
source_element = doc.createElement('sourceUrl')
text_node = doc.createTextNode(self.pydap_url + paths[1])
source_element.appendChild(text_node)
dataset_element.appendChild(source_element)
reload_element = doc.createElement('reloadEveryNMinutes')
text_node = doc.createTextNode('5')
reload_element.appendChild(text_node)
dataset_element.appendChild(reload_element)
add_attributes_element = doc.createElement('addAttributes')
atts = {}
atts['title'] = product_name or urllib.unquote(cov.name)
atts['infoUrl'] = self.pydap_url + paths[1]
atts['institution'] = 'OOI'
atts[
'Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"
atts['license'] = '[standard]'
atts['summary'] = cov.name
atts['cdm_data_type'] = 'Grid'
atts['subsetVariables'] = ','.join(
[erd_name_map[v] for v in vars])
atts['standard_name_vocabulary'] = 'CF-12'
try:
lat_min, lat_max = cov.get_data_bounds("lat")
atts['geospatial_lat_max'] = str(lat_max)
atts['geospatial_lat_min'] = str(lat_min)
pc = cov.get_parameter_context("lat")
atts['geospatial_lat_units'] = str(pc.uom)
lon_min, lon_max = cov.get_data_bounds("lon")
atts['geospatial_lon_max'] = str(lon_max)
atts['geospatial_lon_min'] = str(lon_min)
pc = cov.get_parameter_context("lon")
atts['geospatial_lon_units'] = str(pc.uom)
except:
#silently fail and just don't fill attributes
pass
for key, val in atts.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
if len(add_attributes_element.childNodes) > 0:
dataset_element.appendChild(add_attributes_element)
for var_name in vars:
param = cov.get_parameter(var_name)
var = param.context
units = "unknown"
try:
units = var.uom
except:
pass
if units is None:
units = "unknown"
#if len(param.shape) >=1 and not param.is_coordinate: #dataVariable
data_element = doc.createElement('dataVariable')
source_name_element = doc.createElement('sourceName')
text_node = doc.createTextNode(var.name)
source_name_element.appendChild(text_node)
data_element.appendChild(source_name_element)
destination_name_element = doc.createElement(
'destinationName')
text_node = doc.createTextNode(erd_name_map[var.name])
destination_name_element.appendChild(text_node)
data_element.appendChild(destination_name_element)
add_attributes_element = doc.createElement('addAttributes')
if not var.attributes is None:
for key, val in var.attributes.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'ioos_category')
text_node = doc.createTextNode(
self.get_ioos_category(var.name, units))
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'long_name')
long_name = ""
if var.long_name is not None:
long_name = var.long_name
text_node = doc.createTextNode(long_name)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
standard_name = ""
if var.standard_name is not None:
standard_name = var.standard_name
att_element.setAttribute('name', 'standard_name')
text_node = doc.createTextNode(standard_name)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'units')
text_node = doc.createTextNode(units)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
data_element.appendChild(add_attributes_element)
dataset_element.appendChild(data_element)
index += 1
#bug with prettyxml
#http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/
#result += dataset_element.toprettyxml() + '\n'
result += dataset_element.toxml() + '\n'
cov.close()
return result
