def test_get_dataset_to_xml(self): def init(self): super(RegistrationProcess, self).__init__() self.CFG = CFG RegistrationProcess.__init__ = init self.rp = RegistrationProcess() self.rp.on_start() dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path, 'product_id', 'product_name') dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals('product_name', n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] for p in parameters: self.assertIn(p, cov_params) cov.close()
def test_plot_1(): from coverage_model.test.examples import SimplexCoverage import matplotlib.pyplot as plt cov=SimplexCoverage.load('test_data/usgs.cov') log.debug('Plot the \'water_temperature\' and \'streamflow\' for all times') wtemp = cov.get_parameter_values('water_temperature') wtemp_pc = cov.get_parameter_context('water_temperature') sflow = cov.get_parameter_values('streamflow') sflow_pc = cov.get_parameter_context('streamflow') times = cov.get_parameter_values('time') time_pc = cov.get_parameter_context('time') fig = plt.figure() ax1 = fig.add_subplot(2,1,1) ax1.plot(times,wtemp) ax1.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom)) ax1.set_ylabel('{0} ({1})'.format(wtemp_pc.name, wtemp_pc.uom)) ax2 = fig.add_subplot(2,1,2) ax2.plot(times,sflow) ax2.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom)) ax2.set_ylabel('{0} ({1})'.format(sflow_pc.name, sflow_pc.uom)) plt.show(0)
def test_get_dataset_to_xml(self): dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path) dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals(cov.name, n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] self.assertEquals(parameters, cov_params) cov.close()
def test_get_dataset_to_xml(self): dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path) dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals(cov.name, n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url + cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] self.assertEquals(parameters, cov_params) cov.close()
def methodized_read(): from coverage_model import SimplexCoverage from coverage_model.basic_types import DomainOfApplication import numpy as np import os log.debug('============ Station ============') pth = 'test_data/usgs.cov' if not os.path.exists(pth): raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth)) cov=SimplexCoverage.load(pth) ra=np.zeros([0]) log.debug('\n>> All data for first timestep\n') log.debug(cov.get_parameter_values('water_temperature',0,None,ra)) log.debug('\n>> All data\n') log.debug(cov.get_parameter_values('water_temperature',None,None,None)) log.debug('\n>> All data for second, fifth and sixth timesteps\n') log.debug(cov.get_parameter_values('water_temperature',[[1,4,5]],None,None)) log.debug('\n>> First datapoint (in x) for every 5th timestep\n') log.debug(cov.get_parameter_values('water_temperature',slice(0,None,5),0,None)) log.debug('\n>> First datapoint for first 10 timesteps, passing DOA objects\n') tdoa = DomainOfApplication(slice(0,10)) sdoa = DomainOfApplication(0) log.debug(cov.get_parameter_values('water_temperature',tdoa,sdoa,None)) log.debug('\n============ Grid ============') pth = 'test_data/ncom.cov' if not os.path.exists(pth): raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth)) cov=SimplexCoverage.load(pth) ra=np.zeros([0]) log.debug('\n>> All data for first timestep\n') log.debug(cov.get_parameter_values('water_temp',0,None,ra)) log.debug('\n>> All data\n') log.debug(cov.get_parameter_values('water_temp',None,None,None)) log.debug('\n>> All data for first, fourth, and fifth timesteps\n') log.debug(cov.get_parameter_values('water_temp',[[0,3,4]],None,None)) log.debug('\n>> Data from z=0, y=10, x=10 for every 2nd timestep\n') log.debug(cov.get_parameter_values('water_temp',slice(0,None,2),[0,10,10],None)) log.debug('\n>> Data from z=0-10, y=10, x=10 for the first 2 timesteps, passing DOA objects\n') tdoa = DomainOfApplication(slice(0,2)) sdoa = DomainOfApplication([slice(0,10),10,10]) log.debug(cov.get_parameter_values('water_temp',tdoa,sdoa,None))
def execute(self, quals, req_columns): #WARNING: qualField:time qualOperator:>= qualValue:2011-02-11 00:00:00 #WARNING: qualField:time qualOperator:<= qualValue:2011-02-12 23:59:59 log_to_postgres("dir:"+os.getcwd()) os.chdir("/Users/rpsdev/externalization") log_to_postgres("dir:"+os.getcwd()) #cov_path = self.cov_path[:len(self.cov_path)-len(self.cov_id)] log_to_postgres("LOADING Coverage At Path: "+self.cov_path, WARNING) log_to_postgres("LOADING Coverage ID: "+self.cov_id, WARNING) cov_available = False try: log_to_postgres("LOADING Coverage", WARNING) cov = SimplexCoverage.load(self.cov_path) cov_available = True log_to_postgres("Cov Type:"+type(cov)) except Exception, e: log_to_postgres("failed to load coverage...:" + str(e),WARNING)
def get_dataset_xml(self, coverage_path, product_name=''): #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html result = '' paths = os.path.split(coverage_path) cov = SimplexCoverage.load(coverage_path) doc = xml.dom.minidom.Document() #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} #Get lists of variables with unique sets of dimensions. #Datasets can only have variables with the same sets of dimensions if not cov.list_parameters(): raise BadRequest('Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' %(coverage_path, cov)) datasets = {} for key in cov.list_parameters(): pc = cov.get_parameter_context(key) #if not isinstance(pc.param_type, QuantityType): # continue param = cov.get_parameter(key) dims = (cov.temporal_parameter_name,) if len(param.shape) == 2: dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name) if not dims in datasets.keys(): datasets[dims] = [] datasets[dims].append(key) index = 0 if not datasets: raise BadRequest('Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' %( coverage_path, cov)) for dims, vars in datasets.iteritems(): erd_name_map = self.get_errdap_name_map(vars) if len(vars)==1: raise BadRequest('A dataset needs a proper range, not just the temporal dimension. %s\n%s' %( coverage_path, cov)) if not (len(dims) == 1 and dims[0] == vars[0]): dataset_element = doc.createElement('dataset') dataset_element.setAttribute('type', 'EDDGridFromDap') dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index)) dataset_element.setAttribute('active', 'True') source_element = doc.createElement('sourceUrl') text_node = doc.createTextNode(self.pydap_url + paths[1]) source_element.appendChild(text_node) dataset_element.appendChild(source_element) reload_element = doc.createElement('reloadEveryNMinutes') text_node = doc.createTextNode('5') reload_element.appendChild(text_node) dataset_element.appendChild(reload_element) add_attributes_element = doc.createElement('addAttributes') atts = {} atts['title'] = product_name or urllib.unquote(cov.name) atts['infoUrl'] = self.pydap_url + paths[1] atts['institution'] = 'OOI' atts['Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0" atts['license'] = '[standard]' atts['summary'] = cov.name atts['cdm_data_type'] = 'Grid' atts['subsetVariables'] = ','.join([erd_name_map[v] for v in vars]) atts['standard_name_vocabulary'] = 'CF-12' try: lat_min,lat_max = cov.get_data_bounds("lat") atts['geospatial_lat_max'] = str(lat_max) atts['geospatial_lat_min'] = str(lat_min) pc = cov.get_parameter_context("lat") atts['geospatial_lat_units'] = str(pc.uom) lon_min,lon_max = cov.get_data_bounds("lon") atts['geospatial_lon_max'] = str(lon_max) atts['geospatial_lon_min'] = str(lon_min) pc = cov.get_parameter_context("lon") atts['geospatial_lon_units'] = str(pc.uom) except: #silently fail and just don't fill attributes pass for key, val in atts.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) if len(add_attributes_element.childNodes) > 0: dataset_element.appendChild(add_attributes_element) for var_name in vars: var = cov.get_parameter_context(var_name) units = "unknown" if hasattr(var,'uom') and var.uom: units = var.uom #if len(param.shape) >=1 and not param.is_coordinate: #dataVariable data_element = doc.createElement('dataVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(var.name) source_name_element.appendChild(text_node) data_element.appendChild(source_name_element) destination_name_element = doc.createElement('destinationName') text_node = doc.createTextNode(erd_name_map[var.name]) destination_name_element.appendChild(text_node) data_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if var.ATTRS is not None: for key in var.ATTRS: if not hasattr(var,key): continue val = getattr(var,key) att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode(self.get_ioos_category(var.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'long_name') long_name = "" if hasattr(var,'display_name') and var.display_name is not None: long_name = var.display_name text_node = doc.createTextNode(long_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') standard_name = "" if hasattr(var,'standard_name') and var.standard_name is not None: standard_name = var.standard_name att_element.setAttribute('name', 'standard_name') text_node = doc.createTextNode(standard_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'units') text_node = doc.createTextNode(units) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) data_element.appendChild(add_attributes_element) dataset_element.appendChild(data_element) index += 1 #bug with prettyxml #http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/ #result += dataset_element.toprettyxml() + '\n' result += dataset_element.toxml() + '\n' cov.close() return result
def get_dataset_xml(self, coverage_path, product_name=''): #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html result = '' paths = os.path.split(coverage_path) cov = SimplexCoverage.load(coverage_path) doc = xml.dom.minidom.Document() #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} #Get lists of variables with unique sets of dimensions. #Datasets can only have variables with the same sets of dimensions if not cov.list_parameters(): raise BadRequest( 'Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' % (coverage_path, cov)) datasets = {} for key in cov.list_parameters(): pc = cov.get_parameter_context(key) #if not isinstance(pc.param_type, QuantityType): # continue param = cov.get_parameter(key) dims = (cov.temporal_parameter_name, ) if len(param.shape) == 2: dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name) if not dims in datasets.keys(): datasets[dims] = [] datasets[dims].append(key) index = 0 if not datasets: raise BadRequest( 'Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' % (coverage_path, cov)) for dims, vars in datasets.iteritems(): erd_name_map = self.get_errdap_name_map(vars) if len(vars) == 1: raise BadRequest( 'A dataset needs a proper range, not just the temporal dimension. %s\n%s' % (coverage_path, cov)) if not (len(dims) == 1 and dims[0] == vars[0]): dataset_element = doc.createElement('dataset') dataset_element.setAttribute('type', 'EDDGridFromDap') dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index)) dataset_element.setAttribute('active', 'True') source_element = doc.createElement('sourceUrl') text_node = doc.createTextNode(self.pydap_url + paths[1]) source_element.appendChild(text_node) dataset_element.appendChild(source_element) reload_element = doc.createElement('reloadEveryNMinutes') text_node = doc.createTextNode('5') reload_element.appendChild(text_node) dataset_element.appendChild(reload_element) add_attributes_element = doc.createElement('addAttributes') atts = {} atts['title'] = product_name or urllib.unquote(cov.name) atts['infoUrl'] = self.pydap_url + paths[1] atts['institution'] = 'OOI' atts[ 'Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0" atts['license'] = '[standard]' atts['summary'] = cov.name atts['cdm_data_type'] = 'Grid' atts['subsetVariables'] = ','.join( [erd_name_map[v] for v in vars]) atts['standard_name_vocabulary'] = 'CF-12' try: lat_min, lat_max = cov.get_data_bounds("lat") atts['geospatial_lat_max'] = str(lat_max) atts['geospatial_lat_min'] = str(lat_min) pc = cov.get_parameter_context("lat") atts['geospatial_lat_units'] = str(pc.uom) lon_min, lon_max = cov.get_data_bounds("lon") atts['geospatial_lon_max'] = str(lon_max) atts['geospatial_lon_min'] = str(lon_min) pc = cov.get_parameter_context("lon") atts['geospatial_lon_units'] = str(pc.uom) except: #silently fail and just don't fill attributes pass for key, val in atts.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) if len(add_attributes_element.childNodes) > 0: dataset_element.appendChild(add_attributes_element) for var_name in vars: param = cov.get_parameter(var_name) var = param.context units = "unknown" try: units = var.uom except: pass if units is None: units = "unknown" #if len(param.shape) >=1 and not param.is_coordinate: #dataVariable data_element = doc.createElement('dataVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(var.name) source_name_element.appendChild(text_node) data_element.appendChild(source_name_element) destination_name_element = doc.createElement( 'destinationName') text_node = doc.createTextNode(erd_name_map[var.name]) destination_name_element.appendChild(text_node) data_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if not var.attributes is None: for key, val in var.attributes.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode( self.get_ioos_category(var.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'long_name') long_name = "" if var.long_name is not None: long_name = var.long_name text_node = doc.createTextNode(long_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') standard_name = "" if var.standard_name is not None: standard_name = var.standard_name att_element.setAttribute('name', 'standard_name') text_node = doc.createTextNode(standard_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'units') text_node = doc.createTextNode(units) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) data_element.appendChild(add_attributes_element) dataset_element.appendChild(data_element) index += 1 #bug with prettyxml #http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/ #result += dataset_element.toprettyxml() + '\n' result += dataset_element.toxml() + '\n' cov.close() return result