def oneparamcov_noautoflush(save_coverage=False, in_memory=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory, auto_flush_values=False) # Insert some timesteps (automatically expands other arrays) nt = 100 scov.insert_timesteps(nt) # Add data for the parameter scov.set_parameter_values('time', value=np.arange(nt)) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/sample.cov') return scov
def test_get_dataset_to_xml(self): dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path) dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals(cov.name, n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url + cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] self.assertEquals(parameters, cov_params) cov.close()
def test_plot_1(): from coverage_model.test.examples import SimplexCoverage import matplotlib.pyplot as plt cov=SimplexCoverage.load('test_data/usgs.cov') log.debug('Plot the \'water_temperature\' and \'streamflow\' for all times') wtemp = cov.get_parameter_values('water_temperature') wtemp_pc = cov.get_parameter_context('water_temperature') sflow = cov.get_parameter_values('streamflow') sflow_pc = cov.get_parameter_context('streamflow') times = cov.get_parameter_values('time') time_pc = cov.get_parameter_context('time') fig = plt.figure() ax1 = fig.add_subplot(2,1,1) ax1.plot(times,wtemp) ax1.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom)) ax1.set_ylabel('{0} ({1})'.format(wtemp_pc.name, wtemp_pc.uom)) ax2 = fig.add_subplot(2,1,2) ax2.plot(times,sflow) ax2.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom)) ax2.set_ylabel('{0} ({1})'.format(sflow_pc.name, sflow_pc.uom)) plt.show(0)
def test_get_dataset_to_xml(self): dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path) dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals(cov.name, n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] self.assertEquals(parameters, cov_params) cov.close()
def test_get_dataset_to_xml(self): def init(self): super(RegistrationProcess, self).__init__() self.CFG = CFG RegistrationProcess.__init__ = init self.rp = RegistrationProcess() self.rp.on_start() dataset_id = self._make_dataset() coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) cov = SimplexCoverage.load(coverage_path) xml_str = self.rp.get_dataset_xml(coverage_path, 'product_id', 'product_name') dom = parseString(xml_str) node = dom.getElementsByTagName('addAttributes') metadata = node[0] for n in metadata.childNodes: if n.nodeType != 3: if n.attributes["name"].value == "title": self.assertEquals('product_name', n.childNodes[0].nodeValue) if n.attributes["name"].value == "institution": self.assertEquals('OOI', n.childNodes[0].nodeValue) if n.attributes["name"].value == "infoUrl": self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue) parameters = [] node = dom.getElementsByTagName('sourceName') for n in node: if n.nodeType != 3: parameters.append(str(n.childNodes[0].nodeValue)) cov_params = [key for key in cov.list_parameters()] for p in parameters: self.assertIn(p, cov_params) cov.close()
def methodized_read(): from coverage_model import SimplexCoverage from coverage_model.basic_types import DomainOfApplication import numpy as np import os log.debug('============ Station ============') pth = 'test_data/usgs.cov' if not os.path.exists(pth): raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth)) cov=SimplexCoverage.load(pth) ra=np.zeros([0]) log.debug('\n>> All data for first timestep\n') log.debug(cov.get_parameter_values('water_temperature',0,None,ra)) log.debug('\n>> All data\n') log.debug(cov.get_parameter_values('water_temperature',None,None,None)) log.debug('\n>> All data for second, fifth and sixth timesteps\n') log.debug(cov.get_parameter_values('water_temperature',[[1,4,5]],None,None)) log.debug('\n>> First datapoint (in x) for every 5th timestep\n') log.debug(cov.get_parameter_values('water_temperature',slice(0,None,5),0,None)) log.debug('\n>> First datapoint for first 10 timesteps, passing DOA objects\n') tdoa = DomainOfApplication(slice(0,10)) sdoa = DomainOfApplication(0) log.debug(cov.get_parameter_values('water_temperature',tdoa,sdoa,None)) log.debug('\n============ Grid ============') pth = 'test_data/ncom.cov' if not os.path.exists(pth): raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth)) cov=SimplexCoverage.load(pth) ra=np.zeros([0]) log.debug('\n>> All data for first timestep\n') log.debug(cov.get_parameter_values('water_temp',0,None,ra)) log.debug('\n>> All data\n') log.debug(cov.get_parameter_values('water_temp',None,None,None)) log.debug('\n>> All data for first, fourth, and fifth timesteps\n') log.debug(cov.get_parameter_values('water_temp',[[0,3,4]],None,None)) log.debug('\n>> Data from z=0, y=10, x=10 for every 2nd timestep\n') log.debug(cov.get_parameter_values('water_temp',slice(0,None,2),[0,10,10],None)) log.debug('\n>> Data from z=0-10, y=10, x=10 for the first 2 timesteps, passing DOA objects\n') tdoa = DomainOfApplication(slice(0,2)) sdoa = DomainOfApplication([slice(0,10),10,10]) log.debug(cov.get_parameter_values('water_temp',tdoa,sdoa,None))
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext(name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0,len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def execute(self, quals, req_columns): #WARNING: qualField:time qualOperator:>= qualValue:2011-02-11 00:00:00 #WARNING: qualField:time qualOperator:<= qualValue:2011-02-12 23:59:59 log_to_postgres("dir:"+os.getcwd()) os.chdir("/Users/rpsdev/externalization") log_to_postgres("dir:"+os.getcwd()) #cov_path = self.cov_path[:len(self.cov_path)-len(self.cov_id)] log_to_postgres("LOADING Coverage At Path: "+self.cov_path, WARNING) log_to_postgres("LOADING Coverage ID: "+self.cov_id, WARNING) cov_available = False try: log_to_postgres("LOADING Coverage", WARNING) cov = SimplexCoverage.load(self.cov_path) cov_available = True log_to_postgres("Cov Type:"+type(cov)) except Exception, e: log_to_postgres("failed to load coverage...:" + str(e),WARNING)
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def ncstation2cov(save_coverage=False, in_memory=False): # Open the netcdf dataset ds = Dataset('test_data/usgs.nc') # Itemize the variable names that we want to include in the coverage var_names = ['time','lat','lon','z','streamflow','water_temperature',] # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a ParameterContext object for each of the variables in the dataset and add them to the ParameterDictionaryl for v in var_names: var = ds.variables[v] ptype = QuantityType(var.dtype.char) if v in ('lat','lon','z'): ptype=ConstantType(ptype) pcontext = ParameterContext(v, param_type=ptype) if 'units' in var.ncattrs(): pcontext.uom = var.getncattr('units') if 'long_name' in var.ncattrs(): pcontext.description = var.getncattr('long_name') if '_FillValue' in var.ncattrs(): pcontext.fill_value = var.getncattr('_FillValue') # Set the axis for the coordinate parameters if v == 'time': pcontext.variability = VariabilityEnum.TEMPORAL pcontext.axis = AxisTypeEnum.TIME elif v == 'lat': pcontext.axis = AxisTypeEnum.LAT elif v == 'lon': pcontext.axis = AxisTypeEnum.LON elif v == 'z': pcontext.axis = AxisTypeEnum.HEIGHT pdict.add_context(pcontext) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS.standard_temporal() scrs = CRS.lat_lon_height() # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample station coverage_model', pdict, tdom, sdom, in_memory) # Insert the timesteps (automatically expands other arrays) tvar=ds.variables['time'] scov.insert_timesteps(tvar.size) # Add data to the parameters - NOT using setters at this point, direct assignment to arrays for v in var_names: var = ds.variables[v] var.set_auto_maskandscale(False) if v in ('lat','lon','z'): scov._range_value[v][0] = make_range_expr(var[0]) else: scov._range_value[v][:] = var[:] if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/usgs.cov') return scov
def ncgrid2cov(save_coverage=False, in_memory=False): if True: raise NotImplementedError('Multidimensional support is not available at this time') # Open the netcdf dataset ds = Dataset('test_data/ncom.nc') # Itemize the variable names that we want to include in the coverage var_names = ['time','lat','lon','depth','water_u','water_v','salinity','water_temp',] # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a ParameterContext object for each of the variables in the dataset and add them to the ParameterDictionary for v in var_names: var = ds.variables[v] pcontext = ParameterContext(v, param_type=QuantityType(value_encoding=ds.variables[v].dtype.char)) if 'units' in var.ncattrs(): pcontext.uom = var.getncattr('units') if 'long_name' in var.ncattrs(): pcontext.description = var.getncattr('long_name') if '_FillValue' in var.ncattrs(): pcontext.fill_value = var.getncattr('_FillValue') # Set the axis for the coordinate parameters if v == 'time': pcontext.variability = VariabilityEnum.TEMPORAL pcontext.axis = AxisTypeEnum.TIME elif v == 'lat': pcontext.axis = AxisTypeEnum.LAT elif v == 'lon': pcontext.axis = AxisTypeEnum.LON elif v == 'depth': pcontext.axis = AxisTypeEnum.HEIGHT pdict.add_context(pcontext) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal'), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [34,57,89]), scrs, MutabilityEnum.IMMUTABLE) # 3d spatial topology (grid) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample grid coverage_model', pdict, tdom, sdom, in_memory) # Insert the timesteps (automatically expands other arrays) tvar=ds.variables['time'] scov.insert_timesteps(tvar.size) # Add data to the parameters - NOT using setters at this point, direct assignment to arrays for v in var_names: log.debug('Assign values to %s', v) var = ds.variables[v] var.set_auto_maskandscale(False) arr = var[:] # TODO: Sort out how to leave these sparse internally and only broadcast during read if v == 'depth': z,_,_ = my_meshgrid(arr,np.zeros([57]),np.zeros([89]),indexing='ij',sparse=True) scov._range_value[v][:] = z elif v == 'lat': _,y,_ = my_meshgrid(np.zeros([34]),arr,np.zeros([89]),indexing='ij',sparse=True) scov._range_value[v][:] = y elif v == 'lon': _,_,x = my_meshgrid(np.zeros([34]),np.zeros([57]),arr,indexing='ij',sparse=True) scov._range_value[v][:] = x else: scov._range_value[v][:] = var[:] if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/ncom.cov') return scov
def nospatialcov(save_coverage=False, in_memory=False): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32'))) quant_ctxt.long_name = 'example of a parameter of type QuantityType' quant_ctxt.uom = 'degree_Celsius' pdict.add_context(quant_ctxt) const_ctxt = ParameterContext('constant', param_type=ConstantType()) const_ctxt.long_name = 'example of a parameter of type ConstantType' pdict.add_context(const_ctxt) arr_ctxt = ParameterContext('array', param_type=ArrayType()) arr_ctxt.long_name = 'example of a parameter of type ArrayType with base_type ndarray (resolves to \'object\')' pdict.add_context(arr_ctxt) arr2_ctxt = ParameterContext('array2', param_type=ArrayType()) arr2_ctxt.long_name = 'example of a parameter of type ArrayType with base_type object' pdict.add_context(arr2_ctxt) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, temporal_domain=tdom, in_memory_storage=in_memory) # Insert some timesteps (automatically expands other arrays) nt = 20 scov.insert_timesteps(nt) # Add data for each parameter scov.set_parameter_values('quantity_time', value=np.arange(nt)) scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23) scov.set_parameter_values('constant', value=20) arrval = [] arr2val = [] for x in xrange(nt): # One value (which IS an array) for each member of the domain arrval.append(np.random.bytes(np.random.randint(1,20))) arr2val.append(np.random.random_sample(np.random.randint(1,10))) scov.set_parameter_values('array', value=arrval) scov.set_parameter_values('array2', value=arr2val) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov') return scov
def samplecov2(save_coverage=False, in_memory=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt, is_temporal=True) lat_ctxt = ParameterContext('lat', param_type=ConstantType(), variability=VariabilityEnum.NONE) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=ConstantType(), variability=VariabilityEnum.NONE) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects nt = 20 tdom = GridDomain(GridShape('temporal', [nt]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory) # Insert some timesteps (automatically expands other arrays) # scov.insert_timesteps(10) # Add data for each parameter scov.set_parameter_values('time', value=np.arange(nt)) scov.set_parameter_values('lat', value=make_range_expr(45.32)) scov.set_parameter_values('lon', value=make_range_expr(-71.11)) # make a random sample of 10 values between 23 and 26 # Ref: http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.random_sample.html#numpy.random.random_sample # --> To sample multiply the output of random_sample by (b-a) and add a tvals=np.random.random_sample(nt)*(26-23)+23 scov.set_parameter_values('temp', value=tvals) scov.set_parameter_values('conductivity', value=np.random.random_sample(nt)*(110-90)+90) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/sample2.cov') return scov
def repair(self, backup=True, copy_over=True, keep_temp=False, reanalyze=False, analyze_bricks=False, detailed_analysis=False): """ Heavy repair tool that recreates a blank persisted Coverage from the broken coverage's original construction parameters, then reconstructs the Master and Parameter metadata files by inspection of the ION objects and "valid" brick files. @return: """ if self._ar is None or reanalyze: self._ar = self._do_analysis(analyze_bricks=analyze_bricks, detailed_analysis=detailed_analysis) if self._ar.is_corrupt: if len(self._ar.get_brick_corruptions()) > 0: raise NotImplementedError( 'Brick corruption. Cannot repair at this time!!!') else: # Repair the Master and Parameter metadata files # Need the ParameterDictionary, TemporalDomain and SpatialDomain pdict = ParameterDictionary.load( self._dso.parameter_dictionary) tdom = GridDomain.load(self._dso.temporal_domain) sdom = GridDomain.load(self._dso.spatial_domain) # Set up the working directory for the recovered coverage tempcov_dir = tempfile.mkdtemp('covs') # Create the temporary Coverage tempcov = SimplexCoverage(root_dir=tempcov_dir, persistence_guid=self._guid, name=self._guid, parameter_dictionary=pdict, spatial_domain=sdom, temporal_domain=tdom) # Handle to persistence layer for tempcov pl = tempcov._persistence_layer # Set up the original and temporary coverage path strings orig_dir = os.path.join(self.cov_pth, self._guid) temp_dir = os.path.join(tempcov.persistence_dir, tempcov.persistence_guid) # Insert same number of timesteps into temporary coverage as in broken coverage brick_domains_new, new_brick_list, brick_list_spans, tD, bD, min_data_bound, max_data_bound = self.inspect_bricks( self.cov_pth, self._guid, 'time') empty_cov = brick_list_spans is None # If None, there are no brick files --> no timesteps, empty coverage! if not empty_cov: # If None, there are no brick files --> no timesteps, empty coverage! bls = [s.value for s in brick_list_spans] maxes = [sum(b[3]) for b in new_brick_list.values()] tempcov.insert_timesteps(sum(maxes)) # Replace metadata is the Master file pl.master_manager.brick_domains = brick_domains_new pl.master_manager.brick_list = new_brick_list # Repair ExternalLinks to brick files f = h5py.File(pl.master_manager.file_path, 'a') for param_name in pdict.keys(): del f[param_name] f.create_group(param_name) for brick in bls: link_path = '/{0}/{1}'.format(param_name, brick[0]) brick_file_name = '{0}.hdf5'.format(brick[0]) brick_rel_path = os.path.join( pl.parameter_metadata[param_name].root_dir. replace(tempcov.persistence_dir, '.'), brick_file_name) log.debug('link_path: %s', link_path) log.debug('brick_rel_path: %s', brick_rel_path) pl.master_manager.add_external_link( link_path, brick_rel_path, brick[0]) pl.flush_values() pl.flush() tempcov.close() # Remove 'rtree' dataset from Master file if it already exists (post domain expansion) # to make way for reconstruction f = h5py.File(pl.master_manager.file_path, 'a') if 'rtree' in f.keys(): del f['rtree'] f.close() # Reconstruct 'rtree' dataset # Open temporary Coverage and PersistenceLayer objects fixed_cov = AbstractCoverage.load(tempcov.persistence_dir, mode='a') pl_fixed = fixed_cov._persistence_layer # Call update_rtree for each brick using PersistenceLayer builtin brick_count = 0 if not empty_cov: for brick in bls: rtree_extents, brick_extents, brick_active_size = pl_fixed.calculate_extents( brick[1][1], bD, tD) pl_fixed.master_manager.update_rtree(brick_count, rtree_extents, obj=brick[0]) brick_count += 1 # Update parameter_bounds property based on each parameter's brick data using deep inspection valid_bounds_types = [ 'BooleanType', 'ConstantType', 'QuantityType', 'ConstantRangeType' ] if not empty_cov: for param in pdict.keys(): if pdict.get_context( param ).param_type.__class__.__name__ in valid_bounds_types: brick_domains_new, new_brick_list, brick_list_spans, tD, bD, min_data_bound, max_data_bound = self.inspect_bricks( self.cov_pth, self._guid, param) # Update the metadata pl_fixed.update_parameter_bounds( param, [min_data_bound, max_data_bound]) pl_fixed.flush() fixed_cov.close() # Create backup copy of original Master and Parameter files if backup: import datetime orig_master_file = os.path.join( self.cov_pth, '{0}_master.hdf5'.format(self._guid)) # Generate the timestamp tstamp_format = '%Y%m%d%H%M%S' tstamp = datetime.datetime.now().strftime(tstamp_format) backup_master_file = os.path.join( self.cov_pth, '{0}_master.{1}.hdf5'.format(self._guid, tstamp)) shutil.copy2(orig_master_file, backup_master_file) for param in pdict.keys(): param_orig = os.path.join(orig_dir, param, '{0}.hdf5'.format(param)) param_backup = os.path.join( orig_dir, param, '{0}.{1}.hdf5'.format(param, tstamp)) shutil.copy2(param_orig, param_backup) # Copy Master and Parameter metadata files back to original/broken coverage (cov_pth) location if copy_over == True: shutil.copy2( os.path.join(tempcov.persistence_dir, '{0}_master.hdf5'.format(self._guid)), os.path.join(self.cov_pth, '{0}_master.hdf5'.format(self._guid))) for param in pdict.keys(): shutil.copy2( os.path.join(temp_dir, param, '{0}.hdf5'.format(param)), os.path.join(orig_dir, param, '{0}.hdf5'.format(param))) # Reanalyze the repaired coverage self._ar = self._do_analysis(analyze_bricks=True) # Verify repair worked, clean up if not if self._ar.is_corrupt: # If the files were backed up then revert if backup: # Remove backed up files and clean up the repair attempt log.info( 'Repair attempt failed. Reverting to pre-repair state.' ) # Use backup copy to replace post-repair file. shutil.copy2(backup_master_file, orig_master_file) # Delete the backup os.remove(backup_master_file) # Iterate over parameters and revert to pre-repair state for param in pdict.keys(): param_orig = os.path.join(orig_dir, param, '{0}.hdf5'.format(param)) param_backup = os.path.join( orig_dir, param, '{0}.{1}.hdf5'.format(param, tstamp)) # Use backup copy to replace post-repair file. shutil.copy2(param_backup, param_orig) # Delete the backup os.remove(param_backup) raise ValueError( 'Coverage repair failed! Revert to stored backup version, if possible.' ) # Remove temporary coverage if keep_temp == False: shutil.rmtree(tempcov_dir) else: return tempcov_dir else: log.info('Coverage is not corrupt, nothing to repair!')
def get_dataset_xml(self, coverage_path, product_name=''): #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html result = '' paths = os.path.split(coverage_path) cov = SimplexCoverage.load(coverage_path) doc = xml.dom.minidom.Document() #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} #Get lists of variables with unique sets of dimensions. #Datasets can only have variables with the same sets of dimensions if not cov.list_parameters(): raise BadRequest( 'Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' % (coverage_path, cov)) datasets = {} for key in cov.list_parameters(): pc = cov.get_parameter_context(key) #if not isinstance(pc.param_type, QuantityType): # continue param = cov.get_parameter(key) dims = (cov.temporal_parameter_name, ) if len(param.shape) == 2: dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name) if not dims in datasets.keys(): datasets[dims] = [] datasets[dims].append(key) index = 0 if not datasets: raise BadRequest( 'Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' % (coverage_path, cov)) for dims, vars in datasets.iteritems(): erd_name_map = self.get_errdap_name_map(vars) if len(vars) == 1: raise BadRequest( 'A dataset needs a proper range, not just the temporal dimension. %s\n%s' % (coverage_path, cov)) if not (len(dims) == 1 and dims[0] == vars[0]): dataset_element = doc.createElement('dataset') dataset_element.setAttribute('type', 'EDDGridFromDap') dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index)) dataset_element.setAttribute('active', 'True') source_element = doc.createElement('sourceUrl') text_node = doc.createTextNode(self.pydap_url + paths[1]) source_element.appendChild(text_node) dataset_element.appendChild(source_element) reload_element = doc.createElement('reloadEveryNMinutes') text_node = doc.createTextNode('5') reload_element.appendChild(text_node) dataset_element.appendChild(reload_element) add_attributes_element = doc.createElement('addAttributes') atts = {} atts['title'] = product_name or urllib.unquote(cov.name) atts['infoUrl'] = self.pydap_url + paths[1] atts['institution'] = 'OOI' atts[ 'Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0" atts['license'] = '[standard]' atts['summary'] = cov.name atts['cdm_data_type'] = 'Grid' atts['subsetVariables'] = ','.join( [erd_name_map[v] for v in vars]) atts['standard_name_vocabulary'] = 'CF-12' try: lat_min, lat_max = cov.get_data_bounds("lat") atts['geospatial_lat_max'] = str(lat_max) atts['geospatial_lat_min'] = str(lat_min) pc = cov.get_parameter_context("lat") atts['geospatial_lat_units'] = str(pc.uom) lon_min, lon_max = cov.get_data_bounds("lon") atts['geospatial_lon_max'] = str(lon_max) atts['geospatial_lon_min'] = str(lon_min) pc = cov.get_parameter_context("lon") atts['geospatial_lon_units'] = str(pc.uom) except: #silently fail and just don't fill attributes pass for key, val in atts.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) if len(add_attributes_element.childNodes) > 0: dataset_element.appendChild(add_attributes_element) for var_name in vars: param = cov.get_parameter(var_name) var = param.context units = "unknown" try: units = var.uom except: pass if units is None: units = "unknown" #if len(param.shape) >=1 and not param.is_coordinate: #dataVariable data_element = doc.createElement('dataVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(var.name) source_name_element.appendChild(text_node) data_element.appendChild(source_name_element) destination_name_element = doc.createElement( 'destinationName') text_node = doc.createTextNode(erd_name_map[var.name]) destination_name_element.appendChild(text_node) data_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if not var.attributes is None: for key, val in var.attributes.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode( self.get_ioos_category(var.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'long_name') long_name = "" if var.long_name is not None: long_name = var.long_name text_node = doc.createTextNode(long_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') standard_name = "" if var.standard_name is not None: standard_name = var.standard_name att_element.setAttribute('name', 'standard_name') text_node = doc.createTextNode(standard_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'units') text_node = doc.createTextNode(units) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) data_element.appendChild(add_attributes_element) dataset_element.appendChild(data_element) index += 1 #bug with prettyxml #http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/ #result += dataset_element.toprettyxml() + '\n' result += dataset_element.toxml() + '\n' cov.close() return result
def repair( self, backup=True, copy_over=True, keep_temp=False, reanalyze=False, analyze_bricks=False, detailed_analysis=False, ): """ Heavy repair tool that recreates a blank persisted Coverage from the broken coverage's original construction parameters, then reconstructs the Master and Parameter metadata files by inspection of the ION objects and "valid" brick files. @return: """ if self._ar is None or reanalyze: self._ar = self._do_analysis(analyze_bricks=analyze_bricks, detailed_analysis=detailed_analysis) if self._ar.is_corrupt: if len(self._ar.get_brick_corruptions()) > 0: raise NotImplementedError("Brick corruption. Cannot repair at this time!!!") else: # Repair the Master and Parameter metadata files # Need the ParameterDictionary, TemporalDomain and SpatialDomain pdict = ParameterDictionary.load(self._dso.parameter_dictionary) tdom = GridDomain.load(self._dso.temporal_domain) sdom = GridDomain.load(self._dso.spatial_domain) # Set up the working directory for the recovered coverage tempcov_dir = tempfile.mkdtemp("covs") # Create the temporary Coverage tempcov = SimplexCoverage( root_dir=tempcov_dir, persistence_guid=self._guid, name=self._guid, parameter_dictionary=pdict, spatial_domain=sdom, temporal_domain=tdom, ) # Handle to persistence layer for tempcov pl = tempcov._persistence_layer # Set up the original and temporary coverage path strings orig_dir = os.path.join(self.cov_pth, self._guid) temp_dir = os.path.join(tempcov.persistence_dir, tempcov.persistence_guid) # Insert same number of timesteps into temporary coverage as in broken coverage brick_domains_new, new_brick_list, brick_list_spans, tD, bD, min_data_bound, max_data_bound = self.inspect_bricks( self.cov_pth, self._guid, "time" ) empty_cov = ( brick_list_spans is None ) # If None, there are no brick files --> no timesteps, empty coverage! if not empty_cov: # If None, there are no brick files --> no timesteps, empty coverage! bls = [s.value for s in brick_list_spans] maxes = [sum(b[3]) for b in new_brick_list.values()] # Replace metadata is the Master file pl.master_manager.brick_domains = brick_domains_new pl.master_manager.brick_list = new_brick_list # Repair ExternalLinks to brick files with HDFLockingFile(pl.master_manager.file_path, "r+") as f: for param_name in pdict.keys(): del f[param_name] f.create_group(param_name) for param_name in pdict.keys(): for brick in bls: link_path = "/{0}/{1}".format(param_name, brick[0]) brick_file_name = "{0}.hdf5".format(brick[0]) brick_rel_path = os.path.join( pl.parameter_metadata[param_name].root_dir.replace(tempcov.persistence_dir, "."), brick_file_name, ) log.debug("link_path: %s", link_path) log.debug("brick_rel_path: %s", brick_rel_path) pl.master_manager.add_external_link(link_path, brick_rel_path, brick[0]) pl.flush_values() pl.flush() tempcov.close() # Remove 'rtree' dataset from Master file if it already exists (post domain expansion) # to make way for reconstruction with HDFLockingFile(pl.master_manager.file_path, "r+") as f: if "rtree" in f.keys(): del f["rtree"] # Reconstruct 'rtree' dataset # Open temporary Coverage and PersistenceLayer objects fixed_cov = AbstractCoverage.load(tempcov.persistence_dir, mode="r+") pl_fixed = fixed_cov._persistence_layer # Call update_rtree for each brick using PersistenceLayer builtin brick_count = 0 if not empty_cov: for brick in bls: rtree_extents, brick_extents, brick_active_size = pl_fixed.calculate_extents( brick[1][1], bD, tD ) pl_fixed.master_manager.update_rtree(brick_count, rtree_extents, obj=brick[0]) brick_count += 1 # Update parameter_bounds property based on each parameter's brick data using deep inspection valid_bounds_types = ["BooleanType", "ConstantType", "QuantityType", "ConstantRangeType"] if not empty_cov: for param in pdict.keys(): if pdict.get_context(param).param_type.__class__.__name__ in valid_bounds_types: brick_domains_new, new_brick_list, brick_list_spans, tD, bD, min_data_bound, max_data_bound = self.inspect_bricks( self.cov_pth, self._guid, param ) # Update the metadata pl_fixed.update_parameter_bounds(param, [min_data_bound, max_data_bound]) pl_fixed.flush() fixed_cov.close() # Create backup copy of original Master and Parameter files if backup: import datetime orig_master_file = os.path.join(self.cov_pth, "{0}_master.hdf5".format(self._guid)) # Generate the timestamp tstamp_format = "%Y%m%d%H%M%S" tstamp = datetime.datetime.now().strftime(tstamp_format) backup_master_file = os.path.join(self.cov_pth, "{0}_master.{1}.hdf5".format(self._guid, tstamp)) shutil.copy2(orig_master_file, backup_master_file) for param in pdict.keys(): param_orig = os.path.join(orig_dir, param, "{0}.hdf5".format(param)) param_backup = os.path.join(orig_dir, param, "{0}.{1}.hdf5".format(param, tstamp)) shutil.copy2(param_orig, param_backup) # Copy Master and Parameter metadata files back to original/broken coverage (cov_pth) location if copy_over == True: shutil.copy2( os.path.join(tempcov.persistence_dir, "{0}_master.hdf5".format(self._guid)), os.path.join(self.cov_pth, "{0}_master.hdf5".format(self._guid)), ) for param in pdict.keys(): shutil.copy2( os.path.join(temp_dir, param, "{0}.hdf5".format(param)), os.path.join(orig_dir, param, "{0}.hdf5".format(param)), ) # Reanalyze the repaired coverage self._ar = self._do_analysis(analyze_bricks=True) # Verify repair worked, clean up if not if self._ar.is_corrupt: # If the files were backed up then revert if backup: # Remove backed up files and clean up the repair attempt log.info("Repair attempt failed. Reverting to pre-repair state.") # Use backup copy to replace post-repair file. shutil.copy2(backup_master_file, orig_master_file) # Delete the backup os.remove(backup_master_file) # Iterate over parameters and revert to pre-repair state for param in pdict.keys(): param_orig = os.path.join(orig_dir, param, "{0}.hdf5".format(param)) param_backup = os.path.join(orig_dir, param, "{0}.{1}.hdf5".format(param, tstamp)) # Use backup copy to replace post-repair file. shutil.copy2(param_backup, param_orig) # Delete the backup os.remove(param_backup) raise ValueError("Coverage repair failed! Revert to stored backup version, if possible.") # Remove temporary coverage if keep_temp == False: shutil.rmtree(tempcov_dir) else: return tempcov_dir else: log.info("Coverage is not corrupt, nothing to repair!")
def ptypescov(save_coverage=False, in_memory=False): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary quant_t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) quant_t_ctxt.axis = AxisTypeEnum.TIME quant_t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(quant_t_ctxt) cnst_int_ctxt = ParameterContext('const_int', param_type=ConstantType(QuantityType(value_encoding=np.dtype('int32'))), variability=VariabilityEnum.NONE) cnst_int_ctxt.long_name = 'example of a parameter of type ConstantType, base_type int32' cnst_int_ctxt.axis = AxisTypeEnum.LAT cnst_int_ctxt.uom = 'degree_north' pdict.add_context(cnst_int_ctxt) cnst_flt_ctxt = ParameterContext('const_float', param_type=ConstantType(), variability=VariabilityEnum.NONE) cnst_flt_ctxt.long_name = 'example of a parameter of type QuantityType, base_type float (default)' cnst_flt_ctxt.axis = AxisTypeEnum.LON cnst_flt_ctxt.uom = 'degree_east' pdict.add_context(cnst_flt_ctxt) # func_ctxt = ParameterContext('function', param_type=FunctionType(QuantityType(value_encoding=np.dtype('float32')))) # func_ctxt.long_name = 'example of a parameter of type FunctionType' # pdict.add_context(func_ctxt) quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32'))) quant_ctxt.long_name = 'example of a parameter of type QuantityType' quant_ctxt.uom = 'degree_Celsius' pdict.add_context(quant_ctxt) arr_ctxt = ParameterContext('array', param_type=ArrayType()) arr_ctxt.long_name = 'example of a parameter of type ArrayType, will be filled with variable-length \'byte-string\' data' pdict.add_context(arr_ctxt) rec_ctxt = ParameterContext('record', param_type=RecordType()) rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries' pdict.add_context(rec_ctxt) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory) # Insert some timesteps (automatically expands other arrays) nt = 20 scov.insert_timesteps(nt) # Add data for each parameter scov.set_parameter_values('quantity_time', value=np.arange(nt)) scov.set_parameter_values('const_int', value=45.32) # Set a constant directly, with incorrect data type (fixed under the hood) scov.set_parameter_values('const_float', value=make_range_expr(-71.11)) # Set with a properly formed constant expression scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23) # # Setting three range expressions such that indices 0-2 == 10, 3-7 == 15 and >=8 == 20 # scov.set_parameter_values('function', value=make_range_expr(10, 0, 3, min_incl=True, max_incl=False, else_val=-999.9)) # scov.set_parameter_values('function', value=make_range_expr(15, 3, 8, min_incl=True, max_incl=False, else_val=-999.9)) # scov.set_parameter_values('function', value=make_range_expr(20, 8, min_incl=True, max_incl=False, else_val=-999.9)) arrval = [] recval = [] letts='abcdefghijklmnopqrstuvwxyz' while len(letts) < nt: letts += 'abcdefghijklmnopqrstuvwxyz' for x in xrange(nt): arrval.append(np.random.bytes(np.random.randint(1,20))) # One value (which is a byte string) for each member of the domain d = {letts[x]: letts[x:]} recval.append(d) # One value (which is a dict) for each member of the domain scov.set_parameter_values('array', value=arrval) scov.set_parameter_values('record', value=recval) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov') return scov
def get_dataset_xml(self, coverage_path, product_name=''): #http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html result = '' paths = os.path.split(coverage_path) cov = SimplexCoverage.load(coverage_path) doc = xml.dom.minidom.Document() #erd_type_map = {'d':'double', 'f':"float", 'h':'short', 'i':'int', 'l':'int', 'q':'int', 'b':'byte', 'b':'char', 'S':'String'} #Get lists of variables with unique sets of dimensions. #Datasets can only have variables with the same sets of dimensions if not cov.list_parameters(): raise BadRequest('Attempting to register an empty dataset. The coverage (%s) has no definition.\n%s' %(coverage_path, cov)) datasets = {} for key in cov.list_parameters(): pc = cov.get_parameter_context(key) #if not isinstance(pc.param_type, QuantityType): # continue param = cov.get_parameter(key) dims = (cov.temporal_parameter_name,) if len(param.shape) == 2: dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name) if not dims in datasets.keys(): datasets[dims] = [] datasets[dims].append(key) index = 0 if not datasets: raise BadRequest('Attempting to register a dimensionless dataset. The coverage (%s) has no dimension(s).\n%s' %( coverage_path, cov)) for dims, vars in datasets.iteritems(): erd_name_map = self.get_errdap_name_map(vars) if len(vars)==1: raise BadRequest('A dataset needs a proper range, not just the temporal dimension. %s\n%s' %( coverage_path, cov)) if not (len(dims) == 1 and dims[0] == vars[0]): dataset_element = doc.createElement('dataset') dataset_element.setAttribute('type', 'EDDGridFromDap') dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index)) dataset_element.setAttribute('active', 'True') source_element = doc.createElement('sourceUrl') text_node = doc.createTextNode(self.pydap_url + paths[1]) source_element.appendChild(text_node) dataset_element.appendChild(source_element) reload_element = doc.createElement('reloadEveryNMinutes') text_node = doc.createTextNode('5') reload_element.appendChild(text_node) dataset_element.appendChild(reload_element) add_attributes_element = doc.createElement('addAttributes') atts = {} atts['title'] = product_name or urllib.unquote(cov.name) atts['infoUrl'] = self.pydap_url + paths[1] atts['institution'] = 'OOI' atts['Conventions'] = "COARDS, CF-1.6, Unidata Dataset Discovery v1.0" atts['license'] = '[standard]' atts['summary'] = cov.name atts['cdm_data_type'] = 'Grid' atts['subsetVariables'] = ','.join([erd_name_map[v] for v in vars]) atts['standard_name_vocabulary'] = 'CF-12' try: lat_min,lat_max = cov.get_data_bounds("lat") atts['geospatial_lat_max'] = str(lat_max) atts['geospatial_lat_min'] = str(lat_min) pc = cov.get_parameter_context("lat") atts['geospatial_lat_units'] = str(pc.uom) lon_min,lon_max = cov.get_data_bounds("lon") atts['geospatial_lon_max'] = str(lon_max) atts['geospatial_lon_min'] = str(lon_min) pc = cov.get_parameter_context("lon") atts['geospatial_lon_units'] = str(pc.uom) except: #silently fail and just don't fill attributes pass for key, val in atts.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) if len(add_attributes_element.childNodes) > 0: dataset_element.appendChild(add_attributes_element) for var_name in vars: var = cov.get_parameter_context(var_name) units = "unknown" if hasattr(var,'uom') and var.uom: units = var.uom #if len(param.shape) >=1 and not param.is_coordinate: #dataVariable data_element = doc.createElement('dataVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(var.name) source_name_element.appendChild(text_node) data_element.appendChild(source_name_element) destination_name_element = doc.createElement('destinationName') text_node = doc.createTextNode(erd_name_map[var.name]) destination_name_element.appendChild(text_node) data_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if var.ATTRS is not None: for key in var.ATTRS: if not hasattr(var,key): continue val = getattr(var,key) att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode(self.get_ioos_category(var.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'long_name') long_name = "" if hasattr(var,'display_name') and var.display_name is not None: long_name = var.display_name text_node = doc.createTextNode(long_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') standard_name = "" if hasattr(var,'standard_name') and var.standard_name is not None: standard_name = var.standard_name att_element.setAttribute('name', 'standard_name') text_node = doc.createTextNode(standard_name) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'units') text_node = doc.createTextNode(units) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) data_element.appendChild(add_attributes_element) dataset_element.appendChild(data_element) index += 1 #bug with prettyxml #http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/ #result += dataset_element.toprettyxml() + '\n' result += dataset_element.toxml() + '\n' cov.close() return result
def benchmark_value_setting(num_params=10, num_insertions=100, repeat=1, bulk_ts_insert=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) for i in xrange(num_params-1): pdict.add_context(ParameterContext('param_{0}'.format(i))) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) import time counter = 1 insert_times = [] per_rep_times = [] full_time = time.time() for r in xrange(repeat): # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'empty sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) rep_time = time.time() if bulk_ts_insert: cov.insert_timesteps(num_insertions) for x in xrange(num_insertions): in_time = time.time() if not bulk_ts_insert: cov.insert_timesteps(1) slice_ = slice(cov.num_timesteps - 1, None) cov.set_parameter_values('time', 1, tdoa=slice_) for i in xrange(num_params-1): cov.set_parameter_values('param_{0}'.format(i), 1.1, tdoa=slice_) in_time = time.time() - in_time insert_times.append(in_time) counter += 1 rep_time = time.time() - rep_time per_rep_times.append(rep_time) cov.close() print 'Average Value Insertion Time (%s repetitions): %s' % (repeat, sum(insert_times) / len(insert_times)) print 'Average Total Expansion Time (%s repetitions): %s' % (repeat, sum(per_rep_times) / len(per_rep_times)) print 'Full Time (includes cov creation/closing): %s' % (time.time() - full_time) return cov