def oneparamcov_noautoflush(save_coverage=False, in_memory=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory, auto_flush_values=False) # Insert some timesteps (automatically expands other arrays) nt = 100 scov.insert_timesteps(nt) # Add data for the parameter scov.set_parameter_values('time', value=np.arange(nt)) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/sample.cov') return scov
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext(name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0,len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def nospatialcov(save_coverage=False, in_memory=False): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32'))) quant_ctxt.long_name = 'example of a parameter of type QuantityType' quant_ctxt.uom = 'degree_Celsius' pdict.add_context(quant_ctxt) const_ctxt = ParameterContext('constant', param_type=ConstantType()) const_ctxt.long_name = 'example of a parameter of type ConstantType' pdict.add_context(const_ctxt) arr_ctxt = ParameterContext('array', param_type=ArrayType()) arr_ctxt.long_name = 'example of a parameter of type ArrayType with base_type ndarray (resolves to \'object\')' pdict.add_context(arr_ctxt) arr2_ctxt = ParameterContext('array2', param_type=ArrayType()) arr2_ctxt.long_name = 'example of a parameter of type ArrayType with base_type object' pdict.add_context(arr2_ctxt) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, temporal_domain=tdom, in_memory_storage=in_memory) # Insert some timesteps (automatically expands other arrays) nt = 20 scov.insert_timesteps(nt) # Add data for each parameter scov.set_parameter_values('quantity_time', value=np.arange(nt)) scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23) scov.set_parameter_values('constant', value=20) arrval = [] arr2val = [] for x in xrange(nt): # One value (which IS an array) for each member of the domain arrval.append(np.random.bytes(np.random.randint(1,20))) arr2val.append(np.random.random_sample(np.random.randint(1,10))) scov.set_parameter_values('array', value=arrval) scov.set_parameter_values('array2', value=arr2val) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov') return scov
def samplecov2(save_coverage=False, in_memory=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt, is_temporal=True) lat_ctxt = ParameterContext('lat', param_type=ConstantType(), variability=VariabilityEnum.NONE) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=ConstantType(), variability=VariabilityEnum.NONE) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects nt = 20 tdom = GridDomain(GridShape('temporal', [nt]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory) # Insert some timesteps (automatically expands other arrays) # scov.insert_timesteps(10) # Add data for each parameter scov.set_parameter_values('time', value=np.arange(nt)) scov.set_parameter_values('lat', value=make_range_expr(45.32)) scov.set_parameter_values('lon', value=make_range_expr(-71.11)) # make a random sample of 10 values between 23 and 26 # Ref: http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.random_sample.html#numpy.random.random_sample # --> To sample multiply the output of random_sample by (b-a) and add a tvals=np.random.random_sample(nt)*(26-23)+23 scov.set_parameter_values('temp', value=tvals) scov.set_parameter_values('conductivity', value=np.random.random_sample(nt)*(110-90)+90) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/sample2.cov') return scov
def benchmark_value_setting(num_params=10, num_insertions=100, repeat=1, bulk_ts_insert=False): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) for i in xrange(num_params-1): pdict.add_context(ParameterContext('param_{0}'.format(i))) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) import time counter = 1 insert_times = [] per_rep_times = [] full_time = time.time() for r in xrange(repeat): # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'empty sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) rep_time = time.time() if bulk_ts_insert: cov.insert_timesteps(num_insertions) for x in xrange(num_insertions): in_time = time.time() if not bulk_ts_insert: cov.insert_timesteps(1) slice_ = slice(cov.num_timesteps - 1, None) cov.set_parameter_values('time', 1, tdoa=slice_) for i in xrange(num_params-1): cov.set_parameter_values('param_{0}'.format(i), 1.1, tdoa=slice_) in_time = time.time() - in_time insert_times.append(in_time) counter += 1 rep_time = time.time() - rep_time per_rep_times.append(rep_time) cov.close() print 'Average Value Insertion Time (%s repetitions): %s' % (repeat, sum(insert_times) / len(insert_times)) print 'Average Total Expansion Time (%s repetitions): %s' % (repeat, sum(per_rep_times) / len(per_rep_times)) print 'Full Time (includes cov creation/closing): %s' % (time.time() - full_time) return cov
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def ptypescov(save_coverage=False, in_memory=False): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary quant_t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL) quant_t_ctxt.axis = AxisTypeEnum.TIME quant_t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(quant_t_ctxt) cnst_int_ctxt = ParameterContext('const_int', param_type=ConstantType(QuantityType(value_encoding=np.dtype('int32'))), variability=VariabilityEnum.NONE) cnst_int_ctxt.long_name = 'example of a parameter of type ConstantType, base_type int32' cnst_int_ctxt.axis = AxisTypeEnum.LAT cnst_int_ctxt.uom = 'degree_north' pdict.add_context(cnst_int_ctxt) cnst_flt_ctxt = ParameterContext('const_float', param_type=ConstantType(), variability=VariabilityEnum.NONE) cnst_flt_ctxt.long_name = 'example of a parameter of type QuantityType, base_type float (default)' cnst_flt_ctxt.axis = AxisTypeEnum.LON cnst_flt_ctxt.uom = 'degree_east' pdict.add_context(cnst_flt_ctxt) # func_ctxt = ParameterContext('function', param_type=FunctionType(QuantityType(value_encoding=np.dtype('float32')))) # func_ctxt.long_name = 'example of a parameter of type FunctionType' # pdict.add_context(func_ctxt) quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32'))) quant_ctxt.long_name = 'example of a parameter of type QuantityType' quant_ctxt.uom = 'degree_Celsius' pdict.add_context(quant_ctxt) arr_ctxt = ParameterContext('array', param_type=ArrayType()) arr_ctxt.long_name = 'example of a parameter of type ArrayType, will be filled with variable-length \'byte-string\' data' pdict.add_context(arr_ctxt) rec_ctxt = ParameterContext('record', param_type=RecordType()) rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries' pdict.add_context(rec_ctxt) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory) # Insert some timesteps (automatically expands other arrays) nt = 20 scov.insert_timesteps(nt) # Add data for each parameter scov.set_parameter_values('quantity_time', value=np.arange(nt)) scov.set_parameter_values('const_int', value=45.32) # Set a constant directly, with incorrect data type (fixed under the hood) scov.set_parameter_values('const_float', value=make_range_expr(-71.11)) # Set with a properly formed constant expression scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23) # # Setting three range expressions such that indices 0-2 == 10, 3-7 == 15 and >=8 == 20 # scov.set_parameter_values('function', value=make_range_expr(10, 0, 3, min_incl=True, max_incl=False, else_val=-999.9)) # scov.set_parameter_values('function', value=make_range_expr(15, 3, 8, min_incl=True, max_incl=False, else_val=-999.9)) # scov.set_parameter_values('function', value=make_range_expr(20, 8, min_incl=True, max_incl=False, else_val=-999.9)) arrval = [] recval = [] letts='abcdefghijklmnopqrstuvwxyz' while len(letts) < nt: letts += 'abcdefghijklmnopqrstuvwxyz' for x in xrange(nt): arrval.append(np.random.bytes(np.random.randint(1,20))) # One value (which is a byte string) for each member of the domain d = {letts[x]: letts[x:]} recval.append(d) # One value (which is a dict) for each member of the domain scov.set_parameter_values('array', value=arrval) scov.set_parameter_values('record', value=recval) if in_memory and save_coverage: SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov') return scov