def oneparamcov_noautoflush(save_coverage=False, in_memory=False):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory, auto_flush_values=False)
# Insert some timesteps (automatically expands other arrays)
nt = 100
scov.insert_timesteps(nt)
# Add data for the parameter
scov.set_parameter_values('time', value=np.arange(nt))
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/sample.cov')
return scov
def cov_io(self, context, value_array, comp_val=None):
pdict = ParameterDictionary()
time = ParameterContext(name='time', param_type=QuantityType(value_encoding=np.float64))
pdict.add_context(context)
pdict.add_context(time, True)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
cov.insert_timesteps(len(value_array))
cov.set_parameter_values('test', tdoa=slice(0,len(value_array)), value=value_array)
comp_val = comp_val if comp_val is not None else value_array
testval = cov.get_parameter_values('test')
try:
np.testing.assert_array_equal(testval, comp_val)
except:
print repr(value_array)
raise
def nospatialcov(save_coverage=False, in_memory=False):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL)
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32')))
quant_ctxt.long_name = 'example of a parameter of type QuantityType'
quant_ctxt.uom = 'degree_Celsius'
pdict.add_context(quant_ctxt)
const_ctxt = ParameterContext('constant', param_type=ConstantType())
const_ctxt.long_name = 'example of a parameter of type ConstantType'
pdict.add_context(const_ctxt)
arr_ctxt = ParameterContext('array', param_type=ArrayType())
arr_ctxt.long_name = 'example of a parameter of type ArrayType with base_type ndarray (resolves to \'object\')'
pdict.add_context(arr_ctxt)
arr2_ctxt = ParameterContext('array2', param_type=ArrayType())
arr2_ctxt.long_name = 'example of a parameter of type ArrayType with base_type object'
pdict.add_context(arr2_ctxt)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, temporal_domain=tdom, in_memory_storage=in_memory)
# Insert some timesteps (automatically expands other arrays)
nt = 20
scov.insert_timesteps(nt)
# Add data for each parameter
scov.set_parameter_values('quantity_time', value=np.arange(nt))
scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23)
scov.set_parameter_values('constant', value=20)
arrval = []
arr2val = []
for x in xrange(nt): # One value (which IS an array) for each member of the domain
arrval.append(np.random.bytes(np.random.randint(1,20)))
arr2val.append(np.random.random_sample(np.random.randint(1,10)))
scov.set_parameter_values('array', value=arrval)
scov.set_parameter_values('array2', value=arr2val)
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov')
return scov
def samplecov2(save_coverage=False, in_memory=False):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL)
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
lat_ctxt = ParameterContext('lat', param_type=ConstantType(), variability=VariabilityEnum.NONE)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=ConstantType(), variability=VariabilityEnum.NONE)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
nt = 20
tdom = GridDomain(GridShape('temporal', [nt]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory)
# Insert some timesteps (automatically expands other arrays)
# scov.insert_timesteps(10)
# Add data for each parameter
scov.set_parameter_values('time', value=np.arange(nt))
scov.set_parameter_values('lat', value=make_range_expr(45.32))
scov.set_parameter_values('lon', value=make_range_expr(-71.11))
# make a random sample of 10 values between 23 and 26
# Ref: http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.random_sample.html#numpy.random.random_sample
# --> To sample multiply the output of random_sample by (b-a) and add a
tvals=np.random.random_sample(nt)*(26-23)+23
scov.set_parameter_values('temp', value=tvals)
scov.set_parameter_values('conductivity', value=np.random.random_sample(nt)*(110-90)+90)
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/sample2.cov')
return scov
def benchmark_value_setting(num_params=10, num_insertions=100, repeat=1, bulk_ts_insert=False):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
for i in xrange(num_params-1):
pdict.add_context(ParameterContext('param_{0}'.format(i)))
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
import time
counter = 1
insert_times = []
per_rep_times = []
full_time = time.time()
for r in xrange(repeat):
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
cov = SimplexCoverage('test_data', create_guid(), 'empty sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
rep_time = time.time()
if bulk_ts_insert:
cov.insert_timesteps(num_insertions)
for x in xrange(num_insertions):
in_time = time.time()
if not bulk_ts_insert:
cov.insert_timesteps(1)
slice_ = slice(cov.num_timesteps - 1, None)
cov.set_parameter_values('time', 1, tdoa=slice_)
for i in xrange(num_params-1):
cov.set_parameter_values('param_{0}'.format(i), 1.1, tdoa=slice_)
in_time = time.time() - in_time
insert_times.append(in_time)
counter += 1
rep_time = time.time() - rep_time
per_rep_times.append(rep_time)
cov.close()
print 'Average Value Insertion Time (%s repetitions): %s' % (repeat, sum(insert_times) / len(insert_times))
print 'Average Total Expansion Time (%s repetitions): %s' % (repeat, sum(per_rep_times) / len(per_rep_times))
print 'Full Time (includes cov creation/closing): %s' % (time.time() - full_time)
return cov
def cov_io(self, context, value_array, comp_val=None):
pdict = ParameterDictionary()
time = ParameterContext(
name='time', param_type=QuantityType(value_encoding=np.float64))
pdict.add_context(context)
pdict.add_context(time, True)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
cov = SimplexCoverage('test_data',
create_guid(),
'sample coverage_model',
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
self.addCleanup(shutil.rmtree, cov.persistence_dir)
cov.insert_timesteps(len(value_array))
cov.set_parameter_values('test',
tdoa=slice(0, len(value_array)),
value=value_array)
comp_val = comp_val if comp_val is not None else value_array
testval = cov.get_parameter_values('test')
try:
np.testing.assert_array_equal(testval, comp_val)
except:
print repr(value_array)
raise
def ptypescov(save_coverage=False, in_memory=False):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
quant_t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL)
quant_t_ctxt.axis = AxisTypeEnum.TIME
quant_t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(quant_t_ctxt)
cnst_int_ctxt = ParameterContext('const_int', param_type=ConstantType(QuantityType(value_encoding=np.dtype('int32'))), variability=VariabilityEnum.NONE)
cnst_int_ctxt.long_name = 'example of a parameter of type ConstantType, base_type int32'
cnst_int_ctxt.axis = AxisTypeEnum.LAT
cnst_int_ctxt.uom = 'degree_north'
pdict.add_context(cnst_int_ctxt)
cnst_flt_ctxt = ParameterContext('const_float', param_type=ConstantType(), variability=VariabilityEnum.NONE)
cnst_flt_ctxt.long_name = 'example of a parameter of type QuantityType, base_type float (default)'
cnst_flt_ctxt.axis = AxisTypeEnum.LON
cnst_flt_ctxt.uom = 'degree_east'
pdict.add_context(cnst_flt_ctxt)
# func_ctxt = ParameterContext('function', param_type=FunctionType(QuantityType(value_encoding=np.dtype('float32'))))
# func_ctxt.long_name = 'example of a parameter of type FunctionType'
# pdict.add_context(func_ctxt)
quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32')))
quant_ctxt.long_name = 'example of a parameter of type QuantityType'
quant_ctxt.uom = 'degree_Celsius'
pdict.add_context(quant_ctxt)
arr_ctxt = ParameterContext('array', param_type=ArrayType())
arr_ctxt.long_name = 'example of a parameter of type ArrayType, will be filled with variable-length \'byte-string\' data'
pdict.add_context(arr_ctxt)
rec_ctxt = ParameterContext('record', param_type=RecordType())
rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries'
pdict.add_context(rec_ctxt)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory)
# Insert some timesteps (automatically expands other arrays)
nt = 20
scov.insert_timesteps(nt)
# Add data for each parameter
scov.set_parameter_values('quantity_time', value=np.arange(nt))
scov.set_parameter_values('const_int', value=45.32) # Set a constant directly, with incorrect data type (fixed under the hood)
scov.set_parameter_values('const_float', value=make_range_expr(-71.11)) # Set with a properly formed constant expression
scov.set_parameter_values('quantity', value=np.random.random_sample(nt)*(26-23)+23)
# # Setting three range expressions such that indices 0-2 == 10, 3-7 == 15 and >=8 == 20
# scov.set_parameter_values('function', value=make_range_expr(10, 0, 3, min_incl=True, max_incl=False, else_val=-999.9))
# scov.set_parameter_values('function', value=make_range_expr(15, 3, 8, min_incl=True, max_incl=False, else_val=-999.9))
# scov.set_parameter_values('function', value=make_range_expr(20, 8, min_incl=True, max_incl=False, else_val=-999.9))
arrval = []
recval = []
letts='abcdefghijklmnopqrstuvwxyz'
while len(letts) < nt:
letts += 'abcdefghijklmnopqrstuvwxyz'
for x in xrange(nt):
arrval.append(np.random.bytes(np.random.randint(1,20))) # One value (which is a byte string) for each member of the domain
d = {letts[x]: letts[x:]}
recval.append(d) # One value (which is a dict) for each member of the domain
scov.set_parameter_values('array', value=arrval)
scov.set_parameter_values('record', value=recval)
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov')
return scov
