def __init__(self, span_uuid, coverage_id, param_dict, ingest_time=None, compressors=None, mutable=False):
self.param_dict = param_dict
self.ingest_time = ingest_time
self.ingest_times = []
if ingest_time is None:
self.ingest_time = get_current_ntp_time()
self._set_ingest_times()
self.ingest_time_array
self.id = span_uuid
self.coverage_id = coverage_id
self.compressors = compressors
self.mutable = mutable
def read_and_set_config(self):
one_from_config = False
try:
data = Config(["res/config/coverage.yml"]).data['CoverageConfig']
for k, v in data.iteritems():
self.__setattr__(k, v)
one_from_config = True
self.using_default_config = False
self.config_time = get_current_ntp_time()
except Exception as ex:
if one_from_config:
log.info("Load from config failed with '%s'. Using hybrid default/config file configuration" % ex.message)
return False
else:
log.info("load from config failed with '%s'. Using default config" % ex.message)
return True
def _create_parameter_dictionary_of_numpy_arrays(self, numpy_params, function_params=None, params=None, fill_indexes=False):
return_dict = {}
mask_dict = {}
value_set_time_dict = {}
shape_outer_dimmension = 0
span_order = []
span_size_dict = {}
t_dict = {}
if self.alignment_parameter in numpy_params:
for id, span_data in numpy_params[self.alignment_parameter].iteritems():
span_size_dict[id] = span_data[1].get_data().size
shape_outer_dimmension += span_data[1].get_data().size
span_order.append((span_data[0], id))
span_order.sort()
t_dict = numpy_params[self.alignment_parameter]
dt = np.dtype(self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.value_encoding)
arr = np.empty(shape_outer_dimmension, dtype=dt)
insert_index = 0
for span_tup in span_order:
span_id = span_tup[1]
np_data = t_dict[span_id][1].get_data()
end_idx = insert_index+np_data.size
arr[insert_index:end_idx] = np_data
insert_index += np_data.size
return_dict[self.alignment_parameter] = arr
mask_dict[self.alignment_parameter] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool)
value_set_time_dict[self.alignment_parameter] = self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.create_filled_array(arr.shape[0])
ingest_name_ptype = self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type
for id, span_data in numpy_params.iteritems():
if id == self.alignment_parameter:
continue
npa_list = []
mask_list = []
value_set_list = []
for span_tup in span_order:
span_id = span_tup[1]
span_time = span_tup[0]
if span_id not in span_data:
npa = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_filled_array(span_size_dict[span_id])
npa_list.append(npa)
value_set_list.append(ingest_name_ptype.create_filled_array(npa.shape[0]))
mask_list.append(NumpyUtils.create_filled_array(npa.shape[0], False, dtype=np.bool))
continue
else:
this_data = span_data[span_id][1].get_data()
npa_list.append(this_data)
mask_list.append(NumpyUtils.create_filled_array(this_data.shape[0], True, dtype=np.bool))
value_set_list.append(ingest_name_ptype.create_filled_array(this_data.shape[0], span_time))
return_dict[id] = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_merged_value_array(npa_list)
from coverage_model.parameter_types import BooleanType
mask_dict[id] = BooleanType().create_merged_value_array(mask_list)
value_set_time_dict[id] = ingest_name_ptype.create_merged_value_array(value_set_list)
for param_name, param_dict in function_params.iteritems():
arr = ConstantOverTime.merge_data_as_numpy_array(return_dict[self.alignment_parameter],
param_dict,
param_type=self.master_manager.parameter_metadata[param_name].parameter_context.param_type)
return_dict[param_name] = arr
mask_dict[param_name] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool)
value_set_time_dict[param_name] = ingest_name_ptype.create_filled_array(arr.shape[0], get_current_ntp_time())
return return_dict, mask_dict, value_set_time_dict
def write_parameters(self, write_id, values):
if not isinstance(values, dict):
raise TypeError("values must be dict type. Found %s" % type(values))
arr_len = -1
all_values_constant_over_time = True
write_time = get_current_ntp_time()
bad_keys = []
mutable_params = {}
for k, v in values.iteritems():
if isinstance(v, NumpyParameterData):
arr_len = v.get_data().shape[0]
elif isinstance(v, np.ndarray):
arr_len = v.shape[0]
elif isinstance(v, ConstantOverTime):
continue
break
if arr_len > 0:
values[Span.ingest_time_str] = NumpyParameterData(Span.ingest_time_str, self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type.create_filled_array(arr_len, write_time))
for key, arr in values.iteritems():
if key not in self.value_list:
raise KeyError("Parameter, %s, has not been initialized" % (key))
param_type = self.master_manager.parameter_metadata[key].parameter_context.param_type
if isinstance(param_type, ParameterFunctionType):
bad_keys.append(key)
continue #TODO: throw error instead
if isinstance(arr, np.ndarray):
arr = NumpyParameterData(key, arr)
values[key] = arr
if not isinstance(arr, ParameterData):
raise TypeError("Value for %s must implement <%s>, found <%s>" % (key, ParameterData.__name__, arr.__class__.__name__))
if not isinstance(arr, ConstantOverTime):
if self.master_manager.parameter_metadata[key].parameter_context.param_type.validate_value_set(arr.get_data()):
self.master_manager.parameter_metadata[key].read_only = False
self.master_manager.parameter_metadata[key].flush()
self.master_manager.parameter_metadata[key].read_only = True
all_values_constant_over_time = False
elif isinstance(arr, NumpyParameterData) and arr.get_data().shape[0] != arr_len:
raise ValueError("Array size for %s is inconsistent. Expected %s elements, found %s." % (key, str(arr_len), str(arr.get_data().size)))
min_val, max_val = self.value_list[key].get_statistics(arr)
self.update_parameter_bounds(key, (min_val, max_val))
if param_type.is_mutable:
mutable_params[key] = values[key]
for key in bad_keys:
values.pop(key)
for key in mutable_params:
values.pop(key)
if len(mutable_params) > 0 and self.alignment_parameter in values:
mutable_params[self.alignment_parameter] = values[self.alignment_parameter]
mutable_params[Span.ingest_time_str] = values[Span.ingest_time_str]
if not all_values_constant_over_time and self.alignment_parameter not in values:
raise LookupError("Array must be supplied for parameter, %s, to ensure alignment" % self.alignment_parameter)
write_span = False
if len(values) > 1 and self.alignment_parameter in values:
if len(values) == 2 and len(mutable_params) > 0:
pass
else:
write_span = True
elif all_values_constant_over_time:
write_span = True
if write_span:
span = Span(write_id, self.master_manager.guid, values, compressors=self.value_list)
span_table = SpanStorageFactory.get_span_storage_obj(self.storage_name)
span_table.write_span(span)
if len(mutable_params) > 2:
self._write_mutable_params(mutable_params, write_id + '_mutable', self.master_manager.guid)