def validate_span_data(self):
invalid_spans = []
valid_spans = []
associated_spans = self._get_span_dict(self.value_list.keys())
compressors = self.value_list
for span in associated_spans:
span.compressors = compressors
stored_hash = SpanStorageFactory.get_span_storage_obj(self.storage_name).get_stored_span_hash(span.id)
recalculated_hash = span.get_hash()
if stored_hash == recalculated_hash:
valid_spans.append(span.id)
else:
invalid_spans.append(span.id)
return valid_spans, invalid_spans
def has_data(self):
return SpanStorageFactory.get_span_storage_obj(self.storage_name).has_data(self.master_manager.guid)
def _get_span_dict(self, params, time_range=None, time=None):
return SpanStorageFactory.get_span_storage_obj(self.storage_name).get_spans(coverage_ids=self.master_manager.guid, decompressors=self.value_list)
def get_spans_by_id(self, spans):
return SpanStorageFactory.get_span_storage_obj(self.storage_name).get_spans(coverage_ids=self.master_manager.guid, span_ids=spans, decompressors=self.value_list)
def _write_mutable_params(self, param_dict, write_id, guid):
if len(param_dict) < 1:
return
for param in param_dict:
ptype = self.master_manager.parameter_metadata[param].parameter_context.param_type
if not ptype.is_mutable and param not in [Span.ingest_time_str, self.alignment_parameter]:
raise TypeError("Parameter, %s, is not mutable" % param)
time_arr = param_dict[self.alignment_parameter].get_data()
time_segment = [time_arr.min(), time_arr.max()]
associated_spans = self._get_span_dict(param_dict.keys().remove(self.alignment_parameter), time_segment)
new_alignment_arr = param_dict[self.alignment_parameter].get_data()
spans_needing_rewrite = set()
remove = {}
for span in associated_spans:
if not span.mutable:
continue
span_alignment_arr = span.param_dict[self.alignment_parameter].get_data()
intersection = np.intersect1d(span_alignment_arr, new_alignment_arr)
if intersection.shape[0] < 1:
continue
intersect_indicies = np.nonzero(intersection)[0]
for idx in intersect_indicies:
new_idx = np.nonzero(new_alignment_arr == span_alignment_arr[idx])
for param_name, values in param_dict.iteritems():
if param_name in [Span.ingest_time_str, self.alignment_parameter]:
continue
if param_name in span.param_dict:
span.param_dict[param_name].get_data().setflags(write=True)
spans_needing_rewrite.add(span)
span.param_dict[param_name].get_data()[idx] = param_dict[param_name].get_data()[new_idx]
if param_name not in remove:
remove[param_name] = []
remove[param_name].append(new_idx[0][0])
if len(remove) > 0:
for param , indicies in remove.iteritems():
from copy import deepcopy
align_data = deepcopy(param_dict[self.alignment_parameter])
get_data_indicies = range(len(align_data.get_data()))
from collections import deque
deque((list.pop(get_data_indicies, i) for i in sorted(indicies, reverse=True)), maxlen=0)
if len(get_data_indicies) > 0:
span_dict = {}
align_data._data = align_data.get_data()[get_data_indicies]
span_dict[self.alignment_parameter] = align_data
param_dict[param]._data = param_dict[param].get_data()[get_data_indicies]
span_dict[param] = param_dict[param]
span = Span(write_id+'_'+param, guid, span_dict, compressors=self.value_list, mutable=True)
spans_needing_rewrite.add(span)
param_dict.pop(param)
span_table = SpanStorageFactory.get_span_storage_obj(self.storage_name)
if len(param_dict) > 2: # observation time and ingest time
span = Span(write_id, guid, param_dict, compressors=self.value_list, mutable=True)
span_table.write_span(span)
for span in spans_needing_rewrite:
span.compressors = self.value_list
span_table.write_span(span)
def write_parameters(self, write_id, values):
if not isinstance(values, dict):
raise TypeError("values must be dict type. Found %s" % type(values))
arr_len = -1
all_values_constant_over_time = True
write_time = get_current_ntp_time()
bad_keys = []
mutable_params = {}
for k, v in values.iteritems():
if isinstance(v, NumpyParameterData):
arr_len = v.get_data().shape[0]
elif isinstance(v, np.ndarray):
arr_len = v.shape[0]
elif isinstance(v, ConstantOverTime):
continue
break
if arr_len > 0:
values[Span.ingest_time_str] = NumpyParameterData(Span.ingest_time_str, self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type.create_filled_array(arr_len, write_time))
for key, arr in values.iteritems():
if key not in self.value_list:
raise KeyError("Parameter, %s, has not been initialized" % (key))
param_type = self.master_manager.parameter_metadata[key].parameter_context.param_type
if isinstance(param_type, ParameterFunctionType):
bad_keys.append(key)
continue #TODO: throw error instead
if isinstance(arr, np.ndarray):
arr = NumpyParameterData(key, arr)
values[key] = arr
if not isinstance(arr, ParameterData):
raise TypeError("Value for %s must implement <%s>, found <%s>" % (key, ParameterData.__name__, arr.__class__.__name__))
if not isinstance(arr, ConstantOverTime):
if self.master_manager.parameter_metadata[key].parameter_context.param_type.validate_value_set(arr.get_data()):
self.master_manager.parameter_metadata[key].read_only = False
self.master_manager.parameter_metadata[key].flush()
self.master_manager.parameter_metadata[key].read_only = True
all_values_constant_over_time = False
elif isinstance(arr, NumpyParameterData) and arr.get_data().shape[0] != arr_len:
raise ValueError("Array size for %s is inconsistent. Expected %s elements, found %s." % (key, str(arr_len), str(arr.get_data().size)))
min_val, max_val = self.value_list[key].get_statistics(arr)
self.update_parameter_bounds(key, (min_val, max_val))
if param_type.is_mutable:
mutable_params[key] = values[key]
for key in bad_keys:
values.pop(key)
for key in mutable_params:
values.pop(key)
if len(mutable_params) > 0 and self.alignment_parameter in values:
mutable_params[self.alignment_parameter] = values[self.alignment_parameter]
mutable_params[Span.ingest_time_str] = values[Span.ingest_time_str]
if not all_values_constant_over_time and self.alignment_parameter not in values:
raise LookupError("Array must be supplied for parameter, %s, to ensure alignment" % self.alignment_parameter)
write_span = False
if len(values) > 1 and self.alignment_parameter in values:
if len(values) == 2 and len(mutable_params) > 0:
pass
else:
write_span = True
elif all_values_constant_over_time:
write_span = True
if write_span:
span = Span(write_id, self.master_manager.guid, values, compressors=self.value_list)
span_table = SpanStorageFactory.get_span_storage_obj(self.storage_name)
span_table.write_span(span)
if len(mutable_params) > 2:
self._write_mutable_params(mutable_params, write_id + '_mutable', self.master_manager.guid)
def select(self, db_name=None, limit=-1):
db = SpanStorageFactory.get_span_storage_obj(db_name)
# db = DBFactory.get_db(db_name)
span_dict = db.search(self.search_criteria, limit)
return CoverageSearchResults(span_dict, self.search_criteria, viewable_parameters=self.viewable_parameters,
order_by=self.order_by)
