def __init__(self, span_uuid, coverage_id, param_dict, ingest_time=None, compressors=None, mutable=False): self.param_dict = param_dict self.ingest_time = ingest_time self.ingest_times = [] if ingest_time is None: self.ingest_time = get_current_ntp_time() self._set_ingest_times() self.ingest_time_array self.id = span_uuid self.coverage_id = coverage_id self.compressors = compressors self.mutable = mutable
def read_and_set_config(self): one_from_config = False try: data = Config(["res/config/coverage.yml"]).data['CoverageConfig'] for k, v in data.iteritems(): self.__setattr__(k, v) one_from_config = True self.using_default_config = False self.config_time = get_current_ntp_time() except Exception as ex: if one_from_config: log.info("Load from config failed with '%s'. Using hybrid default/config file configuration" % ex.message) return False else: log.info("load from config failed with '%s'. Using default config" % ex.message) return True
def _create_parameter_dictionary_of_numpy_arrays(self, numpy_params, function_params=None, params=None, fill_indexes=False): return_dict = {} mask_dict = {} value_set_time_dict = {} shape_outer_dimmension = 0 span_order = [] span_size_dict = {} t_dict = {} if self.alignment_parameter in numpy_params: for id, span_data in numpy_params[self.alignment_parameter].iteritems(): span_size_dict[id] = span_data[1].get_data().size shape_outer_dimmension += span_data[1].get_data().size span_order.append((span_data[0], id)) span_order.sort() t_dict = numpy_params[self.alignment_parameter] dt = np.dtype(self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.value_encoding) arr = np.empty(shape_outer_dimmension, dtype=dt) insert_index = 0 for span_tup in span_order: span_id = span_tup[1] np_data = t_dict[span_id][1].get_data() end_idx = insert_index+np_data.size arr[insert_index:end_idx] = np_data insert_index += np_data.size return_dict[self.alignment_parameter] = arr mask_dict[self.alignment_parameter] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool) value_set_time_dict[self.alignment_parameter] = self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.create_filled_array(arr.shape[0]) ingest_name_ptype = self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type for id, span_data in numpy_params.iteritems(): if id == self.alignment_parameter: continue npa_list = [] mask_list = [] value_set_list = [] for span_tup in span_order: span_id = span_tup[1] span_time = span_tup[0] if span_id not in span_data: npa = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_filled_array(span_size_dict[span_id]) npa_list.append(npa) value_set_list.append(ingest_name_ptype.create_filled_array(npa.shape[0])) mask_list.append(NumpyUtils.create_filled_array(npa.shape[0], False, dtype=np.bool)) continue else: this_data = span_data[span_id][1].get_data() npa_list.append(this_data) mask_list.append(NumpyUtils.create_filled_array(this_data.shape[0], True, dtype=np.bool)) value_set_list.append(ingest_name_ptype.create_filled_array(this_data.shape[0], span_time)) return_dict[id] = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_merged_value_array(npa_list) from coverage_model.parameter_types import BooleanType mask_dict[id] = BooleanType().create_merged_value_array(mask_list) value_set_time_dict[id] = ingest_name_ptype.create_merged_value_array(value_set_list) for param_name, param_dict in function_params.iteritems(): arr = ConstantOverTime.merge_data_as_numpy_array(return_dict[self.alignment_parameter], param_dict, param_type=self.master_manager.parameter_metadata[param_name].parameter_context.param_type) return_dict[param_name] = arr mask_dict[param_name] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool) value_set_time_dict[param_name] = ingest_name_ptype.create_filled_array(arr.shape[0], get_current_ntp_time()) return return_dict, mask_dict, value_set_time_dict
def write_parameters(self, write_id, values): if not isinstance(values, dict): raise TypeError("values must be dict type. Found %s" % type(values)) arr_len = -1 all_values_constant_over_time = True write_time = get_current_ntp_time() bad_keys = [] mutable_params = {} for k, v in values.iteritems(): if isinstance(v, NumpyParameterData): arr_len = v.get_data().shape[0] elif isinstance(v, np.ndarray): arr_len = v.shape[0] elif isinstance(v, ConstantOverTime): continue break if arr_len > 0: values[Span.ingest_time_str] = NumpyParameterData(Span.ingest_time_str, self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type.create_filled_array(arr_len, write_time)) for key, arr in values.iteritems(): if key not in self.value_list: raise KeyError("Parameter, %s, has not been initialized" % (key)) param_type = self.master_manager.parameter_metadata[key].parameter_context.param_type if isinstance(param_type, ParameterFunctionType): bad_keys.append(key) continue #TODO: throw error instead if isinstance(arr, np.ndarray): arr = NumpyParameterData(key, arr) values[key] = arr if not isinstance(arr, ParameterData): raise TypeError("Value for %s must implement <%s>, found <%s>" % (key, ParameterData.__name__, arr.__class__.__name__)) if not isinstance(arr, ConstantOverTime): if self.master_manager.parameter_metadata[key].parameter_context.param_type.validate_value_set(arr.get_data()): self.master_manager.parameter_metadata[key].read_only = False self.master_manager.parameter_metadata[key].flush() self.master_manager.parameter_metadata[key].read_only = True all_values_constant_over_time = False elif isinstance(arr, NumpyParameterData) and arr.get_data().shape[0] != arr_len: raise ValueError("Array size for %s is inconsistent. Expected %s elements, found %s." % (key, str(arr_len), str(arr.get_data().size))) min_val, max_val = self.value_list[key].get_statistics(arr) self.update_parameter_bounds(key, (min_val, max_val)) if param_type.is_mutable: mutable_params[key] = values[key] for key in bad_keys: values.pop(key) for key in mutable_params: values.pop(key) if len(mutable_params) > 0 and self.alignment_parameter in values: mutable_params[self.alignment_parameter] = values[self.alignment_parameter] mutable_params[Span.ingest_time_str] = values[Span.ingest_time_str] if not all_values_constant_over_time and self.alignment_parameter not in values: raise LookupError("Array must be supplied for parameter, %s, to ensure alignment" % self.alignment_parameter) write_span = False if len(values) > 1 and self.alignment_parameter in values: if len(values) == 2 and len(mutable_params) > 0: pass else: write_span = True elif all_values_constant_over_time: write_span = True if write_span: span = Span(write_id, self.master_manager.guid, values, compressors=self.value_list) span_table = SpanStorageFactory.get_span_storage_obj(self.storage_name) span_table.write_span(span) if len(mutable_params) > 2: self._write_mutable_params(mutable_params, write_id + '_mutable', self.master_manager.guid)