Esempio n. 1
0
 def __init__(self, span_uuid, coverage_id, param_dict, ingest_time=None, compressors=None, mutable=False):
     self.param_dict = param_dict
     self.ingest_time = ingest_time
     self.ingest_times = []
     if ingest_time is None:
         self.ingest_time = get_current_ntp_time()
     self._set_ingest_times()
     self.ingest_time_array
     self.id = span_uuid
     self.coverage_id = coverage_id
     self.compressors = compressors
     self.mutable = mutable
Esempio n. 2
0
 def read_and_set_config(self):
     one_from_config = False
     try:
         data = Config(["res/config/coverage.yml"]).data['CoverageConfig']
         for k, v in data.iteritems():
             self.__setattr__(k, v)
             one_from_config = True
         self.using_default_config = False
         self.config_time = get_current_ntp_time()
     except Exception as ex:
         if one_from_config:
             log.info("Load from config failed with '%s'.  Using hybrid default/config file configuration" % ex.message)
             return False
         else:
             log.info("load from config failed with '%s'.  Using default config" % ex.message)
     return True
    def _create_parameter_dictionary_of_numpy_arrays(self, numpy_params, function_params=None, params=None, fill_indexes=False):
        return_dict = {}
        mask_dict = {}
        value_set_time_dict = {}
        shape_outer_dimmension = 0
        span_order = []
        span_size_dict = {}
        t_dict = {}
        if self.alignment_parameter in numpy_params:
            for id, span_data in numpy_params[self.alignment_parameter].iteritems():
                span_size_dict[id] = span_data[1].get_data().size
                shape_outer_dimmension += span_data[1].get_data().size
                span_order.append((span_data[0], id))
            span_order.sort()
            t_dict = numpy_params[self.alignment_parameter]
        dt = np.dtype(self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.value_encoding)
        arr = np.empty(shape_outer_dimmension, dtype=dt)

        insert_index = 0
        for span_tup in span_order:
            span_id = span_tup[1]
            np_data = t_dict[span_id][1].get_data()
            end_idx = insert_index+np_data.size
            arr[insert_index:end_idx] = np_data
            insert_index += np_data.size
        return_dict[self.alignment_parameter] = arr
        mask_dict[self.alignment_parameter] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool)
        value_set_time_dict[self.alignment_parameter] = self.master_manager.parameter_metadata[self.alignment_parameter].parameter_context.param_type.create_filled_array(arr.shape[0])

        ingest_name_ptype = self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type
        for id, span_data in numpy_params.iteritems():
            if id == self.alignment_parameter:
                continue
            npa_list = []
            mask_list = []
            value_set_list = []
            for span_tup in span_order:
                span_id = span_tup[1]
                span_time = span_tup[0]
                if span_id not in span_data:
                    npa = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_filled_array(span_size_dict[span_id])
                    npa_list.append(npa)
                    value_set_list.append(ingest_name_ptype.create_filled_array(npa.shape[0]))
                    mask_list.append(NumpyUtils.create_filled_array(npa.shape[0], False, dtype=np.bool))
                    continue
                else:
                    this_data = span_data[span_id][1].get_data()
                    npa_list.append(this_data)
                    mask_list.append(NumpyUtils.create_filled_array(this_data.shape[0], True, dtype=np.bool))
                    value_set_list.append(ingest_name_ptype.create_filled_array(this_data.shape[0], span_time))
            return_dict[id] = self.master_manager.parameter_metadata[id].parameter_context.param_type.create_merged_value_array(npa_list)
            from coverage_model.parameter_types import BooleanType
            mask_dict[id] = BooleanType().create_merged_value_array(mask_list)
            value_set_time_dict[id] = ingest_name_ptype.create_merged_value_array(value_set_list)

        for param_name, param_dict in function_params.iteritems():
            arr = ConstantOverTime.merge_data_as_numpy_array(return_dict[self.alignment_parameter],
                                                             param_dict,
                                                             param_type=self.master_manager.parameter_metadata[param_name].parameter_context.param_type)
            return_dict[param_name] = arr
            mask_dict[param_name] = NumpyUtils.create_filled_array(arr.shape[0], True, dtype=np.bool)
            value_set_time_dict[param_name] = ingest_name_ptype.create_filled_array(arr.shape[0], get_current_ntp_time())

        return return_dict, mask_dict, value_set_time_dict
    def write_parameters(self, write_id, values):
        if not isinstance(values, dict):
            raise TypeError("values must be dict type.  Found %s" % type(values))
        arr_len = -1
        all_values_constant_over_time = True
        write_time = get_current_ntp_time()
        bad_keys = []
        mutable_params = {}
        for k, v in values.iteritems():
            if isinstance(v, NumpyParameterData):
                arr_len = v.get_data().shape[0]
            elif isinstance(v, np.ndarray):
                arr_len = v.shape[0]
            elif isinstance(v, ConstantOverTime):
                continue
            break
        if arr_len > 0:
            values[Span.ingest_time_str] = NumpyParameterData(Span.ingest_time_str, self.master_manager.parameter_metadata[Span.ingest_time_str].parameter_context.param_type.create_filled_array(arr_len, write_time))

        for key, arr in values.iteritems():
            if key not in self.value_list:
                raise KeyError("Parameter, %s, has not been initialized" % (key))

            param_type = self.master_manager.parameter_metadata[key].parameter_context.param_type
            if isinstance(param_type, ParameterFunctionType):
                bad_keys.append(key)
                continue #TODO: throw error instead

            if isinstance(arr, np.ndarray):
                arr = NumpyParameterData(key, arr)
                values[key] = arr
            if not isinstance(arr, ParameterData):
                raise TypeError("Value for %s must implement <%s>, found <%s>" % (key, ParameterData.__name__, arr.__class__.__name__))

            if not isinstance(arr, ConstantOverTime):
                if self.master_manager.parameter_metadata[key].parameter_context.param_type.validate_value_set(arr.get_data()):
                    self.master_manager.parameter_metadata[key].read_only = False
                    self.master_manager.parameter_metadata[key].flush()
                    self.master_manager.parameter_metadata[key].read_only = True

                all_values_constant_over_time = False
            elif isinstance(arr, NumpyParameterData) and arr.get_data().shape[0] != arr_len:
                raise ValueError("Array size for %s is inconsistent.  Expected %s elements, found %s." % (key, str(arr_len), str(arr.get_data().size)))
            min_val, max_val = self.value_list[key].get_statistics(arr)
            self.update_parameter_bounds(key, (min_val, max_val))
            if param_type.is_mutable:
                mutable_params[key] = values[key]

        for key in bad_keys:
            values.pop(key)

        for key in mutable_params:
            values.pop(key)
        if len(mutable_params) > 0 and self.alignment_parameter in values:
            mutable_params[self.alignment_parameter] = values[self.alignment_parameter]
            mutable_params[Span.ingest_time_str] = values[Span.ingest_time_str]
        if not all_values_constant_over_time and self.alignment_parameter not in values:
            raise LookupError("Array must be supplied for parameter, %s, to ensure alignment" % self.alignment_parameter)

        write_span = False
        if len(values) > 1 and self.alignment_parameter in values:
            if len(values) == 2 and len(mutable_params) > 0:
                pass
            else:
                write_span = True
        elif all_values_constant_over_time:
            write_span = True
        if write_span:
            span = Span(write_id, self.master_manager.guid, values, compressors=self.value_list)
            span_table = SpanStorageFactory.get_span_storage_obj(self.storage_name)
            span_table.write_span(span)
        if len(mutable_params) > 2:
            self._write_mutable_params(mutable_params, write_id + '_mutable', self.master_manager.guid)