def get_chunks_count(tsuid, md_list, chunk_size): """ Get the count of chunks for a TSUID split into chunks of <chunk_size> points each :param tsuid: tsuid to get points from :type tsuid: str :param md_list: List of metadata :type md_list: dict :param chunk_size: the size of the chunk :type chunk_size: int :return: the number of chunks generated :rtype: int """ if tsuid not in md_list: raise IkatsException("No metadata for TS %s" % tsuid) if chunk_size <= 0: raise ValueError("Chunk size must be positive") try: number_of_points = int(md_list[tsuid]["qual_nb_points"]) except KeyError: raise IkatsException("qual_nb_points metadata not found for TSUID %s" % tsuid) return int(ceil(number_of_points / chunk_size))
def get_value(self): """ Reads the value as the BLOB content in the process_data row identified by self.id. The decoding process is made by decode() method, which may be overridden. :raises IkatsException: error with its cause: - missing process_id: request is cancelled - read failed (http request failed) - unexpected error """ try: if self.get_process_id() is None: raise IkatsException("Unexpected: process_id is None") content = IkatsApi.pd.read(process_data_id=self.__id) LOG_PROCESS_DATA_READER.debug("Content has been read. Context=%s", str(self)) decoded_content = self.decode_content(content) LOG_PROCESS_DATA_READER.debug( "Content has been decoded. Context=%s", str(self)) return decoded_content except Exception: raise IkatsException( "Read failure in data source ProcessDataReader: %s" % str(self))
def update(cls, business_custo_algo): """ UPDATE operation on the resource CustomizedAlgo :param cls: class parameter. :type cls: CustomizedAlgoDao :param business_obj: business resource needing to be created in custom database :type business_obj: CustomizedAlgo :return: business resource wrapping the created record in custom database: database ids are filled. :rtype: CustomizedAlgo """ if not isinstance(business_custo_algo, CustomizedAlgo): my_mess = "{}: expects arg type CustomizedAlgo: unexpected type={}" raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX, type(business_custo_algo).__name__)) if not business_custo_algo.is_db_id_defined(): my_mess = "{} unexpected: undefined db_id for {}".format(cls.ERROR_UPDATE_PREFIX, business_custo_algo) raise IkatsException(msg=my_mess) if len(business_custo_algo.custom_params) == 0: my_mess = "{} unexpected and useless: no customized value defined for {}" raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX, business_custo_algo)) if ((not business_custo_algo.implementation) or (not business_custo_algo.implementation.is_db_id_defined())): my_mess = "{} self.implementation must be defined in DB: undefined implementation: {}" raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX, business_custo_algo)) # init and save now orm_obj = CustomizedAlgoDao.init_orm(business_custo_algo, save=True) # returned_business_obj will be completed with associated CustomizedParameter returned_business_obj = orm_obj.build_business() returned_business_obj.clear_custom_params() # updating strategy for associated list of CustomizedParameter: # 1: delete all CustomizedParameter from DB # 2: add all CustomizedParameter from business_custo_algo # 1: delete all CustomizedParameter from DB # read each obsolete CustomizedParameter # using reverse relationship defined by CustomizedParameterDao::custom_algo for old_db_edited_value in CustomizedParameterDao.objects.filter(custom_algo__id=orm_obj.id): old_db_edited_value.delete() for param in business_custo_algo.custom_params.values(): # remove obsolete db id : all previous children have been deleted param.db_id = None # 2: add all CustomizedParameter from business_custo_algo # cls.create_edited_params( business_custo_algo, orm_obj, save_now) returned_business_obj = orm_obj.__create_linked_custom_params(specified_business_custo_algo=business_custo_algo, returned_business_obj=returned_business_obj, save_now=True) return returned_business_obj
def __init__(self, cat_parameter, value, db_id=None): """ Constructor :param cat_parameter: either parameter business resource from catalogue or parameter database ID (business db_id) :type cat_parameter: either Parameter (business subclass of ProfileItem) or int :param value: edited value, json-friendly object (str, int, bool, list, dict, ...) :type value: object :param db_id: optional, default None: db_id of self, when self is already saved in DB :type db_id: str or None """ if type(cat_parameter) is int: try: my_param = ProfileItemDao.find_business_elem_with_key( cat_parameter) except ProfileItemDao.DoesNotExist: my_msg = "Not found in catalogue: parameter with id={}" raise IkatsException(msg=my_msg.format(cat_parameter)) elif not isinstance(cat_parameter, Parameter): my_msg = "Unexpected {} type read instead of Parameter: id={} type= {}" raise IkatsException( msg=my_msg.format("cat_parameter", cat_parameter, type(cat_parameter).__name__)) else: my_param = cat_parameter self._parameter = my_param self._value = value # When resource is read from DB: db_id records the primary key self._db_id = db_id
def _back_transform_sax(sax_output): """ Back transformation of the SAX transformation. Each letter of a word is changed into a number. .. note:: Here, the back transformation is just the paa ! :param sax_output: The output of the SAX algorithm (the definition of PAA breakpoints values (paa : dimension n),the word (string), and the breakpoints interval (dimension length alphabet - 1)) :type sax_output: dict :return: The "back transformation" of a SAX word into a list of number : key (tsuid) : value (paa) :rtype: dict """ LOGGER.info("Starting back transformation of SAX words...") # 1/ Input checkout # if type(sax_output) is not dict: msg = "sax_output has unexpected type={}" raise IkatsInputTypeError(msg.format(type(sax_output).__name__)) else: # List of all the TSUID (identifier of TS) of the SAX output tsuid_list = list(sax_output.keys()) # each TSUID has its own "paa" for i in range(0, len(tsuid_list)): # Not a SAX output format if type(sax_output.get(tsuid_list[i])) is not dict: msg = "The TSUID {} is not a dict (type={})" raise IkatsException(msg.format(i, type(sax_output.get(tsuid_list[i])))) elif "paa" not in list(sax_output.get(tsuid_list[i]).keys()): msg = "The TSUID {} has no attribute *paa*" raise IkatsException(msg.format(i)) # 2/ Build the results # result = {} # the SAX output : {TSUID : paa} for ts in range(0, len(tsuid_list)): # List of all the TSUID of the SAX output f_id = tsuid_list[ts] # paa (sub-dict) paa = sax_output.get(f_id).get("paa") result.update({f_id: paa}) LOGGER.info(" ... back transformation finished") return result
def _get_chunk_info(tsuid, index, md_list, chunk_size): """ Get the chunk <index> information for a TSUID split into chunks of <chunk_size> points each :param tsuid: tsuid to get points from :type tsuid: str :param index: the index of the chunk to get :type index: int :param md_list: List of metadata :type md_list: dict :param chunk_size: the size of the chunk :type chunk_size: int :return: information about the chunk (chunk_index, chunk_start_window, chunk_end_window) :rtype: list """ if tsuid not in md_list: raise IkatsException("No metadata for TS %s" % tsuid) # Number of points if "qual_nb_points" not in md_list[tsuid]: raise IkatsException("No qual_nb_points in metadata for TS %s" % tsuid) nb_points = int(md_list[tsuid]["qual_nb_points"]) if nb_points <= 0: raise ValueError("qual_nb_points shall be a positive number for %s" % tsuid) # Timeseries start date if "ikats_start_date" not in md_list[tsuid]: raise IkatsException("No ikats_start_date in metadata for TS %s" % tsuid) start_date = int(md_list[tsuid]["ikats_start_date"]) # Timeseries end date if "ikats_end_date" not in md_list[tsuid]: raise IkatsException("No ikats_end_date in metadata for TS %s" % tsuid) end_date = int(md_list[tsuid]["ikats_end_date"]) # Extrapolation of the number of points delta = int((end_date - start_date) * chunk_size / nb_points) # Chunk start date chunk_start = start_date + index * delta # Chunk end date chunk_end = chunk_start + delta if chunk_end + delta > end_date: chunk_end = end_date return [index, chunk_start, chunk_end]
def _compute_chunk_slope(rdd, fid, save_new_ts=True): """ Compute the slope function on the current chunk :param rdd: RDD containing the chunk data points :param fid: functional identifier for the new timeseries :param save_new_ts: True (default) if TS must be saved to database :type rdd: RDD :type fid: str :type save_new_ts: bool :return: The TSUID corresponding to the new timeseries and the timestamp of the last point :rtype: tuple (str, int) """ # DESCRIPTION : Build the new slope for this chunk and suppress empty chunks # INPUT : (chunk_data_points, ...) # OUTPUT : (computed_ts_data_points, ...) rdd_chunk_slope = rdd \ .map(lambda x: _spark_calc_slope(data=x)) \ .filter(lambda x: x is not None) \ .filter(lambda x: len(x) > 0) # Caching this RDD because reused several times rdd_chunk_slope.cache() computed_tsuid = None if save_new_ts: try: # DESCRIPTION : Save the slope into database and return the TSUID # INPUT : (computed_ts_data_points, ...) # OUTPUT : TSUID computed_tsuid = rdd_chunk_slope \ .map(lambda x: _spark_save(fid=fid, data=x)) \ .collect()[0] except Exception as err: raise IkatsException("TS %s couldn't be saved: %s" % (fid, err)) try: # Get date of last point of the chunks end_date = int(rdd_chunk_slope \ .map(lambda x: x[-1][0]) \ .reduce(lambda x, y: max(x, y))) except Exception as err: raise IkatsException("End date couldn't be extracted : %s" % err) # RDD not needed anymore, unpersist it rdd_chunk_slope.unpersist() return computed_tsuid, end_date
def send_value(self, value, progress_status=None): """ Triggers the receiver action: import the processed data into the non temporal DB, with the defined self attributes. - encodes the blob content with self.encode_content(value) - calls the API to send the blob - returns creates row ID :param value: value to store :type value: depending on algorithm output data type :param progress_status: optional default None: will be used to follow the execution progress :type progress_status: object :return: id of written processed data :rtype: str or int :raises IkatsException: error with its cause: - missing process_id: request is cancelled - http request failed - unexpected error """ try: if self.get_process_id() is None: raise IkatsException( "Unexpected: send_value() called with self.get_process_id()==None" ) encoded_value = self.encode_content(value) LOGGER.debug("Content is encoded, ready to be written. Context=%s", str(self)) res = IkatsApi.pd.create(data=encoded_value, process_id=self.get_process_id(), name=self.get_output_name()) if res['status']: self.__written_data_id = res['id'] LOGGER.debug( "Process-data imported in DB: returned id=%s. Context=%s", self.__written_data_id, str(self)) else: msg = "Failure: IkatsApi.pd.create returned error={}" raise IkatsException(msg.format(res)) return self.__written_data_id except Exception: raise IkatsException("Writing failure in data receiver: %s" % str(self))
def init_orm(cls, business_custom_algo, save=False): """ Build the ORM object based on corresponding business class It will return the ORM object filled with business information :param cls: :type cls: :param business_custom_algo: business object containing information :type business_custom_algo: CustomizedAlgo :param save:True if it is required to save object :type save: bool :return: the ORM object (instance of CustomizedAlgoDao) """ # Step1: creates/updates inherited attributes from ElementDao: # - name # - label # - desc # - db if not business_custom_algo.is_db_id_defined(): db_obj = CustomizedAlgoDao.init_elem_orm(business_custom_algo, save=False) else: db_obj = CustomizedAlgoDao.objects.get(id=business_custom_algo.db_id) if db_obj is None: msg = "Undefined in database: {2} with id={0} name={1}" raise IkatsException(msg.format(business_custom_algo.db_id, business_custom_algo.name, cls.__name__)) db_obj.name = business_custom_algo.name db_obj.desc = business_custom_algo.description db_obj.label = business_custom_algo.label # Step2: creates/updates: # - ref_implementation implem_id = business_custom_algo.implementation.db_id impl = ImplementationDao.objects.get(id=implem_id) if impl is None: my_mess = "Error in CustomizedAlgoDao: did not find ImplementationDao with id={}. See {}" raise IkatsException(my_mess.format(implem_id, business_custom_algo)) db_obj.ref_implementation = impl # If required: save CustomizedAlgo before updating the linked CustomizedParameter list if save: db_obj.save() # init of CustomizedParameter list is done elsewhere return db_obj
def check_customized_values(self): """ Checks the Customized algo: self._checked_resource * check each defined type, according to CheckEngine.get_checking_rules() * see check_type() * check each defined domain * see check_domain() :return: the checkStatus completed with these checks :rtype: checkStatus """ if not isinstance(self._checked_resource, CustomizedAlgo): raise IkatsException( "CheckEngine::check_customized_values() requires self._checked_resource as CustomizedAlgo" ) for _, custom_param in self._checked_resource.custom_params.items(): the_profile_item = custom_param.get_parameter() value = custom_param.get_value() if value is not None: self.check_type(profile_item=the_profile_item, checked_value=value) self.check_domain(profile_item=the_profile_item, checked_value=value) return self._check_status
def _spark_import(fid, data, generate_metadata): """ Create chunks of data in temporal database :param fid: functional identifier :param data: data to store in db :param generate_metadata: True to generate metadata on the fly while creating data in db :type fid: str :type data: numpy array :type generate_metadata: boolean :return: identifier of ts created, start date of chunk, end date of chunk (dates in timestamp) :rtype: tuple (tsuid, sd, ed) """ results = IkatsApi.ts.create(fid=fid, data=data, generate_metadata=generate_metadata, sparkified=True) start_date = data[0][0] end_date = data[-1][0] if results['status']: return results['tsuid'], start_date, end_date else: raise IkatsException("TS %s couldn't be created" % fid)
def find_business_elem_with_name(cls, name): """ Search list of resources matched by name. Note: list size is may be :param cls: :type cls: :param name: name of searched resource(s) :type name: str :return: business objects found for given name """ try: res = [] my_query_set = cls.objects.filter(name=name) for db_obj in my_query_set: res.append(db_obj.build_business()) return res except Exception as err: trace_back = sys.exc_info()[2] raise IkatsException( msg="Failed: find business element_with_name=" + name, cause=err).with_traceback(trace_back)
def __pearson_from_dataset(self, dataset, context, var_one, var_two): """ Computes the pearson correlation coeff for the test, with the function TestCorrelationLoop.__my_pearson :param dataset: the unit test dataset must be a dictionary of - keys: funcIDS - values: [ <metadata dict>, <Timeseries numpy array> ] :type dataset: dict :param context: the context value :type context: int or str :param var_one: the name of the variable one :type var_one: str :param var_two: the name of the variable two :type var_two: str :return: the pearson correlation coeff computed by TestCorrelationLoop.__my_pearson :rtype: float :raise IkatsException: Test preparation error when piece of data is missing """ ts_selection = [value[1] for value in dataset.values() if (value[0].get(CONTEXT, None) == context) and (value[0].get(VARIABLE, None) in [var_one, var_two])] if len(ts_selection) != 2: msg = "Test preparation error: expects 2 TS defined for cts={} vars=[{},{}]" raise IkatsException(msg.format(context, var_one, var_two)) # Read values of timeseries, ignoring the timestamps x = ts_selection[0][:, 1] y = ts_selection[1][:, 1] the_min = min(x.size, y.size) x_val = x[:the_min] y_val = y[:the_min] return self.__my_pearson(x_val, y_val)
def _build_chunk_index_map(rdd, lambda_index): """ Returns a map to convert old chunk index to new one by making all chunk index contiguous, keeping the same order. A small information of the RDD is collected in this function. To convert old index to new index, just use: hash_map = _build_chunk_index_map(rdd, lambda x: x[1][0]) new_index = hash_map[old_index] where the old index is at first sub-element of the second tuple element (Example: (("tsuid",(chunk_index, data)))) :param rdd: RDD to work on :type rdd: RDD :param lambda_index: lambda expression used to extract chunk index position from rdd :type lambda_index: function :return: the map :rtype: dict """ try: # DESCRIPTION : Renumber chunk_index to get contiguous numbers # INPUT : any rdd having the index located with the lambda expression <lambda_index> # OUTPUT : A dict having key=old index and value=new index chunk_renumber_map_flat = rdd.map(lambda_index).collect() chunk_renumber_map = dict( zip(chunk_renumber_map_flat, range(len(chunk_renumber_map_flat)))) return chunk_renumber_map except Exception as err: raise IkatsException("Error during chunk index remapping %s" % err)
def get_json_friendly_dict(self): """ Gets the JSON encoding for this data: Example for Pearson: { "variables": [ "var1", "var2" ], "x_value": { "desc": { "label": "FlightIdentifier"}, "data": [10,20,30,40] }, "y_values": [ { "desc": { "label": "Pearson correlation"}, "data": [correl10, correl20, correl30, correl40] } ], "ts_lists": [ [tsuid_var1_10, tsuid_var2_10], [tsuid_var1_20, tsuid_var2_20], [tsuidx_var1_30, tsuidy_var2_30], [tsuidx_var1_40, tsuidy_var2_40] ] } :return: the CorrelationsBycontext as a dict json-friendly :rtype: dict """ if (self.__json_friendly_dict["variables"][0] is None) or \ (self.__json_friendly_dict["variables"][1] is None): msg = "Inconsistency error: section variables is incomplete: {}" raise IkatsException( msg.format(self.__json_friendly_dict["variables"])) size_x = len(self.__json_friendly_dict["x_value"]["data"]) for index, y_value in enumerate(self.__json_friendly_dict["y_values"]): if len(y_value["data"]) != size_x: msg = "Inconsistency: len(y_values[{}]['data']) != len(x_value) with content={}" raise IkatsException( msg.format(index, self.__json_friendly_dict)) if len(self.__json_friendly_dict["ts_lists"]) != size_x: msg = "Inconsistency: len(ts_lists) != len(x_value) with content={}" raise IkatsException(msg.format(self.__json_friendly_dict)) return self.__json_friendly_dict
def create(cls, business_custo_algo): """ CREATE resource in database. :param cls: class parameter. :type cls: CustomizedAlgoDao :param business_custo_algo: business resource needing to be created in custom database :type business_custo_algo: CustomizedAlgo :return: business resource wrapping the created record in custom database: database ids are filled. :rtype: CustomizedAlgo :raise IkatsException: inconsistency error, or resource already created """ if not isinstance(business_custo_algo, CustomizedAlgo): my_mess = "{}: expects arg type CustomizedAlgo: unexpected type={}" raise IkatsException(msg=my_mess.format(cls.ERROR_CREATE_PREFIX, type(business_custo_algo).__name__)) if business_custo_algo.is_db_id_defined(): my_mess = "{} unexpected: db_id already defined for {}".format(cls.ERROR_CREATE_PREFIX, business_custo_algo) raise IkatsException(msg=my_mess) if len(business_custo_algo.custom_params) == 0: my_mess = "{} unexpected and useless: no customized value defined for {}".format(cls.ERROR_CREATE_PREFIX, business_custo_algo) raise IkatsException(msg=my_mess) if ((not business_custo_algo.implementation) or (not business_custo_algo.implementation.is_db_id_defined())): my_mess = "{} self.implementation must be defined in DB: undefined implementation: {}" raise IkatsException(msg=my_mess.format(cls.ERROR_CREATE_PREFIX, business_custo_algo)) # init and save now orm_obj = CustomizedAlgoDao.init_orm(business_custo_algo, save=True) # returned_business_obj will be completed with associated CustomizedParameter returned_business_obj = orm_obj.build_business() returned_business_obj = orm_obj.__create_linked_custom_params(specified_business_custo_algo=business_custo_algo, returned_business_obj=returned_business_obj, save_now=True) return returned_business_obj
def set_variables(self, labels): """ Sets the list of variables: defines labels[i] defines the label associated to the matrix index i. :param labels: list of labels :type labels: list """ if type(labels) is not list: raise IkatsException("Inconsistency error detected: variables should be a list") self.__json_friendly_dict['variables'] = labels
def __retrieve_funcids(tsuids): """ Internal purpose: retrieves funcIds from a list of tsuids :param tsuids: TSUIDS list to get functional identifier from :type tsuids: list :return: list of funcids :rtype: list of str :raise exception: error occurred """ try: return [IkatsApi.ts.fid(tsuid=tsuid) for tsuid in tsuids] except Exception: raise IkatsException("Failed to convert tsuids={} to funcids".format(tsuids))
def delete_resource(cls, business_obj): """ DELETE operation on the business resource CustomizedAlgo and its related CustomizedParameter resources from Django database. Note: catalogue db resources ImplementationDao and ProfileDao are never deleted by the DELETE on CustomizedAlgo. Note: this method calls the default models.Model.delete() method on instance CustomizedAlgoDao, thus this class method name is delete_resource in order to avoid an unwanted extension of the models.Model.delete() :param cls: class parameter. :type cls: CustomizedAlgoDao :param business_obj: :type business_obj: CustomizedAlgo with required db_id :return: report: message list that is logged on server and returned for Rest API :rtype: list of str """ if business_obj.is_db_id_defined(): orm_obj = CustomizedAlgoDao.objects.get(id=business_obj.db_id) if orm_obj is None: my_mess = "{} resource CustomizedAlgo not found in DB with id={}: delete cancelled on {}" raise IkatsException(my_mess.format(cls.ERROR_DELETE_PREFIX, business_obj)) # read each obsolete CustomizedParameter # using reverse relationship defined by CustomizedParameterDao::custom_algo for cust_param in CustomizedParameterDao.objects.filter(custom_algo__id=orm_obj.id): cust_param.delete() orm_obj.delete() else: my_mess = "{} Undefined db_id: the delete is impossible and is cancelled for {}" raise IkatsException(my_mess.format(cls.ERROR_DELETE_PREFIX, business_obj))
def read(process_data_id): """ Reads the data blob content: for the unique process_data row identified by id. :param process_data_id: the id key of the raw process_data to get data from :return: the content data stored. :rtype: bytes or str or object :raise IkatsNotFoundError: no resource identified by ID :raise IkatsException: failed to read """ response = None try: ntdm = NonTemporalDataMgr() response = ntdm.download_data(process_data_id) if response.status == 200: # Reads the data returned by the service. # default_content is only used if the content_type is unknown by RestClientResponse return response.get_appropriate_content( default_content=response.text) elif response.status == 404: msg = "IkatsProcessData::read({}) resource not found : HTTP response={}" raise IkatsNotFoundError(msg.format(process_data_id, response)) else: msg = "IkatsProcessData::read({}) failed : HTTP response={}" raise IkatsException(msg.format(process_data_id, response)) except IkatsException: raise except Exception as exception: msg = "IkatsProcessData::read({}) failed : unexpected error. got response={}" raise IkatsException(msg.format(exception, response))
def save_rollmean(tsuid, ts_result, short_name="rollmean"): """ Saves the TS to database It copies some attributes from the original TSUID, that is why it needs the tsuid :param tsuid: original TSUID used for computation :type tsuid: str :param ts_result: TS resulting of the operation :type ts_result: TimestampedMonoVal :param short_name: Name used as short name for Functional identifier :type short_name: str :return: the created TSUID and its associated FID :rtype: str, str :raise IOError: if an issue occurs during the import """ try: # Retrieve timeseries information (funcId) original_fid = IkatsApi.ts.fid(tsuid=tsuid) # Generate unique functional id for resulting timeseries new_fid = '%s_%s' % (str(original_fid), short_name) try: IkatsApi.ts.create_ref(new_fid) except IkatsConflictError: # TS already exist, append timestamp to be unique new_fid = '%s_%s_%s' % (str(original_fid), short_name, int(time.time() * 1000)) IkatsApi.ts.create_ref(new_fid) # Import timeseries result in DB res_import = IkatsApi.ts.create(fid=new_fid, data=ts_result.data, parent=tsuid, generate_metadata=True) # Returns the TSUID if no issue happened if res_import['status']: return res_import['tsuid'], new_fid else: raise IOError( " IkatsApi.ts.create(fid={}, data=..., ...) failed".format( original_fid)) except Exception: raise IkatsException("save_rollmean() failed")
def decode_content(self, raw_content): """ Decodes the raw_content as a python object, unpicking the raw content. :param raw_content: the raw content :type raw_content: pickle bytes :return: the unpicked object :rtype: any :raises IkatsException: error occurred while loading the object from the pickled content """ try: obj = pickle.loads(raw_content) return obj except Exception: raise IkatsException( "Failed to load picked object. Context={}".format(str(self)))
def correlation_ts_loop(ds_name, corr_method, context_name): """ Wrapper of correlation_ts_list_loop(): - translates the ds_name into a flat list of TSUIDs: computed_list - and then calls correlation_ts_list_loop(ts_list=computed_list, corr_method=corr_method, context_meta=context_name) .. seealso:: full documentation in correlation_ts_list_loop() :param ds_name: name of the dataset grouping all the timeseries involved by the correlation loop. :type ds_name: str :param corr_method: the method computing the correlation between 2 timeseries. The value must be in CORRELATION_METHODS. Choose PEARSON to apply the pearson correlation. :type corr_method: str :param context_name: name of the metadata identifying each observed context, where correlations are computed. .. note:: this metadata shall exist for each timeseries, otherwise the latter will be ignored. With Airbus example: 'FlightIdentifier' identifies the flight as observed context. :type context_name: str :return: JSON-friendly dict: see correlation_ts_list_loop return documentation :rtype: dict as json-friendly structure for json library """ if ds_name is None or type(ds_name) is not str: msg = "Unexpected arg value: defined str is expected for ds_name={}" raise IkatsException(msg.format(msg.format(ds_name))) LOGGER.info("Evaluating timeseries list from the dataset named: %s", ds_name) ds_content = IkatsApi.ds.read(ds_name) computed_list = ds_content['ts_list'] LOGGER.info("- Found %s TS from %s to %s", len(computed_list), computed_list[0], computed_list[-1]) return correlation_ts_list_loop(ts_list=computed_list, corr_method=corr_method, context_meta=context_name)
def __create_linked_custom_params(self, specified_business_custo_algo, returned_business_obj, save_now=True): """ Evaluates and creates the list of CustomizedParameters specified by specified_business_custo_algo: - create in DB each new CustomizedParameterDao, attached to self instance of CustomizedAlgoDao - computes each CustomizedParameter from created CustomizedParametersDao - append each CustomizedParameter to returned_business_obj - return returned_business_obj :param specified_business_custo_algo: the customized algo specifying the list of CustomizedParameter :type specified_business_custo_algo: CustomizedAlgo :param returned_business_obj: updated business object equivalent to specified_business_custo_algo, but with created db_id :type returned_business_obj: CustomizedAlgo :param save_now: optional, default True: flag activating the save on each CustomizedParameterDao. :type save_now: boolean """ my_implem = specified_business_custo_algo.implementation for buz_edited_value in specified_business_custo_algo.custom_params.values(): # 1: retrieve referenced ProfileItemDao # it is cool to look up the existence of named parameter under the implementation definition my_profile_item = my_implem.find_by_name_input_item(buz_edited_value.parameter.name) ref_id_profile_item = ProfileItemDao.parse_dao_id(my_profile_item.db_id) try: db_profile_item = ProfileItemDao.objects.get(id=ref_id_profile_item) except ProfileItemDao.DoesNotExist: my_mess = "Error in CustomizedAlgoDao: unresolved param definition: " + \ "no ProfileItemDao with id={}. See {}" raise IkatsException(my_mess.format(ref_id_profile_item, specified_business_custo_algo)) db_cust_param = CustomizedParameterDao() # 2: init # foreign keys => assign retrieved DAO models db_cust_param.ref_profile_item = db_profile_item db_cust_param.custom_algo = self # value + is_aliased # # encoding the DB value: # the business value => DB value is the json encoded value db_cust_param.edited_value = json.dumps(buz_edited_value.value) db_cust_param.is_aliased = False if save_now: db_cust_param.save() business_custo_param = db_cust_param.build_business() returned_business_obj.add_custom_param(business_custo_param) return returned_business_obj
def find_orm_from_element_def(cls, name=None, description_part=None, label=None): """ This reading method finds ElementDao matching the criteria (name, description_part, label) :param cls: :type cls: :param name: optional value, default None: if not None, the exact value matching the name of \ searched ElementDao. None disables the name filtering. :type name: str :param description_part: optional value, default None: if not None, the part of description \ searched on the ElementDao. None disables the description_part filtering. :type description_part: str :param label: optional value, default None: if not None, the exact value matching the label of \ searched ElementDao. None disables the name filtering. :type label: str :return: the filtered QuerySet :rtype: QuerySet """ try: if name is not None: my_query_set = cls.objects.filter(name=name) else: my_query_set = cls.objects.all() if label is not None: my_query_set2 = my_query_set.filter(label=label) else: my_query_set2 = my_query_set if description_part is not None: my_query_set3 = my_query_set2.filter( desc__contains=description_part) else: my_query_set3 = my_query_set2 return my_query_set3 except Exception as err: trace_back = sys.exc_info()[2] err_msg = "Failed: find_orm_from_element_def(name={},description_part={},label={}" raise IkatsException(msg=err_msg.format(name, description_part, label), cause=err).with_traceback(trace_back)
def decode_content(self, raw_content): """ Decodes the raw_content as a plain text: converted into a str, using self.__char_encoding :param raw_content: the raw content :type raw_content: bytes or str :return: the decoded content :rtype: str :raises IkatsException: unexpected raw_content type """ # You could have kept type(...) is ... if isinstance(raw_content, str): return raw_content elif isinstance(raw_content, bytes): return str(raw_content, self.__char_encoding) else: msg = "Unhandled raw content type={} instead of bytes or str. Context={}" raise IkatsException(msg.format(type(raw_content), str(self)))
def __save_ts(data, PCId): """ Build one resulting TS and save it. :param data: One column of transformed result (np.array) :param PCId: The Id of the current principal component (PC) :return: Dict containing: * resulted tsuid ("tsuid") * resulted funcId ("funcId") * original tsuid ("origin") """ # Add timestamp to the data column, and store it into custom object result = np.array([timestamps, data]).T # Shape = (n_times, 2) # 4.1/ Generate new FID # ------------------------------- # Add id of current PC (in 1..k), `fid_pattern` contains str '{}' new_fid = fid_pattern.format(PCId) # Example: "PORTFOLIO_pc1" try: IkatsApi.ts.create_ref(new_fid) # Exception: if fid already exist except IkatsConflictError: # TS already exist, append timestamp to be unique new_fid = '%s_%s' % (new_fid, int(time.time() * 1000)) IkatsApi.ts.create_ref(new_fid) # 4.2/ Import time series result in database # ------------------------------- try: res_import = IkatsApi.ts.create(fid=new_fid, data=result, generate_metadata=True, parent=None, sparkified=False) new_tsuid = res_import['tsuid'] except Exception: raise IkatsException("save scale failed") LOGGER.debug("TSUID: %s(%s), saved", new_fid, new_tsuid) return {"tsuid": new_tsuid, "funcId": new_fid}
def save(tsuid, ts_result, short_name="scaled", sparkified=False): """ Saves the TS into database. It copies some attributes from the original TSUID, that is why it needs the tsuid :param tsuid: original TSUID used for computation :type tsuid: str :param ts_result: TS resulting of the operation :type ts_result: TimestampedMonoVal :param short_name: Name used as short name for Functional identifier :type short_name: str :param sparkified: set to True to prevent from having multi-processing, and to handle correctly the creation of TS by chunk :type sparkified: bool :return: the created TSUID and its associated FID :rtype: str, str :raises IkatsException: if an issue occurs during the import :raises TypeError: if `ts_result` not type TimestampedMonoVal """ if type(ts_result) is not TimestampedMonoVal: raise TypeError( 'Arg `ts_result` is {}, expected TimestampedMonoVal'.format( type(ts_result))) try: # Generate new FID new_fid = gen_fid(tsuid=tsuid, short_name=short_name) # Import time series result in database res_import = IkatsApi.ts.create(fid=new_fid, data=ts_result.data, generate_metadata=True, parent=tsuid, sparkified=sparkified) return res_import['tsuid'], new_fid except Exception: raise IkatsException("save scale failed")
def fit_kmeans_internal(data, n_cluster=3, random_state=None): """ The internal wrapper of fit algorithm for k-means of scikit-learn. Perform the k-means algorithm on list of numbers (floats), not SAX words ! :param data: The data-set : key (TS id), values (list of floats) :type data : dict :param n_cluster:The number of clusters to form as well as the number of centroids to generate. :type n_cluster: int :param random_state: The seed used by the random number generator (if int). If None, the random number generator is the RandomState instance used by np.random. :type random_state: int or NoneType .. note:: specify `random_state` to make the results reproducible. :return model: The KMeans model fitted on the input data-set. :rtype model: sklearn.cluster.k_means_.KMeans :raises IkatsException: error occurred. """ LOGGER.info("Starting K-Means fit with scikit-learn ...") try: # Fit the algorithm model = KMeans(n_clusters=n_cluster, random_state=random_state) model.fit(list(data.values())) LOGGER.info(" ... finished fitting K-Means to data") LOGGER.info(" - Exporting results to sklearn.cluster.KMeans format") return model except IkatsException as ike: raise ike except Exception: msg = "Unexpected error: fit_kmeans_internal(..., {}, {}, {})" raise IkatsException(msg.format(data, n_cluster, random_state))
def load_from_dict(cls, ws_dict): """ Reload the resource ImplementationWs with internal model: the 'id' key is required: points to existing Implementation from the catalogue DB. Restriction [#139805]: any other key of ws_dict is ignored, as the model is a read-only catalogue resource via ImplementationWs. :param cls: :type cls: :param ws_dict: :type ws_dict: """ my_implem_id = ws_dict.get('id', None) if my_implem_id is not None: business_implem = ImplementationDao.find_business_elem_with_key(primary_key=my_implem_id) else: raise IkatsException("Unexpected: missing Implementation reference from json ImplementationWs") return ImplementationWs(business_implem)