def get_chunks_count(tsuid, md_list, chunk_size):
"""
Get the count of chunks for a TSUID split into chunks of <chunk_size> points each
:param tsuid: tsuid to get points from
:type tsuid: str
:param md_list: List of metadata
:type md_list: dict
:param chunk_size: the size of the chunk
:type chunk_size: int
:return: the number of chunks generated
:rtype: int
"""
if tsuid not in md_list:
raise IkatsException("No metadata for TS %s" % tsuid)
if chunk_size <= 0:
raise ValueError("Chunk size must be positive")
try:
number_of_points = int(md_list[tsuid]["qual_nb_points"])
except KeyError:
raise IkatsException("qual_nb_points metadata not found for TSUID %s" % tsuid)
return int(ceil(number_of_points / chunk_size))
def get_value(self):
"""
Reads the value as the BLOB content in the process_data row identified by self.id.
The decoding process is made by decode() method, which may be overridden.
:raises IkatsException: error with its cause:
- missing process_id: request is cancelled
- read failed (http request failed)
- unexpected error
"""
try:
if self.get_process_id() is None:
raise IkatsException("Unexpected: process_id is None")
content = IkatsApi.pd.read(process_data_id=self.__id)
LOG_PROCESS_DATA_READER.debug("Content has been read. Context=%s",
str(self))
decoded_content = self.decode_content(content)
LOG_PROCESS_DATA_READER.debug(
"Content has been decoded. Context=%s", str(self))
return decoded_content
except Exception:
raise IkatsException(
"Read failure in data source ProcessDataReader: %s" %
str(self))
def update(cls, business_custo_algo):
"""
UPDATE operation on the resource CustomizedAlgo
:param cls: class parameter.
:type cls: CustomizedAlgoDao
:param business_obj: business resource needing to be created in custom database
:type business_obj: CustomizedAlgo
:return: business resource wrapping the created record in custom database: database ids are filled.
:rtype: CustomizedAlgo
"""
if not isinstance(business_custo_algo, CustomizedAlgo):
my_mess = "{}: expects arg type CustomizedAlgo: unexpected type={}"
raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX,
type(business_custo_algo).__name__))
if not business_custo_algo.is_db_id_defined():
my_mess = "{} unexpected: undefined db_id for {}".format(cls.ERROR_UPDATE_PREFIX,
business_custo_algo)
raise IkatsException(msg=my_mess)
if len(business_custo_algo.custom_params) == 0:
my_mess = "{} unexpected and useless: no customized value defined for {}"
raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX,
business_custo_algo))
if ((not business_custo_algo.implementation) or
(not business_custo_algo.implementation.is_db_id_defined())):
my_mess = "{} self.implementation must be defined in DB: undefined implementation: {}"
raise IkatsException(msg=my_mess.format(cls.ERROR_UPDATE_PREFIX,
business_custo_algo))
# init and save now
orm_obj = CustomizedAlgoDao.init_orm(business_custo_algo, save=True)
# returned_business_obj will be completed with associated CustomizedParameter
returned_business_obj = orm_obj.build_business()
returned_business_obj.clear_custom_params()
# updating strategy for associated list of CustomizedParameter:
# 1: delete all CustomizedParameter from DB
# 2: add all CustomizedParameter from business_custo_algo
# 1: delete all CustomizedParameter from DB
# read each obsolete CustomizedParameter
# using reverse relationship defined by CustomizedParameterDao::custom_algo
for old_db_edited_value in CustomizedParameterDao.objects.filter(custom_algo__id=orm_obj.id):
old_db_edited_value.delete()
for param in business_custo_algo.custom_params.values():
# remove obsolete db id : all previous children have been deleted
param.db_id = None
# 2: add all CustomizedParameter from business_custo_algo
# cls.create_edited_params( business_custo_algo, orm_obj, save_now)
returned_business_obj = orm_obj.__create_linked_custom_params(specified_business_custo_algo=business_custo_algo,
returned_business_obj=returned_business_obj,
save_now=True)
return returned_business_obj
def __init__(self, cat_parameter, value, db_id=None):
"""
Constructor
:param cat_parameter: either parameter business resource from catalogue or parameter database ID
(business db_id)
:type cat_parameter: either Parameter (business subclass of ProfileItem) or int
:param value: edited value, json-friendly object (str, int, bool, list, dict, ...)
:type value: object
:param db_id: optional, default None: db_id of self, when self is already saved in DB
:type db_id: str or None
"""
if type(cat_parameter) is int:
try:
my_param = ProfileItemDao.find_business_elem_with_key(
cat_parameter)
except ProfileItemDao.DoesNotExist:
my_msg = "Not found in catalogue: parameter with id={}"
raise IkatsException(msg=my_msg.format(cat_parameter))
elif not isinstance(cat_parameter, Parameter):
my_msg = "Unexpected {} type read instead of Parameter: id={} type= {}"
raise IkatsException(
msg=my_msg.format("cat_parameter", cat_parameter,
type(cat_parameter).__name__))
else:
my_param = cat_parameter
self._parameter = my_param
self._value = value
# When resource is read from DB: db_id records the primary key
self._db_id = db_id
def _back_transform_sax(sax_output):
"""
Back transformation of the SAX transformation. Each letter of a word is changed into a number.
.. note:: Here, the back transformation is just the paa !
:param sax_output: The output of the SAX algorithm (the definition of PAA breakpoints values (paa :
dimension n),the word (string), and the breakpoints interval (dimension length alphabet - 1))
:type sax_output: dict
:return: The "back transformation" of a SAX word into a list of number : key (tsuid) : value (paa)
:rtype: dict
"""
LOGGER.info("Starting back transformation of SAX words...")
# 1/ Input checkout
#
if type(sax_output) is not dict:
msg = "sax_output has unexpected type={}"
raise IkatsInputTypeError(msg.format(type(sax_output).__name__))
else:
# List of all the TSUID (identifier of TS) of the SAX output
tsuid_list = list(sax_output.keys())
# each TSUID has its own "paa"
for i in range(0, len(tsuid_list)):
# Not a SAX output format
if type(sax_output.get(tsuid_list[i])) is not dict:
msg = "The TSUID {} is not a dict (type={})"
raise IkatsException(msg.format(i, type(sax_output.get(tsuid_list[i]))))
elif "paa" not in list(sax_output.get(tsuid_list[i]).keys()):
msg = "The TSUID {} has no attribute *paa*"
raise IkatsException(msg.format(i))
# 2/ Build the results
#
result = {}
# the SAX output : {TSUID : paa}
for ts in range(0, len(tsuid_list)):
# List of all the TSUID of the SAX output
f_id = tsuid_list[ts]
# paa (sub-dict)
paa = sax_output.get(f_id).get("paa")
result.update({f_id: paa})
LOGGER.info(" ... back transformation finished")
return result
def _get_chunk_info(tsuid, index, md_list, chunk_size):
"""
Get the chunk <index> information for a TSUID split into chunks of <chunk_size> points each
:param tsuid: tsuid to get points from
:type tsuid: str
:param index: the index of the chunk to get
:type index: int
:param md_list: List of metadata
:type md_list: dict
:param chunk_size: the size of the chunk
:type chunk_size: int
:return: information about the chunk (chunk_index, chunk_start_window, chunk_end_window)
:rtype: list
"""
if tsuid not in md_list:
raise IkatsException("No metadata for TS %s" % tsuid)
# Number of points
if "qual_nb_points" not in md_list[tsuid]:
raise IkatsException("No qual_nb_points in metadata for TS %s" % tsuid)
nb_points = int(md_list[tsuid]["qual_nb_points"])
if nb_points <= 0:
raise ValueError("qual_nb_points shall be a positive number for %s" % tsuid)
# Timeseries start date
if "ikats_start_date" not in md_list[tsuid]:
raise IkatsException("No ikats_start_date in metadata for TS %s" % tsuid)
start_date = int(md_list[tsuid]["ikats_start_date"])
# Timeseries end date
if "ikats_end_date" not in md_list[tsuid]:
raise IkatsException("No ikats_end_date in metadata for TS %s" % tsuid)
end_date = int(md_list[tsuid]["ikats_end_date"])
# Extrapolation of the number of points
delta = int((end_date - start_date) * chunk_size / nb_points)
# Chunk start date
chunk_start = start_date + index * delta
# Chunk end date
chunk_end = chunk_start + delta
if chunk_end + delta > end_date:
chunk_end = end_date
return [index, chunk_start, chunk_end]
def _compute_chunk_slope(rdd, fid, save_new_ts=True):
"""
Compute the slope function on the current chunk
:param rdd: RDD containing the chunk data points
:param fid: functional identifier for the new timeseries
:param save_new_ts: True (default) if TS must be saved to database
:type rdd: RDD
:type fid: str
:type save_new_ts: bool
:return: The TSUID corresponding to the new timeseries and the timestamp of the last point
:rtype: tuple (str, int)
"""
# DESCRIPTION : Build the new slope for this chunk and suppress empty chunks
# INPUT : (chunk_data_points, ...)
# OUTPUT : (computed_ts_data_points, ...)
rdd_chunk_slope = rdd \
.map(lambda x: _spark_calc_slope(data=x)) \
.filter(lambda x: x is not None) \
.filter(lambda x: len(x) > 0)
# Caching this RDD because reused several times
rdd_chunk_slope.cache()
computed_tsuid = None
if save_new_ts:
try:
# DESCRIPTION : Save the slope into database and return the TSUID
# INPUT : (computed_ts_data_points, ...)
# OUTPUT : TSUID
computed_tsuid = rdd_chunk_slope \
.map(lambda x: _spark_save(fid=fid, data=x)) \
.collect()[0]
except Exception as err:
raise IkatsException("TS %s couldn't be saved: %s" % (fid, err))
try:
# Get date of last point of the chunks
end_date = int(rdd_chunk_slope \
.map(lambda x: x[-1][0]) \
.reduce(lambda x, y: max(x, y)))
except Exception as err:
raise IkatsException("End date couldn't be extracted : %s" % err)
# RDD not needed anymore, unpersist it
rdd_chunk_slope.unpersist()
return computed_tsuid, end_date
def send_value(self, value, progress_status=None):
"""
Triggers the receiver action: import the processed data into the non
temporal DB, with the defined self attributes.
- encodes the blob content with self.encode_content(value)
- calls the API to send the blob
- returns creates row ID
:param value: value to store
:type value: depending on algorithm output data type
:param progress_status: optional default None: will be used to follow the execution progress
:type progress_status: object
:return: id of written processed data
:rtype: str or int
:raises IkatsException: error with its cause:
- missing process_id: request is cancelled
- http request failed
- unexpected error
"""
try:
if self.get_process_id() is None:
raise IkatsException(
"Unexpected: send_value() called with self.get_process_id()==None"
)
encoded_value = self.encode_content(value)
LOGGER.debug("Content is encoded, ready to be written. Context=%s",
str(self))
res = IkatsApi.pd.create(data=encoded_value,
process_id=self.get_process_id(),
name=self.get_output_name())
if res['status']:
self.__written_data_id = res['id']
LOGGER.debug(
"Process-data imported in DB: returned id=%s. Context=%s",
self.__written_data_id, str(self))
else:
msg = "Failure: IkatsApi.pd.create returned error={}"
raise IkatsException(msg.format(res))
return self.__written_data_id
except Exception:
raise IkatsException("Writing failure in data receiver: %s" %
str(self))
def init_orm(cls, business_custom_algo, save=False):
"""
Build the ORM object based on corresponding business class
It will return the ORM object filled with business information
:param cls:
:type cls:
:param business_custom_algo: business object containing information
:type business_custom_algo: CustomizedAlgo
:param save:True if it is required to save object
:type save: bool
:return: the ORM object (instance of CustomizedAlgoDao)
"""
# Step1: creates/updates inherited attributes from ElementDao:
# - name
# - label
# - desc
# - db
if not business_custom_algo.is_db_id_defined():
db_obj = CustomizedAlgoDao.init_elem_orm(business_custom_algo, save=False)
else:
db_obj = CustomizedAlgoDao.objects.get(id=business_custom_algo.db_id)
if db_obj is None:
msg = "Undefined in database: {2} with id={0} name={1}"
raise IkatsException(msg.format(business_custom_algo.db_id,
business_custom_algo.name,
cls.__name__))
db_obj.name = business_custom_algo.name
db_obj.desc = business_custom_algo.description
db_obj.label = business_custom_algo.label
# Step2: creates/updates:
# - ref_implementation
implem_id = business_custom_algo.implementation.db_id
impl = ImplementationDao.objects.get(id=implem_id)
if impl is None:
my_mess = "Error in CustomizedAlgoDao: did not find ImplementationDao with id={}. See {}"
raise IkatsException(my_mess.format(implem_id, business_custom_algo))
db_obj.ref_implementation = impl
# If required: save CustomizedAlgo before updating the linked CustomizedParameter list
if save:
db_obj.save()
# init of CustomizedParameter list is done elsewhere
return db_obj
def check_customized_values(self):
"""
Checks the Customized algo: self._checked_resource
* check each defined type, according to CheckEngine.get_checking_rules()
* see check_type()
* check each defined domain
* see check_domain()
:return: the checkStatus completed with these checks
:rtype: checkStatus
"""
if not isinstance(self._checked_resource, CustomizedAlgo):
raise IkatsException(
"CheckEngine::check_customized_values() requires self._checked_resource as CustomizedAlgo"
)
for _, custom_param in self._checked_resource.custom_params.items():
the_profile_item = custom_param.get_parameter()
value = custom_param.get_value()
if value is not None:
self.check_type(profile_item=the_profile_item,
checked_value=value)
self.check_domain(profile_item=the_profile_item,
checked_value=value)
return self._check_status
def _spark_import(fid, data, generate_metadata):
"""
Create chunks of data in temporal database
:param fid: functional identifier
:param data: data to store in db
:param generate_metadata: True to generate metadata on the fly while creating data in db
:type fid: str
:type data: numpy array
:type generate_metadata: boolean
:return: identifier of ts created, start date of chunk, end date of chunk (dates in timestamp)
:rtype: tuple (tsuid, sd, ed)
"""
results = IkatsApi.ts.create(fid=fid,
data=data,
generate_metadata=generate_metadata,
sparkified=True)
start_date = data[0][0]
end_date = data[-1][0]
if results['status']:
return results['tsuid'], start_date, end_date
else:
raise IkatsException("TS %s couldn't be created" % fid)
def find_business_elem_with_name(cls, name):
"""
Search list of resources matched by name.
Note: list size is may be
:param cls:
:type cls:
:param name: name of searched resource(s)
:type name: str
:return: business objects found for given name
"""
try:
res = []
my_query_set = cls.objects.filter(name=name)
for db_obj in my_query_set:
res.append(db_obj.build_business())
return res
except Exception as err:
trace_back = sys.exc_info()[2]
raise IkatsException(
msg="Failed: find business element_with_name=" + name,
cause=err).with_traceback(trace_back)
def __pearson_from_dataset(self, dataset, context, var_one, var_two):
"""
Computes the pearson correlation coeff for the test, with the function TestCorrelationLoop.__my_pearson
:param dataset: the unit test dataset must be a dictionary of
- keys: funcIDS
- values: [ <metadata dict>, <Timeseries numpy array> ]
:type dataset: dict
:param context: the context value
:type context: int or str
:param var_one: the name of the variable one
:type var_one: str
:param var_two: the name of the variable two
:type var_two: str
:return: the pearson correlation coeff computed by TestCorrelationLoop.__my_pearson
:rtype: float
:raise IkatsException: Test preparation error when piece of data is missing
"""
ts_selection = [value[1] for value in dataset.values() if
(value[0].get(CONTEXT, None) == context) and
(value[0].get(VARIABLE, None) in [var_one, var_two])]
if len(ts_selection) != 2:
msg = "Test preparation error: expects 2 TS defined for cts={} vars=[{},{}]"
raise IkatsException(msg.format(context, var_one, var_two))
# Read values of timeseries, ignoring the timestamps
x = ts_selection[0][:, 1]
y = ts_selection[1][:, 1]
the_min = min(x.size, y.size)
x_val = x[:the_min]
y_val = y[:the_min]
return self.__my_pearson(x_val, y_val)
def _build_chunk_index_map(rdd, lambda_index):
"""
Returns a map to convert old chunk index to new one by making all chunk index contiguous, keeping the same order.
A small information of the RDD is collected in this function.
To convert old index to new index, just use:
hash_map = _build_chunk_index_map(rdd, lambda x: x[1][0])
new_index = hash_map[old_index]
where the old index is at first sub-element of the second tuple element (Example: (("tsuid",(chunk_index, data))))
:param rdd: RDD to work on
:type rdd: RDD
:param lambda_index: lambda expression used to extract chunk index position from rdd
:type lambda_index: function
:return: the map
:rtype: dict
"""
try:
# DESCRIPTION : Renumber chunk_index to get contiguous numbers
# INPUT : any rdd having the index located with the lambda expression <lambda_index>
# OUTPUT : A dict having key=old index and value=new index
chunk_renumber_map_flat = rdd.map(lambda_index).collect()
chunk_renumber_map = dict(
zip(chunk_renumber_map_flat, range(len(chunk_renumber_map_flat))))
return chunk_renumber_map
except Exception as err:
raise IkatsException("Error during chunk index remapping %s" % err)
def get_json_friendly_dict(self):
"""
Gets the JSON encoding for this data:
Example for Pearson:
{
"variables": [ "var1", "var2" ],
"x_value": { "desc": { "label": "FlightIdentifier"},
"data": [10,20,30,40]
},
"y_values": [ { "desc": { "label": "Pearson correlation"},
"data": [correl10, correl20, correl30, correl40]
}
],
"ts_lists": [ [tsuid_var1_10, tsuid_var2_10],
[tsuid_var1_20, tsuid_var2_20],
[tsuidx_var1_30, tsuidy_var2_30],
[tsuidx_var1_40, tsuidy_var2_40]
]
}
:return: the CorrelationsBycontext as a dict json-friendly
:rtype: dict
"""
if (self.__json_friendly_dict["variables"][0] is None) or \
(self.__json_friendly_dict["variables"][1] is None):
msg = "Inconsistency error: section variables is incomplete: {}"
raise IkatsException(
msg.format(self.__json_friendly_dict["variables"]))
size_x = len(self.__json_friendly_dict["x_value"]["data"])
for index, y_value in enumerate(self.__json_friendly_dict["y_values"]):
if len(y_value["data"]) != size_x:
msg = "Inconsistency: len(y_values[{}]['data']) != len(x_value) with content={}"
raise IkatsException(
msg.format(index, self.__json_friendly_dict))
if len(self.__json_friendly_dict["ts_lists"]) != size_x:
msg = "Inconsistency: len(ts_lists) != len(x_value) with content={}"
raise IkatsException(msg.format(self.__json_friendly_dict))
return self.__json_friendly_dict
def create(cls, business_custo_algo):
"""
CREATE resource in database.
:param cls: class parameter.
:type cls: CustomizedAlgoDao
:param business_custo_algo: business resource needing to be created in custom database
:type business_custo_algo: CustomizedAlgo
:return: business resource wrapping the created record in custom database: database ids are filled.
:rtype: CustomizedAlgo
:raise IkatsException: inconsistency error, or resource already created
"""
if not isinstance(business_custo_algo, CustomizedAlgo):
my_mess = "{}: expects arg type CustomizedAlgo: unexpected type={}"
raise IkatsException(msg=my_mess.format(cls.ERROR_CREATE_PREFIX,
type(business_custo_algo).__name__))
if business_custo_algo.is_db_id_defined():
my_mess = "{} unexpected: db_id already defined for {}".format(cls.ERROR_CREATE_PREFIX,
business_custo_algo)
raise IkatsException(msg=my_mess)
if len(business_custo_algo.custom_params) == 0:
my_mess = "{} unexpected and useless: no customized value defined for {}".format(cls.ERROR_CREATE_PREFIX,
business_custo_algo)
raise IkatsException(msg=my_mess)
if ((not business_custo_algo.implementation) or
(not business_custo_algo.implementation.is_db_id_defined())):
my_mess = "{} self.implementation must be defined in DB: undefined implementation: {}"
raise IkatsException(msg=my_mess.format(cls.ERROR_CREATE_PREFIX,
business_custo_algo))
# init and save now
orm_obj = CustomizedAlgoDao.init_orm(business_custo_algo, save=True)
# returned_business_obj will be completed with associated CustomizedParameter
returned_business_obj = orm_obj.build_business()
returned_business_obj = orm_obj.__create_linked_custom_params(specified_business_custo_algo=business_custo_algo,
returned_business_obj=returned_business_obj,
save_now=True)
return returned_business_obj
def set_variables(self, labels):
"""
Sets the list of variables: defines labels[i] defines the label associated to the matrix index i.
:param labels: list of labels
:type labels: list
"""
if type(labels) is not list:
raise IkatsException("Inconsistency error detected: variables should be a list")
self.__json_friendly_dict['variables'] = labels
def __retrieve_funcids(tsuids):
"""
Internal purpose: retrieves funcIds from a list of tsuids
:param tsuids: TSUIDS list to get functional identifier from
:type tsuids: list
:return: list of funcids
:rtype: list of str
:raise exception: error occurred
"""
try:
return [IkatsApi.ts.fid(tsuid=tsuid) for tsuid in tsuids]
except Exception:
raise IkatsException("Failed to convert tsuids={} to funcids".format(tsuids))
def delete_resource(cls, business_obj):
"""
DELETE operation on the business resource CustomizedAlgo and its related
CustomizedParameter resources from Django database.
Note: catalogue db resources ImplementationDao and ProfileDao are never
deleted by the DELETE on CustomizedAlgo.
Note: this method calls the default models.Model.delete() method on instance CustomizedAlgoDao,
thus this class method name is delete_resource in order to avoid an unwanted extension of
the models.Model.delete()
:param cls: class parameter.
:type cls: CustomizedAlgoDao
:param business_obj:
:type business_obj: CustomizedAlgo with required db_id
:return: report: message list that is logged on server and returned for Rest API
:rtype: list of str
"""
if business_obj.is_db_id_defined():
orm_obj = CustomizedAlgoDao.objects.get(id=business_obj.db_id)
if orm_obj is None:
my_mess = "{} resource CustomizedAlgo not found in DB with id={}: delete cancelled on {}"
raise IkatsException(my_mess.format(cls.ERROR_DELETE_PREFIX,
business_obj))
# read each obsolete CustomizedParameter
# using reverse relationship defined by CustomizedParameterDao::custom_algo
for cust_param in CustomizedParameterDao.objects.filter(custom_algo__id=orm_obj.id):
cust_param.delete()
orm_obj.delete()
else:
my_mess = "{} Undefined db_id: the delete is impossible and is cancelled for {}"
raise IkatsException(my_mess.format(cls.ERROR_DELETE_PREFIX,
business_obj))
def read(process_data_id):
"""
Reads the data blob content: for the unique process_data row identified by id.
:param process_data_id: the id key of the raw process_data to get data from
:return: the content data stored.
:rtype: bytes or str or object
:raise IkatsNotFoundError: no resource identified by ID
:raise IkatsException: failed to read
"""
response = None
try:
ntdm = NonTemporalDataMgr()
response = ntdm.download_data(process_data_id)
if response.status == 200:
# Reads the data returned by the service.
# default_content is only used if the content_type is unknown by RestClientResponse
return response.get_appropriate_content(
default_content=response.text)
elif response.status == 404:
msg = "IkatsProcessData::read({}) resource not found : HTTP response={}"
raise IkatsNotFoundError(msg.format(process_data_id, response))
else:
msg = "IkatsProcessData::read({}) failed : HTTP response={}"
raise IkatsException(msg.format(process_data_id, response))
except IkatsException:
raise
except Exception as exception:
msg = "IkatsProcessData::read({}) failed : unexpected error. got response={}"
raise IkatsException(msg.format(exception, response))
def save_rollmean(tsuid, ts_result, short_name="rollmean"):
"""
Saves the TS to database
It copies some attributes from the original TSUID, that is why it needs the tsuid
:param tsuid: original TSUID used for computation
:type tsuid: str
:param ts_result: TS resulting of the operation
:type ts_result: TimestampedMonoVal
:param short_name: Name used as short name for Functional identifier
:type short_name: str
:return: the created TSUID and its associated FID
:rtype: str, str
:raise IOError: if an issue occurs during the import
"""
try:
# Retrieve timeseries information (funcId)
original_fid = IkatsApi.ts.fid(tsuid=tsuid)
# Generate unique functional id for resulting timeseries
new_fid = '%s_%s' % (str(original_fid), short_name)
try:
IkatsApi.ts.create_ref(new_fid)
except IkatsConflictError:
# TS already exist, append timestamp to be unique
new_fid = '%s_%s_%s' % (str(original_fid), short_name,
int(time.time() * 1000))
IkatsApi.ts.create_ref(new_fid)
# Import timeseries result in DB
res_import = IkatsApi.ts.create(fid=new_fid,
data=ts_result.data,
parent=tsuid,
generate_metadata=True)
# Returns the TSUID if no issue happened
if res_import['status']:
return res_import['tsuid'], new_fid
else:
raise IOError(
" IkatsApi.ts.create(fid={}, data=..., ...) failed".format(
original_fid))
except Exception:
raise IkatsException("save_rollmean() failed")
def decode_content(self, raw_content):
"""
Decodes the raw_content as a python object, unpicking the raw content.
:param raw_content: the raw content
:type raw_content: pickle bytes
:return: the unpicked object
:rtype: any
:raises IkatsException: error occurred while loading the object from the pickled content
"""
try:
obj = pickle.loads(raw_content)
return obj
except Exception:
raise IkatsException(
"Failed to load picked object. Context={}".format(str(self)))
def correlation_ts_loop(ds_name, corr_method, context_name):
"""
Wrapper of correlation_ts_list_loop():
- translates the ds_name into a flat list of TSUIDs: computed_list
- and then calls correlation_ts_list_loop(ts_list=computed_list,
corr_method=corr_method,
context_meta=context_name)
.. seealso:: full documentation in correlation_ts_list_loop()
:param ds_name: name of the dataset grouping all the timeseries involved by the correlation loop.
:type ds_name: str
:param corr_method: the method computing the correlation between 2 timeseries.
The value must be in CORRELATION_METHODS.
Choose PEARSON to apply the pearson correlation.
:type corr_method: str
:param context_name: name of the metadata identifying each observed context,
where correlations are computed.
.. note:: this metadata shall exist for each timeseries, otherwise the
latter will be ignored.
With Airbus example: 'FlightIdentifier' identifies the flight as observed context.
:type context_name: str
:return: JSON-friendly dict: see correlation_ts_list_loop return documentation
:rtype: dict as json-friendly structure for json library
"""
if ds_name is None or type(ds_name) is not str:
msg = "Unexpected arg value: defined str is expected for ds_name={}"
raise IkatsException(msg.format(msg.format(ds_name)))
LOGGER.info("Evaluating timeseries list from the dataset named: %s",
ds_name)
ds_content = IkatsApi.ds.read(ds_name)
computed_list = ds_content['ts_list']
LOGGER.info("- Found %s TS from %s to %s", len(computed_list),
computed_list[0], computed_list[-1])
return correlation_ts_list_loop(ts_list=computed_list,
corr_method=corr_method,
context_meta=context_name)
def __create_linked_custom_params(self, specified_business_custo_algo, returned_business_obj, save_now=True):
"""
Evaluates and creates the list of CustomizedParameters specified by specified_business_custo_algo:
- create in DB each new CustomizedParameterDao, attached to self instance of CustomizedAlgoDao
- computes each CustomizedParameter from created CustomizedParametersDao
- append each CustomizedParameter to returned_business_obj
- return returned_business_obj
:param specified_business_custo_algo: the customized algo specifying the list of CustomizedParameter
:type specified_business_custo_algo: CustomizedAlgo
:param returned_business_obj: updated business object equivalent to specified_business_custo_algo,
but with created db_id
:type returned_business_obj: CustomizedAlgo
:param save_now: optional, default True: flag activating the save on each CustomizedParameterDao.
:type save_now: boolean
"""
my_implem = specified_business_custo_algo.implementation
for buz_edited_value in specified_business_custo_algo.custom_params.values():
# 1: retrieve referenced ProfileItemDao
# it is cool to look up the existence of named parameter under the implementation definition
my_profile_item = my_implem.find_by_name_input_item(buz_edited_value.parameter.name)
ref_id_profile_item = ProfileItemDao.parse_dao_id(my_profile_item.db_id)
try:
db_profile_item = ProfileItemDao.objects.get(id=ref_id_profile_item)
except ProfileItemDao.DoesNotExist:
my_mess = "Error in CustomizedAlgoDao: unresolved param definition: " + \
"no ProfileItemDao with id={}. See {}"
raise IkatsException(my_mess.format(ref_id_profile_item, specified_business_custo_algo))
db_cust_param = CustomizedParameterDao()
# 2: init
# foreign keys => assign retrieved DAO models
db_cust_param.ref_profile_item = db_profile_item
db_cust_param.custom_algo = self
# value + is_aliased
#
# encoding the DB value:
# the business value => DB value is the json encoded value
db_cust_param.edited_value = json.dumps(buz_edited_value.value)
db_cust_param.is_aliased = False
if save_now:
db_cust_param.save()
business_custo_param = db_cust_param.build_business()
returned_business_obj.add_custom_param(business_custo_param)
return returned_business_obj
def find_orm_from_element_def(cls,
name=None,
description_part=None,
label=None):
"""
This reading method finds ElementDao matching the criteria (name, description_part, label)
:param cls:
:type cls:
:param name: optional value, default None: if not None, the exact value matching the name of \
searched ElementDao. None disables the name filtering.
:type name: str
:param description_part: optional value, default None: if not None, the part of description \
searched on the ElementDao. None disables the description_part filtering.
:type description_part: str
:param label: optional value, default None: if not None, the exact value matching the label of \
searched ElementDao. None disables the name filtering.
:type label: str
:return: the filtered QuerySet
:rtype: QuerySet
"""
try:
if name is not None:
my_query_set = cls.objects.filter(name=name)
else:
my_query_set = cls.objects.all()
if label is not None:
my_query_set2 = my_query_set.filter(label=label)
else:
my_query_set2 = my_query_set
if description_part is not None:
my_query_set3 = my_query_set2.filter(
desc__contains=description_part)
else:
my_query_set3 = my_query_set2
return my_query_set3
except Exception as err:
trace_back = sys.exc_info()[2]
err_msg = "Failed: find_orm_from_element_def(name={},description_part={},label={}"
raise IkatsException(msg=err_msg.format(name, description_part,
label),
cause=err).with_traceback(trace_back)
def decode_content(self, raw_content):
"""
Decodes the raw_content as a plain text: converted into a str, using self.__char_encoding
:param raw_content: the raw content
:type raw_content: bytes or str
:return: the decoded content
:rtype: str
:raises IkatsException: unexpected raw_content type
"""
# You could have kept type(...) is ...
if isinstance(raw_content, str):
return raw_content
elif isinstance(raw_content, bytes):
return str(raw_content, self.__char_encoding)
else:
msg = "Unhandled raw content type={} instead of bytes or str. Context={}"
raise IkatsException(msg.format(type(raw_content), str(self)))
def __save_ts(data, PCId):
"""
Build one resulting TS and save it.
:param data: One column of transformed result (np.array)
:param PCId: The Id of the current principal component (PC)
:return: Dict containing:
* resulted tsuid ("tsuid")
* resulted funcId ("funcId")
* original tsuid ("origin")
"""
# Add timestamp to the data column, and store it into custom object
result = np.array([timestamps, data]).T
# Shape = (n_times, 2)
# 4.1/ Generate new FID
# -------------------------------
# Add id of current PC (in 1..k), `fid_pattern` contains str '{}'
new_fid = fid_pattern.format(PCId)
# Example: "PORTFOLIO_pc1"
try:
IkatsApi.ts.create_ref(new_fid)
# Exception: if fid already exist
except IkatsConflictError:
# TS already exist, append timestamp to be unique
new_fid = '%s_%s' % (new_fid, int(time.time() * 1000))
IkatsApi.ts.create_ref(new_fid)
# 4.2/ Import time series result in database
# -------------------------------
try:
res_import = IkatsApi.ts.create(fid=new_fid,
data=result,
generate_metadata=True,
parent=None,
sparkified=False)
new_tsuid = res_import['tsuid']
except Exception:
raise IkatsException("save scale failed")
LOGGER.debug("TSUID: %s(%s), saved", new_fid, new_tsuid)
return {"tsuid": new_tsuid, "funcId": new_fid}
def save(tsuid, ts_result, short_name="scaled", sparkified=False):
"""
Saves the TS into database.
It copies some attributes from the original TSUID, that is why it needs the tsuid
:param tsuid: original TSUID used for computation
:type tsuid: str
:param ts_result: TS resulting of the operation
:type ts_result: TimestampedMonoVal
:param short_name: Name used as short name for Functional identifier
:type short_name: str
:param sparkified: set to True to prevent from having multi-processing,
and to handle correctly the creation of TS by chunk
:type sparkified: bool
:return: the created TSUID and its associated FID
:rtype: str, str
:raises IkatsException: if an issue occurs during the import
:raises TypeError: if `ts_result` not type TimestampedMonoVal
"""
if type(ts_result) is not TimestampedMonoVal:
raise TypeError(
'Arg `ts_result` is {}, expected TimestampedMonoVal'.format(
type(ts_result)))
try:
# Generate new FID
new_fid = gen_fid(tsuid=tsuid, short_name=short_name)
# Import time series result in database
res_import = IkatsApi.ts.create(fid=new_fid,
data=ts_result.data,
generate_metadata=True,
parent=tsuid,
sparkified=sparkified)
return res_import['tsuid'], new_fid
except Exception:
raise IkatsException("save scale failed")
def fit_kmeans_internal(data, n_cluster=3, random_state=None):
"""
The internal wrapper of fit algorithm for k-means of scikit-learn.
Perform the k-means algorithm on list of numbers (floats), not SAX words !
:param data: The data-set : key (TS id), values (list of floats)
:type data : dict
:param n_cluster:The number of clusters to form as well as the number of centroids to generate.
:type n_cluster: int
:param random_state: The seed used by the random number generator (if int).
If None, the random number generator is the RandomState instance used by np.random.
:type random_state: int or NoneType
.. note:: specify `random_state` to make the results reproducible.
:return model: The KMeans model fitted on the input data-set.
:rtype model: sklearn.cluster.k_means_.KMeans
:raises IkatsException: error occurred.
"""
LOGGER.info("Starting K-Means fit with scikit-learn ...")
try:
# Fit the algorithm
model = KMeans(n_clusters=n_cluster, random_state=random_state)
model.fit(list(data.values()))
LOGGER.info(" ... finished fitting K-Means to data")
LOGGER.info(" - Exporting results to sklearn.cluster.KMeans format")
return model
except IkatsException as ike:
raise ike
except Exception:
msg = "Unexpected error: fit_kmeans_internal(..., {}, {}, {})"
raise IkatsException(msg.format(data, n_cluster, random_state))
def load_from_dict(cls, ws_dict):
"""
Reload the resource ImplementationWs with internal model:
the 'id' key is required: points to existing Implementation from the catalogue DB.
Restriction [#139805]: any other key of ws_dict is ignored, as the model is a read-only
catalogue resource via ImplementationWs.
:param cls:
:type cls:
:param ws_dict:
:type ws_dict:
"""
my_implem_id = ws_dict.get('id', None)
if my_implem_id is not None:
business_implem = ImplementationDao.find_business_elem_with_key(primary_key=my_implem_id)
else:
raise IkatsException("Unexpected: missing Implementation reference from json ImplementationWs")
return ImplementationWs(business_implem)