def delete(tsuid, no_exception=False):
"""
Delete the data corresponding to a ts and all associated metadata
If timeseries belongs to a dataset it will not be removed
Setting no_exception to True will prevent any exception to be raised.
Useful to just try to delete a TS if it exists but have no specific action to perform in case of error
:param tsuid: tsuid of the timeseries to remove
:type tsuid: str
:param no_exception: Set to True to not raise any exception, False by default
:type no_exception: bool
:raises TypeError: if *tsuid* is not a str
:raises IkatsNotFoundError: if *tsuid* is not found on server
:raises IkatsConflictError: if *tsuid* belongs to -at least- one dataset
:raises SystemError: if any other unhandled error occurred
"""
tdm = TemporalDataMgr()
try:
tdm.remove_ts(tsuid=tsuid)
except Exception:
if no_exception:
pass
else:
raise
def find_from_meta(constraint=None):
"""
From a meta data constraint provided in parameter, the method get a TS list matching these constraints
Example of constraint:
| {
| frequency: [1, 2],
| flight_phase: 8
| }
will find the TS having the following meta data:
| (frequency == 1 OR frequency == 2)
| AND
| flight_phase == 8
:returns: list of TSUID matching the constraints
:rtype: dict
:param constraint: constraint definition
:type constraint: dict
:raises TypeError: if *constraint* is not a dict
"""
tdm = TemporalDataMgr()
return tdm.get_ts_from_meta_data(constraint=constraint)
def read(tsuid_list, sd=None, ed=None):
"""
Retrieve the data corresponding to a ts (or a list of ts) without knowing date range
.. note::
if omitted, *sd* (start date) and *ed* (end date) will be retrieved from meta data for each TS
if you want a fixed windowed range, set *sd* and *ed* manually (but be aware that the TS may be
not completely gathered)
.. note::
If no range is provided and no meta data are found, this method will compute the 3 elementary statistics:
* ikats_start_date : First date of the TS
* ikats_end_date : Last date of the TS
* qual_nb_points : Number of points of the TS
:param tsuid_list:
:param sd: optional starting date (timestamp in ms from epoch)
:param ed: optional ending date (timestamp in ms from epoch)
:type tsuid_list: str or list
:type sd: int
:type ed: int
:returns: a list of ts data as numpy array
:rtype: list of numpy array
:raises TypeError: if *tsuid_list* is neither a list nor a string
"""
tdm = TemporalDataMgr()
return tdm.get_ts(tsuid_list=tsuid_list, sd=sd, ed=ed)
def update(tsuid, name, value, data_type=DTYPE.string, force_create=False):
"""
Import a meta data into TemporalDataManager
:param tsuid: Functional Identifier of the TS
:param name: Metadata name
:param value: Value of the metadata
:param data_type: data type of the meta data (used only if force_create, no type change on existing meta data)
:param force_create: True to create the meta if not exists (default: False)
:type tsuid: str
:type name: str
:type value: str or number
:type data_type: DTYPE
:type force_create: bool
:return: execution status, True if import successful, False otherwise
:rtype: bool
:raises TypeError: if *tsuid* not a str
:raises TypeError: if *name* not a str
:raises TypeError: if *value* not a str or a number
:raises TypeError: if *force_create* not a bool
:raises ValueError: if *tsuid* is empty
:raises ValueError: if *name* is empty
:raises ValueError: if *value* is empty
"""
tdm = TemporalDataMgr()
return tdm.update_meta_data(tsuid=tsuid,
name=name,
value=value,
data_type=data_type,
force_create=force_create)
def find(criterion_type, criteria_list):
"""
Retrieve the list of functional identifier records.
Each resource record aggregates one tsuid and associated fundId.
Note: partial match will not raise error, contrary to empty match.
:param criterion_type: defines criterion applicable to this search
:type criterion_type: str value accepted by server.
ex: 'tsuids' or 'funcIds'
:param criteria_list: non empty list of possible values for the criterion type
:type criteria_list: list of str
:return: matching list of functional identifier resources: dict having following keys defined:
- 'tsuid',
- and 'funcId'
:rtype: list of dict
:raises exception:
- TypeError: if unexpected arguments OR status 400 (bad request) OR unexpected http status
- ValueError: mismatched result: http status 404: not found
- ServerError: http status for server errors: 500 <= status < 600
"""
tdm = TemporalDataMgr()
return tdm.search_functional_identifiers(criterion_type=criterion_type,
criteria_list=criteria_list)
def list():
"""
Get the list of all data set and their corresponding description
:return: key: data set, value: corresponding description : [{'name':name,'description':description}]
:rtype: list of dict
"""
tdm = TemporalDataMgr()
return tdm.get_data_set_list()
def list():
"""
Get the list of all TSUID in database
:return: the list of TSUID with their associated metrics
:rtype: list
"""
tdm = TemporalDataMgr()
return tdm.get_ts_list()
def delete(name):
"""
Delete a table
:param name: the name of the table to delete
:type name: str
:return: the status of deletion (True=deleted, False otherwise)
:rtype: bool
"""
tdm = TemporalDataMgr()
return tdm.delete_table(name=name)
def get_nb_points_from_tsuid(cls, tsuid, ed=None, timeout=300):
"""
return the effective imported number of points for a given tsuid
:param tsuid: name of the metric
:param ed: end date of the ts to get (EPOCH ms)
:param timeout: timeout for the request (in seconds)
:type tsuid: str
:type ed: int
:type timeout: int
:return: the imported number of points
:rtype: int
:raises ValueError: if no TS with tsuid were found
:raises SystemError: if openTSDB triggers an error
"""
# Get Ikats information
config_reader = ConfigReader()
# Send the request to get the TSUID information
if ed is None:
# Retrieve end date from metadata
tdm = TemporalDataMgr()
metadata = tdm.get_meta_data([tsuid])[tsuid]
# Create or update the metadata
ed = int(metadata['ikats_end_date'])
q_ed = "end=%s&" % int(ed)
url = "http://%s:%s/api/query?start=0&%s&ms=true&tsuid=sum:%sms-count:%s" % (
config_reader.get('cluster', 'opentsdb.read.ip'),
int(config_reader.get(
'cluster', 'opentsdb.read.port')), q_ed, int(ed + 1), tsuid)
results = requests.get(url=url, timeout=timeout).json()
if 'error' in results:
if 'No such name for' in results['error']['message']:
raise ValueError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
else:
raise SystemError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
# Extract Nb points successfully imported
# The for loop may have at most 2 loops because of the way openTSDB output is built
nb_points = 0
for point in results[0]['dps']:
nb_points += int(results[0]['dps'][point])
return nb_points
def delete(tsuid):
"""
Delete a functional ID from its TSUID
:param tsuid: TSUID identifying the TS
:type tsuid: str
:raises TypeError: if *tsuid* not a str
:raises ValueError: if *tsuid* is empty
:raises ValueError: if FID not deleted
"""
tdm = TemporalDataMgr()
tdm.delete_fid(tsuid=tsuid)
def read(tsuid):
"""
Get a functional ID from its TSUID
:param tsuid: TSUID identifying the TS
:type tsuid: str
:raises TypeError: if *tsuid* not a str
:raises ValueError: if *tsuid* is empty
:raises ValueError: if *tsuid* doesn't have a FID
"""
tdm = TemporalDataMgr()
return tdm.get_fid(tsuid=tsuid)
def fid(tsuid):
"""
Retrieve the functional ID associated to the tsuid param.
:param tsuid: one tsuid value
:type tsuid: str
:return: retrieved functional identifier value
:rtype: str
:raises TypeError: if tsuid is not a defined str
:raises ValueError: no functional ID matching the tsuid
:raises ServerError: http answer with status : 500 <= status < 600
"""
tdm = TemporalDataMgr()
return tdm.get_func_id_from_tsuid(tsuid=tsuid)
def read(name):
"""
Reads the data blob content: for the unique table identified by id.
:param name: the id key of the raw table to get data from
:type name: str
:return: the content data stored.
:rtype: bytes or str or object
:raise IkatsNotFoundError: no resource identified by ID
:raise IkatsException: any other error
"""
tdm = TemporalDataMgr()
return tdm.read_table(name=name)
def main_test():
"""
Functional test entry point
"""
tdm = TemporalDataMgr()
ntdm = NonTemporalDataMgr()
nb_tsuid = 0
tsuid_list = []
answer = input(
'output functional Identifiers [0] or tsuids [1] (default value : 1) :'
) or '1'
if answer == '0':
tsuids_out = False
elif answer == '1':
tsuids_out = True
else:
raise ValueError(TypeError)
answer = input(
'input list of tsuids [0] or dataset name [1] (default value : 0) :'
) or '0'
if answer == '0':
print('Example tsuid_list 1 (benchmark) = ')
print(
'[000069000009000AE3, 00006E000009000AE8, 00006A000009000AE4, 00006B000009000AE5,'
' 00006C000009000AE6, 00006D000009000AE7, 000066000009000AE0, 000068000009000AE2 ]'
)
print('Example tsuid_list 2 (functional ids) = ')
print('[00001F00000B000BFE, 00008300000B000BFF, 00008400000B000C08]')
print('Enter your list of tsuids :')
tsuid = input('tsuid no 1 [ENTER to quit]: ')
while tsuid != '':
if tsuid != '':
nb_tsuid += 1
tsuid_list.append(tsuid)
tsuid = input('tsuid no ' + str(nb_tsuid + 1) +
'[ENTER to quit]: ')
else:
print('Example dataset : dsCorrMat')
tsuid_list = input('dataset_name: ')
if tsuid_list:
sd_temp = input('start date (yyyy-mm-dd) : ')
ed_temp = input('end date (yyyy-mm-dd): ')
if sd_temp and ed_temp:
sd = int(time.mktime(time.strptime(sd_temp, "%Y-%m-%d")))
ed = int(time.mktime(time.strptime(ed_temp, "%Y-%m-%d")))
# Correlation Matrix computation and display
print('Correlation Matrix (pearson) = ')
print(pearson_correlation_matrix(tdm, tsuid_list, tsuids_out))
print('PROCESS ID name : PearsonCorrelationMatrix')
else:
print('Start date / end date missing')
else:
print('tsuids list or dataset missing')
def tsuid(fid):
"""
Retrieve the tsuid associated to the func_id param.
:param fid: one func_id value
:type fid: str
:return: retrieved tsuid value
:rtype: str
:raises TypeError: if unexpected func_id parameter OR status 400 (bad request) OR unexpected http status
:raises ValueError: mismatched result: http status 404: not found
:raises ServerError: http status for server errors: 500 <= status < 600
"""
tdm = TemporalDataMgr()
return tdm.get_tsuid_from_func_id(func_id=fid)
def get_ts_to_accu(tsuid):
"""
Function to be called to retrieve one TS content values
:param tsuid: TS to get content from
:return:
"""
tdm = TemporalDataMgr(host=broadcast.value['host'],
port=broadcast.value['port'])
# The slice removes the timestamp column
ts_points = tdm.get_ts(tsuid)[0][:, 1]
accumulator.add(
{tsuid: {
"data": ts_points,
"nb_points": len(ts_points)
}})
def list(name=None, strict=True):
"""
List all tables
If name is specified, filter by name
name can contains "*", this character is considered as "any chars" (equivalent to regexp /.*/)
:param name: name to find
:param strict: consider name without any wildcards
:type name: str or None
:type strict: bool
:return: the list of tables matching the requirements
:rtype: list
"""
tdm = TemporalDataMgr()
return tdm.list_tables(name=name, strict=strict)
def create(cls,
tsuid,
name,
value,
data_type=DTYPE.string,
force_update=False):
"""
Import a meta data into TemporalDataManager
:param tsuid: Functional Identifier of the TS
:param name: Metadata name
:param value: Value of the metadata
:param data_type: data type of the meta data
:param force_update: True to create the meta if not exists (default: False)
:type tsuid: str
:type name: str
:type value: str or number
:type data_type: DTYPE
:type force_update: bool
:return: execution status, True if import successful, False otherwise
:rtype: bool
:raises TypeError: if *tsuid* not a str
:raises TypeError: if *name* not a str
:raises TypeError: if *value* not a str or a number
:raises TypeError: if *data_type* not a DTYPE
:raises TypeError: if *force_update* not a bool
:raises ValueError: if *tsuid* is empty
:raises ValueError: if *name* is empty
:raises ValueError: if *value* is empty
"""
tdm = TemporalDataMgr()
result = tdm.import_meta_data(tsuid=tsuid,
name=name,
value=value,
data_type=data_type,
force_update=force_update)
if not result:
cls.LOGGER.error("Metadata '%s' couldn't be saved for TS %s", name,
tsuid)
return result
def read(ds_name):
"""
Retrieve the list of TS identifier corresponding to a data set provided in arguments
:param ds_name: name of the data set to request TS list from
:type ds_name: str
:return:
information about ts_list and description
* *ts_list* is the list of TS matching the data_set
* *description* is the description sentence of the data set
:rtype: dict
:raises TypeError: if data_set is not a str
"""
tdm = TemporalDataMgr()
return tdm.get_data_set(data_set=ds_name)
def inherit_properties(cls, tsuid, parent):
"""
:param tsuid: TSUID of the TS (which will inherit properties)
:param parent: TSUID of parent TS (from which inheritance will be done)
:type tsuid: str
:type parent: str
"""
tdm = TemporalDataMgr()
try:
metadata = tdm.get_meta_data([parent])[parent]
for meta_name in metadata:
if not Wrapper.NON_INHERITABLE_PATTERN.match(meta_name):
tdm.import_meta_data(tsuid=tsuid,
name=meta_name,
value=metadata[meta_name],
force_update=True)
except (ValueError, TypeError, SystemError) as exception:
cls.logger.warning(
"Can't get metadata of parent TS (%s), nothing will be inherited; \nreason: %s",
parent, exception)
def create(tsuid, fid):
"""
Import a functional ID into TemporalDataManager
:param tsuid: TSUID identifying the TS
:param fid: Functional identifier
:type tsuid: str
:type fid: str
:raises TypeError: if *tsuid* not a str
:raises TypeError: if *fid* not a str
:raises ValueError: if *tsuid* is empty
:raises ValueError: if *fid* is empty
:raises IndexError: if *fid* exists
:raises SystemError: if another issue occurs
"""
tdm = TemporalDataMgr()
tdm.import_fid(tsuid=tsuid, fid=fid)
def create(ds_name, description, tsuid_list):
"""
Create a new data set composed of the *tsuid_list*
:param ds_name: name of the data set
:param description: short functional description about the content
:param tsuid_list: list of tsuid composing the data set
:type ds_name: str
:type description: str
:type tsuid_list: list OR str
:return: execution status (True if success, False otherwise)
:rtype: bool
:raises TypeError: if *tsuid_list* is not a list
"""
tdm = TemporalDataMgr()
return tdm.import_data_set(data_set_id=ds_name,
description=description,
tsuid_list=tsuid_list)
def delete(ds_name, deep=False):
"""
Remove data_set from base
:param ds_name: name of the data set to delete
:type ds_name: str
:param deep: true to deeply remove dataset (tsuids and metadata erased)
:type deep: boolean
:return: True if operation is a success, False if error occurred
:rtype: bool
.. note::
Removing an unknown data set results in a successful operation (server constraint)
The only possible errors may come from server (HTTP status code 5xx)
:raises TypeError: if *data_set* is not a str
"""
tdm = TemporalDataMgr()
return tdm.remove_data_set(data_set=ds_name, deep=deep)
def create(data, name=None, description=None):
"""
Create a table
If name or description is provided,
the method will overwrite the corresponding fields inside the data.
:param data: data to store
:param name: name of the table (optional)
:param description: description of the table (optional)
:type data: dict
:type name: str or None
:type description: str or None
:return: the id of the created table
"""
if name is not None:
data['table_desc']['name'] = name
if description is not None:
data['table_desc']['desc'] = description
tdm = TemporalDataMgr()
return tdm.create_table(data=data)
def read(ts_list, with_type=False):
"""
Request for metadata of a TS or a list of TS
.. note::
Accepted format for list of TS are:
* 'TS1,TS2,TS3,TS4'
* 'TS1'
* ['TS1','TS2','TS3','TS4']
* ['TS1']
:returns: metadata for each TS
:rtype: dict (key is TS identifier, value is list of metadata with its associated data type)
| {
| 'TS1': {'param1':{'value':'value1', 'type': 'dtype'}, 'param2':{'value':'value2', 'type': 'dtype'}},
| 'TS2': {'param1':{'value':'value1', 'type': 'dtype'}, 'param2':{'value':'value2', 'type': 'dtype'}}
| }
:rtype: dict (key is TS identifier, value is list of metadata without its associated data type)
| {
| 'TS1': {'param1':'value1', 'param2':'value2'},
| 'TS2': {'param1':'value1', 'param2':'value2'}
| }
:param ts_list: list of TS identifier
:type ts_list: str or list
:param with_type: boolean indicating the content to return
:type with_type: bool
:raises TypeError: if *ts_list* is neither a str nor a list
"""
tdm = TemporalDataMgr()
if with_type:
return tdm.get_typed_meta_data(ts_list=ts_list)
else:
return tdm.get_meta_data(ts_list=ts_list)
def test_ccf_dataset_real_ts(self):
"""
test of ccf calculation using real timeseries (dataset)
"""
tdm = TemporalDataMgr()
start_time = time.time()
# CCF correlation calculation
results = ccf(tdm=tdm, tsuid_list_or_dataset='Portfolio', cut_ts=True)
LOGGER.info("EXECUTION TIME : %.3f seconds", time.time() - start_time)
print('CORRELATION CCF :\n', results)
# The result must be a 14x14 matrix with a header line and a header column
self.assertEqual(results.shape, (14, 14))
def test_pearson_corr_real_ts(self):
"""
test of pearson correlation calculation using real timeseries (dataset)
"""
tdm = TemporalDataMgr()
# correlation calculation on benchmark
results = pearson_correlation_matrix(tdm=tdm,
tsuid_list_or_dataset='Portfolio',
tsuids_out=True,
cut_ts=True)
print('Pearson Correlation matrix :\n', results)
# The result must be a 14x14 matrix with a header line and a header column
self.assertEqual(results.shape, (14, 14))
def test_pearson_corr(self):
"""
test of pearson correlation calculation using mocked timeseries
"""
tdm = TemporalDataMgr()
# Correlation matrix
results = pearson_correlation_matrix(
tdm=tdm,
tsuid_list_or_dataset=['00001', '00002', '00003', '00004'],
tsuids_out=True)
print('Pearson Correlation matrix :\n', results)
# The result must be a 5x5 matrix with a header line and a header column
self.assertEqual(results.shape, (5, 5))
def run_sax_spark(working_tsuid):
"""
Method called by spark job
:param working_tsuid: rdd item
"""
spark_tdm = TemporalDataMgr(host=broadcast.value['host'],
port=broadcast.value['port'])
results = run_sax_from_tsuid(
tdm=spark_tdm,
tsuid=working_tsuid,
alphabet_size=broadcast.value['alphabet_size'],
word_size=broadcast.value['word_size'],
normalize=broadcast.value['normalize'])
accumulator.add({working_tsuid: results})
def test_ccf(self):
"""
test of ccf calculation using mocked timeseries
result is exactly checked
"""
tdm = TemporalDataMgr()
# CCF correlation calculation
results = ccf(
tdm=tdm,
tsuid_list_or_dataset=['00001', '00002', '00003', '00004'],
cut_ts=True)
print('CORRELATION CCF :\n', results)
# The result must be a 5x5 matrix with a header line and a header column
self.assertEqual(results.shape, (5, 5))
self.assertListEqual(
results.tolist(),
np.array([[
'', 'FuncId_0001', 'FuncId_0002', 'FuncId_0003', 'FuncId_0004'
],
[
'FuncId_0001', '1.000000000000000',
'0.981980506061966', '-1.000000000000000',
'-0.981980506061966'
],
[
'FuncId_0002', '0.981980506061966',
'1.000000000000000', '-0.981980506061966',
'-1.000000000000000'
],
[
'FuncId_0003', '-1.000000000000000',
'-0.981980506061966', '1.000000000000000',
'0.981980506061966'
],
[
'FuncId_0004', '-0.981980506061966',
'-1.000000000000000', '0.981980506061966',
'1.000000000000000'
]]).tolist())