def __init__(self, *args, **kwargs):
super(IkatsTimeseriesMgr, self).__init__(*args, **kwargs)
if self.api.emulate:
self.tsdb_client = OpenTSDBStub(session=self.api.session)
self.dm_client = DatamodelStub(session=self.api.session)
else:
self.tsdb_client = OpenTSDBClient(session=self.api.session)
self.dm_client = DatamodelClient(session=self.api.session)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.api.emulate:
self.dm_client = DatamodelStub(session=self.api.session)
else:
self.dm_client = DatamodelClient(session=self.api.session)
class IkatsMetadataMgr(IkatsGenericApiEndPoint):
"""
Ikats EndPoint specific to Metadata management
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.api.emulate:
self.dm_client = DatamodelStub(session=self.api.session)
else:
self.dm_client = DatamodelClient(session=self.api.session)
def save(self,
tsuid,
name,
value,
dtype=MDType.STRING,
raise_exception=True):
"""
Save a metadata into Datamodel
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param tsuid: timeseries identifier (TSUID)
:param name: name of the metadata to save
:param value: Value of the metadata to save
:param dtype: data type of the metadata save
:param raise_exception: (optional) Indicates if Ikats exceptions shall be raised (True, default) or not (False)
:type tsuid: str
:type name: str
:type value: str or number
:type dtype: DTYPE
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises TypeError: if *tsuid* not a str
:raises TypeError: if *name* not a str
:raises TypeError: if *value* not a str nor a number
:raises TypeError: if *dtype* not a MDType
:raises ValueError: if *tsuid* is empty
:raises ValueError: if *name* is empty
:raises ValueError: if *value* is empty
:raises IkatsConflictError: if metadata couldn't be saved
"""
try:
result = self.dm_client.metadata_create(tsuid=tsuid,
name=name,
value=value,
data_type=dtype,
force_update=True)
return result
except IkatsException:
if raise_exception:
raise
return False
def delete(self, tsuid, name, raise_exception=True):
"""
Delete a metadata associated to a TSUID
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param tsuid: tsuid associated to this metadata
:param name: Name of the metadata
:param raise_exception: (optional) Indicates if Ikats exceptions shall be raised (True, default) or not (False)
:type tsuid: str
:type name: str
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises IkatsNotFoundError: if metadata doesn't exist
"""
return self.dm_client.metadata_delete(tsuid=tsuid,
name=name,
raise_exception=raise_exception)
def fetch(self, metadata):
"""
Fetch and return metadata information about the Metadata object provided
The returned dict has the following format:
{
'md1':{'value':'value1', 'dtype': 'dtype', 'deleted': False},
'md2':{'value':'value2', 'dtype': 'dtype', 'deleted': False}
}
:param metadata: Metadata object containing a valid tsuid
:type metadata: Metadata
:returns: the object containing information about each metadata matching the TSUID.
:rtype: dict
:raises IkatsNotFoundError: if metadata doesn't exist
"""
check_type(value=metadata,
allowed_types=Metadata,
raise_exception=True)
result = self.dm_client.metadata_get_typed(
ts_list=[metadata.tsuid])[metadata.tsuid]
for md in result:
# Converts MDType
result[md]["dtype"] = result[md]["dtype"]
# Flag metadata as "not deleted"
result[md]["deleted"] = False
return result
class IkatsDatasetMgr(IkatsGenericApiEndPoint):
"""
Ikats EndPoint specific to Dataset management
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.api.emulate:
self.dm_client = DatamodelStub(session=self.api.session)
else:
self.dm_client = DatamodelClient(session=self.api.session)
def new(self, name=None, desc=None, ts=None):
"""
Create an empty local Dataset with optional *name*, *desc* and *ts*
:param name: Dataset name to use
:param desc: description of the dataset
:param ts: list of Timeseries composing the dataset
:type name: str
:type desc: str
:type ts: list of Timeseries
:returns: the Dataset object
:rtype: Dataset
:raises IkatsConflictError: if *name* already present in database
"""
try:
self.dm_client.dataset_read(name=name)
except (IkatsNotFoundError, TypeError):
# Type error occur when name is None
return Dataset(api=self.api, name=name, desc=desc, ts=ts)
raise IkatsConflictError(
"Dataset already exist. Try using `get()` method")
def save(self, ds, raise_exception=True):
"""
Save the dataset to database (creation only, no update available)
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param ds: Dataset to create
:param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False)
:type ds: Dataset
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises TypeError: if *ds* is not a valid Dataset object
:raises ValueError: if *ds* doesn't contain any Timeseries
:raises IkatsConflictError: if Dataset name already exists in database
"""
check_is_valid_ds_name(ds.name, raise_exception=True)
if ds.ts is None or (isinstance(ds.ts, list) and not ds.ts):
raise ValueError("No TS to save")
for ts in ds.ts:
if ts.tsuid is None:
raise IkatsInputError("TS %s doesn't have a TSUID" % ts.fid)
try:
self.dm_client.dataset_create(name=ds.name,
description=ds.desc,
ts=[x.tsuid for x in ds.ts])
except IkatsException:
if raise_exception:
raise
return False
return True
def get(self, name):
"""
Reads the dataset information from database
Retrieves description and list of Timeseries
:param name: Dataset name
:type name: str
:returns: the retrieved Dataset object with Timeseries list filled
:rtype: Dataset
:raises TypeError: if *name* is not a Dataset
:raises IkatsNotFoundError: if dataset not found in database
"""
check_type(value=name,
allowed_types=str,
var_name="name",
raise_exception=True)
result = self.dm_client.dataset_read(name)
ts = [
Timeseries(tsuid=x['tsuid'], fid=x['funcId'], api=self.api)
for x in result.get('ts_list', [])
]
description = result.get("description", "")
return Dataset(api=self.api, name=name, desc=description, ts=ts)
def fetch(self, dataset):
"""
Reads the dataset sub-objects from database
Retrieves the list of Timeseries
:param dataset: Dataset object
:type dataset: Dataset
:returns: the list of Timeseries composing the Dataset
:rtype: list of Timeseries
:raises TypeError: if name is not a Dataset
:raises IkatsNotFoundError: if dataset not found in database
"""
check_type(value=dataset,
allowed_types=Dataset,
var_name="dataset",
raise_exception=True)
result = self.dm_client.dataset_read(dataset.name)
ts = [
Timeseries(tsuid=x['tsuid'], fid=x['funcId'], api=self.api)
for x in result.get('ts_list', [])
]
return ts
def delete(self, name, deep=False, raise_exception=True):
"""
Remove dataset from database
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param name: Dataset name to delete
:param deep: true to deeply remove dataset (tsuid and metadata erased)
:param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False)
:type name: str or Dataset
:type deep: bool
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises TypeError: if *name* is not a str
:raises TypeError: if *deep* is not a bool
:raises ValueError: if *name* is a valid name
:raises IkatsNotFoundError: if dataset not found in database
"""
check_type(value=deep,
allowed_types=[bool, None],
var_name="deep",
raise_exception=True)
check_type(value=name,
allowed_types=[str, Dataset],
var_name="name",
raise_exception=True)
if isinstance(name, Dataset):
name = name.name
check_is_valid_ds_name(value=name, raise_exception=True)
try:
self.dm_client.dataset_delete(name=name, deep=deep)
except IkatsException:
if raise_exception:
raise
return False
return True
def list(self):
"""
Get the list of all datasets
:returns: the list of Dataset objects
:rtype: list of Dataset
"""
return [
Dataset(name=x["name"], desc=x["description"], api=self.api)
for x in self.dm_client.dataset_list()
]
class IkatsTimeseriesMgr(IkatsGenericApiEndPoint):
"""
Ikats EndPoint specific to Timeseries management
"""
def __init__(self, *args, **kwargs):
super(IkatsTimeseriesMgr, self).__init__(*args, **kwargs)
if self.api.emulate:
self.tsdb_client = OpenTSDBStub(session=self.api.session)
self.dm_client = DatamodelStub(session=self.api.session)
else:
self.tsdb_client = OpenTSDBClient(session=self.api.session)
self.dm_client = DatamodelClient(session=self.api.session)
def new(self, fid=None, data=None):
"""
Create an empty local Timeseries (if fid not provided)
If fid is set, the identifier will be created to database
:param fid: Identifier to create (if provided)
:param data: List of data points as numpy array or python 2D-list
:type fid: str
:type data: list or np.array
:returns: the Timeseries object
:rtype: Timeseries
:raises IkatsConflictError: if *fid* already present in database (use `get` instead of `new`)
"""
if fid is None:
ts = Timeseries(api=self.api)
else:
ts = self._create_ref(fid=fid)
ts.data = data
return ts
def get(self, fid=None, tsuid=None):
"""
Returns an existing Timeseries object by providing either its FID or TSUID (only one shall be provided)
:param fid: FID of the Timeseries
:param tsuid: TSUID of the Timeseries
:type fid: str
:type tsuid: str
:returns: The Timeseries object
:rtype: Timeseries
:raises ValueError: if both *fid* and *tsuid* are set (or none of them)
:raises IkatsNotFoundError: if the identifier was not found in database
"""
if bool(fid) == bool(tsuid):
raise ValueError("fid and tsuid are mutually exclusive")
if fid is not None:
tsuid = self.fid2tsuid(fid=fid, raise_exception=True)
return Timeseries(api=self.api, tsuid=tsuid, fid=fid)
def save(self,
ts,
parent=None,
generate_metadata=True,
raise_exception=True):
"""
Import timeseries data points to database or update an existing timeseries with new points
if *generate_metadata* is set or if no TSUID is present in *ts* object,
the *ikats_start_date*, *ikats_end_date* and *qual_nb_points* will be
overwritten by the first point date, last point date and number of points in *ts.data*
*parent* is the original timeseries where metadata shall be taken from
(except intrinsic ones, eg. *qual_nb_points*)
If the timeseries is a new one (object has no tsuid defined), the computation of the metadata is forced
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param ts: Timeseries object containing information about what to create
:param parent: (optional) Timeseries object of inheritance parent
:param generate_metadata: Generate metadata (set to False when doing partial import) (Default: True)
:param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False)
:type ts: Timeseries
:type parent: Timeseries
:type generate_metadata: bool
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises TypeError: if *ts* is not a valid Timeseries object
"""
# Input checks
check_type(ts, Timeseries, "ts", raise_exception=True)
check_type(parent, [Timeseries, None], "parent", raise_exception=True)
check_type(generate_metadata,
bool,
"generate_metadata",
raise_exception=True)
check_is_fid_valid(ts.fid, raise_exception=True)
try:
# First, we shall create the TSUID reference (if not provided)
if ts.tsuid is None:
ts.tsuid = self._create_ref(ts.fid).tsuid
# If the TS is fresh, we force the creation of the metadata
generate_metadata = True
# Add points to this TSUID
start_date, end_date, nb_points = self.tsdb_client.add_points(
tsuid=ts.tsuid, data=ts.data)
if generate_metadata:
# ikats_start_date
self.dm_client.metadata_update(tsuid=ts.tsuid,
name='ikats_start_date',
value=start_date,
data_type=MDType.DATE,
force_create=True)
ts.metadata.set(name='ikats_start_date',
value=start_date,
dtype=MDType.DATE)
# ikats_end_date
self.dm_client.metadata_update(tsuid=ts.tsuid,
name='ikats_end_date',
value=end_date,
data_type=MDType.DATE,
force_create=True)
ts.metadata.set(name='ikats_end_date',
value=end_date,
dtype=MDType.DATE)
# qual_nb_points
self.dm_client.metadata_update(tsuid=ts.tsuid,
name='qual_nb_points',
value=nb_points,
data_type=MDType.NUMBER,
force_create=True)
ts.metadata.set(name='qual_nb_points',
value=nb_points,
dtype=MDType.NUMBER)
# Inherit from parent when it is defined
if parent is not None:
self.inherit(ts=ts, parent=parent)
except IkatsException:
if raise_exception:
raise
return False
return True
def delete(self, ts, raise_exception=True):
"""
Delete the data corresponding to a *ts* object and all associated metadata
Note that if timeseries belongs to a dataset it will not be removed
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param ts: tsuid of the timeseries or Timeseries Object to remove
:param raise_exception: (optional) Indicates if IKATS exceptions shall be raised (True, default) or not (False)
:type ts: str or Timeseries
:type raise_exception: bool
:returns: the status of the action
:rtype: bool
:raises TypeError: if *ts* is not a str nor a Timeseries
:raises IkatsNotFoundError: if timeseries is not found on server
:raises IkatsConflictError: if timeseries belongs to -at least- one dataset
"""
check_type(value=ts,
allowed_types=[str, Timeseries],
var_name="ts",
raise_exception=True)
tsuid = ts
if isinstance(ts, Timeseries):
if ts.tsuid is not None:
tsuid = ts.tsuid
elif ts.fid is not None:
try:
tsuid = self.dm_client.get_tsuid_from_fid(fid=ts.fid)
except IkatsException:
if raise_exception:
raise
return False
else:
raise ValueError(
"Timeseries object shall have set at least tsuid or fid")
return self.dm_client.ts_delete(tsuid=tsuid,
raise_exception=raise_exception)
def list(self):
"""
Get the list of all Timeseries from database
.. note::
This action may take a while
:returns: the list of Timeseries object
:rtype: list
"""
return [
Timeseries(tsuid=x["tsuid"], fid=x["funcId"], api=self.api)
for x in self.dm_client.get_ts_list()
]
def fetch(self, ts, sd=None, ed=None):
"""
Retrieve the data corresponding to a Timeseries object as a numpy array
.. note::
if omitted, *sd* (start date) and *ed* (end date) will be retrieved from metadata
if you want a fixed windowed range, set *sd* and *ed* manually (but be aware that the TS may be
not completely gathered)
:param ts: Timeseries object
:param sd: (optional) starting date (timestamp in ms from epoch)
:param ed: (optional) ending date (timestamp in ms from epoch)
:type ts: Timeseries
:type sd: int or None
:type ed: int or None
:returns: The data points
:rtype: np.array
:raises TypeError: if *ts* is not a Timeseries object
:raises TypeError: if *sd* is not an int
:raises TypeError: if *ed* is not an int
:raises IkatsNotFoundError: if TS data points couldn't be retrieved properly
"""
check_type(value=ts,
allowed_types=Timeseries,
var_name="ts",
raise_exception=True)
check_type(value=sd,
allowed_types=[int, None],
var_name="sd",
raise_exception=True)
check_type(value=ed,
allowed_types=[int, None],
var_name="ed",
raise_exception=True)
if sd is None:
sd = ts.metadata.get(name="ikats_start_date")
check_is_valid_epoch(value=sd, raise_exception=True)
if ed is None:
ed = ts.metadata.get(name="ikats_end_date")
check_is_valid_epoch(value=ed, raise_exception=True)
try:
data_points = self.tsdb_client.get_ts_by_tsuid(tsuid=ts.tsuid,
sd=sd,
ed=ed)
# Return the points
return data_points
except ValueError:
raise IkatsNotFoundError(
"TS data points couldn't be retrieved properly")
def inherit(self, ts, parent):
"""
Make a timeseries inherit of parent's metadata according to a pattern (not all metadata inherited)
:param ts: TS object in IKATS (which will inherit)
:param parent: TS object in IKATS of inheritance parent
:type ts: Timeseries
:param parent: Timeseries
"""
try:
result = self.dm_client.metadata_get_typed([parent.tsuid
])[parent.tsuid]
for meta_name in result:
# Flag metadata as "not deleted"
result[meta_name]["deleted"] = False
if not NON_INHERITABLE_PATTERN.match(meta_name):
self.dm_client.metadata_create(
tsuid=ts.tsuid,
name=meta_name,
value=result[meta_name]["value"],
data_type=MDType(result[meta_name]["dtype"]),
force_update=True)
except (ValueError, TypeError, SystemError) as exception:
self.api.session.log.warning(
"Can't get metadata of parent TS (%s), nothing will be inherited; \nreason: %s",
parent, exception)
def find_from_meta(self, constraint=None):
"""
From a metadata constraint provided in parameter, the method get a TS list matching these constraints
Example of constraint:
| {
| frequency: [1, 2],
| flight_phase: 8
| }
will find the TS having the following metadata:
| (frequency == 1 OR frequency == 2)
| AND
| flight_phase == 8
:param constraint: constraint definition
:type constraint: dict
:returns: list of TSUID matching the constraints
:rtype: dict
:raises TypeError: if *constraint* is not a dict
"""
return self.dm_client.get_ts_from_metadata(constraint=constraint)
def tsuid2fid(self, tsuid, raise_exception=True):
"""
Retrieve the functional ID associated to the tsuid param.
:param tsuid: one tsuid value
:param raise_exception: Allow to specify if the action shall assert if not found or not
:type tsuid: str
:type raise_exception: bool
:returns: retrieved functional identifier value
:rtype: str
:raises TypeError: if tsuid is not a defined str
:raises ValueError: no functional ID matching the tsuid
:raises ServerError: http answer with status : 500 <= status < 600
"""
try:
return self.dm_client.get_func_id_from_tsuid(tsuid=tsuid)
except IkatsException:
if raise_exception:
raise
return None
def fid2tsuid(self, fid, raise_exception=True):
"""
Retrieve the TSUID associated to the functional ID param.
:param fid: the functional Identifier
:param raise_exception: Allow to specify if the action shall assert if not found or not
:type fid: str
:type raise_exception: bool
:returns: retrieved TSUID value or None if not found
:rtype: str
:raises TypeError: if fid is not str
:raises IkatsNotFoundError: no match
"""
check_is_fid_valid(fid=fid)
# Check if fid already associated to an existing tsuid
try:
return self.dm_client.get_tsuid_from_fid(fid=fid)
except IkatsException:
if raise_exception:
raise
return None
def _create_ref(self, fid):
"""
Create a reference of timeseries in temporal data database and associate it to fid
in temporal database for future use.
Shall be used before create method in case of parallel creation of data (import data via spark for example)
:param fid: Functional Identifier of the TS in Ikats
:type fid: str
:returns: A prepared Timeseries object
:rtype: Timeseries
:raises IkatsConflictError: if FID already present in database (use `get` instead of `new`)
"""
check_is_fid_valid(fid, raise_exception=True)
try:
# Check if fid already associated to an existing tsuid
tsuid = self.dm_client.get_tsuid_from_fid(fid=fid)
# if fid already exists in database, raise a conflict exception
raise IkatsConflictError(
"%s already associated to an existing tsuid: %s" %
(fid, tsuid))
except IkatsNotFoundError:
# Creation of a new tsuid
metric, tags = self.tsdb_client.gen_metric_tags()
tsuid = self.tsdb_client.assign_metric(metric=metric, tags=tags)
# finally importing tsuid/fid pair in non temporal database
self.dm_client.import_fid(tsuid=tsuid, fid=fid)
return Timeseries(tsuid=tsuid, fid=fid, api=self.api)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.dm_client = DatamodelClient(session=self.api.session)
class IkatsTableMgr(IkatsGenericApiEndPoint):
"""
Ikats EndPoint specific to Table management
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.dm_client = DatamodelClient(session=self.api.session)
def new(self, name=None, data=None):
"""
Creates an empty table locally
:param name: (optional) name of the Table
:param data: (optional) data of the table (as a JSON)
:type name: str or None
:type data: dict or None
:return: the Table object
:rtype: Table
:raises IkatsConflictError: if table already exist
"""
check_type(value=name,
allowed_types=[str, None],
var_name="name",
raise_exception=True)
check_type(value=data,
allowed_types=[dict, None],
var_name="data",
raise_exception=True)
try:
self.dm_client.table_read(name=name)
except IkatsNotFoundError:
return Table(api=self.api, name=name, data=data)
raise IkatsConflictError(
"Table already exist. Try using `get()` method")
def get(self, name):
"""
Reads the data blob content: for the unique table identified by id.
:param name: the id key of the raw table to get data from
:type name: str
:returns: the content data stored.
:rtype: bytes or str or object
:raises IkatsNotFoundError: no resource identified by ID
:raises IkatsException: any other error
"""
check_type(value=name,
allowed_types=str,
var_name="name",
raise_exception=True)
data = self.dm_client.table_read(name=name)
return Table(api=self.api, name=name, data=data)
def save(self, data, name=None, raise_exception=True):
"""
Create a table
If name or description is provided,
the method will overwrite the corresponding fields inside the data.
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param data: data to store
:param name: name of the table (optional)
:param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False)
:type data: dict
:type name: str or None
:type raise_exception: bool
:returns: the status of deletion (True=deleted, False otherwise)
:rtype: bool
:raises IkatsConflictError: if Table *name* already exists in database
"""
check_type(value=data,
allowed_types=dict,
var_name="data",
raise_exception=True)
check_type(value=name,
allowed_types=[str, None],
var_name="name",
raise_exception=True)
check_type(value=raise_exception,
allowed_types=bool,
var_name="raise_exception",
raise_exception=True)
if name is not None:
data['table_desc']['name'] = name
try:
self.dm_client.table_create(data=data)
except IkatsException:
if raise_exception:
raise
return False
return True
def delete(self, name, raise_exception=True):
"""
Delete a table
Returns a boolean status of the action (True means "OK", False means "errors occurred")
:param name: the name of the table to delete
:param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False)
:type name: str
:type raise_exception: bool
:returns: the status of deletion (True=deleted, False otherwise)
:rtype: bool
:raises IkatsNotFoundError: if table not found in database
"""
check_type(value=name,
allowed_types=str,
var_name="name",
raise_exception=True)
check_type(value=raise_exception,
allowed_types=bool,
var_name="raise_exception",
raise_exception=True)
try:
self.dm_client.table_delete(name=name)
except IkatsException:
if raise_exception:
raise
return False
return True
def list(self, name=None, strict=True):
"""
List all tables
If name is specified, filter by name
name can contains "*", this character is considered as "any chars" (equivalent to regexp /.*/)
:param name: name to find
:param strict: consider name without any wildcards
:type name: str or None
:type strict: bool
:returns: the list of tables matching the requirements
:rtype: list
"""
check_type(value=name,
allowed_types=[str, None],
var_name="name",
raise_exception=True)
check_type(value=strict,
allowed_types=bool,
var_name="strict",
raise_exception=True)
return self.dm_client.table_list(name=name, strict=strict)