def __init__(self, *args, **kwargs): super(IkatsTimeseriesMgr, self).__init__(*args, **kwargs) if self.api.emulate: self.tsdb_client = OpenTSDBStub(session=self.api.session) self.dm_client = DatamodelStub(session=self.api.session) else: self.tsdb_client = OpenTSDBClient(session=self.api.session) self.dm_client = DatamodelClient(session=self.api.session)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.api.emulate: self.dm_client = DatamodelStub(session=self.api.session) else: self.dm_client = DatamodelClient(session=self.api.session)
class IkatsMetadataMgr(IkatsGenericApiEndPoint): """ Ikats EndPoint specific to Metadata management """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.api.emulate: self.dm_client = DatamodelStub(session=self.api.session) else: self.dm_client = DatamodelClient(session=self.api.session) def save(self, tsuid, name, value, dtype=MDType.STRING, raise_exception=True): """ Save a metadata into Datamodel Returns a boolean status of the action (True means "OK", False means "errors occurred") :param tsuid: timeseries identifier (TSUID) :param name: name of the metadata to save :param value: Value of the metadata to save :param dtype: data type of the metadata save :param raise_exception: (optional) Indicates if Ikats exceptions shall be raised (True, default) or not (False) :type tsuid: str :type name: str :type value: str or number :type dtype: DTYPE :type raise_exception: bool :returns: the status of the action :rtype: bool :raises TypeError: if *tsuid* not a str :raises TypeError: if *name* not a str :raises TypeError: if *value* not a str nor a number :raises TypeError: if *dtype* not a MDType :raises ValueError: if *tsuid* is empty :raises ValueError: if *name* is empty :raises ValueError: if *value* is empty :raises IkatsConflictError: if metadata couldn't be saved """ try: result = self.dm_client.metadata_create(tsuid=tsuid, name=name, value=value, data_type=dtype, force_update=True) return result except IkatsException: if raise_exception: raise return False def delete(self, tsuid, name, raise_exception=True): """ Delete a metadata associated to a TSUID Returns a boolean status of the action (True means "OK", False means "errors occurred") :param tsuid: tsuid associated to this metadata :param name: Name of the metadata :param raise_exception: (optional) Indicates if Ikats exceptions shall be raised (True, default) or not (False) :type tsuid: str :type name: str :type raise_exception: bool :returns: the status of the action :rtype: bool :raises IkatsNotFoundError: if metadata doesn't exist """ return self.dm_client.metadata_delete(tsuid=tsuid, name=name, raise_exception=raise_exception) def fetch(self, metadata): """ Fetch and return metadata information about the Metadata object provided The returned dict has the following format: { 'md1':{'value':'value1', 'dtype': 'dtype', 'deleted': False}, 'md2':{'value':'value2', 'dtype': 'dtype', 'deleted': False} } :param metadata: Metadata object containing a valid tsuid :type metadata: Metadata :returns: the object containing information about each metadata matching the TSUID. :rtype: dict :raises IkatsNotFoundError: if metadata doesn't exist """ check_type(value=metadata, allowed_types=Metadata, raise_exception=True) result = self.dm_client.metadata_get_typed( ts_list=[metadata.tsuid])[metadata.tsuid] for md in result: # Converts MDType result[md]["dtype"] = result[md]["dtype"] # Flag metadata as "not deleted" result[md]["deleted"] = False return result
class IkatsDatasetMgr(IkatsGenericApiEndPoint): """ Ikats EndPoint specific to Dataset management """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.api.emulate: self.dm_client = DatamodelStub(session=self.api.session) else: self.dm_client = DatamodelClient(session=self.api.session) def new(self, name=None, desc=None, ts=None): """ Create an empty local Dataset with optional *name*, *desc* and *ts* :param name: Dataset name to use :param desc: description of the dataset :param ts: list of Timeseries composing the dataset :type name: str :type desc: str :type ts: list of Timeseries :returns: the Dataset object :rtype: Dataset :raises IkatsConflictError: if *name* already present in database """ try: self.dm_client.dataset_read(name=name) except (IkatsNotFoundError, TypeError): # Type error occur when name is None return Dataset(api=self.api, name=name, desc=desc, ts=ts) raise IkatsConflictError( "Dataset already exist. Try using `get()` method") def save(self, ds, raise_exception=True): """ Save the dataset to database (creation only, no update available) Returns a boolean status of the action (True means "OK", False means "errors occurred") :param ds: Dataset to create :param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False) :type ds: Dataset :type raise_exception: bool :returns: the status of the action :rtype: bool :raises TypeError: if *ds* is not a valid Dataset object :raises ValueError: if *ds* doesn't contain any Timeseries :raises IkatsConflictError: if Dataset name already exists in database """ check_is_valid_ds_name(ds.name, raise_exception=True) if ds.ts is None or (isinstance(ds.ts, list) and not ds.ts): raise ValueError("No TS to save") for ts in ds.ts: if ts.tsuid is None: raise IkatsInputError("TS %s doesn't have a TSUID" % ts.fid) try: self.dm_client.dataset_create(name=ds.name, description=ds.desc, ts=[x.tsuid for x in ds.ts]) except IkatsException: if raise_exception: raise return False return True def get(self, name): """ Reads the dataset information from database Retrieves description and list of Timeseries :param name: Dataset name :type name: str :returns: the retrieved Dataset object with Timeseries list filled :rtype: Dataset :raises TypeError: if *name* is not a Dataset :raises IkatsNotFoundError: if dataset not found in database """ check_type(value=name, allowed_types=str, var_name="name", raise_exception=True) result = self.dm_client.dataset_read(name) ts = [ Timeseries(tsuid=x['tsuid'], fid=x['funcId'], api=self.api) for x in result.get('ts_list', []) ] description = result.get("description", "") return Dataset(api=self.api, name=name, desc=description, ts=ts) def fetch(self, dataset): """ Reads the dataset sub-objects from database Retrieves the list of Timeseries :param dataset: Dataset object :type dataset: Dataset :returns: the list of Timeseries composing the Dataset :rtype: list of Timeseries :raises TypeError: if name is not a Dataset :raises IkatsNotFoundError: if dataset not found in database """ check_type(value=dataset, allowed_types=Dataset, var_name="dataset", raise_exception=True) result = self.dm_client.dataset_read(dataset.name) ts = [ Timeseries(tsuid=x['tsuid'], fid=x['funcId'], api=self.api) for x in result.get('ts_list', []) ] return ts def delete(self, name, deep=False, raise_exception=True): """ Remove dataset from database Returns a boolean status of the action (True means "OK", False means "errors occurred") :param name: Dataset name to delete :param deep: true to deeply remove dataset (tsuid and metadata erased) :param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False) :type name: str or Dataset :type deep: bool :type raise_exception: bool :returns: the status of the action :rtype: bool :raises TypeError: if *name* is not a str :raises TypeError: if *deep* is not a bool :raises ValueError: if *name* is a valid name :raises IkatsNotFoundError: if dataset not found in database """ check_type(value=deep, allowed_types=[bool, None], var_name="deep", raise_exception=True) check_type(value=name, allowed_types=[str, Dataset], var_name="name", raise_exception=True) if isinstance(name, Dataset): name = name.name check_is_valid_ds_name(value=name, raise_exception=True) try: self.dm_client.dataset_delete(name=name, deep=deep) except IkatsException: if raise_exception: raise return False return True def list(self): """ Get the list of all datasets :returns: the list of Dataset objects :rtype: list of Dataset """ return [ Dataset(name=x["name"], desc=x["description"], api=self.api) for x in self.dm_client.dataset_list() ]
class IkatsTimeseriesMgr(IkatsGenericApiEndPoint): """ Ikats EndPoint specific to Timeseries management """ def __init__(self, *args, **kwargs): super(IkatsTimeseriesMgr, self).__init__(*args, **kwargs) if self.api.emulate: self.tsdb_client = OpenTSDBStub(session=self.api.session) self.dm_client = DatamodelStub(session=self.api.session) else: self.tsdb_client = OpenTSDBClient(session=self.api.session) self.dm_client = DatamodelClient(session=self.api.session) def new(self, fid=None, data=None): """ Create an empty local Timeseries (if fid not provided) If fid is set, the identifier will be created to database :param fid: Identifier to create (if provided) :param data: List of data points as numpy array or python 2D-list :type fid: str :type data: list or np.array :returns: the Timeseries object :rtype: Timeseries :raises IkatsConflictError: if *fid* already present in database (use `get` instead of `new`) """ if fid is None: ts = Timeseries(api=self.api) else: ts = self._create_ref(fid=fid) ts.data = data return ts def get(self, fid=None, tsuid=None): """ Returns an existing Timeseries object by providing either its FID or TSUID (only one shall be provided) :param fid: FID of the Timeseries :param tsuid: TSUID of the Timeseries :type fid: str :type tsuid: str :returns: The Timeseries object :rtype: Timeseries :raises ValueError: if both *fid* and *tsuid* are set (or none of them) :raises IkatsNotFoundError: if the identifier was not found in database """ if bool(fid) == bool(tsuid): raise ValueError("fid and tsuid are mutually exclusive") if fid is not None: tsuid = self.fid2tsuid(fid=fid, raise_exception=True) return Timeseries(api=self.api, tsuid=tsuid, fid=fid) def save(self, ts, parent=None, generate_metadata=True, raise_exception=True): """ Import timeseries data points to database or update an existing timeseries with new points if *generate_metadata* is set or if no TSUID is present in *ts* object, the *ikats_start_date*, *ikats_end_date* and *qual_nb_points* will be overwritten by the first point date, last point date and number of points in *ts.data* *parent* is the original timeseries where metadata shall be taken from (except intrinsic ones, eg. *qual_nb_points*) If the timeseries is a new one (object has no tsuid defined), the computation of the metadata is forced Returns a boolean status of the action (True means "OK", False means "errors occurred") :param ts: Timeseries object containing information about what to create :param parent: (optional) Timeseries object of inheritance parent :param generate_metadata: Generate metadata (set to False when doing partial import) (Default: True) :param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False) :type ts: Timeseries :type parent: Timeseries :type generate_metadata: bool :type raise_exception: bool :returns: the status of the action :rtype: bool :raises TypeError: if *ts* is not a valid Timeseries object """ # Input checks check_type(ts, Timeseries, "ts", raise_exception=True) check_type(parent, [Timeseries, None], "parent", raise_exception=True) check_type(generate_metadata, bool, "generate_metadata", raise_exception=True) check_is_fid_valid(ts.fid, raise_exception=True) try: # First, we shall create the TSUID reference (if not provided) if ts.tsuid is None: ts.tsuid = self._create_ref(ts.fid).tsuid # If the TS is fresh, we force the creation of the metadata generate_metadata = True # Add points to this TSUID start_date, end_date, nb_points = self.tsdb_client.add_points( tsuid=ts.tsuid, data=ts.data) if generate_metadata: # ikats_start_date self.dm_client.metadata_update(tsuid=ts.tsuid, name='ikats_start_date', value=start_date, data_type=MDType.DATE, force_create=True) ts.metadata.set(name='ikats_start_date', value=start_date, dtype=MDType.DATE) # ikats_end_date self.dm_client.metadata_update(tsuid=ts.tsuid, name='ikats_end_date', value=end_date, data_type=MDType.DATE, force_create=True) ts.metadata.set(name='ikats_end_date', value=end_date, dtype=MDType.DATE) # qual_nb_points self.dm_client.metadata_update(tsuid=ts.tsuid, name='qual_nb_points', value=nb_points, data_type=MDType.NUMBER, force_create=True) ts.metadata.set(name='qual_nb_points', value=nb_points, dtype=MDType.NUMBER) # Inherit from parent when it is defined if parent is not None: self.inherit(ts=ts, parent=parent) except IkatsException: if raise_exception: raise return False return True def delete(self, ts, raise_exception=True): """ Delete the data corresponding to a *ts* object and all associated metadata Note that if timeseries belongs to a dataset it will not be removed Returns a boolean status of the action (True means "OK", False means "errors occurred") :param ts: tsuid of the timeseries or Timeseries Object to remove :param raise_exception: (optional) Indicates if IKATS exceptions shall be raised (True, default) or not (False) :type ts: str or Timeseries :type raise_exception: bool :returns: the status of the action :rtype: bool :raises TypeError: if *ts* is not a str nor a Timeseries :raises IkatsNotFoundError: if timeseries is not found on server :raises IkatsConflictError: if timeseries belongs to -at least- one dataset """ check_type(value=ts, allowed_types=[str, Timeseries], var_name="ts", raise_exception=True) tsuid = ts if isinstance(ts, Timeseries): if ts.tsuid is not None: tsuid = ts.tsuid elif ts.fid is not None: try: tsuid = self.dm_client.get_tsuid_from_fid(fid=ts.fid) except IkatsException: if raise_exception: raise return False else: raise ValueError( "Timeseries object shall have set at least tsuid or fid") return self.dm_client.ts_delete(tsuid=tsuid, raise_exception=raise_exception) def list(self): """ Get the list of all Timeseries from database .. note:: This action may take a while :returns: the list of Timeseries object :rtype: list """ return [ Timeseries(tsuid=x["tsuid"], fid=x["funcId"], api=self.api) for x in self.dm_client.get_ts_list() ] def fetch(self, ts, sd=None, ed=None): """ Retrieve the data corresponding to a Timeseries object as a numpy array .. note:: if omitted, *sd* (start date) and *ed* (end date) will be retrieved from metadata if you want a fixed windowed range, set *sd* and *ed* manually (but be aware that the TS may be not completely gathered) :param ts: Timeseries object :param sd: (optional) starting date (timestamp in ms from epoch) :param ed: (optional) ending date (timestamp in ms from epoch) :type ts: Timeseries :type sd: int or None :type ed: int or None :returns: The data points :rtype: np.array :raises TypeError: if *ts* is not a Timeseries object :raises TypeError: if *sd* is not an int :raises TypeError: if *ed* is not an int :raises IkatsNotFoundError: if TS data points couldn't be retrieved properly """ check_type(value=ts, allowed_types=Timeseries, var_name="ts", raise_exception=True) check_type(value=sd, allowed_types=[int, None], var_name="sd", raise_exception=True) check_type(value=ed, allowed_types=[int, None], var_name="ed", raise_exception=True) if sd is None: sd = ts.metadata.get(name="ikats_start_date") check_is_valid_epoch(value=sd, raise_exception=True) if ed is None: ed = ts.metadata.get(name="ikats_end_date") check_is_valid_epoch(value=ed, raise_exception=True) try: data_points = self.tsdb_client.get_ts_by_tsuid(tsuid=ts.tsuid, sd=sd, ed=ed) # Return the points return data_points except ValueError: raise IkatsNotFoundError( "TS data points couldn't be retrieved properly") def inherit(self, ts, parent): """ Make a timeseries inherit of parent's metadata according to a pattern (not all metadata inherited) :param ts: TS object in IKATS (which will inherit) :param parent: TS object in IKATS of inheritance parent :type ts: Timeseries :param parent: Timeseries """ try: result = self.dm_client.metadata_get_typed([parent.tsuid ])[parent.tsuid] for meta_name in result: # Flag metadata as "not deleted" result[meta_name]["deleted"] = False if not NON_INHERITABLE_PATTERN.match(meta_name): self.dm_client.metadata_create( tsuid=ts.tsuid, name=meta_name, value=result[meta_name]["value"], data_type=MDType(result[meta_name]["dtype"]), force_update=True) except (ValueError, TypeError, SystemError) as exception: self.api.session.log.warning( "Can't get metadata of parent TS (%s), nothing will be inherited; \nreason: %s", parent, exception) def find_from_meta(self, constraint=None): """ From a metadata constraint provided in parameter, the method get a TS list matching these constraints Example of constraint: | { | frequency: [1, 2], | flight_phase: 8 | } will find the TS having the following metadata: | (frequency == 1 OR frequency == 2) | AND | flight_phase == 8 :param constraint: constraint definition :type constraint: dict :returns: list of TSUID matching the constraints :rtype: dict :raises TypeError: if *constraint* is not a dict """ return self.dm_client.get_ts_from_metadata(constraint=constraint) def tsuid2fid(self, tsuid, raise_exception=True): """ Retrieve the functional ID associated to the tsuid param. :param tsuid: one tsuid value :param raise_exception: Allow to specify if the action shall assert if not found or not :type tsuid: str :type raise_exception: bool :returns: retrieved functional identifier value :rtype: str :raises TypeError: if tsuid is not a defined str :raises ValueError: no functional ID matching the tsuid :raises ServerError: http answer with status : 500 <= status < 600 """ try: return self.dm_client.get_func_id_from_tsuid(tsuid=tsuid) except IkatsException: if raise_exception: raise return None def fid2tsuid(self, fid, raise_exception=True): """ Retrieve the TSUID associated to the functional ID param. :param fid: the functional Identifier :param raise_exception: Allow to specify if the action shall assert if not found or not :type fid: str :type raise_exception: bool :returns: retrieved TSUID value or None if not found :rtype: str :raises TypeError: if fid is not str :raises IkatsNotFoundError: no match """ check_is_fid_valid(fid=fid) # Check if fid already associated to an existing tsuid try: return self.dm_client.get_tsuid_from_fid(fid=fid) except IkatsException: if raise_exception: raise return None def _create_ref(self, fid): """ Create a reference of timeseries in temporal data database and associate it to fid in temporal database for future use. Shall be used before create method in case of parallel creation of data (import data via spark for example) :param fid: Functional Identifier of the TS in Ikats :type fid: str :returns: A prepared Timeseries object :rtype: Timeseries :raises IkatsConflictError: if FID already present in database (use `get` instead of `new`) """ check_is_fid_valid(fid, raise_exception=True) try: # Check if fid already associated to an existing tsuid tsuid = self.dm_client.get_tsuid_from_fid(fid=fid) # if fid already exists in database, raise a conflict exception raise IkatsConflictError( "%s already associated to an existing tsuid: %s" % (fid, tsuid)) except IkatsNotFoundError: # Creation of a new tsuid metric, tags = self.tsdb_client.gen_metric_tags() tsuid = self.tsdb_client.assign_metric(metric=metric, tags=tags) # finally importing tsuid/fid pair in non temporal database self.dm_client.import_fid(tsuid=tsuid, fid=fid) return Timeseries(tsuid=tsuid, fid=fid, api=self.api)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.dm_client = DatamodelClient(session=self.api.session)
class IkatsTableMgr(IkatsGenericApiEndPoint): """ Ikats EndPoint specific to Table management """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.dm_client = DatamodelClient(session=self.api.session) def new(self, name=None, data=None): """ Creates an empty table locally :param name: (optional) name of the Table :param data: (optional) data of the table (as a JSON) :type name: str or None :type data: dict or None :return: the Table object :rtype: Table :raises IkatsConflictError: if table already exist """ check_type(value=name, allowed_types=[str, None], var_name="name", raise_exception=True) check_type(value=data, allowed_types=[dict, None], var_name="data", raise_exception=True) try: self.dm_client.table_read(name=name) except IkatsNotFoundError: return Table(api=self.api, name=name, data=data) raise IkatsConflictError( "Table already exist. Try using `get()` method") def get(self, name): """ Reads the data blob content: for the unique table identified by id. :param name: the id key of the raw table to get data from :type name: str :returns: the content data stored. :rtype: bytes or str or object :raises IkatsNotFoundError: no resource identified by ID :raises IkatsException: any other error """ check_type(value=name, allowed_types=str, var_name="name", raise_exception=True) data = self.dm_client.table_read(name=name) return Table(api=self.api, name=name, data=data) def save(self, data, name=None, raise_exception=True): """ Create a table If name or description is provided, the method will overwrite the corresponding fields inside the data. Returns a boolean status of the action (True means "OK", False means "errors occurred") :param data: data to store :param name: name of the table (optional) :param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False) :type data: dict :type name: str or None :type raise_exception: bool :returns: the status of deletion (True=deleted, False otherwise) :rtype: bool :raises IkatsConflictError: if Table *name* already exists in database """ check_type(value=data, allowed_types=dict, var_name="data", raise_exception=True) check_type(value=name, allowed_types=[str, None], var_name="name", raise_exception=True) check_type(value=raise_exception, allowed_types=bool, var_name="raise_exception", raise_exception=True) if name is not None: data['table_desc']['name'] = name try: self.dm_client.table_create(data=data) except IkatsException: if raise_exception: raise return False return True def delete(self, name, raise_exception=True): """ Delete a table Returns a boolean status of the action (True means "OK", False means "errors occurred") :param name: the name of the table to delete :param raise_exception: Indicates if exceptions shall be raised (True, default) or not (False) :type name: str :type raise_exception: bool :returns: the status of deletion (True=deleted, False otherwise) :rtype: bool :raises IkatsNotFoundError: if table not found in database """ check_type(value=name, allowed_types=str, var_name="name", raise_exception=True) check_type(value=raise_exception, allowed_types=bool, var_name="raise_exception", raise_exception=True) try: self.dm_client.table_delete(name=name) except IkatsException: if raise_exception: raise return False return True def list(self, name=None, strict=True): """ List all tables If name is specified, filter by name name can contains "*", this character is considered as "any chars" (equivalent to regexp /.*/) :param name: name to find :param strict: consider name without any wildcards :type name: str or None :type strict: bool :returns: the list of tables matching the requirements :rtype: list """ check_type(value=name, allowed_types=[str, None], var_name="name", raise_exception=True) check_type(value=strict, allowed_types=bool, var_name="strict", raise_exception=True) return self.dm_client.table_list(name=name, strict=strict)