def __init__(self, host=None, port=None):
"""
Initializer
:param host: host to connect to
:param port: port to use for connection
"""
# Create the logger object
self.logger = logging.getLogger(__name__)
# Configuration file
self.config_reader = ConfigReader()
# Host to connect to (IP address or known host name)
self._host = None
if host is not None:
self.host = host
else:
self.host = self.config_reader.get('cluster', 'tdm.ip')
# Port Number to connect to
self._port = None
if port is not None:
self.port = port
else:
self.port = int(self.config_reader.get('cluster', 'tdm.port'))
def _get_metric_tags_from_tsuid(cls, tsuid):
"""
Get the metric and tags of a TSUID provided from tsdb-uid table
:param tsuid: TSUID to get info from
:type tsuid: str
:return: the metric and tags
:rtype: tuple (metric, tags)
:raises ValueError: if TSUID is unknown
:raises ValueError: if OpenTSDB result can't be parsed
"""
# get Ikats information
config_reader = ConfigReader()
if tsuid:
# extracting uids by cutting tsuid in slices of 6 characters
uids = [tsuid[i:i + 6] for i in range(0, len(tsuid), 6)]
else:
raise ValueError("TSUID incorrect (got:%s)" % tsuid)
metric = None
tags = {}
for i, uid in enumerate(uids):
if i == 0:
item_type = 'metric'
elif i % 2 == 0:
item_type = 'tagv'
else:
item_type = 'tagk'
url = "http://%s:%s/api/uid/uidmeta?uid=%s&type=%s" % (
config_reader.get('cluster', 'opentsdb.read.ip'),
int(config_reader.get('cluster',
'opentsdb.read.port')), uid, item_type)
results = requests.get(url=url)
if 200 <= results.status_code < 300:
try:
result = results.json()['name']
except:
raise ValueError("OpenTSDB result not parsable (got:%s)" %
results.status_code)
else:
raise ValueError("UID unknown (got:%s)" % results.status_code)
if item_type == 'metric':
metric = result
tag_key = ''
elif item_type == 'tagk':
tag_key = result
tags[tag_key] = None
else:
if tag_key in tags:
tags[tag_key] = result
return metric, tags
def get_nb_points_from_tsuid(cls, tsuid, ed=None, timeout=300):
"""
return the effective imported number of points for a given tsuid
:param tsuid: name of the metric
:param ed: end date of the ts to get (EPOCH ms)
:param timeout: timeout for the request (in seconds)
:type tsuid: str
:type ed: int
:type timeout: int
:return: the imported number of points
:rtype: int
:raises ValueError: if no TS with tsuid were found
:raises SystemError: if openTSDB triggers an error
"""
# Get Ikats information
config_reader = ConfigReader()
# Send the request to get the TSUID information
if ed is None:
# Retrieve end date from metadata
tdm = TemporalDataMgr()
metadata = tdm.get_meta_data([tsuid])[tsuid]
# Create or update the metadata
ed = int(metadata['ikats_end_date'])
q_ed = "end=%s&" % int(ed)
url = "http://%s:%s/api/query?start=0&%s&ms=true&tsuid=sum:%sms-count:%s" % (
config_reader.get('cluster', 'opentsdb.read.ip'),
int(config_reader.get(
'cluster', 'opentsdb.read.port')), q_ed, int(ed + 1), tsuid)
results = requests.get(url=url, timeout=timeout).json()
if 'error' in results:
if 'No such name for' in results['error']['message']:
raise ValueError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
else:
raise SystemError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
# Extract Nb points successfully imported
# The for loop may have at most 2 loops because of the way openTSDB output is built
nb_points = 0
for point in results[0]['dps']:
nb_points += int(results[0]['dps'][point])
return nb_points
def __init__(self, host=None, port=None, qsize=1000, threads_count=1):
"""
Main OpenTSDB client.
:param host: Host to connect to (overrides configuration file)
:param port: port to connect to (overrides configuration file)
:param qsize: Size of the send_queue to use (bigger implies big memory usage) (default:1k)
:param threads_count: set the initial threads count (default:1 meaning, no multi-threaded)
:type host: str
:type port: int
:type qsize: int
:type threads_count: int
"""
self.thr_list = []
# Prepare events detecting the end of an import
self._event_done = Event()
self._event_abort = Event()
# Get the cluster configuration
config = ConfigReader()
# Host and port to connect to
self.host = host or config.get('cluster', 'opentsdb.write.ip')
self.port = port or int(config.get('cluster', 'opentsdb.write.port'))
self.use_threads = threads_count > 1
if self.use_threads:
self.send_queue = Queue(maxsize=qsize)
self.result_queue = Queue()
self.queue_max_size = qsize
self.threads_count = threads_count
# Start threads
for _ in range(min(threads_count, os.cpu_count())):
self.__add_thr()
def create_tsuid(cls, fid, show_details=False):
"""
Create a reference of timeseries in openTSDB without creating any data
:param fid: Functional Identifier of the TS in Ikats
:param show_details: Show metric and tags if set to True
:type fid: str
:type show_details: bool
:return: the timeseries reference in database (tsuid)
:rtype: str
:raises IkatsConflictError: if TSUID already exist
"""
tdm = TemporalDataMgr()
try:
# check if fid already associated to an existing tsuid
tsuid = tdm.get_tsuid_from_func_id(func_id=fid)
# if fid already exist in database, raise a conflict exception
raise IkatsConflictError(
"%s already associated to an existing tsuid: %s" %
(fid, tsuid))
except ValueError:
# here creation of a new tsuid
metric, tags = cls._gen_metric_tags()
# get Ikats config information
config_reader = ConfigReader()
# formatting request
url = "http://%s:%s/api/uid/assign?metric=%s&tagk=%s&tagv=%s" \
% (config_reader.get('cluster', 'opentsdb.read.ip'),
int(config_reader.get('cluster', 'opentsdb.read.port')),
metric,
','.join([str(k) for k, v in tags.items()]),
','.join([str(v) for k, v in tags.items()]))
results = requests.get(url=url).json()
# initializing tsuid with metric uid retrieved from opentsdb json response
tsuid = cls._extract_uid_from_json(item_type='metric',
value=metric,
json=results)
# retrieving and concatenating by pair [ tagk + tagv ] uids from opentsdb json response
tagkv_items = [
cls._extract_uid_from_json(
item_type='tagk', value=str(k), json=results) +
cls._extract_uid_from_json(
item_type='tagv', value=str(v), json=results)
for k, v in tags.items()
]
# concatenating [tagk + tagv] uids to previously initialized tsuid, after having sorted them in
# increasing order
tsuid += ''.join(item for item in sorted(tagkv_items))
# finally importing tsuid/fid pair in non temporal database
tdm.import_fid(tsuid=tsuid, fid=fid)
if show_details:
return tsuid, metric, tags
return tsuid
def _get_tsuid_from_metric_tags(cls, metric, ed=None, timeout=120, **tags):
"""
return the TSUID (and effective imported number of points) from a metric name and a list of tags
:param metric: name of the metric
:param ed: end date of the ts to get (EPOCH ms)
:param tags: dict of tags key and tags values
:type metric: str
:type ed: int
:type tags: dict
:return: the TSUID and the imported number of points
:rtype: tuple
:raises ValueError: if more than 1 TS matching the criteria exists in openTSDB
:raises ValueError: if no TS with metric and tags were found
:raises SystemError: if openTSDB triggers an error
"""
# get Ikats information
config_reader = ConfigReader()
# Build a string like "{tagk:tagv,tagk2:tagv2,tagk3:tagv3}"
tag_string = "{%s}" % ','.join(
["%s=%s" % (k, v) for k, v in tags.items()])
# Send the request to get the TSUID information
q_ed = ""
if ed is not None:
q_ed = "end=%s&" % int(ed)
# The "random" query parameter is a trick to not hit opentsdb cache
url = "http://%s:%s/api/query?start=0&%sshow_tsuids=true&ms=true&m=sum:%sms-count:%s%s&_=%s" % (
config_reader.get('cluster', 'opentsdb.read.ip'),
int(config_reader.get('cluster', 'opentsdb.read.port')), q_ed,
int(ed + 1), metric, tag_string, random.random())
results = requests.get(url=url, timeout=timeout).json()
if 'error' in results:
if 'No such name for' in results['error']['message']:
raise ValueError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
else:
raise SystemError("OpenTSDB Error : %s (url: %s)" %
(results['error']['message'], url))
# Extract TSUID
try:
tsuid = results[0]['tsuids'][0]
except IndexError:
cls.logger.error("No results for url : %s", url)
raise ValueError("No results for url : %s" % url)
# Extract Nb points successfully imported
# The for loop may have at most 2 loops because of the way openTSDB output is built
nb_points = 0
for point in results[0]['dps']:
nb_points += int(results[0]['dps'][point])
# Do some checks
if len(results[0]['tsuids']) > 1:
cls.logger.error("Too many results: %s", len(results['results']))
raise ValueError("Too many results: %s" % len(results['results']))
return tsuid, nb_points
import logging
from django.test import TestCase
import mock
from apps.algo.catalogue.models.orm.implem import ImplementationDao
from apps.algo.execute.models.business.scripts.execalgo import run
from ikats.core.config.ConfigReader import ConfigReader
from ikats.core.library.status import State
from ikats.core.resource.client.temporal_data_mgr import DTYPE
from ikats_processing.core.resource_config import ResourceClientSingleton
import numpy as np
LOGGER = logging.getLogger(__name__)
CONFIG_READER = ConfigReader()
HOST = CONFIG_READER.get('cluster', 'tdm.ip')
PORT = int(CONFIG_READER.get('cluster', 'tdm.port'))
# noinspection PyUnusedLocal
def get_fid_from_tsuid_mock(self, tsuid):
"""
Mock of TemporalDataMgr.get_data_set method
Same parameters and types as the original function
"""
return {'funcId': 'funcId' + tsuid, 'tsuid': tsuid}
from unittest import TestCase
import httpretty
import mock
import numpy as np
from ikats.core.config.ConfigReader import ConfigReader
from ikats.core.resource.api import IkatsApi
from ikats.core.resource.client.temporal_data_mgr import DTYPE
from ikats.core.resource.opentsdb.HttpClient import HttpClient
from ikats.core.resource.opentsdb.wrapper import Wrapper
USE_REAL_SERVER = False
# Configuration file
CONFIG_READER = ConfigReader()
# Address of the real server to use for tests
TEST_HOST = CONFIG_READER.get('cluster', 'tdm.ip')
TEST_PORT = int(CONFIG_READER.get('cluster', 'tdm.port'))
TEST_OPENTSDB_HOST = CONFIG_READER.get('cluster', 'opentsdb.read.ip')
TEST_OPENTSDB_PORT = int(CONFIG_READER.get('cluster', 'opentsdb.read.port'))
ROOT_URL = 'http://%s:%s/TemporalDataManagerWebApp/webapi' % (TEST_HOST,
TEST_PORT)
DIRECT_ROOT_URL = 'http://%s:%s/api' % (TEST_OPENTSDB_HOST, TEST_OPENTSDB_PORT)
def log_to_stdout(logger_to_use):
"""
Allow to print some loggers to stdout
class RestClient(object):
"""
Generic class to communicate using REST API
"""
class VERB(Enum):
"""
Definition of possibilities for HTTP verb
Only the following 4 are managed because they are the only allowed verbs for CRUD interface
* CREATE -> POST
* READ -> GET
* UPDATE -> PUT
* DELETE -> DELETE
"""
POST = 0
GET = 1
PUT = 2
DELETE = 3
def __init__(self, host=None, port=None):
"""
Initializer
:param host: host to connect to
:param port: port to use for connection
"""
# Create the logger object
self.logger = logging.getLogger(__name__)
# Configuration file
self.config_reader = ConfigReader()
# Host to connect to (IP address or known host name)
self._host = None
if host is not None:
self.host = host
else:
self.host = self.config_reader.get('cluster', 'tdm.ip')
# Port Number to connect to
self._port = None
if port is not None:
self.port = port
else:
self.port = int(self.config_reader.get('cluster', 'tdm.port'))
@property
def host(self):
"""
Host address
:getter: provide the stored host address
:setter: check host validity
:raises TypeError: if host has a wrong type (in setter)
:raises ValueError: if host is not an understandable URL (in setter)
"""
return self._host
@host.setter
def host(self, value):
"""
See getter
"""
if type(value) is not None:
if type(value) is not str:
raise TypeError("Host must be a string (got %s)" % type(value))
if value == "":
raise ValueError("Host must be filled")
if not is_url_valid(value):
raise ValueError("Host not a valid URL : %s" % value)
self._host = value
@property
def port(self):
"""
port number
:getter: provide the stored port address
:setter: check port validity
:raises TypeError: if port is not a number
:raises ValueError: if port is not a correct port number
"""
return self._port
@port.setter
def port(self, value):
"""
See getter
"""
if type(value) is not int:
raise TypeError("Port must be a number (got %s)" % type(value))
if value <= 0 or value >= 65535:
raise ValueError("Port must be within ]0:65535] (got %s)" % value)
self._port = value
def _send(self,
verb=None,
template="",
uri_params=None,
q_params=None,
files=None,
data=None,
json_data=None,
headers=None):
"""
Generic call command that should not be called directly
It performs the following actions:
* checks the input type validity
* calls the correct verb method from the library (get, post, put, delete)
* formats the output (utf-8)
* Handles the status from the server
* decode the output
* return the data
:param template: optional, default "": key matching the template to use for url building:
see dict ikats.core.resource.client.utils.TEMPLATES
:param uri_params: optional, default None: parameters applied to the template
:param verb: optional, default None: HTTP method to call
:type verb: IkatsRest.VERB
:param q_params: optional, default None: list of query parameters
:type q_params: dict or None
:param files: optional, default None: files full path to attach to request
:type files: str or list or None
:param data: optional, default None: data input consumed by request
-note: when data is not None, json must be None
:type data: object
:param json_data: optional, default None: json input consumed by request
-note: when json is not None, data must be None
:type json_data: object
:return: the response as a anonymous class containing the following attributes:
class Result:
url = *url of the request performed*
json = *body content parsed from json*
text = *body content parsed as text*
raw = *raw response content*
status = *HTTP status code*
reason = *reason (useful in case of HTTP status code 4xx or 5xx)
This way to return results improve readability of the code.
Example:
r = self.send(...)
if r.status == 200:
print(r.text)
:rtype: anonymous class
.. note:
Timeout set to following values:
- 120 seconds for GET and POST
- 120 seconds for PUT and DELETE
:raises TypeError: if VERB is incorrect
:raises TypeError: if FORMAT is incorrect
:raises ValueError: if a parameter of uri_param contains spaces
if there are unexpected argument values
:raises ServerError: if receiving HTTP error code 5xx
"""
# Check the validity of inputs
assert (template
in TEMPLATES), "Template is not defined: %s" % template
if not isinstance(verb, RestClient.VERB):
self.logger.error(
'Verb type is %s whereas IkatsRest.VERB is expected',
type(verb))
raise TypeError(
"Verb type is %s whereas IkatsRest.VERB is expected",
type(verb))
if (data is not None) and (json_data is not None):
raise ValueError(
"Integrity error: arguments data and json_data are mutually exclusive."
)
# Build the URL
if uri_params is None:
uri_params = {}
if 'host' not in uri_params:
uri_params['host'] = self.host
if 'port' not in uri_params:
uri_params['port'] = self.port
url = TEMPLATES[template]['pattern'] % uri_params
# Converts file to 'requests' module format
json_file = build_json_files(files)
# Dispatch method
try:
if verb == RestClient.VERB.POST:
result = requests.post(url,
data=data,
json=json_data,
files=json_file,
params=q_params,
timeout=600,
headers=headers)
elif verb == RestClient.VERB.GET:
result = requests.get(url,
params=q_params,
timeout=600,
headers=headers)
elif verb == RestClient.VERB.PUT:
result = requests.put(url,
params=q_params,
timeout=600,
headers=headers)
elif verb == RestClient.VERB.DELETE:
result = requests.delete(url,
params=q_params,
timeout=600,
headers=headers)
else:
self.logger.error(
"Verb [%s] is unknown, shall be one defined by VERB Enumerate",
verb)
raise RuntimeError(
"Verb [%s] is unknown, shall be one defined by VERB Enumerate"
% verb)
# Format output encoding
result.encoding = 'utf-8'
# Debug information
if result.status_code == 400 or result.status_code >= 500:
self.logger.debug("Sending request:")
self.logger.debug(" %-6s: %s", str(verb)[5:], result.url)
self.logger.debug(" Status: %s", result.status_code)
self.logger.debug(" Data: %s", data)
except Exception as exception:
self.logger.error(
"ERROR OCCURRED DURING THE HTTP SEND ACTION (details below)")
self.logger.error(exception)
raise
finally:
# Close potential opened files
close_files(json_file)
# Error handling
if 500 <= result.status_code < 600:
self.logger.error('%s Server Error: %s', result.status_code,
result.reason)
raise ServerError('%s Server Error: %s %s %s' %
(result.status_code, verb, url, result.text))
return RestClientResponse(result)
import requests
from ikats.core.config.ConfigReader import ConfigReader
from ikats.core.resource.opentsdb.HttpClient import HttpClient
LOGGER = HttpClient.LOGGER
LOGGER.setLevel(logging.DEBUG)
FORMATTER = logging.Formatter('%(asctime)s:%(levelname)s:%(funcName)s:%(message)s')
# Create another handler that will redirect log entries to STDOUT
STREAM_HANDLER = logging.StreamHandler()
STREAM_HANDLER.setLevel(logging.DEBUG)
STREAM_HANDLER.setFormatter(FORMATTER)
LOGGER.addHandler(STREAM_HANDLER)
# Configuration file
CONFIG_READER = ConfigReader()
def delete_ts(metric, **tags):
"""
Delete the TS from OpenTSDB manually
"""
tag_string = ','.join(['%s=%s' % (k, v) for k, v in tags.items()])
# Address of the real server to use for tests
test_opentsdb_host = CONFIG_READER.get('cluster', 'opentsdb.write.ip')
test_opentsdb_port = int(CONFIG_READER.get('cluster', 'opentsdb.write.port'))
direct_root_url = 'http://%s:%s//api/query?start=0&m=sum:%s{%s}&ms=true&delete=true' % (
test_opentsdb_host, test_opentsdb_port, metric, tag_string)