def test_count(http_client2: TSDBClient):
http_client2.is_connected()
http_client2.TEST_METRIC = Counter('test.metric.counter')
metric = http_client2.TEST_METRIC.inc()
assert metric['value'] == 1
metric = http_client2.TEST_METRIC.inc()
assert metric['value'] == 2
def start(self):
self._client = TSDBClient(
static_tags=self.container.config[
self.OPEN_TSDB_CONFIG_KEY]['STATIC_TAGS'],
**self.container.config[self.OPEN_TSDB_CONFIG_KEY]['OPTIONS'])
for name, metric in self.metrics.items():
setattr(self._client, name, metric)
def test_gauge(http_client2: TSDBClient):
http_client2.is_connected()
http_client2.TEST_METRIC = Gauge('test.metric.gauge')
metric = http_client2.TEST_METRIC.inc()
assert metric['value'] == 1
metric = http_client2.TEST_METRIC.set(10)
assert metric['value'] == 10
metric = http_client2.TEST_METRIC.dec()
assert metric['value'] == 9
class SharedOpenTSDB(ProviderCollector, SharedExtension):
OPEN_TSDB_CONFIG_KEY = 'TSDB_CONFIG'
def __init__(self, **metrics):
super(SharedOpenTSDB, self).__init__()
self.metrics = metrics
self._client = None
def get_client(self) -> TSDBClient:
return self._client
def start(self):
self._client = TSDBClient(
static_tags=self.container.config[
self.OPEN_TSDB_CONFIG_KEY]['STATIC_TAGS'],
**self.container.config[self.OPEN_TSDB_CONFIG_KEY]['OPTIONS'])
for name, metric in self.metrics.items():
setattr(self._client, name, metric)
def stop(self):
self._client.close()
self._client.wait()
def kill(self):
self._client.close()
self._client.stop()
self._client = None
def connect(self):
"""
Override from BaseDB.
Create the connection to the openTSDB database with the current
configuration (hostname/port), then check if the connection has
been created without failure.
"""
# close connection if reload
if self.client is not None:
self.client.close()
self.client.wait()
self.client = TSDBClient(host=self.host, port=self.port)
if not self.client.is_connected() and not self.client.is_alive():
raise CantConnectToOpenTSDBException('connexion error')
class DatabaseAdaptor:
db = TSDBClient('127.0.0.1')
@staticmethod
def add(table, data):
if table == 'orderBook10':
DatabaseAdaptor.add_order_book(data)
if table == 'trade':
DatabaseAdaptor.add_trade(data)
@staticmethod
def add_order_book(data):
for record in data:
kwargs = {'symbol': record['symbol']}
count_bid = 1
for bid in record['bids']:
price = bid[0]
size = bid[1]
kwargs['bid' + str(count_bid) + 'price'] = price
kwargs['bid' + str(count_bid) + 'size'] = size
count_bid += 1
count_ask = 1
for ask in record['asks']:
price = ask[0]
size = ask[1]
kwargs['ask' + str(count_ask) + 'price'] = price
kwargs['ask' + str(count_ask) + 'size'] = size
count_ask += 1
dt = dateutil.parser.parse(record['timestamp'])
epoch = pytz.utc.localize(datetime.datetime.utcfromtimestamp(0))
DatabaseAdaptor.db.send("marketdata.orderbook",
(dt - epoch).total_seconds() * 1000.0,
**kwargs)
@staticmethod
def add_trade(data):
for record in data:
kwargs = {'symbol': record['symbol']}
kwargs['side'] = record['side']
kwargs['size'] = record['size']
kwargs['price'] = record['price']
kwargs['tickDirection'] = record['tickDirection']
kwargs['trdMatchID'] = record['trdMatchID']
kwargs['grossValue'] = record['grossValue']
kwargs['homeNotional'] = record['homeNotional']
kwargs['foreignNotional'] = record['foreignNotional']
dt = dateutil.parser.parse(record['timestamp'])
epoch = pytz.utc.localize(datetime.datetime.utcfromtimestamp(0))
DatabaseAdaptor.db.send("marketdata.trade",
(dt - epoch).total_seconds() * 1000.0,
**kwargs)
class OpenTSDB(BaseDB):
"""
OpenTSDB class herited from BaseDB
Allow to handle an OpenTSDB database to save PowerReport.
"""
def __init__(self, report_type: Type[Report], host: str, port,
metric_name: str):
"""
:param host: host of the OpenTSDB server
:param port: port of the OpenTSDB server
:param metric_name: mectric name to store
:param report_type: type of report handled by this database
"""
BaseDB.__init__(self, report_type)
self.host = host
self.port = port
self.metric_name = metric_name
self.client = None
def __iter__(self):
raise NotImplementedError()
def connect(self):
"""
Override from BaseDB.
Create the connection to the openTSDB database with the current
configuration (hostname/port), then check if the connection has
been created without failure.
"""
# close connection if reload
if self.client is not None:
self.client.close()
self.client.wait()
self.client = TSDBClient(host=self.host, port=self.port)
if not self.client.is_connected() and not self.client.is_alive():
raise CantConnectToOpenTSDBException('connexion error')
def save(self, report: PowerReport):
"""
Override from BaseDB
:param report: Report to save
"""
self.client.send(self.metric_name,
report.power,
timestamp=int(report.timestamp.timestamp()),
host=report.target)
def save_many(self, reports: List[Report]):
"""
Save a batch of data
:param reports: Batch of data.
"""
for report in reports:
self.save(report)
def __init__(self):
logger.info("init MarketDataCollector")
self.exchange = ExchangeInterface(settings.DRY_RUN)
self.instrument = self.exchange.get_instrument()
self.db = TSDBClient('http://127.0.0.1:4242/')
import tempfile
import random
import numpy as np
import smtplib
import logging
import sys, getopt
#logging.basicConfig(level=logging.DEBUG)
from ratelimit import limits
from multiprocessing import Process
from ratelimiter import RateLimiter
from tabulate import tabulate
from ratelimit import limits
import threading
from opentsdb import TSDBClient
tsdb = TSDBClient('internal-hugemetric-1216732828.us-east-1.elb.amazonaws.com',
static_tags={'node': 'OutBoundTestNode'})
MAILS_PER_THREAD = 5
MAX_RATE = 1
MAX_THREADS = 1
SMTP_HOST = 'smtp.flockmail.com'
TEST_CLUSTER = 'Outbound'
TIME_PERIOD = 1
LOCAL = False
PRESERVE_SESSIONS = False
#global lists
CONCURRENT_SMTPs = []
SMTP_SENDMAIL_TIME = []
SMTP_LOGIN_TIME = []
FAILED_MAILS = []
def telnet_client(tsdb_host, tsdb_port):
return TSDBClient(tsdb_host,
tsdb_port,
protocol=TSDBConnectProtocols.TELNET,
host_tag=True)
def http_client2(tsdb_host, tsdb_port):
return TSDBClient(tsdb_host, tsdb_port, host_tag=True)
def http_client(tsdb_host, tsdb_port):
return TSDBClient(tsdb_host, tsdb_port, host_tag=False)
import time
from opentsdb import TSDBClient
import potsdb
tsdb = TSDBClient('0.0.0.0')
'''
tsdb.send('metric.test4', 'fwe', t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 'fwe', t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 'fwe', t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 22, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 22, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 22, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 202, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 23, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 24, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 25, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 6, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 206, t1='v1')
time.sleep(2)
tsdb.send('metric.test4', 20, t1='v1')
def test_count_with_tag(http_client2: TSDBClient):
http_client2.is_connected()
http_client2.TEST_METRIC2 = Counter('test.metric2.counter', ['tag1'])
metric = http_client2.TEST_METRIC2.tags('tag1').inc()
assert 'tag1' in metric['tags']
