def _response_qpack(self, data, status=200):
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=qpack.packb(data),
charset='UTF-8',
content_type='application/x-qpack',
status=status)
def create_database(dbname,
dbpath,
time_precision='ms',
duration_log='1d',
duration_num='1w',
timezone=DEFAULT_TIMEZONE,
drop_threshold=DEFAULT_DROP_THRESHOLD,
buffer_size=DEFAULT_BUFFER_SIZE,
config={},
_uuid=None,
_pool=0):
'''
Note: duration_log and duration_num can both be integer or string.
get_duration() understands both.
'''
check_valid_dbname(dbname)
_config = {'buffer_path': dbpath}
_config.update(config)
if time_precision not in ['s', 'ms', 'us', 'ns']:
raise ValueError(
'time_precision must be either \'s\' (seconds), '
'\'ms\' (milliseconds), \'us\' (microseconds) '
'or \'ns\' (nanoseconds) but received {!r}'.format(time_precision))
time_precision = get_time_precision(time_precision)
duration_num = get_duration(time_precision, duration_num)
duration_log = get_duration(time_precision, duration_log)
if _uuid is None:
_uuid = uuid.uuid1()
with open(os.path.join(dbpath, 'database.conf'), 'w',
encoding='utf-8') as f:
f.write(
DEFAULT_CONFIG.format(comment_buffer_path='# '
if _config['buffer_path'] == dbpath else '',
**_config))
db_obj = [
1, # shema version
_uuid.bytes, # uuid
dbname, # dbname
time_precision, # time precision
buffer_size, # buffer size
duration_num, # duration num
duration_log, # duration log
timezone, # timezone
drop_threshold, # drop threshold
]
with open(os.path.join(dbpath, 'database.dat'), 'wb') as f:
f.write(qpack.packb(db_obj))
async def _handshake(self):
data = {
'client_id': str(self._id),
'client_type': self._client_type,
'token': self._token,
'version': self._client_version
}
if self._handshake_data_cb is not None:
handshake_data = await self._handshake_data_cb()
data = {**data, **handshake_data}
data = qpack.packb(data)
await self._send_message(len(data), HANDSHAKE, data)
self._last_heartbeat_send = datetime.datetime.now()
async def _send_worker_cancel_job(cls, worker_id: str, job_id: int):
worker = await ClientManager.get_worker_by_id(worker_id)
if worker is None:
return
try:
logging.error(f"Asking worker {worker_id} to cancel job {job_id}")
data = qpack.packb({'job_id': job_id})
header = create_header(len(data), WORKER_JOB_CANCEL, 0)
worker.writer.write(header + data)
except Exception as e:
logging.error(
f"Something went wrong when sending worker to cancel job")
logging.debug(f"Corresponding error: {e}, "
f'exception class: {e.__class__.__name__}')
def test_decode(self):
if not PYTHON3:
return
bindata = pickle.dumps({})
data = ['normal', bindata]
packed = qpack.packb(data)
s, b = qpack.unpackb(packed)
self.assertEqual(type(s).__name__, 'bytes')
self.assertEqual(type(b).__name__, 'bytes')
with self.assertRaises(ValueError):
s, b = qpack.unpackb(packed, decode='utf8')
s, b = qpack.unpackb(packed, decode='utf8', ignore_decode_errors=True)
self.assertEqual(type(s).__name__, 'str')
self.assertEqual(type(b).__name__, 'bytes')
async def _handle_handshake(self, writer, packet_id, data):
client_data = data
client_id = client_data.get('client_id')
client_token = client_data.get('token')
if self._token is not None and (client_token is None
or client_token != self._token):
response = create_header(0, HANDSHAKE_FAIL, packet_id)
writer.write(response)
return client_id, False
if 'client_type' not in client_data:
response = create_header(0, HANDSHAKE_FAIL, packet_id)
writer.write(response)
return client_id, False
if client_data.get('client_type') == 'listener':
await ClientManager.listener_connected(
writer.get_extra_info('peername'), writer, client_data)
logging.info(f'New listener with id: {client_id}')
elif client_data.get('client_type') == 'worker':
supported_modules = client_data.get('module')
if supported_modules is None:
logging.warning(f"New worker connected with id : {client_id}"
", but has no installed modules")
if version.parse(client_data.get('lib_version')) < version.parse(
ENODO_HUB_WORKER_MIN_VERSION):
logging.warning(
f"Worker with id : {client_id} tried to connect,"
"but has incompatible version")
response = create_header(0, HANDSHAKE_FAIL, packet_id)
writer.write(response)
return client_id, False
await ClientManager.worker_connected(
writer.get_extra_info('peername'), writer, client_data)
response = create_header(0, HANDSHAKE_OK, packet_id)
writer.write(response)
return client_id, True
if client_data.get('client_type') == 'listener':
update = qpack.packb(await
SeriesManager.get_listener_series_info())
header = create_header(len(update), UPDATE_SERIES, packet_id)
writer.write(header + update)
return client_id, True
async def _send_worker_job_request(cls, worker: WorkerClient,
job: EnodoJob):
try:
series = await SeriesManager.get_series_read_only(job.series_name)
job_data = EnodoJobRequestDataModel(
job_id=job.rid,
job_config=job.job_config,
series_name=job.series_name,
series_config=series.config,
series_state=series.state,
siridb_ts_units=ServerState.siridb_ts_unit)
data = qpack.packb(job_data.serialize())
header = create_header(len(data), WORKER_JOB, 0)
worker.writer.write(header + data)
except Exception as e:
logging.error(
f"Something went wrong when sending job request to worker")
import traceback
traceback.print_exc()
logging.debug(f"Corresponding error: {e}, "
f'exception class: {e.__class__.__name__}')
def _unpack(self, unpackb):
for inp, want in self.CASES:
out = unpackb(qpack.packb(inp), decode='utf8')
self.assertEqual(out, inp)
def unpackb(qp, decode=None):
'''De-serialize QPack to Python. (Pure Python implementation)'''
return _unpack(qp, 0, len(qp), decode=decode)[1]
if __name__ == '__main__':
import math
packed = b'\xfd\x82CC\x82AN\x82VE\x82Jl\x82LE\x82S4\x82Gy\x82x2\x82xj\x828B\x82Ux\x82sw\x86secret\xf5\x87session\xf4\x84user\x84iris\x87created\xea\xaf\x8f\x99X\xff'
unpacked = unpackb(packed, decode='utf-8' if PYTHON3 else None)
print(unpacked)
import qpack
qpack.unpackb(packed)
packed = qpack.packb(unpacked)
print(packed)
t = {
's': 4,
't': {
'a': 1,
'U': 2,
'x': 3,
'L': 4,
'V': 5,
'G': 6,
'w': 7
},
'u': 5,
'v': 6,
'w': 7,
def test_packb_unsupported(self):
with self.assertRaises(TypeError):
fallback.packb({'module': sys})
with self.assertRaises(TypeError):
qpack.packb({'module': sys})
async def send_message(self, body, message_type, use_qpack=True):
if use_qpack:
body = qpack.packb(body)
await self._send_message(len(body), message_type, body)
:copyright: 2016, Jeroen van der Heijden (Transceptor Technology)
'''
import asyncio
import logging
import qpack
from . import protomap
from .datapackage import DataPackage
from .exceptions import InsertError
from .exceptions import QueryError
from .exceptions import ServerError
from .exceptions import PoolError
from .exceptions import AuthenticationError
from .exceptions import UserAuthError
_MAP = (lambda data: b'', lambda data: qpack.packb(data), lambda data: data)
def _packdata(tipe, data=None):
assert tipe in protomap.MAP_REQ_DTYPE, \
'No data type found for message type: {}'.format(tipe)
return _MAP[protomap.MAP_REQ_DTYPE[tipe]](data)
class _SiriDBProtocol(asyncio.Protocol):
_connected = False
_MAP = {
# SiriDB Client protocol success response types
protomap.CPROTO_RES_QUERY:
async def query_qpack(auth, q):
data = {'query': q}
headers = await auth.get_header(content_type='application/x-qpack')
res, status = await _query(auth, qpack.packb(data), headers)
return qpack.unpackb(res, decode='utf-8'), status
async def _send_heartbeat(self):
print('Sending heartbeat to hub')
id_encoded = qpack.packb(self._id)
await self._send_message(len(id_encoded), HEARTBEAT, id_encoded)
self._last_heartbeat_send = datetime.datetime.now()
def create_and_register_server(dbname,
dbpath,
pool,
props,
cfg,
new_pool,
allow_retry=True):
def rollback(*args):
logging.warning('Roll-back create database...')
shutil.rmtree(dbpath)
quit_manage(*args)
address = settings.server_address
port = settings.listen_backend_port
_uuid = uuid.uuid1()
create_database(dbname=dbname,
dbpath=dbpath,
time_precision=props['time_precision'],
duration_log=props['duration_log'],
duration_num=props['duration_num'],
timezone=props['timezone'],
drop_threshold=props['drop_threshold'],
config=cfg,
_uuid=_uuid,
_pool=pool)
logging.info('Added database {!r}'.format(props['dbname']))
for fn in ('servers.dat', 'users.dat', 'groups.dat'):
try:
content = siri._get_file(fn)
except ServerError as e:
rollback(1, e)
with open(os.path.join(dbpath, fn), 'wb') as f:
f.write(content)
if new_pool:
with open(os.path.join(dbpath, '.reindex'), 'wb') as f:
pass
with open(os.path.join(dbpath, 'servers.dat'), 'rb') as f:
qp_servers = f.read()
servers_obj = qpack.unpackb(qp_servers)
server = [_uuid.bytes, bytes(address, 'utf-8'), port, pool]
servers_obj.append(server)
with open(os.path.join(dbpath, 'servers.dat'), 'wb') as f:
qp_servers = qpack.packb(servers_obj)
f.write(qp_servers)
result = siri.query('list servers name, status')
expected = 'running'
for srv in result['servers']:
if srv[1] != expected:
rollback(
1, 'All servers must have status {!r} '
'before we can continue. As least {!r} has status {!r}'.format(
expected, srv[0], srv[1]))
asyncio.get_event_loop().run_until_complete(
load_database(dbpath, settings.localhost, settings.listen_client_port))
time.sleep(1)
try:
check_loaded(dbname)
except Exception as e:
rollback(1, e)
else:
logging.info(
'Database loaded... now register the server'.format(dbname))
while True:
try:
siri._register_server(server)
except Exception as e:
if allow_retry:
print_error(e)
answer = menu(title='Do you want to retry the registration?',
options=Options([{
'option': 'r',
'text': 'Retry'
}, {
'option': 'q',
'text': 'Quit'
}]),
default='r')
if answer == 'r':
continue
rollback(0, None)
else:
rollback(1, e)
break
quit_manage(0, 'Finished joining database {!r}...'.format(dbname))
def update_listener(cls, listener: ListenerClient, data: Any):
update = qpack.packb(data)
series_update = create_header(len(update), UPDATE_SERIES, 1)
listener.writer.write(series_update + update)
async def update_listeners(cls, series):
for listener in ClientManager.listeners.values():
update = qpack.packb(series)
series_update = create_header(len(update), UPDATE_SERIES, 0)
listener.writer.write(series_update + update)
def encrypt(self, raw):
d = {_get_random(): _get_random() for _ in range(self._rand)}
d['secret'] = raw
secret = base64.b64encode(qpack.packb(d))
return secret
async def run(self):
await self.client0.connect()
x = requests.get(
f'http://localhost:9020/get-version', auth=('sa', 'siri'))
self.assertEqual(x.status_code, 200)
v = x.json()
self.assertTrue(isinstance(v, list))
self.assertTrue(isinstance(v[0], str))
x = requests.post(
f'http://localhost:9020/insert/dbtest',
auth=('iris', 'siri'),
headers={'Content-Type': 'application/json'})
self.assertEqual(x.status_code, 400)
series_float = gen_points(
tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
series_int = gen_points(
tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
data = {
'my_float': series_float,
'my_int': series_int
}
x = requests.post(
f'http://localhost:9020/insert/dbtest',
data=json.dumps(data),
auth=('iris', 'siri'),
headers={'Content-Type': 'application/json'}
)
self.assertEqual(x.status_code, 200)
self.assertDictEqual(x.json(), {
'success_msg': 'Successfully inserted 20000 point(s).'})
data = {
'dbname': 'dbtest',
'host': 'localhost',
'port': 9000,
'username': '******',
'password': '******'
}
x = requests.post(
f'http://localhost:9021/new-pool',
data=json.dumps(data),
auth=('sa', 'siri'),
headers={'Content-Type': 'application/json'})
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), 'OK')
self.db.servers.append(self.server1)
await self.assertIsRunning(self.db, self.client0, timeout=30)
data = {'data': [[1579521271, 10], [1579521573, 20]]}
x = requests.post(
f'http://localhost:9020/insert/dbtest',
json=data,
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertDictEqual(x.json(), {
'success_msg': 'Successfully inserted 2 point(s).'})
x = requests.post(
f'http://localhost:9020/query/dbtest',
json={'q': 'select * from "data"'},
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), data)
x = requests.post(
f'http://localhost:9020/query/dbtest',
json={'q': 'select * from "data"', 't': 'ms'},
auth=('iris', 'siri'))
data = {
'data': [[p[0] * 1000, p[1]] for p in data['data']]
}
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), data)
x = requests.post(
f'http://localhost:9020/query/dbtest',
data=qpack.packb({
'q': 'select sum(1579600000) from "data"',
't': 'ms'}),
headers={'Content-Type': 'application/qpack'},
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertEqual(
qpack.unpackb(x.content, decode='utf8'),
{'data': [[1579600000000, 30]]})
x = requests.post(
f'http://localhost:9021/new-account',
json={'account': 't', 'password': ''},
auth=('sa', 'siri'))
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg':
'service account name should have at least 2 characters'})
x = requests.post(
f'http://localhost:9021/new-account',
json={'account': 'tt', 'password': '******'},
auth=('sa', 'siri'))
self.assertEqual(x.status_code, 200)
data = {
'dbname': 'dbtest',
'host': 'localhost',
'port': 1234,
'pool': 0,
'username': '******',
'password': '******'
}
auth = ('tt', 'pass')
x = requests.post(
f'http://localhost:9021/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg': "database name already exists: 'dbtest'"})
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 401)
auth = ('sa', 'siri')
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg':
"connecting to server 'localhost:1234' failed with error: "
"connection refused"})
data['port'] = 9000
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), 'OK')
self.db.servers.append(self.server2)
await self.assertIsRunning(self.db, self.client0, timeout=30)
x = requests.get(
f'http://localhost:9022/get-databases', auth=auth)
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), ['dbtest'])
self.client0.close()
import asyncio
import logging
import qpack
from . import protomap
from .datapackage import DataPackage
from .exceptions import InsertError
from .exceptions import QueryError
from .exceptions import ServerError
from .exceptions import PoolError
from .exceptions import AuthenticationError
from .exceptions import UserAuthError
_MAP = (
lambda data: b'',
lambda data: qpack.packb(data),
lambda data: data
)
def _packdata(tipe, data=None):
assert tipe in protomap.MAP_REQ_DTYPE, \
'No data type found for message type: {}'.format(tipe)
return _MAP[protomap.MAP_REQ_DTYPE[tipe]](data)
class _SiriDBProtocol(asyncio.Protocol):
_connected = False
_MAP = {
