def extract_data_from(self, barray):
try:
self.data = qpack.unpackb(
barray[self.__class__.struct_datapackage.size:self.length],
decode='utf-8')
finally:
del barray[:self.length]
async def _read_message(self, header):
packet_type, packet_id, data = await read_packet(self._sock, header)
if len(data):
data = qpack.unpackb(data, decode='utf-8')
if packet_type == 0:
print("Connection lost, trying to reconnect")
try:
await self.setup(self._cbs)
except Exception as e:
print(e)
await asyncio.sleep(5)
elif packet_type == HANDSHAKE_OK:
print(f'Hands shaked with hub')
elif packet_type == HANDSHAKE_FAIL:
print(f'Hub does not want to shake hands')
elif packet_type == HEARTBEAT:
print(f'Heartbeat back from hub')
elif packet_type == RESPONSE_OK:
print(f'Hub received update correctly')
elif packet_type == UNKNOWN_CLIENT:
print(f'Hub does not recognize us')
await self._handshake()
else:
if packet_type in self._cbs.keys():
await self._cbs.get(packet_type)(data)
else:
print(f'Message type not implemented: {packet_type}')
def test_decode(self):
if not PYTHON3:
return
bindata = pickle.dumps({})
data = ['normal', bindata]
packed = qpack.packb(data)
s, b = qpack.unpackb(packed)
self.assertEqual(type(s).__name__, 'bytes')
self.assertEqual(type(b).__name__, 'bytes')
with self.assertRaises(ValueError):
s, b = qpack.unpackb(packed, decode='utf8')
s, b = qpack.unpackb(packed, decode='utf8', ignore_decode_errors=True)
self.assertEqual(type(s).__name__, 'str')
self.assertEqual(type(b).__name__, 'bytes')
async def _handle_client_connection(self, reader, writer):
connected = True
saved_client_id = None
while connected and self._server_running:
packet_type, packet_id, data = await read_packet(reader)
if data is False:
connected = False
continue
if len(data):
data = qpack.unpackb(data, decode='utf-8')
addr = writer.get_extra_info('peername')
logging.debug("Received %r from %r" % (packet_id, addr))
if packet_id == 0:
connected = False
if packet_type == HANDSHAKE:
saved_client_id, connected = await self._handle_handshake(
writer, packet_id, data)
elif packet_type == HEARTBEAT:
await self._handle_heartbeat(writer, packet_id, data)
elif packet_type == CLIENT_SHUTDOWN:
await self._handle_client_shutdown(saved_client_id)
connected = False
else:
if packet_type in self._cbs:
await self._cbs.get(packet_type)(writer, packet_type,
packet_id, data,
saved_client_id)
else:
logging.debug(
f'Package type {packet_type} not implemented')
await writer.drain()
logging.info(f'Closing socket with client {saved_client_id}')
await ClientManager.assert_if_client_is_offline(saved_client_id)
writer.close()
class DataPackage(object):
__slots__ = ('pid', 'length', 'tipe', 'checkbit', 'data')
struct_datapackage = struct.Struct('<IHBB')
_MAP = (
lambda data: None,
lambda data: qpack.unpackb(data, decode='utf-8'),
lambda data: data,
)
def __init__(self, barray):
self.length, self.pid, self.tipe, self.checkbit = \
self.__class__.struct_datapackage.unpack_from(barray, offset=0)
self.length += self.__class__.struct_datapackage.size
self.data = None
def extract_data_from(self, barray):
try:
self.data = self.__class__._MAP[protomap.MAP_RES_DTYPE[self.tipe]](
barray[self.__class__.struct_datapackage.size:self.length])
finally:
del barray[:self.length]
async def run(self):
await self.client0.connect()
x = requests.get(
f'http://localhost:9020/get-version', auth=('sa', 'siri'))
self.assertEqual(x.status_code, 200)
v = x.json()
self.assertTrue(isinstance(v, list))
self.assertTrue(isinstance(v[0], str))
x = requests.post(
f'http://localhost:9020/insert/dbtest',
auth=('iris', 'siri'),
headers={'Content-Type': 'application/json'})
self.assertEqual(x.status_code, 400)
series_float = gen_points(
tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
series_int = gen_points(
tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
data = {
'my_float': series_float,
'my_int': series_int
}
x = requests.post(
f'http://localhost:9020/insert/dbtest',
data=json.dumps(data),
auth=('iris', 'siri'),
headers={'Content-Type': 'application/json'}
)
self.assertEqual(x.status_code, 200)
self.assertDictEqual(x.json(), {
'success_msg': 'Successfully inserted 20000 point(s).'})
data = {
'dbname': 'dbtest',
'host': 'localhost',
'port': 9000,
'username': '******',
'password': '******'
}
x = requests.post(
f'http://localhost:9021/new-pool',
data=json.dumps(data),
auth=('sa', 'siri'),
headers={'Content-Type': 'application/json'})
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), 'OK')
self.db.servers.append(self.server1)
await self.assertIsRunning(self.db, self.client0, timeout=30)
data = {'data': [[1579521271, 10], [1579521573, 20]]}
x = requests.post(
f'http://localhost:9020/insert/dbtest',
json=data,
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertDictEqual(x.json(), {
'success_msg': 'Successfully inserted 2 point(s).'})
x = requests.post(
f'http://localhost:9020/query/dbtest',
json={'q': 'select * from "data"'},
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), data)
x = requests.post(
f'http://localhost:9020/query/dbtest',
json={'q': 'select * from "data"', 't': 'ms'},
auth=('iris', 'siri'))
data = {
'data': [[p[0] * 1000, p[1]] for p in data['data']]
}
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), data)
x = requests.post(
f'http://localhost:9020/query/dbtest',
data=qpack.packb({
'q': 'select sum(1579600000) from "data"',
't': 'ms'}),
headers={'Content-Type': 'application/qpack'},
auth=('iris', 'siri'))
self.assertEqual(x.status_code, 200)
self.assertEqual(
qpack.unpackb(x.content, decode='utf8'),
{'data': [[1579600000000, 30]]})
x = requests.post(
f'http://localhost:9021/new-account',
json={'account': 't', 'password': ''},
auth=('sa', 'siri'))
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg':
'service account name should have at least 2 characters'})
x = requests.post(
f'http://localhost:9021/new-account',
json={'account': 'tt', 'password': '******'},
auth=('sa', 'siri'))
self.assertEqual(x.status_code, 200)
data = {
'dbname': 'dbtest',
'host': 'localhost',
'port': 1234,
'pool': 0,
'username': '******',
'password': '******'
}
auth = ('tt', 'pass')
x = requests.post(
f'http://localhost:9021/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg': "database name already exists: 'dbtest'"})
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 401)
auth = ('sa', 'siri')
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 400)
self.assertEqual(x.json(), {
'error_msg':
"connecting to server 'localhost:1234' failed with error: "
"connection refused"})
data['port'] = 9000
x = requests.post(
f'http://localhost:9022/new-replica', json=data, auth=auth)
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), 'OK')
self.db.servers.append(self.server2)
await self.assertIsRunning(self.db, self.client0, timeout=30)
x = requests.get(
f'http://localhost:9022/get-databases', auth=auth)
self.assertEqual(x.status_code, 200)
self.assertEqual(x.json(), ['dbtest'])
self.client0.close()
class Handlers:
_MAP_ERORR_STATUS = {
InsertError: 500,
QueryError: 500,
ServerError: 503,
PoolError: 503,
UserAuthError: 401,
AuthenticationError: 422}
_SECRET_RX = re.compile('^Secret ([^\s]+)$', re.IGNORECASE)
_TOKEN_RX = re.compile('^Token ([^\s]+)$', re.IGNORECASE)
_REFRESH_RX = re.compile('^Refresh ([^\s]+)$', re.IGNORECASE)
_QUERY_MAP = {
_MSGPACK: lambda content:
msgpack.unpackb(content, use_list=False, encoding='utf-8'),
_CSV: lambda content:
csvhandler.loads(content.decode('utf-8'), is_query=True),
_JSON: lambda content: json.loads(content.decode('utf-8')),
_QPACK: lambda content:
qpack.unpackb(content, decode='utf-8')
}
_INSERT_MAP = {
_MSGPACK: lambda content:
msgpack.unpackb(content, use_list=False, encoding='utf-8'),
_CSV: csvhandler.loads,
_JSON: lambda content: json.loads(content.decode('utf-8')),
_QPACK: lambda content:
qpack.unpackb(content, decode='utf-8')
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.router.add_route('GET', '/db-info', self.handle_db_info)
self.router.add_route('POST', '/insert', self.handle_insert)
self.router.add_route('POST', '/query', self.handle_query)
if self.config.getboolean('Configuration', 'enable_web'):
logging.info('Enable Web Server routes')
# Read static and template paths
static_path = os.path.join(utils.get_path(), 'static')
templates_path = os.path.join(utils.get_path(), 'templates')
# Setup static route
static = self.router.add_static('/static/', static_path)
self.router.add_route('GET', '/', self.handle_main)
self.router.add_route(
'POST',
'/auth/secret',
self.handle_auth_secret)
self.router.add_route(
'POST',
'/auth/login',
self.handle_auth_login)
self.router.add_route(
'GET',
'/auth/fetch',
self.handle_auth_fetch)
self.router.add_route(
'GET',
'/auth/logoff',
self.handle_auth_logoff)
self.router.add_route(
'GET',
'/favicon.ico',
static_factory(static, 'favicon.ico'))
# Setup templates
setup_template_loader(templates_path)
if self.auth is not None:
logging.info('Enable Authentication routes')
self.router.add_route('POST', '/get-token', self.handle_get_token)
self.router.add_route(
'POST',
'/refresh-token',
self.handle_refresh_token)
async def handle_auth_secret(self, request):
secret = (await request.json())['secret']
self.auth.validate_secret(secret)
session = await get_session(request)
session['user'] = self.config.get('Database', 'user')
return self._response_json({'user': session['user']})
async def _save_session(self, request, user):
session = await get_session(request)
session['user'] = user
return {'user': user}
async def handle_auth_login(self, request):
login = await request.json()
if login['username'] == self.config.get('Database', 'user'):
if login['password'] != self.config.get('Database', 'password'):
resp = AuthenticationError('Username or password incorrect')
else:
resp = await self._save_session(
request,
self.config.get('Database', 'user'))
elif self.config.getboolean('Session', 'enable_multi_user'):
if login['username'] not in self.siri_connections:
siri = SiriDBClient(
username=login['username'],
password=login['password'],
dbname=self.config.get('Database', 'dbname'),
hostlist=self.config.hostlist,
keepalive=True)
result = await siri.connect()
if any([not isinstance(r, Exception) for r in result]):
self.siri_connections[login['username']] = \
(siri, login['password'])
resp = await self._save_session(request, login['username'])
else:
resp = result[0]
siri.close()
else:
_, password = self.siri_connections[login['username']]
if login['password'] != password:
resp = \
AuthenticationError('Username or password incorrect')
else:
resp = await self._save_session(request, login['username'])
else:
resp = AuthenticationError('Multiple user login is not allowed')
return self._response_json(resp)
async def handle_auth_fetch(self, request):
if self.auth is None:
return self._response_json({
'user': self.config.get('Database', 'user'),
'authRequired': False
})
session = await get_session(request)
return self._response_json({
'user': session.get('user'),
'authRequired': True
})
async def handle_auth_logoff(self, request):
session = await get_session(request)
session.clear()
return self._response_json({'user': session.get('user')})
@template('waiting.html')
async def handle_waiting(self, request):
return {'debug': self.debug_mode}
@checksiri
@template('main.html')
async def handle_main(self, request):
return {'debug': self.debug_mode}
async def handle_db_info(self, request):
return self._response_json(self.db)
def _response_text(self, text, status=200):
if isinstance(text, Exception):
exc = text
text = str(exc)
status = self._MAP_ERORR_STATUS.get(exc.__class__, 500)
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=text.encode('utf-8'),
charset='UTF-8',
content_type='text/plain',
status=status)
@pack_exception
def _response_csv(self, data, status=200):
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=csvhandler.dumps(data).encode('utf-8'),
charset='UTF-8',
content_type='application/csv',
status=status)
@pack_exception
def _response_json(self, data, status=200):
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=json.dumps(data).encode('utf-8'),
charset='UTF-8',
content_type='application/json',
status=status)
@pack_exception
def _response_msgpack(self, data, status=200):
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=msgpack.packb(data),
charset='UTF-8',
content_type='application/x-msgpack',
status=status)
@pack_exception
def _response_qpack(self, data, status=200):
return aiohttp.web.Response(
headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'},
body=qpack.packb(data),
charset='UTF-8',
content_type='application/x-qpack',
status=status)
@authentication
async def handle_insert(self, request, siri):
content = await request.read()
ct = _UNSUPPORTED
try:
ct = self._get_content_type(request)
data = self._INSERT_MAP[ct](content)
except Exception as e:
resp = InsertError(
'Error while reading data: {}'.format(str(e)))
else:
try:
resp = await siri.insert(data)
except Exception as e:
logging.error(e)
resp = e
finally:
return self._RESPONSE_MAP[ct](self, resp)
@staticmethod
def _get_content_type(request):
ct = request.content_type.lower().split(';')[0]
if ct.endswith('x-msgpack'):
return _MSGPACK
if ct.endswith('json'):
return _JSON
if ct.endswith('x-qpack'):
return _QPACK
if ct.endswith('csv'):
return _CSV
raise TypeError('Unsupported content type: {}'.format(ct))
@authentication
async def handle_query(self, request, siri):
content = await request.read()
ct = _UNSUPPORTED
try:
ct = self._get_content_type(request)
query = self._QUERY_MAP[ct](content)
if isinstance(query, dict):
query = query['query']
except Exception as e:
logging.error(e)
resp = QueryError(
'Error while reading query: {}'.format(str(e)))
else:
logging.debug('Process query: {}'.format(query))
try:
resp = await siri.query(query)
except Exception as e:
logging.error(e)
resp = e
finally:
return self._RESPONSE_MAP[ct](self, resp)
async def handle_get_token(self, request):
ct = _UNSUPPORTED
try:
ct = self._get_content_type(request)
authorization = \
self._SECRET_RX.match(request.headers['Authorization'])
if not authorization:
raise ValueError('Missing "Secret" in headers')
secret = authorization.group(1)
except Exception as e:
logging.error(e)
resp = e
else:
try:
resp = self.auth.get_token(secret)
except Exception as e:
logging.error(e)
resp = e
finally:
return self._RESPONSE_MAP[ct](self, resp)
async def handle_refresh_token(self, request):
ct = _UNSUPPORTED
try:
authorization = \
self._REFRESH_RX.match(request.headers['Authorization'])
if not authorization:
raise ValueError('Missing "Refresh" in headers')
refresh = authorization.group(1)
ct = self._get_content_type(request)
except Exception as e:
resp = e
else:
try:
resp = self.auth.refresh_token(refresh)
except Exception as e:
resp = e
finally:
return self._RESPONSE_MAP[ct](self, resp)
_RESPONSE_MAP = {
_MSGPACK: _response_msgpack,
_CSV: _response_csv,
_QPACK: _response_qpack,
_JSON: _response_json,
_UNSUPPORTED: _response_text
}
async def query_qpack(auth, q):
data = {'query': q}
headers = await auth.get_header(content_type='application/x-qpack')
res, status = await _query(auth, qpack.packb(data), headers)
return qpack.unpackb(res, decode='utf-8'), status
return b''.join(container)
def unpackb(qp, decode=None):
'''De-serialize QPack to Python. (Pure Python implementation)'''
return _unpack(qp, 0, len(qp), decode=decode)[1]
if __name__ == '__main__':
import math
packed = b'\xfd\x82CC\x82AN\x82VE\x82Jl\x82LE\x82S4\x82Gy\x82x2\x82xj\x828B\x82Ux\x82sw\x86secret\xf5\x87session\xf4\x84user\x84iris\x87created\xea\xaf\x8f\x99X\xff'
unpacked = unpackb(packed, decode='utf-8' if PYTHON3 else None)
print(unpacked)
import qpack
qpack.unpackb(packed)
packed = qpack.packb(unpacked)
print(packed)
t = {
's': 4,
't': {
'a': 1,
'U': 2,
'x': 3,
'L': 4,
'V': 5,
'G': 6,
'w': 7
},
'u': 5,
'v': 6,
def decrypt(self, enc):
enc = base64.b64decode(enc)
d = qpack.unpackb(enc, decode='utf-8')
return d['secret']
def create_and_register_server(dbname,
dbpath,
pool,
props,
cfg,
new_pool,
allow_retry=True):
def rollback(*args):
logging.warning('Roll-back create database...')
shutil.rmtree(dbpath)
quit_manage(*args)
address = settings.server_address
port = settings.listen_backend_port
_uuid = uuid.uuid1()
create_database(dbname=dbname,
dbpath=dbpath,
time_precision=props['time_precision'],
duration_log=props['duration_log'],
duration_num=props['duration_num'],
timezone=props['timezone'],
drop_threshold=props['drop_threshold'],
config=cfg,
_uuid=_uuid,
_pool=pool)
logging.info('Added database {!r}'.format(props['dbname']))
for fn in ('servers.dat', 'users.dat', 'groups.dat'):
try:
content = siri._get_file(fn)
except ServerError as e:
rollback(1, e)
with open(os.path.join(dbpath, fn), 'wb') as f:
f.write(content)
if new_pool:
with open(os.path.join(dbpath, '.reindex'), 'wb') as f:
pass
with open(os.path.join(dbpath, 'servers.dat'), 'rb') as f:
qp_servers = f.read()
servers_obj = qpack.unpackb(qp_servers)
server = [_uuid.bytes, bytes(address, 'utf-8'), port, pool]
servers_obj.append(server)
with open(os.path.join(dbpath, 'servers.dat'), 'wb') as f:
qp_servers = qpack.packb(servers_obj)
f.write(qp_servers)
result = siri.query('list servers name, status')
expected = 'running'
for srv in result['servers']:
if srv[1] != expected:
rollback(
1, 'All servers must have status {!r} '
'before we can continue. As least {!r} has status {!r}'.format(
expected, srv[0], srv[1]))
asyncio.get_event_loop().run_until_complete(
load_database(dbpath, settings.localhost, settings.listen_client_port))
time.sleep(1)
try:
check_loaded(dbname)
except Exception as e:
rollback(1, e)
else:
logging.info(
'Database loaded... now register the server'.format(dbname))
while True:
try:
siri._register_server(server)
except Exception as e:
if allow_retry:
print_error(e)
answer = menu(title='Do you want to retry the registration?',
options=Options([{
'option': 'r',
'text': 'Retry'
}, {
'option': 'q',
'text': 'Quit'
}]),
default='r')
if answer == 'r':
continue
rollback(0, None)
else:
rollback(1, e)
break
quit_manage(0, 'Finished joining database {!r}...'.format(dbname))
