def extract_data_from(self, barray): try: self.data = qpack.unpackb( barray[self.__class__.struct_datapackage.size:self.length], decode='utf-8') finally: del barray[:self.length]
async def _read_message(self, header): packet_type, packet_id, data = await read_packet(self._sock, header) if len(data): data = qpack.unpackb(data, decode='utf-8') if packet_type == 0: print("Connection lost, trying to reconnect") try: await self.setup(self._cbs) except Exception as e: print(e) await asyncio.sleep(5) elif packet_type == HANDSHAKE_OK: print(f'Hands shaked with hub') elif packet_type == HANDSHAKE_FAIL: print(f'Hub does not want to shake hands') elif packet_type == HEARTBEAT: print(f'Heartbeat back from hub') elif packet_type == RESPONSE_OK: print(f'Hub received update correctly') elif packet_type == UNKNOWN_CLIENT: print(f'Hub does not recognize us') await self._handshake() else: if packet_type in self._cbs.keys(): await self._cbs.get(packet_type)(data) else: print(f'Message type not implemented: {packet_type}')
def test_decode(self): if not PYTHON3: return bindata = pickle.dumps({}) data = ['normal', bindata] packed = qpack.packb(data) s, b = qpack.unpackb(packed) self.assertEqual(type(s).__name__, 'bytes') self.assertEqual(type(b).__name__, 'bytes') with self.assertRaises(ValueError): s, b = qpack.unpackb(packed, decode='utf8') s, b = qpack.unpackb(packed, decode='utf8', ignore_decode_errors=True) self.assertEqual(type(s).__name__, 'str') self.assertEqual(type(b).__name__, 'bytes')
async def _handle_client_connection(self, reader, writer): connected = True saved_client_id = None while connected and self._server_running: packet_type, packet_id, data = await read_packet(reader) if data is False: connected = False continue if len(data): data = qpack.unpackb(data, decode='utf-8') addr = writer.get_extra_info('peername') logging.debug("Received %r from %r" % (packet_id, addr)) if packet_id == 0: connected = False if packet_type == HANDSHAKE: saved_client_id, connected = await self._handle_handshake( writer, packet_id, data) elif packet_type == HEARTBEAT: await self._handle_heartbeat(writer, packet_id, data) elif packet_type == CLIENT_SHUTDOWN: await self._handle_client_shutdown(saved_client_id) connected = False else: if packet_type in self._cbs: await self._cbs.get(packet_type)(writer, packet_type, packet_id, data, saved_client_id) else: logging.debug( f'Package type {packet_type} not implemented') await writer.drain() logging.info(f'Closing socket with client {saved_client_id}') await ClientManager.assert_if_client_is_offline(saved_client_id) writer.close()
class DataPackage(object): __slots__ = ('pid', 'length', 'tipe', 'checkbit', 'data') struct_datapackage = struct.Struct('<IHBB') _MAP = ( lambda data: None, lambda data: qpack.unpackb(data, decode='utf-8'), lambda data: data, ) def __init__(self, barray): self.length, self.pid, self.tipe, self.checkbit = \ self.__class__.struct_datapackage.unpack_from(barray, offset=0) self.length += self.__class__.struct_datapackage.size self.data = None def extract_data_from(self, barray): try: self.data = self.__class__._MAP[protomap.MAP_RES_DTYPE[self.tipe]]( barray[self.__class__.struct_datapackage.size:self.length]) finally: del barray[:self.length]
async def run(self): await self.client0.connect() x = requests.get( f'http://localhost:9020/get-version', auth=('sa', 'siri')) self.assertEqual(x.status_code, 200) v = x.json() self.assertTrue(isinstance(v, list)) self.assertTrue(isinstance(v[0], str)) x = requests.post( f'http://localhost:9020/insert/dbtest', auth=('iris', 'siri'), headers={'Content-Type': 'application/json'}) self.assertEqual(x.status_code, 400) series_float = gen_points( tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m') series_int = gen_points( tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m') data = { 'my_float': series_float, 'my_int': series_int } x = requests.post( f'http://localhost:9020/insert/dbtest', data=json.dumps(data), auth=('iris', 'siri'), headers={'Content-Type': 'application/json'} ) self.assertEqual(x.status_code, 200) self.assertDictEqual(x.json(), { 'success_msg': 'Successfully inserted 20000 point(s).'}) data = { 'dbname': 'dbtest', 'host': 'localhost', 'port': 9000, 'username': '******', 'password': '******' } x = requests.post( f'http://localhost:9021/new-pool', data=json.dumps(data), auth=('sa', 'siri'), headers={'Content-Type': 'application/json'}) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), 'OK') self.db.servers.append(self.server1) await self.assertIsRunning(self.db, self.client0, timeout=30) data = {'data': [[1579521271, 10], [1579521573, 20]]} x = requests.post( f'http://localhost:9020/insert/dbtest', json=data, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertDictEqual(x.json(), { 'success_msg': 'Successfully inserted 2 point(s).'}) x = requests.post( f'http://localhost:9020/query/dbtest', json={'q': 'select * from "data"'}, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), data) x = requests.post( f'http://localhost:9020/query/dbtest', json={'q': 'select * from "data"', 't': 'ms'}, auth=('iris', 'siri')) data = { 'data': [[p[0] * 1000, p[1]] for p in data['data']] } self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), data) x = requests.post( f'http://localhost:9020/query/dbtest', data=qpack.packb({ 'q': 'select sum(1579600000) from "data"', 't': 'ms'}), headers={'Content-Type': 'application/qpack'}, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertEqual( qpack.unpackb(x.content, decode='utf8'), {'data': [[1579600000000, 30]]}) x = requests.post( f'http://localhost:9021/new-account', json={'account': 't', 'password': ''}, auth=('sa', 'siri')) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': 'service account name should have at least 2 characters'}) x = requests.post( f'http://localhost:9021/new-account', json={'account': 'tt', 'password': '******'}, auth=('sa', 'siri')) self.assertEqual(x.status_code, 200) data = { 'dbname': 'dbtest', 'host': 'localhost', 'port': 1234, 'pool': 0, 'username': '******', 'password': '******' } auth = ('tt', 'pass') x = requests.post( f'http://localhost:9021/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': "database name already exists: 'dbtest'"}) x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 401) auth = ('sa', 'siri') x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': "connecting to server 'localhost:1234' failed with error: " "connection refused"}) data['port'] = 9000 x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), 'OK') self.db.servers.append(self.server2) await self.assertIsRunning(self.db, self.client0, timeout=30) x = requests.get( f'http://localhost:9022/get-databases', auth=auth) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), ['dbtest']) self.client0.close()
class Handlers: _MAP_ERORR_STATUS = { InsertError: 500, QueryError: 500, ServerError: 503, PoolError: 503, UserAuthError: 401, AuthenticationError: 422} _SECRET_RX = re.compile('^Secret ([^\s]+)$', re.IGNORECASE) _TOKEN_RX = re.compile('^Token ([^\s]+)$', re.IGNORECASE) _REFRESH_RX = re.compile('^Refresh ([^\s]+)$', re.IGNORECASE) _QUERY_MAP = { _MSGPACK: lambda content: msgpack.unpackb(content, use_list=False, encoding='utf-8'), _CSV: lambda content: csvhandler.loads(content.decode('utf-8'), is_query=True), _JSON: lambda content: json.loads(content.decode('utf-8')), _QPACK: lambda content: qpack.unpackb(content, decode='utf-8') } _INSERT_MAP = { _MSGPACK: lambda content: msgpack.unpackb(content, use_list=False, encoding='utf-8'), _CSV: csvhandler.loads, _JSON: lambda content: json.loads(content.decode('utf-8')), _QPACK: lambda content: qpack.unpackb(content, decode='utf-8') } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.router.add_route('GET', '/db-info', self.handle_db_info) self.router.add_route('POST', '/insert', self.handle_insert) self.router.add_route('POST', '/query', self.handle_query) if self.config.getboolean('Configuration', 'enable_web'): logging.info('Enable Web Server routes') # Read static and template paths static_path = os.path.join(utils.get_path(), 'static') templates_path = os.path.join(utils.get_path(), 'templates') # Setup static route static = self.router.add_static('/static/', static_path) self.router.add_route('GET', '/', self.handle_main) self.router.add_route( 'POST', '/auth/secret', self.handle_auth_secret) self.router.add_route( 'POST', '/auth/login', self.handle_auth_login) self.router.add_route( 'GET', '/auth/fetch', self.handle_auth_fetch) self.router.add_route( 'GET', '/auth/logoff', self.handle_auth_logoff) self.router.add_route( 'GET', '/favicon.ico', static_factory(static, 'favicon.ico')) # Setup templates setup_template_loader(templates_path) if self.auth is not None: logging.info('Enable Authentication routes') self.router.add_route('POST', '/get-token', self.handle_get_token) self.router.add_route( 'POST', '/refresh-token', self.handle_refresh_token) async def handle_auth_secret(self, request): secret = (await request.json())['secret'] self.auth.validate_secret(secret) session = await get_session(request) session['user'] = self.config.get('Database', 'user') return self._response_json({'user': session['user']}) async def _save_session(self, request, user): session = await get_session(request) session['user'] = user return {'user': user} async def handle_auth_login(self, request): login = await request.json() if login['username'] == self.config.get('Database', 'user'): if login['password'] != self.config.get('Database', 'password'): resp = AuthenticationError('Username or password incorrect') else: resp = await self._save_session( request, self.config.get('Database', 'user')) elif self.config.getboolean('Session', 'enable_multi_user'): if login['username'] not in self.siri_connections: siri = SiriDBClient( username=login['username'], password=login['password'], dbname=self.config.get('Database', 'dbname'), hostlist=self.config.hostlist, keepalive=True) result = await siri.connect() if any([not isinstance(r, Exception) for r in result]): self.siri_connections[login['username']] = \ (siri, login['password']) resp = await self._save_session(request, login['username']) else: resp = result[0] siri.close() else: _, password = self.siri_connections[login['username']] if login['password'] != password: resp = \ AuthenticationError('Username or password incorrect') else: resp = await self._save_session(request, login['username']) else: resp = AuthenticationError('Multiple user login is not allowed') return self._response_json(resp) async def handle_auth_fetch(self, request): if self.auth is None: return self._response_json({ 'user': self.config.get('Database', 'user'), 'authRequired': False }) session = await get_session(request) return self._response_json({ 'user': session.get('user'), 'authRequired': True }) async def handle_auth_logoff(self, request): session = await get_session(request) session.clear() return self._response_json({'user': session.get('user')}) @template('waiting.html') async def handle_waiting(self, request): return {'debug': self.debug_mode} @checksiri @template('main.html') async def handle_main(self, request): return {'debug': self.debug_mode} async def handle_db_info(self, request): return self._response_json(self.db) def _response_text(self, text, status=200): if isinstance(text, Exception): exc = text text = str(exc) status = self._MAP_ERORR_STATUS.get(exc.__class__, 500) return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=text.encode('utf-8'), charset='UTF-8', content_type='text/plain', status=status) @pack_exception def _response_csv(self, data, status=200): return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=csvhandler.dumps(data).encode('utf-8'), charset='UTF-8', content_type='application/csv', status=status) @pack_exception def _response_json(self, data, status=200): return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=json.dumps(data).encode('utf-8'), charset='UTF-8', content_type='application/json', status=status) @pack_exception def _response_msgpack(self, data, status=200): return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=msgpack.packb(data), charset='UTF-8', content_type='application/x-msgpack', status=status) @pack_exception def _response_qpack(self, data, status=200): return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=qpack.packb(data), charset='UTF-8', content_type='application/x-qpack', status=status) @authentication async def handle_insert(self, request, siri): content = await request.read() ct = _UNSUPPORTED try: ct = self._get_content_type(request) data = self._INSERT_MAP[ct](content) except Exception as e: resp = InsertError( 'Error while reading data: {}'.format(str(e))) else: try: resp = await siri.insert(data) except Exception as e: logging.error(e) resp = e finally: return self._RESPONSE_MAP[ct](self, resp) @staticmethod def _get_content_type(request): ct = request.content_type.lower().split(';')[0] if ct.endswith('x-msgpack'): return _MSGPACK if ct.endswith('json'): return _JSON if ct.endswith('x-qpack'): return _QPACK if ct.endswith('csv'): return _CSV raise TypeError('Unsupported content type: {}'.format(ct)) @authentication async def handle_query(self, request, siri): content = await request.read() ct = _UNSUPPORTED try: ct = self._get_content_type(request) query = self._QUERY_MAP[ct](content) if isinstance(query, dict): query = query['query'] except Exception as e: logging.error(e) resp = QueryError( 'Error while reading query: {}'.format(str(e))) else: logging.debug('Process query: {}'.format(query)) try: resp = await siri.query(query) except Exception as e: logging.error(e) resp = e finally: return self._RESPONSE_MAP[ct](self, resp) async def handle_get_token(self, request): ct = _UNSUPPORTED try: ct = self._get_content_type(request) authorization = \ self._SECRET_RX.match(request.headers['Authorization']) if not authorization: raise ValueError('Missing "Secret" in headers') secret = authorization.group(1) except Exception as e: logging.error(e) resp = e else: try: resp = self.auth.get_token(secret) except Exception as e: logging.error(e) resp = e finally: return self._RESPONSE_MAP[ct](self, resp) async def handle_refresh_token(self, request): ct = _UNSUPPORTED try: authorization = \ self._REFRESH_RX.match(request.headers['Authorization']) if not authorization: raise ValueError('Missing "Refresh" in headers') refresh = authorization.group(1) ct = self._get_content_type(request) except Exception as e: resp = e else: try: resp = self.auth.refresh_token(refresh) except Exception as e: resp = e finally: return self._RESPONSE_MAP[ct](self, resp) _RESPONSE_MAP = { _MSGPACK: _response_msgpack, _CSV: _response_csv, _QPACK: _response_qpack, _JSON: _response_json, _UNSUPPORTED: _response_text }
async def query_qpack(auth, q): data = {'query': q} headers = await auth.get_header(content_type='application/x-qpack') res, status = await _query(auth, qpack.packb(data), headers) return qpack.unpackb(res, decode='utf-8'), status
return b''.join(container) def unpackb(qp, decode=None): '''De-serialize QPack to Python. (Pure Python implementation)''' return _unpack(qp, 0, len(qp), decode=decode)[1] if __name__ == '__main__': import math packed = b'\xfd\x82CC\x82AN\x82VE\x82Jl\x82LE\x82S4\x82Gy\x82x2\x82xj\x828B\x82Ux\x82sw\x86secret\xf5\x87session\xf4\x84user\x84iris\x87created\xea\xaf\x8f\x99X\xff' unpacked = unpackb(packed, decode='utf-8' if PYTHON3 else None) print(unpacked) import qpack qpack.unpackb(packed) packed = qpack.packb(unpacked) print(packed) t = { 's': 4, 't': { 'a': 1, 'U': 2, 'x': 3, 'L': 4, 'V': 5, 'G': 6, 'w': 7 }, 'u': 5, 'v': 6,
def decrypt(self, enc): enc = base64.b64decode(enc) d = qpack.unpackb(enc, decode='utf-8') return d['secret']
def create_and_register_server(dbname, dbpath, pool, props, cfg, new_pool, allow_retry=True): def rollback(*args): logging.warning('Roll-back create database...') shutil.rmtree(dbpath) quit_manage(*args) address = settings.server_address port = settings.listen_backend_port _uuid = uuid.uuid1() create_database(dbname=dbname, dbpath=dbpath, time_precision=props['time_precision'], duration_log=props['duration_log'], duration_num=props['duration_num'], timezone=props['timezone'], drop_threshold=props['drop_threshold'], config=cfg, _uuid=_uuid, _pool=pool) logging.info('Added database {!r}'.format(props['dbname'])) for fn in ('servers.dat', 'users.dat', 'groups.dat'): try: content = siri._get_file(fn) except ServerError as e: rollback(1, e) with open(os.path.join(dbpath, fn), 'wb') as f: f.write(content) if new_pool: with open(os.path.join(dbpath, '.reindex'), 'wb') as f: pass with open(os.path.join(dbpath, 'servers.dat'), 'rb') as f: qp_servers = f.read() servers_obj = qpack.unpackb(qp_servers) server = [_uuid.bytes, bytes(address, 'utf-8'), port, pool] servers_obj.append(server) with open(os.path.join(dbpath, 'servers.dat'), 'wb') as f: qp_servers = qpack.packb(servers_obj) f.write(qp_servers) result = siri.query('list servers name, status') expected = 'running' for srv in result['servers']: if srv[1] != expected: rollback( 1, 'All servers must have status {!r} ' 'before we can continue. As least {!r} has status {!r}'.format( expected, srv[0], srv[1])) asyncio.get_event_loop().run_until_complete( load_database(dbpath, settings.localhost, settings.listen_client_port)) time.sleep(1) try: check_loaded(dbname) except Exception as e: rollback(1, e) else: logging.info( 'Database loaded... now register the server'.format(dbname)) while True: try: siri._register_server(server) except Exception as e: if allow_retry: print_error(e) answer = menu(title='Do you want to retry the registration?', options=Options([{ 'option': 'r', 'text': 'Retry' }, { 'option': 'q', 'text': 'Quit' }]), default='r') if answer == 'r': continue rollback(0, None) else: rollback(1, e) break quit_manage(0, 'Finished joining database {!r}...'.format(dbname))