def _response_qpack(self, data, status=200): return aiohttp.web.Response( headers={'ACCESS-CONTROL-ALLOW-ORIGIN': '*'}, body=qpack.packb(data), charset='UTF-8', content_type='application/x-qpack', status=status)
def create_database(dbname, dbpath, time_precision='ms', duration_log='1d', duration_num='1w', timezone=DEFAULT_TIMEZONE, drop_threshold=DEFAULT_DROP_THRESHOLD, buffer_size=DEFAULT_BUFFER_SIZE, config={}, _uuid=None, _pool=0): ''' Note: duration_log and duration_num can both be integer or string. get_duration() understands both. ''' check_valid_dbname(dbname) _config = {'buffer_path': dbpath} _config.update(config) if time_precision not in ['s', 'ms', 'us', 'ns']: raise ValueError( 'time_precision must be either \'s\' (seconds), ' '\'ms\' (milliseconds), \'us\' (microseconds) ' 'or \'ns\' (nanoseconds) but received {!r}'.format(time_precision)) time_precision = get_time_precision(time_precision) duration_num = get_duration(time_precision, duration_num) duration_log = get_duration(time_precision, duration_log) if _uuid is None: _uuid = uuid.uuid1() with open(os.path.join(dbpath, 'database.conf'), 'w', encoding='utf-8') as f: f.write( DEFAULT_CONFIG.format(comment_buffer_path='# ' if _config['buffer_path'] == dbpath else '', **_config)) db_obj = [ 1, # shema version _uuid.bytes, # uuid dbname, # dbname time_precision, # time precision buffer_size, # buffer size duration_num, # duration num duration_log, # duration log timezone, # timezone drop_threshold, # drop threshold ] with open(os.path.join(dbpath, 'database.dat'), 'wb') as f: f.write(qpack.packb(db_obj))
async def _handshake(self): data = { 'client_id': str(self._id), 'client_type': self._client_type, 'token': self._token, 'version': self._client_version } if self._handshake_data_cb is not None: handshake_data = await self._handshake_data_cb() data = {**data, **handshake_data} data = qpack.packb(data) await self._send_message(len(data), HANDSHAKE, data) self._last_heartbeat_send = datetime.datetime.now()
async def _send_worker_cancel_job(cls, worker_id: str, job_id: int): worker = await ClientManager.get_worker_by_id(worker_id) if worker is None: return try: logging.error(f"Asking worker {worker_id} to cancel job {job_id}") data = qpack.packb({'job_id': job_id}) header = create_header(len(data), WORKER_JOB_CANCEL, 0) worker.writer.write(header + data) except Exception as e: logging.error( f"Something went wrong when sending worker to cancel job") logging.debug(f"Corresponding error: {e}, " f'exception class: {e.__class__.__name__}')
def test_decode(self): if not PYTHON3: return bindata = pickle.dumps({}) data = ['normal', bindata] packed = qpack.packb(data) s, b = qpack.unpackb(packed) self.assertEqual(type(s).__name__, 'bytes') self.assertEqual(type(b).__name__, 'bytes') with self.assertRaises(ValueError): s, b = qpack.unpackb(packed, decode='utf8') s, b = qpack.unpackb(packed, decode='utf8', ignore_decode_errors=True) self.assertEqual(type(s).__name__, 'str') self.assertEqual(type(b).__name__, 'bytes')
async def _handle_handshake(self, writer, packet_id, data): client_data = data client_id = client_data.get('client_id') client_token = client_data.get('token') if self._token is not None and (client_token is None or client_token != self._token): response = create_header(0, HANDSHAKE_FAIL, packet_id) writer.write(response) return client_id, False if 'client_type' not in client_data: response = create_header(0, HANDSHAKE_FAIL, packet_id) writer.write(response) return client_id, False if client_data.get('client_type') == 'listener': await ClientManager.listener_connected( writer.get_extra_info('peername'), writer, client_data) logging.info(f'New listener with id: {client_id}') elif client_data.get('client_type') == 'worker': supported_modules = client_data.get('module') if supported_modules is None: logging.warning(f"New worker connected with id : {client_id}" ", but has no installed modules") if version.parse(client_data.get('lib_version')) < version.parse( ENODO_HUB_WORKER_MIN_VERSION): logging.warning( f"Worker with id : {client_id} tried to connect," "but has incompatible version") response = create_header(0, HANDSHAKE_FAIL, packet_id) writer.write(response) return client_id, False await ClientManager.worker_connected( writer.get_extra_info('peername'), writer, client_data) response = create_header(0, HANDSHAKE_OK, packet_id) writer.write(response) return client_id, True if client_data.get('client_type') == 'listener': update = qpack.packb(await SeriesManager.get_listener_series_info()) header = create_header(len(update), UPDATE_SERIES, packet_id) writer.write(header + update) return client_id, True
async def _send_worker_job_request(cls, worker: WorkerClient, job: EnodoJob): try: series = await SeriesManager.get_series_read_only(job.series_name) job_data = EnodoJobRequestDataModel( job_id=job.rid, job_config=job.job_config, series_name=job.series_name, series_config=series.config, series_state=series.state, siridb_ts_units=ServerState.siridb_ts_unit) data = qpack.packb(job_data.serialize()) header = create_header(len(data), WORKER_JOB, 0) worker.writer.write(header + data) except Exception as e: logging.error( f"Something went wrong when sending job request to worker") import traceback traceback.print_exc() logging.debug(f"Corresponding error: {e}, " f'exception class: {e.__class__.__name__}')
def _unpack(self, unpackb): for inp, want in self.CASES: out = unpackb(qpack.packb(inp), decode='utf8') self.assertEqual(out, inp)
def unpackb(qp, decode=None): '''De-serialize QPack to Python. (Pure Python implementation)''' return _unpack(qp, 0, len(qp), decode=decode)[1] if __name__ == '__main__': import math packed = b'\xfd\x82CC\x82AN\x82VE\x82Jl\x82LE\x82S4\x82Gy\x82x2\x82xj\x828B\x82Ux\x82sw\x86secret\xf5\x87session\xf4\x84user\x84iris\x87created\xea\xaf\x8f\x99X\xff' unpacked = unpackb(packed, decode='utf-8' if PYTHON3 else None) print(unpacked) import qpack qpack.unpackb(packed) packed = qpack.packb(unpacked) print(packed) t = { 's': 4, 't': { 'a': 1, 'U': 2, 'x': 3, 'L': 4, 'V': 5, 'G': 6, 'w': 7 }, 'u': 5, 'v': 6, 'w': 7,
def test_packb_unsupported(self): with self.assertRaises(TypeError): fallback.packb({'module': sys}) with self.assertRaises(TypeError): qpack.packb({'module': sys})
async def send_message(self, body, message_type, use_qpack=True): if use_qpack: body = qpack.packb(body) await self._send_message(len(body), message_type, body)
:copyright: 2016, Jeroen van der Heijden (Transceptor Technology) ''' import asyncio import logging import qpack from . import protomap from .datapackage import DataPackage from .exceptions import InsertError from .exceptions import QueryError from .exceptions import ServerError from .exceptions import PoolError from .exceptions import AuthenticationError from .exceptions import UserAuthError _MAP = (lambda data: b'', lambda data: qpack.packb(data), lambda data: data) def _packdata(tipe, data=None): assert tipe in protomap.MAP_REQ_DTYPE, \ 'No data type found for message type: {}'.format(tipe) return _MAP[protomap.MAP_REQ_DTYPE[tipe]](data) class _SiriDBProtocol(asyncio.Protocol): _connected = False _MAP = { # SiriDB Client protocol success response types protomap.CPROTO_RES_QUERY:
async def query_qpack(auth, q): data = {'query': q} headers = await auth.get_header(content_type='application/x-qpack') res, status = await _query(auth, qpack.packb(data), headers) return qpack.unpackb(res, decode='utf-8'), status
async def _send_heartbeat(self): print('Sending heartbeat to hub') id_encoded = qpack.packb(self._id) await self._send_message(len(id_encoded), HEARTBEAT, id_encoded) self._last_heartbeat_send = datetime.datetime.now()
def create_and_register_server(dbname, dbpath, pool, props, cfg, new_pool, allow_retry=True): def rollback(*args): logging.warning('Roll-back create database...') shutil.rmtree(dbpath) quit_manage(*args) address = settings.server_address port = settings.listen_backend_port _uuid = uuid.uuid1() create_database(dbname=dbname, dbpath=dbpath, time_precision=props['time_precision'], duration_log=props['duration_log'], duration_num=props['duration_num'], timezone=props['timezone'], drop_threshold=props['drop_threshold'], config=cfg, _uuid=_uuid, _pool=pool) logging.info('Added database {!r}'.format(props['dbname'])) for fn in ('servers.dat', 'users.dat', 'groups.dat'): try: content = siri._get_file(fn) except ServerError as e: rollback(1, e) with open(os.path.join(dbpath, fn), 'wb') as f: f.write(content) if new_pool: with open(os.path.join(dbpath, '.reindex'), 'wb') as f: pass with open(os.path.join(dbpath, 'servers.dat'), 'rb') as f: qp_servers = f.read() servers_obj = qpack.unpackb(qp_servers) server = [_uuid.bytes, bytes(address, 'utf-8'), port, pool] servers_obj.append(server) with open(os.path.join(dbpath, 'servers.dat'), 'wb') as f: qp_servers = qpack.packb(servers_obj) f.write(qp_servers) result = siri.query('list servers name, status') expected = 'running' for srv in result['servers']: if srv[1] != expected: rollback( 1, 'All servers must have status {!r} ' 'before we can continue. As least {!r} has status {!r}'.format( expected, srv[0], srv[1])) asyncio.get_event_loop().run_until_complete( load_database(dbpath, settings.localhost, settings.listen_client_port)) time.sleep(1) try: check_loaded(dbname) except Exception as e: rollback(1, e) else: logging.info( 'Database loaded... now register the server'.format(dbname)) while True: try: siri._register_server(server) except Exception as e: if allow_retry: print_error(e) answer = menu(title='Do you want to retry the registration?', options=Options([{ 'option': 'r', 'text': 'Retry' }, { 'option': 'q', 'text': 'Quit' }]), default='r') if answer == 'r': continue rollback(0, None) else: rollback(1, e) break quit_manage(0, 'Finished joining database {!r}...'.format(dbname))
def update_listener(cls, listener: ListenerClient, data: Any): update = qpack.packb(data) series_update = create_header(len(update), UPDATE_SERIES, 1) listener.writer.write(series_update + update)
async def update_listeners(cls, series): for listener in ClientManager.listeners.values(): update = qpack.packb(series) series_update = create_header(len(update), UPDATE_SERIES, 0) listener.writer.write(series_update + update)
def encrypt(self, raw): d = {_get_random(): _get_random() for _ in range(self._rand)} d['secret'] = raw secret = base64.b64encode(qpack.packb(d)) return secret
async def run(self): await self.client0.connect() x = requests.get( f'http://localhost:9020/get-version', auth=('sa', 'siri')) self.assertEqual(x.status_code, 200) v = x.json() self.assertTrue(isinstance(v, list)) self.assertTrue(isinstance(v[0], str)) x = requests.post( f'http://localhost:9020/insert/dbtest', auth=('iris', 'siri'), headers={'Content-Type': 'application/json'}) self.assertEqual(x.status_code, 400) series_float = gen_points( tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m') series_int = gen_points( tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m') data = { 'my_float': series_float, 'my_int': series_int } x = requests.post( f'http://localhost:9020/insert/dbtest', data=json.dumps(data), auth=('iris', 'siri'), headers={'Content-Type': 'application/json'} ) self.assertEqual(x.status_code, 200) self.assertDictEqual(x.json(), { 'success_msg': 'Successfully inserted 20000 point(s).'}) data = { 'dbname': 'dbtest', 'host': 'localhost', 'port': 9000, 'username': '******', 'password': '******' } x = requests.post( f'http://localhost:9021/new-pool', data=json.dumps(data), auth=('sa', 'siri'), headers={'Content-Type': 'application/json'}) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), 'OK') self.db.servers.append(self.server1) await self.assertIsRunning(self.db, self.client0, timeout=30) data = {'data': [[1579521271, 10], [1579521573, 20]]} x = requests.post( f'http://localhost:9020/insert/dbtest', json=data, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertDictEqual(x.json(), { 'success_msg': 'Successfully inserted 2 point(s).'}) x = requests.post( f'http://localhost:9020/query/dbtest', json={'q': 'select * from "data"'}, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), data) x = requests.post( f'http://localhost:9020/query/dbtest', json={'q': 'select * from "data"', 't': 'ms'}, auth=('iris', 'siri')) data = { 'data': [[p[0] * 1000, p[1]] for p in data['data']] } self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), data) x = requests.post( f'http://localhost:9020/query/dbtest', data=qpack.packb({ 'q': 'select sum(1579600000) from "data"', 't': 'ms'}), headers={'Content-Type': 'application/qpack'}, auth=('iris', 'siri')) self.assertEqual(x.status_code, 200) self.assertEqual( qpack.unpackb(x.content, decode='utf8'), {'data': [[1579600000000, 30]]}) x = requests.post( f'http://localhost:9021/new-account', json={'account': 't', 'password': ''}, auth=('sa', 'siri')) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': 'service account name should have at least 2 characters'}) x = requests.post( f'http://localhost:9021/new-account', json={'account': 'tt', 'password': '******'}, auth=('sa', 'siri')) self.assertEqual(x.status_code, 200) data = { 'dbname': 'dbtest', 'host': 'localhost', 'port': 1234, 'pool': 0, 'username': '******', 'password': '******' } auth = ('tt', 'pass') x = requests.post( f'http://localhost:9021/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': "database name already exists: 'dbtest'"}) x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 401) auth = ('sa', 'siri') x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 400) self.assertEqual(x.json(), { 'error_msg': "connecting to server 'localhost:1234' failed with error: " "connection refused"}) data['port'] = 9000 x = requests.post( f'http://localhost:9022/new-replica', json=data, auth=auth) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), 'OK') self.db.servers.append(self.server2) await self.assertIsRunning(self.db, self.client0, timeout=30) x = requests.get( f'http://localhost:9022/get-databases', auth=auth) self.assertEqual(x.status_code, 200) self.assertEqual(x.json(), ['dbtest']) self.client0.close()
import asyncio import logging import qpack from . import protomap from .datapackage import DataPackage from .exceptions import InsertError from .exceptions import QueryError from .exceptions import ServerError from .exceptions import PoolError from .exceptions import AuthenticationError from .exceptions import UserAuthError _MAP = ( lambda data: b'', lambda data: qpack.packb(data), lambda data: data ) def _packdata(tipe, data=None): assert tipe in protomap.MAP_REQ_DTYPE, \ 'No data type found for message type: {}'.format(tipe) return _MAP[protomap.MAP_REQ_DTYPE[tipe]](data) class _SiriDBProtocol(asyncio.Protocol): _connected = False _MAP = {