class WampMQTTServerFactory(Factory): log = make_logger() protocol = WampMQTTServerProtocol serializers = { 'json': JsonObjectSerializer(), 'msgpack': MsgPackObjectSerializer(), 'cbor': CBORObjectSerializer(), 'ubjson': UBJSONObjectSerializer(), } def __init__(self, router_session_factory, config, reactor): self._router_session_factory = router_session_factory self._router_factory = router_session_factory._routerFactory self._options = config.get('options', {}) self._realm = self._options.get('realm', None) self._reactor = reactor self._payload_mapping = StringTrie() for topic, pmap in self._options.get('payload_mapping', {}).items(): self._set_payload_format(topic, pmap) def buildProtocol(self, addr): protocol = self.protocol(self._reactor) protocol.factory = self return protocol def _get_payload_format(self, topic): """ Map a WAMP topic URI to MQTT payload format. :param topic: WAMP URI. :type topic: str :returns: Payload format metadata. :rtype: dict """ try: pmap = self._payload_mapping.longest_prefix_value(topic) except KeyError: return None else: return pmap def _set_payload_format(self, topic, pmap=None): if pmap is None: if topic in self._payload_mapping: del self._payload_mapping[topic] else: self._payload_mapping[topic] = pmap @inlineCallbacks def transform_wamp(self, topic, msg): # check for cached transformed payload cache_key = '_{}_{}'.format(self.__class__.__name__, id(self)) cached = msg._serialized.get(cache_key, None) if cached: payload_format, mapped_topic, payload = cached self.log.debug( 'using cached payload for {cache_key} in message {msg_id}!', msg_id=id(msg), cache_key=cache_key) else: # convert WAMP URI to MQTT topic mapped_topic = _wamp_topic_to_mqtt(topic) # for WAMP->MQTT, the payload mapping is determined from the # WAMP URI (not the transformed MQTT topic) payload_format = self._get_payload_format(topic) payload_format_type = payload_format['type'] if payload_format_type == 'passthrough': payload = msg.payload elif payload_format_type == 'native': serializer = payload_format.get('serializer', None) payload = self._transform_wamp_native(serializer, msg) elif payload_format_type == 'dynamic': encoder = payload_format.get('encoder', None) codec_realm = payload_format.get('realm', self._realm) payload = yield self._transform_wamp_dynamic( encoder, codec_realm, mapped_topic, topic, msg) else: raise Exception( 'payload format {} not implemented'.format(payload_format)) msg._serialized[cache_key] = (payload_format, mapped_topic, payload) self.log.debug( 'transform_wamp({topic}, {msg}) -> payload_format={payload_format}, mapped_topic={mapped_topic}, payload={payload}', topic=topic, msg=msg, payload_format=payload_format, mapped_topic=mapped_topic, payload=payload) returnValue((payload_format, mapped_topic, payload)) @inlineCallbacks def _transform_wamp_dynamic(self, encoder, codec_realm, mapped_topic, topic, msg): codec_session = self._router_factory.get(codec_realm)._realm.session payload = yield codec_session.call(encoder, mapped_topic, topic, msg.args, msg.kwargs) returnValue(payload) def _transform_wamp_native(self, serializer, msg): obj = {} for opt in [ 'args', 'kwargs', 'exclude', 'exclude_authid', 'exclude_authrole', 'eligible', 'eligible_authid', 'eligible_authrole' ]: attr = getattr(msg, opt, None) if attr is not None: obj[opt] = attr if serializer in self.serializers: payload = self.serializers[serializer].serialize(obj) else: raise Exception( 'MQTT native mode payload transform: invalid serializer {}'. format(serializer)) return payload @inlineCallbacks def transform_mqtt(self, topic, payload): # transform MQTT topic to WAMP URI mapped_topic = _mqtt_topicname_to_wamp(topic) # for MQTT->WAMP, the payload mapping is determined from the # transformed WAMP URI (not the original MQTT topic) payload_format = self._get_payload_format(mapped_topic) payload_format_type = payload_format['type'] if payload_format_type == 'passthrough': options = {'payload': payload, 'enc_algo': 'mqtt'} elif payload_format_type == 'native': serializer = payload_format.get('serializer', None) options = self._transform_mqtt_native(serializer, payload) elif payload_format_type == 'dynamic': decoder = payload_format.get('decoder', None) codec_realm = payload_format.get('realm', self._realm) options = yield self._transform_mqtt_dynamic( decoder, codec_realm, mapped_topic, topic, payload) else: raise Exception( 'payload format {} not implemented'.format(payload_format)) self.log.debug( 'transform_mqtt({topic}, {payload}) -> payload_format={payload_format}, mapped_topic={mapped_topic}, options={options}', topic=topic, payload=payload, payload_format=payload_format, mapped_topic=mapped_topic, options=options) returnValue((payload_format, mapped_topic, options)) @inlineCallbacks def _transform_mqtt_dynamic(self, decoder, codec_realm, mapped_topic, topic, payload): codec_session = self._router_factory.get(codec_realm)._realm.session options = yield codec_session.call(decoder, mapped_topic, topic, payload) returnValue(options) def _transform_mqtt_native(self, serializer, payload): """ Transform MQTT binary payload from a MQTT Publish to keyword dict suitable for the constructor of a WAMP Publish message, that is :class:`autobahn.wamp.message.Publish`. """ options = {} if serializer in self.serializers: if serializer == 'json': if not _validator.validate(payload)[0]: # invalid UTF-8: drop the event raise Exception( 'invalid UTF8 in JSON encoded MQTT payload') obj = self.serializers[serializer].unserialize(payload)[0] else: raise Exception( '"{}" serializer for encoded MQTT payload not implemented'. format(serializer)) if not isinstance(obj, dict): raise Exception( 'invalid type {} for "{}" encoded MQTT payload'.format( type(obj), serializer)) for opt in [ 'args', 'kwargs', 'exclude', 'exclude_authid', 'exclude_authrole', 'eligible', 'eligible_authid', 'eligible_authrole' ]: if opt in obj: options[opt] = obj[opt] return options
class CatalogResource(resource.Resource): """ Twisted Web resource for API FbsRepository Web service. This resource uses templates loaded into a Jinja2 environment to render HTML pages with data retrieved from an API FbsRepository archive file or on-chain address. """ log = make_logger() ser = JsonObjectSerializer() isLeaf = True def __init__(self, jinja_env: Environment, worker: Union[RouterController, ProxyController], config: Dict[str, Any], path: str): """ :param worker: The router worker controller within this Web service is started. :param config: The Web service configuration item. """ resource.Resource.__init__(self) # remember all ctor args self._jinja_env: Environment = jinja_env self._worker = worker self._config = config self._path = path # setup Werkzeug URL map adapter # https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.Map adapter_map = werkzeug.routing.Map() routes = { '/': 'wamp_catalog_home.html', 'table': 'wamp_catalog_table.html', 'struct': 'wamp_catalog_struct.html', 'enum': 'wamp_catalog_enum.html', 'service': 'wamp_catalog_service.html', } for rpath, route_template in routes.items(): # compute full absolute URL of route to be added - ending in Werkzeug/Routes URL pattern _rp = [] if path != '/': _rp.append(path) if rpath != '/': _rp.append(rpath) route_url = os.path.join('/', '/'.join(_rp)) route_endpoint = jinja_env.get_template(route_template) route_rule = Rule(route_url, methods=['GET'], endpoint=route_endpoint) adapter_map.add(route_rule) # https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.Map.bind self._map_adapter: MapAdapter = adapter_map.bind('localhost', '/') # FIXME self._repo: FbsRepository = FbsRepository('FIXME') self._repo.load(self._config['filename']) def render(self, request): # https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html#render # The encoded path of the request URI (_not_ (!) including query arguments), full_path = request.path.decode('utf-8') # HTTP request method http_method = request.method.decode() if http_method not in ['GET']: request.setResponseCode(511) return self._render_error( 'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format( full_path=full_path), request) # parse and decode any query parameters query_args = {} if request.args: for key, values in request.args.items(): key = key.decode() # we only process the first header value per key (!) value = values[0].decode() query_args[key] = value self.log.info('Parsed query parameters: {query_args}', query_args=query_args) # parse client announced accept-header client_accept = request.getAllHeaders().get(b'accept', None) if client_accept: client_accept = client_accept.decode() # flag indicating the client wants to get plain JSON results (not rendered HTML) # client_return_json = client_accept == 'application/json' # client cookie processing cookie = request.received_cookies.get(b'session_cookie') self.log.debug('Session Cookie is ({})'.format(cookie)) try: # werkzeug.routing.MapAdapter # https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.MapAdapter.match template, kwargs = self._map_adapter.match(full_path, method=http_method, query_args=query_args) if kwargs: if query_args: kwargs.update(query_args) else: kwargs = query_args kwargs['repo'] = self._repo kwargs['created'] = time_ns() self.log.info( 'CatalogResource request on path "{full_path}" mapped to template "{template}" ' 'using kwargs\n{kwargs}', full_path=full_path, template=template, kwargs=pformat(kwargs)) rendered = template.render(**kwargs).encode('utf8') self.log.info('successfully rendered HTML result: {rendered} bytes', rendered=len(rendered)) request.setResponseCode(200) return rendered except NotFound: self.log.warn('URL "{url}" not found (method={method})', url=full_path, method=http_method) request.setResponseCode(404) return self._render_error( 'Path "{full_path}" not found [werkzeug.routing.MapAdapter.match]'.format(full_path=full_path), request) except MethodNotAllowed: self.log.warn('method={method} not allowed on URL "{url}"', url=full_path, method=http_method) request.setResponseCode(511) return self._render_error( 'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format( full_path=full_path), request) except Exception as e: self.log.warn('error while processing method={method} on URL "{url}": {e}', url=full_path, method=http_method, e=e) request.setResponseCode(500) return self._render_error( 'Unknown error with path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format( full_path=full_path), request) def _render_error(self, message, request, client_return_json=False): """ Error renderer, display a basic error message to tell the user that there was a problem and roughly what the problem was. :param message: The current error message :param request: The original HTTP request :return: HTML formatted error string """ if client_return_json: return self.ser.serialize({'error': message}) else: return """ <html> <title>API Error</title> <body> <h3 style="color: #f00">Crossbar WAMP Application Page Error</h3> <pre>{}</pre> </body> </html> """.format(escape(message)).encode('utf8')
import binascii from autobahn.wamp.serializer import JsonObjectSerializer, MsgPackObjectSerializer ser = JsonObjectSerializer(batched = True) ser = MsgPackObjectSerializer(batched = True) o1 = [1, "hello", [1, 2, 3]] o2 = [3, {"a": 23, "b": 24}] d1 = ser.serialize(o1) + ser.serialize(o2) + ser.serialize(o1) m = ser.unserialize(d1) print m
class WapResource(resource.Resource): """ Twisted Web resource for WAMP Application Page web service. This resource uses templates loaded into a Jinja2 environment to render HTML pages with data retrieved from a WAMP procedure call, triggered from the original Web request. """ log = make_logger() ser = JsonObjectSerializer() isLeaf = True def __init__(self, worker, config, path): """ :param worker: The router worker controller within this Web service is started. :type worker: crossbar.worker.router.RouterController :param config: The Web service configuration item. :type config: dict """ resource.Resource.__init__(self) self._worker = worker self._config = config self._session_cache = {} self._realm_name = config.get('wamp', {}).get('realm', None) self._authrole = config.get('wamp', {}).get('authrole', 'anonymous') self._service_agent = worker.realm_by_name(self._realm_name).session # TODO: # We need to lookup the credentials for the current user based on the pre-established # HTTP session cookie, this will establish the 'authrole' the user is running as. # This 'authrole' can then be used to authorize the back-end topic call. # QUESTION: # Does the topic need the authid, if so, how do we pass it? # # This is our default (anonymous) session for unauthenticated users # router = worker._router_factory.get(self._realm_name) self._default_session = ApplicationSession( ComponentConfig(realm=self._realm_name, extra=None)) worker._router_session_factory.add(self._default_session, router, authrole=self._authrole) # Setup Jinja2 to point to our templates folder or a package resource # templates_config = config.get("templates") if type(templates_config) == str: # resolve specified template directory path relative to node directory templates_dir = os.path.abspath( os.path.join(self._worker.config.extra.cbdir, templates_config)) templates_source = 'directory' elif type(templates_config) == dict: # in case we got a dict, that must contain "package" and "resource" attributes if 'package' not in templates_config: raise ApplicationError( 'crossbar.error.invalid_configuration', 'missing attribute "resource" in WAP web service configuration' ) if 'resource' not in templates_config: raise ApplicationError( 'crossbar.error.invalid_configuration', 'missing attribute "resource" in WAP web service configuration' ) try: importlib.import_module(templates_config['package']) except ImportError as e: emsg = 'Could not import resource {} from package {}: {}'.format( templates_config['resource'], templates_config['package'], e) raise ApplicationError('crossbar.error.invalid_configuration', emsg) else: try: # resolve template directory from package resource templates_dir = os.path.abspath( pkg_resources.resource_filename( templates_config['package'], templates_config['resource'])) except Exception as e: emsg = 'Could not import resource {} from package {}: {}'.format( templates_config['resource'], templates_config['package'], e) raise ApplicationError( 'crossbar.error.invalid_configuration', emsg) templates_source = 'package' else: raise ApplicationError( 'crossbar.error.invalid_configuration', 'invalid type "{}" for attribute "templates" in WAP web service configuration' .format(type(templates_config))) if config.get('sandbox', True): # The sandboxed environment. It works like the regular environment but tells the compiler to # generate sandboxed code. # https://jinja.palletsprojects.com/en/2.11.x/sandbox/#jinja2.sandbox.SandboxedEnvironment env = SandboxedEnvironment(loader=FileSystemLoader(templates_dir), autoescape=True) else: env = Environment(loader=FileSystemLoader(templates_dir), autoescape=True) self.log.info( 'WapResource created (realm="{realm}", authrole="{authrole}", templates_dir="{templates_dir}", templates_source="{templates_source}", jinja2_env={jinja2_env})', realm=hlid(self._realm_name), authrole=hlid(self._authrole), templates_dir=hlid(templates_dir), templates_source=hlid(templates_source), jinja2_env=hltype(env.__class__)) # http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.Map map = Map() # Add all our routes into 'map', note each route endpoint is a tuple of the # topic to call, and the template to use when rendering the results. for route in config.get('routes', {}): # compute full absolute URL of route to be added - ending in Werkzeug/Routes URL pattern rpath = route['path'] _rp = [] if path != '/': _rp.append(path) if rpath != '/': _rp.append(rpath) route_url = os.path.join('/', '/'.join(_rp)) route_method = route.get('method', 'GET') assert route_method in [ 'GET', 'POST' ], 'invalid HTTP method "{}" for route on URL "{}"'.format( route_method, route_url) route_methods = [route_method] # note the WAMP procedure to call and the Jinja2 template to render as HTTP response route_endpoint = (route['call'], env.get_template(route['render'])) route_rule = Rule(route_url, methods=route_methods, endpoint=route_endpoint) map.add(route_rule) self.log.info( 'WapResource route added (url={route_url}, methods={route_methods}, endpoint={route_endpoint})', route_url=hlid(route_url), route_methods=hlid(route_methods), route_endpoint=route_endpoint) # http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter # http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter.match self._map_adapter = map.bind('/') def _after_call_success(self, result, request, client_return_json): """ When the WAMP call attached to the URL returns, render the WAMP result into a Jinja2 template and return HTML to client. Alternatively, return the call result as plain JSON. :param result: The dict returned from the WAMP procedure call. :param request: The HTTP request. :param client_return_json: Flag indicating to return plain JSON (no HTML rendering.) """ try: if client_return_json: rendered = self.ser.serialize(result) self.log.info( 'WapResource successfully serialized JSON result:\n{result}', result=pformat(result)) else: rendered = request.template.render(result).encode('utf8') self.log.info( 'WapResource successfully rendered HTML result: {rendered} bytes', rendered=len(rendered)) except Exception as e: self.log.failure() emsg = 'WapResource render error for WAMP result of type "{}": {}'.format( type(result), e) self.log.warn(emsg) request.setResponseCode(500) request.write(self._render_error(emsg, request, client_return_json)) else: request.write(rendered) request.finish() def _after_call_error(self, error, request, client_return_json): """ Deferred error, write out the error template and finish the request :param error: The current deferred error object :param request: The original HTTP request """ self.log.error('WapResource error: {error}', error=error) request.setResponseCode(500) request.write( self._render_error(error.value.error, request, client_return_json)) request.finish() def _render_error(self, message, request, client_return_json=False): """ Error renderer, display a basic error message to tell the user that there was a problem and roughly what the problem was. :param message: The current error message :param request: The original HTTP request :return: HTML formatted error string """ if client_return_json: return self.ser.serialize({'error': message}) else: return """ <html> <title>API Error</title> <body> <h3 style="color: #f00">Crossbar WAMP Application Page Error</h3> <pre>{}</pre> </body> </html> """.format(escape(message)).encode('utf8') def render(self, request): """ Initiate the rendering of a HTTP/GET request by calling a WAMP procedure, the resulting ``dict`` is rendered together with the specified Jinja2 template for this URL. :param request: The HTTP request. :returns: server.NOT_DONE_YET (special) """ # https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html#render # The encoded path of the request URI (_not_ (!) including query arguments), full_path = request.path.decode('utf-8') # HTTP request method (GET, POST, ..) http_method = request.method.decode() if http_method not in ['GET', 'POST']: request.setResponseCode(511) return self._render_error( 'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]' .format(full_path=full_path), request) # in case of HTTP/POST, read request body as one binary string if http_method == 'POST' and request.content: content_type = request.getAllHeaders().get( b'content-type', b'application/octet-stream').decode() # https://stackoverflow.com/a/11549600/884770 # http://marianoiglesias.com.ar/python/file-uploading-with-multi-part-encoding-using-twisted/ body_data = request.content.read() self.log.info('POST data len = {newdata_len}', newdata_len=len(body_data)) else: content_type = None body_data = None # parse and decode any query parameters query_args = {} if request.args: for key, values in request.args.items(): key = key.decode() # we only process the first header value per key (!) value = values[0].decode() query_args[key] = value self.log.info('Parsed query parameters: {query_args}', query_args=query_args) # parse client announced accept header client_accept = request.getAllHeaders().get(b'accept', None) if client_accept: client_accept = client_accept.decode() # flag indicating the client wants to get plain JSON results (not rendered HTML) client_return_json = client_accept == 'application/json' # client cookie processing cookie = request.received_cookies.get(b'session_cookie') self.log.debug('Session Cookie is ({})'.format(cookie)) if cookie: session = self._session_cache.get(cookie) if not session: # FIXME: lookup role for current session self.log.debug( 'Creating a new session for cookie ({})'.format(cookie)) authrole = 'anonymous' session = ApplicationSession( ComponentConfig(realm=self._realm_name, extra=None)) self._worker._router_session_factory.add(session, authrole=authrole) self._session_cache[cookie] = session else: self.log.debug( 'Using a cached session for ({})'.format(cookie)) else: self.log.debug( 'No session cookie, falling back on default session') session = self._default_session try: # werkzeug.routing.MapAdapter # http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter.match (procedure, request.template), kwargs = self._map_adapter.match( full_path, method=http_method, query_args=query_args) if kwargs and query_args: kwargs.update(query_args) else: kwargs = query_args self.log.info( 'WapResource on path "{full_path}" mapped to procedure "{procedure}"', full_path=full_path, procedure=procedure) # FIXME: how do we allow calling WAMP procedures with positional args? if procedure: self.log.info( 'calling procedure "{procedure}" with kwargs={kwargs} and body_data_len={body_data_len}', procedure=procedure, kwargs=kwargs, body_data_len=len(body_data) if body_data else 0) # we need a session to call if not session: self.log.error('could not call procedure - no session') return self._render_error( 'could not call procedure - no session', request) if body_data: if kwargs: d = session.call(procedure, **kwargs, data=body_data, data_type=content_type) else: d = session.call(procedure, data=body_data, data_type=content_type) else: if kwargs: d = session.call(procedure, **kwargs) else: d = session.call(procedure) else: d = succeed({}) d.addCallbacks(self._after_call_success, self._after_call_error, callbackArgs=[request, client_return_json], errbackArgs=[request, client_return_json]) return server.NOT_DONE_YET except NotFound: self.log.info('URL "{url}" not found (method={method})', url=full_path, method=http_method) request.setResponseCode(404) return self._render_error( 'Path "{full_path}" not found [werkzeug.routing.MapAdapter.match]' .format(full_path=full_path), request) except MethodNotAllowed: self.log.info('method={method} not allowed on URL "{url}"', url=full_path, method=http_method) request.setResponseCode(511) return self._render_error( 'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]' .format(full_path=full_path), request) except Exception as e: self.log.info( 'error while processing method={method} on URL "{url}": {e}', url=full_path, method=http_method, e=e) request.setResponseCode(500) request.write( self._render_error( 'Unknown error with path "{full_path}" [werkzeug.routing.MapAdapter.match]' .format(full_path=full_path), request)) raise
def import_database(self, filename=None, include_indexes=False, include_schemata=None, exclude_tables=None, use_json=False, quiet=False, use_binary_hex_encoding=False): """ :param filename: :param include_indexes: :param include_schemata: :param exclude_tables: :param use_json: :param use_binary_hex_encoding: :returns: """ if include_schemata is None: schemata = sorted(self._schemata.keys()) else: assert type(include_schemata) == list schemata = sorted(list(set(include_schemata).intersection(self._schemata.keys()))) if exclude_tables is None: exclude_tables = set() else: assert type(exclude_tables) == list exclude_tables = set(exclude_tables) if filename: with open(filename, 'rb') as f: data = f.read() else: data = sys.stdin.read() if use_json: ser = JsonObjectSerializer(batched=False, use_binary_hex_encoding=use_binary_hex_encoding) db_data = ser.unserialize(data)[0] else: db_data = cbor2.loads(data) if not quiet: print('\nImporting database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'. format(dbpath=self._dbpath, filename=filename, filesize=len(data))) with self._db.begin(write=True) as txn: for schema_name in schemata: for table_name in self._schema_tables[schema_name]: fq_table_name = '{}.{}'.format(schema_name, table_name) if fq_table_name not in exclude_tables: table = self._schemata[schema_name].__dict__[table_name] if not table.is_index() or include_indexes: if schema_name in db_data and table_name in db_data[schema_name]: cnt = 0 for key, val in db_data[schema_name][table_name]: key = table._deserialize_key(key) val = table.parse(val) table[txn, key] = val cnt += 1 if cnt and not quiet: print('{:.<52}: {}'.format( click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True), click.style(str(cnt) + ' records', fg='yellow'))) else: if not quiet: print('No data to import for {}.{}!'.format(schema_name, table_name))
def export_database(self, filename=None, include_indexes=False, include_schemata=None, exclude_tables=None, use_json=False, quiet=False, use_binary_hex_encoding=False): """ :param filename: :param include_indexes: :param include_schemata: :param exclude_tables: :param use_json: :param use_binary_hex_encoding: :returns: """ if include_schemata is None: schemata = sorted(self._schemata.keys()) else: assert type(include_schemata) == list schemata = sorted(list(set(include_schemata).intersection(self._schemata.keys()))) if exclude_tables is None: exclude_tables = set() else: assert type(exclude_tables) == list exclude_tables = set(exclude_tables) result = {} with self._db.begin() as txn: for schema_name in schemata: for table_name in self._schema_tables[schema_name]: fq_table_name = '{}.{}'.format(schema_name, table_name) if fq_table_name not in exclude_tables: table = self._schemata[schema_name].__dict__[table_name] if not table.is_index() or include_indexes: recs = [] for key, val in table.select(txn, return_keys=True, return_values=True): if val: if hasattr(val, 'marshal'): val = val.marshal() recs.append((table._serialize_key(key), val)) if recs: if schema_name not in result: result[schema_name] = {} result[schema_name][table_name] = recs if use_json: ser = JsonObjectSerializer(batched=False, use_binary_hex_encoding=use_binary_hex_encoding) try: data: bytes = ser.serialize(result) except TypeError as e: print(e) pprint(result) sys.exit(1) else: data: bytes = cbor2.dumps(result) if filename: with open(filename, 'wb') as f: f.write(data) else: sys.stdout.buffer.write(data) if not quiet: print('\nExported database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'. format(dbpath=self._dbpath, filename=filename, filesize=len(data))) for schema_name in result: for table_name in result[schema_name]: cnt = len(result[schema_name][table_name]) if cnt: print('{:.<52}: {}'.format( click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True), click.style(str(cnt) + ' records', fg='yellow')))
break return res msg = message.Call(1, u'com.example.add2', args=(1, 2), kwargs={ u'foo': 23, u'bar': u'baz' }, receive_progress=True) obj = msg.marshal() serializers = [ JsonObjectSerializer(), MsgPackObjectSerializer(), CBORObjectSerializer() ] res = {'ser': {}, 'unser': {}} for ser in serializers: print("testing serializing using {}".format(ser.__class__)) ops = test_serialize(ser, obj) ops = ops[len(ops) / 2:] avg = int(round(sum(ops) / float(len(ops)))) res['ser'][ser.__class__.__name__] = avg print for ser in serializers: