class WampMQTTServerFactory(Factory):
log = make_logger()
protocol = WampMQTTServerProtocol
serializers = {
'json': JsonObjectSerializer(),
'msgpack': MsgPackObjectSerializer(),
'cbor': CBORObjectSerializer(),
'ubjson': UBJSONObjectSerializer(),
}
def __init__(self, router_session_factory, config, reactor):
self._router_session_factory = router_session_factory
self._router_factory = router_session_factory._routerFactory
self._options = config.get('options', {})
self._realm = self._options.get('realm', None)
self._reactor = reactor
self._payload_mapping = StringTrie()
for topic, pmap in self._options.get('payload_mapping', {}).items():
self._set_payload_format(topic, pmap)
def buildProtocol(self, addr):
protocol = self.protocol(self._reactor)
protocol.factory = self
return protocol
def _get_payload_format(self, topic):
"""
Map a WAMP topic URI to MQTT payload format.
:param topic: WAMP URI.
:type topic: str
:returns: Payload format metadata.
:rtype: dict
"""
try:
pmap = self._payload_mapping.longest_prefix_value(topic)
except KeyError:
return None
else:
return pmap
def _set_payload_format(self, topic, pmap=None):
if pmap is None:
if topic in self._payload_mapping:
del self._payload_mapping[topic]
else:
self._payload_mapping[topic] = pmap
@inlineCallbacks
def transform_wamp(self, topic, msg):
# check for cached transformed payload
cache_key = '_{}_{}'.format(self.__class__.__name__, id(self))
cached = msg._serialized.get(cache_key, None)
if cached:
payload_format, mapped_topic, payload = cached
self.log.debug(
'using cached payload for {cache_key} in message {msg_id}!',
msg_id=id(msg),
cache_key=cache_key)
else:
# convert WAMP URI to MQTT topic
mapped_topic = _wamp_topic_to_mqtt(topic)
# for WAMP->MQTT, the payload mapping is determined from the
# WAMP URI (not the transformed MQTT topic)
payload_format = self._get_payload_format(topic)
payload_format_type = payload_format['type']
if payload_format_type == 'passthrough':
payload = msg.payload
elif payload_format_type == 'native':
serializer = payload_format.get('serializer', None)
payload = self._transform_wamp_native(serializer, msg)
elif payload_format_type == 'dynamic':
encoder = payload_format.get('encoder', None)
codec_realm = payload_format.get('realm', self._realm)
payload = yield self._transform_wamp_dynamic(
encoder, codec_realm, mapped_topic, topic, msg)
else:
raise Exception(
'payload format {} not implemented'.format(payload_format))
msg._serialized[cache_key] = (payload_format, mapped_topic,
payload)
self.log.debug(
'transform_wamp({topic}, {msg}) -> payload_format={payload_format}, mapped_topic={mapped_topic}, payload={payload}',
topic=topic,
msg=msg,
payload_format=payload_format,
mapped_topic=mapped_topic,
payload=payload)
returnValue((payload_format, mapped_topic, payload))
@inlineCallbacks
def _transform_wamp_dynamic(self, encoder, codec_realm, mapped_topic,
topic, msg):
codec_session = self._router_factory.get(codec_realm)._realm.session
payload = yield codec_session.call(encoder, mapped_topic, topic,
msg.args, msg.kwargs)
returnValue(payload)
def _transform_wamp_native(self, serializer, msg):
obj = {}
for opt in [
'args', 'kwargs', 'exclude', 'exclude_authid',
'exclude_authrole', 'eligible', 'eligible_authid',
'eligible_authrole'
]:
attr = getattr(msg, opt, None)
if attr is not None:
obj[opt] = attr
if serializer in self.serializers:
payload = self.serializers[serializer].serialize(obj)
else:
raise Exception(
'MQTT native mode payload transform: invalid serializer {}'.
format(serializer))
return payload
@inlineCallbacks
def transform_mqtt(self, topic, payload):
# transform MQTT topic to WAMP URI
mapped_topic = _mqtt_topicname_to_wamp(topic)
# for MQTT->WAMP, the payload mapping is determined from the
# transformed WAMP URI (not the original MQTT topic)
payload_format = self._get_payload_format(mapped_topic)
payload_format_type = payload_format['type']
if payload_format_type == 'passthrough':
options = {'payload': payload, 'enc_algo': 'mqtt'}
elif payload_format_type == 'native':
serializer = payload_format.get('serializer', None)
options = self._transform_mqtt_native(serializer, payload)
elif payload_format_type == 'dynamic':
decoder = payload_format.get('decoder', None)
codec_realm = payload_format.get('realm', self._realm)
options = yield self._transform_mqtt_dynamic(
decoder, codec_realm, mapped_topic, topic, payload)
else:
raise Exception(
'payload format {} not implemented'.format(payload_format))
self.log.debug(
'transform_mqtt({topic}, {payload}) -> payload_format={payload_format}, mapped_topic={mapped_topic}, options={options}',
topic=topic,
payload=payload,
payload_format=payload_format,
mapped_topic=mapped_topic,
options=options)
returnValue((payload_format, mapped_topic, options))
@inlineCallbacks
def _transform_mqtt_dynamic(self, decoder, codec_realm, mapped_topic,
topic, payload):
codec_session = self._router_factory.get(codec_realm)._realm.session
options = yield codec_session.call(decoder, mapped_topic, topic,
payload)
returnValue(options)
def _transform_mqtt_native(self, serializer, payload):
"""
Transform MQTT binary payload from a MQTT Publish to keyword dict
suitable for the constructor of a WAMP Publish message,
that is :class:`autobahn.wamp.message.Publish`.
"""
options = {}
if serializer in self.serializers:
if serializer == 'json':
if not _validator.validate(payload)[0]:
# invalid UTF-8: drop the event
raise Exception(
'invalid UTF8 in JSON encoded MQTT payload')
obj = self.serializers[serializer].unserialize(payload)[0]
else:
raise Exception(
'"{}" serializer for encoded MQTT payload not implemented'.
format(serializer))
if not isinstance(obj, dict):
raise Exception(
'invalid type {} for "{}" encoded MQTT payload'.format(
type(obj), serializer))
for opt in [
'args', 'kwargs', 'exclude', 'exclude_authid',
'exclude_authrole', 'eligible', 'eligible_authid',
'eligible_authrole'
]:
if opt in obj:
options[opt] = obj[opt]
return options
class CatalogResource(resource.Resource):
"""
Twisted Web resource for API FbsRepository Web service.
This resource uses templates loaded into a Jinja2 environment to render HTML pages
with data retrieved from an API FbsRepository archive file or on-chain address.
"""
log = make_logger()
ser = JsonObjectSerializer()
isLeaf = True
def __init__(self, jinja_env: Environment, worker: Union[RouterController, ProxyController],
config: Dict[str, Any], path: str):
"""
:param worker: The router worker controller within this Web service is started.
:param config: The Web service configuration item.
"""
resource.Resource.__init__(self)
# remember all ctor args
self._jinja_env: Environment = jinja_env
self._worker = worker
self._config = config
self._path = path
# setup Werkzeug URL map adapter
# https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.Map
adapter_map = werkzeug.routing.Map()
routes = {
'/': 'wamp_catalog_home.html',
'table': 'wamp_catalog_table.html',
'struct': 'wamp_catalog_struct.html',
'enum': 'wamp_catalog_enum.html',
'service': 'wamp_catalog_service.html',
}
for rpath, route_template in routes.items():
# compute full absolute URL of route to be added - ending in Werkzeug/Routes URL pattern
_rp = []
if path != '/':
_rp.append(path)
if rpath != '/':
_rp.append(rpath)
route_url = os.path.join('/', '/'.join(_rp))
route_endpoint = jinja_env.get_template(route_template)
route_rule = Rule(route_url, methods=['GET'], endpoint=route_endpoint)
adapter_map.add(route_rule)
# https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.Map.bind
self._map_adapter: MapAdapter = adapter_map.bind('localhost', '/')
# FIXME
self._repo: FbsRepository = FbsRepository('FIXME')
self._repo.load(self._config['filename'])
def render(self, request):
# https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html#render
# The encoded path of the request URI (_not_ (!) including query arguments),
full_path = request.path.decode('utf-8')
# HTTP request method
http_method = request.method.decode()
if http_method not in ['GET']:
request.setResponseCode(511)
return self._render_error(
'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format(
full_path=full_path), request)
# parse and decode any query parameters
query_args = {}
if request.args:
for key, values in request.args.items():
key = key.decode()
# we only process the first header value per key (!)
value = values[0].decode()
query_args[key] = value
self.log.info('Parsed query parameters: {query_args}', query_args=query_args)
# parse client announced accept-header
client_accept = request.getAllHeaders().get(b'accept', None)
if client_accept:
client_accept = client_accept.decode()
# flag indicating the client wants to get plain JSON results (not rendered HTML)
# client_return_json = client_accept == 'application/json'
# client cookie processing
cookie = request.received_cookies.get(b'session_cookie')
self.log.debug('Session Cookie is ({})'.format(cookie))
try:
# werkzeug.routing.MapAdapter
# https://werkzeug.palletsprojects.com/en/2.1.x/routing/#werkzeug.routing.MapAdapter.match
template, kwargs = self._map_adapter.match(full_path, method=http_method, query_args=query_args)
if kwargs:
if query_args:
kwargs.update(query_args)
else:
kwargs = query_args
kwargs['repo'] = self._repo
kwargs['created'] = time_ns()
self.log.info(
'CatalogResource request on path "{full_path}" mapped to template "{template}" '
'using kwargs\n{kwargs}',
full_path=full_path,
template=template,
kwargs=pformat(kwargs))
rendered = template.render(**kwargs).encode('utf8')
self.log.info('successfully rendered HTML result: {rendered} bytes', rendered=len(rendered))
request.setResponseCode(200)
return rendered
except NotFound:
self.log.warn('URL "{url}" not found (method={method})', url=full_path, method=http_method)
request.setResponseCode(404)
return self._render_error(
'Path "{full_path}" not found [werkzeug.routing.MapAdapter.match]'.format(full_path=full_path),
request)
except MethodNotAllowed:
self.log.warn('method={method} not allowed on URL "{url}"', url=full_path, method=http_method)
request.setResponseCode(511)
return self._render_error(
'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format(
full_path=full_path), request)
except Exception as e:
self.log.warn('error while processing method={method} on URL "{url}": {e}',
url=full_path,
method=http_method,
e=e)
request.setResponseCode(500)
return self._render_error(
'Unknown error with path "{full_path}" [werkzeug.routing.MapAdapter.match]'.format(
full_path=full_path), request)
def _render_error(self, message, request, client_return_json=False):
"""
Error renderer, display a basic error message to tell the user that there
was a problem and roughly what the problem was.
:param message: The current error message
:param request: The original HTTP request
:return: HTML formatted error string
"""
if client_return_json:
return self.ser.serialize({'error': message})
else:
return """
<html>
<title>API Error</title>
<body>
<h3 style="color: #f00">Crossbar WAMP Application Page Error</h3>
<pre>{}</pre>
</body>
</html>
""".format(escape(message)).encode('utf8')
import binascii
from autobahn.wamp.serializer import JsonObjectSerializer, MsgPackObjectSerializer
ser = JsonObjectSerializer(batched = True)
ser = MsgPackObjectSerializer(batched = True)
o1 = [1, "hello", [1, 2, 3]]
o2 = [3, {"a": 23, "b": 24}]
d1 = ser.serialize(o1) + ser.serialize(o2) + ser.serialize(o1)
m = ser.unserialize(d1)
print m
class WapResource(resource.Resource):
"""
Twisted Web resource for WAMP Application Page web service.
This resource uses templates loaded into a Jinja2 environment to render HTML pages with data retrieved
from a WAMP procedure call, triggered from the original Web request.
"""
log = make_logger()
ser = JsonObjectSerializer()
isLeaf = True
def __init__(self, worker, config, path):
"""
:param worker: The router worker controller within this Web service is started.
:type worker: crossbar.worker.router.RouterController
:param config: The Web service configuration item.
:type config: dict
"""
resource.Resource.__init__(self)
self._worker = worker
self._config = config
self._session_cache = {}
self._realm_name = config.get('wamp', {}).get('realm', None)
self._authrole = config.get('wamp', {}).get('authrole', 'anonymous')
self._service_agent = worker.realm_by_name(self._realm_name).session
# TODO:
# We need to lookup the credentials for the current user based on the pre-established
# HTTP session cookie, this will establish the 'authrole' the user is running as.
# This 'authrole' can then be used to authorize the back-end topic call.
# QUESTION:
# Does the topic need the authid, if so, how do we pass it?
#
# This is our default (anonymous) session for unauthenticated users
#
router = worker._router_factory.get(self._realm_name)
self._default_session = ApplicationSession(
ComponentConfig(realm=self._realm_name, extra=None))
worker._router_session_factory.add(self._default_session,
router,
authrole=self._authrole)
# Setup Jinja2 to point to our templates folder or a package resource
#
templates_config = config.get("templates")
if type(templates_config) == str:
# resolve specified template directory path relative to node directory
templates_dir = os.path.abspath(
os.path.join(self._worker.config.extra.cbdir,
templates_config))
templates_source = 'directory'
elif type(templates_config) == dict:
# in case we got a dict, that must contain "package" and "resource" attributes
if 'package' not in templates_config:
raise ApplicationError(
'crossbar.error.invalid_configuration',
'missing attribute "resource" in WAP web service configuration'
)
if 'resource' not in templates_config:
raise ApplicationError(
'crossbar.error.invalid_configuration',
'missing attribute "resource" in WAP web service configuration'
)
try:
importlib.import_module(templates_config['package'])
except ImportError as e:
emsg = 'Could not import resource {} from package {}: {}'.format(
templates_config['resource'], templates_config['package'],
e)
raise ApplicationError('crossbar.error.invalid_configuration',
emsg)
else:
try:
# resolve template directory from package resource
templates_dir = os.path.abspath(
pkg_resources.resource_filename(
templates_config['package'],
templates_config['resource']))
except Exception as e:
emsg = 'Could not import resource {} from package {}: {}'.format(
templates_config['resource'],
templates_config['package'], e)
raise ApplicationError(
'crossbar.error.invalid_configuration', emsg)
templates_source = 'package'
else:
raise ApplicationError(
'crossbar.error.invalid_configuration',
'invalid type "{}" for attribute "templates" in WAP web service configuration'
.format(type(templates_config)))
if config.get('sandbox', True):
# The sandboxed environment. It works like the regular environment but tells the compiler to
# generate sandboxed code.
# https://jinja.palletsprojects.com/en/2.11.x/sandbox/#jinja2.sandbox.SandboxedEnvironment
env = SandboxedEnvironment(loader=FileSystemLoader(templates_dir),
autoescape=True)
else:
env = Environment(loader=FileSystemLoader(templates_dir),
autoescape=True)
self.log.info(
'WapResource created (realm="{realm}", authrole="{authrole}", templates_dir="{templates_dir}", templates_source="{templates_source}", jinja2_env={jinja2_env})',
realm=hlid(self._realm_name),
authrole=hlid(self._authrole),
templates_dir=hlid(templates_dir),
templates_source=hlid(templates_source),
jinja2_env=hltype(env.__class__))
# http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.Map
map = Map()
# Add all our routes into 'map', note each route endpoint is a tuple of the
# topic to call, and the template to use when rendering the results.
for route in config.get('routes', {}):
# compute full absolute URL of route to be added - ending in Werkzeug/Routes URL pattern
rpath = route['path']
_rp = []
if path != '/':
_rp.append(path)
if rpath != '/':
_rp.append(rpath)
route_url = os.path.join('/', '/'.join(_rp))
route_method = route.get('method', 'GET')
assert route_method in [
'GET', 'POST'
], 'invalid HTTP method "{}" for route on URL "{}"'.format(
route_method, route_url)
route_methods = [route_method]
# note the WAMP procedure to call and the Jinja2 template to render as HTTP response
route_endpoint = (route['call'], env.get_template(route['render']))
route_rule = Rule(route_url,
methods=route_methods,
endpoint=route_endpoint)
map.add(route_rule)
self.log.info(
'WapResource route added (url={route_url}, methods={route_methods}, endpoint={route_endpoint})',
route_url=hlid(route_url),
route_methods=hlid(route_methods),
route_endpoint=route_endpoint)
# http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter
# http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter.match
self._map_adapter = map.bind('/')
def _after_call_success(self, result, request, client_return_json):
"""
When the WAMP call attached to the URL returns, render the WAMP result
into a Jinja2 template and return HTML to client. Alternatively, return
the call result as plain JSON.
:param result: The dict returned from the WAMP procedure call.
:param request: The HTTP request.
:param client_return_json: Flag indicating to return plain JSON (no HTML rendering.)
"""
try:
if client_return_json:
rendered = self.ser.serialize(result)
self.log.info(
'WapResource successfully serialized JSON result:\n{result}',
result=pformat(result))
else:
rendered = request.template.render(result).encode('utf8')
self.log.info(
'WapResource successfully rendered HTML result: {rendered} bytes',
rendered=len(rendered))
except Exception as e:
self.log.failure()
emsg = 'WapResource render error for WAMP result of type "{}": {}'.format(
type(result), e)
self.log.warn(emsg)
request.setResponseCode(500)
request.write(self._render_error(emsg, request,
client_return_json))
else:
request.write(rendered)
request.finish()
def _after_call_error(self, error, request, client_return_json):
"""
Deferred error, write out the error template and finish the request
:param error: The current deferred error object
:param request: The original HTTP request
"""
self.log.error('WapResource error: {error}', error=error)
request.setResponseCode(500)
request.write(
self._render_error(error.value.error, request, client_return_json))
request.finish()
def _render_error(self, message, request, client_return_json=False):
"""
Error renderer, display a basic error message to tell the user that there
was a problem and roughly what the problem was.
:param message: The current error message
:param request: The original HTTP request
:return: HTML formatted error string
"""
if client_return_json:
return self.ser.serialize({'error': message})
else:
return """
<html>
<title>API Error</title>
<body>
<h3 style="color: #f00">Crossbar WAMP Application Page Error</h3>
<pre>{}</pre>
</body>
</html>
""".format(escape(message)).encode('utf8')
def render(self, request):
"""
Initiate the rendering of a HTTP/GET request by calling a WAMP procedure, the
resulting ``dict`` is rendered together with the specified Jinja2 template
for this URL.
:param request: The HTTP request.
:returns: server.NOT_DONE_YET (special)
"""
# https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html#render
# The encoded path of the request URI (_not_ (!) including query arguments),
full_path = request.path.decode('utf-8')
# HTTP request method (GET, POST, ..)
http_method = request.method.decode()
if http_method not in ['GET', 'POST']:
request.setResponseCode(511)
return self._render_error(
'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'
.format(full_path=full_path), request)
# in case of HTTP/POST, read request body as one binary string
if http_method == 'POST' and request.content:
content_type = request.getAllHeaders().get(
b'content-type', b'application/octet-stream').decode()
# https://stackoverflow.com/a/11549600/884770
# http://marianoiglesias.com.ar/python/file-uploading-with-multi-part-encoding-using-twisted/
body_data = request.content.read()
self.log.info('POST data len = {newdata_len}',
newdata_len=len(body_data))
else:
content_type = None
body_data = None
# parse and decode any query parameters
query_args = {}
if request.args:
for key, values in request.args.items():
key = key.decode()
# we only process the first header value per key (!)
value = values[0].decode()
query_args[key] = value
self.log.info('Parsed query parameters: {query_args}',
query_args=query_args)
# parse client announced accept header
client_accept = request.getAllHeaders().get(b'accept', None)
if client_accept:
client_accept = client_accept.decode()
# flag indicating the client wants to get plain JSON results (not rendered HTML)
client_return_json = client_accept == 'application/json'
# client cookie processing
cookie = request.received_cookies.get(b'session_cookie')
self.log.debug('Session Cookie is ({})'.format(cookie))
if cookie:
session = self._session_cache.get(cookie)
if not session:
# FIXME: lookup role for current session
self.log.debug(
'Creating a new session for cookie ({})'.format(cookie))
authrole = 'anonymous'
session = ApplicationSession(
ComponentConfig(realm=self._realm_name, extra=None))
self._worker._router_session_factory.add(session,
authrole=authrole)
self._session_cache[cookie] = session
else:
self.log.debug(
'Using a cached session for ({})'.format(cookie))
else:
self.log.debug(
'No session cookie, falling back on default session')
session = self._default_session
try:
# werkzeug.routing.MapAdapter
# http://werkzeug.pocoo.org/docs/dev/routing/#werkzeug.routing.MapAdapter.match
(procedure, request.template), kwargs = self._map_adapter.match(
full_path, method=http_method, query_args=query_args)
if kwargs and query_args:
kwargs.update(query_args)
else:
kwargs = query_args
self.log.info(
'WapResource on path "{full_path}" mapped to procedure "{procedure}"',
full_path=full_path,
procedure=procedure)
# FIXME: how do we allow calling WAMP procedures with positional args?
if procedure:
self.log.info(
'calling procedure "{procedure}" with kwargs={kwargs} and body_data_len={body_data_len}',
procedure=procedure,
kwargs=kwargs,
body_data_len=len(body_data) if body_data else 0)
# we need a session to call
if not session:
self.log.error('could not call procedure - no session')
return self._render_error(
'could not call procedure - no session', request)
if body_data:
if kwargs:
d = session.call(procedure,
**kwargs,
data=body_data,
data_type=content_type)
else:
d = session.call(procedure,
data=body_data,
data_type=content_type)
else:
if kwargs:
d = session.call(procedure, **kwargs)
else:
d = session.call(procedure)
else:
d = succeed({})
d.addCallbacks(self._after_call_success,
self._after_call_error,
callbackArgs=[request, client_return_json],
errbackArgs=[request, client_return_json])
return server.NOT_DONE_YET
except NotFound:
self.log.info('URL "{url}" not found (method={method})',
url=full_path,
method=http_method)
request.setResponseCode(404)
return self._render_error(
'Path "{full_path}" not found [werkzeug.routing.MapAdapter.match]'
.format(full_path=full_path), request)
except MethodNotAllowed:
self.log.info('method={method} not allowed on URL "{url}"',
url=full_path,
method=http_method)
request.setResponseCode(511)
return self._render_error(
'Method not allowed on path "{full_path}" [werkzeug.routing.MapAdapter.match]'
.format(full_path=full_path), request)
except Exception as e:
self.log.info(
'error while processing method={method} on URL "{url}": {e}',
url=full_path,
method=http_method,
e=e)
request.setResponseCode(500)
request.write(
self._render_error(
'Unknown error with path "{full_path}" [werkzeug.routing.MapAdapter.match]'
.format(full_path=full_path), request))
raise
def import_database(self,
filename=None,
include_indexes=False,
include_schemata=None,
exclude_tables=None,
use_json=False,
quiet=False,
use_binary_hex_encoding=False):
"""
:param filename:
:param include_indexes:
:param include_schemata:
:param exclude_tables:
:param use_json:
:param use_binary_hex_encoding:
:returns:
"""
if include_schemata is None:
schemata = sorted(self._schemata.keys())
else:
assert type(include_schemata) == list
schemata = sorted(list(set(include_schemata).intersection(self._schemata.keys())))
if exclude_tables is None:
exclude_tables = set()
else:
assert type(exclude_tables) == list
exclude_tables = set(exclude_tables)
if filename:
with open(filename, 'rb') as f:
data = f.read()
else:
data = sys.stdin.read()
if use_json:
ser = JsonObjectSerializer(batched=False, use_binary_hex_encoding=use_binary_hex_encoding)
db_data = ser.unserialize(data)[0]
else:
db_data = cbor2.loads(data)
if not quiet:
print('\nImporting database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'.
format(dbpath=self._dbpath, filename=filename, filesize=len(data)))
with self._db.begin(write=True) as txn:
for schema_name in schemata:
for table_name in self._schema_tables[schema_name]:
fq_table_name = '{}.{}'.format(schema_name, table_name)
if fq_table_name not in exclude_tables:
table = self._schemata[schema_name].__dict__[table_name]
if not table.is_index() or include_indexes:
if schema_name in db_data and table_name in db_data[schema_name]:
cnt = 0
for key, val in db_data[schema_name][table_name]:
key = table._deserialize_key(key)
val = table.parse(val)
table[txn, key] = val
cnt += 1
if cnt and not quiet:
print('{:.<52}: {}'.format(
click.style('{}.{}'.format(schema_name, table_name),
fg='white',
bold=True), click.style(str(cnt) + ' records',
fg='yellow')))
else:
if not quiet:
print('No data to import for {}.{}!'.format(schema_name, table_name))
def export_database(self,
filename=None,
include_indexes=False,
include_schemata=None,
exclude_tables=None,
use_json=False,
quiet=False,
use_binary_hex_encoding=False):
"""
:param filename:
:param include_indexes:
:param include_schemata:
:param exclude_tables:
:param use_json:
:param use_binary_hex_encoding:
:returns:
"""
if include_schemata is None:
schemata = sorted(self._schemata.keys())
else:
assert type(include_schemata) == list
schemata = sorted(list(set(include_schemata).intersection(self._schemata.keys())))
if exclude_tables is None:
exclude_tables = set()
else:
assert type(exclude_tables) == list
exclude_tables = set(exclude_tables)
result = {}
with self._db.begin() as txn:
for schema_name in schemata:
for table_name in self._schema_tables[schema_name]:
fq_table_name = '{}.{}'.format(schema_name, table_name)
if fq_table_name not in exclude_tables:
table = self._schemata[schema_name].__dict__[table_name]
if not table.is_index() or include_indexes:
recs = []
for key, val in table.select(txn, return_keys=True, return_values=True):
if val:
if hasattr(val, 'marshal'):
val = val.marshal()
recs.append((table._serialize_key(key), val))
if recs:
if schema_name not in result:
result[schema_name] = {}
result[schema_name][table_name] = recs
if use_json:
ser = JsonObjectSerializer(batched=False, use_binary_hex_encoding=use_binary_hex_encoding)
try:
data: bytes = ser.serialize(result)
except TypeError as e:
print(e)
pprint(result)
sys.exit(1)
else:
data: bytes = cbor2.dumps(result)
if filename:
with open(filename, 'wb') as f:
f.write(data)
else:
sys.stdout.buffer.write(data)
if not quiet:
print('\nExported database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'.
format(dbpath=self._dbpath, filename=filename, filesize=len(data)))
for schema_name in result:
for table_name in result[schema_name]:
cnt = len(result[schema_name][table_name])
if cnt:
print('{:.<52}: {}'.format(
click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True),
click.style(str(cnt) + ' records', fg='yellow')))
break
return res
msg = message.Call(1,
u'com.example.add2',
args=(1, 2),
kwargs={
u'foo': 23,
u'bar': u'baz'
},
receive_progress=True)
obj = msg.marshal()
serializers = [
JsonObjectSerializer(),
MsgPackObjectSerializer(),
CBORObjectSerializer()
]
res = {'ser': {}, 'unser': {}}
for ser in serializers:
print("testing serializing using {}".format(ser.__class__))
ops = test_serialize(ser, obj)
ops = ops[len(ops) / 2:]
avg = int(round(sum(ops) / float(len(ops))))
res['ser'][ser.__class__.__name__] = avg
print
for ser in serializers: