def _pubsub_check_credentials(self):
auth = self.wsgi_environ.get('HTTP_AUTHORIZATION')
if not auth:
raise Forbidden(self.cid)
try:
username, password = parse_basic_auth(auth)
except ValueError:
raise Forbidden(self.cid)
basic_auth = self.server.worker_store.request_dispatcher.url_data.basic_auth_config.itervalues(
)
for item in basic_auth:
config = item['config']
if config['is_active']:
if config['username'] == username and config[
'password'] == password:
auth_ok = True
security_id = config['id']
break
else:
auth_ok = False
if not auth_ok:
raise Forbidden(self.cid)
return security_id
def handle_DELETE(self):
""" DELETE /zato/pubsub/subscribe/topic/{topic_name}?sub_key=..
"""
# Local aliases
sub_key = self.request.input.sub_key
# Checks credentials and returns endpoint_id if valid
endpoint_id = self._pubsub_check_credentials()
# To unsubscribe, we also need to have the right subscription permissions first (patterns) ..
self._check_sub_access(endpoint_id)
# .. also check that sub_key exists and that we are not using another endpoint's sub_key.
try:
sub = self.pubsub.get_subscription_by_sub_key(sub_key)
except KeyError:
self.logger.warn(
'Could not find subscription by sub_key:`%s`, endpoint:`%s`',
sub_key,
self.pubsub.get_endpoint_by_id(endpoint_id).name)
raise Forbidden(self.cid)
else:
if not sub:
self.logger.info('No such sub_key: `%s`', sub_key)
return
# Raise an exception if current endpoint is not the one that created the subscription originally,
# but only if current endpoint is not the default internal one; in such a case we want to let
# the call succeed - this lets other services use self.invoke in order to unsubscribe.
if sub.endpoint_id != endpoint_id:
if endpoint_id != self.server.default_internal_pubsub_endpoint_id:
sub_endpoint = self.pubsub.get_endpoint_by_id(
sub.endpoint_id)
self_endpoint = self.pubsub.get_endpoint_by_id(endpoint_id)
self.logger.warn(
'Endpoint `%s` cannot unsubscribe sk:`%s` (%s) created by `%s`',
self_endpoint.name, sub_key,
self.pubsub.get_topic_by_sub_key(sub_key).name,
sub_endpoint.name)
raise Forbidden(self.cid)
# We have all permissions checked now and can proceed to the actual calls
self.response.payload = self.invoke(
'zato.pubsub.endpoint.delete-endpoint-queue', {
'cluster_id': self.server.cluster_id,
'sub_key': sub_key
})
if sub.is_wsx:
self.invoke(
'zato.channel.web-socket.client.unregister-ws-sub-key', {
'sub_key_list': [sub_key],
})
def _get_pattern_matched(self, topic_name, ws_channel_id, sql_ws_client_id,
security_id, endpoint_id):
pubsub = self.server.worker_store.pubsub
if ws_channel_id and (not sql_ws_client_id):
raise BadRequest(
self.cid,
'sql_ws_client_id must not be empty if ws_channel_id is given on input'
)
# Confirm if this client may subscribe at all to the topic it chose
if endpoint_id:
pattern_matched = pubsub.is_allowed_sub_topic_by_endpoint_id(
topic_name, endpoint_id)
else:
kwargs = {
'security_id': security_id
} if security_id else {
'ws_channel_id': ws_channel_id
}
pattern_matched = pubsub.is_allowed_sub_topic(topic_name, **kwargs)
# Not allowed - raise an exception then
if not pattern_matched:
raise Forbidden(self.cid)
# Alright, we can proceed
else:
return pattern_matched
def handle(self,
_pubsub_prefix='zato.pubsub.pubapi',
_default_allowed=default_services_allowed):
# Local aliases
input = self.request.input
service = input.service
if service \
and service not in _default_allowed \
and service not in self.services_allowed:
self.logger.warn('Service `%s` is not among %s', service,
self.services_allowed) # noqa: E117
raise Forbidden(self.cid)
# We need to special-pub/sub subscriptions
# because they will require calling self.pubsub on behalf of the current WSX connection.
if service == 'zato.pubsub.pubapi.subscribe-wsx':
topic_name = input.request['topic_name']
unsub_on_wsx_close = input.request.get('unsub_on_wsx_close', True)
sub_key = self.pubsub.subscribe(
topic_name,
use_current_wsx=True,
unsub_on_wsx_close=unsub_on_wsx_close,
service=self)
self.response.payload.sub_key = sub_key
else:
self.wsgi_environ['zato.orig_channel'] = self.channel
response = self.invoke(service,
self.request.input.request,
wsgi_environ=self.wsgi_environ)
self.response.payload = response
def _check_sub_access(self, endpoint_id):
# At this point we know that the credentials are valid and in principle, there is such an endpoint,
# but we still don't know if it has permissions to subscribe to this topic and we don't want to reveal
# information about what topics exist or not.
try:
topic = self.pubsub.get_topic_by_name(self.request.input.topic_name)
except KeyError:
self.logger.warn(format_exc())
raise Forbidden(self.cid)
# We know the topic exists but we also need to make sure the endpoint can subscribe to it
if not self.pubsub.is_allowed_sub_topic_by_endpoint_id(topic.name, endpoint_id):
endpoint = self.pubsub.get_endpoint_by_id(endpoint_id)
self.logger.warn('Endpoint `%s` is not allowed to subscribe to `%s`', endpoint.name, self.request.input.topic_name)
raise Forbidden(self.cid)
def get_pub_pattern_matched(self, endpoint_id, input):
""" Returns a publication pattern matched that allows the endpoint to publish messages
or raises an exception if no pattern was matched. Takes into account various IDs possibly given on input,
depending on what our caller wanted to provide.
"""
pubsub = self.server.worker_store.pubsub
security_id = input.security_id or None
ws_channel_id = input.ws_channel_id or None
if not endpoint_id:
if security_id:
endpoint_id = pubsub.get_endpoint_id_by_sec_id(security_id)
elif ws_channel_id:
endpoint_id = pubsub.get_endpoint_id_by_ws_channel_id(ws_channel_id)
else:
raise Exception('Either security_id or ws_channel_id is required if there is no endpoint_id')
kwargs = {'security_id':security_id} if security_id else {'ws_channel_id':ws_channel_id}
pub_pattern_matched = pubsub.is_allowed_pub_topic(input.topic_name, **kwargs)
else:
pub_pattern_matched = pubsub.is_allowed_pub_topic_by_endpoint_id(input.topic_name, endpoint_id)
# Not allowed, raise an exception in that case
if not pub_pattern_matched:
raise Forbidden(self.cid)
# Alright, we are in
return endpoint_id, pub_pattern_matched
def _pubsub_check_credentials(self,
_invoke_channels=(CHANNEL.INVOKE,
CHANNEL.INVOKE_ASYNC)):
# If we are being through a CHANNEL.INVOKE* channel, it means that our caller used self.invoke
# or self.invoke_async, so there will never by any credentials in HTTP headers (there is no HTTP request after all),
# and we can run as an internal endpoint in this situation.
if self.channel.type in _invoke_channels:
return self.server.default_internal_pubsub_endpoint_id
auth = self.wsgi_environ.get('HTTP_AUTHORIZATION')
if not auth:
raise Forbidden(self.cid)
try:
username, password = parse_basic_auth(auth)
except ValueError:
raise Forbidden(self.cid)
basic_auth = itervalues(self.server.worker_store.request_dispatcher.
url_data.basic_auth_config)
for item in basic_auth:
config = item['config']
if config['is_active']:
if config['username'] == username and config[
'password'] == password:
auth_ok = True
security_id = config['id']
break
else:
auth_ok = False
if not auth_ok:
raise Forbidden(self.cid)
try:
endpoint_id = self.pubsub.get_endpoint_id_by_sec_id(security_id)
except KeyError:
self.logger.warn(
'Client credentials are valid but there is no pub/sub endpoint using them, sec_id:`%s`, e:`%s`',
security_id, format_exc())
raise Forbidden(self.cid)
else:
return endpoint_id
def handle(self):
# Make sure this is one of allowed services that we are to invoke
if self.request.input.service not in self.server.fs_server_config.pubsub.wsx_gateway_service_allowed:
raise Forbidden(self.cid)
# All good, we can invoke this service
else:
self.response.payload = self.invoke(self.request.input.service, self.request.input.request,
wsgi_environ=self.wsgi_environ)
def _get_messages(self, ctx):
""" POST /zato/pubsub/topic/{topic_name}?sub_key=...
"""
sub_key = self.request.input.sub_key
try:
self.pubsub.get_subscription_by_sub_key(sub_key)
except KeyError:
self.logger.warn('Could not find sub_key:`%s`, e:`%s`', sub_key, format_exc())
raise Forbidden(self.cid)
else:
return self.pubsub.get_messages(self.request.input.topic_name, sub_key)
def handle_DELETE(self):
""" DELETE /zato/pubsub/subscribe/topic/{topic_name}?sub_key=..
"""
# Local aliases
sub_key = self.request.input.sub_key
# Checks credentials and returns endpoint_id if valid
endpoint_id = self._pubsub_check_credentials()
# To unsubscribe, we also need to have the right subscription permissions first (patterns) ..
self._check_sub_access(endpoint_id)
# .. also check that sub_key exists and that we are not using another endpoint's sub_key.
try:
sub = self.pubsub.get_subscription_by_sub_key(sub_key)
except KeyError:
self.logger.warn(
'Could not find subscription by sub_key:`%s`, endpoint:`%s`',
sub_key,
self.pubsub.get_endpoint_by_id(endpoint_id).name)
raise Forbidden(self.cid)
else:
if sub.endpoint_id != endpoint_id:
sub_endpoint = self.pubsub.get_endpoint_by_id(sub.endpoint_id)
self_endpoint = self.pubsub.get_endpoint_by_id(endpoint_id)
self.logger.warn(
'Endpoint `%s` cannot unsubscribe sk:`%s` (%s) created by `%s`',
self_endpoint.name, sub_key,
self.pubsub.get_topic_by_sub_key(sub_key).name,
sub_endpoint.name)
raise Forbidden(self.cid)
# We have all permissions checked now and can proceed to the actual call
self.invoke('zato.pubsub.endpoint.delete-endpoint-queue', {
'cluster_id': self.server.cluster_id,
'sub_key': sub_key
})
def handle(self):
# Local aliases
topic_name = self.request.input.topic_name
topic_name_list = set(self.request.input.topic_name_list)
environ = self.wsgi_environ['zato.request_ctx.async_msg']['environ']
ws_channel_id = environ['ws_channel_config'].id
# Make sure the WSX channel actually points to an endpoint. If it does not,
# we cannot proceed, i.e. there is no such API client.
try:
self.pubsub.get_endpoint_id_by_ws_channel_id(ws_channel_id)
except KeyError:
self.logger.warn('There is no endpoint for WSX chan ID `%s`',
ws_channel_id)
raise Forbidden(self.cid)
# Either an exact topic name or a list thereof is needed ..
if not (topic_name or topic_name_list):
raise BadRequest(
self.cid,
'Either or topic_name or topic_name_list is required')
# .. but we cannot accept both of them.
elif topic_name and topic_name_list:
raise BadRequest(
self.cid,
'Cannot provide both topic_name and topic_name_list on input')
subscribe_to = [topic_name] if topic_name else topic_name_list
responses = {}
for item in subscribe_to:
response = self.invoke(
'zato.pubsub.subscription.subscribe-websockets', {
'topic_name': item,
'ws_channel_id': ws_channel_id,
'ext_client_id': environ['ext_client_id'],
'ws_pub_client_id': environ['pub_client_id'],
'ws_channel_name': environ['ws_channel_config']['name'],
'sql_ws_client_id': environ['sql_ws_client_id'],
'web_socket': environ['web_socket'],
})['response']
responses[item] = response
# There was only one topic on input ..
if topic_name:
self.response.payload = responses[topic_name]
# .. or a list of topics on was given on input.
else:
out = []
for key, value in responses.items():
out.append({
'topic_name': key,
'sub_key': value['sub_key'],
'current_depth': value['queue_depth'],
})
self.response.payload.sub_data = out
def handle(self,
_expected_endpoint_type=PUBSUB.ENDPOINT_TYPE.WEB_SOCKETS.id):
# Local aliases
sub_key_list = [self.request.input.sub_key]
async_msg = self.wsgi_environ['zato.request_ctx.async_msg']
# This will exist if are being invoked directly ..
environ = async_msg.get('environ')
# .. however, if there is a service on whose behalf we are invoked, the 'environ' key will be further nested.
if not environ:
_wsgi_environ = async_msg['wsgi_environ']
_async_msg = _wsgi_environ['zato.request_ctx.async_msg']
environ = _async_msg['environ']
# We now have environ in one way or another
wsx = environ['web_socket']
pubsub_tool = wsx.pubsub_tool
# Need to confirm that our WebSocket previously created all the input sub_keys
wsx_channel_id = environ['ws_channel_config'].id
wsx_endpoint = self.pubsub.get_endpoint_by_ws_channel_id(
wsx_channel_id)
# First off, make sure that input sub_key(s) were previously created by current WebSocket
for sub_key in sub_key_list:
sub = self.pubsub.get_subscription_by_sub_key(sub_key)
if sub.config.endpoint_type != _expected_endpoint_type:
self.logger.warn(
'Subscription `%s` endpoint_type:`%s` did not match `%s`',
sub_key, sub.config.endpoint_type, _expected_endpoint_type)
raise Forbidden(self.cid)
if wsx_endpoint.name != sub.config.endpoint_name:
expected_endpoint = self.pubsub.get_endpoint_by_id(
sub.config.endpoint_id)
self.logger.warn(
'Current WSX endpoint did not match sub_key `%s` endpoint, current:%s (%s) vs. expected:%s (%s)',
sub_key, wsx_endpoint.name, wsx_endpoint.id,
expected_endpoint.name, expected_endpoint.id)
raise Forbidden(self.cid)
try:
with closing(self.odb.session()) as session:
# Everything is performed using that WebSocket's pub/sub lock to ensure that both
# in-RAM and SQL (non-GD and GD) messages are made available to the WebSocket as a single unit.
with pubsub_tool.lock:
get_in_ram_service = 'zato.pubsub.topic.get-in-ram-message-list'
_, non_gd_messages = self.servers.invoke_all(
get_in_ram_service, {'sub_key_list': sub_key_list},
timeout=120)
# Parse non-GD messages on output from all servers, if any at all, into per-sub_key lists ..
if non_gd_messages:
non_gd_messages = self._parse_non_gd_messages(
sub_key_list, non_gd_messages)
# If there are any non-GD messages, add them to this WebSocket's pubsub tool.
if non_gd_messages:
for sub_key, messages in non_gd_messages.items():
pubsub_tool.add_sub_key_no_lock(sub_key)
pubsub_tool.add_non_gd_messages_by_sub_key(
sub_key, messages)
# For each sub_key from input ..
for sub_key in sub_key_list:
# .. add relevant SQL objects ..
self.pubsub.add_wsx_client_pubsub_keys(
session, environ['sql_ws_client_id'], sub_key,
environ['ws_channel_config']['name'],
environ['pub_client_id'],
environ['web_socket'].get_peer_info_dict())
# .. update state of that WebSocket's pubsub tool that keeps track of message delivery
pubsub_tool.add_sub_key_no_lock(sub_key)
# Everything is ready - note that pubsub_tool itself will enqueue any initial messages
# using its enqueue_initial_messages method which does it in batches.
session.commit()
except Exception:
self.logger.warn(
'Error while resuming WSX pub/sub for keys `%s`, e:`%s`',
sub_key_list, format_exc())
raise
else:
# No exception = all good and we can register this pubsub_tool with self.pubsub now
for sub_key in sub_key_list:
self.pubsub.set_pubsub_tool_for_sub_key(sub_key, pubsub_tool)
# No exceptions here = we have resumed the subscription(s) successfully and we can report it
_log_info = {}
for _sub_key in sub_key_list:
_log_info[_sub_key] = self.pubsub.get_topic_by_sub_key(
_sub_key).name
self.logger.info('Subscription%sresumed: `%s',
' ' if len(sub_key_list) == 1 else 's ',
_log_info)
def handle(self):
# Local aliases
topic_name = self.request.input.topic_name
topic_name_list = set(self.request.input.topic_name_list)
async_msg = self.wsgi_environ['zato.request_ctx.async_msg']
unsub_on_wsx_close = async_msg['wsgi_environ'].get(
'zato.request_ctx.pubsub.unsub_on_wsx_close')
# This will exist if we are being invoked directly ..
environ = async_msg.get('environ')
# .. however, if there is a service on whose behalf we are invoked, the 'environ' key will be further nested.
if not environ:
_wsgi_environ = async_msg['wsgi_environ']
_async_msg = _wsgi_environ['zato.request_ctx.async_msg']
environ = _async_msg['environ']
ws_channel_id = environ['ws_channel_config'].id
# Make sure the WSX channel actually points to an endpoint. If it does not,
# we cannot proceed, i.e. there is no such API client.
try:
self.pubsub.get_endpoint_id_by_ws_channel_id(ws_channel_id)
except KeyError:
self.logger.warn('There is no endpoint for WSX channel ID `%s`',
ws_channel_id)
raise Forbidden(self.cid)
# Either an exact topic name or a list thereof is needed ..
if not (topic_name or topic_name_list):
raise BadRequest(
self.cid,
'Either or topic_name or topic_name_list is required')
# .. but we cannot accept both of them.
elif topic_name and topic_name_list:
raise BadRequest(
self.cid,
'Cannot provide both topic_name and topic_name_list on input')
subscribe_to = [topic_name] if topic_name else topic_name_list
responses = {}
for item in subscribe_to:
request = {
'topic_name': item,
'ws_channel_id': ws_channel_id,
'ext_client_id': environ['ext_client_id'],
'ws_pub_client_id': environ['pub_client_id'],
'ws_channel_name': environ['ws_channel_config']['name'],
'sql_ws_client_id': environ['sql_ws_client_id'],
'unsub_on_wsx_close': unsub_on_wsx_close,
'web_socket': environ['web_socket'],
}
for name in 'wrap_one_msg_in_list', 'delivery_batch_size':
request[name] = self.request.input.get(name)
response = self.invoke(
'zato.pubsub.subscription.subscribe-websockets',
request)['response']
responses[item] = response
# There was only one topic on input ..
if topic_name:
self.response.payload = responses[topic_name]
# .. or a list of topics on was given on input.
else:
out = []
for key, value in responses.items():
out.append({
'topic_name': key,
'sub_key': value['sub_key'],
'current_depth': value['queue_depth'],
})
self.response.payload.sub_data = out
