def test_get_callback_consumers(self):
ps = RedisPubSub(self.kvdb, self.key_prefix)
msg_value = '"msg_value"'
topic = Topic('/test/delete')
ps.add_topic(topic)
producer = Client('Producer', 'producer')
ps.add_producer(producer, topic)
id1 = 'Consumer CB1'
name1 = 'consumer-cb1'
sub_key1 = new_cid()
callback_id1 = rand_int()
id2 = 'Consumer CB2'
name2 = 'consumer-cb2'
sub_key2 = new_cid()
callback_id2 = rand_int()
consumer_cb1 = Consumer(id1, name1, sub_key=sub_key1, delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
callback_id=callback_id1)
consumer_cb2 = Consumer(id2, name2, sub_key=sub_key2, delivery_mode=PUB_SUB.DELIVERY_MODE.CALLBACK_URL.id,
callback_id=callback_id2)
consumer_pull = Consumer('Consumer pull', 'consumer-pull', sub_key=new_cid(), delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
consumer_inactive = Consumer(
'Consumer pull', 'consumer-pull', is_active=False, sub_key=new_cid(), delivery_mode=PUB_SUB.DELIVERY_MODE.PULL.id)
ps.add_consumer(consumer_cb1, topic)
ps.add_consumer(consumer_cb2, topic)
ps.add_consumer(consumer_pull, topic) # This one should not be returned because it's a pull one
ps.add_consumer(consumer_inactive, topic) # This one should not be returned because it's inactive
consumers = list(ps.get_callback_consumers())
# Only 2 are returned, the rest won't make it
eq_(len(consumers), 2)
# Sort by each consumer's ID, i.e. in lexicographical order
consumers.sort(key=attrgetter('id'))
consumer1 = consumers[0]
eq_(consumer1.id, id1)
eq_(consumer1.name, name1)
eq_(consumer1.is_active, True)
eq_(consumer1.sub_key, sub_key1)
eq_(consumer1.callback_id, callback_id1)
consumer2 = consumers[1]
eq_(consumer2.id, id2)
eq_(consumer2.name, name2)
eq_(consumer2.is_active, True)
eq_(consumer2.sub_key, sub_key2)
eq_(consumer2.callback_id, callback_id2)
def dispatcher_callback(self, event, ctx, **opaque):
self.dispatcher_backlog.append(bunchify({
'event_id': new_cid(),
'event': event,
'ctx': ctx,
'opaque': opaque
}))
def get_data(self, data_format, transport, add_string=NON_ASCII_STRING, needs_payload=True,
payload='', service_class=DummyAdminService):
bmh = channel._BaseMessageHandler()
expected = {
'key': 'a' + uuid4().hex + add_string,
'value': uuid4().hex + NON_ASCII_STRING,
'result': uuid4().hex,
'details': uuid4().hex,
'cid': new_cid(),
}
if needs_payload:
if not payload:
if data_format == SIMPLE_IO.FORMAT.JSON:
payload_value = {expected['key']: expected['value']}
else:
# str.format can't handle Unicode arguments http://bugs.python.org/issue7300
payload_value = '<%(key)s>%(value)s</%(key)s>' % (expected)
payload = DummyPayload(payload_value)
else:
payload = None
response = DummyResponse(payload, expected['result'], expected['details'])
service = service_class(response, expected['cid'])
bmh.set_payload(response, data_format, transport, service)
return expected, service
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
status_code = rand_int()
service_name = rand_string()
service_response_name = '{}_response'.format(service_name)
service_response_payload = {'service_id':5207, 'has_wsdl':True}
service_response_dict = {'zato_service_has_wsdl_response':service_response_payload}
service_response = dumps(service_response_dict).encode('base64')
text = dumps({
'zato_env':{'result':ZATO_OK, 'details':''},
service_response_name: {
'response':service_response
}
})
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(service_name, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), service_response_payload.items())
eq_(response.has_data, True)
eq_(response.cid, cid)
def _invoke_callbacks(self, target, target_type, delivery, target_ok, in_doubt, invoker):
""" Asynchronously notifies all callback services of the outcome of the target's invocation.
"""
callback_list = delivery.definition.callback_list
callback_list = callback_list.split(',') or []
payload = dumps({
'target_ok': target_ok,
'in_doubt': in_doubt,
'task_id': delivery.task_id,
'target': target,
'target_type': target_type,
'invoker': invoker
})
for service in callback_list:
if service:
broker_msg = {}
broker_msg['action'] = SERVICE.PUBLISH
broker_msg['task_id'] = delivery.task_id
broker_msg['channel'] = CHANNEL.DELIVERY
broker_msg['data_format'] = DATA_FORMAT.JSON
broker_msg['service'] = service
broker_msg['payload'] = payload
broker_msg['cid'] = new_cid()
try:
self.broker_client.invoke_async(broker_msg)
except Exception, e:
msg = 'Could not invoke callback:[%s], task_id:[%s], e:[%s]'.format(
service, delivery.task_id, format_exc(e))
self.logger.warn(msg)
def get_data(self, data_format, transport, add_string=NON_ASCII_STRING, needs_payload=True,
payload='', service_class=DummyAdminService):
handler = channel.RequestHandler(get_dummy_server())
expected = {
'key': 'a' + uuid4().hex + add_string,
'value': uuid4().hex + NON_ASCII_STRING,
'result': uuid4().hex,
'details': uuid4().hex,
'cid': new_cid(),
'zato':zato_namespace
}
if needs_payload:
if not payload:
if data_format == SIMPLE_IO.FORMAT.JSON:
payload_value = {expected['key']: expected['value']}
else:
# NOTE: str.format can't handle Unicode arguments http://bugs.python.org/issue7300
payload_value = """<%(key)s xmlns="%(zato)s">%(value)s<zato_env>
<cid>%(cid)s</cid>
<result>%(result)s</result>
<details>%(details)s</details>
</zato_env>
</%(key)s>""" % (expected)
payload = DummyPayload(payload_value)
else:
payload = None
response = DummyResponse(payload, expected['result'], expected['details'])
service = service_class(response, expected['cid'])
handler.set_payload(response, data_format, transport, service)
return expected, service
def _on_job_execution(self, name, service, extra, broker_msg_type,
job_type):
""" Invoked by the underlying APScheduler when a job is executed. Sends
the actual execution request to the broker so it can be picked up by
one of the parallel server's broker clients.
"""
msg = {
'action': SCHEDULER.JOB_EXECUTED,
'name': name,
'service': service,
'payload': extra,
'cid': new_cid(),
'job_type': job_type
}
# Special case an internal job that needs to be delivered to all parallel
# servers.
if name == ENSURE_SINGLETON_JOB:
self.singleton.broker_client.publish(msg)
else:
self.singleton.broker_client.invoke_async(msg)
if logger.isEnabledFor(logging.DEBUG):
msg = 'Sent a job execution request, name [{0}], service [{1}], extra [{2}]'.format(
name, service, extra)
logger.debug(msg)
def on_broker_msg_HOT_DEPLOY_CREATE(self, msg, *args):
msg.cid = new_cid()
msg.service = 'zato.hot-deploy.create'
msg.payload = {'package_id': msg.package_id}
msg.data_format = SIMPLE_IO.FORMAT.JSON
return self._on_message_invoke_service(msg, 'hot-deploy',
'HOT_DEPLOY_CREATE', args)
def test_invoke_retry_exception_has_async(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback, callback_impl_name, cid, expected_result, raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke_retry(target, 1, 2, 3, **kwargs)
except NeedsRetry, e:
self.assertEquals(e.cid, cid)
self.assertEquals(e.cid, e.inner_exc.message)
def dispatcher_callback(self, event, ctx, **opaque):
self.dispatcher_backlog.append(bunchify({
'event_id': new_cid(),
'event': event,
'ctx': ctx,
'opaque': opaque
}))
def __init__(
self,
payload="",
topic=None,
mime_type=PUB_SUB.DEFAULT_MIME_TYPE,
priority=PUB_SUB.DEFAULT_PRIORITY,
expiration=PUB_SUB.DEFAULT_EXPIRATION,
msg_id=None,
producer=None,
creation_time_utc=None,
expire_at_utc=None,
):
self.payload = payload
self.topic = topic
self.mime_type = mime_type
self.priority = priority # In 1-9 range where 9 is top priority
self.msg_id = msg_id or new_cid()
self.producer = producer
self.expiration = expiration
self.creation_time_utc = creation_time_utc or datetime.utcnow()
self.expire_at_utc = expire_at_utc or (self.creation_time_utc + timedelta(seconds=self.expiration))
# These two, in local timezone, are used by web-admin.
self.creation_time = None
self.expire_at = None
self.id = None # Used by frontend only
self.payload_html = None # Used by frontend only
def __init__(self,
payload='',
topic=None,
mime_type=PUB_SUB.DEFAULT_MIME_TYPE,
priority=PUB_SUB.DEFAULT_PRIORITY,
expiration=PUB_SUB.DEFAULT_EXPIRATION,
msg_id=None,
producer=None,
creation_time_utc=None,
expire_at_utc=None):
self.payload = payload
self.topic = topic
self.mime_type = mime_type
self.priority = priority # In 1-9 range where 9 is top priority
self.msg_id = msg_id or new_cid()
self.producer = producer
self.expiration = expiration
self.creation_time_utc = creation_time_utc or datetime.utcnow()
self.expire_at_utc = expire_at_utc or (
self.creation_time_utc + timedelta(seconds=self.expiration))
# These two, in local timezone, are used by web-admin.
self.creation_time = None
self.expire_at = None
self.id = None # Used by frontend only
self.payload_html = None # Used by frontend only
def __init__(self, kvdb, client_type, topic_callbacks):
Thread.__init__(self)
self.kvdb = kvdb
self.decrypt_func = kvdb.decrypt_func
self.name = '{}-{}'.format(client_type, new_cid())
self.topic_callbacks = topic_callbacks
self._to_parallel_any_topic = TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]
def _received_message(self,
data,
_now=datetime.utcnow,
_default_data='111',
*args,
**kwargs):
try:
request = self._parse_func(data or _default_data)
cid = new_cid()
now = _now()
logger.info('Request received cid:`%s`', cid)
# If client is authenticated we allow either for it to re-authenticate, which grants a new token, or to invoke a service.
# Otherwise, authentication is required.
if self.is_authenticated:
self.handle_invoke_service(
cid, request
) if not request.has_credentials else self.handle_authenticate(
request)
else:
self.handle_authenticate(request)
logger.info('Response returned cid:`%s`, time:`%s`', cid,
_now() - now)
except Exception, e:
logger.warn(format_exc(e))
def test_invoke_retry_ok(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback, callback_impl_name,
cid, expected_result)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
}
result = ir.invoke_retry(target, 1, 2, 3, **kwargs)
self.assertEquals(expected_result, result)
self.assertTrue(len(invoking_service.invoke_args), 2)
self.assertEquals(invoking_service.invoke_args, (target, 1, 2, 3))
self.assertEquals(invoking_service.invoke_kwargs, {})
def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs):
self.kvdb = kvdb
self.decrypt_func = kvdb.decrypt_func
self.name = '{}-{}'.format(client_type, new_cid())
self.topic_callbacks = topic_callbacks
self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs)
self.ready = False
def test_invoke_retry_exception_no_async(self):
class Sleep(object):
def __init__(self):
self.times_called = 0
self.retry_seconds = []
def __call__(self, retry_seconds):
self.times_called += 1
self.retry_seconds.append(retry_seconds)
sleep = Sleep()
with patch('zato.server.pattern.invoke_retry.sleep', sleep):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback,
callback_impl_name,
cid,
expected_result,
raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': False,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke(target, 1, 2, 3, **kwargs)
except ZatoException as e:
expected_msg = retry_limit_reached_msg(kwargs_copy['repeats'],
target,
kwargs_copy['seconds'],
invoking_service.cid)
self.assertEquals(e.cid, cid)
self.assertEquals(e.message, expected_msg)
self.assertEquals(invoking_service.invoke_called_times,
kwargs_copy['repeats'])
self.assertEquals(sleep.times_called,
kwargs_copy['repeats'] - 1)
self.assertEquals(sleep.retry_seconds,
[kwargs_copy['seconds']] *
(kwargs_copy['repeats'] - 1))
else:
self.fail('Expected a ZatoException')
def deliver_pubsub_msg(self, sub_key, msg):
""" Delivers one or more pub/sub messages to the connected WSX client.
"""
ctx = {}
if isinstance(msg, PubSubMessage):
len_msg = 1
else:
len_msg = len(msg)
msg = msg[0] if len_msg == 1 else msg
# A list of messages is given on input so we need to serialize each of them individually
if isinstance(msg, list):
cid = new_cid()
data = []
for elem in msg:
data.append(elem.serialized if elem.serialized else elem.to_external_dict())
if elem.reply_to_sk:
ctx_reply_to_sk = ctx.setdefault('', [])
ctx_reply_to_sk.append(elem.reply_to_sk)
# A single message was given on input
else:
cid = msg.pub_msg_id
data = msg.serialized if msg.serialized else msg.to_external_dict()
if msg.reply_to_sk:
ctx['reply_to_sk'] = msg.reply_to_sk
logger.info('Delivering %d pub/sub message{} to sub_key `%s` (ctx:%s)'.format('s' if len_msg > 1 else ''),
len_msg, sub_key, ctx)
self.invoke_client(cid, data, ctx=ctx, _Class=InvokeClientPubSubRequest)
def handle(self):
input = self.request.input
self._validate_input(input)
with closing(self.odb.session()) as session:
try:
# Find a topic by its name so it can be paired with client_id later on
topic = session.query(PubSubTopic).\
filter(PubSubTopic.cluster_id==input.cluster_id).\
filter(PubSubTopic.name==input.topic_name).\
one()
callback = self._get_callback(session, input)
sub_key = new_cid()
consumer = PubSubConsumer(
None, input.is_active, sub_key, input.max_backlog, input.delivery_mode, callback[0],
callback[2], topic.id, input.client_id, input.cluster_id)
session.add(consumer)
session.commit()
except Exception, e:
msg = 'Could not create a consumer, e:`{}`'.format(format_exc(e))
self.logger.error(msg)
session.rollback()
raise
else:
def handle(self):
input = self.request.input
self._validate_input(input)
with closing(self.odb.session()) as session:
try:
# Find a topic by its name so it can be paired with client_id later on
topic = session.query(PubSubTopic).\
filter(PubSubTopic.cluster_id==input.cluster_id).\
filter(PubSubTopic.name==input.topic_name).\
one()
callback = self._get_callback(session, input)
sub_key = new_cid()
consumer = PubSubConsumer(None, input.is_active, sub_key,
input.max_depth, input.delivery_mode,
callback[0], callback[2], topic.id,
input.client_id, input.cluster_id)
session.add(consumer)
session.commit()
except Exception, e:
msg = 'Could not create a consumer, e:`{}`'.format(
format_exc(e))
self.logger.error(msg)
session.rollback()
raise
else:
def _invoke_callbacks(self, target, target_type, delivery, target_ok, in_doubt, invoker):
""" Asynchronously notifies all callback services of the outcome of the target's invocation.
"""
callback_list = delivery.definition.callback_list
callback_list = callback_list.split(',') or []
payload = dumps({
'target_ok': target_ok,
'in_doubt': in_doubt,
'task_id': delivery.task_id,
'target': target,
'target_type': target_type,
'invoker': invoker
})
for service in callback_list:
if service:
broker_msg = {}
broker_msg['action'] = SERVICE.PUBLISH.value
broker_msg['task_id'] = delivery.task_id
broker_msg['channel'] = CHANNEL.DELIVERY
broker_msg['data_format'] = DATA_FORMAT.JSON
broker_msg['service'] = service
broker_msg['payload'] = payload
broker_msg['cid'] = new_cid()
try:
self.broker_client.invoke_async(broker_msg)
except Exception, e:
msg = 'Could not invoke callback:[%s], task_id:[%s], e:[%s]'.format(
service, delivery.task_id, format_exc(e))
self.logger.warn(msg)
def test_client_ok(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
_rand = rand_string()
soap_action = rand_string()
text = """
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
<soapenv:Body>
<abc>{}</abc>
</soapenv:Body>
</soapenv:Envelope>""".format(_rand).strip()
status_code = rand_int()
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action)
expected_response_data = """
<abc xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">{}</abc>
""".format(_rand).strip()
eq_(response.details, None)
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(etree.tostring(response.data), expected_response_data)
eq_(response.has_data, True)
eq_(response.cid, cid)
def test_invoke_retry_exception_has_async(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback,
callback_impl_name,
cid,
expected_result,
raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
try:
ir.invoke_retry(target, 1, 2, 3, **kwargs)
except NeedsRetry, e:
self.assertEquals(e.cid, cid)
self.assertEquals(e.cid, e.inner_exc.message)
def test_repr(self):
class MyResponse(_Response):
def init(self):
pass
cid = new_cid()
ok = True
text = rand_string()
status_code = rand_int()
inner_params = ({'x-zato-cid':cid}, ok, text, status_code)
max_repr = ((3,3), (len(text), CID_NO_CLIP))
for(max_response_repr, max_cid_repr) in max_repr:
inner = FakeInnerResponse(*inner_params)
response = MyResponse(inner, False, max_response_repr, max_cid_repr, None)
response.ok = ok
cid_ellipsis = '' if max_cid_repr == CID_NO_CLIP else '..'
expected = 'ok:[{}] inner.status_code:[{}] cid:[{}{}{}], inner.text:[{}]>'.format(
ok, status_code, cid[:max_cid_repr], cid_ellipsis, cid[-max_cid_repr:], text[:max_response_repr])
eq_(repr(response).endswith(expected), True)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
env = {
'details': rand_string(),
'result': ZATO_OK,
'cid': cid
}
sio_payload_key = rand_string()
sio_payload = {rand_string(): rand_string()}
sio_response = {
'zato_env': env,
sio_payload_key: sio_payload
}
text = dumps(sio_response)
status_code = rand_int()
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), sio_response[sio_payload_key].items())
eq_(response.has_data, True)
eq_(response.cid, cid)
eq_(response.cid, sio_response['zato_env']['cid'])
eq_(response.details, sio_response['zato_env']['details'])
def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs):
self.kvdb = kvdb
self.decrypt_func = kvdb.decrypt_func
self.name = '{}-{}'.format(client_type, new_cid())
self.topic_callbacks = topic_callbacks
self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs)
self.ready = False
def _deploy_file(self, current_work_dir, payload, file_name):
f = open(file_name, 'w')
f.write(
payload.decode('utf8') if isinstance(payload, bytes) else payload)
f.close()
services_deployed = []
info = self.server.service_store.import_services_from_anywhere(
file_name, current_work_dir)
for service in info.to_process: # type: InRAMService
service_id = self.server.service_store.impl_name_to_id[
service.impl_name]
services_deployed.append(service_id)
msg = {}
msg['cid'] = new_cid()
msg['service_id'] = service_id
msg['service_name'] = service.name
msg['service_impl_name'] = service.impl_name
msg['action'] = HOT_DEPLOY.AFTER_DEPLOY.value
self.broker_client.publish(msg)
return services_deployed
def test_handle_security_apikey(self):
username, password = uuid4().hex, uuid4().hex
ud = url_data.URLData(None, [])
cid = new_cid()
sec_def = Bunch(username=username, password=password)
path_info = '/'
body = ''
# No header at that point
wsgi_environ = {}
try:
ud._handle_security_apikey(cid, sec_def, path_info, body, wsgi_environ)
except Unauthorized:
pass
else:
self.fail('No header sent, expected Unauthorized')
# Correct header name but invalid key
wsgi_environ[username] = uuid4().hex
try:
ud._handle_security_apikey(cid, sec_def, path_info, body, wsgi_environ)
except Unauthorized:
pass
else:
self.fail('Invalid key, expected Unauthorized')
# Both header and key are valid, not exception at this point
wsgi_environ[username] = password
ud._handle_security_apikey(cid, sec_def, path_info, body, wsgi_environ)
def invoke_async(self,
name,
payload='',
channel=CHANNEL.INVOKE_ASYNC,
data_format=None,
transport=None,
expiration=BROKER.DEFAULT_EXPIRATION,
to_json_string=False):
""" Invokes a service asynchronously by its name.
"""
if to_json_string:
payload = dumps(payload)
cid = new_cid()
msg = {}
msg['action'] = SERVICE.PUBLISH
msg['service'] = name
msg['payload'] = payload
msg['cid'] = cid
msg['channel'] = channel
msg['data_format'] = data_format
msg['transport'] = transport
self.broker_client.invoke_async(msg, expiration=expiration)
return cid
def test_repr(self):
class MyResponse(_Response):
def init(self):
pass
cid = new_cid()
ok = True
text = rand_string()
status_code = rand_int()
inner_params = ({'x-zato-cid': cid}, ok, text, status_code)
max_repr = ((3, 3), (len(text), CID_NO_CLIP))
for (max_response_repr, max_cid_repr) in max_repr:
inner = FakeInnerResponse(*inner_params)
response = MyResponse(inner, False, max_response_repr,
max_cid_repr, None)
response.ok = ok
cid_ellipsis = '' if max_cid_repr == CID_NO_CLIP else '..'
expected = 'ok:[{}] inner.status_code:[{}] cid:[{}{}{}], inner.text:[{}]>'.format(
ok, status_code, cid[:max_cid_repr], cid_ellipsis,
cid[-max_cid_repr:], text[:max_response_repr])
eq_(repr(response).endswith(expected), True)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
env = {'details': rand_string(), 'result': ZATO_OK, 'cid': cid}
sio_payload_key = rand_string()
sio_payload = {rand_string(): rand_string()}
sio_response = {'zato_env': env, sio_payload_key: sio_payload}
text = dumps(sio_response)
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), sio_response[sio_payload_key].items())
eq_(response.has_data, True)
eq_(response.cid, cid)
eq_(response.cid, sio_response['zato_env']['cid'])
eq_(response.details, sio_response['zato_env']['details'])
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
status_code = rand_int()
service_name = rand_string()
service_response_name = '{}_response'.format(service_name)
service_response_payload = {'service_id': 5207, 'has_wsdl': True}
service_response_dict = {
'zato_service_has_wsdl_response': service_response_payload
}
service_response = dumps(service_response_dict).encode('base64')
text = dumps({
'zato_env': {
'result': ZATO_OK,
'details': ''
},
service_response_name: {
'response': service_response
}
})
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(service_name)
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), service_response_payload.items())
eq_(response.has_data, True)
eq_(response.cid, cid)
def test_client_ok(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
_rand = rand_string()
soap_action = rand_string()
text = """
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
<soapenv:Body>
<abc>{}</abc>
</soapenv:Body>
</soapenv:Envelope>""".format(_rand).strip()
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action)
expected_response_data = """
<abc xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">{}</abc>
""".format(_rand).strip()
eq_(response.details, None)
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(etree.tostring(response.data), expected_response_data)
eq_(response.has_data, True)
eq_(response.cid, cid)
def get_data(self, data_format, transport, add_string=NON_ASCII_STRING, needs_payload=True,
payload='', service_class=DummyAdminService):
handler = channel.RequestHandler(get_dummy_server())
expected = {
'key': 'a' + uuid4().hex + add_string,
'value': uuid4().hex + NON_ASCII_STRING,
'result': uuid4().hex,
'details': uuid4().hex,
'cid': new_cid(),
'zato':zato_namespace
}
if needs_payload:
if not payload:
if data_format == SIMPLE_IO.FORMAT.JSON:
payload_value = {expected['key']: expected['value']}
else:
# NOTE: str.format can't handle Unicode arguments http://bugs.python.org/issue7300
payload_value = """<%(key)s xmlns="%(zato)s">%(value)s<zato_env>
<cid>%(cid)s</cid>
<result>%(result)s</result>
<details>%(details)s</details>
</zato_env>
</%(key)s>""" % (expected)
payload = DummyPayload(payload_value)
else:
payload = None
response = DummyResponse(payload, expected['result'], expected['details'])
service = service_class(response, expected['cid'])
handler.set_payload(response, data_format, transport, service)
return expected, service
def invoke_service(self, service_name, data, cid=None, needs_response=True, _channel=CHANNEL.WEB_SOCKET,
_data_format=DATA_FORMAT.DICT, serialize=False):
# It is possible that this method will be invoked before self.__init__ completes,
# because self's parent manages the underlying TCP stream, in which can self
# will not be fully initialized yet so we need to wait a bit until it is.
while not self._initialized:
sleep(0.1)
return self.config.on_message_callback({
'cid': cid or new_cid(),
'data_format': _data_format,
'service': service_name,
'payload': data,
'environ': {
'web_socket': self,
'sql_ws_client_id': self.sql_ws_client_id,
'ws_channel_config': self.config,
'ws_token': self.token,
'ext_token': self.ext_token,
'pub_client_id': self.pub_client_id,
'ext_client_id': self.ext_client_id,
'ext_client_name': self.ext_client_name,
'peer_conn_info_pretty': self.peer_conn_info_pretty,
'connection_time': self.connection_time,
'pings_missed': self.pings_missed,
'pings_missed_threshold': self.pings_missed_threshold,
'peer_host': self._peer_host,
'peer_fqdn': self._peer_fqdn,
'forwarded_for': self.forwarded_for,
'forwarded_for_fqdn': self.forwarded_for_fqdn,
'initial_http_wsgi_environ': self.initial_http_wsgi_environ,
},
}, CHANNEL.WEB_SOCKET, None, needs_response=needs_response, serialize=serialize)
def test_invoke_retry_ok(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback, callback_impl_name, cid, expected_result)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
'cid': cid,
}
result = ir.invoke(target, 1, 2, 3, **kwargs)
self.assertEquals(expected_result, result)
self.assertTrue(len(invoking_service.invoke_args), 2)
self.assertEquals(invoking_service.invoke_args, (target, 1, 2, 3))
self.assertEquals(invoking_service.invoke_kwargs, {'cid':cid})
def __init__(self, id_prefix, in_reply_to, status=OK, error_message=''):
self.id = '{}.{}'.format(id_prefix, new_cid())
self.in_reply_to = in_reply_to
self.meta = Bunch(id=self.id, in_reply_to=in_reply_to, status=status)
self.data = Bunch()
if error_message:
self.meta.error_message = error_message
def on_forbidden(self, action, data=copy_forbidden):
cid = new_cid()
logger.warn(
'Peer %s (%s) %s, closing its connection to %s (%s), cid:`%s` (%s)', self._peer_address, self._peer_fqdn, action,
self._local_address, self.config.name, cid, self.peer_conn_info_pretty)
self.send(Forbidden(cid, data).serialize())
self.server_terminated = True
self.client_terminated = True
def invoke(self, target, *args, **kwargs):
async_fallback, callback, callback_context, retry_repeats, retry_seconds, kwargs = self._get_retry_settings(
target, **kwargs)
# Let's invoke the service and find out if it works, maybe we don't need
# to retry anything.
kwargs['cid'] = kwargs.get('cid', new_cid())
try:
result = self.invoking_service.invoke(target, *args, **kwargs)
except Exception as e:
msg = 'Could not invoke:`{}`, cid:`{}`, e:`{}`'.format(
target, self.invoking_service.cid, format_exc())
logger.warn(msg)
# How we handle the exception depends on whether the caller wants us
# to block or prefers if we retry in background.
if async_fallback:
# .. invoke the background service and return CID to the caller.
return self._invoke_async_retry(target, retry_repeats,
retry_seconds,
self.invoking_service.cid,
kwargs['cid'], callback,
callback_context, args, kwargs)
# We are to block while repeating
else:
# Repeat the given number of times sleeping for as many seconds as we are told
remaining = retry_repeats
result = None
while remaining > 1:
try:
result = self.invoking_service.invoke(
target, *args, **kwargs)
except Exception as e:
msg = retry_failed_msg(
(retry_repeats - remaining) + 1, retry_repeats,
target, retry_seconds, self.invoking_service.cid,
e)
logger.info(msg)
sleep(retry_seconds)
remaining -= 1
# OK, give up now, there's nothing more we can do
if not result:
msg = retry_limit_reached_msg(retry_repeats, target,
retry_seconds,
self.invoking_service.cid)
raise ZatoException(self.invoking_service.cid, msg)
else:
# All good, simply return the response
return result
def _add_workers(self, service, n):
""" Adds n workers to a service
"""
logger.info(
'ZeroMQ MDP channel `%s` adding %s worker{} for `%s`'.format(
'' if n == 1 else 's'), self.config.name, n, service.name)
for x in range(n):
self._add_worker('mdp.{}'.format(new_cid()), service.name,
self.y100, const.worker_type.zato)
def setUp(self):
self.key_prefix = 'zato:pubsub:{}:'.format(new_cid())
self.kvdb = Redis()
try:
self.kvdb.ping()
except ConnectionError:
self.has_redis = False
else:
self.has_redis = True
def authenticate(self, request):
if self.config.auth_func(request.cid, request.username,
request.password, self.config.sec_name):
with self.update_lock:
self.token = 'ws.token.{}'.format(new_cid())
self.is_authenticated = True
return AuthenticateResponse(self.token.value,
request.id).serialize()
def setUp(self):
self.key_prefix = 'zato:pubsub:{}:'.format(new_cid())
self.kvdb = Redis()
try:
self.kvdb.ping()
except ConnectionError:
self.has_redis = False
else:
self.has_redis = True
def invoke_async(self, name, payload='', channel=CHANNEL.INVOKE_ASYNC, data_format=DATA_FORMAT.DICT,
transport=None, expiration=BROKER.DEFAULT_EXPIRATION, to_json_string=False, cid=None, callback=None,
zato_ctx={}, environ={}):
""" Invokes a service asynchronously by its name.
"""
if self.component_enabled_target_matcher:
name, target = self.extract_target(name)
zato_ctx['zato.request_ctx.target'] = target
else:
target = None
# Let's first find out if the service can be invoked at all
impl_name = self.server.service_store.name_to_impl_name[name]
if self.component_enabled_invoke_matcher:
if not self._worker_store.invoke_matcher.is_allowed(impl_name):
raise ZatoException(self.cid, 'Service `{}` (impl_name) cannot be invoked'.format(impl_name))
if to_json_string:
payload = dumps(payload)
cid = cid or new_cid()
# If there is any callback at all, we need to figure out its name because that's how it will be invoked by.
if callback:
# The same service
if callback is self:
callback = self.name
else:
sink = '{}-async-callback'.format(self.name)
if sink in self.server.service_store.name_to_impl_name:
callback = sink
# Otherwise the callback must be a string pointing to the actual service to reply to so we don't need to do anything.
msg = {}
msg['action'] = SERVICE.PUBLISH.value
msg['service'] = name
msg['payload'] = payload
msg['cid'] = cid
msg['channel'] = channel
msg['data_format'] = data_format
msg['transport'] = transport
msg['is_async'] = True
msg['callback'] = callback
msg['zato_ctx'] = zato_ctx
msg['environ'] = environ
# If we have a target we need to invoke all the servers
# and these which are not able to handle the target will drop the message.
(self.broker_client.publish if target else self.broker_client.invoke_async)(msg, expiration=expiration)
return cid
def send(self, msg):
msg['msg_type'] = MESSAGE_TYPE.TO_PARALLEL_ANY
msg = dumps(msg)
topic = TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]
key = broker_msg = b'zato:broker:to-parallel:any:{}'.format(new_cid())
self.kvdb.conn.set(key, str(msg))
self.kvdb.conn.expire(key, 15) # In seconds, TODO: Document it and make configurable
self.pub_client.publish(topic, broker_msg)
def _invoke_service(self, service, request):
return self.on_message_callback(
{
'cid': new_cid(),
'service': service,
'channel': self.zato_channel,
'payload': request
},
self.zato_channel,
None,
needs_response=True)
def invoke_async(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ANY, expiration=BROKER.DEFAULT_EXPIRATION):
msg['msg_type'] = msg_type
msg = dumps(msg)
topic = TOPICS[msg_type]
key = broker_msg = b'zato:broker{}:{}'.format(KEYS[msg_type], new_cid())
self.kvdb.conn.set(key, str(msg))
self.kvdb.conn.expire(key, expiration) # In seconds
self.pub_client.publish(topic, broker_msg)
def _on_message(self, msg, args):
""" Invoked for each message taken off a ZMQ socket.
"""
with self.channel_lock:
params = {}
params['action'] = CHANNEL.ZMQ_MESSAGE_RECEIVED
params['service'] = self.channel.service
params['cid'] = new_cid()
params['payload'] = msg
params['data_format'] = self.channel.data_format
self.broker_client.invoke_async(params)
def subscribe(self, ctx, sub_key=None):
""" Subscribes the client to one or more topics, or topic patterns. Returns subscription key
to use in subsequent calls to fetch messages by.
"""
sub_key = sub_key or new_cid()
with self.update_lock:
for topic in ctx.topics:
# TODO: Resolve topic here - it can be a pattern instead of a concrete name
self.add_subscription(sub_key, ctx.client_id, topic)
self.logger.info('Client `%s` sub to topics `%s`', ctx.client_id, ', '.join(ctx.topics))
return sub_key
def deliver(self, def_name, payload, task_id=None, *args, **kwargs):
""" Uses guaranteed delivery to send payload using a delivery definition known by def_name.
*args and **kwargs will be passed directly as-is to the target behind the def_name.
"""
task_id = task_id or new_cid()
self.delivery_store.deliver(
self.server.cluster_id, def_name, payload, task_id, self.invoke,
kwargs.pop('is_resubmit', False),
kwargs.pop('is_auto', False),
*args, **kwargs)
return task_id
def _on_message(self, method_frame, header_frame, body):
""" A callback to be invoked by ConsumingConnection on each new AMQP message.
"""
with self.def_amqp_lock:
with self.channel_amqp_lock:
params = {}
params['action'] = CHANNEL.AMQP_MESSAGE_RECEIVED
params['service'] = self.channel_amqp.service
params['data_format'] = self.channel_amqp.data_format
params['cid'] = new_cid()
params['payload'] = body
self.broker_client.invoke_async(params)
def invoke_async(self, name, payload='', channel=CHANNEL.INVOKE_ASYNC, data_format=DATA_FORMAT.DICT,
transport=None, expiration=BROKER.DEFAULT_EXPIRATION, to_json_string=False, cid=None, callback=None,
zato_ctx={}, environ={}):
""" Invokes a service asynchronously by its name.
"""
name, target = self.extract_target(name)
zato_ctx['zato.request_ctx.target'] = target
# Let's first find out if the service can be invoked at all
impl_name = self.server.service_store.name_to_impl_name[name]
if not self.worker_store.invoke_matcher.is_allowed(impl_name):
raise ZatoException(self.cid, 'Service `{}` (impl_name) cannot be invoked'.format(impl_name))
if to_json_string:
payload = dumps(payload)
cid = cid or new_cid()
# If there is any callback at all, we need to figure out its name because that's how it will be invoked by.
if callback:
# The same service
if callback is self:
callback = self.name
else:
sink = '{}-async-callback'.format(self.name)
if sink in self.server.service_store.name_to_impl_name:
callback = sink
# Otherwise the callback must be a string pointing to the actual service to reply to so we don't need to do anything.
msg = {}
msg['action'] = SERVICE.PUBLISH.value
msg['service'] = name
msg['payload'] = payload
msg['cid'] = cid
msg['channel'] = channel
msg['data_format'] = data_format
msg['transport'] = transport
msg['is_async'] = True
msg['callback'] = callback
msg['zato_ctx'] = zato_ctx
msg['environ'] = environ
# If we have a target we need to invoke all the servers
# and these which are not able to handle the target will drop the message.
(self.broker_client.publish if target else self.broker_client.invoke_async)(msg, expiration=expiration)
return cid
def invoke(self, class_, request_data, expected, mock_data={}, channel=CHANNEL.HTTP_SOAP, job_type=None,
data_format=DATA_FORMAT.JSON, service_store_name_to_impl_name=None, service_store_impl_name_to_service=None):
""" Sets up a service's invocation environment, then invokes and returns
an instance of the service.
"""
instance = class_()
worker_store = MagicMock()
worker_store.worker_config = MagicMock
worker_store.worker_config.outgoing_connections = MagicMock(return_value=(None, None, None, None))
worker_store.worker_config.cloud_openstack_swift = MagicMock(return_value=None)
worker_store.worker_config.cloud_aws_s3 = MagicMock(return_value=None)
worker_store.invoke_matcher.is_allowed = MagicMock(return_value=True)
simple_io_config = {
'int_parameters': SIMPLE_IO.INT_PARAMETERS.VALUES,
'int_parameter_suffixes': SIMPLE_IO.INT_PARAMETERS.SUFFIXES,
'bool_parameter_prefixes': SIMPLE_IO.BOOL_PARAMETERS.SUFFIXES,
}
class_.update(
instance, channel, FakeServer(service_store_name_to_impl_name, service_store_impl_name_to_service),
None, worker_store, new_cid(), request_data, request_data, simple_io_config=simple_io_config,
data_format=data_format, job_type=job_type)
def get_data(self, *ignored_args, **ignored_kwargs):
return expected.get_data()
instance.get_data = get_data
for attr_name, mock_path_data_list in mock_data.iteritems():
setattr(instance, attr_name, Mock())
attr = getattr(instance, attr_name)
for mock_path_data in mock_path_data_list:
for path, value in mock_path_data.iteritems():
splitted = path.split('.')
new_path = '.return_value.'.join(elem for elem in splitted) + '.return_value'
attr.configure_mock(**{new_path:value})
broker_client_publish = getattr(self, 'broker_client_publish', None)
if broker_client_publish:
instance.broker_client = FakeBrokerClient()
instance.broker_client.publish = broker_client_publish
instance.call_hooks('before')
instance.handle()
instance.call_hooks('after')
return instance
def test_client_ok(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
status_code = rand_int()
rand_id, rand_name, soap_action = rand_string(), rand_string(), rand_string()
sio_response = """<zato_outgoing_amqp_edit_response xmlns="https://zato.io/ns/20130518">
<zato_env>
<cid>{}</cid>
<result>ZATO_OK</result>
</zato_env>
<item>
<id>{}</id>
<name>crm.account</name>
</item>
</zato_outgoing_amqp_edit_response>
""".format(cid, rand_id, rand_name)
text = """<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns="https://zato.io/ns/20130518">
<soap:Body>
{}
</soap:Body>
</soap:Envelope>""".format(sio_response).strip()
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.has_data, True)
eq_(response.cid, cid)
path_items = (
('zato_env', 'cid'),
('zato_env', 'result'),
('item', 'id'),
('item', 'name'),
)
for items in path_items:
path = '//zato:zato_outgoing_amqp_edit_response/zato:' + '/zato:'.join(items)
xpath = etree.XPath(path, namespaces=common_namespaces)
expected = xpath(etree.fromstring(text))[0].text
actual = xpath(response.data)[0]
self.assertEquals(expected, actual)
def test_client_soap_fault(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = False
status_code = rand_int()
soap_action = rand_string()
text = b"""<?xml version='1.0' encoding='UTF-8'?>
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/"
xmlns:xsi="http://www.w3.org/1999/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/1999/XMLSchema">
<SOAP-ENV:Body>
<SOAP-ENV:Fault>
<faultcode>SOAP-ENV:Client</faultcode>
<faultstring><![CDATA[cid [K68438211212681798524426103126], faultstring
[Traceback (most recent call last):
File
"/opt/zato/code/zato-server/src/zato/server/connection/http_soap/
channel.py", line 126, in dispatch
service_info, response = handler.handle(cid, wsgi_environ, payload, transport,
worker_store, self.simple_io_config, data_format, path_info)
File
"/opt/zato/code/zato-server/src/zato/server/connection/http_soap/
channel.py", line 227, in handle
service_instance.handle()
File
"/opt/zato/code/zato-server/src/zato/server/service/internal/
definition/amqp.py", line 174, in handle
filter(ConnDefAMQP.id==self.request.input.id).\
File
"/opt/zato/code/eggs/SQLAlchemy-0.7.9-py2.7-linux-x86_64.egg/sqlalchemy/
orm/query.py", line 2190, in one
raise orm_exc.NoResultFound("No row was found for one()")
NoResultFound: No row was found for one()
]]]></faultstring>
</SOAP-ENV:Fault>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>"""
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.has_data, False)
eq_(response.cid, cid)
eq_('NoResultFound: No row was found for one()' in response.details.getchildren()[1].text, True)
def publish(self, name, payload, to_json=True):
if to_json:
payload = dumps(payload)
cid = new_cid()
msg = dict()
msg['action'] = SERVICE.PUBLISH
msg['service'] = self.server.service_store.name_to_impl_name[name]
msg['payload'] = payload
msg['cid'] = cid
self.broker_client.send(msg)
return cid
def _on_message(self, msg):
""" Invoked for each message taken off a WebSphere MQ queue.
"""
with self.def_lock:
with self.channel_lock:
params = {}
params['action'] = CHANNEL.JMS_WMQ_MESSAGE_RECEIVED
params['service'] = self.channel.service
params['cid'] = new_cid()
params['payload'] = msg.text
params['data_format'] = self.channel.data_format
for attr in MESSAGE_ATTRS:
params[attr] = getattr(msg, attr, None)
self.broker_client.async_invoke(params)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
text = '<abc>{}</abc>'.format(rand_string())
status_code = rand_int()
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(etree.tostring(response.data), text)
eq_(response.has_data, True)
eq_(response.cid, cid)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid':cid}
ok = True
text = dumps({rand_string(): rand_string()})
status_code = rand_int()
client = self.get_client(FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), loads(text).items())
eq_(response.has_data, True)
eq_(response.cid, cid)
