def test_client_ok(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
_rand = rand_string()
soap_action = rand_string()
text = """
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
<soapenv:Body>
<abc>{}</abc>
</soapenv:Body>
</soapenv:Envelope>""".format(_rand).strip()
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action)
expected_response_data = """
<abc xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">{}</abc>
""".format(_rand).strip()
eq_(response.details, None)
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(etree.tostring(response.data), expected_response_data)
eq_(response.has_data, True)
eq_(response.cid, cid)
def _invoke_service(self, service, request):
return self.on_message_callback({
'cid': new_cid(),
'service': service,
'channel': self.zato_channel,
'payload': request
}, self.zato_channel, None, needs_response=True)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
status_code = rand_int()
service_name = rand_string()
service_response_name = '{}_response'.format(service_name)
service_response_payload = {'service_id': 5207, 'has_wsdl': True}
service_response_dict = {
'zato_service_has_wsdl_response': service_response_payload
}
service_response = b64encode(dumps(service_response_dict))
text = dumps({
'zato_env': {
'result': ZATO_OK,
'details': ''
},
service_response_name: {
'response': service_response
}
})
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(service_name, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), service_response_payload.items())
eq_(response.has_data, True)
eq_(response.cid, cid)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
env = {'details': rand_string(), 'result': ZATO_OK, 'cid': cid}
sio_payload_key = rand_string()
sio_payload = {rand_string(): rand_string()}
sio_response = {'zato_env': env, sio_payload_key: sio_payload}
text = dumps(sio_response)
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(iteritems(response.data), iteritems(
(sio_response[sio_payload_key])))
eq_(response.has_data, True)
eq_(response.cid, cid)
eq_(response.cid, sio_response['zato_env']['cid'])
eq_(response.details, sio_response['zato_env']['details'])
def _deploy_file(self, current_work_dir, payload, file_name):
if PY3:
f = open(file_name, 'w', encoding='utf-8')
else:
f = open(file_name, 'w')
f.write(
payload.decode('utf8') if isinstance(payload, bytes) else payload)
f.close()
services_deployed = []
info = self.server.service_store.import_services_from_anywhere(
file_name, current_work_dir)
for service in info.to_process: # type: InRAMService
service_id = self.server.service_store.impl_name_to_id[
service.impl_name]
services_deployed.append(service_id)
msg = {}
msg['cid'] = new_cid()
msg['service_id'] = service_id
msg['service_name'] = service.name
msg['service_impl_name'] = service.impl_name
msg['action'] = HOT_DEPLOY.AFTER_DEPLOY.value
self.broker_client.publish(msg)
return services_deployed
def test_repr(self):
class MyResponse(_Response):
def init(self):
pass
cid = new_cid()
ok = True
text = rand_string()
status_code = rand_int()
inner_params = ({'x-zato-cid': cid}, ok, text, status_code)
max_repr = ((3, 3), (len(text), CID_NO_CLIP))
for (max_response_repr, max_cid_repr) in max_repr:
inner = FakeInnerResponse(*inner_params)
response = MyResponse(inner, False, max_response_repr,
max_cid_repr, None)
response.ok = ok
cid_ellipsis = '' if max_cid_repr == CID_NO_CLIP else '..'
expected = 'ok:[{}] inner.status_code:[{}] cid:[{}{}{}], inner.text:[{}]>'.format(
ok, status_code, cid[:max_cid_repr], cid_ellipsis,
cid[-max_cid_repr:], text[:max_response_repr])
eq_(repr(response).endswith(expected), True)
def dispatcher_callback(self, event, ctx, **opaque):
self.dispatcher_backlog.append(
bunchify({
'event_id': new_cid(),
'event': event,
'ctx': ctx,
'opaque': opaque
}))
def _add_workers(self, service, n):
""" Adds n workers to a service
"""
logger.info(
'ZeroMQ MDP channel `%s` adding %s worker{} for `%s`'.format('' if n==1 else 's'),
self.config.name, n, service.name)
for x in range(n):
self._add_worker('mdp.{}'.format(new_cid()), service.name, self.y100, const.worker_type.zato)
def __init__(self, kvdb, client_type, topic_callbacks,
initial_lua_programs):
self.kvdb = kvdb
self.decrypt_func = kvdb.decrypt_func
self.name = '{}-{}'.format(client_type, new_cid())
self.topic_callbacks = topic_callbacks
self.lua_container = LuaContainer(self.kvdb.conn,
initial_lua_programs)
self.ready = False
def invoke(self, target, *args, **kwargs):
async_fallback, callback, callback_context, retry_repeats, retry_seconds, kwargs = self._get_retry_settings(
target, **kwargs)
# Let's invoke the service and find out if it works, maybe we don't need
# to retry anything.
kwargs['cid'] = kwargs.get('cid', new_cid())
try:
result = self.invoking_service.invoke(target, *args, **kwargs)
except Exception as e:
msg = 'Could not invoke:`{}`, cid:`{}`, e:`{}`'.format(
target, self.invoking_service.cid, format_exc())
logger.warn(msg)
# How we handle the exception depends on whether the caller wants us
# to block or prefers if we retry in background.
if async_fallback:
# .. invoke the background service and return CID to the caller.
return self._invoke_async_retry(target, retry_repeats,
retry_seconds,
self.invoking_service.cid,
kwargs['cid'], callback,
callback_context, args, kwargs)
# We are to block while repeating
else:
# Repeat the given number of times sleeping for as many seconds as we are told
remaining = retry_repeats
result = None
while remaining > 1:
try:
result = self.invoking_service.invoke(
target, *args, **kwargs)
except Exception as e:
msg = retry_failed_msg(
(retry_repeats - remaining) + 1, retry_repeats,
target, retry_seconds, self.invoking_service.cid,
e)
logger.info(msg)
sleep(retry_seconds)
remaining -= 1
# OK, give up now, there's nothing more we can do
if not result:
msg = retry_limit_reached_msg(retry_repeats, target,
retry_seconds,
self.invoking_service.cid)
raise ZatoException(self.invoking_service.cid, msg)
else:
# All good, simply return the response
return result
def invoke_service(self, class_, request=None, **kwargs):
# type: (Service, object, **object)
class_.name = class_.get_name()
class_.impl_name = class_.get_impl_name()
class_.component_enabled_ibm_mq = True
class_.component_enabled_zeromq = False
class_.component_enabled_sms = True
class_.component_enabled_cassandra = False
class_.component_enabled_email = False
class_.component_enabled_search = False
class_.component_enabled_msg_path = False
class_.component_enabled_patterns = False
class_.has_sio = True
class_._worker_config = self.worker_config
class_._worker_store = self.worker_store
class_.crypto = self.server.crypto_manager
service = class_() # type: Service
service.out.vault = self.vault_conn_api
self.service_store.services[service.impl_name] = {
'slow_threshold': 100,
}
channel = kwargs.get('channel') or CHANNEL.INVOKE
data_format = kwargs.get('data_format') or DATA_FORMAT.DICT
transport = ''
broker_client = None
cid = kwargs.get('cid') or new_cid()
simple_io_config = {'bytes_to_str': {'encoding': 'utf8'}}
response = service.update_handle(
self.request_handler._set_response_data,
service,
request,
channel,
data_format,
transport,
self.server,
broker_client,
self.worker_store,
cid,
simple_io_config,
environ=kwargs.get('environ'))
if kwargs.get('as_bunch'):
if isinstance(response.payload, basestring):
payload = loads(response.payload)
payload = bunchify(payload)
response._payload = payload
return response
def test_client_ok(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
status_code = rand_int()
rand_id, rand_name, soap_action = rand_string(), rand_string(
), rand_string()
sio_response = """<zato_outgoing_amqp_edit_response xmlns="https://zato.io/ns/20130518">
<zato_env>
<cid>{}</cid>
<result>ZATO_OK</result>
</zato_env>
<item>
<id>{}</id>
<name>crm.account</name>
</item>
</zato_outgoing_amqp_edit_response>
""".format(cid, rand_id, rand_name)
text = """<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns="https://zato.io/ns/20130518">
<soap:Body>
{}
</soap:Body>
</soap:Envelope>""".format(sio_response).strip()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.has_data, True)
eq_(response.cid, cid)
path_items = (
('zato_env', 'cid'),
('zato_env', 'result'),
('item', 'id'),
('item', 'name'),
)
for items in path_items:
path = '//zato:zato_outgoing_amqp_edit_response/zato:' + '/zato:'.join(
items)
xpath = etree.XPath(path, namespaces=common_namespaces)
expected = xpath(etree.fromstring(text))[0].text
actual = xpath(response.data)[0]
self.assertEquals(expected, actual)
def test_client_soap_fault(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = False
status_code = rand_int()
soap_action = rand_string()
text = b"""<?xml version='1.0' encoding='UTF-8'?>
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/"
xmlns:xsi="http://www.w3.org/1999/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/1999/XMLSchema">
<SOAP-ENV:Body>
<SOAP-ENV:Fault>
<faultcode>SOAP-ENV:Client</faultcode>
<faultstring><![CDATA[cid [K68438211212681798524426103126], faultstring
[Traceback (most recent call last):
File
"/opt/zato/code/zato-server/src/zato/server/connection/http_soap/
channel.py", line 126, in dispatch
service_info, response = handler.handle(cid, wsgi_environ, payload, transport,
worker_store, self.simple_io_config, data_format, path_info)
File
"/opt/zato/code/zato-server/src/zato/server/connection/http_soap/
channel.py", line 227, in handle
service_instance.handle()
File
"/opt/zato/code/zato-server/src/zato/server/service/internal/
definition/amqp.py", line 174, in handle
filter(ConnDefAMQP.id==self.request.input.id).\
File
"/opt/zato/code/eggs/SQLAlchemy-0.7.9-py2.7-linux-x86_64.egg/sqlalchemy/
orm/query.py", line 2190, in one
raise orm_exc.NoResultFound("No row was found for one()")
NoResultFound: No row was found for one()
]]]></faultstring>
</SOAP-ENV:Fault>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>"""
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.has_data, False)
eq_(response.cid, cid)
eq_(
'NoResultFound: No row was found for one()'
in response.details.getchildren()[1].text, True)
def __init__(self, *args, **kwargs):
super(ZatoWSXClient, self).__init__(*args, **kwargs)
self._zato_client_config = _ZatoWSXConfigImpl()
self._zato_client_config.client_name = 'WSX outconn - {}'.format(self.config.name)
self._zato_client_config.client_id = 'wsx.out.{}'.format(new_cid(8))
self._zato_client_config.address = self.config.address
self._zato_client_config.on_request_callback = self.on_message_cb
self._zato_client_config.on_closed_callback = self.on_close_cb
if self.config.get('username'):
self._zato_client_config.username = self.config.username
self._zato_client_config.secret = self.config.secret
self._zato_client = _ZatoWSXClientImpl(self.opened, self._zato_client_config)
self.invoke = self.send = self._zato_client.invoke
def get_context(self):
ctx = {
'cid': new_cid(),
'start_time': self.start_time.isoformat(),
'cb_kwargs': self.cb_kwargs
}
if self.type == SCHEDULER.JOB_TYPE.CRON_STYLE:
ctx['cron_definition'] = self.cron_definition
else:
ctx['interval_in_seconds'] = self.interval.in_seconds
for name in 'id', 'name', 'current_run', 'max_repeats_reached', 'max_repeats', 'type':
ctx[name] = getattr(self, name)
return ctx
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
text = dumps({rand_string(): rand_string()})
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(iteritems(response.data), iteritems(loads(text)))
eq_(response.has_data, True)
eq_(response.cid, cid)
def __init__(self):
self.action = None
self.service = None
self.sec_type = None
self.username = None
self.password = None
self.id = None
self.timestamp = None
self.cid = new_cid()
self.in_reply_to = None
self.data = Bunch()
self.has_credentials = None
self.token = None
self.ext_client_name = None
self.ext_client_id = None
self.reply_to_sk = None
self.deliver_to_sk = None
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
text = '<abc>{}</abc>'.format(rand_string())
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(etree.tostring(response.data), text)
eq_(response.has_data, True)
eq_(response.cid, cid)
def __init__(self,
publisher_tag,
publisher_pid,
payload='',
request_id=None):
self.publisher_tag = publisher_tag
self.publisher_pid = publisher_pid
self.action = NO_DEFAULT_VALUE
self.service = ''
self._payload = payload
self.payload_type = type(payload).__name__
self.data_format = DATA_FORMAT.DICT
self.request_id = request_id or 'ipc.{}'.format(new_cid())
self.target_pid = None
self.reply_to_tag = ''
self.reply_to_fifo = ''
self.in_reply_to = ''
self.creation_time_utc = datetime.utcnow()
def on_job_executed(self, ctx, extra_data_format=ZATO_NONE):
""" Invoked by the underlying scheduler when a job is executed. Sends the actual execution request to the broker
so it can be picked up by one of the parallel server's broker clients.
"""
name = ctx['name']
payload = ctx['cb_kwargs']['extra']
if isinstance(payload, bytes):
payload = payload.decode('utf8')
msg = {
'action': SCHEDULER_MSG.JOB_EXECUTED.value,
'name': name,
'service': ctx['cb_kwargs']['service'],
'payload': payload,
'cid': ctx['cid'],
'job_type': ctx['type']
}
if extra_data_format != ZATO_NONE:
msg['data_format'] = extra_data_format
self.broker_client.invoke_async(msg)
if _has_debug:
msg = 'Sent a job execution request, name [{}], service [{}], extra [{}]'.format(
name, ctx['cb_kwargs']['service'], ctx['cb_kwargs']['extra'])
logger.debug(msg)
# Now, if it was a one-time job, it needs to be deactivated.
if ctx['type'] == SCHEDULER.JOB_TYPE.ONE_TIME:
msg = {
'action': SERVICE.PUBLISH.value,
'service': 'zato.scheduler.job.set-active-status',
'payload': {
'id': ctx['id'],
'is_active': False
},
'cid': new_cid(),
'channel': CHANNEL.SCHEDULER_AFTER_ONE_TIME,
'data_format': DATA_FORMAT.JSON,
}
self.broker_client.publish(msg)
def send(self,
data,
_socket_socket=socket.socket,
_family=socket.AF_INET,
_type=socket.SOCK_STREAM):
# type: (bytes) -> bytes
try:
data = data if isinstance(data, bytes) else data.encode('utf8')
# Wrap the message in an MLLP envelope
msg = self.start_seq + data + self.end_seq
# This will auto-close the socket ..
with _socket_socket(_family, _type) as sock:
# .. connect to the remote end ..
sock.connect((self.host, self.port))
# .. send our data ..
sock.send(msg)
# .. encapsulate configuration for our socket reader function ..
ctx = SocketReaderCtx(new_cid(), sock, self.max_wait_time,
self.max_msg_size, self.read_buffer_size,
self.recv_timeout,
self.should_log_messages)
# .. wait for the response ..
response = read_from_socket(ctx)
if self.should_log_messages:
logger.info('Response received `%s`', response)
return response
except Exception:
logger.warn(
'Client caught an exception while sending HL7 MLLP data to `%s (%s)`; e:`%s`',
self.name, self.address, format_exc())
raise
def test_client_no_soap_response(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = False
soap_action = rand_string()
text = '<abc/>'
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(soap_action)
eq_(response.ok, ok)
eq_(response.details,
'No /soapenv:Envelope/soapenv:Body/*[1] in SOAP response')
eq_(response.inner.text, text)
eq_(response.has_data, False)
eq_(response.cid, cid)
def invoke_async(self,
msg,
msg_type=MESSAGE_TYPE.TO_PARALLEL_ANY,
expiration=BROKER.DEFAULT_EXPIRATION):
msg['msg_type'] = msg_type
try:
msg = dumps(msg)
except Exception:
error_msg = 'JSON serialization failed for msg:`%r`, e:`%s`'
logger.error(error_msg, msg, format_exc())
raise
else:
topic = TOPICS[msg_type]
key = broker_msg = 'zato:broker{}:{}'.format(
KEYS[msg_type], new_cid())
self.kvdb.conn.set(key, str(msg))
self.kvdb.conn.expire(key, expiration) # In seconds
self.pub_client.publish(topic, broker_msg)
def _invoke_rest_outconn_callback(self, item_id, request):
# type: (str, dict) -> None
cid = new_cid()
item = self.worker_store.get_outconn_rest_by_id(item_id)
ping_response = item.ping(cid, return_response=True,
log_verbose=True) # type: Response
if ping_response.status_code != OK:
logger.warn(
'Could not ping file transfer connection for `%s` (%s); config:`%s`, r:`%s`, h:`%s`',
request['full_path'], request['config'].name, item.config,
ping_response.text, ping_response.headers)
else:
file_name = request['file_name']
mime_type = guess_mime_type(file_name, strict=False)
mime_type = mime_type[0] if mime_type[
0] else 'application/octet-stream'
payload = request['raw_data']
params = {'file_name': file_name, 'mime_type': mime_type}
headers = {
'X-Zato-File-Name': file_name,
'X-Zato-Mime-Type': mime_type,
}
response = item.conn.post(cid, payload, params,
headers=headers) # type: Response
if response.status_code != OK:
logger.warn(
'Could not send file `%s` (%s) to `%s` (p:`%s`, h:`%s`), r:`%s`, h:`%s`',
request['full_path'], request['config'].name, item.config,
params, headers, response.text, response.headers)
def invoke(self,
class_,
request_data,
expected,
mock_data={},
channel=CHANNEL.HTTP_SOAP,
job_type=None,
data_format=DATA_FORMAT.JSON,
service_store_name_to_impl_name=None,
service_store_impl_name_to_service=None):
""" Sets up a service's invocation environment, then invokes and returns
an instance of the service.
"""
class_.component_enabled_cassandra = True
class_.component_enabled_email = True
class_.component_enabled_search = True
class_.component_enabled_msg_path = True
class_.has_sio = getattr(class_, 'SimpleIO', False)
instance = class_()
server = MagicMock()
server.component_enabled.stats = False
worker_store = MagicMock()
worker_store.worker_config = MagicMock
worker_store.worker_config.outgoing_connections = MagicMock(
return_value=(None, None, None, None))
worker_store.worker_config.cloud_openstack_swift = MagicMock(
return_value=None)
worker_store.worker_config.cloud_aws_s3 = MagicMock(return_value=None)
worker_store.invoke_matcher.is_allowed = MagicMock(return_value=True)
simple_io_config = {
'int_parameters': SIMPLE_IO.INT_PARAMETERS.VALUES,
'int_parameter_suffixes': SIMPLE_IO.INT_PARAMETERS.SUFFIXES,
'bool_parameter_prefixes': SIMPLE_IO.BOOL_PARAMETERS.SUFFIXES,
}
class_.update(instance,
channel,
FakeServer(service_store_name_to_impl_name,
service_store_impl_name_to_service,
worker_store),
None,
worker_store,
new_cid(),
request_data,
request_data,
simple_io_config=simple_io_config,
data_format=data_format,
job_type=job_type)
def get_data(self, *ignored_args, **ignored_kwargs):
return expected.get_data()
instance.get_data = get_data
for attr_name, mock_path_data_list in mock_data.items():
setattr(instance, attr_name, Mock())
attr = getattr(instance, attr_name)
for mock_path_data in mock_path_data_list:
for path, value in mock_path_data.items():
split = path.split('.')
new_path = '.return_value.'.join(
elem for elem in split) + '.return_value'
attr.configure_mock(**{new_path: value})
broker_client_publish = getattr(self, 'broker_client_publish', None)
if broker_client_publish:
instance.broker_client = FakeBrokerClient()
instance.broker_client.publish = broker_client_publish
def set_response_func(*args, **kwargs):
pass
instance.handle()
#instance.update_handle(
# set_response_func, instance, request_data, channel, data_format, None, server, None, worker_store, new_cid(),
# None)
return instance
def __init__(self, kvdb, client_type, topic_callbacks):
Thread.__init__(self)
self.kvdb = kvdb
self.decrypt_func = kvdb.decrypt_func
self.name = '{}-{}'.format(client_type, new_cid())
self.topic_callbacks = topic_callbacks
def __init__(self):
self.broker_client_id = '{}-{}'.format(ZATO_NONE, new_cid())
self.broker_callbacks = {}
self.broker_messages = []
