def test_client_ok(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True _rand = rand_string() soap_action = rand_string() text = """ <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"> <soapenv:Body> <abc>{}</abc> </soapenv:Body> </soapenv:Envelope>""".format(_rand).strip() status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(soap_action) expected_response_data = """ <abc xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">{}</abc> """.format(_rand).strip() eq_(response.details, None) eq_(response.ok, ok) eq_(response.inner.text, text) eq_(etree.tostring(response.data), expected_response_data) eq_(response.has_data, True) eq_(response.cid, cid)
def _invoke_service(self, service, request): return self.on_message_callback({ 'cid': new_cid(), 'service': service, 'channel': self.zato_channel, 'payload': request }, self.zato_channel, None, needs_response=True)
def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True status_code = rand_int() service_name = rand_string() service_response_name = '{}_response'.format(service_name) service_response_payload = {'service_id': 5207, 'has_wsdl': True} service_response_dict = { 'zato_service_has_wsdl_response': service_response_payload } service_response = b64encode(dumps(service_response_dict)) text = dumps({ 'zato_env': { 'result': ZATO_OK, 'details': '' }, service_response_name: { 'response': service_response } }) client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(service_name, '') eq_(response.ok, ok) eq_(response.inner.text, text) eq_(response.data.items(), service_response_payload.items()) eq_(response.has_data, True) eq_(response.cid, cid)
def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True env = {'details': rand_string(), 'result': ZATO_OK, 'cid': cid} sio_payload_key = rand_string() sio_payload = {rand_string(): rand_string()} sio_response = {'zato_env': env, sio_payload_key: sio_payload} text = dumps(sio_response) status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke() eq_(response.ok, ok) eq_(response.inner.text, text) eq_(iteritems(response.data), iteritems( (sio_response[sio_payload_key]))) eq_(response.has_data, True) eq_(response.cid, cid) eq_(response.cid, sio_response['zato_env']['cid']) eq_(response.details, sio_response['zato_env']['details'])
def _deploy_file(self, current_work_dir, payload, file_name): if PY3: f = open(file_name, 'w', encoding='utf-8') else: f = open(file_name, 'w') f.write( payload.decode('utf8') if isinstance(payload, bytes) else payload) f.close() services_deployed = [] info = self.server.service_store.import_services_from_anywhere( file_name, current_work_dir) for service in info.to_process: # type: InRAMService service_id = self.server.service_store.impl_name_to_id[ service.impl_name] services_deployed.append(service_id) msg = {} msg['cid'] = new_cid() msg['service_id'] = service_id msg['service_name'] = service.name msg['service_impl_name'] = service.impl_name msg['action'] = HOT_DEPLOY.AFTER_DEPLOY.value self.broker_client.publish(msg) return services_deployed
def test_repr(self): class MyResponse(_Response): def init(self): pass cid = new_cid() ok = True text = rand_string() status_code = rand_int() inner_params = ({'x-zato-cid': cid}, ok, text, status_code) max_repr = ((3, 3), (len(text), CID_NO_CLIP)) for (max_response_repr, max_cid_repr) in max_repr: inner = FakeInnerResponse(*inner_params) response = MyResponse(inner, False, max_response_repr, max_cid_repr, None) response.ok = ok cid_ellipsis = '' if max_cid_repr == CID_NO_CLIP else '..' expected = 'ok:[{}] inner.status_code:[{}] cid:[{}{}{}], inner.text:[{}]>'.format( ok, status_code, cid[:max_cid_repr], cid_ellipsis, cid[-max_cid_repr:], text[:max_response_repr]) eq_(repr(response).endswith(expected), True)
def dispatcher_callback(self, event, ctx, **opaque): self.dispatcher_backlog.append( bunchify({ 'event_id': new_cid(), 'event': event, 'ctx': ctx, 'opaque': opaque }))
def _add_workers(self, service, n): """ Adds n workers to a service """ logger.info( 'ZeroMQ MDP channel `%s` adding %s worker{} for `%s`'.format('' if n==1 else 's'), self.config.name, n, service.name) for x in range(n): self._add_worker('mdp.{}'.format(new_cid()), service.name, self.y100, const.worker_type.zato)
def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs): self.kvdb = kvdb self.decrypt_func = kvdb.decrypt_func self.name = '{}-{}'.format(client_type, new_cid()) self.topic_callbacks = topic_callbacks self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs) self.ready = False
def invoke(self, target, *args, **kwargs): async_fallback, callback, callback_context, retry_repeats, retry_seconds, kwargs = self._get_retry_settings( target, **kwargs) # Let's invoke the service and find out if it works, maybe we don't need # to retry anything. kwargs['cid'] = kwargs.get('cid', new_cid()) try: result = self.invoking_service.invoke(target, *args, **kwargs) except Exception as e: msg = 'Could not invoke:`{}`, cid:`{}`, e:`{}`'.format( target, self.invoking_service.cid, format_exc()) logger.warn(msg) # How we handle the exception depends on whether the caller wants us # to block or prefers if we retry in background. if async_fallback: # .. invoke the background service and return CID to the caller. return self._invoke_async_retry(target, retry_repeats, retry_seconds, self.invoking_service.cid, kwargs['cid'], callback, callback_context, args, kwargs) # We are to block while repeating else: # Repeat the given number of times sleeping for as many seconds as we are told remaining = retry_repeats result = None while remaining > 1: try: result = self.invoking_service.invoke( target, *args, **kwargs) except Exception as e: msg = retry_failed_msg( (retry_repeats - remaining) + 1, retry_repeats, target, retry_seconds, self.invoking_service.cid, e) logger.info(msg) sleep(retry_seconds) remaining -= 1 # OK, give up now, there's nothing more we can do if not result: msg = retry_limit_reached_msg(retry_repeats, target, retry_seconds, self.invoking_service.cid) raise ZatoException(self.invoking_service.cid, msg) else: # All good, simply return the response return result
def invoke_service(self, class_, request=None, **kwargs): # type: (Service, object, **object) class_.name = class_.get_name() class_.impl_name = class_.get_impl_name() class_.component_enabled_ibm_mq = True class_.component_enabled_zeromq = False class_.component_enabled_sms = True class_.component_enabled_cassandra = False class_.component_enabled_email = False class_.component_enabled_search = False class_.component_enabled_msg_path = False class_.component_enabled_patterns = False class_.has_sio = True class_._worker_config = self.worker_config class_._worker_store = self.worker_store class_.crypto = self.server.crypto_manager service = class_() # type: Service service.out.vault = self.vault_conn_api self.service_store.services[service.impl_name] = { 'slow_threshold': 100, } channel = kwargs.get('channel') or CHANNEL.INVOKE data_format = kwargs.get('data_format') or DATA_FORMAT.DICT transport = '' broker_client = None cid = kwargs.get('cid') or new_cid() simple_io_config = {'bytes_to_str': {'encoding': 'utf8'}} response = service.update_handle( self.request_handler._set_response_data, service, request, channel, data_format, transport, self.server, broker_client, self.worker_store, cid, simple_io_config, environ=kwargs.get('environ')) if kwargs.get('as_bunch'): if isinstance(response.payload, basestring): payload = loads(response.payload) payload = bunchify(payload) response._payload = payload return response
def test_client_ok(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True status_code = rand_int() rand_id, rand_name, soap_action = rand_string(), rand_string( ), rand_string() sio_response = """<zato_outgoing_amqp_edit_response xmlns="https://zato.io/ns/20130518"> <zato_env> <cid>{}</cid> <result>ZATO_OK</result> </zato_env> <item> <id>{}</id> <name>crm.account</name> </item> </zato_outgoing_amqp_edit_response> """.format(cid, rand_id, rand_name) text = """<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns="https://zato.io/ns/20130518"> <soap:Body> {} </soap:Body> </soap:Envelope>""".format(sio_response).strip() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(soap_action, '') eq_(response.ok, ok) eq_(response.inner.text, text) eq_(response.has_data, True) eq_(response.cid, cid) path_items = ( ('zato_env', 'cid'), ('zato_env', 'result'), ('item', 'id'), ('item', 'name'), ) for items in path_items: path = '//zato:zato_outgoing_amqp_edit_response/zato:' + '/zato:'.join( items) xpath = etree.XPath(path, namespaces=common_namespaces) expected = xpath(etree.fromstring(text))[0].text actual = xpath(response.data)[0] self.assertEquals(expected, actual)
def test_client_soap_fault(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = False status_code = rand_int() soap_action = rand_string() text = b"""<?xml version='1.0' encoding='UTF-8'?> <SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/1999/XMLSchema-instance" xmlns:xsd="http://www.w3.org/1999/XMLSchema"> <SOAP-ENV:Body> <SOAP-ENV:Fault> <faultcode>SOAP-ENV:Client</faultcode> <faultstring><![CDATA[cid [K68438211212681798524426103126], faultstring [Traceback (most recent call last): File "/opt/zato/code/zato-server/src/zato/server/connection/http_soap/ channel.py", line 126, in dispatch service_info, response = handler.handle(cid, wsgi_environ, payload, transport, worker_store, self.simple_io_config, data_format, path_info) File "/opt/zato/code/zato-server/src/zato/server/connection/http_soap/ channel.py", line 227, in handle service_instance.handle() File "/opt/zato/code/zato-server/src/zato/server/service/internal/ definition/amqp.py", line 174, in handle filter(ConnDefAMQP.id==self.request.input.id).\ File "/opt/zato/code/eggs/SQLAlchemy-0.7.9-py2.7-linux-x86_64.egg/sqlalchemy/ orm/query.py", line 2190, in one raise orm_exc.NoResultFound("No row was found for one()") NoResultFound: No row was found for one() ]]]></faultstring> </SOAP-ENV:Fault> </SOAP-ENV:Body> </SOAP-ENV:Envelope>""" client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(soap_action, '') eq_(response.ok, ok) eq_(response.inner.text, text) eq_(response.has_data, False) eq_(response.cid, cid) eq_( 'NoResultFound: No row was found for one()' in response.details.getchildren()[1].text, True)
def __init__(self, *args, **kwargs): super(ZatoWSXClient, self).__init__(*args, **kwargs) self._zato_client_config = _ZatoWSXConfigImpl() self._zato_client_config.client_name = 'WSX outconn - {}'.format(self.config.name) self._zato_client_config.client_id = 'wsx.out.{}'.format(new_cid(8)) self._zato_client_config.address = self.config.address self._zato_client_config.on_request_callback = self.on_message_cb self._zato_client_config.on_closed_callback = self.on_close_cb if self.config.get('username'): self._zato_client_config.username = self.config.username self._zato_client_config.secret = self.config.secret self._zato_client = _ZatoWSXClientImpl(self.opened, self._zato_client_config) self.invoke = self.send = self._zato_client.invoke
def get_context(self): ctx = { 'cid': new_cid(), 'start_time': self.start_time.isoformat(), 'cb_kwargs': self.cb_kwargs } if self.type == SCHEDULER.JOB_TYPE.CRON_STYLE: ctx['cron_definition'] = self.cron_definition else: ctx['interval_in_seconds'] = self.interval.in_seconds for name in 'id', 'name', 'current_run', 'max_repeats_reached', 'max_repeats', 'type': ctx[name] = getattr(self, name) return ctx
def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True text = dumps({rand_string(): rand_string()}) status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke() eq_(response.ok, ok) eq_(response.inner.text, text) eq_(iteritems(response.data), iteritems(loads(text))) eq_(response.has_data, True) eq_(response.cid, cid)
def __init__(self): self.action = None self.service = None self.sec_type = None self.username = None self.password = None self.id = None self.timestamp = None self.cid = new_cid() self.in_reply_to = None self.data = Bunch() self.has_credentials = None self.token = None self.ext_client_name = None self.ext_client_id = None self.reply_to_sk = None self.deliver_to_sk = None
def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True text = '<abc>{}</abc>'.format(rand_string()) status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke() eq_(response.ok, ok) eq_(response.inner.text, text) eq_(etree.tostring(response.data), text) eq_(response.has_data, True) eq_(response.cid, cid)
def __init__(self, publisher_tag, publisher_pid, payload='', request_id=None): self.publisher_tag = publisher_tag self.publisher_pid = publisher_pid self.action = NO_DEFAULT_VALUE self.service = '' self._payload = payload self.payload_type = type(payload).__name__ self.data_format = DATA_FORMAT.DICT self.request_id = request_id or 'ipc.{}'.format(new_cid()) self.target_pid = None self.reply_to_tag = '' self.reply_to_fifo = '' self.in_reply_to = '' self.creation_time_utc = datetime.utcnow()
def on_job_executed(self, ctx, extra_data_format=ZATO_NONE): """ Invoked by the underlying scheduler when a job is executed. Sends the actual execution request to the broker so it can be picked up by one of the parallel server's broker clients. """ name = ctx['name'] payload = ctx['cb_kwargs']['extra'] if isinstance(payload, bytes): payload = payload.decode('utf8') msg = { 'action': SCHEDULER_MSG.JOB_EXECUTED.value, 'name': name, 'service': ctx['cb_kwargs']['service'], 'payload': payload, 'cid': ctx['cid'], 'job_type': ctx['type'] } if extra_data_format != ZATO_NONE: msg['data_format'] = extra_data_format self.broker_client.invoke_async(msg) if _has_debug: msg = 'Sent a job execution request, name [{}], service [{}], extra [{}]'.format( name, ctx['cb_kwargs']['service'], ctx['cb_kwargs']['extra']) logger.debug(msg) # Now, if it was a one-time job, it needs to be deactivated. if ctx['type'] == SCHEDULER.JOB_TYPE.ONE_TIME: msg = { 'action': SERVICE.PUBLISH.value, 'service': 'zato.scheduler.job.set-active-status', 'payload': { 'id': ctx['id'], 'is_active': False }, 'cid': new_cid(), 'channel': CHANNEL.SCHEDULER_AFTER_ONE_TIME, 'data_format': DATA_FORMAT.JSON, } self.broker_client.publish(msg)
def send(self, data, _socket_socket=socket.socket, _family=socket.AF_INET, _type=socket.SOCK_STREAM): # type: (bytes) -> bytes try: data = data if isinstance(data, bytes) else data.encode('utf8') # Wrap the message in an MLLP envelope msg = self.start_seq + data + self.end_seq # This will auto-close the socket .. with _socket_socket(_family, _type) as sock: # .. connect to the remote end .. sock.connect((self.host, self.port)) # .. send our data .. sock.send(msg) # .. encapsulate configuration for our socket reader function .. ctx = SocketReaderCtx(new_cid(), sock, self.max_wait_time, self.max_msg_size, self.read_buffer_size, self.recv_timeout, self.should_log_messages) # .. wait for the response .. response = read_from_socket(ctx) if self.should_log_messages: logger.info('Response received `%s`', response) return response except Exception: logger.warn( 'Client caught an exception while sending HL7 MLLP data to `%s (%s)`; e:`%s`', self.name, self.address, format_exc()) raise
def test_client_no_soap_response(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = False soap_action = rand_string() text = '<abc/>' status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(soap_action) eq_(response.ok, ok) eq_(response.details, 'No /soapenv:Envelope/soapenv:Body/*[1] in SOAP response') eq_(response.inner.text, text) eq_(response.has_data, False) eq_(response.cid, cid)
def invoke_async(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ANY, expiration=BROKER.DEFAULT_EXPIRATION): msg['msg_type'] = msg_type try: msg = dumps(msg) except Exception: error_msg = 'JSON serialization failed for msg:`%r`, e:`%s`' logger.error(error_msg, msg, format_exc()) raise else: topic = TOPICS[msg_type] key = broker_msg = 'zato:broker{}:{}'.format( KEYS[msg_type], new_cid()) self.kvdb.conn.set(key, str(msg)) self.kvdb.conn.expire(key, expiration) # In seconds self.pub_client.publish(topic, broker_msg)
def _invoke_rest_outconn_callback(self, item_id, request): # type: (str, dict) -> None cid = new_cid() item = self.worker_store.get_outconn_rest_by_id(item_id) ping_response = item.ping(cid, return_response=True, log_verbose=True) # type: Response if ping_response.status_code != OK: logger.warn( 'Could not ping file transfer connection for `%s` (%s); config:`%s`, r:`%s`, h:`%s`', request['full_path'], request['config'].name, item.config, ping_response.text, ping_response.headers) else: file_name = request['file_name'] mime_type = guess_mime_type(file_name, strict=False) mime_type = mime_type[0] if mime_type[ 0] else 'application/octet-stream' payload = request['raw_data'] params = {'file_name': file_name, 'mime_type': mime_type} headers = { 'X-Zato-File-Name': file_name, 'X-Zato-Mime-Type': mime_type, } response = item.conn.post(cid, payload, params, headers=headers) # type: Response if response.status_code != OK: logger.warn( 'Could not send file `%s` (%s) to `%s` (p:`%s`, h:`%s`), r:`%s`, h:`%s`', request['full_path'], request['config'].name, item.config, params, headers, response.text, response.headers)
def invoke(self, class_, request_data, expected, mock_data={}, channel=CHANNEL.HTTP_SOAP, job_type=None, data_format=DATA_FORMAT.JSON, service_store_name_to_impl_name=None, service_store_impl_name_to_service=None): """ Sets up a service's invocation environment, then invokes and returns an instance of the service. """ class_.component_enabled_cassandra = True class_.component_enabled_email = True class_.component_enabled_search = True class_.component_enabled_msg_path = True class_.has_sio = getattr(class_, 'SimpleIO', False) instance = class_() server = MagicMock() server.component_enabled.stats = False worker_store = MagicMock() worker_store.worker_config = MagicMock worker_store.worker_config.outgoing_connections = MagicMock( return_value=(None, None, None, None)) worker_store.worker_config.cloud_openstack_swift = MagicMock( return_value=None) worker_store.worker_config.cloud_aws_s3 = MagicMock(return_value=None) worker_store.invoke_matcher.is_allowed = MagicMock(return_value=True) simple_io_config = { 'int_parameters': SIMPLE_IO.INT_PARAMETERS.VALUES, 'int_parameter_suffixes': SIMPLE_IO.INT_PARAMETERS.SUFFIXES, 'bool_parameter_prefixes': SIMPLE_IO.BOOL_PARAMETERS.SUFFIXES, } class_.update(instance, channel, FakeServer(service_store_name_to_impl_name, service_store_impl_name_to_service, worker_store), None, worker_store, new_cid(), request_data, request_data, simple_io_config=simple_io_config, data_format=data_format, job_type=job_type) def get_data(self, *ignored_args, **ignored_kwargs): return expected.get_data() instance.get_data = get_data for attr_name, mock_path_data_list in mock_data.items(): setattr(instance, attr_name, Mock()) attr = getattr(instance, attr_name) for mock_path_data in mock_path_data_list: for path, value in mock_path_data.items(): split = path.split('.') new_path = '.return_value.'.join( elem for elem in split) + '.return_value' attr.configure_mock(**{new_path: value}) broker_client_publish = getattr(self, 'broker_client_publish', None) if broker_client_publish: instance.broker_client = FakeBrokerClient() instance.broker_client.publish = broker_client_publish def set_response_func(*args, **kwargs): pass instance.handle() #instance.update_handle( # set_response_func, instance, request_data, channel, data_format, None, server, None, worker_store, new_cid(), # None) return instance
def __init__(self, kvdb, client_type, topic_callbacks): Thread.__init__(self) self.kvdb = kvdb self.decrypt_func = kvdb.decrypt_func self.name = '{}-{}'.format(client_type, new_cid()) self.topic_callbacks = topic_callbacks
def __init__(self): self.broker_client_id = '{}-{}'.format(ZATO_NONE, new_cid()) self.broker_callbacks = {} self.broker_messages = []