def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
status_code = rand_int()
service_name = rand_string()
service_response_name = '{}_response'.format(service_name)
service_response_payload = {'service_id': 5207, 'has_wsdl': True}
service_response_dict = {
'zato_service_has_wsdl_response': service_response_payload
}
service_response = b64encode(dumps(service_response_dict))
text = dumps({
'zato_env': {
'result': ZATO_OK,
'details': ''
},
service_response_name: {
'response': service_response
}
})
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke(service_name, '')
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(response.data.items(), service_response_payload.items())
eq_(response.has_data, True)
eq_(response.cid, cid)
def on_before_append_item(self, item):
# type: (FileTransferChannel) -> FileTransferChannel
if item.service_list:
item.service_list = item.service_list if isinstance(
item.service_list, list) else [item.service_list]
item.service_list = sorted(item.service_list)
item.service_list_json = dumps(item.service_list)
if item.topic_list:
item.topic_list = item.topic_list if isinstance(
item.topic_list, list) else [item.topic_list]
item.topic_list = sorted(item.topic_list)
item.topic_list_json = dumps(item.topic_list)
if item.outconn_rest_list:
# All REST outgoing connections for the cluster
all_outconn_rest_list = get_outconn_rest_list(self.req)
item.outconn_rest_list = item.outconn_rest_list if isinstance(item.outconn_rest_list, list) else \
[item.outconn_rest_list]
item.outconn_rest_list_by_name = sorted(
all_outconn_rest_list[int(elem)]
for elem in item.outconn_rest_list if elem)
item.outconn_rest_list_json = dumps(item.outconn_rest_list)
return item
def invoke(self, targets, on_final, on_target=None, cid=None):
""" Invokes targets collecting their responses, can be both as a whole or individual ones,
and executes callback(s).
"""
# Can be user-provided or what our source gave us
cid = cid or self.cid
on_final = [on_final] if isinstance(on_final, basestring) else on_final
on_target = [on_target] if isinstance(on_target, basestring) else on_target
on_final = on_final or ''
on_target = on_target or ''
# Keep everything under a distributed lock
with self.source.lock(self.lock_pattern.format(cid)):
# Store information how many targets there were + info on what to invoke when they complete
self.source.kvdb.conn.set(self.counter_pattern.format(cid), len(targets))
self.source.kvdb.conn.hmset(self.data_pattern.format(cid), {
'source': self.source.name,
'on_final': dumps(on_final),
'on_target': dumps(on_target),
'req_ts_utc': self.source.time.utcnow()
})
# Invoke targets
for name, payload in targets.items():
to_json_string = False if isinstance(payload, basestring) else True
self.source.invoke_async(name, payload, self.call_channel, to_json_string=to_json_string,
zato_ctx={self.request_ctx_cid_key: cid})
return cid
def set_payload(self,
response,
data_format,
transport,
service_instance,
_sio_json=SIMPLE_IO.FORMAT.JSON,
_url_type_soap=URL_TYPE.SOAP,
_dict_like=(DATA_FORMAT.JSON, DATA_FORMAT.DICT),
_AdminService=AdminService,
_dumps=dumps,
_basestring=basestring):
""" Sets the actual payload to represent the service's response out of what the service produced.
This includes converting dictionaries into JSON, adding Zato metadata and wrapping the mesasge in SOAP if need be.
"""
# type: (Response, str, str, Service)
if isinstance(service_instance, AdminService):
if data_format == _sio_json:
zato_env = {
'zato_env': {
'result': response.result,
'cid': service_instance.cid,
'details': response.result_details
}
}
if response.payload:
payload = response.payload.getvalue(False)
payload.update(zato_env)
else:
payload = zato_env
response.payload = dumps(payload)
else:
if transport == _url_type_soap:
zato_message_template = zato_message_soap
else:
zato_message_template = zato_message_declaration_uni
if response.payload:
if not isinstance(response.payload, _basestring):
response.payload = self._get_xml_admin_payload(
service_instance, zato_message_template,
response.payload)
else:
response.payload = self._get_xml_admin_payload(
service_instance, zato_message_template, None)
else:
if not isinstance(response.payload, _basestring):
if isinstance(response.payload,
dict) and data_format in _dict_like:
response.payload = dumps(response.payload)
else:
response.payload = response.payload.getvalue(
) if response.payload else ''
if transport == _url_type_soap:
if not isinstance(service_instance, _AdminService):
if self.use_soap_envelope:
response.payload = soap_doc.format(body=response.payload)
def remote_command_execute(req):
""" Executes a command against the key/value DB.
"""
try:
response = req.zato.client.invoke('zato.kvdb.remote-command.execute', {'command': req.POST['command']})
if response.has_data:
return HttpResponse(dumps({'message': dumps(response.data.result)}), content_type='application/javascript')
else:
raise ZatoException(msg=response.details)
except Exception as e:
return HttpResponseServerError(e.args[0])
def http_request(self,
method,
cid,
data='',
params=None,
_has_debug=has_debug,
*args,
**kwargs):
self._enforce_is_active()
# We never touch strings/unicode because apparently the user already serialized outgoing data
needs_serialize = not isinstance(data, basestring)
if needs_serialize:
if self.config['data_format'] == DATA_FORMAT.JSON:
data = dumps(data)
elif data and self.config['data_format'] == DATA_FORMAT.XML:
data = tostring(data)
headers = self._create_headers(cid, kwargs.pop('headers', {}))
if self.config['transport'] == 'soap':
data, headers = self._soap_data(data, headers)
params = params or {}
if self.path_params:
address, qs_params = self.format_address(cid, params)
else:
address, qs_params = self.address, dict(params)
if isinstance(data, unicode):
data = data.encode('utf-8')
logger.info(
'CID:`%s`, address:`%s`, qs:`%s`, auth_user:`%s`, kwargs:`%s`',
cid, address, qs_params, self.username, kwargs)
response = self.invoke_http(cid,
method,
address,
data,
headers, {},
params=qs_params,
*args,
**kwargs)
if _has_debug:
logger.debug('CID:`%s`, response:`%s`', cid, response.text)
if needs_serialize:
if self.config['data_format'] == DATA_FORMAT.JSON:
response.data = loads(response.text)
elif self.config['data_format'] == DATA_FORMAT.XML:
if response.text and response.headers.get('Content-Type') in (
'application/xml', 'text/xml'):
response.data = fromstring(response.text)
return response
def settings_basic_save(req):
# Process explicitly named attributes
for attr in profile_attrs:
# Use False as default value so as to convert blank checkboxes into a boolean value
value = req.POST.get(attr, False)
setattr(req.zato.user_profile, attr, value)
# Process opaque attributes
opaque_attrs = {}
for attr in profile_attrs_opaque:
value = req.POST.get(attr)
opaque_attrs[attr] = value
# Encrypt TOTP before it is saved to the database
totp_key = opaque_attrs.get('totp_key')
if totp_key:
# set_user_profile_totp_key(profile, zato_secret_key, key, label, opaque_attrs=None)
set_user_profile_totp_key(req.zato.user_profile,
zato_settings.zato_secret_key, totp_key,
opaque_attrs.get('totp_key_label'),
opaque_attrs)
# Make sure all values are Unicode objects before serializing opaque attrs to JSON
for key, value in opaque_attrs.items():
if not isinstance(value, unicode):
opaque_attrs[key] = value.decode('utf8')
# Save all opaque attributes along with the profile
req.zato.user_profile.opaque1 = dumps(opaque_attrs)
# Save the profile
req.zato.user_profile.save()
# Save preferred cluster colour markers
for key, value in req.POST.items():
if key.startswith('color_') and value != DEFAULT_PROMPT:
cluster_id = key.replace('color_', '')
if 'checkbox_{}'.format(cluster_id) in req.POST:
try:
ccm = ClusterColorMarker.objects.get(
cluster_id=cluster_id,
user_profile=req.zato.user_profile)
except ClusterColorMarker.DoesNotExist:
ccm = ClusterColorMarker()
ccm.cluster_id = cluster_id
ccm.user_profile = req.zato.user_profile
ccm.color = value
ccm.save()
else:
ClusterColorMarker.objects.filter(
cluster_id=cluster_id,
user_profile=req.zato.user_profile).delete()
msg = 'Settings saved'
messages.add_message(req, messages.INFO, msg, extra_tags='success')
return redirect(reverse('account-settings-basic'))
def _on_OUTGOING_SFTP_EXECUTE(self, msg, is_reconnect=False, _utcnow=datetime.utcnow):
out = {}
connection = self.connections[msg.id] # type: SFTPConnection
start_time = _utcnow()
try:
result = connection.execute(msg.cid, msg.data, msg.log_level) # type: Output
except ErrorReturnCode as e:
out['stdout'] = e.stdout
out['stderr'] = e.stderr
except Exception as e:
out['stderr'] = format_exc()
out['is_ok'] = False
else:
out.update(result.to_dict())
finally:
out['cid'] = msg.cid
out['command_no'] = connection.command_no
out['response_time'] = str(_utcnow() - start_time)
print()
print(111, out)
print()
return Response(data=dumps(out))
def invoke_action(req,
pub_client_id,
send_attrs=('id', 'pub_client_id', 'request_data',
'timeout')):
try:
request = {'cluster_id': req.zato.cluster_id}
for name in send_attrs:
request[name] = req.POST.get(name, '')
response = req.zato.client.invoke('zato.channel.web-socket.invoke-wsx',
request)
if response.ok:
response_data = response.data['response_data']
if isinstance(response_data, dict):
response_data = dict(response_data)
return HttpResponse(dumps(response_data),
content_type='application/javascript')
else:
raise Exception(response.details)
except Exception:
msg = 'Caught an exception, e:`{}`'.format(format_exc())
logger.error(msg)
return HttpResponseServerError(msg)
def set_instance_opaque_attrs(instance, input, skip=None, only=None, _zato_skip=_zato_opaque_skip_attrs):
""" Given an SQLAlchemy object instance and incoming SimpleIO-based input,
populates all opaque values of that instance.
"""
only = only or []
instance_opaque_attrs = None
instance_attrs = set(instance.asdict())
input_attrs = set(input)
if only:
input_attrs = set([elem for elem in input_attrs if elem in only])
instance_attrs = set([elem for elem in instance_attrs if elem not in only])
# Any extra input attributes will be treated as opaque ones
input_opaque_attrs = input_attrs - instance_attrs
# Skip attributes related to pagination
for name in chain(skip or [], _zato_skip):
input_opaque_attrs.discard(name)
# Prepare generic attributes for instance
if GENERIC.ATTR_NAME in instance_attrs:
instance_opaque_attrs = getattr(instance, GENERIC.ATTR_NAME)
if instance_opaque_attrs:
instance_opaque_attrs = loads(instance_opaque_attrs)
else:
instance_opaque_attrs = {}
for name in input_opaque_attrs:
instance_opaque_attrs[name] = input[name]
# Set generic attributes for instance
if instance_opaque_attrs is not None:
setattr(instance, GENERIC.ATTR_NAME, dumps(instance_opaque_attrs))
def invoke_action_handler(req, service_name, send_attrs):
# type: (str, tuple) -> object
try:
request = {'cluster_id': req.zato.cluster_id}
for name in send_attrs:
request[name] = req.POST.get(name, '')
logger.info('Invoking `%s` with `%s`', service_name, request)
response = req.zato.client.invoke(service_name, request)
if response.ok:
response_data = response.data['response_data']
if isinstance(response_data, dict):
response_data = dict(response_data)
logger.info('Returning `%s` from `%s`', response_data,
service_name)
return HttpResponse(dumps(response_data),
content_type='application/javascript')
else:
raise Exception(response.details)
except Exception:
msg = 'Caught an exception, e:`{}`'.format(format_exc())
logger.error(msg)
return HttpResponseServerError(msg)
def _get_key_value_list(req, service_name, input_dict):
return_data = []
for item in req.zato.client.invoke(service_name, input_dict):
return_data.append({'name': item.name})
return HttpResponse(dumps(return_data),
content_type='application/javascript')
def command_shell_action(req, id, cluster_id, name_slug):
try:
response = req.zato.client.invoke(
'zato.outgoing.sftp.execute', {
'cluster_id': req.zato.cluster_id,
'id': id,
'data': req.POST['data'],
'log_level': req.POST['log_level'],
})
if response.ok:
data = response.data
return HttpResponse(dumps({
'msg':
'Response time: {} (#{})'.format(data.response_time,
data.command_no),
'stdout':
data.get('stdout') or '(None)',
'stderr':
data.get('stderr') or '(None)',
}),
content_type='application/javascript')
else:
raise Exception(response.details)
except Exception:
msg = 'Caught an exception, e:`{}`'.format(format_exc())
logger.error(msg)
return HttpResponseServerError(msg)
def test_client(self):
cid = new_cid()
headers = {'x-zato-cid': cid}
ok = True
env = {'details': rand_string(), 'result': ZATO_OK, 'cid': cid}
sio_payload_key = rand_string()
sio_payload = {rand_string(): rand_string()}
sio_response = {'zato_env': env, sio_payload_key: sio_payload}
text = dumps(sio_response)
status_code = rand_int()
client = self.get_client(
FakeInnerResponse(headers, ok, text, status_code))
response = client.invoke()
eq_(response.ok, ok)
eq_(response.inner.text, text)
eq_(iteritems(response.data), iteritems(
(sio_response[sio_payload_key])))
eq_(response.has_data, True)
eq_(response.cid, cid)
eq_(response.cid, sio_response['zato_env']['cid'])
eq_(response.details, sio_response['zato_env']['details'])
def _edit_create_response(req, id, verb, transport, connection, name):
return_data = {
'id':
id,
'transport':
transport,
'message':
'Successfully {} the {} {} `{}`, check server logs for details'.format(
verb, TRANSPORT[transport], CONNECTION[connection], name),
}
# If current item has a cache assigned, provide its human-friendly name to the caller
response = req.zato.client.invoke('zato.http-soap.get', {
'cluster_id': req.zato.cluster_id,
'id': id,
})
if response.data.cache_id:
cache_type = response.data.cache_type
cache_name = '{}/{}'.format(CACHE_TYPE[cache_type],
response.data.cache_name)
else:
cache_type = None
cache_name = None
return_data['cache_type'] = cache_type
return_data['cache_name'] = cache_name
return HttpResponse(dumps(return_data),
content_type='application/javascript')
def _invoke(self,
func,
func_name,
url_path,
request,
expect_ok,
auth=None,
_not_given='_test_not_given'):
address = Config.server_address.format(url_path)
request['current_app'] = Config.current_app
data = dumps(request)
logger.info('Invoking %s %s with %s', func_name, address, data)
response = func(address, data=data, auth=auth)
logger.info('Response received %s %s', response.status_code,
response.text)
data = loads(response.text)
data = bunchify(data)
# Most tests require status OK and CID
if expect_ok:
self.assertNotEquals(data.get('cid', _not_given), _not_given)
self.assertEquals(data.status, status_code.ok)
return data
def _handle_attr_call(self, attr):
try:
key = self.request.input.key
# This may be potentially an integer key that we received as string
# so we need to try to convert it to one.
try:
key = int(key)
except ValueError:
pass # That is fine, it was not an integer
values = attr[key]
except KeyError:
raise KeyError('No such key `{}` ({}) among `{}`'.format(key, type(key), sorted(attr.keys())))
values = values if isinstance(values, list) else [values]
out = [elem.to_dict() for elem in values]
out.sort(key=itemgetter(*self.request.input.sort_by), reverse=True)
for item in out:
for key, value in item.items():
if isinstance(value, datetime):
item[key] = value.isoformat()
self.response.payload = dumps(out)
def _invoke_async_retry(self,
target,
retry_repeats,
retry_seconds,
orig_cid,
call_cid,
callback,
callback_context,
args,
kwargs,
_utcnow=utcnow):
# Request to invoke the background service with ..
retry_request = {
'source': self.invoking_service.name,
'target': target,
'retry_repeats': retry_repeats,
'retry_seconds': retry_seconds,
'orig_cid': orig_cid,
'call_cid': call_cid,
'callback': callback,
'callback_context': callback_context,
'args': args,
'kwargs': kwargs,
'req_ts_utc': _utcnow()
}
return self.invoking_service.invoke_async(
'zato.pattern.invoke-retry.invoke-retry',
dumps(retry_request),
cid=call_cid)
def _request(self,
op,
key,
value=NotGiven,
pattern='/zato/cache/{}',
op_verb_map=op_verb_map):
# type: (str, str, str) -> str
# Build a full address
path = pattern.format(key)
address = '{}{}'.format(self.address, path)
# Get the HTTP verb to use in the request
verb = op_verb_map[op] # type: str
data = {'cache': self.cache_name, 'return_prev': True}
if value is not NotGiven:
data['value'] = value
data = dumps(data)
response = self.session.request(verb, address,
data=data) # type: RequestsResponse
return response.text
def _edit_create_response(service_response, action, name, password_type):
return_data = {'id': service_response.data.id,
'message': 'Successfully {0} the WS-Security definition [{1}]'.format(action, name),
'password_type_raw':password_type,
'password_type':ZATO_WSS_PASSWORD_TYPES[password_type]}
return HttpResponse(dumps(return_data), content_type='application/javascript')
def start_connector(self, ipc_tcp_start_port, timeout=5):
""" Starts an HTTP server acting as an connector process. Its port will be greater than ipc_tcp_start_port,
which is the starting point to find a free port from.
"""
if self.check_enabled:
self._check_enabled()
self.ipc_tcp_port = get_free_port(ipc_tcp_start_port)
logger.info('Starting {} connector for server `%s` on port `%s`'.format(self.connector_name),
self.server.name, self.ipc_tcp_port)
# Credentials for both servers and connectors
username, password = self.get_credentials()
# Employ IPC to exchange subprocess startup configuration
self.server.connector_config_ipc.set_config(self.ipc_config_name, dumps({
'port': self.ipc_tcp_port,
'username': username,
'password': password,
'server_port': self.server.port,
'server_name': self.server.name,
'server_path': '/zato/internal/callback/{}'.format(self.callback_suffix),
'base_dir': self.server.base_dir,
'needs_pidfile': not self.server.has_fg,
'pidfile_suffix': self.pidfile_suffix,
'logging_conf_path': self.server.logging_conf_path
}))
# Start connector in a sub-process
start_python_process('{} connector'.format(self.connector_name), False, self.connector_module, '', extra_options={
'deployment_key': self.server.deployment_key,
'shmem_size': self.server.shmem_size
}, stderr_path=self.server.stderr_path)
# Wait up to timeout seconds for the connector to start as indicated by its responding to a PING request
now = datetime.utcnow()
warn_after = now + timedelta(seconds=60)
should_warn = False
until = now + timedelta(seconds=timeout)
is_ok = False
address = address_pattern.format(self.ipc_tcp_port, 'ping')
auth = self.get_credentials()
while not is_ok or now >= until:
if not should_warn:
if now >= warn_after:
should_warn = True
is_ok = self._ping_connector(address, auth, should_warn)
if is_ok:
break
else:
sleep(2)
now = datetime.utcnow()
if not is_ok:
logger.warn('{} connector (%s) could not be started after %s'.format(self.connector_name), address, timeout)
else:
return is_ok
def on_call_finished(self, invoked_service, response, exception):
now = invoked_service.time.utcnow()
cid = invoked_service.wsgi_environ['zato.request_ctx.{}'.format(self.request_ctx_cid_key)]
data_key = self.data_pattern.format(cid)
counter_key = self.counter_pattern.format(cid)
source, req_ts_utc, on_target = invoked_service.kvdb.conn.hmget(data_key, 'source', 'req_ts_utc', 'on_target')
with invoked_service.lock(self.lock_pattern.format(cid)):
data = Bunch()
data.cid = cid
data.resp_ts_utc = now
data.response = response
data.exception = exception
data.ok = False if exception else True
data.source = source
data.target = invoked_service.name
data.req_ts_utc = req_ts_utc
# First store our response and exception (if any)
json_data = dumps(data)
invoked_service.kvdb.conn.hset(data_key, invoked_service.get_name(), json_data)
on_target = loads(on_target)
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_target', on_target, invoked_service)
# We always invoke 'on_target' callbacks, if there are any
self.invoke_callbacks(invoked_service, data, on_target, self.on_target_channel, cid)
# Was it the last parallel call?
if not invoked_service.kvdb.conn.decr(counter_key):
# Not every subclass will need final callbacks
if self.needs_on_final:
payload = invoked_service.kvdb.conn.hgetall(data_key)
payload['data'] = {}
for key in (key for key in payload.keys() if key not in JSON_KEYS):
payload['data'][key] = loads(payload.pop(key))
for key in JSON_KEYS:
if key not in ('source', 'data', 'req_ts_utc'):
payload[key] = loads(payload[key])
on_final = payload['on_final']
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_final', on_final, invoked_service)
self.invoke_callbacks(invoked_service, payload, on_final, self.on_final_channel, cid)
invoked_service.kvdb.conn.delete(counter_key)
invoked_service.kvdb.conn.delete(data_key)
def serialize(self, to_string=False):
# type: (bool) -> object
out = {
'is_ok': False,
'cid': self.cid,
'message': self.get_error_message()
}
return dumps(out) if to_string else out
def publish(self,
msg,
msg_type=MESSAGE_TYPE.TO_PARALLEL_ALL,
*ignored_args,
**ignored_kwargs):
msg['msg_type'] = msg_type
topic = TOPICS[msg_type]
msg = dumps(msg)
self.pub_client.publish(topic, msg)
def client_json_error(cid, faultstring):
zato_env = {
'zato_env': {
'result': ZATO_ERROR,
'cid': cid,
'details': faultstring
}
}
return dumps(zato_env)
def _edit_create_response(verb, id, name, engine_display_name, cluster_id):
""" A common function for producing return data for create and edit actions.
"""
return_data = {'id': id,
'message': 'Successfully {} outgoing SQL connection `{}`'.format(verb, name.encode('utf-8')),
'engine_display_name': engine_display_name,
'cluster_id': cluster_id,
}
return HttpResponse(dumps(return_data), content_type='application/javascript')
def handle(self):
response = {}
response['url_sec'] = sorted(
self.worker_store.request_handler.security.url_sec.items())
response['plain_http_handler.http_soap'] = sorted(
self.worker_store.request_handler.plain_http_handler.http_soap.
items())
response['soap_handler.http_soap'] = sorted(
self.worker_store.request_handler.soap_handler.http_soap.items())
self.response.payload = dumps(response, sort_keys=True, indent=4)
self.response.content_type = 'application/json'
def invoke_service_with_json_response(req, service, input_dict, ok_msg, error_template='', content_type='application/javascript',
extra=None):
try:
req.zato.client.invoke(service, input_dict)
except Exception as e:
return HttpResponseServerError(e.message, content_type=content_type)
else:
response = {'msg': ok_msg}
response.update(extra or {})
response = dumps(response)
return HttpResponse(response, content_type=content_type)
def _edit_create_response(client, verb, id, name, def_id, cluster_id):
response = client.invoke('zato.definition.amqp.get-by-id', {
'id': def_id,
'cluster_id': cluster_id
})
return_data = {
'id': id,
'message': 'Successfully {} the AMQP channel `{}`'.format(verb, name),
'def_name': response.data.name
}
return HttpResponse(dumps(return_data),
content_type='application/javascript')
def to_sql_dict(self, needs_bunch=False, skip=None):
out = {}
skip = skip or []
for name in self.__slots__:
if name in skip:
continue
if name != 'opaque':
out[name] = getattr(self, name)
else:
out[GENERIC.ATTR_NAME] = dumps(self.opaque)
return bunchify(out) if needs_bunch else out
