def on_call_finished(self, invoked_service, response, exception):
now = invoked_service.time.utcnow()
cid = invoked_service.wsgi_environ['zato.request_ctx.{}'.format(self.request_ctx_cid_key)]
data_key = self.data_pattern.format(cid)
counter_key = self.counter_pattern.format(cid)
source, req_ts_utc, on_target = invoked_service.kvdb.conn.hmget(data_key, 'source', 'req_ts_utc', 'on_target')
with invoked_service.lock(self.lock_pattern.format(cid)):
data = Bunch()
data.cid = cid
data.resp_ts_utc = now
data.response = response
data.exception = exception
data.ok = False if exception else True
data.source = source
data.target = invoked_service.name
data.req_ts_utc = req_ts_utc
# First store our response and exception (if any)
json_data = dumps(data)
invoked_service.kvdb.conn.hset(data_key, invoked_service.get_name(), json_data)
on_target = loads(on_target)
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_target', on_target, invoked_service)
# We always invoke 'on_target' callbacks, if there are any
self.invoke_callbacks(invoked_service, data, on_target, self.on_target_channel, cid)
# Was it the last parallel call?
if not invoked_service.kvdb.conn.decr(counter_key):
# Not every subclass will need final callbacks
if self.needs_on_final:
payload = invoked_service.kvdb.conn.hgetall(data_key)
payload['data'] = {}
for key in (key for key in payload.keys() if key not in JSON_KEYS):
payload['data'][key] = loads(payload.pop(key))
for key in JSON_KEYS:
if key not in ('source', 'data', 'req_ts_utc'):
payload[key] = loads(payload[key])
on_final = payload['on_final']
if logger.isEnabledFor(DEBUG):
self._log_before_callbacks('on_final', on_final, invoked_service)
self.invoke_callbacks(invoked_service, payload, on_final, self.on_final_channel, cid)
invoked_service.kvdb.conn.delete(counter_key)
invoked_service.kvdb.conn.delete(data_key)
def from_sql(item, needs_decrypt, decrypt_func, serialize_dt):
""" Builds a new AttrEntity out of an SQL row.
"""
return AttrEntity(
item.name,
decrypt_func(loads(item.value)) if
(needs_decrypt and item.is_encrypted) else loads(item.value),
item.creation_time.isoformat()
if serialize_dt else item.creation_time,
item.last_modified.isoformat()
if serialize_dt else item.last_modified,
item.expiration_time.isoformat() if serialize_dt else
item.expiration_time, item.is_encrypted, item.is_session_attr)
def check_impl(self, service_class, request_data, response_data, response_elem, mock_data={}):
expected_data = sorted(response_data.items())
instance = self.invoke(service_class, request_data, None, mock_data)
self._check_sio_request_input(instance, request_data)
if response_data:
if not isinstance(instance.response.payload, basestring):
response = loads(instance.response.payload.getvalue())[response_elem] # Raises KeyError if 'response_elem' doesn't match
else:
response = loads(instance.response.payload)[response_elem]
self.assertEqual(sorted(response.items()), expected_data)
def http_request(self,
method,
cid,
data='',
params=None,
_has_debug=has_debug,
*args,
**kwargs):
self._enforce_is_active()
# We never touch strings/unicode because apparently the user already serialized outgoing data
needs_serialize = not isinstance(data, basestring)
if needs_serialize:
if self.config['data_format'] == DATA_FORMAT.JSON:
data = dumps(data)
elif data and self.config['data_format'] == DATA_FORMAT.XML:
data = tostring(data)
headers = self._create_headers(cid, kwargs.pop('headers', {}))
if self.config['transport'] == 'soap':
data, headers = self._soap_data(data, headers)
params = params or {}
if self.path_params:
address, qs_params = self.format_address(cid, params)
else:
address, qs_params = self.address, dict(params)
if isinstance(data, unicode):
data = data.encode('utf-8')
logger.info(
'CID:`%s`, address:`%s`, qs:`%s`, auth_user:`%s`, kwargs:`%s`',
cid, address, qs_params, self.username, kwargs)
response = self.invoke_http(cid,
method,
address,
data,
headers, {},
params=qs_params,
*args,
**kwargs)
if _has_debug:
logger.debug('CID:`%s`, response:`%s`', cid, response.text)
if needs_serialize:
if self.config['data_format'] == DATA_FORMAT.JSON:
response.data = loads(response.text)
elif self.config['data_format'] == DATA_FORMAT.XML:
if response.text and response.headers.get('Content-Type') in (
'application/xml', 'text/xml'):
response.data = fromstring(response.text)
return response
def set_initial_opaque_attrs(username, initial, opaque_attrs):
# By default, opaque attributes are not set for user
if opaque_attrs:
opaque_attrs = loads(opaque_attrs)
for attr in profile_attrs_opaque:
initial[attr] = opaque_attrs.get(attr) or ''
# Generate or use the existing TOTP key
totp_key = initial.get('totp_key')
if not totp_key:
totp_key = pyotp.random_base32()
initial['totp_key_label'] = 'Zato web-admin'
else:
cm = CryptoManager(secret_key=zato_settings.zato_secret_key)
# TOTP key is always decrypted so we need to decrypt it here
totp_key = cm.decrypt(totp_key)
# .. same goes for its label
initial['totp_key_label'] = cm.decrypt(initial['totp_key_label'])
# Build the actual TOTP object for later use
totp = pyotp.totp.TOTP(totp_key)
# Update template data with TOTP information
initial['totp_key'] = totp.secret
initial['totp_key_provision_uri'] = totp.provisioning_uri(
username, issuer_name=initial['totp_key_label'])
def check_impl_list(
self,
service_class,
item_class,
request_data, # noqa
response_data,
request_elem,
response_elem,
mock_data={}): # noqa
expected_keys = response_data.keys()
expected_data = tuple(response_data for x in range(rand_int(10)))
expected = Expected()
for datum in expected_data:
item = item_class()
for key in expected_keys:
value = getattr(datum, key)
setattr(item, key, value)
expected.add(item)
instance = self.invoke(service_class, request_data, expected,
mock_data)
response = loads(instance.response.payload.getvalue())[response_elem]
for idx, item in enumerate(response):
expected = expected_data[idx]
given = Bunch(item)
for key in expected_keys:
given_value = getattr(given, key)
expected_value = getattr(expected, key)
eq_(given_value, expected_value)
self._check_sio_request_input(instance, request_data)
def handle(self):
# Convert to bunch so it's easier to read everything
self.req_bunch = Bunch(loads(self.request.payload))
# Initial retry linked to a retry callback
g = spawn(self._retry, self.req_bunch.retry_repeats)
g.link(self._on_retry_finished)
def set_config(self):
""" Sets self attributes, as configured in shmem by our parent process.
"""
config = self.config_ipc.get_config('zato-{}'.format(self.ipc_name))
config = loads(config)
config = bunchify(config)
self.username = config.username
self.password = config.password
self.server_auth = (self.username, self.password)
self.base_dir = config.base_dir
self.port = config.port
self.server_port = config.server_port
self.server_path = config.server_path
self.server_address = self.server_address.format(
self.server_port, self.server_path)
with open(config.logging_conf_path) as f:
logging_config = yaml.load(f, yaml.FullLoader)
if not 'zato_{}'.format(self.conn_type) in logging_config['loggers']:
logging_config = get_logging_config(self.conn_type,
self.logging_file_name)
# Configure logging for this connector
self.set_up_logging(logging_config)
# Store our process's pidfile
if config.needs_pidfile:
self.store_pidfile(config.pidfile_suffix)
def _invoke(self,
func,
func_name,
url_path,
request,
expect_ok,
auth=None,
_not_given='_test_not_given'):
address = Config.server_address.format(url_path)
request['current_app'] = Config.current_app
data = dumps(request)
logger.info('Invoking %s %s with %s', func_name, address, data)
response = func(address, data=data, auth=auth)
logger.info('Response received %s %s', response.status_code,
response.text)
data = loads(response.text)
data = bunchify(data)
# Most tests require status OK and CID
if expect_ok:
self.assertNotEquals(data.get('cid', _not_given), _not_given)
self.assertEquals(data.status, status_code.ok)
return data
def parse_instance_opaque_attr(instance):
opaque = getattr(instance, GENERIC.ATTR_NAME)
opaque = loads(opaque) if opaque else None
if not opaque:
return {}
ElemsWithOpaqueMaker.process_config_dict(opaque)
return bunchify(opaque)
def set_instance_opaque_attrs(instance, input, skip=None, only=None, _zato_skip=_zato_opaque_skip_attrs):
""" Given an SQLAlchemy object instance and incoming SimpleIO-based input,
populates all opaque values of that instance.
"""
only = only or []
instance_opaque_attrs = None
instance_attrs = set(instance.asdict())
input_attrs = set(input)
if only:
input_attrs = set([elem for elem in input_attrs if elem in only])
instance_attrs = set([elem for elem in instance_attrs if elem not in only])
# Any extra input attributes will be treated as opaque ones
input_opaque_attrs = input_attrs - instance_attrs
# Skip attributes related to pagination
for name in chain(skip or [], _zato_skip):
input_opaque_attrs.discard(name)
# Prepare generic attributes for instance
if GENERIC.ATTR_NAME in instance_attrs:
instance_opaque_attrs = getattr(instance, GENERIC.ATTR_NAME)
if instance_opaque_attrs:
instance_opaque_attrs = loads(instance_opaque_attrs)
else:
instance_opaque_attrs = {}
for name in input_opaque_attrs:
instance_opaque_attrs[name] = input[name]
# Set generic attributes for instance
if instance_opaque_attrs is not None:
setattr(instance, GENERIC.ATTR_NAME, dumps(instance_opaque_attrs))
def _handle_sso_GET(self, ctx):
# Local aliases
default_value = self.SimpleIO.default_value
# We either have a single UST on input or both target and current ones, but not both kinds
if ctx.input.ust:
if ctx.input.current_ust != default_value or ctx.input.target_ust != default_value:
raise ValidationError(status_code.common.invalid_input)
else:
# Without ctx.input.ust we must require both of the other elements
if not (ctx.input.current_ust != default_value
and ctx.input.target_ust != default_value):
raise ValidationError(status_code.common.invalid_input)
result = self.sso.user.session.get_list(self.cid, ctx.input.ust,
ctx.input.target_ust,
ctx.input.current_ust,
ctx.input.current_app,
ctx.remote_addr)
for item in result: # type: dict
item['creation_time'] = item['creation_time'].isoformat()
item['expiration_time'] = item['expiration_time'].isoformat()
opaque = item.pop(GENERIC.ATTR_NAME, None)
if opaque:
opaque = loads(opaque) # type: dict
item['session_state_change_list'] = opaque.get(
'session_state_change_list', [])
self.response.payload.result = result
def _set_opaque(elem, drop_opaque=False):
opaque = ElemsWithOpaqueMaker.get_opaque_data(elem)
opaque = loads(opaque) if opaque else {}
elem.update(opaque)
if drop_opaque:
del elem[GENERIC.ATTR_NAME]
def handle(self):
data = deepcopy(self.request.input)
raw_request = self.request.raw_request
if isinstance(raw_request, basestring):
raw_request = loads(raw_request)
for key, value in raw_request.items():
if key not in data:
value = parse_simple_type(value)
value = self._sio.eval_(key, value, self.server.encrypt)
data[key] = value
conn = GenericConnection.from_dict(data)
# Make sure not to overwrite the seceret in Edit
if not self.is_edit:
conn.secret = self.server.encrypt('auto.generated.{}'.format(
self.crypto.generate_secret()))
conn_dict = conn.to_sql_dict()
with closing(self.server.odb.session()) as session:
if self.is_edit:
model = self._get_instance_by_id(session, ModelGenericConn,
data.id)
else:
model = self._new_zato_instance_with_cluster(ModelGenericConn)
# This will be needed in case this is a rename
old_name = model.name
for key, value in sorted(conn_dict.items()):
if key == 'secret':
continue
setattr(model, key, value)
hook_func = hook.get(data.type_)
if hook_func:
hook_func(self, data, model, old_name)
session.add(model)
session.commit()
instance = self._get_instance_by_name(session, ModelGenericConn,
data.type_, data.name)
self.response.payload.id = instance.id
self.response.payload.name = instance.name
data['old_name'] = old_name
data[
'action'] = GENERIC.CONNECTION_EDIT.value if self.is_edit else GENERIC.CONNECTION_CREATE.value
data['id'] = instance.id
self.broker_client.publish(data)
def _session_with_opaque(session, _opaque_attr=GENERIC.ATTR_NAME):
if session:
opaque = getattr(session, _opaque_attr, None)
if opaque:
opaque = loads(opaque)
session = session._asdict()
session[_opaque_attr] = opaque
return bunchify(session)
return session
def get_totp_data(self):
totp_data = TOTPData()
if self.opaque1:
opaque = loads(self.opaque1)
totp_data.key = opaque.get('totp_key')
totp_data.label = opaque.get('totp_label')
return totp_data
def get_json_conf(self, conf_name, repo_dir=None):
# stdlib
from os.path import join
# Zato
from zato.common.json_internal import loads
repo_dir = repo_dir or join(self.config_dir, 'repo')
return loads(open(join(repo_dir, conf_name)).read())
def invoke(self, *args, **kwargs):
response = super(Client, self).invoke(
*args,
headers={'X-Zato-Forwarded-For': self.forwarded_for},
**kwargs)
if response.inner.status_code != OK:
zato_env = loads(response.inner.text).get('zato_env', {})
raise Exception('CID: {}\nDetails: {}'.format(
zato_env.get('cid'), zato_env.get('details')))
return response
def on_message(self, msg):
if has_debug:
logger.warn('Got broker message `%s`', msg)
if msg.type == 'message':
# Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside.
if msg.channel in NEEDS_TMP_KEY:
tmp_key = '{}.tmp'.format(msg.data)
if self.lua_container.run_lua(
'zato.rename_if_exists',
[msg.data, tmp_key]) == CODE_NO_SUCH_FROM_KEY:
payload = None
else:
payload = self.kvdb.conn.get(tmp_key)
self.kvdb.conn.delete(
tmp_key) # Note that it would've expired anyway
if not payload:
logger.info(
'No KVDB payload for key `%s` (already expired?)',
tmp_key)
else:
if isinstance(payload, bytes):
payload = payload
payload = loads(payload)
else:
if isinstance(msg.data, bytes):
msg.data = msg.data
payload = loads(msg.data)
if payload:
payload = Bunch(payload)
if has_debug:
logger.debug('Got broker message payload `%s`',
payload)
callback = self.topic_callbacks[msg.channel]
spawn_greenlet(callback, payload)
else:
if has_debug:
logger.debug('No payload in msg: `%s`', msg)
def handle(self):
out = self.kvdb.conn.get(_meta_endpoint_key % (self.request.input.cluster_id, self.request.input.endpoint_id))
out = loads(out) if out else []
for elem in out:
elem['pub_time'] = datetime_from_ms(elem['pub_time'] * 1000.0)
if elem['ext_pub_time']:
elem['ext_pub_time'] = datetime_from_ms(float(elem['ext_pub_time']) * 1000.0)
self.response.payload[:] = out
def to_bunch(self):
""" Returns a bunchified (converted into bunch.Bunch) version of self.raw_request,
deep copied if it's a dict (or a subclass). Note that it makes sense to use this method
only with dicts or JSON input.
"""
# We have a dict
if isinstance(self.raw_request, dict):
return bunchify(deepcopy(self.raw_request))
# Must be a JSON input, raises exception when attempting to load it if it's not
return bunchify(loads(self.raw_request))
def on_message(self, msg):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Got broker message:[{}]'.format(msg))
if msg.type == 'message':
# Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside.
if msg.channel in NEEDS_TMP_KEY:
tmp_key = '{}.tmp'.format(msg.data)
try:
self.kvdb.conn.rename(msg.data, tmp_key)
except redis.ResponseError as e:
if e.message != 'ERR no such key': # Doh, I hope Redis guys don't change it out of a sudden :/
raise
else:
payload = None
else:
payload = self.kvdb.conn.get(tmp_key)
self.kvdb.conn.delete(
tmp_key) # Note that it would've expired anyway
if not payload:
logger.warning(
'No KVDB payload for key [{}] (already expired?)'.
format(tmp_key))
else:
payload = loads(payload)
else:
payload = loads(msg.data)
if payload:
payload = Bunch(payload)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
'Got broker message payload [{}]'.format(payload))
return self.topic_callbacks[msg.channel](payload)
else:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('No payload in msg:[{}]'.format(msg))
def handle(self):
# Let's prepare as much as we can upfront.
sec_def = self.server.worker_store.basic_auth_get(
'admin.invoke').config
channel = self.server.worker_store.get_channel_rest(
'admin.invoke.json')
out = {}
with closing(self.odb.session()) as session:
for item in server_list(session, self.server.cluster_id, None,
None, False):
server_info = out.setdefault(item.name, {})
server_info['cluster_name'] = item.cluster_name
server_info['up_mod_date'] = item.up_mod_date.isoformat(
) if item.up_status == SERVER_UP_STATUS.RUNNING else None
server_info['last_join_mod_date'] = item.last_join_mod_date.isoformat() if \
item.last_join_status == SERVER_JOIN_STATUS.ACCEPTED else None
for name in 'id', 'name', 'bind_host', 'bind_port', 'last_join_status', 'last_join_mod_by', 'up_status':
server_info[name] = getattr(item, name)
if item.up_status == SERVER_UP_STATUS.RUNNING:
client = AnyServiceInvoker(
'http://{}:{}'.format(item.bind_host, item.bind_port),
channel.url_path, (sec_def.username, sec_def.password))
response = client.invoke('zato.info.get-server-info')
if response.ok:
response = loads(
response.inner.text
)['zato_service_invoke_response']['response']
response = b64decode(response)
response = loads(response)['response']
server_info['info'] = loads(response['info'])
else:
self.logger.warn(response)
self.response.content_type = 'application/json'
self.response.payload = dumps(out)
def get_data(self, session):
items = session.query(DeployedService.details,
Server.name.label('server_name'),
Server.id.label('server_id')).\
outerjoin(Server, DeployedService.server_id==Server.id).\
filter(DeployedService.service_id==self.request.input.id).\
all()
for item in items:
item.details = loads(item.details)
return items
def _invoke(self, method, data=None, *args, **kwargs):
to_bunch = kwargs.get('to_bunch', True)
result = self.session.post(self.address.format(method=method), data=data, *args, **kwargs) # type: Response
if not result.status_code == OK:
raise Exception(result.text)
if to_bunch:
out = loads(result.text)
return bunchify(out)
else:
return result
def _get_current(self, _url_info, self_major, self_version):
try:
response = requests_get(_url_info.format(self_major),
params={'v': self_version})
except ConnectionError as e:
# We ignore ENETUNREACH because it simply means that we could not connect to the server,
# which is fine, e.g. no Internet connectivity is allowed in that system.
if e.errno != ENETUNREACH:
raise
else:
if response.status_code == OK:
return bunchify(loads(response.text))
def send_message(self, msg):
if self.check_enabled:
self._check_enabled()
msg['action'] = self.action_send.value
response = self.invoke_connector(msg)
# If we are here, it means that there was no error because otherwise an exception
# would have been raised by invoke_connector.
response = loads(response.text)
return response
def from_model(data):
instance = GenericConnection()
opaque_value = getattr(data, GENERIC.ATTR_NAME, None)
if opaque_value:
instance.opaque.update(loads(opaque_value))
for name in instance.__slots__:
if name != 'opaque':
value = getattr(data, name, '<no-value-given-{}>'.format(name))
setattr(instance, name, value)
return instance
def invoke_service(self, class_, request=None, **kwargs):
# type: (Service, object, **object)
class_.name = class_.get_name()
class_.impl_name = class_.get_impl_name()
class_.component_enabled_ibm_mq = True
class_.component_enabled_zeromq = False
class_.component_enabled_sms = True
class_.component_enabled_cassandra = False
class_.component_enabled_email = False
class_.component_enabled_search = False
class_.component_enabled_msg_path = False
class_.component_enabled_patterns = False
class_.has_sio = True
class_._worker_config = self.worker_config
class_._worker_store = self.worker_store
class_.crypto = self.server.crypto_manager
service = class_() # type: Service
service.out.vault = self.vault_conn_api
self.service_store.services[service.impl_name] = {
'slow_threshold': 100,
}
channel = kwargs.get('channel') or CHANNEL.INVOKE
data_format = kwargs.get('data_format') or DATA_FORMAT.DICT
transport = ''
broker_client = None
cid = kwargs.get('cid') or new_cid()
simple_io_config = {'bytes_to_str': {'encoding': 'utf8'}}
response = service.update_handle(
self.request_handler._set_response_data,
service,
request,
channel,
data_format,
transport,
self.server,
broker_client,
self.worker_store,
cid,
simple_io_config,
environ=kwargs.get('environ'))
if kwargs.get('as_bunch'):
if isinstance(response.payload, basestring):
payload = loads(response.payload)
payload = bunchify(payload)
response._payload = payload
return response
def handle(self, _channel=CHANNEL.IBM_MQ, ts_format='YYYYMMDDHHmmssSS'):
request = loads(self.request.raw_request)
msg = request['msg']
service_name = request['service_name']
# Make MQ-level attributes easier to handle
correlation_id = unhexlify(
msg['correlation_id']) if msg['correlation_id'] else None
expiration = datetime_from_ms(
msg['expiration']) if msg['expiration'] else None
timestamp = '{}{}'.format(msg['put_date'], msg['put_time'])
timestamp = arrow_get(timestamp,
ts_format).replace(tzinfo='UTC').datetime
# Extract MQMD
mqmd = msg['mqmd']
mqmd = b64decode(mqmd)
mqmd = pickle_loads(mqmd)
# Find the message's CCSID
request_ccsid = mqmd.CodedCharSetId
# Try to find an encoding matching the CCSID,
# if not found, use the default one.
try:
encoding = CCSIDConfig.encoding_map[request_ccsid]
except KeyError:
encoding = CCSIDConfig.default_encoding
# Encode the input Unicode data into bytes
msg['text'] = msg['text'].encode(encoding)
# Extract the business payload
data = payload_from_request(self.server.json_parser, self.cid,
msg['text'], request['data_format'], None)
# Invoke the target service
self.invoke(service_name,
data,
_channel,
wmq_ctx={
'msg_id': unhexlify(msg['msg_id']),
'correlation_id': correlation_id,
'timestamp': timestamp,
'put_time': msg['put_time'],
'put_date': msg['put_date'],
'expiration': expiration,
'reply_to': msg['reply_to'],
'data': data,
'mqmd': mqmd
})
