def on_call_finished(self, invoked_service, response, exception): now = invoked_service.time.utcnow() cid = invoked_service.wsgi_environ['zato.request_ctx.{}'.format(self.request_ctx_cid_key)] data_key = self.data_pattern.format(cid) counter_key = self.counter_pattern.format(cid) source, req_ts_utc, on_target = invoked_service.kvdb.conn.hmget(data_key, 'source', 'req_ts_utc', 'on_target') with invoked_service.lock(self.lock_pattern.format(cid)): data = Bunch() data.cid = cid data.resp_ts_utc = now data.response = response data.exception = exception data.ok = False if exception else True data.source = source data.target = invoked_service.name data.req_ts_utc = req_ts_utc # First store our response and exception (if any) json_data = dumps(data) invoked_service.kvdb.conn.hset(data_key, invoked_service.get_name(), json_data) on_target = loads(on_target) if logger.isEnabledFor(DEBUG): self._log_before_callbacks('on_target', on_target, invoked_service) # We always invoke 'on_target' callbacks, if there are any self.invoke_callbacks(invoked_service, data, on_target, self.on_target_channel, cid) # Was it the last parallel call? if not invoked_service.kvdb.conn.decr(counter_key): # Not every subclass will need final callbacks if self.needs_on_final: payload = invoked_service.kvdb.conn.hgetall(data_key) payload['data'] = {} for key in (key for key in payload.keys() if key not in JSON_KEYS): payload['data'][key] = loads(payload.pop(key)) for key in JSON_KEYS: if key not in ('source', 'data', 'req_ts_utc'): payload[key] = loads(payload[key]) on_final = payload['on_final'] if logger.isEnabledFor(DEBUG): self._log_before_callbacks('on_final', on_final, invoked_service) self.invoke_callbacks(invoked_service, payload, on_final, self.on_final_channel, cid) invoked_service.kvdb.conn.delete(counter_key) invoked_service.kvdb.conn.delete(data_key)
def from_sql(item, needs_decrypt, decrypt_func, serialize_dt): """ Builds a new AttrEntity out of an SQL row. """ return AttrEntity( item.name, decrypt_func(loads(item.value)) if (needs_decrypt and item.is_encrypted) else loads(item.value), item.creation_time.isoformat() if serialize_dt else item.creation_time, item.last_modified.isoformat() if serialize_dt else item.last_modified, item.expiration_time.isoformat() if serialize_dt else item.expiration_time, item.is_encrypted, item.is_session_attr)
def check_impl(self, service_class, request_data, response_data, response_elem, mock_data={}): expected_data = sorted(response_data.items()) instance = self.invoke(service_class, request_data, None, mock_data) self._check_sio_request_input(instance, request_data) if response_data: if not isinstance(instance.response.payload, basestring): response = loads(instance.response.payload.getvalue())[response_elem] # Raises KeyError if 'response_elem' doesn't match else: response = loads(instance.response.payload)[response_elem] self.assertEqual(sorted(response.items()), expected_data)
def http_request(self, method, cid, data='', params=None, _has_debug=has_debug, *args, **kwargs): self._enforce_is_active() # We never touch strings/unicode because apparently the user already serialized outgoing data needs_serialize = not isinstance(data, basestring) if needs_serialize: if self.config['data_format'] == DATA_FORMAT.JSON: data = dumps(data) elif data and self.config['data_format'] == DATA_FORMAT.XML: data = tostring(data) headers = self._create_headers(cid, kwargs.pop('headers', {})) if self.config['transport'] == 'soap': data, headers = self._soap_data(data, headers) params = params or {} if self.path_params: address, qs_params = self.format_address(cid, params) else: address, qs_params = self.address, dict(params) if isinstance(data, unicode): data = data.encode('utf-8') logger.info( 'CID:`%s`, address:`%s`, qs:`%s`, auth_user:`%s`, kwargs:`%s`', cid, address, qs_params, self.username, kwargs) response = self.invoke_http(cid, method, address, data, headers, {}, params=qs_params, *args, **kwargs) if _has_debug: logger.debug('CID:`%s`, response:`%s`', cid, response.text) if needs_serialize: if self.config['data_format'] == DATA_FORMAT.JSON: response.data = loads(response.text) elif self.config['data_format'] == DATA_FORMAT.XML: if response.text and response.headers.get('Content-Type') in ( 'application/xml', 'text/xml'): response.data = fromstring(response.text) return response
def set_initial_opaque_attrs(username, initial, opaque_attrs): # By default, opaque attributes are not set for user if opaque_attrs: opaque_attrs = loads(opaque_attrs) for attr in profile_attrs_opaque: initial[attr] = opaque_attrs.get(attr) or '' # Generate or use the existing TOTP key totp_key = initial.get('totp_key') if not totp_key: totp_key = pyotp.random_base32() initial['totp_key_label'] = 'Zato web-admin' else: cm = CryptoManager(secret_key=zato_settings.zato_secret_key) # TOTP key is always decrypted so we need to decrypt it here totp_key = cm.decrypt(totp_key) # .. same goes for its label initial['totp_key_label'] = cm.decrypt(initial['totp_key_label']) # Build the actual TOTP object for later use totp = pyotp.totp.TOTP(totp_key) # Update template data with TOTP information initial['totp_key'] = totp.secret initial['totp_key_provision_uri'] = totp.provisioning_uri( username, issuer_name=initial['totp_key_label'])
def check_impl_list( self, service_class, item_class, request_data, # noqa response_data, request_elem, response_elem, mock_data={}): # noqa expected_keys = response_data.keys() expected_data = tuple(response_data for x in range(rand_int(10))) expected = Expected() for datum in expected_data: item = item_class() for key in expected_keys: value = getattr(datum, key) setattr(item, key, value) expected.add(item) instance = self.invoke(service_class, request_data, expected, mock_data) response = loads(instance.response.payload.getvalue())[response_elem] for idx, item in enumerate(response): expected = expected_data[idx] given = Bunch(item) for key in expected_keys: given_value = getattr(given, key) expected_value = getattr(expected, key) eq_(given_value, expected_value) self._check_sio_request_input(instance, request_data)
def handle(self): # Convert to bunch so it's easier to read everything self.req_bunch = Bunch(loads(self.request.payload)) # Initial retry linked to a retry callback g = spawn(self._retry, self.req_bunch.retry_repeats) g.link(self._on_retry_finished)
def set_config(self): """ Sets self attributes, as configured in shmem by our parent process. """ config = self.config_ipc.get_config('zato-{}'.format(self.ipc_name)) config = loads(config) config = bunchify(config) self.username = config.username self.password = config.password self.server_auth = (self.username, self.password) self.base_dir = config.base_dir self.port = config.port self.server_port = config.server_port self.server_path = config.server_path self.server_address = self.server_address.format( self.server_port, self.server_path) with open(config.logging_conf_path) as f: logging_config = yaml.load(f, yaml.FullLoader) if not 'zato_{}'.format(self.conn_type) in logging_config['loggers']: logging_config = get_logging_config(self.conn_type, self.logging_file_name) # Configure logging for this connector self.set_up_logging(logging_config) # Store our process's pidfile if config.needs_pidfile: self.store_pidfile(config.pidfile_suffix)
def _invoke(self, func, func_name, url_path, request, expect_ok, auth=None, _not_given='_test_not_given'): address = Config.server_address.format(url_path) request['current_app'] = Config.current_app data = dumps(request) logger.info('Invoking %s %s with %s', func_name, address, data) response = func(address, data=data, auth=auth) logger.info('Response received %s %s', response.status_code, response.text) data = loads(response.text) data = bunchify(data) # Most tests require status OK and CID if expect_ok: self.assertNotEquals(data.get('cid', _not_given), _not_given) self.assertEquals(data.status, status_code.ok) return data
def parse_instance_opaque_attr(instance): opaque = getattr(instance, GENERIC.ATTR_NAME) opaque = loads(opaque) if opaque else None if not opaque: return {} ElemsWithOpaqueMaker.process_config_dict(opaque) return bunchify(opaque)
def set_instance_opaque_attrs(instance, input, skip=None, only=None, _zato_skip=_zato_opaque_skip_attrs): """ Given an SQLAlchemy object instance and incoming SimpleIO-based input, populates all opaque values of that instance. """ only = only or [] instance_opaque_attrs = None instance_attrs = set(instance.asdict()) input_attrs = set(input) if only: input_attrs = set([elem for elem in input_attrs if elem in only]) instance_attrs = set([elem for elem in instance_attrs if elem not in only]) # Any extra input attributes will be treated as opaque ones input_opaque_attrs = input_attrs - instance_attrs # Skip attributes related to pagination for name in chain(skip or [], _zato_skip): input_opaque_attrs.discard(name) # Prepare generic attributes for instance if GENERIC.ATTR_NAME in instance_attrs: instance_opaque_attrs = getattr(instance, GENERIC.ATTR_NAME) if instance_opaque_attrs: instance_opaque_attrs = loads(instance_opaque_attrs) else: instance_opaque_attrs = {} for name in input_opaque_attrs: instance_opaque_attrs[name] = input[name] # Set generic attributes for instance if instance_opaque_attrs is not None: setattr(instance, GENERIC.ATTR_NAME, dumps(instance_opaque_attrs))
def _handle_sso_GET(self, ctx): # Local aliases default_value = self.SimpleIO.default_value # We either have a single UST on input or both target and current ones, but not both kinds if ctx.input.ust: if ctx.input.current_ust != default_value or ctx.input.target_ust != default_value: raise ValidationError(status_code.common.invalid_input) else: # Without ctx.input.ust we must require both of the other elements if not (ctx.input.current_ust != default_value and ctx.input.target_ust != default_value): raise ValidationError(status_code.common.invalid_input) result = self.sso.user.session.get_list(self.cid, ctx.input.ust, ctx.input.target_ust, ctx.input.current_ust, ctx.input.current_app, ctx.remote_addr) for item in result: # type: dict item['creation_time'] = item['creation_time'].isoformat() item['expiration_time'] = item['expiration_time'].isoformat() opaque = item.pop(GENERIC.ATTR_NAME, None) if opaque: opaque = loads(opaque) # type: dict item['session_state_change_list'] = opaque.get( 'session_state_change_list', []) self.response.payload.result = result
def _set_opaque(elem, drop_opaque=False): opaque = ElemsWithOpaqueMaker.get_opaque_data(elem) opaque = loads(opaque) if opaque else {} elem.update(opaque) if drop_opaque: del elem[GENERIC.ATTR_NAME]
def handle(self): data = deepcopy(self.request.input) raw_request = self.request.raw_request if isinstance(raw_request, basestring): raw_request = loads(raw_request) for key, value in raw_request.items(): if key not in data: value = parse_simple_type(value) value = self._sio.eval_(key, value, self.server.encrypt) data[key] = value conn = GenericConnection.from_dict(data) # Make sure not to overwrite the seceret in Edit if not self.is_edit: conn.secret = self.server.encrypt('auto.generated.{}'.format( self.crypto.generate_secret())) conn_dict = conn.to_sql_dict() with closing(self.server.odb.session()) as session: if self.is_edit: model = self._get_instance_by_id(session, ModelGenericConn, data.id) else: model = self._new_zato_instance_with_cluster(ModelGenericConn) # This will be needed in case this is a rename old_name = model.name for key, value in sorted(conn_dict.items()): if key == 'secret': continue setattr(model, key, value) hook_func = hook.get(data.type_) if hook_func: hook_func(self, data, model, old_name) session.add(model) session.commit() instance = self._get_instance_by_name(session, ModelGenericConn, data.type_, data.name) self.response.payload.id = instance.id self.response.payload.name = instance.name data['old_name'] = old_name data[ 'action'] = GENERIC.CONNECTION_EDIT.value if self.is_edit else GENERIC.CONNECTION_CREATE.value data['id'] = instance.id self.broker_client.publish(data)
def _session_with_opaque(session, _opaque_attr=GENERIC.ATTR_NAME): if session: opaque = getattr(session, _opaque_attr, None) if opaque: opaque = loads(opaque) session = session._asdict() session[_opaque_attr] = opaque return bunchify(session) return session
def get_totp_data(self): totp_data = TOTPData() if self.opaque1: opaque = loads(self.opaque1) totp_data.key = opaque.get('totp_key') totp_data.label = opaque.get('totp_label') return totp_data
def get_json_conf(self, conf_name, repo_dir=None): # stdlib from os.path import join # Zato from zato.common.json_internal import loads repo_dir = repo_dir or join(self.config_dir, 'repo') return loads(open(join(repo_dir, conf_name)).read())
def invoke(self, *args, **kwargs): response = super(Client, self).invoke( *args, headers={'X-Zato-Forwarded-For': self.forwarded_for}, **kwargs) if response.inner.status_code != OK: zato_env = loads(response.inner.text).get('zato_env', {}) raise Exception('CID: {}\nDetails: {}'.format( zato_env.get('cid'), zato_env.get('details'))) return response
def on_message(self, msg): if has_debug: logger.warn('Got broker message `%s`', msg) if msg.type == 'message': # Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside. if msg.channel in NEEDS_TMP_KEY: tmp_key = '{}.tmp'.format(msg.data) if self.lua_container.run_lua( 'zato.rename_if_exists', [msg.data, tmp_key]) == CODE_NO_SUCH_FROM_KEY: payload = None else: payload = self.kvdb.conn.get(tmp_key) self.kvdb.conn.delete( tmp_key) # Note that it would've expired anyway if not payload: logger.info( 'No KVDB payload for key `%s` (already expired?)', tmp_key) else: if isinstance(payload, bytes): payload = payload payload = loads(payload) else: if isinstance(msg.data, bytes): msg.data = msg.data payload = loads(msg.data) if payload: payload = Bunch(payload) if has_debug: logger.debug('Got broker message payload `%s`', payload) callback = self.topic_callbacks[msg.channel] spawn_greenlet(callback, payload) else: if has_debug: logger.debug('No payload in msg: `%s`', msg)
def handle(self): out = self.kvdb.conn.get(_meta_endpoint_key % (self.request.input.cluster_id, self.request.input.endpoint_id)) out = loads(out) if out else [] for elem in out: elem['pub_time'] = datetime_from_ms(elem['pub_time'] * 1000.0) if elem['ext_pub_time']: elem['ext_pub_time'] = datetime_from_ms(float(elem['ext_pub_time']) * 1000.0) self.response.payload[:] = out
def to_bunch(self): """ Returns a bunchified (converted into bunch.Bunch) version of self.raw_request, deep copied if it's a dict (or a subclass). Note that it makes sense to use this method only with dicts or JSON input. """ # We have a dict if isinstance(self.raw_request, dict): return bunchify(deepcopy(self.raw_request)) # Must be a JSON input, raises exception when attempting to load it if it's not return bunchify(loads(self.raw_request))
def on_message(self, msg): if logger.isEnabledFor(logging.DEBUG): logger.debug('Got broker message:[{}]'.format(msg)) if msg.type == 'message': # Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside. if msg.channel in NEEDS_TMP_KEY: tmp_key = '{}.tmp'.format(msg.data) try: self.kvdb.conn.rename(msg.data, tmp_key) except redis.ResponseError as e: if e.message != 'ERR no such key': # Doh, I hope Redis guys don't change it out of a sudden :/ raise else: payload = None else: payload = self.kvdb.conn.get(tmp_key) self.kvdb.conn.delete( tmp_key) # Note that it would've expired anyway if not payload: logger.warning( 'No KVDB payload for key [{}] (already expired?)'. format(tmp_key)) else: payload = loads(payload) else: payload = loads(msg.data) if payload: payload = Bunch(payload) if logger.isEnabledFor(logging.DEBUG): logger.debug( 'Got broker message payload [{}]'.format(payload)) return self.topic_callbacks[msg.channel](payload) else: if logger.isEnabledFor(logging.DEBUG): logger.debug('No payload in msg:[{}]'.format(msg))
def handle(self): # Let's prepare as much as we can upfront. sec_def = self.server.worker_store.basic_auth_get( 'admin.invoke').config channel = self.server.worker_store.get_channel_rest( 'admin.invoke.json') out = {} with closing(self.odb.session()) as session: for item in server_list(session, self.server.cluster_id, None, None, False): server_info = out.setdefault(item.name, {}) server_info['cluster_name'] = item.cluster_name server_info['up_mod_date'] = item.up_mod_date.isoformat( ) if item.up_status == SERVER_UP_STATUS.RUNNING else None server_info['last_join_mod_date'] = item.last_join_mod_date.isoformat() if \ item.last_join_status == SERVER_JOIN_STATUS.ACCEPTED else None for name in 'id', 'name', 'bind_host', 'bind_port', 'last_join_status', 'last_join_mod_by', 'up_status': server_info[name] = getattr(item, name) if item.up_status == SERVER_UP_STATUS.RUNNING: client = AnyServiceInvoker( 'http://{}:{}'.format(item.bind_host, item.bind_port), channel.url_path, (sec_def.username, sec_def.password)) response = client.invoke('zato.info.get-server-info') if response.ok: response = loads( response.inner.text )['zato_service_invoke_response']['response'] response = b64decode(response) response = loads(response)['response'] server_info['info'] = loads(response['info']) else: self.logger.warn(response) self.response.content_type = 'application/json' self.response.payload = dumps(out)
def get_data(self, session): items = session.query(DeployedService.details, Server.name.label('server_name'), Server.id.label('server_id')).\ outerjoin(Server, DeployedService.server_id==Server.id).\ filter(DeployedService.service_id==self.request.input.id).\ all() for item in items: item.details = loads(item.details) return items
def _invoke(self, method, data=None, *args, **kwargs): to_bunch = kwargs.get('to_bunch', True) result = self.session.post(self.address.format(method=method), data=data, *args, **kwargs) # type: Response if not result.status_code == OK: raise Exception(result.text) if to_bunch: out = loads(result.text) return bunchify(out) else: return result
def _get_current(self, _url_info, self_major, self_version): try: response = requests_get(_url_info.format(self_major), params={'v': self_version}) except ConnectionError as e: # We ignore ENETUNREACH because it simply means that we could not connect to the server, # which is fine, e.g. no Internet connectivity is allowed in that system. if e.errno != ENETUNREACH: raise else: if response.status_code == OK: return bunchify(loads(response.text))
def send_message(self, msg): if self.check_enabled: self._check_enabled() msg['action'] = self.action_send.value response = self.invoke_connector(msg) # If we are here, it means that there was no error because otherwise an exception # would have been raised by invoke_connector. response = loads(response.text) return response
def from_model(data): instance = GenericConnection() opaque_value = getattr(data, GENERIC.ATTR_NAME, None) if opaque_value: instance.opaque.update(loads(opaque_value)) for name in instance.__slots__: if name != 'opaque': value = getattr(data, name, '<no-value-given-{}>'.format(name)) setattr(instance, name, value) return instance
def invoke_service(self, class_, request=None, **kwargs): # type: (Service, object, **object) class_.name = class_.get_name() class_.impl_name = class_.get_impl_name() class_.component_enabled_ibm_mq = True class_.component_enabled_zeromq = False class_.component_enabled_sms = True class_.component_enabled_cassandra = False class_.component_enabled_email = False class_.component_enabled_search = False class_.component_enabled_msg_path = False class_.component_enabled_patterns = False class_.has_sio = True class_._worker_config = self.worker_config class_._worker_store = self.worker_store class_.crypto = self.server.crypto_manager service = class_() # type: Service service.out.vault = self.vault_conn_api self.service_store.services[service.impl_name] = { 'slow_threshold': 100, } channel = kwargs.get('channel') or CHANNEL.INVOKE data_format = kwargs.get('data_format') or DATA_FORMAT.DICT transport = '' broker_client = None cid = kwargs.get('cid') or new_cid() simple_io_config = {'bytes_to_str': {'encoding': 'utf8'}} response = service.update_handle( self.request_handler._set_response_data, service, request, channel, data_format, transport, self.server, broker_client, self.worker_store, cid, simple_io_config, environ=kwargs.get('environ')) if kwargs.get('as_bunch'): if isinstance(response.payload, basestring): payload = loads(response.payload) payload = bunchify(payload) response._payload = payload return response
def handle(self, _channel=CHANNEL.IBM_MQ, ts_format='YYYYMMDDHHmmssSS'): request = loads(self.request.raw_request) msg = request['msg'] service_name = request['service_name'] # Make MQ-level attributes easier to handle correlation_id = unhexlify( msg['correlation_id']) if msg['correlation_id'] else None expiration = datetime_from_ms( msg['expiration']) if msg['expiration'] else None timestamp = '{}{}'.format(msg['put_date'], msg['put_time']) timestamp = arrow_get(timestamp, ts_format).replace(tzinfo='UTC').datetime # Extract MQMD mqmd = msg['mqmd'] mqmd = b64decode(mqmd) mqmd = pickle_loads(mqmd) # Find the message's CCSID request_ccsid = mqmd.CodedCharSetId # Try to find an encoding matching the CCSID, # if not found, use the default one. try: encoding = CCSIDConfig.encoding_map[request_ccsid] except KeyError: encoding = CCSIDConfig.default_encoding # Encode the input Unicode data into bytes msg['text'] = msg['text'].encode(encoding) # Extract the business payload data = payload_from_request(self.server.json_parser, self.cid, msg['text'], request['data_format'], None) # Invoke the target service self.invoke(service_name, data, _channel, wmq_ctx={ 'msg_id': unhexlify(msg['msg_id']), 'correlation_id': correlation_id, 'timestamp': timestamp, 'put_time': msg['put_time'], 'put_date': msg['put_date'], 'expiration': expiration, 'reply_to': msg['reply_to'], 'data': data, 'mqmd': mqmd })