def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True status_code = rand_int() service_name = rand_string() service_response_name = '{}_response'.format(service_name) service_response_payload = {'service_id': 5207, 'has_wsdl': True} service_response_dict = { 'zato_service_has_wsdl_response': service_response_payload } service_response = b64encode(dumps(service_response_dict)) text = dumps({ 'zato_env': { 'result': ZATO_OK, 'details': '' }, service_response_name: { 'response': service_response } }) client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke(service_name, '') eq_(response.ok, ok) eq_(response.inner.text, text) eq_(response.data.items(), service_response_payload.items()) eq_(response.has_data, True) eq_(response.cid, cid)
def on_before_append_item(self, item): # type: (FileTransferChannel) -> FileTransferChannel if item.service_list: item.service_list = item.service_list if isinstance( item.service_list, list) else [item.service_list] item.service_list = sorted(item.service_list) item.service_list_json = dumps(item.service_list) if item.topic_list: item.topic_list = item.topic_list if isinstance( item.topic_list, list) else [item.topic_list] item.topic_list = sorted(item.topic_list) item.topic_list_json = dumps(item.topic_list) if item.outconn_rest_list: # All REST outgoing connections for the cluster all_outconn_rest_list = get_outconn_rest_list(self.req) item.outconn_rest_list = item.outconn_rest_list if isinstance(item.outconn_rest_list, list) else \ [item.outconn_rest_list] item.outconn_rest_list_by_name = sorted( all_outconn_rest_list[int(elem)] for elem in item.outconn_rest_list if elem) item.outconn_rest_list_json = dumps(item.outconn_rest_list) return item
def invoke(self, targets, on_final, on_target=None, cid=None): """ Invokes targets collecting their responses, can be both as a whole or individual ones, and executes callback(s). """ # Can be user-provided or what our source gave us cid = cid or self.cid on_final = [on_final] if isinstance(on_final, basestring) else on_final on_target = [on_target] if isinstance(on_target, basestring) else on_target on_final = on_final or '' on_target = on_target or '' # Keep everything under a distributed lock with self.source.lock(self.lock_pattern.format(cid)): # Store information how many targets there were + info on what to invoke when they complete self.source.kvdb.conn.set(self.counter_pattern.format(cid), len(targets)) self.source.kvdb.conn.hmset(self.data_pattern.format(cid), { 'source': self.source.name, 'on_final': dumps(on_final), 'on_target': dumps(on_target), 'req_ts_utc': self.source.time.utcnow() }) # Invoke targets for name, payload in targets.items(): to_json_string = False if isinstance(payload, basestring) else True self.source.invoke_async(name, payload, self.call_channel, to_json_string=to_json_string, zato_ctx={self.request_ctx_cid_key: cid}) return cid
def set_payload(self, response, data_format, transport, service_instance, _sio_json=SIMPLE_IO.FORMAT.JSON, _url_type_soap=URL_TYPE.SOAP, _dict_like=(DATA_FORMAT.JSON, DATA_FORMAT.DICT), _AdminService=AdminService, _dumps=dumps, _basestring=basestring): """ Sets the actual payload to represent the service's response out of what the service produced. This includes converting dictionaries into JSON, adding Zato metadata and wrapping the mesasge in SOAP if need be. """ # type: (Response, str, str, Service) if isinstance(service_instance, AdminService): if data_format == _sio_json: zato_env = { 'zato_env': { 'result': response.result, 'cid': service_instance.cid, 'details': response.result_details } } if response.payload: payload = response.payload.getvalue(False) payload.update(zato_env) else: payload = zato_env response.payload = dumps(payload) else: if transport == _url_type_soap: zato_message_template = zato_message_soap else: zato_message_template = zato_message_declaration_uni if response.payload: if not isinstance(response.payload, _basestring): response.payload = self._get_xml_admin_payload( service_instance, zato_message_template, response.payload) else: response.payload = self._get_xml_admin_payload( service_instance, zato_message_template, None) else: if not isinstance(response.payload, _basestring): if isinstance(response.payload, dict) and data_format in _dict_like: response.payload = dumps(response.payload) else: response.payload = response.payload.getvalue( ) if response.payload else '' if transport == _url_type_soap: if not isinstance(service_instance, _AdminService): if self.use_soap_envelope: response.payload = soap_doc.format(body=response.payload)
def remote_command_execute(req): """ Executes a command against the key/value DB. """ try: response = req.zato.client.invoke('zato.kvdb.remote-command.execute', {'command': req.POST['command']}) if response.has_data: return HttpResponse(dumps({'message': dumps(response.data.result)}), content_type='application/javascript') else: raise ZatoException(msg=response.details) except Exception as e: return HttpResponseServerError(e.args[0])
def http_request(self, method, cid, data='', params=None, _has_debug=has_debug, *args, **kwargs): self._enforce_is_active() # We never touch strings/unicode because apparently the user already serialized outgoing data needs_serialize = not isinstance(data, basestring) if needs_serialize: if self.config['data_format'] == DATA_FORMAT.JSON: data = dumps(data) elif data and self.config['data_format'] == DATA_FORMAT.XML: data = tostring(data) headers = self._create_headers(cid, kwargs.pop('headers', {})) if self.config['transport'] == 'soap': data, headers = self._soap_data(data, headers) params = params or {} if self.path_params: address, qs_params = self.format_address(cid, params) else: address, qs_params = self.address, dict(params) if isinstance(data, unicode): data = data.encode('utf-8') logger.info( 'CID:`%s`, address:`%s`, qs:`%s`, auth_user:`%s`, kwargs:`%s`', cid, address, qs_params, self.username, kwargs) response = self.invoke_http(cid, method, address, data, headers, {}, params=qs_params, *args, **kwargs) if _has_debug: logger.debug('CID:`%s`, response:`%s`', cid, response.text) if needs_serialize: if self.config['data_format'] == DATA_FORMAT.JSON: response.data = loads(response.text) elif self.config['data_format'] == DATA_FORMAT.XML: if response.text and response.headers.get('Content-Type') in ( 'application/xml', 'text/xml'): response.data = fromstring(response.text) return response
def settings_basic_save(req): # Process explicitly named attributes for attr in profile_attrs: # Use False as default value so as to convert blank checkboxes into a boolean value value = req.POST.get(attr, False) setattr(req.zato.user_profile, attr, value) # Process opaque attributes opaque_attrs = {} for attr in profile_attrs_opaque: value = req.POST.get(attr) opaque_attrs[attr] = value # Encrypt TOTP before it is saved to the database totp_key = opaque_attrs.get('totp_key') if totp_key: # set_user_profile_totp_key(profile, zato_secret_key, key, label, opaque_attrs=None) set_user_profile_totp_key(req.zato.user_profile, zato_settings.zato_secret_key, totp_key, opaque_attrs.get('totp_key_label'), opaque_attrs) # Make sure all values are Unicode objects before serializing opaque attrs to JSON for key, value in opaque_attrs.items(): if not isinstance(value, unicode): opaque_attrs[key] = value.decode('utf8') # Save all opaque attributes along with the profile req.zato.user_profile.opaque1 = dumps(opaque_attrs) # Save the profile req.zato.user_profile.save() # Save preferred cluster colour markers for key, value in req.POST.items(): if key.startswith('color_') and value != DEFAULT_PROMPT: cluster_id = key.replace('color_', '') if 'checkbox_{}'.format(cluster_id) in req.POST: try: ccm = ClusterColorMarker.objects.get( cluster_id=cluster_id, user_profile=req.zato.user_profile) except ClusterColorMarker.DoesNotExist: ccm = ClusterColorMarker() ccm.cluster_id = cluster_id ccm.user_profile = req.zato.user_profile ccm.color = value ccm.save() else: ClusterColorMarker.objects.filter( cluster_id=cluster_id, user_profile=req.zato.user_profile).delete() msg = 'Settings saved' messages.add_message(req, messages.INFO, msg, extra_tags='success') return redirect(reverse('account-settings-basic'))
def _on_OUTGOING_SFTP_EXECUTE(self, msg, is_reconnect=False, _utcnow=datetime.utcnow): out = {} connection = self.connections[msg.id] # type: SFTPConnection start_time = _utcnow() try: result = connection.execute(msg.cid, msg.data, msg.log_level) # type: Output except ErrorReturnCode as e: out['stdout'] = e.stdout out['stderr'] = e.stderr except Exception as e: out['stderr'] = format_exc() out['is_ok'] = False else: out.update(result.to_dict()) finally: out['cid'] = msg.cid out['command_no'] = connection.command_no out['response_time'] = str(_utcnow() - start_time) print() print(111, out) print() return Response(data=dumps(out))
def invoke_action(req, pub_client_id, send_attrs=('id', 'pub_client_id', 'request_data', 'timeout')): try: request = {'cluster_id': req.zato.cluster_id} for name in send_attrs: request[name] = req.POST.get(name, '') response = req.zato.client.invoke('zato.channel.web-socket.invoke-wsx', request) if response.ok: response_data = response.data['response_data'] if isinstance(response_data, dict): response_data = dict(response_data) return HttpResponse(dumps(response_data), content_type='application/javascript') else: raise Exception(response.details) except Exception: msg = 'Caught an exception, e:`{}`'.format(format_exc()) logger.error(msg) return HttpResponseServerError(msg)
def set_instance_opaque_attrs(instance, input, skip=None, only=None, _zato_skip=_zato_opaque_skip_attrs): """ Given an SQLAlchemy object instance and incoming SimpleIO-based input, populates all opaque values of that instance. """ only = only or [] instance_opaque_attrs = None instance_attrs = set(instance.asdict()) input_attrs = set(input) if only: input_attrs = set([elem for elem in input_attrs if elem in only]) instance_attrs = set([elem for elem in instance_attrs if elem not in only]) # Any extra input attributes will be treated as opaque ones input_opaque_attrs = input_attrs - instance_attrs # Skip attributes related to pagination for name in chain(skip or [], _zato_skip): input_opaque_attrs.discard(name) # Prepare generic attributes for instance if GENERIC.ATTR_NAME in instance_attrs: instance_opaque_attrs = getattr(instance, GENERIC.ATTR_NAME) if instance_opaque_attrs: instance_opaque_attrs = loads(instance_opaque_attrs) else: instance_opaque_attrs = {} for name in input_opaque_attrs: instance_opaque_attrs[name] = input[name] # Set generic attributes for instance if instance_opaque_attrs is not None: setattr(instance, GENERIC.ATTR_NAME, dumps(instance_opaque_attrs))
def invoke_action_handler(req, service_name, send_attrs): # type: (str, tuple) -> object try: request = {'cluster_id': req.zato.cluster_id} for name in send_attrs: request[name] = req.POST.get(name, '') logger.info('Invoking `%s` with `%s`', service_name, request) response = req.zato.client.invoke(service_name, request) if response.ok: response_data = response.data['response_data'] if isinstance(response_data, dict): response_data = dict(response_data) logger.info('Returning `%s` from `%s`', response_data, service_name) return HttpResponse(dumps(response_data), content_type='application/javascript') else: raise Exception(response.details) except Exception: msg = 'Caught an exception, e:`{}`'.format(format_exc()) logger.error(msg) return HttpResponseServerError(msg)
def _get_key_value_list(req, service_name, input_dict): return_data = [] for item in req.zato.client.invoke(service_name, input_dict): return_data.append({'name': item.name}) return HttpResponse(dumps(return_data), content_type='application/javascript')
def command_shell_action(req, id, cluster_id, name_slug): try: response = req.zato.client.invoke( 'zato.outgoing.sftp.execute', { 'cluster_id': req.zato.cluster_id, 'id': id, 'data': req.POST['data'], 'log_level': req.POST['log_level'], }) if response.ok: data = response.data return HttpResponse(dumps({ 'msg': 'Response time: {} (#{})'.format(data.response_time, data.command_no), 'stdout': data.get('stdout') or '(None)', 'stderr': data.get('stderr') or '(None)', }), content_type='application/javascript') else: raise Exception(response.details) except Exception: msg = 'Caught an exception, e:`{}`'.format(format_exc()) logger.error(msg) return HttpResponseServerError(msg)
def test_client(self): cid = new_cid() headers = {'x-zato-cid': cid} ok = True env = {'details': rand_string(), 'result': ZATO_OK, 'cid': cid} sio_payload_key = rand_string() sio_payload = {rand_string(): rand_string()} sio_response = {'zato_env': env, sio_payload_key: sio_payload} text = dumps(sio_response) status_code = rand_int() client = self.get_client( FakeInnerResponse(headers, ok, text, status_code)) response = client.invoke() eq_(response.ok, ok) eq_(response.inner.text, text) eq_(iteritems(response.data), iteritems( (sio_response[sio_payload_key]))) eq_(response.has_data, True) eq_(response.cid, cid) eq_(response.cid, sio_response['zato_env']['cid']) eq_(response.details, sio_response['zato_env']['details'])
def _edit_create_response(req, id, verb, transport, connection, name): return_data = { 'id': id, 'transport': transport, 'message': 'Successfully {} the {} {} `{}`, check server logs for details'.format( verb, TRANSPORT[transport], CONNECTION[connection], name), } # If current item has a cache assigned, provide its human-friendly name to the caller response = req.zato.client.invoke('zato.http-soap.get', { 'cluster_id': req.zato.cluster_id, 'id': id, }) if response.data.cache_id: cache_type = response.data.cache_type cache_name = '{}/{}'.format(CACHE_TYPE[cache_type], response.data.cache_name) else: cache_type = None cache_name = None return_data['cache_type'] = cache_type return_data['cache_name'] = cache_name return HttpResponse(dumps(return_data), content_type='application/javascript')
def _invoke(self, func, func_name, url_path, request, expect_ok, auth=None, _not_given='_test_not_given'): address = Config.server_address.format(url_path) request['current_app'] = Config.current_app data = dumps(request) logger.info('Invoking %s %s with %s', func_name, address, data) response = func(address, data=data, auth=auth) logger.info('Response received %s %s', response.status_code, response.text) data = loads(response.text) data = bunchify(data) # Most tests require status OK and CID if expect_ok: self.assertNotEquals(data.get('cid', _not_given), _not_given) self.assertEquals(data.status, status_code.ok) return data
def _handle_attr_call(self, attr): try: key = self.request.input.key # This may be potentially an integer key that we received as string # so we need to try to convert it to one. try: key = int(key) except ValueError: pass # That is fine, it was not an integer values = attr[key] except KeyError: raise KeyError('No such key `{}` ({}) among `{}`'.format(key, type(key), sorted(attr.keys()))) values = values if isinstance(values, list) else [values] out = [elem.to_dict() for elem in values] out.sort(key=itemgetter(*self.request.input.sort_by), reverse=True) for item in out: for key, value in item.items(): if isinstance(value, datetime): item[key] = value.isoformat() self.response.payload = dumps(out)
def _invoke_async_retry(self, target, retry_repeats, retry_seconds, orig_cid, call_cid, callback, callback_context, args, kwargs, _utcnow=utcnow): # Request to invoke the background service with .. retry_request = { 'source': self.invoking_service.name, 'target': target, 'retry_repeats': retry_repeats, 'retry_seconds': retry_seconds, 'orig_cid': orig_cid, 'call_cid': call_cid, 'callback': callback, 'callback_context': callback_context, 'args': args, 'kwargs': kwargs, 'req_ts_utc': _utcnow() } return self.invoking_service.invoke_async( 'zato.pattern.invoke-retry.invoke-retry', dumps(retry_request), cid=call_cid)
def _request(self, op, key, value=NotGiven, pattern='/zato/cache/{}', op_verb_map=op_verb_map): # type: (str, str, str) -> str # Build a full address path = pattern.format(key) address = '{}{}'.format(self.address, path) # Get the HTTP verb to use in the request verb = op_verb_map[op] # type: str data = {'cache': self.cache_name, 'return_prev': True} if value is not NotGiven: data['value'] = value data = dumps(data) response = self.session.request(verb, address, data=data) # type: RequestsResponse return response.text
def _edit_create_response(service_response, action, name, password_type): return_data = {'id': service_response.data.id, 'message': 'Successfully {0} the WS-Security definition [{1}]'.format(action, name), 'password_type_raw':password_type, 'password_type':ZATO_WSS_PASSWORD_TYPES[password_type]} return HttpResponse(dumps(return_data), content_type='application/javascript')
def start_connector(self, ipc_tcp_start_port, timeout=5): """ Starts an HTTP server acting as an connector process. Its port will be greater than ipc_tcp_start_port, which is the starting point to find a free port from. """ if self.check_enabled: self._check_enabled() self.ipc_tcp_port = get_free_port(ipc_tcp_start_port) logger.info('Starting {} connector for server `%s` on port `%s`'.format(self.connector_name), self.server.name, self.ipc_tcp_port) # Credentials for both servers and connectors username, password = self.get_credentials() # Employ IPC to exchange subprocess startup configuration self.server.connector_config_ipc.set_config(self.ipc_config_name, dumps({ 'port': self.ipc_tcp_port, 'username': username, 'password': password, 'server_port': self.server.port, 'server_name': self.server.name, 'server_path': '/zato/internal/callback/{}'.format(self.callback_suffix), 'base_dir': self.server.base_dir, 'needs_pidfile': not self.server.has_fg, 'pidfile_suffix': self.pidfile_suffix, 'logging_conf_path': self.server.logging_conf_path })) # Start connector in a sub-process start_python_process('{} connector'.format(self.connector_name), False, self.connector_module, '', extra_options={ 'deployment_key': self.server.deployment_key, 'shmem_size': self.server.shmem_size }, stderr_path=self.server.stderr_path) # Wait up to timeout seconds for the connector to start as indicated by its responding to a PING request now = datetime.utcnow() warn_after = now + timedelta(seconds=60) should_warn = False until = now + timedelta(seconds=timeout) is_ok = False address = address_pattern.format(self.ipc_tcp_port, 'ping') auth = self.get_credentials() while not is_ok or now >= until: if not should_warn: if now >= warn_after: should_warn = True is_ok = self._ping_connector(address, auth, should_warn) if is_ok: break else: sleep(2) now = datetime.utcnow() if not is_ok: logger.warn('{} connector (%s) could not be started after %s'.format(self.connector_name), address, timeout) else: return is_ok
def on_call_finished(self, invoked_service, response, exception): now = invoked_service.time.utcnow() cid = invoked_service.wsgi_environ['zato.request_ctx.{}'.format(self.request_ctx_cid_key)] data_key = self.data_pattern.format(cid) counter_key = self.counter_pattern.format(cid) source, req_ts_utc, on_target = invoked_service.kvdb.conn.hmget(data_key, 'source', 'req_ts_utc', 'on_target') with invoked_service.lock(self.lock_pattern.format(cid)): data = Bunch() data.cid = cid data.resp_ts_utc = now data.response = response data.exception = exception data.ok = False if exception else True data.source = source data.target = invoked_service.name data.req_ts_utc = req_ts_utc # First store our response and exception (if any) json_data = dumps(data) invoked_service.kvdb.conn.hset(data_key, invoked_service.get_name(), json_data) on_target = loads(on_target) if logger.isEnabledFor(DEBUG): self._log_before_callbacks('on_target', on_target, invoked_service) # We always invoke 'on_target' callbacks, if there are any self.invoke_callbacks(invoked_service, data, on_target, self.on_target_channel, cid) # Was it the last parallel call? if not invoked_service.kvdb.conn.decr(counter_key): # Not every subclass will need final callbacks if self.needs_on_final: payload = invoked_service.kvdb.conn.hgetall(data_key) payload['data'] = {} for key in (key for key in payload.keys() if key not in JSON_KEYS): payload['data'][key] = loads(payload.pop(key)) for key in JSON_KEYS: if key not in ('source', 'data', 'req_ts_utc'): payload[key] = loads(payload[key]) on_final = payload['on_final'] if logger.isEnabledFor(DEBUG): self._log_before_callbacks('on_final', on_final, invoked_service) self.invoke_callbacks(invoked_service, payload, on_final, self.on_final_channel, cid) invoked_service.kvdb.conn.delete(counter_key) invoked_service.kvdb.conn.delete(data_key)
def serialize(self, to_string=False): # type: (bool) -> object out = { 'is_ok': False, 'cid': self.cid, 'message': self.get_error_message() } return dumps(out) if to_string else out
def publish(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ALL, *ignored_args, **ignored_kwargs): msg['msg_type'] = msg_type topic = TOPICS[msg_type] msg = dumps(msg) self.pub_client.publish(topic, msg)
def client_json_error(cid, faultstring): zato_env = { 'zato_env': { 'result': ZATO_ERROR, 'cid': cid, 'details': faultstring } } return dumps(zato_env)
def _edit_create_response(verb, id, name, engine_display_name, cluster_id): """ A common function for producing return data for create and edit actions. """ return_data = {'id': id, 'message': 'Successfully {} outgoing SQL connection `{}`'.format(verb, name.encode('utf-8')), 'engine_display_name': engine_display_name, 'cluster_id': cluster_id, } return HttpResponse(dumps(return_data), content_type='application/javascript')
def handle(self): response = {} response['url_sec'] = sorted( self.worker_store.request_handler.security.url_sec.items()) response['plain_http_handler.http_soap'] = sorted( self.worker_store.request_handler.plain_http_handler.http_soap. items()) response['soap_handler.http_soap'] = sorted( self.worker_store.request_handler.soap_handler.http_soap.items()) self.response.payload = dumps(response, sort_keys=True, indent=4) self.response.content_type = 'application/json'
def invoke_service_with_json_response(req, service, input_dict, ok_msg, error_template='', content_type='application/javascript', extra=None): try: req.zato.client.invoke(service, input_dict) except Exception as e: return HttpResponseServerError(e.message, content_type=content_type) else: response = {'msg': ok_msg} response.update(extra or {}) response = dumps(response) return HttpResponse(response, content_type=content_type)
def _edit_create_response(client, verb, id, name, def_id, cluster_id): response = client.invoke('zato.definition.amqp.get-by-id', { 'id': def_id, 'cluster_id': cluster_id }) return_data = { 'id': id, 'message': 'Successfully {} the AMQP channel `{}`'.format(verb, name), 'def_name': response.data.name } return HttpResponse(dumps(return_data), content_type='application/javascript')
def to_sql_dict(self, needs_bunch=False, skip=None): out = {} skip = skip or [] for name in self.__slots__: if name in skip: continue if name != 'opaque': out[name] = getattr(self, name) else: out[GENERIC.ATTR_NAME] = dumps(self.opaque) return bunchify(out) if needs_bunch else out