def test__startup_services(self): class FakeBrokerClient(object): def __init__(self): self.messages = {} def invoke_async(self, msg): self.messages[msg['service']] = msg broker_client = FakeBrokerClient() startup_services = Bunch() for x in range(10): name = rand_string() payload = rand_string() startup_services[name] = payload ps = ParallelServer() ps.broker_client = broker_client ps.fs_server_config = Bunch() ps.fs_server_config.startup_services = startup_services ps.invoke_startup_services() for expected_service, expected_payload in startup_services.items(): msg = Bunch(broker_client.messages[expected_service]) eq_(msg.action, SERVICE.PUBLISH) eq_(msg.channel, CHANNEL.STARTUP_SERVICE) eq_(msg.payload, expected_payload) eq_(msg.service, expected_service) ok_(msg.cid.startswith('K')) self.assertEquals(len(msg.cid), 40)
def test__set_tls_info(self): expected_cert_dict = rand_string() expected_cert_der = rand_string() expected_cert_sha1 = sha1(expected_cert_der).hexdigest().upper() for wsgi_url_scheme in('https', 'http'): wsgi_environ = { 'wsgi.url_scheme': wsgi_url_scheme, 'gunicorn.socket': FakeGunicornSocket(expected_cert_der, expected_cert_dict), 'zato.http.response.status': rand_string(), 'zato.http.channel_item': Bunch(audit_enabled=False), 'PATH_INFO': rand_string(), 'REQUEST_METHOD': rand_string(), 'SERVER_PROTOCOL': rand_string(), 'HTTP_USER_AGENT': rand_string(), } ps = ParallelServer() ps.worker_store = FakeWorkerStore() ps.on_wsgi_request(wsgi_environ, StartResponse()) if wsgi_url_scheme == 'https': eq_(wsgi_environ['zato.tls.client_cert.dict'], expected_cert_dict) eq_(wsgi_environ['zato.tls.client_cert.der'], expected_cert_der) eq_(wsgi_environ['zato.tls.client_cert.sha1'], expected_cert_sha1) else: self.assertTrue('zato.tls.client_cert.dict' not in wsgi_environ) self.assertTrue('zato.tls.client_cert.der' not in wsgi_environ) self.assertTrue('zato.tls.client_cert.sha1' not in wsgi_environ)
def parallel_server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() #server.request_handler = self.request_handler() #server.request_handler.soap_handler.server = server #server.request_handler.plain_http_handler.server = server return server
def test_set_tls_info(self): expected_cert_dict = rand_string() expected_cert_der = rand_string() expected_cert_sha1 = sha1(expected_cert_der).hexdigest().upper() class FakeRequestDispatcher(object): def dispatch(self, *ignored_args, **ignored_kwargs): return rand_string() class FakeWorkerStore(object): request_dispatcher = FakeRequestDispatcher() class FakeGunicornSocket(object): def getpeercert(self, needs_der=False): if needs_der: return expected_cert_der return expected_cert_dict def start_response(*ignored_args, **ignored_kwargs): pass for wsgi_url_scheme in('https', 'http'): wsgi_environ = { 'wsgi.url_scheme': wsgi_url_scheme, 'gunicorn.socket': FakeGunicornSocket(), 'zato.http.response.status': rand_string() } ps = ParallelServer() ps.worker_store = FakeWorkerStore() ps.on_wsgi_request(wsgi_environ, start_response) if wsgi_url_scheme == 'https': eq_(wsgi_environ['zato.tls.client_cert.dict'], expected_cert_dict) eq_(wsgi_environ['zato.tls.client_cert.der'], expected_cert_der) eq_(wsgi_environ['zato.tls.client_cert.sha1'], expected_cert_sha1) else: self.assertTrue('zato.tls.client_cert.dict' not in wsgi_environ) self.assertTrue('zato.tls.client_cert.der' not in wsgi_environ) self.assertTrue('zato.tls.client_cert.sha1' not in wsgi_environ)
def test_access_log(self): def _utcnow(self): return datetime(year=2014, month=1, day=12, hour=16, minute=22, second=12, tzinfo=UTC) local_tz = get_localzone() _now = _utcnow(None) local_dt = _now.replace(tzinfo=UTC).astimezone(local_tz) local_dt = local_tz.normalize(local_dt) request_timestamp = local_dt.strftime(ACCESS_LOG_DT_FORMAT) with patch('arrow.factory.ArrowFactory.utcnow', _utcnow): response = rand_string() * rand_int() cid = new_cid() cluster_id = 1 channel_name = rand_string() url_path = '/{}'.format(rand_string()) user_agent = rand_string() http_version = rand_string() request_method = rand_string() remote_ip = '10.{}.{}.{}'.format(rand_int(), rand_int(), rand_int()) req_timestamp_utc = utcnow() channel_item = { 'name': channel_name, 'audit_enabled': False, 'is_active': True, 'transport': 'plain_http', 'data_format': None, 'match_target': url_path } wsgi_environ = { 'gunicorn.socket': FakeGunicornSocket(None, None), 'wsgi.url_scheme': 'http', 'wsgi.input': StringIO(response), 'zato.http.response.status': httplib.OK, 'zato.http.channel_item': channel_item, 'zato.request_timestamp_utc': req_timestamp_utc, 'HTTP_X_FORWARDED_FOR': remote_ip, 'PATH_INFO': url_path, 'REQUEST_METHOD': request_method, 'SERVER_PROTOCOL': http_version, 'HTTP_USER_AGENT': user_agent, } class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=cluster_id) def session(self): return fake_session class FakeURLData(URLData): def __init__(self): self.url_sec = {url_path: Bunch(sec_def=ZATO_NONE)} def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=response, content_type='text/plain', headers={}, status_code=httplib.OK) class FakeAccessLogger(object): def __init__(self): self.extra = {} def info(self, msg, extra): self.extra = extra def isEnabledFor(self, ignored): return True bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler() ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.access_logger = FakeAccessLogger() ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=cid) extra = Bunch(ps.access_logger.extra) eq_(extra.channel_name, channel_name) eq_(extra.user_agent, user_agent) eq_(extra.status_code, '200') eq_(extra.http_version, http_version) eq_(extra.response_size, len(response)) eq_(extra.cid, cid) eq_(extra.path, url_path) eq_(extra.method, request_method) eq_(extra.remote_ip, remote_ip) eq_(extra.req_timestamp_utc, '12/Jan/2014:16:22:12 +0000') eq_(extra.req_timestamp, request_timestamp)
def test_audit(self): for expected_audit_enabled in(True, False): for expected_status_code in(httplib.OK, httplib.FORBIDDEN): for use_x_remote_addr in(True, False): expected_auth_ok = True if expected_status_code == httplib.OK else False expected_invoke_ok = True if expected_auth_ok is True else False expected_cid = new_cid() expected_url_scheme = rand_string() expected_payload = rand_string() expected_audit_repl_patt_type = rand_string() expected_replace_patterns_elem_path = [] expected_replace_patterns_xpath = [] expected_cluster_id = rand_int() expected_id = rand_int() expected_name = rand_string() expected_password = '******' expected_username = rand_string() expected_transport = rand_string() expected_connection = rand_string() expected_data_format = DATA_FORMAT.JSON expected_is_active = True expected_request = rand_string() expected_audit_max_payload = len(expected_request) - 7 # Substracting any value would do expected_channel_item_key1 = rand_string() expected_channel_item_value1 = rand_string() expected_match_target = rand_string() channel_item = { 'id': expected_id, 'name': expected_name, 'transport': expected_transport, 'connection': expected_connection, 'audit_enabled': expected_audit_enabled, expected_channel_item_key1:expected_channel_item_value1, 'audit_repl_patt_type': expected_audit_repl_patt_type, 'replace_patterns_elem_path': expected_replace_patterns_elem_path, 'replace_patterns_xpath': expected_replace_patterns_xpath, 'audit_max_payload': expected_audit_max_payload, 'is_active': expected_is_active, 'data_format': DATA_FORMAT.JSON, 'match_target': expected_match_target, 'username': expected_username, } wsgi_environ = { 'wsgi.url_scheme': expected_url_scheme, 'gunicorn.socket': FakeGunicornSocket(None, None), 'zato.http.response.status': expected_status_code, 'zato.http.channel_item': channel_item, 'PATH_INFO': rand_string(), 'wsgi.input': StringIO(expected_request), 'REQUEST_METHOD': rand_string(), 'SERVER_PROTOCOL': rand_string(), 'HTTP_USER_AGENT': rand_string(), } expected_remote_addr = rand_string() if use_x_remote_addr: expected_remote_addr_header = 'HTTP_X_FORWARDED_FOR' wsgi_environ[expected_remote_addr_header] = expected_remote_addr else: expected_remote_addr_header = 'REMOTE_ADDR' wsgi_environ[expected_remote_addr_header] = expected_remote_addr class FakeSession: def __init__(self, audit=None): self.audit = audit self.commit_called = False def close(self): pass def commit(self): self.commit_called = True def add(self, audit): self.audit = audit fake_session = FakeSession() class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=expected_cluster_id) def session(self): return fake_session class FakeURLData(URLData): def __init__(self): self.url_sec = {expected_match_target: Bunch(sec_def=ZATO_NONE)} def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=expected_payload, content_type='text/plain', headers={}, status_code=expected_status_code) bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler() ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=expected_cid) if expected_audit_enabled: # # Audit 1/2 - Request # # Parsing will confirm the proper value was used datetime.strptime(fake_session.audit.req_time.isoformat(), '%Y-%m-%dT%H:%M:%S.%f') self.assertEquals(fake_session.audit.name, expected_name) self.assertEquals(fake_session.audit.cid, expected_cid) self.assertEquals(fake_session.audit.transport, expected_transport) self.assertEquals(fake_session.audit.connection, expected_connection) self.assertEquals(fake_session.audit.resp_time, None) self.assertEquals(fake_session.audit.user_token, expected_username) self.assertEquals(fake_session.audit.auth_ok, None) self.assertEquals(fake_session.audit.invoke_ok, None) self.assertEquals(fake_session.audit.remote_addr, expected_remote_addr) self.assertEquals(fake_session.audit.req_payload, expected_request[:expected_audit_max_payload]) self.assertEquals(fake_session.audit.resp_headers, None) self.assertEquals(fake_session.audit.resp_payload, None) req_headers = literal_eval(fake_session.audit.req_headers) self.assertEquals(req_headers[expected_remote_addr_header], repr(expected_remote_addr)) self.assertEquals(req_headers['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(req_headers['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) channel_item = literal_eval(req_headers['zato.http.channel_item']) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['username'], expected_username) self.assertEquals(channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals(channel_item['replace_patterns_elem_path'], expected_replace_patterns_elem_path) self.assertEquals(channel_item['is_active'], expected_is_active) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['match_target'], expected_match_target) # # Audit 2/2 - Response # self.assertEquals(bc.msg['action'], CHANNEL_BROKER_MESSAGE.HTTP_SOAP_AUDIT_RESPONSE) self.assertEquals(bc.msg['cid'], expected_cid) self.assertEquals(bc.msg['data_format'], DATA_FORMAT.JSON) self.assertEquals(bc.msg['service'], 'zato.http-soap.set-audit-response-data') payload = loads(bc.msg['payload']) self.assertEquals(payload['auth_ok'], expected_auth_ok) self.assertEquals(payload['invoke_ok'], expected_invoke_ok) self.assertEquals(payload['resp_payload'], expected_payload) # Parsing alone will check its format is valid datetime.strptime(payload['resp_time'], '%Y-%m-%dT%H:%M:%S.%f') wsgi_environ = loads(payload['resp_headers']) self.assertEquals(wsgi_environ['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(wsgi_environ['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) self.assertEquals(wsgi_environ['zato.http.response.status'], "'{} {}'".format( expected_status_code, httplib.responses[expected_status_code], )) channel_item = literal_eval(wsgi_environ['zato.http.channel_item']) self.assertEquals(channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals(channel_item['replace_patterns_elem_path'], expected_replace_patterns_elem_path) self.assertEquals(channel_item['replace_patterns_xpath'], expected_replace_patterns_xpath) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['id'], expected_id) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['connection'], expected_connection) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['is_active'], expected_is_active) else: # Audit not enabled so no response audit message was published on the broker self.assertTrue(bc.msg is None)
def parallel_server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() server.sql_pool_store = self.sql_pool_store() server.int_parameters = self.int_parameters() server.int_parameter_suffixes = self.int_parameter_suffixes() server.bool_parameter_prefixes = self.bool_parameter_prefixes() server.internal_service_modules = self.internal_service_modules() server.service_modules = self.service_modules() server.kvdb = self.kvdb() server.user_config = Bunch() return server
def test_access_log(self): def _utcnow(): return datetime(year=2014, month=1, day=12, hour=16, minute=22, second=12, tzinfo=UTC) local_tz = get_localzone() _now = _utcnow() local_dt = _now.replace(tzinfo=UTC).astimezone(local_tz) local_dt = local_tz.normalize(local_dt) request_timestamp = local_dt.strftime(ACCESS_LOG_DT_FORMAT) response = rand_string() * rand_int() cid = new_cid() cluster_id = 1 channel_name = rand_string() url_path = '/{}'.format(rand_string()) user_agent = rand_string() http_version = rand_string() request_method = rand_string() remote_ip = '10.{}.{}.{}'.format(rand_int(), rand_int(), rand_int()) req_timestamp_utc = utcnow() channel_item = { 'name': channel_name, 'is_active': True, 'transport': 'plain_http', 'data_format': None, 'match_target': url_path, 'method': '', } wsgi_environ = { 'gunicorn.socket': FakeGunicornSocket(None, None), 'wsgi.url_scheme': 'http', 'wsgi.input': StringIO(response), 'zato.http.response.status': OK, 'zato.channel_item': channel_item, 'zato.request_timestamp_utc': req_timestamp_utc, 'HTTP_X_FORWARDED_FOR': remote_ip, 'PATH_INFO': url_path, 'REQUEST_METHOD': request_method, 'SERVER_PROTOCOL': http_version, 'HTTP_USER_AGENT': user_agent, } class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=cluster_id) class FakeURLData(URLData): def __init__(self): self.url_sec = { url_path: Bunch(sec_def=ZATO_NONE, sec_use_rbac=False) } def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=response, content_type='text/plain', headers={}, status_code=OK) class FakeAccessLogger(object): def __init__(self): self.level = object() self.msg = object() self.args = object() self.exc_info = object() self.extra = object() def _log(self, level, msg, args, exc_info, extra): self.level = level self.msg = msg self.args self.exc_info = exc_info self.extra = extra def isEnabledFor(self, ignored): return True bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler() ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.request_dispatcher_dispatch = ws.request_dispatcher.dispatch ps.access_logger = FakeAccessLogger() ps.access_logger_log = ps.access_logger._log ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=cid, _utcnow=_utcnow) extra = Bunch(ps.access_logger.extra) eq_(extra.channel_name, channel_name) eq_(extra.user_agent, user_agent) eq_(extra.status_code, '200') eq_(extra.http_version, http_version) eq_(extra.response_size, len(response)) eq_(extra.path, url_path) eq_(extra.cid_resp_time, '{}/0.0'.format( cid)) # It's 0.0 because we mock utcnow to be a constant value eq_(extra.method, request_method) eq_(extra.remote_ip, remote_ip) eq_(extra.req_timestamp_utc, '12/Jan/2014:16:22:12 +0000') eq_(extra.req_timestamp, request_timestamp)
def server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() server.service_store.server = server server.sql_pool_store = self.sql_pool_store() server.int_parameters = self.int_parameters() server.int_parameter_suffixes = self.int_parameter_suffixes() server.bool_parameter_prefixes = self.bool_parameter_prefixes() server.internal_service_modules = self.internal_service_modules() server.service_modules = self.service_modules() server.kvdb = self.kvdb() server.user_config = Bunch() return server
def test_audit(self): for expected_audit_enabled in (True, False): for expected_status_code in (httplib.OK, httplib.FORBIDDEN): for use_x_remote_addr in (True, False): expected_auth_ok = True if expected_status_code == httplib.OK else False expected_invoke_ok = True if expected_auth_ok is True else False expected_cid = new_cid() expected_url_scheme = rand_string() expected_payload = rand_string() expected_audit_repl_patt_type = rand_string() expected_replace_patterns_json_pointer = [] expected_replace_patterns_xpath = [] expected_cluster_id = rand_int() expected_id = rand_int() expected_name = rand_string() expected_password = '******' expected_username = rand_string() expected_transport = rand_string() expected_connection = rand_string() expected_data_format = DATA_FORMAT.JSON expected_is_active = True expected_request = rand_string() expected_audit_max_payload = len( expected_request ) - 7 # Substracting any value would do expected_channel_item_key1 = rand_string() expected_channel_item_value1 = rand_string() expected_match_target = rand_string() channel_item = { 'id': expected_id, 'name': expected_name, 'transport': expected_transport, 'connection': expected_connection, 'audit_enabled': expected_audit_enabled, expected_channel_item_key1: expected_channel_item_value1, 'audit_repl_patt_type': expected_audit_repl_patt_type, 'replace_patterns_json_pointer': expected_replace_patterns_json_pointer, 'replace_patterns_xpath': expected_replace_patterns_xpath, 'audit_max_payload': expected_audit_max_payload, 'is_active': expected_is_active, 'data_format': DATA_FORMAT.JSON, 'match_target': expected_match_target, 'username': expected_username, 'method': '', } wsgi_environ = { 'wsgi.url_scheme': expected_url_scheme, 'gunicorn.socket': FakeGunicornSocket(None, None), 'zato.http.response.status': expected_status_code, 'zato.http.channel_item': channel_item, 'PATH_INFO': rand_string(), 'wsgi.input': StringIO(expected_request), 'REQUEST_METHOD': rand_string(), 'SERVER_PROTOCOL': rand_string(), 'HTTP_USER_AGENT': rand_string(), } expected_remote_addr = rand_string() if use_x_remote_addr: expected_remote_addr_header = 'HTTP_X_FORWARDED_FOR' wsgi_environ[ expected_remote_addr_header] = expected_remote_addr else: expected_remote_addr_header = 'REMOTE_ADDR' wsgi_environ[ expected_remote_addr_header] = expected_remote_addr class FakeSession: def __init__(self, audit=None): self.audit = audit self.commit_called = False def close(self): pass def commit(self): self.commit_called = True def add(self, audit): self.audit = audit fake_session = FakeSession() class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=expected_cluster_id) def session(self): return fake_session class FakeURLData(URLData): def __init__(self): self.url_sec = { expected_match_target: Bunch(sec_def=ZATO_NONE) } def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=expected_payload, content_type='text/plain', headers={}, status_code=expected_status_code) bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler( ) ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=expected_cid) if expected_audit_enabled: # # Audit 1/2 - Request # # Parsing will confirm the proper value was used datetime.strptime( fake_session.audit.req_time.isoformat(), '%Y-%m-%dT%H:%M:%S.%f') self.assertEquals(fake_session.audit.name, expected_name) self.assertEquals(fake_session.audit.cid, expected_cid) self.assertEquals(fake_session.audit.transport, expected_transport) self.assertEquals(fake_session.audit.connection, expected_connection) self.assertEquals(fake_session.audit.resp_time, None) self.assertEquals(fake_session.audit.user_token, expected_username) self.assertEquals(fake_session.audit.auth_ok, None) self.assertEquals(fake_session.audit.invoke_ok, None) self.assertEquals(fake_session.audit.remote_addr, expected_remote_addr) self.assertEquals( fake_session.audit.req_payload, expected_request[:expected_audit_max_payload]) self.assertEquals(fake_session.audit.resp_headers, None) self.assertEquals(fake_session.audit.resp_payload, None) req_headers = literal_eval( fake_session.audit.req_headers) self.assertEquals( req_headers[expected_remote_addr_header], repr(expected_remote_addr)) self.assertEquals(req_headers['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(req_headers['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) channel_item = literal_eval( req_headers['zato.http.channel_item']) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['username'], expected_username) self.assertEquals( channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals( channel_item['replace_patterns_json_pointer'], expected_replace_patterns_json_pointer) self.assertEquals(channel_item['is_active'], expected_is_active) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['match_target'], expected_match_target) # # Audit 2/2 - Response # self.assertEquals( bc.msg['action'], CHANNEL_BROKER_MESSAGE. HTTP_SOAP_AUDIT_RESPONSE.value) self.assertEquals(bc.msg['cid'], expected_cid) self.assertEquals(bc.msg['data_format'], DATA_FORMAT.JSON) self.assertEquals( bc.msg['service'], 'zato.http-soap.set-audit-response-data') payload = loads(bc.msg['payload']) self.assertEquals(payload['auth_ok'], expected_auth_ok) self.assertEquals(payload['invoke_ok'], expected_invoke_ok) self.assertEquals(payload['resp_payload'], expected_payload) # Parsing alone will check its format is valid datetime.strptime(payload['resp_time'], '%Y-%m-%dT%H:%M:%S.%f') wsgi_environ = loads(payload['resp_headers']) self.assertEquals(wsgi_environ['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(wsgi_environ['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) self.assertEquals( wsgi_environ['zato.http.response.status'], "'{} {}'".format( expected_status_code, httplib.responses[expected_status_code], )) channel_item = literal_eval( wsgi_environ['zato.http.channel_item']) self.assertEquals( channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals( channel_item['replace_patterns_json_pointer'], expected_replace_patterns_json_pointer) self.assertEquals( channel_item['replace_patterns_xpath'], expected_replace_patterns_xpath) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['id'], expected_id) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['connection'], expected_connection) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['is_active'], expected_is_active) else: # Audit not enabled so no response audit message was published on the broker self.assertTrue(bc.msg is None)
def run(base_dir, start_gunicorn_app=True, options=None): # type: (str, bool, dict) options = options or {} # Store a pidfile before doing anything else store_pidfile(base_dir) # For dumping stacktraces register_diag_handlers() # Capture warnings to log files logging.captureWarnings(True) # Start initializing the server now os.chdir(base_dir) try: import pymysql pymysql.install_as_MySQLdb() except ImportError: pass # We know we don't need warnings because users may explicitly configure no certificate validation. # We don't want for urllib3 to warn us about it. import requests as _r _r.packages.urllib3.disable_warnings() repo_location = os.path.join(base_dir, 'config', 'repo') # Configure the logging first, before configuring the actual server. logging.addLevelName('TRACE1', TRACE1) logging_conf_path = os.path.join(repo_location, 'logging.conf') with open(logging_conf_path) as f: logging_config = yaml.load(f, yaml.FullLoader) dictConfig(logging_config) logger = logging.getLogger(__name__) kvdb_logger = logging.getLogger('zato_kvdb') crypto_manager = ServerCryptoManager(repo_location, secret_key=options['secret_key'], stdin_data=read_stdin_data()) secrets_config = ConfigObj(os.path.join(repo_location, 'secrets.conf'), use_zato=False) server_config = get_config(repo_location, 'server.conf', crypto_manager=crypto_manager, secrets_conf=secrets_config) pickup_config = get_config(repo_location, 'pickup.conf') sio_config = get_config(repo_location, 'simple-io.conf', needs_user_config=False) sio_config = get_sio_server_config(sio_config) sso_config = get_config(repo_location, 'sso.conf', needs_user_config=False) normalize_sso_config(sso_config) # Now that we have access to server.conf, greenify libraries required to be made greenlet-friendly, # assuming that there are any - otherwise do not do anything. to_greenify = [] for key, value in server_config.get('greenify', {}).items(): if asbool(value): if not os.path.exists(key): raise ValueError('No such path `{}`'.format(key)) else: to_greenify.append(key) # Go ahead only if we actually have anything to greenify if to_greenify: import greenify greenify.greenify() for name in to_greenify: result = greenify.patch_lib(name) if not result: raise ValueError( 'Library `{}` could not be greenified'.format(name)) else: logger.info('Greenified library `%s`', name) server_config.main.token = server_config.main.token.encode('utf8') # Do not proceed unless we can be certain our own preferred address or IP can be obtained. preferred_address = server_config.preferred_address.get('address') if not preferred_address: preferred_address = get_preferred_ip(server_config.main.gunicorn_bind, server_config.preferred_address) if not preferred_address and not server_config.server_to_server.boot_if_preferred_not_found: msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf' logger.warn(msg) raise Exception(msg) # Create the startup callable tool as soon as practical startup_callable_tool = StartupCallableTool(server_config) # Run the hook before there is any server object created startup_callable_tool.invoke(SERVER_STARTUP.PHASE.FS_CONFIG_ONLY, kwargs={ 'server_config': server_config, 'pickup_config': pickup_config, 'sio_config': sio_config, 'sso_config': sso_config, 'base_dir': base_dir, }) # New in 2.0 - Start monitoring as soon as possible if server_config.get('newrelic', {}).get('config'): import newrelic.agent newrelic.agent.initialize(server_config.newrelic.config, server_config.newrelic.environment or None, server_config.newrelic.ignore_errors or None, server_config.newrelic.log_file or None, server_config.newrelic.log_level or None) zunicorn.SERVER_SOFTWARE = server_config.misc.get('http_server_header', 'Zato') # Store KVDB config in logs, possibly replacing its password if told to kvdb_config = get_kvdb_config_for_log(server_config.kvdb) kvdb_logger.info('Main process config `%s`', kvdb_config) # New in 2.0 hence optional user_locale = server_config.misc.get('locale', None) if user_locale: locale.setlocale(locale.LC_ALL, user_locale) value = 12345 logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value, locale.currency(value, grouping=True).decode('utf-8')) if server_config.misc.http_proxy: os.environ['http_proxy'] = server_config.misc.http_proxy # Basic components needed for the server to boot up kvdb = KVDB() odb_manager = ODBManager(well_known_data=ZATO_CRYPTO_WELL_KNOWN_DATA) sql_pool_store = PoolStore() service_store = ServiceStore() service_store.odb = odb_manager service_store.services = {} server = ParallelServer() server.odb = odb_manager server.service_store = service_store server.service_store.server = server server.sql_pool_store = sql_pool_store server.service_modules = [] server.kvdb = kvdb server.stderr_path = options.get('stderr_path') # Assigned here because it is a circular dependency odb_manager.parallel_server = server zato_gunicorn_app = ZatoGunicornApplication(server, repo_location, server_config.main, server_config.crypto) server.has_fg = options.get('fg') server.crypto_manager = crypto_manager server.odb_data = server_config.odb server.host = zato_gunicorn_app.zato_host server.port = zato_gunicorn_app.zato_port server.repo_location = repo_location server.user_conf_location = os.path.join(server.repo_location, 'user-conf') server.base_dir = base_dir server.logs_dir = os.path.join(server.base_dir, 'logs') server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls') server.static_dir = os.path.join(server.base_dir, 'config', 'repo', 'static') server.json_schema_dir = os.path.join(server.base_dir, 'config', 'repo', 'schema', 'json') server.fs_server_config = server_config server.fs_sql_config = get_config(repo_location, 'sql.conf', needs_user_config=False) server.pickup_config = pickup_config server.logging_config = logging_config server.logging_conf_path = logging_conf_path server.sio_config = sio_config server.sso_config = sso_config server.user_config.update(server_config.user_config_items) server.preferred_address = preferred_address server.sync_internal = options['sync_internal'] server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8') server.startup_callable_tool = startup_callable_tool server.is_sso_enabled = server.fs_server_config.component_enabled.sso if server.is_sso_enabled: server.sso_api = SSOAPI(server, sso_config, None, crypto_manager.encrypt, crypto_manager.decrypt, crypto_manager.hash_secret, crypto_manager.verify_hash, new_user_id) # Remove all locks possibly left over by previous server instances kvdb.component = 'master-proc' clear_locks(kvdb, server_config.main.token, server_config.kvdb, crypto_manager.decrypt) # New in 2.0.8 server.return_tracebacks = asbool( server_config.misc.get('return_tracebacks', True)) server.default_error_message = server_config.misc.get( 'default_error_message', 'An error has occurred') # Turn the repo dir into an actual repository and commit any new/modified files RepoManager(repo_location).ensure_repo_consistency() # New in 2.0 so it's optional. profiler_enabled = server_config.get('profiler', {}).get('enabled', False) # New in 2.0 so it's optional. sentry_config = server_config.get('sentry') dsn = sentry_config.pop('dsn', None) if dsn: from raven import Client from raven.handlers.logging import SentryHandler handler_level = sentry_config.pop('level') client = Client(dsn, **sentry_config) handler = SentryHandler(client=client) handler.setLevel(getattr(logging, handler_level)) logger = logging.getLogger('') logger.addHandler(handler) for name in logging.Logger.manager.loggerDict: if name.startswith('zato'): logger = logging.getLogger(name) logger.addHandler(handler) if asbool(profiler_enabled): # Repoze from repoze.profile import ProfileMiddleware profiler_dir = os.path.abspath( os.path.join(base_dir, server_config.profiler.profiler_dir)) server.on_wsgi_request = ProfileMiddleware( server.on_wsgi_request, log_filename=os.path.join(profiler_dir, server_config.profiler.log_filename), cachegrind_filename=os.path.join( profiler_dir, server_config.profiler.cachegrind_filename), discard_first_request=server_config.profiler.discard_first_request, flush_at_shutdown=server_config.profiler.flush_at_shutdown, path=server_config.profiler.url_path, unwind=server_config.profiler.unwind) # New in 2.0 - set environmet variables for servers to inherit os_environ = server_config.get('os_environ', {}) for key, value in os_environ.items(): os.environ[key] = value # Run the hook right before the Gunicorn-level server actually starts startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IMPL_BEFORE_RUN, kwargs={ 'zato_gunicorn_app': zato_gunicorn_app, }) # Run the app at last if start_gunicorn_app: zato_gunicorn_app.run() else: return zato_gunicorn_app.zato_wsgi_app
def parallel_server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() server.sql_pool_store = self.sql_pool_store() server.int_parameters = self.int_parameters() server.int_parameter_suffixes = self.int_parameter_suffixes() server.bool_parameter_prefixes = self.bool_parameter_prefixes() server.soap11_content_type = self.soap11_content_type() server.soap12_content_type = self.soap12_content_type() server.plain_xml_content_type = self.plain_xml_content_type() server.json_content_type = self.json_content_type() server.internal_service_modules = self.internal_service_modules() server.service_modules = self.service_modules() server.kvdb = self.kvdb() return server
def setUp(self): # For mocking out Vault responses self.vault_adapter = RequestsAdapter() # We are always the first process in a server os.environ['ZATO_SERVER_WORKER_IDX'] = '1' # Represents the server.conf file self.fs_server_config = FSServerConfig() self.worker_config = ConfigStore() self.fernet_key = Fernet.generate_key() # type: str self.crypto_manager = CryptoManager(secret_key=self.fernet_key) self.vault_conn_api = VaultConnAPI(requests_adapter=self.vault_adapter) self.server = ParallelServer() self.server.fs_server_config = self.fs_server_config self.server.kvdb = KVDB() self.server.component_enabled.stats = False self.server.component_enabled.slow_response = False self.server.crypto_manager = self.crypto_manager self.service_store = ServiceStore(is_testing=True) self.service_store.server = self.server self.service_store.services = {} self.server.service_store = self.service_store self.fs_sql_config = { UNITTEST.SQL_ENGINE: { 'ping_query': 'SELECT 1+1' } } self.cache = Cache() self.sql_pool_store = PoolStore() self.worker_store = WorkerStore(self.worker_config, self.server) self.worker_store.sql_pool_store = self.sql_pool_store self.worker_store.stomp_outconn_api = None self.worker_store.outconn_wsx = None self.worker_store.vault_conn_api = self.vault_conn_api self.worker_store.sms_twilio_api = None self.worker_store.out_sap = None self.worker_store.out_sftp = None self.worker_store.outconn_ldap = {} self.worker_store.outconn_mongodb = {} self.worker_store.def_kafka = {} self.worker_store.cache_api = CacheAPI(self.server) self.worker_store.cache_api.default = self.cache self.request_handler = RequestHandler(self.server) self.wsgi_environ = { 'HTTP_HOST': 'api.localhost' } # Callback methods for particular SQL queries self.sql_callback_by_idx = {}