def test__startup_services(self):
class FakeBrokerClient(object):
def __init__(self):
self.messages = {}
def invoke_async(self, msg):
self.messages[msg['service']] = msg
broker_client = FakeBrokerClient()
startup_services = Bunch()
for x in range(10):
name = rand_string()
payload = rand_string()
startup_services[name] = payload
ps = ParallelServer()
ps.broker_client = broker_client
ps.fs_server_config = Bunch()
ps.fs_server_config.startup_services = startup_services
ps.invoke_startup_services()
for expected_service, expected_payload in startup_services.items():
msg = Bunch(broker_client.messages[expected_service])
eq_(msg.action, SERVICE.PUBLISH)
eq_(msg.channel, CHANNEL.STARTUP_SERVICE)
eq_(msg.payload, expected_payload)
eq_(msg.service, expected_service)
ok_(msg.cid.startswith('K'))
self.assertEquals(len(msg.cid), 40)
def test__set_tls_info(self):
expected_cert_dict = rand_string()
expected_cert_der = rand_string()
expected_cert_sha1 = sha1(expected_cert_der).hexdigest().upper()
for wsgi_url_scheme in('https', 'http'):
wsgi_environ = {
'wsgi.url_scheme': wsgi_url_scheme,
'gunicorn.socket': FakeGunicornSocket(expected_cert_der, expected_cert_dict),
'zato.http.response.status': rand_string(),
'zato.http.channel_item': Bunch(audit_enabled=False),
'PATH_INFO': rand_string(),
'REQUEST_METHOD': rand_string(),
'SERVER_PROTOCOL': rand_string(),
'HTTP_USER_AGENT': rand_string(),
}
ps = ParallelServer()
ps.worker_store = FakeWorkerStore()
ps.on_wsgi_request(wsgi_environ, StartResponse())
if wsgi_url_scheme == 'https':
eq_(wsgi_environ['zato.tls.client_cert.dict'], expected_cert_dict)
eq_(wsgi_environ['zato.tls.client_cert.der'], expected_cert_der)
eq_(wsgi_environ['zato.tls.client_cert.sha1'], expected_cert_sha1)
else:
self.assertTrue('zato.tls.client_cert.dict' not in wsgi_environ)
self.assertTrue('zato.tls.client_cert.der' not in wsgi_environ)
self.assertTrue('zato.tls.client_cert.sha1' not in wsgi_environ)
def parallel_server(self):
server = ParallelServer()
server.odb = self.odb_manager()
server.service_store = self.service_store()
#server.request_handler = self.request_handler()
#server.request_handler.soap_handler.server = server
#server.request_handler.plain_http_handler.server = server
return server
def test_set_tls_info(self):
expected_cert_dict = rand_string()
expected_cert_der = rand_string()
expected_cert_sha1 = sha1(expected_cert_der).hexdigest().upper()
class FakeRequestDispatcher(object):
def dispatch(self, *ignored_args, **ignored_kwargs):
return rand_string()
class FakeWorkerStore(object):
request_dispatcher = FakeRequestDispatcher()
class FakeGunicornSocket(object):
def getpeercert(self, needs_der=False):
if needs_der:
return expected_cert_der
return expected_cert_dict
def start_response(*ignored_args, **ignored_kwargs):
pass
for wsgi_url_scheme in('https', 'http'):
wsgi_environ = {
'wsgi.url_scheme': wsgi_url_scheme,
'gunicorn.socket': FakeGunicornSocket(),
'zato.http.response.status': rand_string()
}
ps = ParallelServer()
ps.worker_store = FakeWorkerStore()
ps.on_wsgi_request(wsgi_environ, start_response)
if wsgi_url_scheme == 'https':
eq_(wsgi_environ['zato.tls.client_cert.dict'], expected_cert_dict)
eq_(wsgi_environ['zato.tls.client_cert.der'], expected_cert_der)
eq_(wsgi_environ['zato.tls.client_cert.sha1'], expected_cert_sha1)
else:
self.assertTrue('zato.tls.client_cert.dict' not in wsgi_environ)
self.assertTrue('zato.tls.client_cert.der' not in wsgi_environ)
self.assertTrue('zato.tls.client_cert.sha1' not in wsgi_environ)
def test_access_log(self):
def _utcnow(self):
return datetime(year=2014, month=1, day=12, hour=16, minute=22, second=12, tzinfo=UTC)
local_tz = get_localzone()
_now = _utcnow(None)
local_dt = _now.replace(tzinfo=UTC).astimezone(local_tz)
local_dt = local_tz.normalize(local_dt)
request_timestamp = local_dt.strftime(ACCESS_LOG_DT_FORMAT)
with patch('arrow.factory.ArrowFactory.utcnow', _utcnow):
response = rand_string() * rand_int()
cid = new_cid()
cluster_id = 1
channel_name = rand_string()
url_path = '/{}'.format(rand_string())
user_agent = rand_string()
http_version = rand_string()
request_method = rand_string()
remote_ip = '10.{}.{}.{}'.format(rand_int(), rand_int(), rand_int())
req_timestamp_utc = utcnow()
channel_item = {
'name': channel_name,
'audit_enabled': False,
'is_active': True,
'transport': 'plain_http',
'data_format': None,
'match_target': url_path
}
wsgi_environ = {
'gunicorn.socket': FakeGunicornSocket(None, None),
'wsgi.url_scheme': 'http',
'wsgi.input': StringIO(response),
'zato.http.response.status': httplib.OK,
'zato.http.channel_item': channel_item,
'zato.request_timestamp_utc': req_timestamp_utc,
'HTTP_X_FORWARDED_FOR': remote_ip,
'PATH_INFO': url_path,
'REQUEST_METHOD': request_method,
'SERVER_PROTOCOL': http_version,
'HTTP_USER_AGENT': user_agent,
}
class FakeBrokerClient(object):
def __init__(self):
self.msg = None
def publish(self, msg):
self.msg = msg
class FakeODB(ODBManager):
def __init__(self):
self.msg = None
self.cluster = Bunch(id=cluster_id)
def session(self):
return fake_session
class FakeURLData(URLData):
def __init__(self):
self.url_sec = {url_path: Bunch(sec_def=ZATO_NONE)}
def match(self, *ignored_args, **ignored_kwargs):
return True, channel_item
class FakeRequestHandler(object):
def handle(self, *ignored_args, **ignored_kwargs):
return Bunch(payload=response, content_type='text/plain', headers={}, status_code=httplib.OK)
class FakeAccessLogger(object):
def __init__(self):
self.extra = {}
def info(self, msg, extra):
self.extra = extra
def isEnabledFor(self, ignored):
return True
bc = FakeBrokerClient()
ws = FakeWorkerStore()
ws.request_dispatcher = RequestDispatcher()
ws.request_dispatcher.request_handler = FakeRequestHandler()
ws.request_dispatcher.url_data = FakeURLData()
ws.request_dispatcher.url_data.broker_client = bc
ws.request_dispatcher.url_data.odb = FakeODB()
ps = ParallelServer()
ps.worker_store = ws
ps.access_logger = FakeAccessLogger()
ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=cid)
extra = Bunch(ps.access_logger.extra)
eq_(extra.channel_name, channel_name)
eq_(extra.user_agent, user_agent)
eq_(extra.status_code, '200')
eq_(extra.http_version, http_version)
eq_(extra.response_size, len(response))
eq_(extra.cid, cid)
eq_(extra.path, url_path)
eq_(extra.method, request_method)
eq_(extra.remote_ip, remote_ip)
eq_(extra.req_timestamp_utc, '12/Jan/2014:16:22:12 +0000')
eq_(extra.req_timestamp, request_timestamp)
def test_audit(self):
for expected_audit_enabled in(True, False):
for expected_status_code in(httplib.OK, httplib.FORBIDDEN):
for use_x_remote_addr in(True, False):
expected_auth_ok = True if expected_status_code == httplib.OK else False
expected_invoke_ok = True if expected_auth_ok is True else False
expected_cid = new_cid()
expected_url_scheme = rand_string()
expected_payload = rand_string()
expected_audit_repl_patt_type = rand_string()
expected_replace_patterns_elem_path = []
expected_replace_patterns_xpath = []
expected_cluster_id = rand_int()
expected_id = rand_int()
expected_name = rand_string()
expected_password = '******'
expected_username = rand_string()
expected_transport = rand_string()
expected_connection = rand_string()
expected_data_format = DATA_FORMAT.JSON
expected_is_active = True
expected_request = rand_string()
expected_audit_max_payload = len(expected_request) - 7 # Substracting any value would do
expected_channel_item_key1 = rand_string()
expected_channel_item_value1 = rand_string()
expected_match_target = rand_string()
channel_item = {
'id': expected_id,
'name': expected_name,
'transport': expected_transport,
'connection': expected_connection,
'audit_enabled': expected_audit_enabled,
expected_channel_item_key1:expected_channel_item_value1,
'audit_repl_patt_type': expected_audit_repl_patt_type,
'replace_patterns_elem_path': expected_replace_patterns_elem_path,
'replace_patterns_xpath': expected_replace_patterns_xpath,
'audit_max_payload': expected_audit_max_payload,
'is_active': expected_is_active,
'data_format': DATA_FORMAT.JSON,
'match_target': expected_match_target,
'username': expected_username,
}
wsgi_environ = {
'wsgi.url_scheme': expected_url_scheme,
'gunicorn.socket': FakeGunicornSocket(None, None),
'zato.http.response.status': expected_status_code,
'zato.http.channel_item': channel_item,
'PATH_INFO': rand_string(),
'wsgi.input': StringIO(expected_request),
'REQUEST_METHOD': rand_string(),
'SERVER_PROTOCOL': rand_string(),
'HTTP_USER_AGENT': rand_string(),
}
expected_remote_addr = rand_string()
if use_x_remote_addr:
expected_remote_addr_header = 'HTTP_X_FORWARDED_FOR'
wsgi_environ[expected_remote_addr_header] = expected_remote_addr
else:
expected_remote_addr_header = 'REMOTE_ADDR'
wsgi_environ[expected_remote_addr_header] = expected_remote_addr
class FakeSession:
def __init__(self, audit=None):
self.audit = audit
self.commit_called = False
def close(self):
pass
def commit(self):
self.commit_called = True
def add(self, audit):
self.audit = audit
fake_session = FakeSession()
class FakeBrokerClient(object):
def __init__(self):
self.msg = None
def publish(self, msg):
self.msg = msg
class FakeODB(ODBManager):
def __init__(self):
self.msg = None
self.cluster = Bunch(id=expected_cluster_id)
def session(self):
return fake_session
class FakeURLData(URLData):
def __init__(self):
self.url_sec = {expected_match_target: Bunch(sec_def=ZATO_NONE)}
def match(self, *ignored_args, **ignored_kwargs):
return True, channel_item
class FakeRequestHandler(object):
def handle(self, *ignored_args, **ignored_kwargs):
return Bunch(payload=expected_payload, content_type='text/plain', headers={}, status_code=expected_status_code)
bc = FakeBrokerClient()
ws = FakeWorkerStore()
ws.request_dispatcher = RequestDispatcher()
ws.request_dispatcher.request_handler = FakeRequestHandler()
ws.request_dispatcher.url_data = FakeURLData()
ws.request_dispatcher.url_data.broker_client = bc
ws.request_dispatcher.url_data.odb = FakeODB()
ps = ParallelServer()
ps.worker_store = ws
ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=expected_cid)
if expected_audit_enabled:
#
# Audit 1/2 - Request
#
# Parsing will confirm the proper value was used
datetime.strptime(fake_session.audit.req_time.isoformat(), '%Y-%m-%dT%H:%M:%S.%f')
self.assertEquals(fake_session.audit.name, expected_name)
self.assertEquals(fake_session.audit.cid, expected_cid)
self.assertEquals(fake_session.audit.transport, expected_transport)
self.assertEquals(fake_session.audit.connection, expected_connection)
self.assertEquals(fake_session.audit.resp_time, None)
self.assertEquals(fake_session.audit.user_token, expected_username)
self.assertEquals(fake_session.audit.auth_ok, None)
self.assertEquals(fake_session.audit.invoke_ok, None)
self.assertEquals(fake_session.audit.remote_addr, expected_remote_addr)
self.assertEquals(fake_session.audit.req_payload, expected_request[:expected_audit_max_payload])
self.assertEquals(fake_session.audit.resp_headers, None)
self.assertEquals(fake_session.audit.resp_payload, None)
req_headers = literal_eval(fake_session.audit.req_headers)
self.assertEquals(req_headers[expected_remote_addr_header], repr(expected_remote_addr))
self.assertEquals(req_headers['wsgi.url_scheme'], repr(expected_url_scheme))
self.assertEquals(req_headers['gunicorn.socket'], repr(FakeGunicornSocket(None, None)))
channel_item = literal_eval(req_headers['zato.http.channel_item'])
self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload)
self.assertEquals(channel_item['name'], expected_name)
self.assertEquals(channel_item['username'], expected_username)
self.assertEquals(channel_item[expected_channel_item_key1], expected_channel_item_value1)
self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type)
self.assertEquals(channel_item['replace_patterns_elem_path'], expected_replace_patterns_elem_path)
self.assertEquals(channel_item['is_active'], expected_is_active)
self.assertEquals(channel_item['data_format'], expected_data_format)
self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled)
self.assertEquals(channel_item['password'], expected_password)
self.assertEquals(channel_item['transport'], expected_transport)
self.assertEquals(channel_item['match_target'], expected_match_target)
#
# Audit 2/2 - Response
#
self.assertEquals(bc.msg['action'], CHANNEL_BROKER_MESSAGE.HTTP_SOAP_AUDIT_RESPONSE)
self.assertEquals(bc.msg['cid'], expected_cid)
self.assertEquals(bc.msg['data_format'], DATA_FORMAT.JSON)
self.assertEquals(bc.msg['service'], 'zato.http-soap.set-audit-response-data')
payload = loads(bc.msg['payload'])
self.assertEquals(payload['auth_ok'], expected_auth_ok)
self.assertEquals(payload['invoke_ok'], expected_invoke_ok)
self.assertEquals(payload['resp_payload'], expected_payload)
# Parsing alone will check its format is valid
datetime.strptime(payload['resp_time'], '%Y-%m-%dT%H:%M:%S.%f')
wsgi_environ = loads(payload['resp_headers'])
self.assertEquals(wsgi_environ['wsgi.url_scheme'], repr(expected_url_scheme))
self.assertEquals(wsgi_environ['gunicorn.socket'], repr(FakeGunicornSocket(None, None)))
self.assertEquals(wsgi_environ['zato.http.response.status'],
"'{} {}'".format(
expected_status_code,
httplib.responses[expected_status_code],
))
channel_item = literal_eval(wsgi_environ['zato.http.channel_item'])
self.assertEquals(channel_item[expected_channel_item_key1], expected_channel_item_value1)
self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled)
self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type)
self.assertEquals(channel_item['replace_patterns_elem_path'], expected_replace_patterns_elem_path)
self.assertEquals(channel_item['replace_patterns_xpath'], expected_replace_patterns_xpath)
self.assertEquals(channel_item['name'], expected_name)
self.assertEquals(channel_item['id'], expected_id)
self.assertEquals(channel_item['password'], expected_password)
self.assertEquals(channel_item['data_format'], expected_data_format)
self.assertEquals(channel_item['transport'], expected_transport)
self.assertEquals(channel_item['connection'], expected_connection)
self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload)
self.assertEquals(channel_item['is_active'], expected_is_active)
else:
# Audit not enabled so no response audit message was published on the broker
self.assertTrue(bc.msg is None)
def parallel_server(self):
server = ParallelServer()
server.odb = self.odb_manager()
server.service_store = self.service_store()
server.sql_pool_store = self.sql_pool_store()
server.int_parameters = self.int_parameters()
server.int_parameter_suffixes = self.int_parameter_suffixes()
server.bool_parameter_prefixes = self.bool_parameter_prefixes()
server.internal_service_modules = self.internal_service_modules()
server.service_modules = self.service_modules()
server.kvdb = self.kvdb()
server.user_config = Bunch()
return server
def test_access_log(self):
def _utcnow():
return datetime(year=2014,
month=1,
day=12,
hour=16,
minute=22,
second=12,
tzinfo=UTC)
local_tz = get_localzone()
_now = _utcnow()
local_dt = _now.replace(tzinfo=UTC).astimezone(local_tz)
local_dt = local_tz.normalize(local_dt)
request_timestamp = local_dt.strftime(ACCESS_LOG_DT_FORMAT)
response = rand_string() * rand_int()
cid = new_cid()
cluster_id = 1
channel_name = rand_string()
url_path = '/{}'.format(rand_string())
user_agent = rand_string()
http_version = rand_string()
request_method = rand_string()
remote_ip = '10.{}.{}.{}'.format(rand_int(), rand_int(), rand_int())
req_timestamp_utc = utcnow()
channel_item = {
'name': channel_name,
'is_active': True,
'transport': 'plain_http',
'data_format': None,
'match_target': url_path,
'method': '',
}
wsgi_environ = {
'gunicorn.socket': FakeGunicornSocket(None, None),
'wsgi.url_scheme': 'http',
'wsgi.input': StringIO(response),
'zato.http.response.status': OK,
'zato.channel_item': channel_item,
'zato.request_timestamp_utc': req_timestamp_utc,
'HTTP_X_FORWARDED_FOR': remote_ip,
'PATH_INFO': url_path,
'REQUEST_METHOD': request_method,
'SERVER_PROTOCOL': http_version,
'HTTP_USER_AGENT': user_agent,
}
class FakeBrokerClient(object):
def __init__(self):
self.msg = None
def publish(self, msg):
self.msg = msg
class FakeODB(ODBManager):
def __init__(self):
self.msg = None
self.cluster = Bunch(id=cluster_id)
class FakeURLData(URLData):
def __init__(self):
self.url_sec = {
url_path: Bunch(sec_def=ZATO_NONE, sec_use_rbac=False)
}
def match(self, *ignored_args, **ignored_kwargs):
return True, channel_item
class FakeRequestHandler(object):
def handle(self, *ignored_args, **ignored_kwargs):
return Bunch(payload=response,
content_type='text/plain',
headers={},
status_code=OK)
class FakeAccessLogger(object):
def __init__(self):
self.level = object()
self.msg = object()
self.args = object()
self.exc_info = object()
self.extra = object()
def _log(self, level, msg, args, exc_info, extra):
self.level = level
self.msg = msg
self.args
self.exc_info = exc_info
self.extra = extra
def isEnabledFor(self, ignored):
return True
bc = FakeBrokerClient()
ws = FakeWorkerStore()
ws.request_dispatcher = RequestDispatcher()
ws.request_dispatcher.request_handler = FakeRequestHandler()
ws.request_dispatcher.url_data = FakeURLData()
ws.request_dispatcher.url_data.broker_client = bc
ws.request_dispatcher.url_data.odb = FakeODB()
ps = ParallelServer()
ps.worker_store = ws
ps.request_dispatcher_dispatch = ws.request_dispatcher.dispatch
ps.access_logger = FakeAccessLogger()
ps.access_logger_log = ps.access_logger._log
ps.on_wsgi_request(wsgi_environ,
StartResponse(),
cid=cid,
_utcnow=_utcnow)
extra = Bunch(ps.access_logger.extra)
eq_(extra.channel_name, channel_name)
eq_(extra.user_agent, user_agent)
eq_(extra.status_code, '200')
eq_(extra.http_version, http_version)
eq_(extra.response_size, len(response))
eq_(extra.path, url_path)
eq_(extra.cid_resp_time, '{}/0.0'.format(
cid)) # It's 0.0 because we mock utcnow to be a constant value
eq_(extra.method, request_method)
eq_(extra.remote_ip, remote_ip)
eq_(extra.req_timestamp_utc, '12/Jan/2014:16:22:12 +0000')
eq_(extra.req_timestamp, request_timestamp)
def server(self):
server = ParallelServer()
server.odb = self.odb_manager()
server.service_store = self.service_store()
server.service_store.server = server
server.sql_pool_store = self.sql_pool_store()
server.int_parameters = self.int_parameters()
server.int_parameter_suffixes = self.int_parameter_suffixes()
server.bool_parameter_prefixes = self.bool_parameter_prefixes()
server.internal_service_modules = self.internal_service_modules()
server.service_modules = self.service_modules()
server.kvdb = self.kvdb()
server.user_config = Bunch()
return server
def test_audit(self):
for expected_audit_enabled in (True, False):
for expected_status_code in (httplib.OK, httplib.FORBIDDEN):
for use_x_remote_addr in (True, False):
expected_auth_ok = True if expected_status_code == httplib.OK else False
expected_invoke_ok = True if expected_auth_ok is True else False
expected_cid = new_cid()
expected_url_scheme = rand_string()
expected_payload = rand_string()
expected_audit_repl_patt_type = rand_string()
expected_replace_patterns_json_pointer = []
expected_replace_patterns_xpath = []
expected_cluster_id = rand_int()
expected_id = rand_int()
expected_name = rand_string()
expected_password = '******'
expected_username = rand_string()
expected_transport = rand_string()
expected_connection = rand_string()
expected_data_format = DATA_FORMAT.JSON
expected_is_active = True
expected_request = rand_string()
expected_audit_max_payload = len(
expected_request
) - 7 # Substracting any value would do
expected_channel_item_key1 = rand_string()
expected_channel_item_value1 = rand_string()
expected_match_target = rand_string()
channel_item = {
'id': expected_id,
'name': expected_name,
'transport': expected_transport,
'connection': expected_connection,
'audit_enabled': expected_audit_enabled,
expected_channel_item_key1:
expected_channel_item_value1,
'audit_repl_patt_type': expected_audit_repl_patt_type,
'replace_patterns_json_pointer':
expected_replace_patterns_json_pointer,
'replace_patterns_xpath':
expected_replace_patterns_xpath,
'audit_max_payload': expected_audit_max_payload,
'is_active': expected_is_active,
'data_format': DATA_FORMAT.JSON,
'match_target': expected_match_target,
'username': expected_username,
'method': '',
}
wsgi_environ = {
'wsgi.url_scheme': expected_url_scheme,
'gunicorn.socket': FakeGunicornSocket(None, None),
'zato.http.response.status': expected_status_code,
'zato.http.channel_item': channel_item,
'PATH_INFO': rand_string(),
'wsgi.input': StringIO(expected_request),
'REQUEST_METHOD': rand_string(),
'SERVER_PROTOCOL': rand_string(),
'HTTP_USER_AGENT': rand_string(),
}
expected_remote_addr = rand_string()
if use_x_remote_addr:
expected_remote_addr_header = 'HTTP_X_FORWARDED_FOR'
wsgi_environ[
expected_remote_addr_header] = expected_remote_addr
else:
expected_remote_addr_header = 'REMOTE_ADDR'
wsgi_environ[
expected_remote_addr_header] = expected_remote_addr
class FakeSession:
def __init__(self, audit=None):
self.audit = audit
self.commit_called = False
def close(self):
pass
def commit(self):
self.commit_called = True
def add(self, audit):
self.audit = audit
fake_session = FakeSession()
class FakeBrokerClient(object):
def __init__(self):
self.msg = None
def publish(self, msg):
self.msg = msg
class FakeODB(ODBManager):
def __init__(self):
self.msg = None
self.cluster = Bunch(id=expected_cluster_id)
def session(self):
return fake_session
class FakeURLData(URLData):
def __init__(self):
self.url_sec = {
expected_match_target: Bunch(sec_def=ZATO_NONE)
}
def match(self, *ignored_args, **ignored_kwargs):
return True, channel_item
class FakeRequestHandler(object):
def handle(self, *ignored_args, **ignored_kwargs):
return Bunch(payload=expected_payload,
content_type='text/plain',
headers={},
status_code=expected_status_code)
bc = FakeBrokerClient()
ws = FakeWorkerStore()
ws.request_dispatcher = RequestDispatcher()
ws.request_dispatcher.request_handler = FakeRequestHandler(
)
ws.request_dispatcher.url_data = FakeURLData()
ws.request_dispatcher.url_data.broker_client = bc
ws.request_dispatcher.url_data.odb = FakeODB()
ps = ParallelServer()
ps.worker_store = ws
ps.on_wsgi_request(wsgi_environ,
StartResponse(),
cid=expected_cid)
if expected_audit_enabled:
#
# Audit 1/2 - Request
#
# Parsing will confirm the proper value was used
datetime.strptime(
fake_session.audit.req_time.isoformat(),
'%Y-%m-%dT%H:%M:%S.%f')
self.assertEquals(fake_session.audit.name,
expected_name)
self.assertEquals(fake_session.audit.cid, expected_cid)
self.assertEquals(fake_session.audit.transport,
expected_transport)
self.assertEquals(fake_session.audit.connection,
expected_connection)
self.assertEquals(fake_session.audit.resp_time, None)
self.assertEquals(fake_session.audit.user_token,
expected_username)
self.assertEquals(fake_session.audit.auth_ok, None)
self.assertEquals(fake_session.audit.invoke_ok, None)
self.assertEquals(fake_session.audit.remote_addr,
expected_remote_addr)
self.assertEquals(
fake_session.audit.req_payload,
expected_request[:expected_audit_max_payload])
self.assertEquals(fake_session.audit.resp_headers,
None)
self.assertEquals(fake_session.audit.resp_payload,
None)
req_headers = literal_eval(
fake_session.audit.req_headers)
self.assertEquals(
req_headers[expected_remote_addr_header],
repr(expected_remote_addr))
self.assertEquals(req_headers['wsgi.url_scheme'],
repr(expected_url_scheme))
self.assertEquals(req_headers['gunicorn.socket'],
repr(FakeGunicornSocket(None, None)))
channel_item = literal_eval(
req_headers['zato.http.channel_item'])
self.assertEquals(channel_item['audit_max_payload'],
expected_audit_max_payload)
self.assertEquals(channel_item['name'], expected_name)
self.assertEquals(channel_item['username'],
expected_username)
self.assertEquals(
channel_item[expected_channel_item_key1],
expected_channel_item_value1)
self.assertEquals(channel_item['audit_repl_patt_type'],
expected_audit_repl_patt_type)
self.assertEquals(
channel_item['replace_patterns_json_pointer'],
expected_replace_patterns_json_pointer)
self.assertEquals(channel_item['is_active'],
expected_is_active)
self.assertEquals(channel_item['data_format'],
expected_data_format)
self.assertEquals(channel_item['audit_enabled'],
expected_audit_enabled)
self.assertEquals(channel_item['password'],
expected_password)
self.assertEquals(channel_item['transport'],
expected_transport)
self.assertEquals(channel_item['match_target'],
expected_match_target)
#
# Audit 2/2 - Response
#
self.assertEquals(
bc.msg['action'], CHANNEL_BROKER_MESSAGE.
HTTP_SOAP_AUDIT_RESPONSE.value)
self.assertEquals(bc.msg['cid'], expected_cid)
self.assertEquals(bc.msg['data_format'],
DATA_FORMAT.JSON)
self.assertEquals(
bc.msg['service'],
'zato.http-soap.set-audit-response-data')
payload = loads(bc.msg['payload'])
self.assertEquals(payload['auth_ok'], expected_auth_ok)
self.assertEquals(payload['invoke_ok'],
expected_invoke_ok)
self.assertEquals(payload['resp_payload'],
expected_payload)
# Parsing alone will check its format is valid
datetime.strptime(payload['resp_time'],
'%Y-%m-%dT%H:%M:%S.%f')
wsgi_environ = loads(payload['resp_headers'])
self.assertEquals(wsgi_environ['wsgi.url_scheme'],
repr(expected_url_scheme))
self.assertEquals(wsgi_environ['gunicorn.socket'],
repr(FakeGunicornSocket(None, None)))
self.assertEquals(
wsgi_environ['zato.http.response.status'],
"'{} {}'".format(
expected_status_code,
httplib.responses[expected_status_code],
))
channel_item = literal_eval(
wsgi_environ['zato.http.channel_item'])
self.assertEquals(
channel_item[expected_channel_item_key1],
expected_channel_item_value1)
self.assertEquals(channel_item['audit_enabled'],
expected_audit_enabled)
self.assertEquals(channel_item['audit_repl_patt_type'],
expected_audit_repl_patt_type)
self.assertEquals(
channel_item['replace_patterns_json_pointer'],
expected_replace_patterns_json_pointer)
self.assertEquals(
channel_item['replace_patterns_xpath'],
expected_replace_patterns_xpath)
self.assertEquals(channel_item['name'], expected_name)
self.assertEquals(channel_item['id'], expected_id)
self.assertEquals(channel_item['password'],
expected_password)
self.assertEquals(channel_item['data_format'],
expected_data_format)
self.assertEquals(channel_item['transport'],
expected_transport)
self.assertEquals(channel_item['connection'],
expected_connection)
self.assertEquals(channel_item['audit_max_payload'],
expected_audit_max_payload)
self.assertEquals(channel_item['is_active'],
expected_is_active)
else:
# Audit not enabled so no response audit message was published on the broker
self.assertTrue(bc.msg is None)
def run(base_dir, start_gunicorn_app=True, options=None):
# type: (str, bool, dict)
options = options or {}
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging_conf_path = os.path.join(repo_location, 'logging.conf')
with open(logging_conf_path) as f:
logging_config = yaml.load(f, yaml.FullLoader)
dictConfig(logging_config)
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
crypto_manager = ServerCryptoManager(repo_location,
secret_key=options['secret_key'],
stdin_data=read_stdin_data())
secrets_config = ConfigObj(os.path.join(repo_location, 'secrets.conf'),
use_zato=False)
server_config = get_config(repo_location,
'server.conf',
crypto_manager=crypto_manager,
secrets_conf=secrets_config)
pickup_config = get_config(repo_location, 'pickup.conf')
sio_config = get_config(repo_location,
'simple-io.conf',
needs_user_config=False)
sio_config = get_sio_server_config(sio_config)
sso_config = get_config(repo_location, 'sso.conf', needs_user_config=False)
normalize_sso_config(sso_config)
# Now that we have access to server.conf, greenify libraries required to be made greenlet-friendly,
# assuming that there are any - otherwise do not do anything.
to_greenify = []
for key, value in server_config.get('greenify', {}).items():
if asbool(value):
if not os.path.exists(key):
raise ValueError('No such path `{}`'.format(key))
else:
to_greenify.append(key)
# Go ahead only if we actually have anything to greenify
if to_greenify:
import greenify
greenify.greenify()
for name in to_greenify:
result = greenify.patch_lib(name)
if not result:
raise ValueError(
'Library `{}` could not be greenified'.format(name))
else:
logger.info('Greenified library `%s`', name)
server_config.main.token = server_config.main.token.encode('utf8')
# Do not proceed unless we can be certain our own preferred address or IP can be obtained.
preferred_address = server_config.preferred_address.get('address')
if not preferred_address:
preferred_address = get_preferred_ip(server_config.main.gunicorn_bind,
server_config.preferred_address)
if not preferred_address and not server_config.server_to_server.boot_if_preferred_not_found:
msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf'
logger.warn(msg)
raise Exception(msg)
# Create the startup callable tool as soon as practical
startup_callable_tool = StartupCallableTool(server_config)
# Run the hook before there is any server object created
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.FS_CONFIG_ONLY,
kwargs={
'server_config': server_config,
'pickup_config': pickup_config,
'sio_config': sio_config,
'sso_config': sso_config,
'base_dir': base_dir,
})
# New in 2.0 - Start monitoring as soon as possible
if server_config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(server_config.newrelic.config,
server_config.newrelic.environment or None,
server_config.newrelic.ignore_errors or None,
server_config.newrelic.log_file or None,
server_config.newrelic.log_level or None)
zunicorn.SERVER_SOFTWARE = server_config.misc.get('http_server_header',
'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(server_config.kvdb)
kvdb_logger.info('Main process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = server_config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
if server_config.misc.http_proxy:
os.environ['http_proxy'] = server_config.misc.http_proxy
# Basic components needed for the server to boot up
kvdb = KVDB()
odb_manager = ODBManager(well_known_data=ZATO_CRYPTO_WELL_KNOWN_DATA)
sql_pool_store = PoolStore()
service_store = ServiceStore()
service_store.odb = odb_manager
service_store.services = {}
server = ParallelServer()
server.odb = odb_manager
server.service_store = service_store
server.service_store.server = server
server.sql_pool_store = sql_pool_store
server.service_modules = []
server.kvdb = kvdb
server.stderr_path = options.get('stderr_path')
# Assigned here because it is a circular dependency
odb_manager.parallel_server = server
zato_gunicorn_app = ZatoGunicornApplication(server, repo_location,
server_config.main,
server_config.crypto)
server.has_fg = options.get('fg')
server.crypto_manager = crypto_manager
server.odb_data = server_config.odb
server.host = zato_gunicorn_app.zato_host
server.port = zato_gunicorn_app.zato_port
server.repo_location = repo_location
server.user_conf_location = os.path.join(server.repo_location, 'user-conf')
server.base_dir = base_dir
server.logs_dir = os.path.join(server.base_dir, 'logs')
server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls')
server.static_dir = os.path.join(server.base_dir, 'config', 'repo',
'static')
server.json_schema_dir = os.path.join(server.base_dir, 'config', 'repo',
'schema', 'json')
server.fs_server_config = server_config
server.fs_sql_config = get_config(repo_location,
'sql.conf',
needs_user_config=False)
server.pickup_config = pickup_config
server.logging_config = logging_config
server.logging_conf_path = logging_conf_path
server.sio_config = sio_config
server.sso_config = sso_config
server.user_config.update(server_config.user_config_items)
server.preferred_address = preferred_address
server.sync_internal = options['sync_internal']
server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8')
server.startup_callable_tool = startup_callable_tool
server.is_sso_enabled = server.fs_server_config.component_enabled.sso
if server.is_sso_enabled:
server.sso_api = SSOAPI(server, sso_config, None,
crypto_manager.encrypt, crypto_manager.decrypt,
crypto_manager.hash_secret,
crypto_manager.verify_hash, new_user_id)
# Remove all locks possibly left over by previous server instances
kvdb.component = 'master-proc'
clear_locks(kvdb, server_config.main.token, server_config.kvdb,
crypto_manager.decrypt)
# New in 2.0.8
server.return_tracebacks = asbool(
server_config.misc.get('return_tracebacks', True))
server.default_error_message = server_config.misc.get(
'default_error_message', 'An error has occurred')
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = server_config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = server_config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
# Repoze
from repoze.profile import ProfileMiddleware
profiler_dir = os.path.abspath(
os.path.join(base_dir, server_config.profiler.profiler_dir))
server.on_wsgi_request = ProfileMiddleware(
server.on_wsgi_request,
log_filename=os.path.join(profiler_dir,
server_config.profiler.log_filename),
cachegrind_filename=os.path.join(
profiler_dir, server_config.profiler.cachegrind_filename),
discard_first_request=server_config.profiler.discard_first_request,
flush_at_shutdown=server_config.profiler.flush_at_shutdown,
path=server_config.profiler.url_path,
unwind=server_config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = server_config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the hook right before the Gunicorn-level server actually starts
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IMPL_BEFORE_RUN,
kwargs={
'zato_gunicorn_app': zato_gunicorn_app,
})
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def parallel_server(self):
server = ParallelServer()
server.odb = self.odb_manager()
server.service_store = self.service_store()
server.sql_pool_store = self.sql_pool_store()
server.int_parameters = self.int_parameters()
server.int_parameter_suffixes = self.int_parameter_suffixes()
server.bool_parameter_prefixes = self.bool_parameter_prefixes()
server.soap11_content_type = self.soap11_content_type()
server.soap12_content_type = self.soap12_content_type()
server.plain_xml_content_type = self.plain_xml_content_type()
server.json_content_type = self.json_content_type()
server.internal_service_modules = self.internal_service_modules()
server.service_modules = self.service_modules()
server.kvdb = self.kvdb()
return server
def parallel_server(self):
server = ParallelServer()
server.odb = self.odb_manager()
server.service_store = self.service_store()
server.sql_pool_store = self.sql_pool_store()
server.int_parameters = self.int_parameters()
server.int_parameter_suffixes = self.int_parameter_suffixes()
server.bool_parameter_prefixes = self.bool_parameter_prefixes()
server.soap11_content_type = self.soap11_content_type()
server.soap12_content_type = self.soap12_content_type()
server.plain_xml_content_type = self.plain_xml_content_type()
server.json_content_type = self.json_content_type()
server.internal_service_modules = self.internal_service_modules()
server.service_modules = self.service_modules()
server.kvdb = self.kvdb()
return server
def setUp(self):
# For mocking out Vault responses
self.vault_adapter = RequestsAdapter()
# We are always the first process in a server
os.environ['ZATO_SERVER_WORKER_IDX'] = '1'
# Represents the server.conf file
self.fs_server_config = FSServerConfig()
self.worker_config = ConfigStore()
self.fernet_key = Fernet.generate_key() # type: str
self.crypto_manager = CryptoManager(secret_key=self.fernet_key)
self.vault_conn_api = VaultConnAPI(requests_adapter=self.vault_adapter)
self.server = ParallelServer()
self.server.fs_server_config = self.fs_server_config
self.server.kvdb = KVDB()
self.server.component_enabled.stats = False
self.server.component_enabled.slow_response = False
self.server.crypto_manager = self.crypto_manager
self.service_store = ServiceStore(is_testing=True)
self.service_store.server = self.server
self.service_store.services = {}
self.server.service_store = self.service_store
self.fs_sql_config = {
UNITTEST.SQL_ENGINE: {
'ping_query': 'SELECT 1+1'
}
}
self.cache = Cache()
self.sql_pool_store = PoolStore()
self.worker_store = WorkerStore(self.worker_config, self.server)
self.worker_store.sql_pool_store = self.sql_pool_store
self.worker_store.stomp_outconn_api = None
self.worker_store.outconn_wsx = None
self.worker_store.vault_conn_api = self.vault_conn_api
self.worker_store.sms_twilio_api = None
self.worker_store.out_sap = None
self.worker_store.out_sftp = None
self.worker_store.outconn_ldap = {}
self.worker_store.outconn_mongodb = {}
self.worker_store.def_kafka = {}
self.worker_store.cache_api = CacheAPI(self.server)
self.worker_store.cache_api.default = self.cache
self.request_handler = RequestHandler(self.server)
self.wsgi_environ = {
'HTTP_HOST': 'api.localhost'
}
# Callback methods for particular SQL queries
self.sql_callback_by_idx = {}
