def init(self): plain_http_config = MultiDict() soap_config = MultiDict() dol = deepcopy(self.worker_config.http_soap).dict_of_lists() for url_path in dol: for item in dol[url_path]: for soap_action, channel_info in item.items(): if channel_info['connection'] == 'channel': if channel_info.transport == 'plain_http': config = plain_http_config.setdefault(url_path, Bunch()) config[soap_action] = deepcopy(channel_info) else: config = soap_config.setdefault(url_path, Bunch()) config[soap_action] = deepcopy(channel_info) self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io) self.request_dispatcher.soap_handler = SOAPHandler(soap_config, self.server) self.request_dispatcher.plain_http_handler = PlainHTTPHandler(plain_http_config, self.server) # Statistics maintenance self.stats_maint = MaintenanceTool(self.kvdb.conn) self.request_dispatcher.security = ConnectionHTTPSOAPSecurity( self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0], self.worker_config.basic_auth, self.worker_config.tech_acc, self.worker_config.wss) # Create all the expected connections self.init_sql() self.init_ftp() self.init_http_soap()
def init(self): # Statistics maintenance self.stats_maint = MaintenanceTool(self.kvdb.conn) # Request dispatcher - matches URLs, checks security and dispatches HTTP # requests to services. self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io) self.request_dispatcher.url_data = URLData( deepcopy(self.worker_config.http_soap), self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0], self.worker_config.basic_auth, self.worker_config.tech_acc, self.worker_config.wss) self.request_dispatcher.request_handler = RequestHandler(self.server) # Create all the expected connections self.init_sql() self.init_ftp() self.init_http_soap()
def init(self): # Statistics maintenance self.stats_maint = MaintenanceTool(self.kvdb.conn) self.msg_ns_store = NamespaceStore() self.elem_path_store = ElemPathStore() self.xpath_store = XPathStore() # Message-related config - init_msg_ns_store must come before init_xpath_store # so the latter has access to the former's namespace map. self.init_msg_ns_store() self.init_elem_path_store() self.init_xpath_store() # Request dispatcher - matches URLs, checks security and dispatches HTTP # requests to services. self.request_dispatcher = RequestDispatcher( simple_io_config=self.worker_config.simple_io) self.request_dispatcher.url_data = URLData( deepcopy(self.worker_config.http_soap), self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0], self.worker_config.basic_auth, self.worker_config.ntlm, self.worker_config.oauth, self.worker_config.tech_acc, self.worker_config.wss, self.kvdb, self.broker_client, self.server.odb, self.elem_path_store, self.xpath_store) self.request_dispatcher.request_handler = RequestHandler(self.server) # Create all the expected connections self.init_sql() self.init_ftp() self.init_http_soap() # All set, whoever is waiting for us, if anyone at all, can now proceed self.is_ready = True
def test_access_log(self): def _utcnow(self): return datetime(year=2014, month=1, day=12, hour=16, minute=22, second=12, tzinfo=UTC) local_tz = get_localzone() _now = _utcnow(None) local_dt = _now.replace(tzinfo=UTC).astimezone(local_tz) local_dt = local_tz.normalize(local_dt) request_timestamp = local_dt.strftime(ACCESS_LOG_DT_FORMAT) with patch('arrow.factory.ArrowFactory.utcnow', _utcnow): response = rand_string() * rand_int() cid = new_cid() cluster_id = 1 channel_name = rand_string() url_path = '/{}'.format(rand_string()) user_agent = rand_string() http_version = rand_string() request_method = rand_string() remote_ip = '10.{}.{}.{}'.format(rand_int(), rand_int(), rand_int()) req_timestamp_utc = utcnow() channel_item = { 'name': channel_name, 'audit_enabled': False, 'is_active': True, 'transport': 'plain_http', 'data_format': None, 'match_target': url_path, 'method': '', } wsgi_environ = { 'gunicorn.socket': FakeGunicornSocket(None, None), 'wsgi.url_scheme': 'http', 'wsgi.input': StringIO(response), 'zato.http.response.status': httplib.OK, 'zato.http.channel_item': channel_item, 'zato.request_timestamp_utc': req_timestamp_utc, 'HTTP_X_FORWARDED_FOR': remote_ip, 'PATH_INFO': url_path, 'REQUEST_METHOD': request_method, 'SERVER_PROTOCOL': http_version, 'HTTP_USER_AGENT': user_agent, } class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=cluster_id) class FakeURLData(URLData): def __init__(self): self.url_sec = {url_path: Bunch(sec_def=ZATO_NONE)} def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=response, content_type='text/plain', headers={}, status_code=httplib.OK) class FakeAccessLogger(object): def __init__(self): self.extra = {} def info(self, msg, extra): self.extra = extra def isEnabledFor(self, ignored): return True bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler() ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.access_logger = FakeAccessLogger() ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=cid) extra = Bunch(ps.access_logger.extra) eq_(extra.channel_name, channel_name) eq_(extra.user_agent, user_agent) eq_(extra.status_code, '200') eq_(extra.http_version, http_version) eq_(extra.response_size, len(response)) eq_(extra.path, url_path) eq_(extra.cid_resp_time, '{}/0.0'.format( cid)) # It's 0.0 because we mock utcnow to be a constant value eq_(extra.method, request_method) eq_(extra.remote_ip, remote_ip) eq_(extra.req_timestamp_utc, '12/Jan/2014:16:22:12 +0000') eq_(extra.req_timestamp, request_timestamp)
def test_audit(self): for expected_audit_enabled in (True, False): for expected_status_code in (httplib.OK, httplib.FORBIDDEN): for use_x_remote_addr in (True, False): expected_auth_ok = True if expected_status_code == httplib.OK else False expected_invoke_ok = True if expected_auth_ok is True else False expected_cid = new_cid() expected_url_scheme = rand_string() expected_payload = rand_string() expected_audit_repl_patt_type = rand_string() expected_replace_patterns_json_pointer = [] expected_replace_patterns_xpath = [] expected_cluster_id = rand_int() expected_id = rand_int() expected_name = rand_string() expected_password = '******' expected_username = rand_string() expected_transport = rand_string() expected_connection = rand_string() expected_data_format = DATA_FORMAT.JSON expected_is_active = True expected_request = rand_string() expected_audit_max_payload = len( expected_request ) - 7 # Substracting any value would do expected_channel_item_key1 = rand_string() expected_channel_item_value1 = rand_string() expected_match_target = rand_string() channel_item = { 'id': expected_id, 'name': expected_name, 'transport': expected_transport, 'connection': expected_connection, 'audit_enabled': expected_audit_enabled, expected_channel_item_key1: expected_channel_item_value1, 'audit_repl_patt_type': expected_audit_repl_patt_type, 'replace_patterns_json_pointer': expected_replace_patterns_json_pointer, 'replace_patterns_xpath': expected_replace_patterns_xpath, 'audit_max_payload': expected_audit_max_payload, 'is_active': expected_is_active, 'data_format': DATA_FORMAT.JSON, 'match_target': expected_match_target, 'username': expected_username, 'method': '', } wsgi_environ = { 'wsgi.url_scheme': expected_url_scheme, 'gunicorn.socket': FakeGunicornSocket(None, None), 'zato.http.response.status': expected_status_code, 'zato.http.channel_item': channel_item, 'PATH_INFO': rand_string(), 'wsgi.input': StringIO(expected_request), 'REQUEST_METHOD': rand_string(), 'SERVER_PROTOCOL': rand_string(), 'HTTP_USER_AGENT': rand_string(), } expected_remote_addr = rand_string() if use_x_remote_addr: expected_remote_addr_header = 'HTTP_X_FORWARDED_FOR' wsgi_environ[ expected_remote_addr_header] = expected_remote_addr else: expected_remote_addr_header = 'REMOTE_ADDR' wsgi_environ[ expected_remote_addr_header] = expected_remote_addr class FakeSession: def __init__(self, audit=None): self.audit = audit self.commit_called = False def close(self): pass def commit(self): self.commit_called = True def add(self, audit): self.audit = audit fake_session = FakeSession() class FakeBrokerClient(object): def __init__(self): self.msg = None def publish(self, msg): self.msg = msg class FakeODB(ODBManager): def __init__(self): self.msg = None self.cluster = Bunch(id=expected_cluster_id) def session(self): return fake_session class FakeURLData(URLData): def __init__(self): self.url_sec = { expected_match_target: Bunch(sec_def=ZATO_NONE) } def match(self, *ignored_args, **ignored_kwargs): return True, channel_item class FakeRequestHandler(object): def handle(self, *ignored_args, **ignored_kwargs): return Bunch(payload=expected_payload, content_type='text/plain', headers={}, status_code=expected_status_code) bc = FakeBrokerClient() ws = FakeWorkerStore() ws.request_dispatcher = RequestDispatcher() ws.request_dispatcher.request_handler = FakeRequestHandler( ) ws.request_dispatcher.url_data = FakeURLData() ws.request_dispatcher.url_data.broker_client = bc ws.request_dispatcher.url_data.odb = FakeODB() ps = ParallelServer() ps.worker_store = ws ps.on_wsgi_request(wsgi_environ, StartResponse(), cid=expected_cid) if expected_audit_enabled: # # Audit 1/2 - Request # # Parsing will confirm the proper value was used datetime.strptime( fake_session.audit.req_time.isoformat(), '%Y-%m-%dT%H:%M:%S.%f') self.assertEquals(fake_session.audit.name, expected_name) self.assertEquals(fake_session.audit.cid, expected_cid) self.assertEquals(fake_session.audit.transport, expected_transport) self.assertEquals(fake_session.audit.connection, expected_connection) self.assertEquals(fake_session.audit.resp_time, None) self.assertEquals(fake_session.audit.user_token, expected_username) self.assertEquals(fake_session.audit.auth_ok, None) self.assertEquals(fake_session.audit.invoke_ok, None) self.assertEquals(fake_session.audit.remote_addr, expected_remote_addr) self.assertEquals( fake_session.audit.req_payload, expected_request[:expected_audit_max_payload]) self.assertEquals(fake_session.audit.resp_headers, None) self.assertEquals(fake_session.audit.resp_payload, None) req_headers = literal_eval( fake_session.audit.req_headers) self.assertEquals( req_headers[expected_remote_addr_header], repr(expected_remote_addr)) self.assertEquals(req_headers['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(req_headers['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) channel_item = literal_eval( req_headers['zato.http.channel_item']) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['username'], expected_username) self.assertEquals( channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals( channel_item['replace_patterns_json_pointer'], expected_replace_patterns_json_pointer) self.assertEquals(channel_item['is_active'], expected_is_active) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['match_target'], expected_match_target) # # Audit 2/2 - Response # self.assertEquals( bc.msg['action'], CHANNEL_BROKER_MESSAGE. HTTP_SOAP_AUDIT_RESPONSE.value) self.assertEquals(bc.msg['cid'], expected_cid) self.assertEquals(bc.msg['data_format'], DATA_FORMAT.JSON) self.assertEquals( bc.msg['service'], 'zato.http-soap.set-audit-response-data') payload = loads(bc.msg['payload']) self.assertEquals(payload['auth_ok'], expected_auth_ok) self.assertEquals(payload['invoke_ok'], expected_invoke_ok) self.assertEquals(payload['resp_payload'], expected_payload) # Parsing alone will check its format is valid datetime.strptime(payload['resp_time'], '%Y-%m-%dT%H:%M:%S.%f') wsgi_environ = loads(payload['resp_headers']) self.assertEquals(wsgi_environ['wsgi.url_scheme'], repr(expected_url_scheme)) self.assertEquals(wsgi_environ['gunicorn.socket'], repr(FakeGunicornSocket(None, None))) self.assertEquals( wsgi_environ['zato.http.response.status'], "'{} {}'".format( expected_status_code, httplib.responses[expected_status_code], )) channel_item = literal_eval( wsgi_environ['zato.http.channel_item']) self.assertEquals( channel_item[expected_channel_item_key1], expected_channel_item_value1) self.assertEquals(channel_item['audit_enabled'], expected_audit_enabled) self.assertEquals(channel_item['audit_repl_patt_type'], expected_audit_repl_patt_type) self.assertEquals( channel_item['replace_patterns_json_pointer'], expected_replace_patterns_json_pointer) self.assertEquals( channel_item['replace_patterns_xpath'], expected_replace_patterns_xpath) self.assertEquals(channel_item['name'], expected_name) self.assertEquals(channel_item['id'], expected_id) self.assertEquals(channel_item['password'], expected_password) self.assertEquals(channel_item['data_format'], expected_data_format) self.assertEquals(channel_item['transport'], expected_transport) self.assertEquals(channel_item['connection'], expected_connection) self.assertEquals(channel_item['audit_max_payload'], expected_audit_max_payload) self.assertEquals(channel_item['is_active'], expected_is_active) else: # Audit not enabled so no response audit message was published on the broker self.assertTrue(bc.msg is None)