def test_post_normal_addition_dimensions_metadata(): met_buffer = Helper.get_buffer(10) met_config = Helper.default_config() configs = { 'dimension_tags': ('dim_key1', 'dim_val1', 'dim_key2', 'dim_val2'), 'meta_tags': ('meta_key1', 'meta_val1', 'meta_key2', 'meta_val2') } for (key, value) in configs.items(): node = ConfigNode(getattr(ConfigOptions, key), value) config = CollectdConfig([Helper.url_node(), Helper.types_db_node(), node]) met_config.parse_config(config) for i in range(10): met_buffer.put_pending_batch(['batch_%s' % i]) met_sender = Helper.get_sender(met_config.conf, met_buffer) sleep_helper(10, 0.100, 100) assert requests.mock_server.url == met_config.conf[ConfigOptions.url] assert requests.mock_server.headers == { HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type], HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding], HeaderKeys.x_sumo_dimensions: 'dim_key1=dim_val1,dim_key2=dim_val2', HeaderKeys.x_sumo_metadata: 'meta_key1=meta_val1,meta_key2=meta_val2', } for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i) met_sender.cancel_timer()
def test_post_recoverable_requests_exception(): request_exception = requests.exceptions.RequestException() exception_cases = [ requests.exceptions.ConnectionError(request_exception), requests.exceptions.Timeout(request_exception), requests.exceptions.TooManyRedirects(request_exception), requests.exceptions.StreamConsumedError(request_exception), requests.exceptions.RetryError(request_exception), requests.exceptions.ChunkedEncodingError(request_exception), requests.exceptions.ContentDecodingError(request_exception), requests.exceptions.URLRequired(request_exception), requests.exceptions.MissingSchema(request_exception), requests.exceptions.InvalidSchema(request_exception), requests.exceptions.InvalidURL(request_exception), Exception('unknown_exception') ] for exception_case in exception_cases: reset_test_env() met_buffer = Helper.get_buffer(10) met_config = Helper.default_config() helper_test_post_recoverable_exception(met_config, met_buffer, exception_case, "unknown_status_code", 5) for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
def test_post_normal_additional_keys(): met_buffer = Helper.get_buffer(10) met_config = Helper.default_config() configs = { 'source_name': 'test_source', 'host_name': 'test_host', 'source_category': 'test_category' } Helper.parse_configs(met_config, configs) for i in range(10): met_buffer.put_pending_batch(['batch_%s' % i]) met_sender = Helper.get_sender(met_config.conf, met_buffer) sleep_helper(10, 0.100, 100) assert requests.mock_server.url == met_config.conf[ConfigOptions.url] assert requests.mock_server.headers == { HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type], HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding], HeaderKeys.x_sumo_source: 'test_source', HeaderKeys.x_sumo_host: 'test_host', HeaderKeys.x_sumo_category: 'test_category' } for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i) met_sender.cancel_timer()
def test_parse_retry_config_values_non_negative(): with pytest.raises(Exception) as e: met_config = Helper.default_config() configs = {'retry_initial_delay': '-1'} Helper.parse_configs(met_config, configs) assert 'Value -1 for key RetryInitialDelay is a negative number' in str(e)
def test_post_different_content_encodings(): met_buffer = Helper.get_buffer(10) deflate_config = { 'content_encoding': 'deflate' } gzip_config = { 'content_encoding': 'gzip' } none_config = { 'content_encoding': 'none' } configs = [deflate_config, gzip_config, none_config] for config in configs: reset_test_env() met_config = Helper.default_config() Helper.parse_configs(met_config, config) for i in range(10): met_buffer.put_pending_batch(['batch_%s' % i]) met_sender = Helper.get_sender(met_config.conf, met_buffer) sleep_helper(10, 0.100, 100) for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i) met_sender.cancel_timer()
def helper_test_post_recoverable_exception(met_config, met_buffer, exception, error_code, stop_raise_exception_after): configs = { 'retry_initial_delay': '0', 'retry_max_attempts': '5', 'retry_max_delay': '5', 'retry_backoff': '1', 'retry_jitter_min': '0', 'retry_jitter_max': '0' } Helper.parse_configs(met_config, configs) requests.post_response_decider.set(True, False, exception, stop_raise_exception_after, 0) requests.mock_response.set(error_code) for i in range(10): met_buffer.put_pending_batch(['batch_%s' % i]) met_sender = Helper.get_sender(met_config.conf, met_buffer) sleep_helper(10, 0.100, 100) assert requests.mock_server.url == met_config.conf[ConfigOptions.url] assert requests.mock_server.headers == { HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type], HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding] } met_sender.cancel_timer()
def test_invalid_ds_in_types_db(): met_config = Helper.default_config() types_db_node = ConfigNode(ConfigOptions.types_db, [cwd + '/test/types_invalid_ds.db']) config = CollectdConfig([Helper.url_node(), types_db_node]) met_config.parse_config(config) assert 'bytes' not in met_config.conf.keys()
def test_init_callback(): metrics_writer = Helper.default_writer() config = CollectdConfig([Helper.url_node(), Helper.types_db_node()]) metrics_writer.parse_config(config) metrics_writer.init_callback() assert metrics_writer.met_buffer is not None assert metrics_writer.met_batcher is not None assert metrics_writer.met_sender is not None
def test_parse_retry_config_jitter_min_greater_than_max(): with pytest.raises(Exception) as e: met_config = Helper.default_config() configs = {'retry_jitter_min': '2', 'retry_jitter_max': '1'} Helper.parse_configs(met_config, configs) assert 'Specify RetryJitterMin 2 to be less or equal to RetryJitterMax 1' in str( e)
def test_parse_retry_config_values_positive(): with pytest.raises(Exception) as e: met_config = Helper.default_config() configs = {'max_batch_size': '10', 'max_batch_interval': '-5'} Helper.parse_configs(met_config, configs) assert 'Value -5 for key MaxBatchInterval is not a positive number' in str( e)
def test_parse_unknown_content_encoding(): with pytest.raises(Exception) as e: met_config = Helper.default_config() unknown_config = {'content_encoding': 'unknown'} Helper.parse_configs(met_config, unknown_config) assert 'Unknown ContentEncoding unknown specified. ' \ 'ContentEncoding must be deflate, gzip, or none' in str(e)
def test_post_fail_after_retries_with_buffer_full(): met_buffer = Helper.get_buffer(10) met_config = Helper.default_config() met_buffer.put_pending_batch(['batch_first']) exception_to_raise = requests.exceptions.HTTPError(requests.exceptions.RequestException()) helper_test_post_recoverable_exception(met_config, met_buffer, exception_to_raise, 429, 10) for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
def test_types_db_no_exist_exception(): with pytest.raises(Exception) as e: met_config = Helper.default_config() types_db_node = ConfigNode(ConfigOptions.types_db, [cwd + '/test/types_not_exist.db']) config = CollectdConfig([Helper.url_node(), types_db_node]) met_config.parse_config(config) assert 'No such file or directory' in str(e)
def test_write_callback(): metrics_writer = Helper.default_writer() config = CollectdConfig([Helper.url_node(), Helper.types_db_node()]) metrics_writer.parse_config(config) metrics_writer.init_callback() d = Values() metrics_writer.write_callback(d) assert metrics_writer.met_batcher.queue.qsize() == 1 assert [metrics_writer.met_batcher.queue.get()] == d.metrics_str()
def test_parse_unknown_config_option(): met_config = Helper.default_config() unknown_config = 'unknown_config' unknown_config_node = ConfigNode('unknown_config', unknown_config) config = CollectdConfig( [Helper.url_node(), Helper.types_db_node(), unknown_config_node]) met_config.parse_config(config) assert hasattr(met_config, 'unknown_config') is False
def test_parse_int_exception(): with pytest.raises(ValueError): met_config = Helper.default_config() max_batch_size = '' max_batch_size_node = ConfigNode(ConfigOptions.max_batch_size, [max_batch_size]) config = CollectdConfig( [Helper.url_node(), Helper.types_db_node(), max_batch_size_node]) met_config.parse_config(config)
def test_non_ascii_strings(): met_config = Helper.default_config() configs = { 'source_name': '数据源', } Helper.parse_configs(met_config, configs) assert met_config.conf[ConfigOptions.source_name] == '数据源'
def test_post_client_recoverable_http_error(): error_codes = [404, 408, 429] for error_code in error_codes: reset_test_env() exception_to_raise = requests.exceptions.HTTPError(requests.exceptions.RequestException()) requests.mock_response.status_code = error_codes met_buffer = Helper.get_buffer(10) met_config = Helper.default_config() helper_test_post_recoverable_exception(met_config, met_buffer, exception_to_raise, error_code, 5) for i in range(10): assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
def test_parse_content_encoding(): deflate_config = {'content_encoding': 'deflate'} gzip_config = {'content_encoding': 'gzip'} none_config = {'content_encoding': 'none'} configs = [deflate_config, gzip_config, none_config] for config in configs: met_config = Helper.default_config() Helper.parse_configs(met_config, config) assert met_config.conf[ConfigOptions.content_encoding] == \ config['content_encoding']
def test_parse_http_post_interval(): met_config = Helper.default_config() http_post_interval = '0.5' http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval, [http_post_interval]) config = CollectdConfig( [Helper.url_node(), Helper.types_db_node(), http_post_interval_node]) met_config.parse_config(config) assert met_config.conf[ConfigOptions.http_post_interval] == float( http_post_interval)
def test_parse_meta_tags(): met_config = Helper.default_config() tags = ('meta_key1', 'meta_val1', 'meta_key2', 'meta_val2') config = CollectdConfig([ Helper.url_node(), Helper.types_db_node(), tags_node(ConfigOptions.meta_tags, tags) ]) met_config.parse_config(config) assert list(met_config.conf[ConfigOptions.meta_tags]) == list( tuple_to_pair(tags))
def test_parse_meta_tags_missing_value(): with pytest.raises(Exception) as e: met_config = Helper.default_config() tags = ('meta_key1', 'meta_val1', 'meta_key2') config = CollectdConfig([ Helper.url_node(), Helper.types_db_node(), tags_node(ConfigOptions.meta_tags, tags) ]) met_config.parse_config(config) assert "Missing tags key/value in options ('meta_key1', 'meta_val1', 'meta_key2')." in str( e)
def test_invalid_http_post_interval_exception(): with pytest.raises(Exception) as e: met_config = Helper.default_config() http_post_interval = '100.0' http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval, [http_post_interval]) config = CollectdConfig([ Helper.url_node(), Helper.types_db_node(), http_post_interval_node ]) met_config.parse_config(config) assert 'Specify HttpPostInterval' in str(e)
def test_contains_reserved_symbols_exception(): with pytest.raises(Exception) as e: met_config = Helper.default_config() tags = ('meta_key1', 'meta_val1', 'meta_key2', 'meta val2') config = CollectdConfig([ Helper.url_node(), Helper.types_db_node(), tags_node(ConfigOptions.meta_tags, tags) ]) met_config.parse_config(config) assert 'Value meta val2 for Key Metadata must not contain reserved symbol " "' in str( e)
def test_write_callback_host_with_equal_char(): metrics_writer = Helper.default_writer() config = CollectdConfig([Helper.url_node(), Helper.types_db_node()]) metrics_writer.parse_config(config) metrics_writer.init_callback() d = Values(host="[invalid=host]") expected_value = ['host=[invalid:host] plugin=test_plugin plugin_instance=test_plugin_instance' \ ' type=test_type type_instance=test_type_instance ds_name=test_ds_name ds_type=test_ds_type' \ ' test_meta_key=test_meta_val 3.140000 1501775008'] metrics_writer.write_callback(d) assert metrics_writer.met_batcher.queue.qsize() == 1 assert [metrics_writer.met_batcher.queue.get()] == expected_value
def helper_test_post_unrecoverable_exception(exception, error_code): with pytest.raises(Exception) as e: met_buffer = Helper.get_buffer(10) helper = Helper() requests.mock_response.set(error_code) requests.post_response_decider.set(False, True, exception, 5, 0) for i in range(10): met_buffer.put_pending_batch(['batch_%s' % i]) Helper.get_sender(helper.conf, met_buffer) assert e.type == type(exception)
def test_parse_string_config(): met_config = Helper.default_config() configs = { 'source_name': 'test_source', 'host_name': 'test_host', 'source_category': 'test_category' } Helper.parse_configs(met_config, configs) for (key, value) in configs.items(): assert met_config.conf[getattr(ConfigOptions, key)] == value
def test_shutdown_call_back(): metrics_writer = Helper.default_writer() config = CollectdConfig([Helper.url_node(), Helper.types_db_node()]) metrics_writer.parse_config(config) metrics_writer.init_callback() for i in range(10): metrics_writer.met_buffer.put_pending_batch(['batch_%s' % i]) metrics_writer.shutdown_callback() time.sleep(2) assert metrics_writer.met_buffer.empty() == True
def test_parse_http_post_interval_exception(): with pytest.raises(Exception) as e: met_config = Helper.default_config() http_post_interval = '0' http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval, [http_post_interval]) config = CollectdConfig([ Helper.url_node(), Helper.types_db_node(), http_post_interval_node ]) met_config.parse_config(config) assert 'Value 0.0 for key HttpPostInterval is not a positive number' in str( e)
def test_metrics_batcher_max_interval(): met_buffer = Helper.get_buffer(100) max_batch_interval = 0.05 met_batcher = Helper.get_batcher(1000, max_batch_interval, met_buffer) for i in range(50): time.sleep(0.010) met_batcher.push_item('item_%s' % i) while not met_buffer.pending_queue.empty(): batch = met_buffer.pending_queue.get() assert len(batch) < 10 met_batcher.cancel_timer()