def test_post_normal_addition_dimensions_metadata():
met_buffer = Helper.get_buffer(10)
met_config = Helper.default_config()
configs = {
'dimension_tags': ('dim_key1', 'dim_val1', 'dim_key2', 'dim_val2'),
'meta_tags': ('meta_key1', 'meta_val1', 'meta_key2', 'meta_val2')
}
for (key, value) in configs.items():
node = ConfigNode(getattr(ConfigOptions, key), value)
config = CollectdConfig([Helper.url_node(), Helper.types_db_node(), node])
met_config.parse_config(config)
for i in range(10):
met_buffer.put_pending_batch(['batch_%s' % i])
met_sender = Helper.get_sender(met_config.conf, met_buffer)
sleep_helper(10, 0.100, 100)
assert requests.mock_server.url == met_config.conf[ConfigOptions.url]
assert requests.mock_server.headers == {
HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type],
HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding],
HeaderKeys.x_sumo_dimensions: 'dim_key1=dim_val1,dim_key2=dim_val2',
HeaderKeys.x_sumo_metadata: 'meta_key1=meta_val1,meta_key2=meta_val2',
}
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
met_sender.cancel_timer()
def test_post_recoverable_requests_exception():
request_exception = requests.exceptions.RequestException()
exception_cases = [
requests.exceptions.ConnectionError(request_exception),
requests.exceptions.Timeout(request_exception),
requests.exceptions.TooManyRedirects(request_exception),
requests.exceptions.StreamConsumedError(request_exception),
requests.exceptions.RetryError(request_exception),
requests.exceptions.ChunkedEncodingError(request_exception),
requests.exceptions.ContentDecodingError(request_exception),
requests.exceptions.URLRequired(request_exception),
requests.exceptions.MissingSchema(request_exception),
requests.exceptions.InvalidSchema(request_exception),
requests.exceptions.InvalidURL(request_exception),
Exception('unknown_exception')
]
for exception_case in exception_cases:
reset_test_env()
met_buffer = Helper.get_buffer(10)
met_config = Helper.default_config()
helper_test_post_recoverable_exception(met_config, met_buffer,
exception_case,
"unknown_status_code", 5)
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i],
'batch_%s' % i)
def test_post_normal_additional_keys():
met_buffer = Helper.get_buffer(10)
met_config = Helper.default_config()
configs = {
'source_name': 'test_source',
'host_name': 'test_host',
'source_category': 'test_category'
}
Helper.parse_configs(met_config, configs)
for i in range(10):
met_buffer.put_pending_batch(['batch_%s' % i])
met_sender = Helper.get_sender(met_config.conf, met_buffer)
sleep_helper(10, 0.100, 100)
assert requests.mock_server.url == met_config.conf[ConfigOptions.url]
assert requests.mock_server.headers == {
HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type],
HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding],
HeaderKeys.x_sumo_source: 'test_source',
HeaderKeys.x_sumo_host: 'test_host',
HeaderKeys.x_sumo_category: 'test_category'
}
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
met_sender.cancel_timer()
def test_parse_retry_config_values_non_negative():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
configs = {'retry_initial_delay': '-1'}
Helper.parse_configs(met_config, configs)
assert 'Value -1 for key RetryInitialDelay is a negative number' in str(e)
def test_post_different_content_encodings():
met_buffer = Helper.get_buffer(10)
deflate_config = {
'content_encoding': 'deflate'
}
gzip_config = {
'content_encoding': 'gzip'
}
none_config = {
'content_encoding': 'none'
}
configs = [deflate_config, gzip_config, none_config]
for config in configs:
reset_test_env()
met_config = Helper.default_config()
Helper.parse_configs(met_config, config)
for i in range(10):
met_buffer.put_pending_batch(['batch_%s' % i])
met_sender = Helper.get_sender(met_config.conf, met_buffer)
sleep_helper(10, 0.100, 100)
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
met_sender.cancel_timer()
def helper_test_post_recoverable_exception(met_config, met_buffer, exception, error_code,
stop_raise_exception_after):
configs = {
'retry_initial_delay': '0',
'retry_max_attempts': '5',
'retry_max_delay': '5',
'retry_backoff': '1',
'retry_jitter_min': '0',
'retry_jitter_max': '0'
}
Helper.parse_configs(met_config, configs)
requests.post_response_decider.set(True, False, exception, stop_raise_exception_after, 0)
requests.mock_response.set(error_code)
for i in range(10):
met_buffer.put_pending_batch(['batch_%s' % i])
met_sender = Helper.get_sender(met_config.conf, met_buffer)
sleep_helper(10, 0.100, 100)
assert requests.mock_server.url == met_config.conf[ConfigOptions.url]
assert requests.mock_server.headers == {
HeaderKeys.content_type: met_config.conf[ConfigOptions.content_type],
HeaderKeys.content_encoding: met_config.conf[ConfigOptions.content_encoding]
}
met_sender.cancel_timer()
def test_invalid_ds_in_types_db():
met_config = Helper.default_config()
types_db_node = ConfigNode(ConfigOptions.types_db,
[cwd + '/test/types_invalid_ds.db'])
config = CollectdConfig([Helper.url_node(), types_db_node])
met_config.parse_config(config)
assert 'bytes' not in met_config.conf.keys()
def test_init_callback():
metrics_writer = Helper.default_writer()
config = CollectdConfig([Helper.url_node(), Helper.types_db_node()])
metrics_writer.parse_config(config)
metrics_writer.init_callback()
assert metrics_writer.met_buffer is not None
assert metrics_writer.met_batcher is not None
assert metrics_writer.met_sender is not None
def test_parse_retry_config_jitter_min_greater_than_max():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
configs = {'retry_jitter_min': '2', 'retry_jitter_max': '1'}
Helper.parse_configs(met_config, configs)
assert 'Specify RetryJitterMin 2 to be less or equal to RetryJitterMax 1' in str(
e)
def test_parse_retry_config_values_positive():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
configs = {'max_batch_size': '10', 'max_batch_interval': '-5'}
Helper.parse_configs(met_config, configs)
assert 'Value -5 for key MaxBatchInterval is not a positive number' in str(
e)
def test_parse_unknown_content_encoding():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
unknown_config = {'content_encoding': 'unknown'}
Helper.parse_configs(met_config, unknown_config)
assert 'Unknown ContentEncoding unknown specified. ' \
'ContentEncoding must be deflate, gzip, or none' in str(e)
def test_post_fail_after_retries_with_buffer_full():
met_buffer = Helper.get_buffer(10)
met_config = Helper.default_config()
met_buffer.put_pending_batch(['batch_first'])
exception_to_raise = requests.exceptions.HTTPError(requests.exceptions.RequestException())
helper_test_post_recoverable_exception(met_config, met_buffer, exception_to_raise, 429, 10)
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
def test_types_db_no_exist_exception():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
types_db_node = ConfigNode(ConfigOptions.types_db,
[cwd + '/test/types_not_exist.db'])
config = CollectdConfig([Helper.url_node(), types_db_node])
met_config.parse_config(config)
assert 'No such file or directory' in str(e)
def test_write_callback():
metrics_writer = Helper.default_writer()
config = CollectdConfig([Helper.url_node(), Helper.types_db_node()])
metrics_writer.parse_config(config)
metrics_writer.init_callback()
d = Values()
metrics_writer.write_callback(d)
assert metrics_writer.met_batcher.queue.qsize() == 1
assert [metrics_writer.met_batcher.queue.get()] == d.metrics_str()
def test_parse_unknown_config_option():
met_config = Helper.default_config()
unknown_config = 'unknown_config'
unknown_config_node = ConfigNode('unknown_config', unknown_config)
config = CollectdConfig(
[Helper.url_node(),
Helper.types_db_node(), unknown_config_node])
met_config.parse_config(config)
assert hasattr(met_config, 'unknown_config') is False
def test_parse_int_exception():
with pytest.raises(ValueError):
met_config = Helper.default_config()
max_batch_size = ''
max_batch_size_node = ConfigNode(ConfigOptions.max_batch_size,
[max_batch_size])
config = CollectdConfig(
[Helper.url_node(),
Helper.types_db_node(), max_batch_size_node])
met_config.parse_config(config)
def test_non_ascii_strings():
met_config = Helper.default_config()
configs = {
'source_name': '数据源',
}
Helper.parse_configs(met_config, configs)
assert met_config.conf[ConfigOptions.source_name] == '数据源'
def test_post_client_recoverable_http_error():
error_codes = [404, 408, 429]
for error_code in error_codes:
reset_test_env()
exception_to_raise = requests.exceptions.HTTPError(requests.exceptions.RequestException())
requests.mock_response.status_code = error_codes
met_buffer = Helper.get_buffer(10)
met_config = Helper.default_config()
helper_test_post_recoverable_exception(met_config, met_buffer, exception_to_raise,
error_code, 5)
for i in range(10):
assert_utf8_equal(met_config.conf, requests.mock_server.data[i], 'batch_%s' % i)
def test_parse_content_encoding():
deflate_config = {'content_encoding': 'deflate'}
gzip_config = {'content_encoding': 'gzip'}
none_config = {'content_encoding': 'none'}
configs = [deflate_config, gzip_config, none_config]
for config in configs:
met_config = Helper.default_config()
Helper.parse_configs(met_config, config)
assert met_config.conf[ConfigOptions.content_encoding] == \
config['content_encoding']
def test_parse_http_post_interval():
met_config = Helper.default_config()
http_post_interval = '0.5'
http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval,
[http_post_interval])
config = CollectdConfig(
[Helper.url_node(),
Helper.types_db_node(), http_post_interval_node])
met_config.parse_config(config)
assert met_config.conf[ConfigOptions.http_post_interval] == float(
http_post_interval)
def test_parse_meta_tags():
met_config = Helper.default_config()
tags = ('meta_key1', 'meta_val1', 'meta_key2', 'meta_val2')
config = CollectdConfig([
Helper.url_node(),
Helper.types_db_node(),
tags_node(ConfigOptions.meta_tags, tags)
])
met_config.parse_config(config)
assert list(met_config.conf[ConfigOptions.meta_tags]) == list(
tuple_to_pair(tags))
def test_parse_meta_tags_missing_value():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
tags = ('meta_key1', 'meta_val1', 'meta_key2')
config = CollectdConfig([
Helper.url_node(),
Helper.types_db_node(),
tags_node(ConfigOptions.meta_tags, tags)
])
met_config.parse_config(config)
assert "Missing tags key/value in options ('meta_key1', 'meta_val1', 'meta_key2')." in str(
e)
def test_invalid_http_post_interval_exception():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
http_post_interval = '100.0'
http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval,
[http_post_interval])
config = CollectdConfig([
Helper.url_node(),
Helper.types_db_node(), http_post_interval_node
])
met_config.parse_config(config)
assert 'Specify HttpPostInterval' in str(e)
def test_contains_reserved_symbols_exception():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
tags = ('meta_key1', 'meta_val1', 'meta_key2', 'meta val2')
config = CollectdConfig([
Helper.url_node(),
Helper.types_db_node(),
tags_node(ConfigOptions.meta_tags, tags)
])
met_config.parse_config(config)
assert 'Value meta val2 for Key Metadata must not contain reserved symbol " "' in str(
e)
def test_write_callback_host_with_equal_char():
metrics_writer = Helper.default_writer()
config = CollectdConfig([Helper.url_node(), Helper.types_db_node()])
metrics_writer.parse_config(config)
metrics_writer.init_callback()
d = Values(host="[invalid=host]")
expected_value = ['host=[invalid:host] plugin=test_plugin plugin_instance=test_plugin_instance' \
' type=test_type type_instance=test_type_instance ds_name=test_ds_name ds_type=test_ds_type' \
' test_meta_key=test_meta_val 3.140000 1501775008']
metrics_writer.write_callback(d)
assert metrics_writer.met_batcher.queue.qsize() == 1
assert [metrics_writer.met_batcher.queue.get()] == expected_value
def helper_test_post_unrecoverable_exception(exception, error_code):
with pytest.raises(Exception) as e:
met_buffer = Helper.get_buffer(10)
helper = Helper()
requests.mock_response.set(error_code)
requests.post_response_decider.set(False, True, exception, 5, 0)
for i in range(10):
met_buffer.put_pending_batch(['batch_%s' % i])
Helper.get_sender(helper.conf, met_buffer)
assert e.type == type(exception)
def test_parse_string_config():
met_config = Helper.default_config()
configs = {
'source_name': 'test_source',
'host_name': 'test_host',
'source_category': 'test_category'
}
Helper.parse_configs(met_config, configs)
for (key, value) in configs.items():
assert met_config.conf[getattr(ConfigOptions, key)] == value
def test_shutdown_call_back():
metrics_writer = Helper.default_writer()
config = CollectdConfig([Helper.url_node(), Helper.types_db_node()])
metrics_writer.parse_config(config)
metrics_writer.init_callback()
for i in range(10):
metrics_writer.met_buffer.put_pending_batch(['batch_%s' % i])
metrics_writer.shutdown_callback()
time.sleep(2)
assert metrics_writer.met_buffer.empty() == True
def test_parse_http_post_interval_exception():
with pytest.raises(Exception) as e:
met_config = Helper.default_config()
http_post_interval = '0'
http_post_interval_node = ConfigNode(ConfigOptions.http_post_interval,
[http_post_interval])
config = CollectdConfig([
Helper.url_node(),
Helper.types_db_node(), http_post_interval_node
])
met_config.parse_config(config)
assert 'Value 0.0 for key HttpPostInterval is not a positive number' in str(
e)
def test_metrics_batcher_max_interval():
met_buffer = Helper.get_buffer(100)
max_batch_interval = 0.05
met_batcher = Helper.get_batcher(1000, max_batch_interval, met_buffer)
for i in range(50):
time.sleep(0.010)
met_batcher.push_item('item_%s' % i)
while not met_buffer.pending_queue.empty():
batch = met_buffer.pending_queue.get()
assert len(batch) < 10
met_batcher.cancel_timer()
