def test_skip_samples_with_linked_samples(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(
samples_dict={
'volume':
SampleGenerator(samples_dict={
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
'tmp': ['ra', 'rb', 'rc', 'rd', 're', 'rf', 'rg', 'rh']
},
turn_to_list=True),
'id': [1, 2, 3, 4, 5, 6, 7, 8],
'name': ['a1', 'b2', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8']
},
dict_name='servers',
page_link_name='server_link')
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
keystone_mock.session.get.side_effect = generator.mock_request
fake_manager = self.FakeManager(keystone=keystone_mock)
pollster_definition = dict(self.multi_metric_pollster_definition)
pollster_definition['name'] = 'test-pollster.{name}'
pollster_definition['value_attribute'] = '[volume].tmp'
pollster_definition['skip_sample_values'] = ['rb']
pollster_definition['url_path'] = 'v1/test-volumes'
pollster_definition['response_entries_key'] = 'servers'
pollster_definition['next_sample_url_attribute'] = \
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
'list(value) | value[0] | value.get("href")'
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
list(map(lambda s: s.volume, samples)))
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'name'
pollster_definition['skip_sample_values'] = ['b2']
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
list(map(lambda s: s.volume, samples)))
def test_default_discovery_method(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.assertEqual(
"endpoint:test",
pollster.definitions.sample_gatherer.default_discovery)
def test_get_samples(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
fake_manager = self.FakeManager()
samples = pollster.get_samples(fake_manager, None,
["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(2, len(samples_list))
first_element = [
s for s in samples_list
if s.resource_id == "e335c317-dfdd-4f22-809a-625bd9a5992d"
][0]
self.assertEqual(5, first_element.volume)
self.assertEqual("6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
first_element.project_id)
self.assertEqual("924d1f77-5d75-4b96-a755-1774d6be17af",
first_element.user_id)
second_element = [
s for s in samples_list
if s.resource_id == "2e350554-6c05-4fda-8109-e47b595a714c"
][0]
self.assertEqual(2, second_element.volume)
self.assertEqual("6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
second_element.project_id)
self.assertEqual("20b5a704-b481-4603-a99e-2636c144b876",
second_element.user_id)
def test_create_request_arguments_PollsterSampleGatherer(self):
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition["headers"] = [{
"x-openstack-nova-api-version":
"2.46"
}, {
"custom_header": "custom"
}, {
"some_other_header": "something"
}]
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
request_args = pollster.definitions.sample_gatherer\
.create_request_arguments()
self.assertTrue("headers" in request_args)
self.assertTrue("authenticated" in request_args)
self.assertTrue(request_args["authenticated"])
self.assertEqual(3, len(request_args["headers"]))
self.assertEqual([
'x-openstack-nova-api-version', 'custom_header',
"some_other_header"
], list(map(lambda h: list(h.keys())[0], request_args["headers"])))
self.assertEqual(['2.46', 'custom', 'something'],
list(
map(lambda h: list(h.values())[0],
request_args["headers"])))
def test_retrieve_entries_using_second_entry_from_response(self):
self.pollster_definition_only_required_fields[
'response_entries_key'] = "second"
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
first_entries_from_response = [{
"object1-attr1": 1
}, {
"object1-attr2": 2
}]
second_entries_from_response = [{
"object1-attr3": 3
}, {
"object1-attr4": 33
}]
response = {
"first": first_entries_from_response,
"second": second_entries_from_response
}
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response)
self.assertEqual(second_entries_from_response, entries)
def test_retrieve_attribute_self_reference_sample(self):
key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0"
sub_value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value2 = [{"r": 245}, {"h": {"yu": "yu"}}]
json_object = {
"key1": {
"subKey1": sub_value1
},
"key2": {
"subkey2": sub_value2
}
}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(2, returned_value)
del json_object['key1']
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(0, returned_value)
def test_retrieve_attribute_nested_value_simulate_radosgw_processing(self):
key = "user | value.split('$') | value[0] | value.strip()"
json_object = {
"categories": [{
"bytes_received": 0,
"bytes_sent": 357088,
"category": "complete_multipart",
"ops": 472,
"successful_ops": 472
}],
"total": {
"bytes_received": 206739531986,
"bytes_sent": 273793180,
"ops": 119690,
"successful_ops": 119682
},
"user":
"******"
}
expected_value_after_operations = "00ab8d7e76fc4"
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
def test_get_samples_multi_metric_pollster(self):
pollster = dynamic_pollster.DynamicPollster(
self.multi_metric_pollster_definition)
fake_manager = self.FakeManager()
samples = pollster.get_samples(fake_manager, None,
["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(4, len(samples_list))
create_bucket_sample = [
s for s in samples_list if s.name == "test-pollster.create_bucket"
][0]
get_obj_sample = [
s for s in samples_list if s.name == "test-pollster.get_obj"
][0]
list_bucket_sample = [
s for s in samples_list if s.name == "test-pollster.list_bucket"
][0]
put_obj_sample = [
s for s in samples_list if s.name == "test-pollster.put_obj"
][0]
self.assertEqual(2, create_bucket_sample.volume)
self.assertEqual(46, get_obj_sample.volume)
self.assertEqual(8, list_bucket_sample.volume)
self.assertEqual(46, put_obj_sample.volume)
def test_retrieve_entries_from_response_response_is_a_list(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
response = [{"object1-attr1": 1}, {"object1-attr2": 2}]
entries = pollster.retrieve_entries_from_response(response)
self.assertEqual(response, entries)
def test_all_valid_sample_type(self):
for sample_type in sample.TYPES:
self.pollster_definition_only_required_fields[
'sample_type'] = sample_type
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.execute_basic_asserts(
pollster, self.pollster_definition_only_required_fields)
def test_create_request_arguments_PollsterSampleGatherer_no_headers(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
request_args =\
pollster.definitions.sample_gatherer.create_request_arguments()
self.assertTrue("headers" not in request_args)
self.assertTrue("authenticated" in request_args)
self.assertTrue(request_args["authenticated"])
def test_execute_value_mapping_no_value_mapping(self):
self.pollster_definition_only_required_fields['value_mapping'] = {}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = value_to_be_mapped
value = pollster.execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_generate_new_metadata_fields_no_metadata_mapping(self):
metadata = {'name': 'someName', 'value': 1}
metadata_before_call = copy.deepcopy(metadata)
self.pollster_definition_only_required_fields['metadata_mapping'] = {}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.generate_new_metadata_fields(metadata)
self.assertEqual(metadata_before_call, metadata)
def test_execute_value_mapping_no_value_mapping_found_with_default(self):
self.pollster_definition_only_required_fields['value_mapping'] = {
'some-possible-value': 15
}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = -1
value = pollster.execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_retrieve_attribute_nested_value_non_nested_key(self):
key = "key"
value = [{"d": 2}, {"g": {"h": "val"}}]
json_object = {"key": value}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(value, returned_value)
def test_execute_value_mapping(self):
self.pollster_definition_only_required_fields['value_mapping'] = {
'test': 'new-value'
}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = 'new-value'
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_get_request_linked_samples_url_next_sample_url(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = "http://test.com/next_page"
kwargs = {'resource': base_url, 'next_sample_url': expected_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)
def test_execute_request_get_samples_response_non_empty(self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {"firstElement": [{}, {}, {}]}
client_mock.session.get.return_value = return_value
samples = pollster.execute_request_get_samples(
client_mock, "https://endpoint.server.name/")
self.assertEqual(3, len(samples))
def test_retrieve_attribute_nested_value_nested_key(self):
key = "key.subKey"
value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value = [{"r": 245}, {"h": {"yu": "yu"}}]
json_object = {"key": {"subKey": sub_value, "subkey2": value1}}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.retrieve_attribute_nested_value(
json_object, key)
self.assertEqual(sub_value, returned_value)
def test_execute_request_get_samples_exception_on_request(
self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.bad
client_mock.session.get.return_value = return_value
exception = self.assertRaises(requests.HTTPError,
pollster.execute_request_get_samples,
client_mock,
"https://endpoint.server.name/")
self.assertEqual("Mock HTTP error.", str(exception))
def test_generate_new_metadata_fields_preserve_old_key(self):
metadata = {'name': 'someName', 'value': 2}
expected_metadata = copy.deepcopy(metadata)
expected_metadata['balance'] = metadata['value']
self.pollster_definition_only_required_fields['metadata_mapping'] = {
'value': 'balance'
}
self.pollster_definition_only_required_fields[
'preserve_mapped_metadata'] = True
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.generate_new_metadata_fields(metadata)
self.assertEqual(expected_metadata, metadata)
def test_execute_request_get_samples_empty_response(self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {}
client_mock.session.get.return_value = return_value
samples = pollster.definitions.sample_gatherer. \
execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual(0, len(samples))
def test_all_required_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.execute_basic_asserts(
pollster, self.pollster_definition_only_required_fields)
self.assertEqual(
0, len(pollster.pollster_definitions['skip_sample_values']))
self.assertEqual(0,
len(pollster.pollster_definitions['value_mapping']))
self.assertEqual(-1, pollster.pollster_definitions['default_value'])
self.assertEqual(
0, len(pollster.pollster_definitions['metadata_mapping']))
self.assertEqual(
True, pollster.pollster_definitions['preserve_mapped_metadata'])
def test_execute_request_get_samples_custom_ids(self):
sample = {
'user_id_attribute': "1",
'project_id_attribute': "2",
'resource_id_attribute': "3",
'user_id': "234",
'project_id': "2334",
'id': "35"
}
def internal_execute_request_get_samples_mock(self, arg):
class Response:
def json(self):
return [sample]
return Response(), "url"
original_method = dynamic_pollster.PollsterSampleGatherer.\
internal_execute_request_get_samples
try:
dynamic_pollster.PollsterSampleGatherer. \
internal_execute_request_get_samples = \
internal_execute_request_get_samples_mock
self.pollster_definition_all_fields[
'user_id_attribute'] = 'user_id_attribute'
self.pollster_definition_all_fields[
'project_id_attribute'] = 'project_id_attribute'
self.pollster_definition_all_fields[
'resource_id_attribute'] = 'resource_id_attribute'
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_all_fields)
params = {"d": "d"}
response = pollster.definitions.sample_gatherer. \
execute_request_get_samples(**params)
self.assertEqual(sample['user_id_attribute'],
response[0]['user_id'])
self.assertEqual(sample['project_id_attribute'],
response[0]['project_id'])
self.assertEqual(sample['resource_id_attribute'],
response[0]['id'])
finally:
dynamic_pollster.PollsterSampleGatherer. \
internal_execute_request_get_samples = original_method
def test_retrieve_attribute_nested_value_with_operation_on_attribute(self):
# spaces here are added on purpose at the end to make sure we
# execute the strip in the code before the eval
key = "key.subKey | value + 1|value / 2 | value * 3"
value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value = 1
expected_value_after_operations = 3
json_object = {"key": {"subKey": sub_value, "subkey2": value1}}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
def test_generate_new_metadata_fields_preserve_old_key_equals_false(self):
metadata = {'name': 'someName', 'value': 1}
expected_clean_metadata = copy.deepcopy(metadata)
expected_clean_metadata['balance'] = metadata['value']
expected_clean_metadata.pop('value')
self.pollster_definition_only_required_fields['metadata_mapping'] = {
'value': 'balance'
}
self.pollster_definition_only_required_fields[
'preserve_mapped_metadata'] = False
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(expected_clean_metadata, metadata)
def test_generate_sample_and_extract_metadata(self):
definition = self.pollster_definition_only_required_fields.copy()
definition['metadata_fields'] = ["metadata1", 'metadata2']
pollster = dynamic_pollster.DynamicPollster(definition)
pollster_sample = {
'metadata1': 'metadata1',
'metadata2': 'metadata2',
'value': 1
}
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample)
self.assertEqual(1, sample.volume)
self.assertEqual(2, len(sample.resource_metadata))
self.assertEqual('metadata1', sample.resource_metadata['metadata1'])
self.assertEqual('metadata2', sample.resource_metadata['metadata2'])
def test_get_samples_empty_samples(self, execute_request_get_samples_mock):
execute_request_get_samples_mock.side_effect = []
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
fake_manager = self.FakeManager()
samples = pollster.get_samples(fake_manager, None,
["https://endpoint.server.name.com/"])
samples_list = list()
try:
for s in samples:
samples_list.append(s)
except RuntimeError as e:
LOG.debug("Generator threw a StopIteration "
"and we need to catch it [%s]." % e)
self.assertEqual(0, len(samples_list))
def test_create_request_arguments_NonOpenStackApisSamplesGatherer(self):
pollster_definition = {
'name': "test-pollster",
'sample_type': "gauge",
'unit': "test",
'value_attribute': "volume",
'url_path': "https://test.com/v1/test/endpoint/fake",
"module": "someModule",
"authentication_object": "objectAuthentication",
"authentication_parameters": "authParam",
"headers": [{
"header1": "val1"
}, {
"header2": "val2"
}]
}
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
request_args = pollster.definitions.sample_gatherer\
.create_request_arguments()
self.assertTrue("headers" in request_args)
self.assertEqual(2, len(request_args["headers"]))
self.assertEqual(['header1', 'header2'],
list(
map(lambda h: list(h.keys())[0],
request_args["headers"])))
self.assertEqual(['val1', 'val2'],
list(
map(lambda h: list(h.values())[0],
request_args["headers"])))
self.assertTrue("authenticated" not in request_args)
def create_dynamic_pollsters(self):
"""Creates dynamic pollsters
This method Creates dynamic pollsters based on configurations placed on
'pollsters_definitions_dirs'
:return: a list with the dynamic pollsters defined by the operator.
"""
pollsters_definitions_dirs = self.conf.pollsters_definitions_dirs
if not pollsters_definitions_dirs:
LOG.info("Variable 'pollsters_definitions_dirs' not defined.")
return []
LOG.info("Looking for dynamic pollsters configurations at [%s].",
pollsters_definitions_dirs)
pollsters_definitions_files = []
for directory in pollsters_definitions_dirs:
files = glob.glob(os.path.join(directory, "*.yaml"))
if not files:
LOG.info("No dynamic pollsters found in folder [%s].",
directory)
continue
for filepath in sorted(files):
if filepath is not None:
pollsters_definitions_files.append(filepath)
if not pollsters_definitions_files:
LOG.info("No dynamic pollsters file found in dirs [%s].",
pollsters_definitions_dirs)
return []
pollsters_definitions = {}
for pollsters_definitions_file in pollsters_definitions_files:
pollsters_cfg = declarative.load_definitions(
self.conf, {}, pollsters_definitions_file)
LOG.info("File [%s] has [%s] dynamic pollster configurations.",
pollsters_definitions_file, len(pollsters_cfg))
for pollster_cfg in pollsters_cfg:
pollster_name = pollster_cfg['name']
if pollster_name not in pollsters_definitions:
LOG.info("Loading dynamic pollster [%s] from file [%s].",
pollster_name, pollsters_definitions_file)
try:
pollsters_definitions[pollster_name] =\
dynamic_pollster.DynamicPollster(
pollster_cfg, self.conf)
except Exception as e:
LOG.error(
"Error [%s] while loading dynamic pollster [%s].",
e, pollster_name)
else:
LOG.info(
"Dynamic pollster [%s] is already defined."
"Therefore, we are skipping it.", pollster_name)
LOG.debug("Total of dynamic pollsters [%s] loaded.",
len(pollsters_definitions))
return pollsters_definitions.values()
