def test_instances_one(self):
sample1 = sample.Sample(
"instance",
"cumulative",
"",
1,
"user-id",
"project-id",
"resource-id",
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={"display_name": "test-server", "tag": "self.sample"},
source="test",
)
msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
"instance",
"cumulative",
"",
1,
"user-id",
"project-id",
"resource-id-alternate",
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={"display_name": "test-server", "tag": "self.sample2"},
source="test",
)
msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg2)
data = self.get_json("/resources/resource-id")
self.assertEqual("resource-id", data["resource_id"])
def setUp(self):
super(TestListProjects, self).setUp()
sample1 = sample.Sample(
"instance",
"cumulative",
"instance",
1,
"user-id",
"project-id",
"resource-id",
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={"display_name": "test-server", "tag": "self.sample"},
source="test_list_projects",
)
msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.metering_secret)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
"instance",
"cumulative",
"instance",
1,
"user-id2",
"project-id2",
"resource-id-alternate",
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={"display_name": "test-server", "tag": "self.sample2"},
source="test_list_users",
)
msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.metering_secret)
self.conn.record_metering_data(msg2)
def test_with_project(self):
sample1 = sample.Sample(
"instance",
"cumulative",
"",
1,
"user-id",
"project-id",
"resource-id",
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={"display_name": "test-server", "tag": "self.sample"},
source="test_list_resources",
)
msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
"instance",
"cumulative",
"",
1,
"user-id2",
"project-id2",
"resource-id-alternate",
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={"display_name": "test-server", "tag": "self.sample2"},
source="not-test",
)
msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg2)
data = self.get_json("/resources", q=[{"field": "project_id", "value": "project-id"}])
ids = [r["resource_id"] for r in data]
self.assertEqual(["resource-id"], ids)
def test_with_invalid_resource_id(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id-1',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id',
'resource-id-2',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2',
},
source='test_list_resources',
)
msg2 = utils.meter_message_from_counter(
sample2, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg2)
resp1 = self.get_json('/resources/resource-id-1')
self.assertEqual("resource-id-1", resp1["resource_id"])
resp2 = self.get_json('/resources/resource-id-2')
self.assertEqual("resource-id-2", resp2["resource_id"])
resp3 = self.get_json('/resources/resource-id-3', expect_errors=True)
self.assertEqual(404, resp3.status_code)
json_data = resp3.body
if six.PY3:
json_data = json_data.decode('utf-8')
self.assertEqual("Resource resource-id-3 Not Found",
json.loads(json_data)['error_message']
['faultstring'])
def test_with_user(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample2',
},
source='not-test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources',
q=[{
'field': 'user_id',
'value': 'user-id',
}])
ids = [r['resource_id'] for r in data]
self.assertEqual(['resource-id'], ids)
def test_instances(self):
timestamps = {
'resource-id': datetime.datetime(2012, 7, 2, 10, 40),
'resource-id-alternate': datetime.datetime(2012, 7, 2, 10, 41),
}
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=timestamps['resource-id'],
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
},
source='test',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id-alternate',
timestamp=timestamps['resource-id-alternate'],
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2',
},
source='test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources')
self.assertEqual(2, len(data))
for res in data:
timestamp = timestamps.get(res['resource_id'])
self._verify_sample_timestamps(res, timestamp, timestamp)
def test_instances(self):
timestamps = {
'resource-id': datetime.datetime(2012, 7, 2, 10, 40),
'resource-id-alternate': datetime.datetime(2012, 7, 2, 10, 41),
}
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=timestamps['resource-id'],
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
},
source='test',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id-alternate',
timestamp=timestamps['resource-id-alternate'],
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2',
},
source='test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources')
self.assertEqual(2, len(data))
for res in data:
timestamp = timestamps.get(res['resource_id'])
self._verify_sample_timestamps(res, timestamp, timestamp)
def setUp(self):
super(TestListEvents, self).setUp()
timeutils.utcnow.override_time = datetime.datetime(2014, 2, 11, 16, 42)
self.sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project1',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
'dict_properties': {
'key': 'value'
},
'not_ignored_list': ['returned'],
},
source='test_source',
)
msg = utils.meter_message_from_counter(
self.sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
self.sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project2',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample2',
},
source='source2',
)
msg2 = utils.meter_message_from_counter(
self.sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
def test_with_source(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2',
},
source='not-test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources', q=[{'field': 'source',
'value': 'test_list_resources',
}])
ids = [r['resource_id'] for r in data]
self.assertEqual(['resource-id'], ids)
sources = [r['source'] for r in data]
self.assertEqual(['test_list_resources'], sources)
def setUp(self):
super(TestAPIACL, self).setUp()
self.environ = {'fake.cache': FakeMemcache()}
for cnt in [
sample.Sample('meter.test',
'cumulative',
'',
1,
'user-good',
'project-good',
'resource-good',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample'
},
source='test_source'),
sample.Sample('meter.mine',
'gauge',
'',
1,
'user-fred',
'project-good',
'resource-56',
timestamp=datetime.datetime(2012, 7, 2, 10, 43),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample4'
},
source='test_source')
]:
msg = utils.meter_message_from_counter(
cnt, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg)
def setUp(self):
super(TestListResourcesRestriction, self).setUp()
self.CONF.set_override('default_api_return_limit', 10, group='api')
for i in range(20):
s = sample.Sample(
'volume.size',
'gauge',
'GiB',
5 + i,
'user-id',
'project1',
'resource-id%s' % i,
timestamp=(datetime.datetime(2012, 9, 25, 10, 30) +
datetime.timedelta(seconds=i)),
resource_metadata={
'display_name': 'test-volume',
'tag': 'self.sample',
},
source='source1',
)
msg = utils.meter_message_from_counter(
s,
self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
def start_fixture(self):
"""Create some samples."""
conf = fixture_config.Config().conf
self.conn = storage.get_connection_from_config(conf)
timestamp = datetime.datetime.utcnow()
project_id = str(uuid.uuid4())
self.source = str(uuid.uuid4())
resource_metadata = {"farmed_by": "nancy"}
for name in ["cow", "pig", "sheep"]:
resource_metadata.update({"breed": name}),
c = sample.Sample(
name="livestock",
type="gauge",
unit="head",
volume=int(10 * random.random()),
user_id="farmerjon",
project_id=project_id,
resource_id=project_id,
timestamp=timestamp,
resource_metadata=resource_metadata,
source=self.source,
)
data = utils.meter_message_from_counter(c, conf.publisher.telemetry_secret)
self.conn.record_metering_data(data)
def test_resource_id_with_slash(self):
s = sample.Sample(
'storage.containers.objects',
'gauge',
'',
1,
'19fbed01c21f4912901057021b9e7111',
'45acc90399134206b3b41f3d3a0a06d6',
'29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance',
timestamp=datetime.datetime(2012, 7, 2, 10, 40).isoformat(),
resource_metadata={},
source='test_show_special_resource',
)
msg = utils.meter_message_from_counter(
s,
self.CONF.publisher.telemetry_secret,
)
msg['timestamp'] = datetime.datetime(2012, 7, 2, 10, 40)
self.conn.record_metering_data(msg)
rid_encoded = '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb%252Fglance'
resp = self.get_json('/resources/%s' % rid_encoded)
self.assertEqual("19fbed01c21f4912901057021b9e7111", resp["user_id"])
self.assertEqual('29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance',
resp["resource_id"])
def test_resource_meter_links(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources')
links = data[0]['links']
self.assertEqual(2, len(links))
self.assertEqual('self', links[0]['rel'])
self.assertIn((self.PATH_PREFIX + '/resources/resource-id'),
links[0]['href'])
self.assertEqual('instance', links[1]['rel'])
self.assertIn((self.PATH_PREFIX + '/meters/instance?'
'q.field=resource_id&q.value=resource-id'),
links[1]['href'])
def test_instance_multiple_samples(self):
timestamps = [
datetime.datetime(2012, 7, 2, 10, 41),
datetime.datetime(2012, 7, 2, 10, 42),
datetime.datetime(2012, 7, 2, 10, 40),
]
for timestamp in timestamps:
datapoint = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=timestamp,
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample-%s' % timestamp,
},
source='test',
)
msg = utils.meter_message_from_counter(
datapoint,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources')
self.assertEqual(1, len(data))
self._verify_resource_timestamps(data[0], timestamps[-1],
timestamps[1])
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(self.CONF,
netutils.urlsplit('udp://somehost'))
publisher.publish_samples(self.test_data)
self.assertEqual(5, len(self.data_sent))
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data, encoding="utf-8")
sent_counters.append(counter)
# Check destination
self.assertEqual(('somehost', 4952), dest)
# Check that counters are equal
def sort_func(counter):
return counter['counter_name']
counters = [
utils.meter_message_from_counter(d, "not-so-secret")
for d in self.test_data
]
counters.sort(key=sort_func)
sent_counters.sort(key=sort_func)
self.assertEqual(counters, sent_counters)
def test_with_user_wrong_tenant(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id2',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample1',
},
source='not-test',
)
msg2 = utils.meter_message_from_counter(
sample1, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources',
headers={"X-Roles": "Member",
"X-Project-Id": "project-wrong"})
ids = set(r['resource_id'] for r in data)
self.assertEqual(set(), ids)
def start_fixture(self):
"""Create some samples."""
conf = fixture_config.Config().conf
self.conn = storage.get_connection_from_config(conf)
timestamp = datetime.datetime.utcnow()
project_id = str(uuid.uuid4())
self.source = str(uuid.uuid4())
resource_metadata = {'farmed_by': 'nancy'}
for name in ['cow', 'pig', 'sheep']:
resource_metadata.update({'breed': name}),
c = sample.Sample(name='livestock',
type='gauge',
unit='head',
volume=int(10 * random.random()),
user_id='farmerjon',
project_id=project_id,
resource_id=project_id,
timestamp=timestamp,
resource_metadata=resource_metadata,
source=self.source,
)
data = utils.meter_message_from_counter(
c, conf.publisher.telemetry_secret)
self.conn.record_metering_data(data)
def test_with_user_wrong_tenant(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id2',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample1',
},
source='not-test',
)
msg2 = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources',
headers={
"X-Roles": "Member",
"X-Project-Id": "project-wrong"
})
ids = set(r['resource_id'] for r in data)
self.assertEqual(set(), ids)
def test_resource_skip_meter_links(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources?meter_links=0')
links = data[0]['links']
self.assertEqual(len(links), 1)
self.assertEqual(links[0]['rel'], 'self')
self.assertTrue((self.PATH_PREFIX +
'/resources/resource-id') in links[0]['href'])
def publish_samples(self, samples):
"""Publish samples on RPC.
:param samples: Samples from pipeline.
"""
meters = [
utils.meter_message_from_counter(
sample, self.conf.publisher.telemetry_secret)
for sample in samples
]
topic = self.conf.publisher_notifier.metering_topic
self.local_queue.append((topic, meters))
if self.per_meter_topic:
for meter_name, meter_list in itertools.groupby(
sorted(meters, key=operator.itemgetter('counter_name')),
operator.itemgetter('counter_name')):
meter_list = list(meter_list)
topic_name = topic + '.' + meter_name
LOG.debug('Publishing %(m)d samples on %(n)s',
{'m': len(meter_list), 'n': topic_name})
self.local_queue.append((topic_name, meter_list))
self.flush()
def test_resource_id_with_slash(self):
s = sample.Sample(
'storage.containers.objects',
'gauge',
'',
1,
'19fbed01c21f4912901057021b9e7111',
'45acc90399134206b3b41f3d3a0a06d6',
'29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance',
timestamp=datetime.datetime(2012, 7, 2, 10, 40).isoformat(),
resource_metadata={},
source='test_show_special_resource',
)
msg = utils.meter_message_from_counter(
s, self.CONF.publisher.telemetry_secret,
)
msg['timestamp'] = datetime.datetime(2012, 7, 2, 10, 40)
self.conn.record_metering_data(msg)
rid_encoded = '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb%252Fglance'
resp = self.get_json('/resources/%s' % rid_encoded)
self.assertEqual("19fbed01c21f4912901057021b9e7111", resp["user_id"])
self.assertEqual('29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance',
resp["resource_id"])
def setUp(self):
super(TestListResourcesBase, self).setUp()
for cnt in [
sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample'},
source='test_list_resources',
),
sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2'},
source='test_list_resources',
),
sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id2',
'resource-id2',
timestamp=datetime.datetime(2012, 7, 2, 10, 42),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample3'},
source='test_list_resources',
),
sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 43),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample4'},
source='test_list_resources',
)]:
msg = utils.meter_message_from_counter(
cnt,
self.CONF.publisher.metering_secret)
self.conn.record_metering_data(msg)
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://somehost'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(5, len(self.data_sent))
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data)
sent_counters.append(counter)
# Check destination
self.assertEqual(('somehost',
self.CONF.collector.udp_port), dest)
# Check that counters are equal
self.assertEqual(sorted(
[utils.meter_message_from_counter(d, "not-so-secret")
for d in self.test_data]), sorted(sent_counters))
def test_resource_skip_meter_links(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_resources',
)
msg = utils.meter_message_from_counter(
sample1, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources?meter_links=0')
links = data[0]['links']
self.assertEqual(len(links), 1)
self.assertEqual(links[0]['rel'], 'self')
self.assertTrue((self.PATH_PREFIX + '/resources/resource-id')
in links[0]['href'])
def setUp(self):
super(TestAPIACL, self).setUp()
self.environ = {'fake.cache': FakeMemcache()}
for cnt in [
sample.Sample(
'meter.test',
'cumulative',
'',
1,
'user-good',
'project-good',
'resource-good',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample'},
source='test_source'),
sample.Sample(
'meter.mine',
'gauge',
'',
1,
'user-fred',
'project-good',
'resource-56',
timestamp=datetime.datetime(2012, 7, 2, 10, 43),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample4'},
source='test_source')]:
msg = utils.meter_message_from_counter(
cnt,
self.CONF.publisher.metering_secret)
self.conn.record_metering_data(msg)
def test_instance_multiple_samples(self):
timestamps = [
datetime.datetime(2012, 7, 2, 10, 41),
datetime.datetime(2012, 7, 2, 10, 42),
datetime.datetime(2012, 7, 2, 10, 40),
]
for timestamp in timestamps:
datapoint = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=timestamp,
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample-%s' % timestamp,
},
source='test',
)
msg = utils.meter_message_from_counter(
datapoint,
self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources')
self.assertEqual(1, len(data))
self._verify_resource_timestamps(data[0],
timestamps[-1], timestamps[1])
def setUp(self):
super(TestCollector, self).setUp()
messaging.setup('fake://')
self.addCleanup(messaging.cleanup)
self.CONF = self.useFixture(config.Config()).conf
self.CONF.set_override("connection", "log://", group='database')
self.srv = collector.CollectorService()
self.CONF.publisher.metering_secret = 'not-so-secret'
self.counter = sample.Sample(
name='foobar',
type='bad',
unit='F',
volume=1,
user_id='jd',
project_id='ceilometer',
resource_id='cat',
timestamp='NOW!',
resource_metadata={},
).as_dict()
self.utf8_msg = utils.meter_message_from_counter(
sample.Sample(
name=u'test',
type=sample.TYPE_CUMULATIVE,
unit=u'',
volume=1,
user_id=u'test',
project_id=u'test',
resource_id=u'test_run_tasks',
timestamp=u'NOW!',
resource_metadata={u'name': [([u'TestPublish'])]},
source=u'testsource',
), 'not-so-secret')
def start_fixture(self):
"""Create some samples."""
global LOAD_APP_KWARGS
conf = LOAD_APP_KWARGS['conf']
self.conn = storage.get_connection_from_config(conf)
timestamp = datetime.datetime.utcnow()
project_id = str(uuid.uuid4())
self.source = str(uuid.uuid4())
resource_metadata = {'farmed_by': 'nancy'}
for name in ['cow', 'pig', 'sheep']:
resource_metadata.update({'breed': name}),
c = sample.Sample(name='livestock',
type='gauge',
unit='head',
volume=int(10 * random.random()),
user_id='farmerjon',
project_id=project_id,
resource_id=project_id,
timestamp=timestamp,
resource_metadata=resource_metadata,
source=self.source)
data = utils.meter_message_from_counter(
c, conf.publisher.telemetry_secret)
self.conn.record_metering_data(data)
def test_metadata(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
'dict_properties': {
'key': 'value'
},
'not_ignored_list': ['returned'],
},
source='test',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources')
metadata = data[0]['metadata']
self.assertEqual([(u'dict_properties.key', u'value'),
(u'display_name', u'test-server'),
(u'not_ignored_list', u"['returned']"),
(u'tag', u'self.sample')],
list(sorted(six.iteritems(metadata))))
def to_samples_and_publish(self, context, notification):
"""Return samples produced by *process_notification*.
Samples produced for the given notification.
:param context: Execution context from the service or RPC call
:param notification: The notification to process.
"""
# TODO(sileht): this will be moved into oslo.messaging
# see oslo.messaging bp notification-dispatcher-filter
if not self._handle_event_type(notification['event_type'],
self.event_types):
return
if self.requeue:
meters = [
utils.meter_message_from_counter(
sample, cfg.CONF.publisher.metering_secret)
for sample in self.process_notification(notification)
]
for notifier in self.transporter:
notifier.sample(context.to_dict(),
event_type='ceilometer.pipeline',
payload=meters)
else:
with self.transporter.publisher(context) as p:
p(list(self.process_notification(notification)))
def test_metadata(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
'dict_properties': {'key.$1': {'$key': 'val'}},
'not_ignored_list': ['returned'],
},
source='test',
)
msg = utils.meter_message_from_counter(
sample1, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
data = self.get_json('/resources')
metadata = data[0]['metadata']
self.assertEqual([(u'dict_properties.key:$1:$key', u'val'),
(u'display_name', u'test-server'),
(u'not_ignored_list', u"['returned']"),
(u'tag', u'self.sample')],
list(sorted(six.iteritems(metadata))))
def test_published(self):
self.data_sent = []
with mock.patch("socket.socket", self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(self.CONF, netutils.urlsplit("udp://somehost"))
publisher.publish_samples(self.test_data)
self.assertEqual(5, len(self.data_sent))
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data, encoding="utf-8")
sent_counters.append(counter)
# Check destination
self.assertEqual(("somehost", self.CONF.collector.udp_port), dest)
# Check that counters are equal
def sort_func(counter):
return counter["counter_name"]
counters = [utils.meter_message_from_counter(d, "not-so-secret") for d in self.test_data]
counters.sort(key=sort_func)
sent_counters.sort(key=sort_func)
self.assertEqual(counters, sent_counters)
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(
netutils.urlsplit('udp://somehost'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(5, len(self.data_sent))
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data)
sent_counters.append(counter)
# Check destination
self.assertEqual(('somehost',
self.CONF.collector.udp_port), dest)
# Check that counters are equal
self.assertEqual(sorted(
[utils.meter_message_from_counter(d, "not-so-secret")
for d in self.test_data]), sorted(sent_counters))
def to_samples_and_publish(self, context, notification):
"""Return samples produced by *process_notification*.
Samples produced for the given notification.
:param context: Execution context from the service or RPC call
:param notification: The notification to process.
"""
# TODO(sileht): this will be moved into oslo.messaging
# see oslo.messaging bp notification-dispatcher-filter
if not self._handle_event_type(notification['event_type'],
self.event_types):
return
if self.requeue:
meters = [
utils.meter_message_from_counter(
sample, cfg.CONF.publisher.telemetry_secret)
for sample in self.process_notification(notification)
]
for notifier in self.transporter:
notifier.sample(context.to_dict(),
event_type='ceilometer.pipeline',
payload=meters)
else:
with self.transporter.publisher(context) as p:
p(list(self.process_notification(notification)))
def publish_samples(self, samples):
"""Publish samples on RPC.
:param samples: Samples from pipeline.
"""
meters = [
utils.meter_message_from_counter(
sample, self.conf.publisher.telemetry_secret)
for sample in samples
]
topic = self.conf.publisher_notifier.metering_topic
self.local_queue.append((topic, meters))
if self.per_meter_topic:
for meter_name, meter_list in itertools.groupby(
sorted(meters, key=operator.itemgetter('counter_name')),
operator.itemgetter('counter_name')):
meter_list = list(meter_list)
topic_name = topic + '.' + meter_name
LOG.debug('Publishing %(m)d samples on %(n)s', {
'm': len(meter_list),
'n': topic_name
})
self.local_queue.append((topic_name, meter_list))
self.flush()
def publish_samples(self, context, samples):
"""Publish samples on RPC.
:param context: Execution context from the service or RPC call.
:param samples: Samples from pipeline after transformation.
"""
meters = [
utils.meter_message_from_counter(
sample, cfg.CONF.publisher.telemetry_secret)
for sample in samples
]
topic = cfg.CONF.publisher_rpc.metering_topic
self.local_queue.append((context, topic, meters))
if self.per_meter_topic:
for meter_name, meter_list in itertools.groupby(
sorted(meters, key=operator.itemgetter('counter_name')),
operator.itemgetter('counter_name')):
meter_list = list(meter_list)
topic_name = topic + '.' + meter_name
LOG.debug('Publishing %(m)d samples on %(n)s',
{'m': len(meter_list), 'n': topic_name})
self.local_queue.append((context, topic_name, meter_list))
self.flush()
def setUp(self):
super(TestQueryMetersController, self).setUp()
self.url = '/query/samples'
for cnt in [
sample.Sample('meter.test',
'cumulative',
'',
1,
'user-id1',
'project-id1',
'resource-id1',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server1',
'tag': 'self.sample',
'size': 456,
'util': 0.25,
'is_public': True
},
source='test_source'),
sample.Sample('meter.test',
'cumulative',
'',
2,
'user-id2',
'project-id2',
'resource-id2',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server2',
'tag': 'self.sample',
'size': 123,
'util': 0.75,
'is_public': True
},
source='test_source'),
sample.Sample('meter.test',
'cumulative',
'',
3,
'user-id3',
'project-id3',
'resource-id3',
timestamp=datetime.datetime(2012, 7, 2, 10, 42),
resource_metadata={
'display_name': 'test-server3',
'tag': 'self.sample',
'size': 789,
'util': 0.95,
'is_public': True
},
source='test_source')
]:
msg = utils.meter_message_from_counter(
cnt, self.CONF.publisher.metering_secret)
self.conn.record_metering_data(msg)
def test_instances_one(self):
sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
},
source='test',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample2',
},
source='test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get_json('/resources/resource-id')
self.assertEqual('resource-id', data['resource_id'])
def setUp(self):
super(TestListSamples, self).setUp()
patcher = mock.patch.object(timeutils, 'utcnow')
self.addCleanup(patcher.stop)
self.mock_utcnow = patcher.start()
self.mock_utcnow.return_value = datetime.datetime(2014, 2, 11, 16, 42)
self.sample1 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id',
'project1',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample',
'dict_properties': {'key': 'value'},
'not_ignored_list': ['returned'],
},
source='test_source',
)
msg = utils.meter_message_from_counter(
self.sample1, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg)
self.sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project2',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2',
},
source='source2',
)
msg2 = utils.meter_message_from_counter(
self.sample2, self.CONF.publisher.telemetry_secret,
)
self.conn.record_metering_data(msg2)
def setUp(self):
super(TestListUsers, self).setUp()
sample1 = sample.Sample(
'instance',
'cumulative',
'instance',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample',
},
source='test_list_users',
)
msg = utils.meter_message_from_counter(
sample1,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg)
sample2 = sample.Sample(
'instance',
'cumulative',
'',
1,
'user-id2',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample2',
},
source='not-test',
)
msg2 = utils.meter_message_from_counter(
sample2,
self.CONF.publisher.metering_secret,
)
self.conn.record_metering_data(msg2)
def make_test_data(conn, name, meter_type, unit, volume, random_min,
random_max, user_id, project_id, resource_id, start,
end, interval, resource_metadata={}, source='artificial',):
# Compute start and end timestamps for the new data.
if isinstance(start, datetime.datetime):
timestamp = start
else:
timestamp = timeutils.parse_strtime(start)
if not isinstance(end, datetime.datetime):
end = timeutils.parse_strtime(end)
increment = datetime.timedelta(minutes=interval)
print('Adding new events for meter %s.' % (name))
# Generate events
n = 0
total_volume = volume
while timestamp <= end:
if (random_min >= 0 and random_max >= 0):
# If there is a random element defined, we will add it to
# user given volume.
if isinstance(random_min, int) and isinstance(random_max, int):
total_volume += random.randint(random_min, random_max)
else:
total_volume += random.uniform(random_min, random_max)
c = sample.Sample(name=name,
type=meter_type,
unit=unit,
volume=total_volume,
user_id=user_id,
project_id=project_id,
resource_id=resource_id,
timestamp=timestamp,
resource_metadata=resource_metadata,
source=source,
)
data = utils.meter_message_from_counter(
c,
cfg.CONF.publisher.metering_secret)
conn.record_metering_data(data)
n += 1
timestamp = timestamp + increment
if (meter_type == 'gauge' or meter_type == 'delta'):
# For delta and gauge, we don't want to increase the value
# in time by random element. So we always set it back to
# volume.
total_volume = volume
print('Added %d new events for meter %s.' % (n, name))
def prepare_data(self):
def old_record_metering_data(self, data):
received_timestamp = datetime.datetime.utcnow()
self.db.resource.update(
{'_id': data['resource_id']},
{
'$set': {
'project_id': data['project_id'],
'user_id': data['user_id'],
# Current metadata being used and when it was
# last updated.
'timestamp': data['timestamp'],
'received_timestamp': received_timestamp,
'metadata': data['resource_metadata'],
'source': data['source'],
},
'$addToSet': {
'meter': {
'counter_name': data['counter_name'],
'counter_type': data['counter_type'],
},
},
},
upsert=True,
)
record = copy.copy(data)
self.db.meter.insert(record)
# Stubout with the old version DB schema, the one w/o 'counter_unit'
with mock.patch.object(self.conn,
'record_metering_data',
side_effect=old_record_metering_data):
self.counters = []
c = sample.Sample(
'volume.size',
'gauge',
'GiB',
5,
None,
None,
None,
timestamp=datetime.datetime(2012, 9, 25, 10, 30),
resource_metadata={
'display_name': 'test-volume',
'tag': 'self.counter',
},
source='test',
)
self.counters.append(c)
msg = utils.meter_message_from_counter(c, secret='not-so-secret')
self.conn.record_metering_data(self.conn, msg)
def publish_samples(self, samples):
if not self.sample_driver:
LOG.error("Can't publish samples to a non-existing dispatcher "
"'%s'", self.dispatcher_name)
return
if not isinstance(samples, list):
samples = [samples]
# not published externally; skip signing
self.get_sample_dispatcher().record_metering_data([
utils.meter_message_from_counter(sample, secret=None)
for sample in samples])
def setUp(self):
super(TestCollector, self).setUp()
conf = service.prepare_service([], [])
self.CONF = self.useFixture(fixture_config.Config(conf)).conf
self.CONF.import_opt("connection", "oslo_db.options", group="database")
self.CONF.set_override("connection", "log://", group='database')
self.CONF.set_override('telemetry_secret',
'not-so-secret',
group='publisher')
self._setup_messaging()
self.sample = utils.meter_message_from_counter(
sample.Sample(
name='foobar',
type='bad',
unit='F',
volume=1,
user_id='jd',
project_id='ceilometer',
resource_id='cat',
timestamp=timeutils.utcnow().isoformat(),
resource_metadata={},
), self.CONF.publisher.telemetry_secret)
self.utf8_msg = utils.meter_message_from_counter(
sample.Sample(
name=u'test',
type=sample.TYPE_CUMULATIVE,
unit=u'',
volume=1,
user_id=u'test',
project_id=u'test',
resource_id=u'test_run_tasks',
timestamp=timeutils.utcnow().isoformat(),
resource_metadata={u'name': [([u'TestPublish'])]},
source=u'testsource',
), 'not-so-secret')
self.mock_dispatcher = self._setup_fake_dispatcher()
self.srv = collector.CollectorService(0, self.CONF)
def setUp(self):
super(TestAPIACL, self).setUp()
self.auth_token_fixture = self.useFixture(
ksm_fixture.AuthTokenFixture())
self.auth_token_fixture.add_token_data(
token_id=VALID_TOKEN,
# FIXME(morganfainberg): The project-id should be a proper uuid
project_id='123i2910',
role_list=['admin'],
user_name='user_id2',
user_id='user_id2',
is_v2=True)
self.auth_token_fixture.add_token_data(
token_id=VALID_TOKEN2,
# FIXME(morganfainberg): The project-id should be a proper uuid
project_id='project-good',
role_list=['Member'],
user_name='user_id1',
user_id='user_id1',
is_v2=True)
for cnt in [
sample.Sample('meter.test',
'cumulative',
'',
1,
'user-good',
'project-good',
'resource-good',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample'
},
source='test_source'),
sample.Sample('meter.mine',
'gauge',
'',
1,
'user-fred',
'project-good',
'resource-56',
timestamp=datetime.datetime(2012, 7, 2, 10, 43),
resource_metadata={
'display_name': 'test-server',
'tag': 'self.sample4'
},
source='test_source')
]:
msg = utils.meter_message_from_counter(
cnt, self.CONF.publisher.telemetry_secret)
self.conn.record_metering_data(msg)
def publish_samples(self, samples):
if not self.sample_driver:
LOG.error(_LE("Can't publish samples to a non-existing dispatcher "
"'%s'"), self.dispatcher_name)
return
if not isinstance(samples, list):
samples = [samples]
self.get_sample_dispatcher().record_metering_data([
utils.meter_message_from_counter(
sample, cfg.CONF.publisher.telemetry_secret)
for sample in samples
])
def publish_samples(self, context, samples):
"""Publish samples on RPC.
:param context: Execution context from the service or RPC call.
:param samples: Samples from pipeline after transformation.
"""
meters = [
utils.meter_message_from_counter(
sample, cfg.CONF.publisher.metering_secret)
for sample in samples
]
topic = cfg.CONF.publisher_rpc.metering_topic
msg = {
'method': self.target,
'version': '1.0',
'args': {
'data': meters
},
}
LOG.audit(
_('Publishing %(m)d samples on %(t)s') %
({
'm': len(msg['args']['data']),
't': topic
}))
self.local_queue.append((context, topic, msg))
if self.per_meter_topic:
for meter_name, meter_list in itertools.groupby(
sorted(meters, key=operator.itemgetter('counter_name')),
operator.itemgetter('counter_name')):
msg = {
'method': self.target,
'version': '1.0',
'args': {
'data': list(meter_list)
},
}
topic_name = topic + '.' + meter_name
LOG.audit(
_('Publishing %(m)d samples on %(n)s') %
({
'm': len(msg['args']['data']),
'n': topic_name
}))
self.local_queue.append((context, topic_name, msg))
self.flush()
