def test_instances_one(self): sample1 = sample.Sample( "instance", "cumulative", "", 1, "user-id", "project-id", "resource-id", timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={"display_name": "test-server", "tag": "self.sample"}, source="test", ) msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg) sample2 = sample.Sample( "instance", "cumulative", "", 1, "user-id", "project-id", "resource-id-alternate", timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={"display_name": "test-server", "tag": "self.sample2"}, source="test", ) msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg2) data = self.get_json("/resources/resource-id") self.assertEqual("resource-id", data["resource_id"])
def setUp(self): super(TestListProjects, self).setUp() sample1 = sample.Sample( "instance", "cumulative", "instance", 1, "user-id", "project-id", "resource-id", timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={"display_name": "test-server", "tag": "self.sample"}, source="test_list_projects", ) msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.metering_secret) self.conn.record_metering_data(msg) sample2 = sample.Sample( "instance", "cumulative", "instance", 1, "user-id2", "project-id2", "resource-id-alternate", timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={"display_name": "test-server", "tag": "self.sample2"}, source="test_list_users", ) msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.metering_secret) self.conn.record_metering_data(msg2)
def test_with_project(self): sample1 = sample.Sample( "instance", "cumulative", "", 1, "user-id", "project-id", "resource-id", timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={"display_name": "test-server", "tag": "self.sample"}, source="test_list_resources", ) msg = utils.meter_message_from_counter(sample1, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg) sample2 = sample.Sample( "instance", "cumulative", "", 1, "user-id2", "project-id2", "resource-id-alternate", timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={"display_name": "test-server", "tag": "self.sample2"}, source="not-test", ) msg2 = utils.meter_message_from_counter(sample2, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg2) data = self.get_json("/resources", q=[{"field": "project_id", "value": "project-id"}]) ids = [r["resource_id"] for r in data] self.assertEqual(["resource-id"], ids)
def test_with_invalid_resource_id(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id-1', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id', 'resource-id-2', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2', }, source='test_list_resources', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg2) resp1 = self.get_json('/resources/resource-id-1') self.assertEqual("resource-id-1", resp1["resource_id"]) resp2 = self.get_json('/resources/resource-id-2') self.assertEqual("resource-id-2", resp2["resource_id"]) resp3 = self.get_json('/resources/resource-id-3', expect_errors=True) self.assertEqual(404, resp3.status_code) json_data = resp3.body if six.PY3: json_data = json_data.decode('utf-8') self.assertEqual("Resource resource-id-3 Not Found", json.loads(json_data)['error_message'] ['faultstring'])
def test_with_user(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample2', }, source='not-test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources', q=[{ 'field': 'user_id', 'value': 'user-id', }]) ids = [r['resource_id'] for r in data] self.assertEqual(['resource-id'], ids)
def test_instances(self): timestamps = { 'resource-id': datetime.datetime(2012, 7, 2, 10, 40), 'resource-id-alternate': datetime.datetime(2012, 7, 2, 10, 41), } sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=timestamps['resource-id'], resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', }, source='test', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id-alternate', timestamp=timestamps['resource-id-alternate'], resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2', }, source='test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources') self.assertEqual(2, len(data)) for res in data: timestamp = timestamps.get(res['resource_id']) self._verify_sample_timestamps(res, timestamp, timestamp)
def test_instances(self): timestamps = { 'resource-id': datetime.datetime(2012, 7, 2, 10, 40), 'resource-id-alternate': datetime.datetime(2012, 7, 2, 10, 41), } sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=timestamps['resource-id'], resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', }, source='test', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id-alternate', timestamp=timestamps['resource-id-alternate'], resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2', }, source='test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources') self.assertEqual(2, len(data)) for res in data: timestamp = timestamps.get(res['resource_id']) self._verify_sample_timestamps(res, timestamp, timestamp)
def setUp(self): super(TestListEvents, self).setUp() timeutils.utcnow.override_time = datetime.datetime(2014, 2, 11, 16, 42) self.sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project1', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', 'dict_properties': { 'key': 'value' }, 'not_ignored_list': ['returned'], }, source='test_source', ) msg = utils.meter_message_from_counter( self.sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) self.sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project2', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample2', }, source='source2', ) msg2 = utils.meter_message_from_counter( self.sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2)
def test_with_source(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2', }, source='not-test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources', q=[{'field': 'source', 'value': 'test_list_resources', }]) ids = [r['resource_id'] for r in data] self.assertEqual(['resource-id'], ids) sources = [r['source'] for r in data] self.assertEqual(['test_list_resources'], sources)
def setUp(self): super(TestAPIACL, self).setUp() self.environ = {'fake.cache': FakeMemcache()} for cnt in [ sample.Sample('meter.test', 'cumulative', '', 1, 'user-good', 'project-good', 'resource-good', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample' }, source='test_source'), sample.Sample('meter.mine', 'gauge', '', 1, 'user-fred', 'project-good', 'resource-56', timestamp=datetime.datetime(2012, 7, 2, 10, 43), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample4' }, source='test_source') ]: msg = utils.meter_message_from_counter( cnt, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg)
def setUp(self): super(TestListResourcesRestriction, self).setUp() self.CONF.set_override('default_api_return_limit', 10, group='api') for i in range(20): s = sample.Sample( 'volume.size', 'gauge', 'GiB', 5 + i, 'user-id', 'project1', 'resource-id%s' % i, timestamp=(datetime.datetime(2012, 9, 25, 10, 30) + datetime.timedelta(seconds=i)), resource_metadata={ 'display_name': 'test-volume', 'tag': 'self.sample', }, source='source1', ) msg = utils.meter_message_from_counter( s, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg)
def start_fixture(self): """Create some samples.""" conf = fixture_config.Config().conf self.conn = storage.get_connection_from_config(conf) timestamp = datetime.datetime.utcnow() project_id = str(uuid.uuid4()) self.source = str(uuid.uuid4()) resource_metadata = {"farmed_by": "nancy"} for name in ["cow", "pig", "sheep"]: resource_metadata.update({"breed": name}), c = sample.Sample( name="livestock", type="gauge", unit="head", volume=int(10 * random.random()), user_id="farmerjon", project_id=project_id, resource_id=project_id, timestamp=timestamp, resource_metadata=resource_metadata, source=self.source, ) data = utils.meter_message_from_counter(c, conf.publisher.telemetry_secret) self.conn.record_metering_data(data)
def test_resource_id_with_slash(self): s = sample.Sample( 'storage.containers.objects', 'gauge', '', 1, '19fbed01c21f4912901057021b9e7111', '45acc90399134206b3b41f3d3a0a06d6', '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance', timestamp=datetime.datetime(2012, 7, 2, 10, 40).isoformat(), resource_metadata={}, source='test_show_special_resource', ) msg = utils.meter_message_from_counter( s, self.CONF.publisher.telemetry_secret, ) msg['timestamp'] = datetime.datetime(2012, 7, 2, 10, 40) self.conn.record_metering_data(msg) rid_encoded = '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb%252Fglance' resp = self.get_json('/resources/%s' % rid_encoded) self.assertEqual("19fbed01c21f4912901057021b9e7111", resp["user_id"]) self.assertEqual('29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance', resp["resource_id"])
def test_resource_meter_links(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources') links = data[0]['links'] self.assertEqual(2, len(links)) self.assertEqual('self', links[0]['rel']) self.assertIn((self.PATH_PREFIX + '/resources/resource-id'), links[0]['href']) self.assertEqual('instance', links[1]['rel']) self.assertIn((self.PATH_PREFIX + '/meters/instance?' 'q.field=resource_id&q.value=resource-id'), links[1]['href'])
def test_instance_multiple_samples(self): timestamps = [ datetime.datetime(2012, 7, 2, 10, 41), datetime.datetime(2012, 7, 2, 10, 42), datetime.datetime(2012, 7, 2, 10, 40), ] for timestamp in timestamps: datapoint = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=timestamp, resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample-%s' % timestamp, }, source='test', ) msg = utils.meter_message_from_counter( datapoint, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources') self.assertEqual(1, len(data)) self._verify_resource_timestamps(data[0], timestamps[-1], timestamps[1])
def test_published(self): self.data_sent = [] with mock.patch('socket.socket', self._make_fake_socket(self.data_sent)): publisher = udp.UDPPublisher(self.CONF, netutils.urlsplit('udp://somehost')) publisher.publish_samples(self.test_data) self.assertEqual(5, len(self.data_sent)) sent_counters = [] for data, dest in self.data_sent: counter = msgpack.loads(data, encoding="utf-8") sent_counters.append(counter) # Check destination self.assertEqual(('somehost', 4952), dest) # Check that counters are equal def sort_func(counter): return counter['counter_name'] counters = [ utils.meter_message_from_counter(d, "not-so-secret") for d in self.test_data ] counters.sort(key=sort_func) sent_counters.sort(key=sort_func) self.assertEqual(counters, sent_counters)
def test_with_user_wrong_tenant(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id2', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample1', }, source='not-test', ) msg2 = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources', headers={"X-Roles": "Member", "X-Project-Id": "project-wrong"}) ids = set(r['resource_id'] for r in data) self.assertEqual(set(), ids)
def start_fixture(self): """Create some samples.""" conf = fixture_config.Config().conf self.conn = storage.get_connection_from_config(conf) timestamp = datetime.datetime.utcnow() project_id = str(uuid.uuid4()) self.source = str(uuid.uuid4()) resource_metadata = {'farmed_by': 'nancy'} for name in ['cow', 'pig', 'sheep']: resource_metadata.update({'breed': name}), c = sample.Sample(name='livestock', type='gauge', unit='head', volume=int(10 * random.random()), user_id='farmerjon', project_id=project_id, resource_id=project_id, timestamp=timestamp, resource_metadata=resource_metadata, source=self.source, ) data = utils.meter_message_from_counter( c, conf.publisher.telemetry_secret) self.conn.record_metering_data(data)
def test_with_user_wrong_tenant(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id2', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample1', }, source='not-test', ) msg2 = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources', headers={ "X-Roles": "Member", "X-Project-Id": "project-wrong" }) ids = set(r['resource_id'] for r in data) self.assertEqual(set(), ids)
def test_resource_skip_meter_links(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources?meter_links=0') links = data[0]['links'] self.assertEqual(len(links), 1) self.assertEqual(links[0]['rel'], 'self') self.assertTrue((self.PATH_PREFIX + '/resources/resource-id') in links[0]['href'])
def publish_samples(self, samples): """Publish samples on RPC. :param samples: Samples from pipeline. """ meters = [ utils.meter_message_from_counter( sample, self.conf.publisher.telemetry_secret) for sample in samples ] topic = self.conf.publisher_notifier.metering_topic self.local_queue.append((topic, meters)) if self.per_meter_topic: for meter_name, meter_list in itertools.groupby( sorted(meters, key=operator.itemgetter('counter_name')), operator.itemgetter('counter_name')): meter_list = list(meter_list) topic_name = topic + '.' + meter_name LOG.debug('Publishing %(m)d samples on %(n)s', {'m': len(meter_list), 'n': topic_name}) self.local_queue.append((topic_name, meter_list)) self.flush()
def test_resource_id_with_slash(self): s = sample.Sample( 'storage.containers.objects', 'gauge', '', 1, '19fbed01c21f4912901057021b9e7111', '45acc90399134206b3b41f3d3a0a06d6', '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance', timestamp=datetime.datetime(2012, 7, 2, 10, 40).isoformat(), resource_metadata={}, source='test_show_special_resource', ) msg = utils.meter_message_from_counter( s, self.CONF.publisher.telemetry_secret, ) msg['timestamp'] = datetime.datetime(2012, 7, 2, 10, 40) self.conn.record_metering_data(msg) rid_encoded = '29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb%252Fglance' resp = self.get_json('/resources/%s' % rid_encoded) self.assertEqual("19fbed01c21f4912901057021b9e7111", resp["user_id"]) self.assertEqual('29f809d9-88bb-4c40-b1ba-a77a1fcf8ceb/glance', resp["resource_id"])
def setUp(self): super(TestListResourcesBase, self).setUp() for cnt in [ sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample'}, source='test_list_resources', ), sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2'}, source='test_list_resources', ), sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id2', 'resource-id2', timestamp=datetime.datetime(2012, 7, 2, 10, 42), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample3'}, source='test_list_resources', ), sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 43), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample4'}, source='test_list_resources', )]: msg = utils.meter_message_from_counter( cnt, self.CONF.publisher.metering_secret) self.conn.record_metering_data(msg)
def test_published(self): self.data_sent = [] with mock.patch('socket.socket', self._make_fake_socket(self.data_sent)): publisher = udp.UDPPublisher( network_utils.urlsplit('udp://somehost')) publisher.publish_samples(None, self.test_data) self.assertEqual(5, len(self.data_sent)) sent_counters = [] for data, dest in self.data_sent: counter = msgpack.loads(data) sent_counters.append(counter) # Check destination self.assertEqual(('somehost', self.CONF.collector.udp_port), dest) # Check that counters are equal self.assertEqual(sorted( [utils.meter_message_from_counter(d, "not-so-secret") for d in self.test_data]), sorted(sent_counters))
def test_resource_skip_meter_links(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_resources', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources?meter_links=0') links = data[0]['links'] self.assertEqual(len(links), 1) self.assertEqual(links[0]['rel'], 'self') self.assertTrue((self.PATH_PREFIX + '/resources/resource-id') in links[0]['href'])
def setUp(self): super(TestAPIACL, self).setUp() self.environ = {'fake.cache': FakeMemcache()} for cnt in [ sample.Sample( 'meter.test', 'cumulative', '', 1, 'user-good', 'project-good', 'resource-good', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample'}, source='test_source'), sample.Sample( 'meter.mine', 'gauge', '', 1, 'user-fred', 'project-good', 'resource-56', timestamp=datetime.datetime(2012, 7, 2, 10, 43), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample4'}, source='test_source')]: msg = utils.meter_message_from_counter( cnt, self.CONF.publisher.metering_secret) self.conn.record_metering_data(msg)
def test_instance_multiple_samples(self): timestamps = [ datetime.datetime(2012, 7, 2, 10, 41), datetime.datetime(2012, 7, 2, 10, 42), datetime.datetime(2012, 7, 2, 10, 40), ] for timestamp in timestamps: datapoint = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=timestamp, resource_metadata={'display_name': 'test-server', 'tag': 'self.sample-%s' % timestamp, }, source='test', ) msg = utils.meter_message_from_counter( datapoint, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources') self.assertEqual(1, len(data)) self._verify_resource_timestamps(data[0], timestamps[-1], timestamps[1])
def setUp(self): super(TestCollector, self).setUp() messaging.setup('fake://') self.addCleanup(messaging.cleanup) self.CONF = self.useFixture(config.Config()).conf self.CONF.set_override("connection", "log://", group='database') self.srv = collector.CollectorService() self.CONF.publisher.metering_secret = 'not-so-secret' self.counter = sample.Sample( name='foobar', type='bad', unit='F', volume=1, user_id='jd', project_id='ceilometer', resource_id='cat', timestamp='NOW!', resource_metadata={}, ).as_dict() self.utf8_msg = utils.meter_message_from_counter( sample.Sample( name=u'test', type=sample.TYPE_CUMULATIVE, unit=u'', volume=1, user_id=u'test', project_id=u'test', resource_id=u'test_run_tasks', timestamp=u'NOW!', resource_metadata={u'name': [([u'TestPublish'])]}, source=u'testsource', ), 'not-so-secret')
def start_fixture(self): """Create some samples.""" global LOAD_APP_KWARGS conf = LOAD_APP_KWARGS['conf'] self.conn = storage.get_connection_from_config(conf) timestamp = datetime.datetime.utcnow() project_id = str(uuid.uuid4()) self.source = str(uuid.uuid4()) resource_metadata = {'farmed_by': 'nancy'} for name in ['cow', 'pig', 'sheep']: resource_metadata.update({'breed': name}), c = sample.Sample(name='livestock', type='gauge', unit='head', volume=int(10 * random.random()), user_id='farmerjon', project_id=project_id, resource_id=project_id, timestamp=timestamp, resource_metadata=resource_metadata, source=self.source) data = utils.meter_message_from_counter( c, conf.publisher.telemetry_secret) self.conn.record_metering_data(data)
def test_metadata(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', 'dict_properties': { 'key': 'value' }, 'not_ignored_list': ['returned'], }, source='test', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources') metadata = data[0]['metadata'] self.assertEqual([(u'dict_properties.key', u'value'), (u'display_name', u'test-server'), (u'not_ignored_list', u"['returned']"), (u'tag', u'self.sample')], list(sorted(six.iteritems(metadata))))
def to_samples_and_publish(self, context, notification): """Return samples produced by *process_notification*. Samples produced for the given notification. :param context: Execution context from the service or RPC call :param notification: The notification to process. """ # TODO(sileht): this will be moved into oslo.messaging # see oslo.messaging bp notification-dispatcher-filter if not self._handle_event_type(notification['event_type'], self.event_types): return if self.requeue: meters = [ utils.meter_message_from_counter( sample, cfg.CONF.publisher.metering_secret) for sample in self.process_notification(notification) ] for notifier in self.transporter: notifier.sample(context.to_dict(), event_type='ceilometer.pipeline', payload=meters) else: with self.transporter.publisher(context) as p: p(list(self.process_notification(notification)))
def test_metadata(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', 'dict_properties': {'key.$1': {'$key': 'val'}}, 'not_ignored_list': ['returned'], }, source='test', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) data = self.get_json('/resources') metadata = data[0]['metadata'] self.assertEqual([(u'dict_properties.key:$1:$key', u'val'), (u'display_name', u'test-server'), (u'not_ignored_list', u"['returned']"), (u'tag', u'self.sample')], list(sorted(six.iteritems(metadata))))
def test_published(self): self.data_sent = [] with mock.patch("socket.socket", self._make_fake_socket(self.data_sent)): publisher = udp.UDPPublisher(self.CONF, netutils.urlsplit("udp://somehost")) publisher.publish_samples(self.test_data) self.assertEqual(5, len(self.data_sent)) sent_counters = [] for data, dest in self.data_sent: counter = msgpack.loads(data, encoding="utf-8") sent_counters.append(counter) # Check destination self.assertEqual(("somehost", self.CONF.collector.udp_port), dest) # Check that counters are equal def sort_func(counter): return counter["counter_name"] counters = [utils.meter_message_from_counter(d, "not-so-secret") for d in self.test_data] counters.sort(key=sort_func) sent_counters.sort(key=sort_func) self.assertEqual(counters, sent_counters)
def test_published(self): self.data_sent = [] with mock.patch('socket.socket', self._make_fake_socket(self.data_sent)): publisher = udp.UDPPublisher( netutils.urlsplit('udp://somehost')) publisher.publish_samples(None, self.test_data) self.assertEqual(5, len(self.data_sent)) sent_counters = [] for data, dest in self.data_sent: counter = msgpack.loads(data) sent_counters.append(counter) # Check destination self.assertEqual(('somehost', self.CONF.collector.udp_port), dest) # Check that counters are equal self.assertEqual(sorted( [utils.meter_message_from_counter(d, "not-so-secret") for d in self.test_data]), sorted(sent_counters))
def to_samples_and_publish(self, context, notification): """Return samples produced by *process_notification*. Samples produced for the given notification. :param context: Execution context from the service or RPC call :param notification: The notification to process. """ # TODO(sileht): this will be moved into oslo.messaging # see oslo.messaging bp notification-dispatcher-filter if not self._handle_event_type(notification['event_type'], self.event_types): return if self.requeue: meters = [ utils.meter_message_from_counter( sample, cfg.CONF.publisher.telemetry_secret) for sample in self.process_notification(notification) ] for notifier in self.transporter: notifier.sample(context.to_dict(), event_type='ceilometer.pipeline', payload=meters) else: with self.transporter.publisher(context) as p: p(list(self.process_notification(notification)))
def publish_samples(self, samples): """Publish samples on RPC. :param samples: Samples from pipeline. """ meters = [ utils.meter_message_from_counter( sample, self.conf.publisher.telemetry_secret) for sample in samples ] topic = self.conf.publisher_notifier.metering_topic self.local_queue.append((topic, meters)) if self.per_meter_topic: for meter_name, meter_list in itertools.groupby( sorted(meters, key=operator.itemgetter('counter_name')), operator.itemgetter('counter_name')): meter_list = list(meter_list) topic_name = topic + '.' + meter_name LOG.debug('Publishing %(m)d samples on %(n)s', { 'm': len(meter_list), 'n': topic_name }) self.local_queue.append((topic_name, meter_list)) self.flush()
def publish_samples(self, context, samples): """Publish samples on RPC. :param context: Execution context from the service or RPC call. :param samples: Samples from pipeline after transformation. """ meters = [ utils.meter_message_from_counter( sample, cfg.CONF.publisher.telemetry_secret) for sample in samples ] topic = cfg.CONF.publisher_rpc.metering_topic self.local_queue.append((context, topic, meters)) if self.per_meter_topic: for meter_name, meter_list in itertools.groupby( sorted(meters, key=operator.itemgetter('counter_name')), operator.itemgetter('counter_name')): meter_list = list(meter_list) topic_name = topic + '.' + meter_name LOG.debug('Publishing %(m)d samples on %(n)s', {'m': len(meter_list), 'n': topic_name}) self.local_queue.append((context, topic_name, meter_list)) self.flush()
def setUp(self): super(TestQueryMetersController, self).setUp() self.url = '/query/samples' for cnt in [ sample.Sample('meter.test', 'cumulative', '', 1, 'user-id1', 'project-id1', 'resource-id1', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server1', 'tag': 'self.sample', 'size': 456, 'util': 0.25, 'is_public': True }, source='test_source'), sample.Sample('meter.test', 'cumulative', '', 2, 'user-id2', 'project-id2', 'resource-id2', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server2', 'tag': 'self.sample', 'size': 123, 'util': 0.75, 'is_public': True }, source='test_source'), sample.Sample('meter.test', 'cumulative', '', 3, 'user-id3', 'project-id3', 'resource-id3', timestamp=datetime.datetime(2012, 7, 2, 10, 42), resource_metadata={ 'display_name': 'test-server3', 'tag': 'self.sample', 'size': 789, 'util': 0.95, 'is_public': True }, source='test_source') ]: msg = utils.meter_message_from_counter( cnt, self.CONF.publisher.metering_secret) self.conn.record_metering_data(msg)
def test_instances_one(self): sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', }, source='test', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project-id', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample2', }, source='test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2) data = self.get_json('/resources/resource-id') self.assertEqual('resource-id', data['resource_id'])
def setUp(self): super(TestListSamples, self).setUp() patcher = mock.patch.object(timeutils, 'utcnow') self.addCleanup(patcher.stop) self.mock_utcnow = patcher.start() self.mock_utcnow.return_value = datetime.datetime(2014, 2, 11, 16, 42) self.sample1 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id', 'project1', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample', 'dict_properties': {'key': 'value'}, 'not_ignored_list': ['returned'], }, source='test_source', ) msg = utils.meter_message_from_counter( self.sample1, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg) self.sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project2', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={'display_name': 'test-server', 'tag': 'self.sample2', }, source='source2', ) msg2 = utils.meter_message_from_counter( self.sample2, self.CONF.publisher.telemetry_secret, ) self.conn.record_metering_data(msg2)
def setUp(self): super(TestListUsers, self).setUp() sample1 = sample.Sample( 'instance', 'cumulative', 'instance', 1, 'user-id', 'project-id', 'resource-id', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample', }, source='test_list_users', ) msg = utils.meter_message_from_counter( sample1, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg) sample2 = sample.Sample( 'instance', 'cumulative', '', 1, 'user-id2', 'project-id', 'resource-id-alternate', timestamp=datetime.datetime(2012, 7, 2, 10, 41), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample2', }, source='not-test', ) msg2 = utils.meter_message_from_counter( sample2, self.CONF.publisher.metering_secret, ) self.conn.record_metering_data(msg2)
def make_test_data(conn, name, meter_type, unit, volume, random_min, random_max, user_id, project_id, resource_id, start, end, interval, resource_metadata={}, source='artificial',): # Compute start and end timestamps for the new data. if isinstance(start, datetime.datetime): timestamp = start else: timestamp = timeutils.parse_strtime(start) if not isinstance(end, datetime.datetime): end = timeutils.parse_strtime(end) increment = datetime.timedelta(minutes=interval) print('Adding new events for meter %s.' % (name)) # Generate events n = 0 total_volume = volume while timestamp <= end: if (random_min >= 0 and random_max >= 0): # If there is a random element defined, we will add it to # user given volume. if isinstance(random_min, int) and isinstance(random_max, int): total_volume += random.randint(random_min, random_max) else: total_volume += random.uniform(random_min, random_max) c = sample.Sample(name=name, type=meter_type, unit=unit, volume=total_volume, user_id=user_id, project_id=project_id, resource_id=resource_id, timestamp=timestamp, resource_metadata=resource_metadata, source=source, ) data = utils.meter_message_from_counter( c, cfg.CONF.publisher.metering_secret) conn.record_metering_data(data) n += 1 timestamp = timestamp + increment if (meter_type == 'gauge' or meter_type == 'delta'): # For delta and gauge, we don't want to increase the value # in time by random element. So we always set it back to # volume. total_volume = volume print('Added %d new events for meter %s.' % (n, name))
def prepare_data(self): def old_record_metering_data(self, data): received_timestamp = datetime.datetime.utcnow() self.db.resource.update( {'_id': data['resource_id']}, { '$set': { 'project_id': data['project_id'], 'user_id': data['user_id'], # Current metadata being used and when it was # last updated. 'timestamp': data['timestamp'], 'received_timestamp': received_timestamp, 'metadata': data['resource_metadata'], 'source': data['source'], }, '$addToSet': { 'meter': { 'counter_name': data['counter_name'], 'counter_type': data['counter_type'], }, }, }, upsert=True, ) record = copy.copy(data) self.db.meter.insert(record) # Stubout with the old version DB schema, the one w/o 'counter_unit' with mock.patch.object(self.conn, 'record_metering_data', side_effect=old_record_metering_data): self.counters = [] c = sample.Sample( 'volume.size', 'gauge', 'GiB', 5, None, None, None, timestamp=datetime.datetime(2012, 9, 25, 10, 30), resource_metadata={ 'display_name': 'test-volume', 'tag': 'self.counter', }, source='test', ) self.counters.append(c) msg = utils.meter_message_from_counter(c, secret='not-so-secret') self.conn.record_metering_data(self.conn, msg)
def publish_samples(self, samples): if not self.sample_driver: LOG.error("Can't publish samples to a non-existing dispatcher " "'%s'", self.dispatcher_name) return if not isinstance(samples, list): samples = [samples] # not published externally; skip signing self.get_sample_dispatcher().record_metering_data([ utils.meter_message_from_counter(sample, secret=None) for sample in samples])
def setUp(self): super(TestCollector, self).setUp() conf = service.prepare_service([], []) self.CONF = self.useFixture(fixture_config.Config(conf)).conf self.CONF.import_opt("connection", "oslo_db.options", group="database") self.CONF.set_override("connection", "log://", group='database') self.CONF.set_override('telemetry_secret', 'not-so-secret', group='publisher') self._setup_messaging() self.sample = utils.meter_message_from_counter( sample.Sample( name='foobar', type='bad', unit='F', volume=1, user_id='jd', project_id='ceilometer', resource_id='cat', timestamp=timeutils.utcnow().isoformat(), resource_metadata={}, ), self.CONF.publisher.telemetry_secret) self.utf8_msg = utils.meter_message_from_counter( sample.Sample( name=u'test', type=sample.TYPE_CUMULATIVE, unit=u'', volume=1, user_id=u'test', project_id=u'test', resource_id=u'test_run_tasks', timestamp=timeutils.utcnow().isoformat(), resource_metadata={u'name': [([u'TestPublish'])]}, source=u'testsource', ), 'not-so-secret') self.mock_dispatcher = self._setup_fake_dispatcher() self.srv = collector.CollectorService(0, self.CONF)
def setUp(self): super(TestAPIACL, self).setUp() self.auth_token_fixture = self.useFixture( ksm_fixture.AuthTokenFixture()) self.auth_token_fixture.add_token_data( token_id=VALID_TOKEN, # FIXME(morganfainberg): The project-id should be a proper uuid project_id='123i2910', role_list=['admin'], user_name='user_id2', user_id='user_id2', is_v2=True) self.auth_token_fixture.add_token_data( token_id=VALID_TOKEN2, # FIXME(morganfainberg): The project-id should be a proper uuid project_id='project-good', role_list=['Member'], user_name='user_id1', user_id='user_id1', is_v2=True) for cnt in [ sample.Sample('meter.test', 'cumulative', '', 1, 'user-good', 'project-good', 'resource-good', timestamp=datetime.datetime(2012, 7, 2, 10, 40), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample' }, source='test_source'), sample.Sample('meter.mine', 'gauge', '', 1, 'user-fred', 'project-good', 'resource-56', timestamp=datetime.datetime(2012, 7, 2, 10, 43), resource_metadata={ 'display_name': 'test-server', 'tag': 'self.sample4' }, source='test_source') ]: msg = utils.meter_message_from_counter( cnt, self.CONF.publisher.telemetry_secret) self.conn.record_metering_data(msg)
def publish_samples(self, samples): if not self.sample_driver: LOG.error(_LE("Can't publish samples to a non-existing dispatcher " "'%s'"), self.dispatcher_name) return if not isinstance(samples, list): samples = [samples] self.get_sample_dispatcher().record_metering_data([ utils.meter_message_from_counter( sample, cfg.CONF.publisher.telemetry_secret) for sample in samples ])
def publish_samples(self, context, samples): """Publish samples on RPC. :param context: Execution context from the service or RPC call. :param samples: Samples from pipeline after transformation. """ meters = [ utils.meter_message_from_counter( sample, cfg.CONF.publisher.metering_secret) for sample in samples ] topic = cfg.CONF.publisher_rpc.metering_topic msg = { 'method': self.target, 'version': '1.0', 'args': { 'data': meters }, } LOG.audit( _('Publishing %(m)d samples on %(t)s') % ({ 'm': len(msg['args']['data']), 't': topic })) self.local_queue.append((context, topic, msg)) if self.per_meter_topic: for meter_name, meter_list in itertools.groupby( sorted(meters, key=operator.itemgetter('counter_name')), operator.itemgetter('counter_name')): msg = { 'method': self.target, 'version': '1.0', 'args': { 'data': list(meter_list) }, } topic_name = topic + '.' + meter_name LOG.audit( _('Publishing %(m)d samples on %(n)s') % ({ 'm': len(msg['args']['data']), 'n': topic_name })) self.local_queue.append((context, topic_name, msg)) self.flush()