def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector, self.queue,
self.exchange, self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect(
'%s://%s:%s@%s/%s' %
(self.config.get('db', 'vendor'), self.config.get('db', 'user'),
self.config.get('db', 'pass'), self.config.get(
'db', 'host'), self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(
username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def setUp(self):
if ClientTestCases.python_version < 270:
self.config = Config('../../bin/giraffe.cfg')
self.gc = GiraffeClient(
username=self.config.get('client', 'user'),
password=self.config.get('client', 'pass'),
tenant_name=self.config.get('client', 'tenant_name'),
tenant_id=self.config.get('client', 'tenant_id'),
auth_url=self.config.get('auth', 'public_url'))
def __init__(self, auth_token=None, \
# username=None, \
# password=None, \
# tenant_name=None, \
# tenant_id=None, \
# protocol=None, \
# endpoint=None, \
# auth_url=None, \
**kwargs):
"""
Creates a new GiraffeClient instance:
- if auth_token is not None, auth_token is used to authenticate
client requests
- alternatively, credentials (username, password, tenant_id, etc.)
to be used to authenticate client requests can be passed to
__init__() via named parameters
"""
# [DELETED]
# - right now, also a fallback to giraffe.cfg is implemented (but not
# to env. variables)
self.config = Config('giraffe.cfg')
if not auth_token:
_username = kwargs.get('username')
# , self.config.get('client', 'user'))
_password = kwargs.get('password')
# , self.config.get('client', 'pass'))
_tenant_name = kwargs.get('tenant_name')
# , self.config.get('client', 'tenant_name'))
_tenant_id = kwargs.get('tenant_id')
# , self.config.get('client', 'tenant_id'))
_auth_url = kwargs.get('auth_url', \
self.config.get('auth', 'public_url'))
auth_token = AuthProxy.get_token(username=_username,
password=_password,
tenant_name=_tenant_name,
tenant_id=_tenant_id,
auth_url=_auth_url)
self.auth_header = dict([('X-Auth-Token', auth_token)])
#@[fbahr] - TODO: Exception handling
self.protocol = kwargs.get('protocol', 'http')
self.endpoint = kwargs.get('endpoint')
if not self.endpoint:
host = kwargs.get('host', self.config.get('rest_api', 'host'))
port = kwargs.get('port', self.config.get('rest_api', 'port'))
self.endpoint = ':'.join((host, port))
def setUpClass(cls):
cls.config = Config('../../bin/giraffe.cfg')
cls.gc = GiraffeClient(username=cls.config.get('client', 'user'),
password=cls.config.get('client', 'pass'),
tenant_name=cls.config.get(
'client', 'tenant_name'),
tenant_id=cls.config.get('client', 'tenant_id'),
auth_url=cls.config.get('auth', 'public_url'))
def setUp(self):
if ClientTestCases.python_version < 270:
self.config = Config('../../bin/giraffe.cfg')
self.gc = GiraffeClient(username=self.config.get('client', 'user'),
password=self.config.get('client', 'pass'),
tenant_name=self.config.get('client', 'tenant_name'),
tenant_id=self.config.get('client', 'tenant_id'),
auth_url=self.config.get('auth', 'public_url'))
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector,
self.queue,
self.exchange,
self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect('%s://%s:%s@%s/%s'
% (self.config.get('db', 'vendor'),
self.config.get('db', 'user'),
self.config.get('db', 'pass'),
self.config.get('db', 'host'),
self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
class ClientTestCases(unittest.TestCase):
python_version = int(''.join([str(i) for i in sys.version_info[0:3]]))
@classmethod
def setUpClass(cls):
cls.config = Config('../../bin/giraffe.cfg')
cls.gc = GiraffeClient(username=cls.config.get('client', 'user'),
password=cls.config.get('client', 'pass'),
tenant_name=cls.config.get('client', 'tenant_name'),
tenant_id=cls.config.get('client', 'tenant_id'),
auth_url=cls.config.get('auth', 'public_url'))
def setUp(self):
if ClientTestCases.python_version < 270:
self.config = Config('../../bin/giraffe.cfg')
self.gc = GiraffeClient(username=self.config.get('client', 'user'),
password=self.config.get('client', 'pass'),
tenant_name=self.config.get('client', 'tenant_name'),
tenant_id=self.config.get('client', 'tenant_id'),
auth_url=self.config.get('auth', 'public_url'))
def test_get_set_auth_token(self):
auth_token = self.gc.auth_token
self.gc.auth_token = auth_token
self.assertEqual(auth_token, self.gc.auth_token)
def test_get_root(self):
root = self.gc.get_root()
self.assertIsNotNone(root)
self.assertTrue(isinstance(root, (str, unicode)))
def test_get_meters(self):
meters = self.gc.get_meters() # tuple (ResultSet) of dicts
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters, (tuple)))
meters = meters.as_(Meter) # tuple of Meter objects
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters[0], (Meter)))
def test_get_projects(self):
projects = self.gc.get_projects() # tuple (ResultSet) of dicts
# for p in projects:
# print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects, (tuple)))
projects = projects.as_(Project) # tuple of Host objects
# for p in projects:
# print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects[0], (Project)))
def test_get_projects_detailled(self):
projects = self.gc.get_projects(params={'details': True}) # tuple (ResultSet) of dicts
for p in projects:
print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects, (tuple)))
projects = projects.as_(Project) # tuple of Host objects
for p in projects:
print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects[0], (Project)))
def test_get_hosts(self):
hosts = self.gc.get_hosts() # tuple (ResultSet) of dicts
# for h in hosts:
# print h
self.assertIsNotNone(hosts)
self.assertTrue(isinstance(hosts, (tuple)))
hosts = hosts.as_(Host) # tuple of Host objects
self.assertIsNotNone(hosts)
self.assertTrue(isinstance(hosts[0], (Host)))
def test_get_host_by_id(self):
host_id = 'uncinus'
# host_id = 600
host = self.gc.get_host(host=host_id) # remember: tuple (ResultSet)
self.assertIsNotNone(host) # of dicts
self.assertTrue(isinstance(host, (tuple)))
host = host.as_(Host) # also: tuple of Host objects
self.assertTrue(isinstance(host[0], (Host)))
def test_get_host_meters(self):
host_id = 'uncinus'
# host_id = 603
meters = self.gc.get_host_meters(host_id) # tuple (ResultSet) of dicts
# for h in hosts:
# print h
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters, (tuple)))
meters = meters.as_(Meter) # tuple of Host objects
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters[0], (Meter)))
def test_get_host_meter_records(self):
meter_records = self.gc.get_host_meter_records( \
host='uncinus',
meter="host.cpu.load.avg_15m")
# ^ tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_get_host_meter_records_with_display_limit(self):
limit = 2
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'limit': limit})
self.assertEqual(len(meter_records), limit)
def test_count_host_meter_records(self):
len_meter_records = len(self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m"))
# ^ int
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
self.assertEqual(count, len_meter_records)
def test_count_host_meter_records_with_time_limits(self):
start_time = '2013-02-13_17-00-00'
end_time = '2013-02-13_23-59-59'
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
num_all_records = count
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
self.assertLess(count, num_all_records)
def test_get_min_max_host_meter_record(self):
_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'min'}) \
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(_min, (tuple)))
self.assertEqual(len(_min), 1)
_min = _min.as_(MeterRecord)[0] # MeterRecord object
self.assertTrue(isinstance(_min, (MeterRecord)))
_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'max'}) \
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(_max, (tuple)))
self.assertEqual(len(_max), 1)
_max = _max.as_(MeterRecord)[0] # MeterRecord object
self.assertTrue(isinstance(_max, (MeterRecord)))
self.assertLessEqual(_min.value, _max.value)
def test_get_host_meter_records_with_time_limits(self):
start_time = '2013-03-06_12-00-00'
end_time = '2013-03-06_23-59-59'
regex = re.compile('^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})$')
matches = regex.match(start_time)
start_time_repr = '%s-%s-%s %s:%s:%s' % \
(matches.group(1), matches.group(2), \
matches.group(3), matches.group(4), \
matches.group(5), matches.group(6))
matches = regex.match(end_time)
end_time_repr = '%s-%s-%s %s:%s:%s' % \
(matches.group(1), matches.group(2), \
matches.group(3), matches.group(4), \
matches.group(5), matches.group(6))
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'order': 'asc'})
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
start_asc = meter_records[0]
end_asc = meter_records[-1]
self.assertLessEqual(start_time_repr, start_asc.timestamp)
self.assertLessEqual(end_asc.timestamp, end_time_repr)
self.assertLessEqual(start_asc.timestamp, end_asc.timestamp)
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'order': 'desc'})
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
start_desc = meter_records[0]
end_desc = meter_records[-1]
self.assertEqual(start_asc.timestamp, end_desc.timestamp)
self.assertTrue(start_desc.timestamp, end_asc.timestamp)
self.assertTrue(start_desc.timestamp >= end_desc.timestamp)
def test_get_min_max_host_meter_record_within_time_limits(self):
start_time = '2013-02-13_20-15-00'
end_time = {}
end_time[0] = '2013-02-13_20-25-59'
end_time[1] = '2013-02-13_20-40-59'
abs_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'min'}) # tuple (ResultSet) of dicts
self.assertEqual(len(abs_min), 1)
abs_min = abs_min.as_(MeterRecord)[0] # MeterRecord object
# print abs_min.value, abs_min
for _, endtime in end_time.iteritems():
rel_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': endtime, \
'aggregation': 'min'}) # tuple (ResultSet) of dicts
if isinstance(rel_min, (tuple)):
self.assertEqual(len(rel_min), 1)
rel_min = rel_min.as_(MeterRecord)[0] # MeterRecord object
# print rel_min.value, rel_min
# print abs_min.value, '<=', rel_min.value
self.assertLessEqual(abs_min.value, rel_min.value)
else:
self.assertIsNone(rel_min)
abs_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'max'}) # tuple (ResultSet) of dicts
self.assertEqual(len(abs_max), 1)
abs_max = abs_max.as_(MeterRecord)[0] # MeterRecord object
for _, endtime in end_time.iteritems():
rel_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': endtime, \
'aggregation': 'max'}) \
# ^ tuple (ResultSet) of dicts
if isinstance(rel_max, (tuple)):
self.assertEqual(len(rel_max), 1)
rel_max = rel_max.as_(MeterRecord)[0] # MeterRecord object
self.assertGreaterEqual(abs_max.value, rel_max.value)
else:
self.assertIsNone(rel_max)
def test_get_instances_as_Text_CsvFormatted(self):
instances = self.gc.get_instances() # tuple (ResultSet) of dicts
self.assertIsNotNone(instances)
self.assertTrue(isinstance(instances, (tuple)))
instances = instances.as_(Text, formatter=CsvFormatter)
# for i in instances:
# print type(i), i
# self.assertTrue(isinstance(i, unicode)) # unicoded string
self.assertTrue(isinstance(instances[0], unicode))
def test_get_inst_meter_records(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
meter_records = self.gc.get_inst_meter_records( \
inst=uuid,
meter="inst.uptime")
# ^ tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
self.assertEqual(meter_records[0].resource_id, uuid)
def test_avg_inst_meter_records(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
avg = self.gc.get_inst_meter_records( \
inst=uuid, \
meter="inst.uptime", \
params={'aggregation': 'avg'}) # float
self.assertTrue(isinstance(avg, (float)))
def test_daily_avg_inst_meter_records(self):
start_time = '2013-02-07_12-00-00'
end_time = '2013-02-17_23-59-59'
avgs = self.gc.get_inst_meter_records( \
'd2b24038-9dee-45d3-876f-d736ddd02d84',
'inst.cpu.time.ratio',
params={'start_time': start_time, \
'end_time': end_time, \
'aggregation': 'daily_avg'})
self.assertIsNotNone(avgs)
self.assertTrue(isinstance(avgs, (tuple)))
for day, avg in zip(*[range(1,len(avgs)+1), avgs]):
# print day, avg
self.assertTrue(avg is None or isinstance(avg, (float)))
def test_get_records(self):
meter_records = self.gc.get_records() # tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord) # tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_get_record(self):
record_ids = [1000, 346607]
for record_id in record_ids:
try:
meter_records = self.gc.get_record(record_id)
except Exception as e:
self.assertTrue(isinstance(e, requests.exceptions.HTTPError))
continue
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
# ^ tuple (ResultSet) of dicts
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertEqual(len(meter_records), 1)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_count_records(self):
meter_records = self.gc.get_records() # tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
len_result_set = len(meter_records)
meter_records = meter_records.as_(MeterRecord) # tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
len_tuple = len(meter_records)
self.assertTrue(len_result_set, len_tuple)
count = self.gc.get_records(params={'aggregation': 'count'})
self.assertTrue(2500 < count or count == len_result_set)
def test_get_proj_meter_records(self):
project = 1
meter = 'host.cpu.load.avg_15m'
proj_meter_records = self.gc.get_proj_meter_records(project, meter)
self.assertIsNotNone(proj_meter_records)
self.assertTrue(isinstance(proj_meter_records, (tuple)))
def test_meter_value_type(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
meters = [('inst.network.io.incoming.packets', long),
('inst.memory.physical', float)]
for m in meters:
r = self.gc.get_inst_meter_records(uuid, m[0], params={'limit': '1'})
r = r.as_(MeterRecord)[0]
self.assertTrue(isinstance(r.value, (m[1])))
def test_with_giraffe_user(self):
# @[fbahr] - TODO: ...
# auth_token = AuthProxy.get_token()
pass
class ClientTestCases(unittest.TestCase):
python_version = int(''.join([str(i) for i in sys.version_info[0:3]]))
@classmethod
def setUpClass(cls):
cls.config = Config('../../bin/giraffe.cfg')
cls.gc = GiraffeClient(username=cls.config.get('client', 'user'),
password=cls.config.get('client', 'pass'),
tenant_name=cls.config.get(
'client', 'tenant_name'),
tenant_id=cls.config.get('client', 'tenant_id'),
auth_url=cls.config.get('auth', 'public_url'))
def setUp(self):
if ClientTestCases.python_version < 270:
self.config = Config('../../bin/giraffe.cfg')
self.gc = GiraffeClient(
username=self.config.get('client', 'user'),
password=self.config.get('client', 'pass'),
tenant_name=self.config.get('client', 'tenant_name'),
tenant_id=self.config.get('client', 'tenant_id'),
auth_url=self.config.get('auth', 'public_url'))
def test_get_set_auth_token(self):
auth_token = self.gc.auth_token
self.gc.auth_token = auth_token
self.assertEqual(auth_token, self.gc.auth_token)
def test_get_root(self):
root = self.gc.get_root()
self.assertIsNotNone(root)
self.assertTrue(isinstance(root, (str, unicode)))
def test_get_meters(self):
meters = self.gc.get_meters() # tuple (ResultSet) of dicts
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters, (tuple)))
meters = meters.as_(Meter) # tuple of Meter objects
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters[0], (Meter)))
def test_get_projects(self):
projects = self.gc.get_projects() # tuple (ResultSet) of dicts
# for p in projects:
# print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects, (tuple)))
projects = projects.as_(Project) # tuple of Host objects
# for p in projects:
# print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects[0], (Project)))
def test_get_projects_detailled(self):
projects = self.gc.get_projects(
params={'details': True}) # tuple (ResultSet) of dicts
for p in projects:
print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects, (tuple)))
projects = projects.as_(Project) # tuple of Host objects
for p in projects:
print p
self.assertIsNotNone(projects)
self.assertTrue(isinstance(projects[0], (Project)))
def test_get_hosts(self):
hosts = self.gc.get_hosts() # tuple (ResultSet) of dicts
# for h in hosts:
# print h
self.assertIsNotNone(hosts)
self.assertTrue(isinstance(hosts, (tuple)))
hosts = hosts.as_(Host) # tuple of Host objects
self.assertIsNotNone(hosts)
self.assertTrue(isinstance(hosts[0], (Host)))
def test_get_host_by_id(self):
host_id = 'uncinus'
# host_id = 600
host = self.gc.get_host(host=host_id) # remember: tuple (ResultSet)
self.assertIsNotNone(host) # of dicts
self.assertTrue(isinstance(host, (tuple)))
host = host.as_(Host) # also: tuple of Host objects
self.assertTrue(isinstance(host[0], (Host)))
def test_get_host_meters(self):
host_id = 'uncinus'
# host_id = 603
meters = self.gc.get_host_meters(host_id) # tuple (ResultSet) of dicts
# for h in hosts:
# print h
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters, (tuple)))
meters = meters.as_(Meter) # tuple of Host objects
self.assertIsNotNone(meters)
self.assertTrue(isinstance(meters[0], (Meter)))
def test_get_host_meter_records(self):
meter_records = self.gc.get_host_meter_records( \
host='uncinus',
meter="host.cpu.load.avg_15m")
# ^ tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_get_host_meter_records_with_display_limit(self):
limit = 2
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'limit': limit})
self.assertEqual(len(meter_records), limit)
def test_count_host_meter_records(self):
len_meter_records = len(self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m"))
# ^ int
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
self.assertEqual(count, len_meter_records)
def test_count_host_meter_records_with_time_limits(self):
start_time = '2013-02-13_17-00-00'
end_time = '2013-02-13_23-59-59'
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
num_all_records = count
count = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'aggregation': 'count'})
# ^ int
self.assertFalse(isinstance(count, (tuple)))
self.assertTrue(isinstance(count, (int)))
self.assertLess(count, num_all_records)
def test_get_min_max_host_meter_record(self):
_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'min'}) \
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(_min, (tuple)))
self.assertEqual(len(_min), 1)
_min = _min.as_(MeterRecord)[0] # MeterRecord object
self.assertTrue(isinstance(_min, (MeterRecord)))
_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'max'}) \
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(_max, (tuple)))
self.assertEqual(len(_max), 1)
_max = _max.as_(MeterRecord)[0] # MeterRecord object
self.assertTrue(isinstance(_max, (MeterRecord)))
self.assertLessEqual(_min.value, _max.value)
def test_get_host_meter_records_with_time_limits(self):
start_time = '2013-03-06_12-00-00'
end_time = '2013-03-06_23-59-59'
regex = re.compile('^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})$')
matches = regex.match(start_time)
start_time_repr = '%s-%s-%s %s:%s:%s' % \
(matches.group(1), matches.group(2), \
matches.group(3), matches.group(4), \
matches.group(5), matches.group(6))
matches = regex.match(end_time)
end_time_repr = '%s-%s-%s %s:%s:%s' % \
(matches.group(1), matches.group(2), \
matches.group(3), matches.group(4), \
matches.group(5), matches.group(6))
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'order': 'asc'})
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
start_asc = meter_records[0]
end_asc = meter_records[-1]
self.assertLessEqual(start_time_repr, start_asc.timestamp)
self.assertLessEqual(end_asc.timestamp, end_time_repr)
self.assertLessEqual(start_asc.timestamp, end_asc.timestamp)
meter_records = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': end_time, \
'order': 'desc'})
# ^ tuple (ResultSet) of dicts
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
start_desc = meter_records[0]
end_desc = meter_records[-1]
self.assertEqual(start_asc.timestamp, end_desc.timestamp)
self.assertTrue(start_desc.timestamp, end_asc.timestamp)
self.assertTrue(start_desc.timestamp >= end_desc.timestamp)
def test_get_min_max_host_meter_record_within_time_limits(self):
start_time = '2013-02-13_20-15-00'
end_time = {}
end_time[0] = '2013-02-13_20-25-59'
end_time[1] = '2013-02-13_20-40-59'
abs_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'min'}) # tuple (ResultSet) of dicts
self.assertEqual(len(abs_min), 1)
abs_min = abs_min.as_(MeterRecord)[0] # MeterRecord object
# print abs_min.value, abs_min
for _, endtime in end_time.iteritems():
rel_min = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': endtime, \
'aggregation': 'min'}) # tuple (ResultSet) of dicts
if isinstance(rel_min, (tuple)):
self.assertEqual(len(rel_min), 1)
rel_min = rel_min.as_(MeterRecord)[0] # MeterRecord object
# print rel_min.value, rel_min
# print abs_min.value, '<=', rel_min.value
self.assertLessEqual(abs_min.value, rel_min.value)
else:
self.assertIsNone(rel_min)
abs_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'aggregation': 'max'}) # tuple (ResultSet) of dicts
self.assertEqual(len(abs_max), 1)
abs_max = abs_max.as_(MeterRecord)[0] # MeterRecord object
for _, endtime in end_time.iteritems():
rel_max = self.gc.get_host_meter_records( \
host='uncinus', \
meter="host.cpu.load.avg_15m", \
params={'start_time': start_time, \
'end_time': endtime, \
'aggregation': 'max'}) \
# ^ tuple (ResultSet) of dicts
if isinstance(rel_max, (tuple)):
self.assertEqual(len(rel_max), 1)
rel_max = rel_max.as_(MeterRecord)[0] # MeterRecord object
self.assertGreaterEqual(abs_max.value, rel_max.value)
else:
self.assertIsNone(rel_max)
def test_get_instances_as_Text_CsvFormatted(self):
instances = self.gc.get_instances() # tuple (ResultSet) of dicts
self.assertIsNotNone(instances)
self.assertTrue(isinstance(instances, (tuple)))
instances = instances.as_(Text, formatter=CsvFormatter)
# for i in instances:
# print type(i), i
# self.assertTrue(isinstance(i, unicode)) # unicoded string
self.assertTrue(isinstance(instances[0], unicode))
def test_get_inst_meter_records(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
meter_records = self.gc.get_inst_meter_records( \
inst=uuid,
meter="inst.uptime")
# ^ tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
self.assertEqual(meter_records[0].resource_id, uuid)
def test_avg_inst_meter_records(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
avg = self.gc.get_inst_meter_records( \
inst=uuid, \
meter="inst.uptime", \
params={'aggregation': 'avg'}) # float
self.assertTrue(isinstance(avg, (float)))
def test_daily_avg_inst_meter_records(self):
start_time = '2013-02-07_12-00-00'
end_time = '2013-02-17_23-59-59'
avgs = self.gc.get_inst_meter_records( \
'd2b24038-9dee-45d3-876f-d736ddd02d84',
'inst.cpu.time.ratio',
params={'start_time': start_time, \
'end_time': end_time, \
'aggregation': 'daily_avg'})
self.assertIsNotNone(avgs)
self.assertTrue(isinstance(avgs, (tuple)))
for day, avg in zip(*[range(1, len(avgs) + 1), avgs]):
# print day, avg
self.assertTrue(avg is None or isinstance(avg, (float)))
def test_get_records(self):
meter_records = self.gc.get_records() # tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
meter_records = meter_records.as_(
MeterRecord) # tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_get_record(self):
record_ids = [1000, 346607]
for record_id in record_ids:
try:
meter_records = self.gc.get_record(record_id)
except Exception as e:
self.assertTrue(isinstance(e, requests.exceptions.HTTPError))
continue
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
# ^ tuple (ResultSet) of dicts
meter_records = meter_records.as_(MeterRecord)
# ^ tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertEqual(len(meter_records), 1)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
def test_count_records(self):
meter_records = self.gc.get_records() # tuple (ResultSet) of dicts
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records, (tuple)))
len_result_set = len(meter_records)
meter_records = meter_records.as_(
MeterRecord) # tuple of MeterRecord objects
self.assertIsNotNone(meter_records)
self.assertTrue(isinstance(meter_records[0], (MeterRecord)))
len_tuple = len(meter_records)
self.assertTrue(len_result_set, len_tuple)
count = self.gc.get_records(params={'aggregation': 'count'})
self.assertTrue(2500 < count or count == len_result_set)
def test_get_proj_meter_records(self):
project = 1
meter = 'host.cpu.load.avg_15m'
proj_meter_records = self.gc.get_proj_meter_records(project, meter)
self.assertIsNotNone(proj_meter_records)
self.assertTrue(isinstance(proj_meter_records, (tuple)))
def test_meter_value_type(self):
uuid = 'd2b24038-9dee-45d3-876f-d736ddd02d84'
meters = [('inst.network.io.incoming.packets', long),
('inst.memory.physical', float)]
for m in meters:
r = self.gc.get_inst_meter_records(uuid,
m[0],
params={'limit': '1'})
r = r.as_(MeterRecord)[0]
self.assertTrue(isinstance(r.value, (m[1])))
def test_with_giraffe_user(self):
# @[fbahr] - TODO: ...
# auth_token = AuthProxy.get_token()
pass
class GiraffeClient(object):
def __init__(self, auth_token=None, \
# username=None, \
# password=None, \
# tenant_name=None, \
# tenant_id=None, \
# protocol=None, \
# endpoint=None, \
# auth_url=None, \
**kwargs):
"""
Creates a new GiraffeClient instance:
- if auth_token is not None, auth_token is used to authenticate
client requests
- alternatively, credentials (username, password, tenant_id, etc.)
to be used to authenticate client requests can be passed to
__init__() via named parameters
"""
# [DELETED]
# - right now, also a fallback to giraffe.cfg is implemented (but not
# to env. variables)
self.config = Config('giraffe.cfg')
if not auth_token:
_username = kwargs.get('username')
# , self.config.get('client', 'user'))
_password = kwargs.get('password')
# , self.config.get('client', 'pass'))
_tenant_name = kwargs.get('tenant_name')
# , self.config.get('client', 'tenant_name'))
_tenant_id = kwargs.get('tenant_id')
# , self.config.get('client', 'tenant_id'))
_auth_url = kwargs.get('auth_url', \
self.config.get('auth', 'public_url'))
auth_token = AuthProxy.get_token(username=_username,
password=_password,
tenant_name=_tenant_name,
tenant_id=_tenant_id,
auth_url=_auth_url)
self.auth_header = dict([('X-Auth-Token', auth_token)])
#@[fbahr] - TODO: Exception handling
self.protocol = kwargs.get('protocol', 'http')
self.endpoint = kwargs.get('endpoint')
if not self.endpoint:
host = kwargs.get('host', self.config.get('rest_api', 'host'))
port = kwargs.get('port', self.config.get('rest_api', 'port'))
self.endpoint = ':'.join((host, port))
@property
def auth_token(self):
return self.auth_header['X-Auth-Token']
@auth_token.setter
def auth_token(self, auth_token):
self.auth_header['X-Auth-Token'] = auth_token
def _get(self, path, params=None):
# ---------------------------------------------------------------------
class ResultSet(tuple):
def __new__(cls, first=(), *next, **kwargs):
cls._giraffe_client = kwargs.get('giraffe_client')
if isinstance(first, (tuple, list)) and not next:
return tuple.__new__(cls, tuple(first))
else:
return tuple.__new__(cls, (first, ) + next)
def as_(self, cls, **kwargs):
"""
Returns a tuple of cls-serialized representations of
ResultSet elements; cls is supposed to be a db.Base or
formatter.FormattableObject subclass.
"""
if not issubclass(cls, (Base, FormattableObject)):
raise TypeError('Expects FormattableObject.')
if not self: # ...not len(self), or: len(self) == 0
return None
formatter = kwargs.get('formatter', DEFAULT_FORMATTERS.get(cls))
#@[fbahr] - TODO: Come up with a smarter solution...
catalog = {}
if self._giraffe_client and issubclass(cls, (MeterRecord)):
meters = self._giraffe_client.get_meters().as_(Meter)
# constructs a catalog of meter_record.value types, based
# on (meter.id, meter.data_type) information
catalog = dict((int(m.id), str(m.data_type)) for m in meters)
return tuple(formatter.serialize(elem, catalog) for elem in self)
# end of class _ResultSet ---------------------------------------------
url = URLBuilder.build(self.protocol, self.endpoint, path, params)
logger.debug('Query: %s' % url)
response = requests.get(url, headers=self.auth_header)
logger.debug('HTTP response status code: %s' % response.status_code)
response.raise_for_status()
return ResultSet(response.json, giraffe_client=self) \
if isinstance(response.json, (tuple, list, dict)) \
else response.json
# ...was: else response.text
def get_root(self, params=None):
"""
Welcome ...
"""
path = '/'
return self._get(path, None)
def get_hosts(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{'id': '<server_id (int)>',
'name': '<server_name>'}
dicts
"""
path = '/hosts'
return self._get(path, params) # .as_(Host)
def get_host(self, host, params=None):
"""
WARNING: get_host - like every other API method - returns a _tuple_
(ResultSet), despite containing only a single element; the same
applies to ResultSet::as_().
"""
path = '/'.join(['/hosts', str(host)])
return self._get(path, params) # .as_(Host)
def get_host_meters(self, host, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{'id': '<meter_id (int)>',
'name': '<meter_name (string)>',
'description': '<description>',
'unit_name': '[seconds|..]',
'data_type': '[float|..]'}
dicts
"""
path = '/'.join(['/hosts', str(host), 'meters'])
return self._get(path, params) # .as_(Meter)
def get_host_meter_records(self, host, meter, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{'id': '<record_id (int)',
'host_id': '<host_id (string)>',
'resource_id': '<intance_id (string)>',
'project_id': '...',
'user_id': '...',
'meter_id': '... (int)',
'timestamp': '2013-01-27 08:46:35',
'value': '...',
'duration': '... (int)',
'signature': '...'}
dicts
"""
path = '/'.join(['/hosts', str(host), 'meters', str(meter), 'records'])
return self._get(path, params) # .as_(MeterRecord)
def get_instances(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
...
dicts
"""
path = '/instances'
return self._get(path, params)
def get_instance(self, inst, params=None):
"""
WARNING: get_host - like every other API method - returns a _tuple_
(ResultSet), despite containing only a single element; the same
applies to ResultSet::as_().
"""
path = '/'.join(['/instances', str(inst)])
raise NotImplementedError('Route /instances/[id|name] '
'not (yet) implemented.')
def get_inst_meters(self, inst, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meters }
dicts
"""
path = '/'.join(['/instances', str(inst), 'meters'])
raise NotImplementedError('Route /instances/[id|name]/meters '
'not (yet) implemented.')
def get_inst_meter_records(self, inst, meter, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meter_records }
dicts
"""
path = '/'.join(['/instances', str(inst), 'meters', str(meter), 'records'])
return self._get(path, params) # .as_(MeterRecord)
def get_projects(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
...
dicts
"""
path = '/projects'
return self._get(path, params)
def get_project(self, proj, params=None):
"""
WARNING: get_host - like every other API method - returns a _tuple_
(ResultSet), despite containing only a single element; the same
applies to ResultSet::as_().
"""
path = '/'.join(['/projects', str(proj)])
return self._get(path, params)
def get_proj_instances(self, proj, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
...
dicts
"""
path = '/'.join(['/projects', str(proj), 'instances'])
return self._get(path, params)
def get_proj_meters(self, proj, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meters }
dicts
"""
path = '/'.join(['/projects', str(proj), 'meters'])
return self._get(path, params) # .as_(Meter)
def get_proj_meter_records(self, proj, meter, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meter_records }
dicts
"""
path = '/'.join(['/projects', str(proj), 'meters', str(meter), 'records'])
return self._get(path, params) # .as_(MeterRecord)
def get_users(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
...
dicts
"""
path = '/users'
return self._get(path, params)
def get_user_meter_records(self, user, meter, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meter_records }
dicts
"""
path = '/'.join(['/users', str(user), 'meters', str(meter), 'records'])
return self._get(path, params) # .as_(MeterRecord)
def get_meters(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{'id': '<meter_id (int)>',
'name': '<meter_name (string)>',
'description': '<description>',
'unit_name': '[seconds|..]',
'data_type': '[float|..]'}
dicts
"""
path = '/meters'
return self._get(path, params) # .as_(Meter)
def get_meter(self, meter, params=None):
"""
WARNING: get_host - like every other API method - returns a _tuple_
(ResultSet), despite containing only a single element; the same
applies to ResultSet::as_().
"""
path = '/'.join(['/meters', str(meter)])
return self._get(path, params) # .as_(Meter)
def get_records(self, params=None):
"""
Returns a tuple (actually, a ResultSet instance) of
{ see GiraffeClient::get_host_meter_records }
dicts
"""
path = '/records'
return self._get(path, params) # .as_(MeterRecord)
def get_record(self, record, params=None):
"""
WARNING: get_host - like every other API method - returns a _tuple_
(ResultSet), despite containing only a single element; the same
applies to ResultSet::as_().
"""
path = '/'.join(['/records', str(record)])
return self._get(path, params) # .as_(MeterRecord)
class Collector(object):
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector,
self.queue,
self.exchange,
self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect('%s://%s:%s@%s/%s'
% (self.config.get('db', 'vendor'),
self.config.get('db', 'user'),
self.config.get('db', 'pass'),
self.config.get('db', 'host'),
self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def launch(self):
try:
self.connector.connect()
self.start_collecting()
except KeyboardInterrupt:
_logger.info("Ctrl-c received!")
except:
_logger.exception("Error: Unable to start collector service")
finally:
self.stop_collecting()
_logger.info("Shutdown collector service")
def start_collecting(self):
_logger.debug("Start collecting from broker")
self.consumer.consume()
def stop_collecting(self):
_logger.debug("Stop collecting from broker")
self.consumer.stop_consuming()
def _str_to_datetime(self, timestamp_str):
return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
def _datetime_to_str(self, datetime_obj):
return datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
def _collector_callback(self, params):
envelope = EnvelopeAdapter()
# check whether incorrectly formatted message
try:
envelope.deserialize_from_str(params)
except:
return
message = MessageAdapter(envelope.message)
# validate signature
if not validateSignature(str(message), self.shared_secret,
envelope.signature):
return
self.db.session_open()
# load all meters now to avoid queries later
meters = self.db.load(Meter)
meter_dict = {}
for meter in meters:
meter_dict[meter.name] = meter
# insert host if it does not exist yet
hosts = self.db.load(Host, {'name': message.host_name}, limit=1)
if not hosts:
host = Host(name=message.host_name)
self.db.save(host)
self.db.commit()
else:
host = hosts[0]
# insert all host records
for r in message.host_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=None,
resource_id=None,
project_id=None,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
except Exception as e:
_logger.exception(e)
# insert all instance records
for r in message.inst_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
# @[fbahr] - TODO: `self.known_instances` grows over time
# towards inf. - clean-up?
if not r.inst_id in self.known_instances:
self.known_instances[r.inst_id] = self._metadata(uuid=r.inst_id)
r.project_id, r.user_id = self.known_instances[r.inst_id]
# insert project if it does not exist yet
projects = self.db.load(Project,
{'uuid': r.project_id},
limit=1)
if not projects:
project = Project(uuid=r.project_id,
created_at=self._str_to_datetime(r.timestamp))
self.db.save(project)
self.db.commit()
else:
project = projects[0]
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=r.user_id,
resource_id=r.inst_id,
project_id=r.project_id,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host and project activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
if not project.updated_at or record_timestamp > project.updated_at:
project.updated_at = record_timestamp
except Exception as e:
_logger.exception(e)
try:
self.db.commit()
except Exception as e:
self.db.rollback()
_logger.exception(e)
self.db.session_close()
def _metadata(self, uuid):
"""
Connect to nova database (by means of nova-client);
fetches (project_id, user_id) information for a given instance uuid.
"""
nova_client = NovaClient(username=self._credentials['username'],
api_key=self._credentials['password'],
project_id=self._credentials['tenant_name'],
auth_url=self._credentials['auth_url'],
service_type='compute',
insecure=True)
# self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials)
# nova_client.client.auth_token = self._credentials['auth_token']
# nova_client.client.authenticate()
# unfortunately, nova_client.servers.find(id=...) is restricted
# to instances in an user's tentant (project) [can't tweek that,
# hard-coded in find()] - but:
search_opts = {'all_tenants': 1}
server = next((s.user_id, s.tenant_id)
for s in nova_client.servers.list(True, search_opts)
if s.id == uuid)
return map(str, server)
def __init__(self, cfg=None):
"""
Initializes a new Agent.
"""
# fetch config parameters
config = Config(cfg if cfg else 'giraffe.cfg')
# configure agent
self.timer = False
self.tasks = []
self.publisher = publisher.AgentPublisher(self)
# add publisher to tasks
self.tasks.append(self.publisher)
# HOST METERS ---------------------------------------------------------
meters = config.items('agent')
for (meter, value) in meters:
try:
if self.HOST_METER.get(meter):
self.tasks.append(self.HOST_METER[meter](getattr(
self, '_callback_' + meter), int(value)))
except ValueError:
logging.exception('Host meter %s ommitted due to ValueError: '
'\'%s\' is not of type \'int\'' %
(meter, value))
# uptime = config.get('agent', 'host_uptime')
# cpu_load = config.get('agent', 'host_cpu_load')
# mem_usage = config.get('agent', 'host_memory_usage')
# disk_io = config.get('agent', 'host_disk_io')
# network_io = config.get('agent', 'host_network_io')
# host meter `uptime`
# if uptime:
# self.tasks.append(Host_UPTIME(
# self._callback_host_uptime,
# int(uptime)))
# host meter `CPU load`
# if cpu_load:
# self.tasks.append(Host_CPU_Load(
# self._callback_host_cpu_load,
# int(loadavg)))
# host meter `memory`
# if mem_usage:
# self.tasks.append(Host_MEMORY_Usage(
# self._callback_host_memory_usage,
# int(mem_usage)))
# host meter `network I/O`
# if network_io:
# self.tasks.append(Host_NETWORK_IO(
# self._callback_host_network_io,
# int(network_io)))
# INSTANCE METERS -----------------------------------------------------
for (meter, value) in meters:
try:
if self.INST_METER.get(meter):
self.tasks.append(self.INST_METER[meter](getattr(
self, '_callback_' + meter), int(value)))
except ValueError:
logging.exception(
'Instance meter %s ommitted due to ValueError: '
'\'%s\' is not of type \'int\'' % (meter, value))
import subprocess
from xml.etree import ElementTree as ETree
import psutil
# try:
# from nova.virt import driver
# except ImportError:
import libvirt
_LIBVIRT_SOCKET_URL = 'qemu:///system'
from giraffe.common.task import PeriodicMeterTask
from novaclient.v1_1.client import Client as NovaClient
from giraffe.common.auth import AuthProxy
from giraffe.common.config import Config
_config = Config('giraffe.cfg')
import logging
logger = logging.getLogger("agent.instance_meter")
# logger.setLevel(logging.DEBUG)
# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# fh = logging.FileHandler("agent.log")
# fh.setFormatter(formatter)
# logger.addHandler(fh)
class PeriodicInstMeterTask(PeriodicMeterTask):
def __init__(self, callback, period):
super(PeriodicInstMeterTask, self).__init__(callback, period)
self.conn = libvirt.openReadOnly(_LIBVIRT_SOCKET_URL)
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def test_config(self):
Config("giraffe.cfg")
class Collector(object):
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector, self.queue,
self.exchange, self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect(
'%s://%s:%s@%s/%s' %
(self.config.get('db', 'vendor'), self.config.get('db', 'user'),
self.config.get('db', 'pass'), self.config.get(
'db', 'host'), self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(
username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def launch(self):
try:
self.connector.connect()
self.start_collecting()
except KeyboardInterrupt:
_logger.info("Ctrl-c received!")
except:
_logger.exception("Error: Unable to start collector service")
finally:
self.stop_collecting()
_logger.info("Shutdown collector service")
def start_collecting(self):
_logger.debug("Start collecting from broker")
self.consumer.consume()
def stop_collecting(self):
_logger.debug("Stop collecting from broker")
self.consumer.stop_consuming()
def _str_to_datetime(self, timestamp_str):
return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
def _datetime_to_str(self, datetime_obj):
return datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
def _collector_callback(self, params):
envelope = EnvelopeAdapter()
# check whether incorrectly formatted message
try:
envelope.deserialize_from_str(params)
except:
return
message = MessageAdapter(envelope.message)
# validate signature
if not validateSignature(str(message), self.shared_secret,
envelope.signature):
return
self.db.session_open()
# load all meters now to avoid queries later
meters = self.db.load(Meter)
meter_dict = {}
for meter in meters:
meter_dict[meter.name] = meter
# insert host if it does not exist yet
hosts = self.db.load(Host, {'name': message.host_name}, limit=1)
if not hosts:
host = Host(name=message.host_name)
self.db.save(host)
self.db.commit()
else:
host = hosts[0]
# insert all host records
for r in message.host_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=None,
resource_id=None,
project_id=None,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
except Exception as e:
_logger.exception(e)
# insert all instance records
for r in message.inst_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
# @[fbahr] - TODO: `self.known_instances` grows over time
# towards inf. - clean-up?
if not r.inst_id in self.known_instances:
self.known_instances[r.inst_id] = self._metadata(
uuid=r.inst_id)
r.project_id, r.user_id = self.known_instances[r.inst_id]
# insert project if it does not exist yet
projects = self.db.load(Project, {'uuid': r.project_id},
limit=1)
if not projects:
project = Project(uuid=r.project_id,
created_at=self._str_to_datetime(
r.timestamp))
self.db.save(project)
self.db.commit()
else:
project = projects[0]
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=r.user_id,
resource_id=r.inst_id,
project_id=r.project_id,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host and project activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
if not project.updated_at or record_timestamp > project.updated_at:
project.updated_at = record_timestamp
except Exception as e:
_logger.exception(e)
try:
self.db.commit()
except Exception as e:
self.db.rollback()
_logger.exception(e)
self.db.session_close()
def _metadata(self, uuid):
"""
Connect to nova database (by means of nova-client);
fetches (project_id, user_id) information for a given instance uuid.
"""
nova_client = NovaClient(username=self._credentials['username'],
api_key=self._credentials['password'],
project_id=self._credentials['tenant_name'],
auth_url=self._credentials['auth_url'],
service_type='compute',
insecure=True)
# self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials)
# nova_client.client.auth_token = self._credentials['auth_token']
# nova_client.client.authenticate()
# unfortunately, nova_client.servers.find(id=...) is restricted
# to instances in an user's tentant (project) [can't tweek that,
# hard-coded in find()] - but:
search_opts = {'all_tenants': 1}
server = next((s.user_id, s.tenant_id)
for s in nova_client.servers.list(True, search_opts)
if s.id == uuid)
return map(str, server)
class Meta:
"""
Model that serves as a container for arguments and options
common to all Giraffe CLI controllers
"""
label = 'abstract'
# aliases = [...]
# stacked_on = ...
# stacked_type = ...
# hide = [True|False]
# usage = ...
# description = ...
# epilog = ...
_config = Config('giraffe.cfg')
# command line arguments
# ! Warning: os.getenv over config.get, i.e., environment variables
# override params defined in giraffe.cfg
_credentials = [
(['--username', '-u'], \
dict(action='store', help='$OS_USERNAME', \
default=os.getenv('OS_USERNAME') or \
_config.get('client', 'user'))),
(['--password', '-p'], \
dict(action='store', help='$OS_PASSWORD', \
default=os.getenv('OS_PASSWORD') or \
_config.get('client', 'pass'))),
(['--tenant_id'], \
dict(action='store', help='$OS_TENANT_ID', \
default=os.getenv('OS_TENANT_ID') or \
_config.get('client', 'tenant_id'))),
(['--tenant_name'], \
dict(action='store', help='$OS_TENANT_NAME', \
default=os.getenv('OS_TENANT_NAME') or \
_config.get('client', 'tenant_name')))
]
_services = [
(['--auth_url', '-a'], \
dict(action='store', help='$OS_AUTH_URL', \
default=os.getenv('OS_AUTH_URL') or \
_config.get('auth', 'public_url'))),
(['--endpoint', '-e'], \
dict(action='store', help='Giraffe REST API endpoint (domain:port)', \
default=':'.join([_config.get('rest_api', 'host'), \
_config.get('rest_api', 'port')])))
]
_domains = [
(['--host'], \
dict(action='store', help='host name or ID',
default=None)),
(['--project', '--proj'], \
dict(action='store', help='project name or ID',
default=None)),
(['--user'], \
dict(action='store', help='user name or ID', \
default=None)),
(['--instance', '--inst'], \
dict(action='store', help='instance name or ID', \
default=None))
]
_modifiers = [
(['--start'], \
dict(action='store', dest='start_time', help='', \
default=None)),
(['--end'], \
dict(action='store', dest='end_time', help='', \
default=None)),
(['--min'], \
dict(action='store_true', help='', \
default=None)),
(['--max'], \
dict(action='store_true', help='', \
default=None)),
(['--avg'], \
dict(action='store_true', help='', \
default=None)),
(['--hourly'], \
dict(action='store_true', dest='hourly_avg', help='hourly '
'averages of meter values within an period '
'[to be defined by --start and --end]', \
default=None)),
(['--daily'], \
dict(action='store_true', dest='daily_avg', help='daily '
'averages of meter values within an period '
'[to be defined by --start and --end]', \
default=None)),
(['--sum'], \
dict(action='store_true', help='', \
default=None)),
(['--count'], \
dict(action='store_true', help='', \
default=None)),
(['--limit'], \
dict(action='store', help='max. number of objects to be '
'retrieved', \
default=0)),
(['--order'], \
dict(action='store', help='ORDER (ASC [default] or DESC)', \
default=None)),
]
_aggregations = [
'min', 'max', 'avg', 'hourly_avg', 'daily_avg', 'sum', 'count'
]
_filters = ['start_time', 'end_time', 'limit', 'order']
_params = sum([_aggregations, _filters], [])
_formats = [
# (['--jsn'], \
# dict(action='store_true', help='display output as plain JSON', \
# default=None)),
(['--csv'], \
dict(action='store_true', help='output formatted as CSV', \
default=None)),
(['--tab'], \
dict(action='store_true', help='output formatted as table', \
default=None))
]
arguments = sum([_modifiers, _formats, _credentials, _services], [])
def __init__(self, cfg=None):
"""
Initializes a new Agent.
"""
# fetch config parameters
config = Config(cfg if cfg else 'giraffe.cfg')
# configure agent
self.timer = False
self.tasks = []
self.publisher = publisher.AgentPublisher(self)
# add publisher to tasks
self.tasks.append(self.publisher)
# HOST METERS ---------------------------------------------------------
meters = config.items('agent')
for (meter, value) in meters:
try:
if self.HOST_METER.get(meter):
self.tasks.append(self.HOST_METER[meter](
getattr(self, '_callback_' + meter),
int(value)))
except ValueError:
logging.exception('Host meter %s ommitted due to ValueError: '
'\'%s\' is not of type \'int\'' % (meter, value))
# uptime = config.get('agent', 'host_uptime')
# cpu_load = config.get('agent', 'host_cpu_load')
# mem_usage = config.get('agent', 'host_memory_usage')
# disk_io = config.get('agent', 'host_disk_io')
# network_io = config.get('agent', 'host_network_io')
# host meter `uptime`
# if uptime:
# self.tasks.append(Host_UPTIME(
# self._callback_host_uptime,
# int(uptime)))
# host meter `CPU load`
# if cpu_load:
# self.tasks.append(Host_CPU_Load(
# self._callback_host_cpu_load,
# int(loadavg)))
# host meter `memory`
# if mem_usage:
# self.tasks.append(Host_MEMORY_Usage(
# self._callback_host_memory_usage,
# int(mem_usage)))
# host meter `network I/O`
# if network_io:
# self.tasks.append(Host_NETWORK_IO(
# self._callback_host_network_io,
# int(network_io)))
# INSTANCE METERS -----------------------------------------------------
for (meter, value) in meters:
try:
if self.INST_METER.get(meter):
self.tasks.append(self.INST_METER[meter](
getattr(self, '_callback_' + meter),
int(value)))
except ValueError:
logging.exception('Instance meter %s ommitted due to ValueError: '
'\'%s\' is not of type \'int\'' % (meter, value))
