def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector, self.queue,
self.exchange, self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect(
'%s://%s:%s@%s/%s' %
(self.config.get('db', 'vendor'), self.config.get('db', 'user'),
self.config.get('db', 'pass'), self.config.get(
'db', 'host'), self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(
username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector,
self.queue,
self.exchange,
self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect('%s://%s:%s@%s/%s'
% (self.config.get('db', 'vendor'),
self.config.get('db', 'user'),
self.config.get('db', 'pass'),
self.config.get('db', 'host'),
self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
class AgentPublisher(threading.Thread):
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def _timestamp_now(self, datetime_now=None):
"""
Returns current system time as formatted string "%Y-%m-%d %H:%M:%S"
"""
if not datetime_now:
datetime_now = datetime.now()
return datetime_now.strftime("%Y-%m-%d %H:%M:%S")
def _build_message(self):
"""
Returns a new MessageAdapter object with hostname and signature
:rtype : MessageAdapter
"""
envelope = EnvelopeAdapter()
envelope.message.host_name = self.host_name
# message.signature = _SIGNATURE
return envelope
def add_meter_record(self, meter_name, meter_value, meter_duration):
"""
Add new meter record to meter message
Params: meter type [as string], value(s), and duration [in seconds]
"""
logger.debug("Add meter record: %s=%s" % (meter_name, meter_value))
if not self.lock.locked():
self.lock.acquire()
try:
if meter_name.startswith('inst'):
for record in meter_value:
self.envelope.add_inst_record(
timestamp=self._timestamp_now(record[1]),
meter_name=meter_name,
value=record[2],
duration=meter_duration,
project_id='',
inst_id=record[0],
user_id='')
else: # .startswith('host'):
self.envelope.add_host_record(
self._timestamp_now(),
meter_name,
meter_value,
meter_duration)
finally:
self.lock.release()
def flush(self):
"""
Sends current state of agents message to the broker.
"""
logger.debug("Flush meter message")
if not self.lock.locked():
self.lock.acquire()
try:
if self.envelope.len() > 0:
# flush message
self._send(self.envelope)
# build new message
self.envelope = self._build_message()
# except Exception as e:
# logger.exception(e)
finally:
self.lock.release()
def _send(self, envelope):
"""
Create message signature and send envelop to broker
"""
messageAdapter = MessageAdapter(envelope.message)
sig = createSignature(str(messageAdapter), self.shared_secret)
envelope.signature = sig
self.producer.send(exchange=self.exchange,
routing_key=self.routing_key,
message=envelope.serialize_to_str())
def run(self):
self.connector.connect()
while self.stopRequest is False:
time.sleep(self.flush_duration)
self.flush()
def stop(self):
self.stopRequest = True
self._Thread__stop()
class Collector(object):
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector, self.queue,
self.exchange, self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect(
'%s://%s:%s@%s/%s' %
(self.config.get('db', 'vendor'), self.config.get('db', 'user'),
self.config.get('db', 'pass'), self.config.get(
'db', 'host'), self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(
username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def launch(self):
try:
self.connector.connect()
self.start_collecting()
except KeyboardInterrupt:
_logger.info("Ctrl-c received!")
except:
_logger.exception("Error: Unable to start collector service")
finally:
self.stop_collecting()
_logger.info("Shutdown collector service")
def start_collecting(self):
_logger.debug("Start collecting from broker")
self.consumer.consume()
def stop_collecting(self):
_logger.debug("Stop collecting from broker")
self.consumer.stop_consuming()
def _str_to_datetime(self, timestamp_str):
return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
def _datetime_to_str(self, datetime_obj):
return datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
def _collector_callback(self, params):
envelope = EnvelopeAdapter()
# check whether incorrectly formatted message
try:
envelope.deserialize_from_str(params)
except:
return
message = MessageAdapter(envelope.message)
# validate signature
if not validateSignature(str(message), self.shared_secret,
envelope.signature):
return
self.db.session_open()
# load all meters now to avoid queries later
meters = self.db.load(Meter)
meter_dict = {}
for meter in meters:
meter_dict[meter.name] = meter
# insert host if it does not exist yet
hosts = self.db.load(Host, {'name': message.host_name}, limit=1)
if not hosts:
host = Host(name=message.host_name)
self.db.save(host)
self.db.commit()
else:
host = hosts[0]
# insert all host records
for r in message.host_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=None,
resource_id=None,
project_id=None,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
except Exception as e:
_logger.exception(e)
# insert all instance records
for r in message.inst_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
# @[fbahr] - TODO: `self.known_instances` grows over time
# towards inf. - clean-up?
if not r.inst_id in self.known_instances:
self.known_instances[r.inst_id] = self._metadata(
uuid=r.inst_id)
r.project_id, r.user_id = self.known_instances[r.inst_id]
# insert project if it does not exist yet
projects = self.db.load(Project, {'uuid': r.project_id},
limit=1)
if not projects:
project = Project(uuid=r.project_id,
created_at=self._str_to_datetime(
r.timestamp))
self.db.save(project)
self.db.commit()
else:
project = projects[0]
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=r.user_id,
resource_id=r.inst_id,
project_id=r.project_id,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host and project activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
if not project.updated_at or record_timestamp > project.updated_at:
project.updated_at = record_timestamp
except Exception as e:
_logger.exception(e)
try:
self.db.commit()
except Exception as e:
self.db.rollback()
_logger.exception(e)
self.db.session_close()
def _metadata(self, uuid):
"""
Connect to nova database (by means of nova-client);
fetches (project_id, user_id) information for a given instance uuid.
"""
nova_client = NovaClient(username=self._credentials['username'],
api_key=self._credentials['password'],
project_id=self._credentials['tenant_name'],
auth_url=self._credentials['auth_url'],
service_type='compute',
insecure=True)
# self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials)
# nova_client.client.auth_token = self._credentials['auth_token']
# nova_client.client.authenticate()
# unfortunately, nova_client.servers.find(id=...) is restricted
# to instances in an user's tentant (project) [can't tweek that,
# hard-coded in find()] - but:
search_opts = {'all_tenants': 1}
server = next((s.user_id, s.tenant_id)
for s in nova_client.servers.list(True, search_opts)
if s.id == uuid)
return map(str, server)
class Collector(object):
def __init__(self):
self.config = Config('giraffe.cfg')
# configure RabbitMQ connector (and consumer)
self.connector = Connector(username=self.config.get('rabbit', 'user'),
password=self.config.get('rabbit', 'pass'),
host=self.config.get('rabbit', 'host'),
port=self.config.getint('rabbit', 'port'))
self.queue = self.config.get('rabbit', 'queue')
self.exchange = self.config.get('rabbit', 'exchange')
self.routing_key = self.config.get('rabbit', 'routing_key')
self.consumer = BasicConsumer(self.connector,
self.queue,
self.exchange,
self._collector_callback)
self.shared_secret = self.config.get('collector', 'shared_secret')
# connect to giraffe database
self.db = db.connect('%s://%s:%s@%s/%s'
% (self.config.get('db', 'vendor'),
self.config.get('db', 'user'),
self.config.get('db', 'pass'),
self.config.get('db', 'host'),
self.config.get('db', 'schema')))
# prepare connection to nova-client
self._credentials = dict(username=self.config.get('agent', 'user'),
password=self.config.get('agent', 'pass'),
# tenant_id=self.config.get('agent', 'tenant_id'),
tenant_name=self.config.get('agent', 'tenant_name'),
auth_url=self.config.get('auth', 'admin_url'),
insecure=True)
# known servers/instances
self.known_instances = {}
def launch(self):
try:
self.connector.connect()
self.start_collecting()
except KeyboardInterrupt:
_logger.info("Ctrl-c received!")
except:
_logger.exception("Error: Unable to start collector service")
finally:
self.stop_collecting()
_logger.info("Shutdown collector service")
def start_collecting(self):
_logger.debug("Start collecting from broker")
self.consumer.consume()
def stop_collecting(self):
_logger.debug("Stop collecting from broker")
self.consumer.stop_consuming()
def _str_to_datetime(self, timestamp_str):
return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
def _datetime_to_str(self, datetime_obj):
return datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
def _collector_callback(self, params):
envelope = EnvelopeAdapter()
# check whether incorrectly formatted message
try:
envelope.deserialize_from_str(params)
except:
return
message = MessageAdapter(envelope.message)
# validate signature
if not validateSignature(str(message), self.shared_secret,
envelope.signature):
return
self.db.session_open()
# load all meters now to avoid queries later
meters = self.db.load(Meter)
meter_dict = {}
for meter in meters:
meter_dict[meter.name] = meter
# insert host if it does not exist yet
hosts = self.db.load(Host, {'name': message.host_name}, limit=1)
if not hosts:
host = Host(name=message.host_name)
self.db.save(host)
self.db.commit()
else:
host = hosts[0]
# insert all host records
for r in message.host_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=None,
resource_id=None,
project_id=None,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
except Exception as e:
_logger.exception(e)
# insert all instance records
for r in message.inst_records:
if r.meter_name not in meter_dict:
_logger.warning('Unknown meter_name "%s"' % r.meter_name)
continue
try:
# @[fbahr] - TODO: `self.known_instances` grows over time
# towards inf. - clean-up?
if not r.inst_id in self.known_instances:
self.known_instances[r.inst_id] = self._metadata(uuid=r.inst_id)
r.project_id, r.user_id = self.known_instances[r.inst_id]
# insert project if it does not exist yet
projects = self.db.load(Project,
{'uuid': r.project_id},
limit=1)
if not projects:
project = Project(uuid=r.project_id,
created_at=self._str_to_datetime(r.timestamp))
self.db.save(project)
self.db.commit()
else:
project = projects[0]
record = MeterRecord(meter_id=meter_dict[r.meter_name].id,
host_id=host.id,
user_id=r.user_id,
resource_id=r.inst_id,
project_id=r.project_id,
value=r.value,
duration=r.duration,
timestamp=r.timestamp)
self.db.save(record)
_logger.debug("New %s" % record)
# update host and project activity
record_timestamp = self._str_to_datetime(r.timestamp)
if not host.activity or record_timestamp > host.activity:
host.activity = record_timestamp
if not project.updated_at or record_timestamp > project.updated_at:
project.updated_at = record_timestamp
except Exception as e:
_logger.exception(e)
try:
self.db.commit()
except Exception as e:
self.db.rollback()
_logger.exception(e)
self.db.session_close()
def _metadata(self, uuid):
"""
Connect to nova database (by means of nova-client);
fetches (project_id, user_id) information for a given instance uuid.
"""
nova_client = NovaClient(username=self._credentials['username'],
api_key=self._credentials['password'],
project_id=self._credentials['tenant_name'],
auth_url=self._credentials['auth_url'],
service_type='compute',
insecure=True)
# self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials)
# nova_client.client.auth_token = self._credentials['auth_token']
# nova_client.client.authenticate()
# unfortunately, nova_client.servers.find(id=...) is restricted
# to instances in an user's tentant (project) [can't tweek that,
# hard-coded in find()] - but:
search_opts = {'all_tenants': 1}
server = next((s.user_id, s.tenant_id)
for s in nova_client.servers.list(True, search_opts)
if s.id == uuid)
return map(str, server)
class AgentPublisher(threading.Thread):
def __init__(self, agent):
threading.Thread.__init__(self)
self.agent = agent
config = Config('giraffe.cfg')
self.host_name = config.get('agent', 'host_name')
self.flush_duration = config.getint('agent', 'duration')
self.connector = Connector(username=config.get('rabbit', 'user'),
password=config.get('rabbit', 'pass'),
host=config.get('rabbit', 'host'),
port=config.getint('rabbit', 'port'))
self.queue = config.get('rabbit', 'queue')
self.exchange = config.get('rabbit', 'exchange')
self.routing_key = config.get('rabbit', 'routing_key')
self.producer = BasicProducer(self.connector, self.exchange)
self.shared_secret = config.get('agent', 'shared_secret')
self.envelope = self._build_message()
self.stopRequest = False
self.lock = threading.Lock()
def _timestamp_now(self, datetime_now=None):
"""
Returns current system time as formatted string "%Y-%m-%d %H:%M:%S"
"""
if not datetime_now:
datetime_now = datetime.now()
return datetime_now.strftime("%Y-%m-%d %H:%M:%S")
def _build_message(self):
"""
Returns a new MessageAdapter object with hostname and signature
:rtype : MessageAdapter
"""
envelope = EnvelopeAdapter()
envelope.message.host_name = self.host_name
# message.signature = _SIGNATURE
return envelope
def add_meter_record(self, meter_name, meter_value, meter_duration):
"""
Add new meter record to meter message
Params: meter type [as string], value(s), and duration [in seconds]
"""
logger.debug("Add meter record: %s=%s" % (meter_name, meter_value))
if not self.lock.locked():
self.lock.acquire()
try:
if meter_name.startswith('inst'):
for record in meter_value:
self.envelope.add_inst_record(
timestamp=self._timestamp_now(record[1]),
meter_name=meter_name,
value=record[2],
duration=meter_duration,
project_id='',
inst_id=record[0],
user_id='')
else: # .startswith('host'):
self.envelope.add_host_record(self._timestamp_now(),
meter_name, meter_value,
meter_duration)
finally:
self.lock.release()
def flush(self):
"""
Sends current state of agents message to the broker.
"""
logger.debug("Flush meter message")
if not self.lock.locked():
self.lock.acquire()
try:
if self.envelope.len() > 0:
# flush message
self._send(self.envelope)
# build new message
self.envelope = self._build_message()
# except Exception as e:
# logger.exception(e)
finally:
self.lock.release()
def _send(self, envelope):
"""
Create message signature and send envelop to broker
"""
messageAdapter = MessageAdapter(envelope.message)
sig = createSignature(str(messageAdapter), self.shared_secret)
envelope.signature = sig
self.producer.send(exchange=self.exchange,
routing_key=self.routing_key,
message=envelope.serialize_to_str())
def run(self):
self.connector.connect()
while self.stopRequest is False:
time.sleep(self.flush_duration)
self.flush()
def stop(self):
self.stopRequest = True
self._Thread__stop()
