def __init__(self): self.config = Config('giraffe.cfg') # configure RabbitMQ connector (and consumer) self.connector = Connector(username=self.config.get('rabbit', 'user'), password=self.config.get('rabbit', 'pass'), host=self.config.get('rabbit', 'host'), port=self.config.getint('rabbit', 'port')) self.queue = self.config.get('rabbit', 'queue') self.exchange = self.config.get('rabbit', 'exchange') self.routing_key = self.config.get('rabbit', 'routing_key') self.consumer = BasicConsumer(self.connector, self.queue, self.exchange, self._collector_callback) self.shared_secret = self.config.get('collector', 'shared_secret') # connect to giraffe database self.db = db.connect( '%s://%s:%s@%s/%s' % (self.config.get('db', 'vendor'), self.config.get('db', 'user'), self.config.get('db', 'pass'), self.config.get( 'db', 'host'), self.config.get('db', 'schema'))) # prepare connection to nova-client self._credentials = dict( username=self.config.get('agent', 'user'), password=self.config.get('agent', 'pass'), # tenant_id=self.config.get('agent', 'tenant_id'), tenant_name=self.config.get('agent', 'tenant_name'), auth_url=self.config.get('auth', 'admin_url'), insecure=True) # known servers/instances self.known_instances = {}
def __init__(self): self.config = Config('giraffe.cfg') # configure RabbitMQ connector (and consumer) self.connector = Connector(username=self.config.get('rabbit', 'user'), password=self.config.get('rabbit', 'pass'), host=self.config.get('rabbit', 'host'), port=self.config.getint('rabbit', 'port')) self.queue = self.config.get('rabbit', 'queue') self.exchange = self.config.get('rabbit', 'exchange') self.routing_key = self.config.get('rabbit', 'routing_key') self.consumer = BasicConsumer(self.connector, self.queue, self.exchange, self._collector_callback) self.shared_secret = self.config.get('collector', 'shared_secret') # connect to giraffe database self.db = db.connect('%s://%s:%s@%s/%s' % (self.config.get('db', 'vendor'), self.config.get('db', 'user'), self.config.get('db', 'pass'), self.config.get('db', 'host'), self.config.get('db', 'schema'))) # prepare connection to nova-client self._credentials = dict(username=self.config.get('agent', 'user'), password=self.config.get('agent', 'pass'), # tenant_id=self.config.get('agent', 'tenant_id'), tenant_name=self.config.get('agent', 'tenant_name'), auth_url=self.config.get('auth', 'admin_url'), insecure=True) # known servers/instances self.known_instances = {}
class Collector(object): def __init__(self): self.config = Config('giraffe.cfg') # configure RabbitMQ connector (and consumer) self.connector = Connector(username=self.config.get('rabbit', 'user'), password=self.config.get('rabbit', 'pass'), host=self.config.get('rabbit', 'host'), port=self.config.getint('rabbit', 'port')) self.queue = self.config.get('rabbit', 'queue') self.exchange = self.config.get('rabbit', 'exchange') self.routing_key = self.config.get('rabbit', 'routing_key') self.consumer = BasicConsumer(self.connector, self.queue, self.exchange, self._collector_callback) self.shared_secret = self.config.get('collector', 'shared_secret') # connect to giraffe database self.db = db.connect( '%s://%s:%s@%s/%s' % (self.config.get('db', 'vendor'), self.config.get('db', 'user'), self.config.get('db', 'pass'), self.config.get( 'db', 'host'), self.config.get('db', 'schema'))) # prepare connection to nova-client self._credentials = dict( username=self.config.get('agent', 'user'), password=self.config.get('agent', 'pass'), # tenant_id=self.config.get('agent', 'tenant_id'), tenant_name=self.config.get('agent', 'tenant_name'), auth_url=self.config.get('auth', 'admin_url'), insecure=True) # known servers/instances self.known_instances = {} def launch(self): try: self.connector.connect() self.start_collecting() except KeyboardInterrupt: _logger.info("Ctrl-c received!") except: _logger.exception("Error: Unable to start collector service") finally: self.stop_collecting() _logger.info("Shutdown collector service") def start_collecting(self): _logger.debug("Start collecting from broker") self.consumer.consume() def stop_collecting(self): _logger.debug("Stop collecting from broker") self.consumer.stop_consuming() def _str_to_datetime(self, timestamp_str): return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S') def _datetime_to_str(self, datetime_obj): return datetime_obj.strftime("%Y-%m-%d %H:%M:%S") def _collector_callback(self, params): envelope = EnvelopeAdapter() # check whether incorrectly formatted message try: envelope.deserialize_from_str(params) except: return message = MessageAdapter(envelope.message) # validate signature if not validateSignature(str(message), self.shared_secret, envelope.signature): return self.db.session_open() # load all meters now to avoid queries later meters = self.db.load(Meter) meter_dict = {} for meter in meters: meter_dict[meter.name] = meter # insert host if it does not exist yet hosts = self.db.load(Host, {'name': message.host_name}, limit=1) if not hosts: host = Host(name=message.host_name) self.db.save(host) self.db.commit() else: host = hosts[0] # insert all host records for r in message.host_records: if r.meter_name not in meter_dict: _logger.warning('Unknown meter_name "%s"' % r.meter_name) continue try: record = MeterRecord(meter_id=meter_dict[r.meter_name].id, host_id=host.id, user_id=None, resource_id=None, project_id=None, value=r.value, duration=r.duration, timestamp=r.timestamp) self.db.save(record) _logger.debug("New %s" % record) # update host activity record_timestamp = self._str_to_datetime(r.timestamp) if not host.activity or record_timestamp > host.activity: host.activity = record_timestamp except Exception as e: _logger.exception(e) # insert all instance records for r in message.inst_records: if r.meter_name not in meter_dict: _logger.warning('Unknown meter_name "%s"' % r.meter_name) continue try: # @[fbahr] - TODO: `self.known_instances` grows over time # towards inf. - clean-up? if not r.inst_id in self.known_instances: self.known_instances[r.inst_id] = self._metadata( uuid=r.inst_id) r.project_id, r.user_id = self.known_instances[r.inst_id] # insert project if it does not exist yet projects = self.db.load(Project, {'uuid': r.project_id}, limit=1) if not projects: project = Project(uuid=r.project_id, created_at=self._str_to_datetime( r.timestamp)) self.db.save(project) self.db.commit() else: project = projects[0] record = MeterRecord(meter_id=meter_dict[r.meter_name].id, host_id=host.id, user_id=r.user_id, resource_id=r.inst_id, project_id=r.project_id, value=r.value, duration=r.duration, timestamp=r.timestamp) self.db.save(record) _logger.debug("New %s" % record) # update host and project activity record_timestamp = self._str_to_datetime(r.timestamp) if not host.activity or record_timestamp > host.activity: host.activity = record_timestamp if not project.updated_at or record_timestamp > project.updated_at: project.updated_at = record_timestamp except Exception as e: _logger.exception(e) try: self.db.commit() except Exception as e: self.db.rollback() _logger.exception(e) self.db.session_close() def _metadata(self, uuid): """ Connect to nova database (by means of nova-client); fetches (project_id, user_id) information for a given instance uuid. """ nova_client = NovaClient(username=self._credentials['username'], api_key=self._credentials['password'], project_id=self._credentials['tenant_name'], auth_url=self._credentials['auth_url'], service_type='compute', insecure=True) # self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials) # nova_client.client.auth_token = self._credentials['auth_token'] # nova_client.client.authenticate() # unfortunately, nova_client.servers.find(id=...) is restricted # to instances in an user's tentant (project) [can't tweek that, # hard-coded in find()] - but: search_opts = {'all_tenants': 1} server = next((s.user_id, s.tenant_id) for s in nova_client.servers.list(True, search_opts) if s.id == uuid) return map(str, server)
class Collector(object): def __init__(self): self.config = Config('giraffe.cfg') # configure RabbitMQ connector (and consumer) self.connector = Connector(username=self.config.get('rabbit', 'user'), password=self.config.get('rabbit', 'pass'), host=self.config.get('rabbit', 'host'), port=self.config.getint('rabbit', 'port')) self.queue = self.config.get('rabbit', 'queue') self.exchange = self.config.get('rabbit', 'exchange') self.routing_key = self.config.get('rabbit', 'routing_key') self.consumer = BasicConsumer(self.connector, self.queue, self.exchange, self._collector_callback) self.shared_secret = self.config.get('collector', 'shared_secret') # connect to giraffe database self.db = db.connect('%s://%s:%s@%s/%s' % (self.config.get('db', 'vendor'), self.config.get('db', 'user'), self.config.get('db', 'pass'), self.config.get('db', 'host'), self.config.get('db', 'schema'))) # prepare connection to nova-client self._credentials = dict(username=self.config.get('agent', 'user'), password=self.config.get('agent', 'pass'), # tenant_id=self.config.get('agent', 'tenant_id'), tenant_name=self.config.get('agent', 'tenant_name'), auth_url=self.config.get('auth', 'admin_url'), insecure=True) # known servers/instances self.known_instances = {} def launch(self): try: self.connector.connect() self.start_collecting() except KeyboardInterrupt: _logger.info("Ctrl-c received!") except: _logger.exception("Error: Unable to start collector service") finally: self.stop_collecting() _logger.info("Shutdown collector service") def start_collecting(self): _logger.debug("Start collecting from broker") self.consumer.consume() def stop_collecting(self): _logger.debug("Stop collecting from broker") self.consumer.stop_consuming() def _str_to_datetime(self, timestamp_str): return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S') def _datetime_to_str(self, datetime_obj): return datetime_obj.strftime("%Y-%m-%d %H:%M:%S") def _collector_callback(self, params): envelope = EnvelopeAdapter() # check whether incorrectly formatted message try: envelope.deserialize_from_str(params) except: return message = MessageAdapter(envelope.message) # validate signature if not validateSignature(str(message), self.shared_secret, envelope.signature): return self.db.session_open() # load all meters now to avoid queries later meters = self.db.load(Meter) meter_dict = {} for meter in meters: meter_dict[meter.name] = meter # insert host if it does not exist yet hosts = self.db.load(Host, {'name': message.host_name}, limit=1) if not hosts: host = Host(name=message.host_name) self.db.save(host) self.db.commit() else: host = hosts[0] # insert all host records for r in message.host_records: if r.meter_name not in meter_dict: _logger.warning('Unknown meter_name "%s"' % r.meter_name) continue try: record = MeterRecord(meter_id=meter_dict[r.meter_name].id, host_id=host.id, user_id=None, resource_id=None, project_id=None, value=r.value, duration=r.duration, timestamp=r.timestamp) self.db.save(record) _logger.debug("New %s" % record) # update host activity record_timestamp = self._str_to_datetime(r.timestamp) if not host.activity or record_timestamp > host.activity: host.activity = record_timestamp except Exception as e: _logger.exception(e) # insert all instance records for r in message.inst_records: if r.meter_name not in meter_dict: _logger.warning('Unknown meter_name "%s"' % r.meter_name) continue try: # @[fbahr] - TODO: `self.known_instances` grows over time # towards inf. - clean-up? if not r.inst_id in self.known_instances: self.known_instances[r.inst_id] = self._metadata(uuid=r.inst_id) r.project_id, r.user_id = self.known_instances[r.inst_id] # insert project if it does not exist yet projects = self.db.load(Project, {'uuid': r.project_id}, limit=1) if not projects: project = Project(uuid=r.project_id, created_at=self._str_to_datetime(r.timestamp)) self.db.save(project) self.db.commit() else: project = projects[0] record = MeterRecord(meter_id=meter_dict[r.meter_name].id, host_id=host.id, user_id=r.user_id, resource_id=r.inst_id, project_id=r.project_id, value=r.value, duration=r.duration, timestamp=r.timestamp) self.db.save(record) _logger.debug("New %s" % record) # update host and project activity record_timestamp = self._str_to_datetime(r.timestamp) if not host.activity or record_timestamp > host.activity: host.activity = record_timestamp if not project.updated_at or record_timestamp > project.updated_at: project.updated_at = record_timestamp except Exception as e: _logger.exception(e) try: self.db.commit() except Exception as e: self.db.rollback() _logger.exception(e) self.db.session_close() def _metadata(self, uuid): """ Connect to nova database (by means of nova-client); fetches (project_id, user_id) information for a given instance uuid. """ nova_client = NovaClient(username=self._credentials['username'], api_key=self._credentials['password'], project_id=self._credentials['tenant_name'], auth_url=self._credentials['auth_url'], service_type='compute', insecure=True) # self._credentials['auth_token'] = AuthProxy.get_token(**self._credentials) # nova_client.client.auth_token = self._credentials['auth_token'] # nova_client.client.authenticate() # unfortunately, nova_client.servers.find(id=...) is restricted # to instances in an user's tentant (project) [can't tweek that, # hard-coded in find()] - but: search_opts = {'all_tenants': 1} server = next((s.user_id, s.tenant_id) for s in nova_client.servers.list(True, search_opts) if s.id == uuid) return map(str, server)