def _parseConfig(self, cfgdata): '''This method overrides the plugin base class method. It is used to parse the plugin specific configuration directives.''' for kw, args in cfgdata: if kw == "path": self.path = args elif kw == "api": self.api = args elif kw == "cloud_name": self.cloud_name = args path.insert(0, self.path) from lib.api.api_service_client import APIClient self.api = APIClient(self.api) self.api.dashboard_conn_check(self.cloud_name, check_for_vpn=True) self.msci = self.api.msci self.obj_cache = {} tzoffset = datetime.utcfromtimestamp(-1).hour - datetime.fromtimestamp( 0).hour + 1 if tzoffset == 24: tzoffset = 0 _now = int(time()) + (3600 * tzoffset) self.last_refresh = str(_now) self.expid = self.api.cldshow(self.cloud_name, "time")["experiment_id"] self.username = self.api.username self.latest_collection = {"HOST" : "latest_runtime_os_HOST_" + self.username, \ "VM" : "latest_runtime_os_VM_" + self.username} self.data_collection = {"HOST" : "runtime_os_HOST_" + self.username, \ "VM" : "runtime_os_VM_" + self.username} self.manage_collection = {"HOST" : "latest_management_HOST_" + self.username, \ "VM" : "latest_management_VM_" + self.username} _msg = "API contacted successfully. The current experiment id is \"" _msg += self.expid + "\" and the current username is \"" + self.username _msg += "\"." logging.debug(_msg)
def _parseConfig(self, cfgdata): '''This method overrides the plugin base class method. It is used to parse the plugin specific configuration directives.''' for kw,args in cfgdata: if kw == "path" : self.path = args elif kw == "api" : self.api = args elif kw == "cloud_name" : self.cloud_name = args path.insert(0, self.path) from lib.api.api_service_client import APIClient self.api = APIClient(self.api) self.api.dashboard_conn_check(self.cloud_name, check_for_vpn = True) self.msci = self.api.msci self.obj_cache = {} tzoffset = datetime.utcfromtimestamp(-1).hour - datetime.fromtimestamp(0).hour + 1 if tzoffset == 24 : tzoffset = 0 _now = int(time()) + (3600 * tzoffset) self.last_refresh = str(_now) self.expid = self.api.cldshow(self.cloud_name, "time")["experiment_id"] self.username = self.api.username self.latest_collection = {"HOST" : "latest_runtime_os_HOST_" + self.username, \ "VM" : "latest_runtime_os_VM_" + self.username} self.data_collection = {"HOST" : "runtime_os_HOST_" + self.username, \ "VM" : "runtime_os_VM_" + self.username} self.manage_collection = {"HOST" : "latest_management_HOST_" + self.username, \ "VM" : "latest_management_VM_" + self.username} _msg = "API contacted successfully. The current experiment id is \"" _msg += self.expid + "\" and the current username is \"" + self.username _msg += "\"." logging.debug(_msg)
def connect_to_cb(cloud_name) : ''' TBD ''' home = os.environ["HOME"] username = pwd.getpwuid(os.getuid())[0] api_file_name = "/tmp/cb_api_" + username if os.access(api_file_name, os.F_OK) : try : _fd = open(api_file_name, 'r') _api_conn_info = _fd.read() _fd.close() print _msg = "#" * 75 _msg += "\nFound CB API connection information in \"" + api_file_name + "\"\n" _msg += "#" * 75 print _msg sleep(3) except : _msg = "Unable to open file containing API connection information " _msg += "(" + api_file_name + ")." print _msg exit(4) else : _msg = "Unable to locate file containing API connection information " _msg += "(" + api_file_name + ")." print _msg exit(4) _path_set = False for _location in [ os.path.abspath(path[0] + "/../"), home ]: for _path, _dirs, _files in os.walk(_location): for _filename in fnmatch.filter(_files, "code_instrumentation.py") : if _path.count("/lib/auxiliary") : _path_set = _path.replace("/lib/auxiliary",'') path.append(_path_set) break if _path_set : break if not _path_set : _msg = "Unable to find CB's client API library" print _msg exit(4) else : _msg = "#" * 75 _msg += "\nCB API client library found in \"" + _path + "\".\n" _msg += "#" * 75 print _msg sleep(3) #from lib.api.api_service_client import * from lib.api.api_service_client import APIException,APINoSuchMetricException,APINoDataException,makeTimestamp,APIVM, APIClient _msg = "\nConnecting to API daemon (" + _api_conn_info + ")...\n" print _msg api = APIClient(_api_conn_info) _msg = "#" * 75 _msg += "\nChecking connection to cloud \"" + cloud_name + "\"..." print _msg, _cloud_not_found = True for _cld in api.cldlist() : if _cld["name"] == cloud_name : _cloud_not_found = False if _cloud_not_found : print "Unable to find cloud \"" + str(cloud_name) + "\"" exit(1) api.hostlist(cloud_name) print "OK" print "#" * 75 return api
def connect_to_cb(cloud_name): ''' TBD ''' home = os.environ["HOME"] username = pwd.getpwuid(os.getuid())[0] api_file_name = "/tmp/cb_api_" + username if os.access(api_file_name, os.F_OK): try: _fd = open(api_file_name, 'r') _api_conn_info = _fd.read() _fd.close() print _msg = "#" * 75 _msg += "\nFound CB API connection information in \"" + api_file_name + "\"\n" _msg += "#" * 75 print _msg sleep(3) except: _msg = "Unable to open file containing API connection information " _msg += "(" + api_file_name + ")." print _msg exit(4) else: _msg = "Unable to locate file containing API connection information " _msg += "(" + api_file_name + ")." print _msg exit(4) _path_set = False for _location in [os.path.abspath(path[0] + "/../"), home]: for _path, _dirs, _files in os.walk(_location): for _filename in fnmatch.filter(_files, "code_instrumentation.py"): if _path.count("/lib/auxiliary"): _path_set = _path.replace("/lib/auxiliary", '') path.append(_path_set) break if _path_set: break if not _path_set: _msg = "Unable to find CB's client API library" print _msg exit(4) else: _msg = "#" * 75 _msg += "\nCB API client library found in \"" + _path + "\".\n" _msg += "#" * 75 print _msg sleep(3) #from lib.api.api_service_client import * from lib.api.api_service_client import APIException, APINoSuchMetricException, APINoDataException, makeTimestamp, APIVM, APIClient _msg = "\nConnecting to API daemon (" + _api_conn_info + ")...\n" print _msg api = APIClient(_api_conn_info) _msg = "#" * 75 _msg += "\nChecking connection to cloud \"" + cloud_name + "\"..." print _msg, _cloud_not_found = True for _cld in api.cldlist(): if _cld["name"] == cloud_name: _cloud_not_found = False if _cloud_not_found: print "Unable to find cloud \"" + str(cloud_name) + "\"" exit(1) api.hostlist(cloud_name) print "OK" print "#" * 75 return api
class MongodbPlugin(GmetadPlugin) : ''' TBD ''' def __init__(self, cfgid): self.path = None self.api = None # The call to the parent class __init__ must be last GmetadPlugin.__init__(self, cfgid) def _resetConfig(self): pass def _parseConfig(self, cfgdata): '''This method overrides the plugin base class method. It is used to parse the plugin specific configuration directives.''' for kw,args in cfgdata: if kw == "path" : self.path = args elif kw == "api" : self.api = args elif kw == "cloud_name" : self.cloud_name = args path.insert(0, self.path) from lib.api.api_service_client import APIClient self.api = APIClient(self.api) self.api.dashboard_conn_check(self.cloud_name, check_for_vpn = True) self.msci = self.api.msci self.obj_cache = {} tzoffset = datetime.utcfromtimestamp(-1).hour - datetime.fromtimestamp(0).hour + 1 if tzoffset == 24 : tzoffset = 0 _now = int(time()) + (3600 * tzoffset) self.last_refresh = str(_now) self.expid = self.api.cldshow(self.cloud_name, "time")["experiment_id"] self.username = self.api.username self.latest_collection = {"HOST" : "latest_runtime_os_HOST_" + self.username, \ "VM" : "latest_runtime_os_VM_" + self.username} self.data_collection = {"HOST" : "runtime_os_HOST_" + self.username, \ "VM" : "runtime_os_VM_" + self.username} self.manage_collection = {"HOST" : "latest_management_HOST_" + self.username, \ "VM" : "latest_management_VM_" + self.username} _msg = "API contacted successfully. The current experiment id is \"" _msg += self.expid + "\" and the current username is \"" + self.username _msg += "\"." logging.debug(_msg) def find_object(self, ip, plugin, name) : ''' TBD ''' verbose = False msci = plugin.msci if plugin.api.should_refresh(plugin.cloud_name, plugin.last_refresh) : if verbose : logging.debug("Should refresh returned true!") plugin.obj_cache = {} tzoffset = datetime.utcfromtimestamp(-1).hour - datetime.fromtimestamp(0).hour + 1 if tzoffset == 24 : tzoffset = 0 _now = int(time()) + (3600 * tzoffset) plugin.last_refresh = str(_now) if ip in plugin.obj_cache : (exists, unused_obj) = plugin.obj_cache[ip] if verbose : if exists : logging.debug("Cache hit for ip " + ip) else : logging.debug("Expired hit for ip " + ip) return plugin.obj_cache[ip] if verbose : logging.debug("Cache miss for ip " + ip) try : vm = True obj = msci.find_document(plugin.manage_collection["VM"], { "$or" : [{"cloud_ip" : ip}, {"cloud_hostname" : name}], "mgt_901_deprovisioning_request_originated" : { "$exists" : False}, "mgt_903_deprovisioning_request_completed" : { "$exists" : False} }) if obj is not None : obj = plugin.api.vmshow(plugin.cloud_name, obj["_id"]) else : vm = False obj = msci.find_document(plugin.manage_collection["HOST"], { "$or" : [{"cloud_ip" : ip}, {"cloud_hostname" : name}] }) if obj is not None : obj = plugin.api.hostshow(plugin.cloud_name, obj["_id"]) else : vm = None obj = None except Exception, e : _status = 23 _fmsg = str(e) obj = None vm = None logging.debug("Can't get VM or HOST object for ip: " + ip) finally :
class MongodbPlugin(GmetadPlugin): ''' TBD ''' def __init__(self, cfgid): self.path = None self.api = None # The call to the parent class __init__ must be last GmetadPlugin.__init__(self, cfgid) def _resetConfig(self): pass def _parseConfig(self, cfgdata): '''This method overrides the plugin base class method. It is used to parse the plugin specific configuration directives.''' for kw, args in cfgdata: if kw == "path": self.path = args elif kw == "api": self.api = args elif kw == "cloud_name": self.cloud_name = args path.insert(0, self.path) from lib.api.api_service_client import APIClient self.api = APIClient(self.api) self.api.dashboard_conn_check(self.cloud_name, check_for_vpn=True) self.msci = self.api.msci self.obj_cache = {} tzoffset = datetime.utcfromtimestamp(-1).hour - datetime.fromtimestamp( 0).hour + 1 if tzoffset == 24: tzoffset = 0 _now = int(time()) + (3600 * tzoffset) self.last_refresh = str(_now) self.expid = self.api.cldshow(self.cloud_name, "time")["experiment_id"] self.username = self.api.username self.latest_collection = {"HOST" : "latest_runtime_os_HOST_" + self.username, \ "VM" : "latest_runtime_os_VM_" + self.username} self.data_collection = {"HOST" : "runtime_os_HOST_" + self.username, \ "VM" : "runtime_os_VM_" + self.username} self.manage_collection = {"HOST" : "latest_management_HOST_" + self.username, \ "VM" : "latest_management_VM_" + self.username} _msg = "API contacted successfully. The current experiment id is \"" _msg += self.expid + "\" and the current username is \"" + self.username _msg += "\"." logging.debug(_msg) def find_object(self, ip, plugin, name): ''' TBD ''' verbose = False msci = plugin.msci if plugin.api.should_refresh(plugin.cloud_name, plugin.last_refresh): if verbose: logging.debug("Should refresh returned true!") plugin.obj_cache = {} tzoffset = datetime.utcfromtimestamp( -1).hour - datetime.fromtimestamp(0).hour + 1 if tzoffset == 24: tzoffset = 0 _now = int(time()) + (3600 * tzoffset) plugin.last_refresh = str(_now) if ip in plugin.obj_cache: (exists, unused_obj) = plugin.obj_cache[ip] if verbose: if exists: logging.debug("Cache hit for ip " + ip) else: logging.debug("Expired hit for ip " + ip) return plugin.obj_cache[ip] if verbose: logging.debug("Cache miss for ip " + ip) try: vm = True obj = msci.find_document( plugin.manage_collection["VM"], { "$or": [{ "cloud_ip": ip }, { "cloud_hostname": name }], "mgt_901_deprovisioning_request_originated": { "$exists": False }, "mgt_903_deprovisioning_request_completed": { "$exists": False }, "expid": self.expid }) if obj is not None: obj = plugin.api.vmshow(plugin.cloud_name, obj["_id"]) else: vm = False obj = msci.find_document( plugin.manage_collection["HOST"], { "$or": [{ "cloud_ip": ip }, { "cloud_hostname": name }], "expid": self.expid }) if obj is not None: obj = plugin.api.hostshow(plugin.cloud_name, obj["_id"]) else: vm = None obj = None except Exception as e: _status = 23 _fmsg = str(e) obj = None vm = None logging.debug("Can't get VM or HOST object for ip: " + ip + ": " + str(e)) finally: if vm and obj: plugin.obj_cache[ip] = (vm, obj) return vm, obj def start(self): ''' Called by the engine during initialization to get the plugin going. ''' logging.debug("MongodbWriter start called") def stop(self): ''' Called by the engine during shutdown to allow the plugin to shutdown. ''' logging.debug("MongodbWriter stop called") def notify(self, clusterNode): ''' TBD ''' from lib.api.api_service_client import APIClient, makeTimestamp for hostNode in clusterNode: ''' Available hostNode keys: gmond_started, name, tags, ip, tmax, tn, reported, location, dmax ''' # Cache object attributes using an ordered dictionary path.insert(0, self.path) from lib.api.api_service_client import APIException try: vm, obj = self.find_object(hostNode.getAttr('ip'), self, hostNode.getAttr('name')) except APIException as obj: logging.error("Problem with API connectivity: " + str(obj)) continue except Exception as obj2: logging.error("Problem with API object lookup: " + str(obj2)) continue if vm is None: # error during lookup # print "VM " + str(hostNode.getAttr('ip')) + " " + hostNode.getAttr('name') + " not found. not going through data..." continue _data = {"uuid": obj["uuid"], "expid": self.expid} obj_type = "VM" if vm else "HOST" empty = True # reported = "" for metricNode in hostNode: ''' Available metricNode keys: name, val, tmax, tn, source, units, dmax, type, slope ''' # reported += " " + metricNode.getAttr('name') # Don't evaluate metrics that aren't numeric values. if metricNode.getAttr('type') in ['string', 'timestamp']: continue attrs = metricNode.getAttrs() last = attrs['tn'] if attrs["units"].strip() == "": attrs["units"] = "N/A" #The pymongo client does not allow keys whose names contains "." # to be written back in Mongo _data[attrs["name"].replace('.', '-')] = attrs empty = False if not empty: tzoffset = datetime.utcfromtimestamp( -1).hour - datetime.fromtimestamp(0).hour + 1 if tzoffset == 24: tzoffset = 0 _now = int(time()) + (3600 * tzoffset) _data["time_h"] = makeTimestamp(_now) _data["time"] = _now _data["latest_update"] = _now try: if obj_type == "VM": pass name = obj["name"] # print_list(_data, "data") self.msci.add_document(self.data_collection[obj_type], _data) old = self.msci.find_document( self.latest_collection[obj_type], { "_id": obj["uuid"], "expid": self.expid }) if old is not None: # print_list(old, "old latest") old.update(_data) else: old = _data # print_list(old, "final latest") old["_id"] = obj["uuid"] self.msci.update_document(self.latest_collection[obj_type], old) except Exception as e: _status = 23 _fmsg = str(e) logging.error("Could not write to metric store: " + str(_fmsg))