def redis_objectstore_setup(global_objects, operation, cloud_name=None): ''' TBD ''' _protocol = global_objects["objectstore"]["protocol"] _hostname = global_objects["objectstore"]["host"] _databaseid = int(global_objects["objectstore"]["dbid"]) _timeout = float(global_objects["objectstore"]["timout"]) _username = global_objects["objectstore"]["username"] _usage = global_objects["objectstore"]["usage"].lower() try: _instance_dir = global_objects["space"]["instance_dir"] if operation == "check": _stores_path = global_objects["space"]["stores_working_dir"] if not path.exists(_stores_path): cbdebug("Making stores working directory: " + _stores_path) mkdir(_stores_path) if _usage == "shared": _hostport = int(global_objects["objectstore"]["port"]) _proc_man = ProcessManagement(username="******") if not pre_check_port(_hostname, _hostport, _protocol): _redis_pid = _proc_man.get_pid_from_cmdline("redis-server") _cmd = "/usr/local/bin/redis-server /etc/redis.conf" if not _redis_pid: _msg = "Unable to detect a shared Redis server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _config_file_fn = _stores_path + '/' + _username + "_redis.conf" _cmd = "redis-server " + _config_file_fn _proc_man = ProcessManagement(username=_username) _redis_pid = _proc_man.get_pid_from_cmdline("redis-server") if not _redis_pid: global_objects["objectstore"][ "port"] = _proc_man.get_free_port( global_objects["objectstore"]["port"], protocol="tcp") _hostport = int(global_objects["objectstore"]["port"]) _config_file_contents = global_objects["objectstore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _redis_pid = _proc_man.start_daemon(_cmd) if not _redis_pid: _msg = "Unable to detect a private Redis server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: global_objects["objectstore"][ "port"] = _proc_man.get_port_from_pid(_redis_pid[0]) _hostport = int(global_objects["objectstore"]["port"]) _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "An Object Store of the kind \"Redis\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + ", database id \"" _msg += str(_databaseid) + "\" seems to be running." _status = 0 else: if not cloud_name: raise StoreSetupException( "Name of cloud is required for the 'initialize' mode", 22) operation = "initialize" _hostport = int(global_objects["objectstore"]["port"]) _collection_names = [ "reported_management_vm_metric_names", \ "reported_runtime_os_host_metric_names", \ "reported_runtime_os_vm_metric_names", \ "host_management_metrics_header", \ "vm_management_metrics_header", \ "host_runtime_os_metrics_header", \ "vm_runtime_os_metrics_header", \ "vm_runtime_app_metrics_header", \ "trace_header" ] for _collection_name in _collection_names: for _component in global_objects["mon_defaults"][ _collection_name].split(','): if _component.lower() in global_objects["mon_defaults"]: global_objects["mon_defaults"][_collection_name] = \ global_objects["mon_defaults"][_collection_name].replace(_component, \ global_objects["mon_defaults"][_component.lower()] + ',') global_objects["mon_defaults"][_collection_name] = \ global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',') _rmc = RedisMgdConn(global_objects["objectstore"]) # First we remove the leftovers from previous experiments. if _rmc.initialize_object_store(cloud_name, global_objects, True): if not path.exists(_instance_dir): mkdir(_instance_dir) for _file_name in listdir(_instance_dir): _file_name = path.join(_instance_dir, _file_name) if path.isdir(_file_name): rmtree(_file_name) _msg = "Folders (but not data) underneath experiment " _msg += "directory " + _instance_dir + " were removed." cbdebug(_msg) _msg = "The Redis datastore was successfully initialized on server " + _hostname _msg += ", port " + str(_hostport) + ", database id \"" + str( _databaseid) _msg += "\"." cbdebug(_msg) _status = 0 else: _msg = "The Object Store of the kind \"Redis\" was successfully initialized " _msg += "on node " + _hostname + ". To change its " _msg += "attributes/state, use the *alter commands" _msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach " _msg += "and attach this cloud back to this experiment." cbdebug(_msg) _status = 0 return _status, _msg except NetworkException, obj: _msg = "An Object Store of the kind \"Redis\" on node " _msg += _hostname + ", " + _protocol + " port " + str(_hostport) _msg += ", database id \"" + str(_databaseid) _msg += "\" seems to be down: " + str(obj.msg) + '.' cberr(_msg) raise StoreSetupException(_msg, 8)
def mongodb_metricstore_setup(global_objects, operation="check"): ''' TBD ''' _protocol = global_objects["metricstore"]["protocol"] _hostname = global_objects["metricstore"]["host"] _databaseid = global_objects["metricstore"]["database"] _timeout = float(global_objects["metricstore"]["timeout"]) _username = global_objects["mon_defaults"]["username"] _usage = global_objects["metricstore"]["usage"].lower() try: if operation == "check": if _usage == "shared": _hostport = int(global_objects["metricstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f") _cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid" if not _mongodb_pid: _msg = "Unable to detect a shared MongoDB server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _config_file_fn = global_objects["space"][ "stores_working_dir"] + '/' + _username + "_mongod.conf" _cmd = "mkdir -p " + global_objects["space"][ "stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects[ "space"]["stores_working_dir"] + "/mongod.pid" _proc_man = ProcessManagement(username=_username) _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f") if not _mongodb_pid: global_objects["metricstore"][ "port"] = _proc_man.get_free_port( global_objects["metricstore"]["port"], protocol="tcp") _hostport = int(global_objects["metricstore"]["port"]) _config_file_contents = global_objects["metricstore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace( "--", '=') _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_fn = global_objects["space"][ "stores_working_dir"] + '/' + _username + "_mongod.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _mongodb_pid = _proc_man.start_daemon(_cmd) sleep(5) if not _mongodb_pid: _msg = "Unable to detect a private MongoDB server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: global_objects["metricstore"][ "port"] = _proc_man.get_port_from_pid(_mongodb_pid[0]) _hostport = int(global_objects["metricstore"]["port"]) _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + ", database id \"" + str( _databaseid) _msg += "\" seems to be running." cbdebug(_msg) _status = 0 else: operation = "initialize" _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.initialize_metric_store(_username) _msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized " _msg += "on node: " + str(global_objects["metricstore"]) cbdebug(_msg) _status = 0 _status = 0 return _status, _msg except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def rsync_filestore_setup(global_objects, operation="check"): ''' TBD ''' _hostname = global_objects["filestore"]["hostname"] _protocol = global_objects["filestore"]["protocol"] _username = global_objects["filestore"]["username"] _port = global_objects["filestore"]["port"] _usage = global_objects["filestore"]["usage"].lower() _base_dir = global_objects["space"]["base_dir"] _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try: _name, _ip = hostname2ip(_hostname) if operation == "check": if _usage == "shared": _hostport = int(global_objects["filestore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _rsync_pid = _proc_man.get_pid_from_cmdline( "rsync --daemon") _cmd = "rsync --daemon" if not _rsync_pid: _msg = "Unable to detect a shared rsync server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _proc_man = ProcessManagement(username=_username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _cmd = "rsync --daemon --config " + _config_file_fn if not access(_config_file_fn, F_OK): # File was deleted, but the rsync process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid: _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid") global_objects["filestore"][ "port"] = _proc_man.get_free_port( global_objects["filestore"]["port"], protocol="tcp") _hostport = int(global_objects["filestore"]["port"]) _config_file_contents = global_objects["filestore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "DOLLAR", '$') _config_file_contents = _config_file_contents.replace( "REPLEQUAL", '=') _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace( "REPLUSERU", _username + '_') _config_file_contents = _config_file_contents.replace( "REPLUSER", _username) _config_file_contents = _config_file_contents.replace( "REPLBASEDIR", _base_dir) _config_file_contents = _config_file_contents.replace( "REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_contents = _config_file_contents.replace( "--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd) if not _rsyslog_pid: _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents: if _line.count("port="): global_objects["filestore"]["port"] = _line.split( '=')[1] _hostport = int( global_objects["filestore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A File Store of the kind \"rsync\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsync " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsync" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def redis_objectstore_setup(global_objects, operation, cloud_name = None) : ''' TBD ''' _protocol = global_objects["objectstore"]["protocol"] _hostname = global_objects["objectstore"]["host"] _databaseid = int(global_objects["objectstore"]["dbid"]) _timeout = float(global_objects["objectstore"]["timout"]) _username = global_objects["objectstore"]["username"] _usage = global_objects["objectstore"]["usage"].lower() try : _instance_dir = global_objects["space"]["instance_dir"] if operation == "check" : _stores_path = global_objects["space"]["stores_working_dir"] if not path.exists(_stores_path) : cbdebug("Making stores working directory: " + _stores_path) mkdir(_stores_path) if _usage == "shared" : _hostport = int(global_objects["objectstore"]["port"]) _proc_man = ProcessManagement(username = "******") if not pre_check_port(_hostname, _hostport, _protocol) : _redis_pid = _proc_man.get_pid_from_cmdline("redis-server") _cmd = "/usr/local/bin/redis-server /etc/redis.conf" if not _redis_pid : _msg = "Unable to detect a shared Redis server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _config_file_fn = _stores_path + '/' + _username + "_redis.conf" _cmd = "redis-server " + _config_file_fn _proc_man = ProcessManagement(username = _username) _redis_pid = _proc_man.get_pid_from_cmdline("redis-server") if not _redis_pid : global_objects["objectstore"]["port"] = _proc_man.get_free_port(global_objects["objectstore"]["port"], protocol = "tcp") _hostport = int(global_objects["objectstore"]["port"]) _config_file_contents = global_objects["objectstore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace(';','\n') _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _redis_pid = _proc_man.start_daemon(_cmd) if not _redis_pid : _msg = "Unable to detect a private Redis server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : global_objects["objectstore"]["port"] = _proc_man.get_port_from_pid(_redis_pid[0]) _hostport = int(global_objects["objectstore"]["port"]) _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "An Object Store of the kind \"Redis\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + ", database id \"" _msg += str(_databaseid) + "\" seems to be running." _status = 0 else : if not cloud_name : raise StoreSetupException("Name of cloud is required for the 'initialize' mode", 22) operation = "initialize" _hostport = int(global_objects["objectstore"]["port"]) _collection_names = [ "reported_management_vm_metric_names", \ "reported_runtime_os_host_metric_names", \ "reported_runtime_os_vm_metric_names", \ "host_management_metrics_header", \ "vm_management_metrics_header", \ "host_runtime_os_metrics_header", \ "vm_runtime_os_metrics_header", \ "vm_runtime_app_metrics_header", \ "trace_header" ] for _collection_name in _collection_names : for _component in global_objects["mon_defaults"][_collection_name].split(',') : if _component.lower() in global_objects["mon_defaults"] : global_objects["mon_defaults"][_collection_name] = \ global_objects["mon_defaults"][_collection_name].replace(_component, \ global_objects["mon_defaults"][_component.lower()] + ',') global_objects["mon_defaults"][_collection_name] = \ global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',') _rmc = RedisMgdConn(global_objects["objectstore"]) # First we remove the leftovers from previous experiments. if _rmc.initialize_object_store(cloud_name, global_objects, True) : if not path.exists(_instance_dir) : mkdir(_instance_dir) for _file_name in listdir(_instance_dir) : _file_name = path.join(_instance_dir, _file_name) if path.isdir(_file_name) : rmtree(_file_name) _msg = "Folders (but not data) underneath experiment " _msg += "directory " + _instance_dir + " were removed." cbdebug(_msg) _msg = "The Redis datastore was successfully initialized on server " + _hostname _msg += ", port " + str(_hostport) + ", database id \"" + str(_databaseid) _msg += "\"." cbdebug(_msg) _status = 0 else : _msg = "The Object Store of the kind \"Redis\" was successfully initialized " _msg += "on node " + _hostname + ". To change its " _msg += "attributes/state, use the *alter commands" _msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach " _msg += "and attach this cloud back to this experiment." cbdebug(_msg) _status = 0 return _status, _msg except NetworkException, obj : _msg = "An Object Store of the kind \"Redis\" on node " _msg += _hostname + ", " + _protocol + " port " + str(_hostport) _msg += ", database id \"" + str(_databaseid) _msg += "\" seems to be down: " + str(obj.msg) + '.' cberr(_msg) raise StoreSetupException(_msg, 8)
def syslog_logstore_setup(global_objects, operation="check"): ''' TBD ''' _hostname = global_objects["logstore"]["hostname"] _protocol = global_objects["logstore"]["protocol"] _username = global_objects["logstore"]["username"] _usage = global_objects["logstore"]["usage"].lower() _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try: _name, _ip = hostname2ip(_hostname) if operation == "check": if _usage == "shared": _hostport = int(global_objects["logstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd") _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid " if not _rsyslog_pid: _msg = "Unable to detect a shared rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _proc_man = ProcessManagement(username=_username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid" if not access(_config_file_fn, F_OK): # File was deleted, but the rsyslog process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) if not access(_log_dir, W_OK): # The directory does not even exist, kill any rsyslog processes still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid: global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\ protocol = "udp") _hostport = int(global_objects["logstore"]["port"]) _config_file_contents = global_objects["logstore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "DOLLAR", '$') _config_file_contents = _config_file_contents.replace( "RSYSLOG", "RSYSLOG_") _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace( "REPLUSER", _username + '_') _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_contents = _config_file_contents.replace( "--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon(_cmd) if not _rsyslog_pid: _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents: if _line.count("UDPServerRun"): global_objects["logstore"]["port"] = _line.split( )[1] _hostport = int(global_objects["logstore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def rsync_filestore_setup(global_objects, operation = "check") : ''' TBD ''' _hostname = global_objects["filestore"]["hostname"] _protocol = global_objects["filestore"]["protocol"] _username = global_objects["filestore"]["username"] _port = global_objects["filestore"]["port"] _usage = global_objects["filestore"]["usage"].lower() _base_dir = global_objects["space"]["base_dir"] _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try : _name, _ip = hostname2ip(_hostname) if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["filestore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _rsync_pid = _proc_man.get_pid_from_cmdline("rsync --daemon") _cmd = "rsync --daemon" if not _rsync_pid : _msg = "Unable to detect a shared rsync server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _proc_man = ProcessManagement(username = _username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _cmd = "rsync --daemon --config " + _config_file_fn if not access(_config_file_fn, F_OK) : # File was deleted, but the rsync process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid : _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid") global_objects["filestore"]["port"] = _proc_man.get_free_port(global_objects["filestore"]["port"], protocol = "tcp") _hostport = int(global_objects["filestore"]["port"]) _config_file_contents = global_objects["filestore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("DOLLAR", '$') _config_file_contents = _config_file_contents.replace("REPLEQUAL", '=') _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace("REPLUSERU", _username + '_') _config_file_contents = _config_file_contents.replace("REPLUSER", _username) _config_file_contents = _config_file_contents.replace("REPLBASEDIR", _base_dir) _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace(';','\n') _config_file_contents = _config_file_contents.replace("--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd) if not _rsyslog_pid : _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents : if _line.count("port=") : global_objects["filestore"]["port"] = _line.split('=')[1] _hostport = int(global_objects["filestore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A File Store of the kind \"rsync\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsync " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsync" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def mongodb_metricstore_setup(global_objects, operation = "check") : ''' TBD ''' _protocol = global_objects["metricstore"]["protocol"] _hostname = global_objects["metricstore"]["host"] _databaseid = global_objects["metricstore"]["database"] _timeout = float(global_objects["metricstore"]["timeout"]) _username = global_objects["mon_defaults"]["username"] _usage = global_objects["metricstore"]["usage"].lower() try : if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["metricstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f") _cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid" if not _mongodb_pid : _msg = "Unable to detect a shared MongoDB server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf" _cmd = "mkdir -p " + global_objects["space"]["stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects["space"]["stores_working_dir"] + "/mongod.pid" _proc_man = ProcessManagement(username = _username) _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f") if not _mongodb_pid : global_objects["metricstore"]["port"] = _proc_man.get_free_port(global_objects["metricstore"]["port"], protocol = "tcp") _hostport = int(global_objects["metricstore"]["port"]) _config_file_contents = global_objects["metricstore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace("--", '=') _config_file_contents = _config_file_contents.replace(';','\n') _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _mongodb_pid = _proc_man.start_daemon(_cmd) sleep(5) if not _mongodb_pid : _msg = "Unable to detect a private MongoDB server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : global_objects["metricstore"]["port"] = _proc_man.get_port_from_pid(_mongodb_pid[0]) _hostport = int(global_objects["metricstore"]["port"]) _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + ", database id \"" + str(_databaseid) _msg += "\" seems to be running." cbdebug(_msg) _status = 0 else: operation = "initialize" _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.initialize_metric_store(_username) _msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized " _msg += "on node: " + str(global_objects["metricstore"]) cbdebug(_msg) _status = 0 _status = 0 return _status, _msg except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def syslog_logstore_setup(global_objects, operation = "check") : ''' TBD ''' _hostname = global_objects["logstore"]["hostname"] _protocol = global_objects["logstore"]["protocol"] _username = global_objects["logstore"]["username"] _usage = global_objects["logstore"]["usage"].lower() _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try : _name, _ip = hostname2ip(_hostname) if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["logstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd") _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid " if not _rsyslog_pid : _msg = "Unable to detect a shared rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _proc_man = ProcessManagement(username = _username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid" if not access(_config_file_fn, F_OK) : # File was deleted, but the rsyslog process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) if not access(_log_dir, W_OK) : # The directory does not even exist, kill any rsyslog processes still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid : global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\ protocol = "udp") _hostport = int(global_objects["logstore"]["port"]) _config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("DOLLAR", '$') _config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_") _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace("REPLUSER", _username + '_') _config_file_contents = _config_file_contents.replace(';','\n') _config_file_contents = _config_file_contents.replace("--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon(_cmd) if not _rsyslog_pid : _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents : if _line.count("UDPServerRun") : global_objects["logstore"]["port"] = _line.split()[1] _hostport = int(global_objects["logstore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def mysql_metricstore_setup(global_objects, operation="check"): _protocol = global_objects["metricstore"]["protocol"] _hostname = global_objects["metricstore"]["host"] _databaseid = global_objects["metricstore"]["database"] _timeout = float(global_objects["metricstore"]["timeout"]) _username = global_objects["mon_defaults"]["username"] _usage = global_objects["metricstore"]["usage"].lower() _hostport = int(global_objects["metricstore"]["mysql_port"]) try: if operation == "check": if _usage == "shared": _hostport = int(global_objects["metricstore"]["mysql_port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _mysql_pid = _proc_man.get_pid_from_cmdline("mysqld") if not _mysql_pid: _msg = "Unable to detect a shared Mysql server daemon running. " _msg += "Please try to start one." print(_msg) exit(8) else: _usage = "private" _config_file_fn = global_objects["space"][ "stores_working_dir"] + '/' + _username + "_mysqld.conf" _cmd = "mkdir -p " + global_objects["space"][ "stores_working_dir"] + "/logs; mysqld --defaults-file=" + _config_file_fn _proc_man = ProcessManagement(username=_username) _pid = _proc_man.get_pid_from_cmdline( "mysqld --defaults-file=" + _config_file_fn) if not _pid: _hostport = int( global_objects["metricstore"]["mysql_port"]) _config_file_contents = global_objects["metricstore"][ "mysql_config_string"] _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLUSER", _username) _config_file_contents = _config_file_contents.replace( "REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace( "--", '=') _config_file_contents = _config_file_contents.replace( '**', '-') _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _pid = _proc_man.start_daemon(_cmd) sleep(5) if not _pid: _msg = "Unable to detect a private MysqlDB server daemon running. " _msg += "You may need to issue $ sudo apt-get install apparmor-utils && sudo aa-complain /usr/bin/mysqld, followed by: " + _cmd + ")" print(_msg) exit(8) else: global_objects["metricstore"][ "mysql_port"] = _proc_man.get_port_from_pid(_pid[0]) _hostport = int( global_objects["metricstore"]["mysql_port"]) _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Metric Store of the kind \"Mysql\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + ", database id \"" + str( _databaseid) _msg += "\" seems to be running." cbdebug(_msg) _status = 0 _mmc = MysqlMgdConn(global_objects["metricstore"]) _mmc.initialize_metric_store(_username) _msg = "The Metric Store of the kind \"Mysql\" was successfully initialized " _msg += "on node: " + str( global_objects["metricstore"] ["host"]) + " " + _protocol + " port " + str(_hostport) cbdebug(_msg) _status = 0 return _status, _msg except ProcessManagement.ProcessManagementException as obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9) except NetworkException as obj: _msg = "A Metric Store of the kind \"Mysql\" on node " _msg += _hostname + ", " + _protocol + " port " + str(_hostport) _msg += ", database id \"" + str(_databaseid) + "\" seems to be down: " _msg += str(obj.msg) + '.' cberr(_msg) raise StoreSetupException(_msg, 8) except MysqlMgdConn.MetricStoreMgdConnException as obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9) except Exception as e: for line in traceback.format_exc().splitlines(): cberr(line, True) _status = 23 _msg = str(e) raise StoreSetupException(_msg, 9)
def syslog_logstore_setup(global_objects, operation = "check") : ''' TBD ''' _hostname = global_objects["logstore"]["hostname"] _protocol = global_objects["logstore"]["protocol"] _username = global_objects["logstore"]["username"] _usage = global_objects["logstore"]["usage"].lower() try : if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["logstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd") _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid -c 5" if not _rsyslog_pid : _msg = "Unable to detect a shared rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf" _cmd = "rsyslogd -f " + _config_file_fn + " -c 4 " + "-i " + global_objects["space"]["stores_working_dir"] + "/rsyslog.pid" _proc_man = ProcessManagement(username = _username) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid : global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"], protocol = "udp") _hostport = int(global_objects["logstore"]["port"]) _config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("DOLLAR", '$') _config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_") _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace(';','\n') _config_file_contents = _config_file_contents.replace("--", ';') _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon(_cmd) if not _rsyslog_pid : _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents : if _line.count("UDPServerRun") : global_objects["logstore"]["port"] = _line.split()[1] _hostport = int(global_objects["logstore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)