def redis_objectstore_setup(global_objects, operation, cloud_name=None):
'''
TBD
'''
_protocol = global_objects["objectstore"]["protocol"]
_hostname = global_objects["objectstore"]["host"]
_databaseid = int(global_objects["objectstore"]["dbid"])
_timeout = float(global_objects["objectstore"]["timout"])
_username = global_objects["objectstore"]["username"]
_usage = global_objects["objectstore"]["usage"].lower()
try:
_instance_dir = global_objects["space"]["instance_dir"]
if operation == "check":
_stores_path = global_objects["space"]["stores_working_dir"]
if not path.exists(_stores_path):
cbdebug("Making stores working directory: " + _stores_path)
mkdir(_stores_path)
if _usage == "shared":
_hostport = int(global_objects["objectstore"]["port"])
_proc_man = ProcessManagement(username="******")
if not pre_check_port(_hostname, _hostport, _protocol):
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
_cmd = "/usr/local/bin/redis-server /etc/redis.conf"
if not _redis_pid:
_msg = "Unable to detect a shared Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = _stores_path + '/' + _username + "_redis.conf"
_cmd = "redis-server " + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
if not _redis_pid:
global_objects["objectstore"][
"port"] = _proc_man.get_free_port(
global_objects["objectstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["objectstore"]["port"])
_config_file_contents = global_objects["objectstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_redis_pid = _proc_man.start_daemon(_cmd)
if not _redis_pid:
_msg = "Unable to detect a private Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["objectstore"][
"port"] = _proc_man.get_port_from_pid(_redis_pid[0])
_hostport = int(global_objects["objectstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \""
_msg += str(_databaseid) + "\" seems to be running."
_status = 0
else:
if not cloud_name:
raise StoreSetupException(
"Name of cloud is required for the 'initialize' mode", 22)
operation = "initialize"
_hostport = int(global_objects["objectstore"]["port"])
_collection_names = [ "reported_management_vm_metric_names", \
"reported_runtime_os_host_metric_names", \
"reported_runtime_os_vm_metric_names", \
"host_management_metrics_header", \
"vm_management_metrics_header", \
"host_runtime_os_metrics_header", \
"vm_runtime_os_metrics_header", \
"vm_runtime_app_metrics_header", \
"trace_header" ]
for _collection_name in _collection_names:
for _component in global_objects["mon_defaults"][
_collection_name].split(','):
if _component.lower() in global_objects["mon_defaults"]:
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name].replace(_component, \
global_objects["mon_defaults"][_component.lower()] + ',')
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')
_rmc = RedisMgdConn(global_objects["objectstore"])
# First we remove the leftovers from previous experiments.
if _rmc.initialize_object_store(cloud_name, global_objects, True):
if not path.exists(_instance_dir):
mkdir(_instance_dir)
for _file_name in listdir(_instance_dir):
_file_name = path.join(_instance_dir, _file_name)
if path.isdir(_file_name):
rmtree(_file_name)
_msg = "Folders (but not data) underneath experiment "
_msg += "directory " + _instance_dir + " were removed."
cbdebug(_msg)
_msg = "The Redis datastore was successfully initialized on server " + _hostname
_msg += ", port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\"."
cbdebug(_msg)
_status = 0
else:
_msg = "The Object Store of the kind \"Redis\" was successfully initialized "
_msg += "on node " + _hostname + ". To change its "
_msg += "attributes/state, use the *alter commands"
_msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
_msg += "and attach this cloud back to this experiment."
cbdebug(_msg)
_status = 0
return _status, _msg
except NetworkException, obj:
_msg = "An Object Store of the kind \"Redis\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid)
_msg += "\" seems to be down: " + str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
def mongodb_metricstore_setup(global_objects, operation="check"):
'''
TBD
'''
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
_cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
if not _mongodb_pid:
_msg = "Unable to detect a shared MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects[
"space"]["stores_working_dir"] + "/mongod.pid"
_proc_man = ProcessManagement(username=_username)
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
if not _mongodb_pid:
global_objects["metricstore"][
"port"] = _proc_man.get_free_port(
global_objects["metricstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["metricstore"]["port"])
_config_file_contents = global_objects["metricstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_mongodb_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _mongodb_pid:
_msg = "Unable to detect a private MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["metricstore"][
"port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
_hostport = int(global_objects["metricstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
else:
operation = "initialize"
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
_msg += "on node: " + str(global_objects["metricstore"])
cbdebug(_msg)
_status = 0
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def mongodb_metricstore_setup(global_objects, operation = "check") :
'''
TBD
'''
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
try :
if operation == "check" :
if _usage == "shared" :
_hostport = int(global_objects["metricstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol) :
_proc_man = ProcessManagement(username = "******")
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
_cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
if not _mongodb_pid :
_msg = "Unable to detect a shared MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
_cmd = "mkdir -p " + global_objects["space"]["stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects["space"]["stores_working_dir"] + "/mongod.pid"
_proc_man = ProcessManagement(username = _username)
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
if not _mongodb_pid :
global_objects["metricstore"]["port"] = _proc_man.get_free_port(global_objects["metricstore"]["port"], protocol = "tcp")
_hostport = int(global_objects["metricstore"]["port"])
_config_file_contents = global_objects["metricstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace("--", '=')
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_mongodb_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _mongodb_pid :
_msg = "Unable to detect a private MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
global_objects["metricstore"]["port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
_hostport = int(global_objects["metricstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
else:
operation = "initialize"
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
_msg += "on node: " + str(global_objects["metricstore"])
cbdebug(_msg)
_status = 0
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def redis_objectstore_setup(global_objects, operation, cloud_name = None) :
'''
TBD
'''
_protocol = global_objects["objectstore"]["protocol"]
_hostname = global_objects["objectstore"]["host"]
_databaseid = int(global_objects["objectstore"]["dbid"])
_timeout = float(global_objects["objectstore"]["timout"])
_username = global_objects["objectstore"]["username"]
_usage = global_objects["objectstore"]["usage"].lower()
try :
_instance_dir = global_objects["space"]["instance_dir"]
if operation == "check" :
_stores_path = global_objects["space"]["stores_working_dir"]
if not path.exists(_stores_path) :
cbdebug("Making stores working directory: " + _stores_path)
mkdir(_stores_path)
if _usage == "shared" :
_hostport = int(global_objects["objectstore"]["port"])
_proc_man = ProcessManagement(username = "******")
if not pre_check_port(_hostname, _hostport, _protocol) :
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
_cmd = "/usr/local/bin/redis-server /etc/redis.conf"
if not _redis_pid :
_msg = "Unable to detect a shared Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_config_file_fn = _stores_path + '/' + _username + "_redis.conf"
_cmd = "redis-server " + _config_file_fn
_proc_man = ProcessManagement(username = _username)
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
if not _redis_pid :
global_objects["objectstore"]["port"] = _proc_man.get_free_port(global_objects["objectstore"]["port"], protocol = "tcp")
_hostport = int(global_objects["objectstore"]["port"])
_config_file_contents = global_objects["objectstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_redis_pid = _proc_man.start_daemon(_cmd)
if not _redis_pid :
_msg = "Unable to detect a private Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
global_objects["objectstore"]["port"] = _proc_man.get_port_from_pid(_redis_pid[0])
_hostport = int(global_objects["objectstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \""
_msg += str(_databaseid) + "\" seems to be running."
_status = 0
else :
if not cloud_name :
raise StoreSetupException("Name of cloud is required for the 'initialize' mode", 22)
operation = "initialize"
_hostport = int(global_objects["objectstore"]["port"])
_collection_names = [ "reported_management_vm_metric_names", \
"reported_runtime_os_host_metric_names", \
"reported_runtime_os_vm_metric_names", \
"host_management_metrics_header", \
"vm_management_metrics_header", \
"host_runtime_os_metrics_header", \
"vm_runtime_os_metrics_header", \
"vm_runtime_app_metrics_header", \
"trace_header" ]
for _collection_name in _collection_names :
for _component in global_objects["mon_defaults"][_collection_name].split(',') :
if _component.lower() in global_objects["mon_defaults"] :
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name].replace(_component, \
global_objects["mon_defaults"][_component.lower()] + ',')
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')
_rmc = RedisMgdConn(global_objects["objectstore"])
# First we remove the leftovers from previous experiments.
if _rmc.initialize_object_store(cloud_name, global_objects, True) :
if not path.exists(_instance_dir) :
mkdir(_instance_dir)
for _file_name in listdir(_instance_dir) :
_file_name = path.join(_instance_dir, _file_name)
if path.isdir(_file_name) :
rmtree(_file_name)
_msg = "Folders (but not data) underneath experiment "
_msg += "directory " + _instance_dir + " were removed."
cbdebug(_msg)
_msg = "The Redis datastore was successfully initialized on server " + _hostname
_msg += ", port " + str(_hostport) + ", database id \"" + str(_databaseid)
_msg += "\"."
cbdebug(_msg)
_status = 0
else :
_msg = "The Object Store of the kind \"Redis\" was successfully initialized "
_msg += "on node " + _hostname + ". To change its "
_msg += "attributes/state, use the *alter commands"
_msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
_msg += "and attach this cloud back to this experiment."
cbdebug(_msg)
_status = 0
return _status, _msg
except NetworkException, obj :
_msg = "An Object Store of the kind \"Redis\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid)
_msg += "\" seems to be down: " + str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
def mysql_metricstore_setup(global_objects, operation="check"):
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
_hostport = int(global_objects["metricstore"]["mysql_port"])
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["mysql_port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mysql_pid = _proc_man.get_pid_from_cmdline("mysqld")
if not _mysql_pid:
_msg = "Unable to detect a shared Mysql server daemon running. "
_msg += "Please try to start one."
print(_msg)
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mysqld.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mysqld --defaults-file=" + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_pid = _proc_man.get_pid_from_cmdline(
"mysqld --defaults-file=" + _config_file_fn)
if not _pid:
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_config_file_contents = global_objects["metricstore"][
"mysql_config_string"]
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username)
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
'**', '-')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _pid:
_msg = "Unable to detect a private MysqlDB server daemon running. "
_msg += "You may need to issue $ sudo apt-get install apparmor-utils && sudo aa-complain /usr/bin/mysqld, followed by: " + _cmd + ")"
print(_msg)
exit(8)
else:
global_objects["metricstore"][
"mysql_port"] = _proc_man.get_port_from_pid(_pid[0])
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"Mysql\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
_mmc = MysqlMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"Mysql\" was successfully initialized "
_msg += "on node: " + str(
global_objects["metricstore"]
["host"]) + " " + _protocol + " port " + str(_hostport)
cbdebug(_msg)
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except NetworkException as obj:
_msg = "A Metric Store of the kind \"Mysql\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid) + "\" seems to be down: "
_msg += str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
except MysqlMgdConn.MetricStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except Exception as e:
for line in traceback.format_exc().splitlines():
cberr(line, True)
_status = 23
_msg = str(e)
raise StoreSetupException(_msg, 9)
