def check_mongo_binary(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking MongoDB version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"mongod --version")
if not _status and _result_stdout.count("db version"):
_version = "N/A"
for _word in _result_stdout.split():
if _word.count("v") and not _word.count("version"):
_version = _word.replace('v', '').replace(',', '')
break
_msg += compare_versions('2.0.0', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_rsyslogd_version(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking rsyslog version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"rsyslogd -v")
if not _status and _result_stdout.count("compiled with"):
_version = "N/A"
for _word in _result_stdout.split():
if _word.count(".") and not _word.count("//"):
_version = _word.replace(',', '')
break
_msg += compare_versions('4.6.0', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_redis_binary(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking Redis version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"redis-server -v")
if not _status and _result_stdout.count("Redis server"):
_version = "N/A"
for _word in _result_stdout.split():
if _word.count("v="):
_version = _word.replace("v=", '')
break
elif _word.count('.') == 2:
_version = _word
break
_msg += compare_versions('2.5.0', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_gmond_version(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking gmond version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"gmond --version")
if not _status and _result_stdout.count("gmond"):
for _word in _result_stdout.split():
if _word.count('.') >= 2:
_parts = _word.split(".")
_version = _parts[0] + "." + _parts[1]
break
_msg += compare_versions('3.0', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_rsync_version(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking rsync version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"rsync --version | grep version")
if not _status and _result_stdout.count("protocol"):
for _word in _result_stdout.split():
if _word.count('.') == 2:
_version = _word
break
_msg += compare_versions('2.6', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def additional_host_discovery(self, obj_attr_list):
'''
TBD
'''
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
try:
if "additional_discovery" in obj_attr_list:
if len(obj_attr_list["additional_discovery"]) > 1:
_proc_man = ProcessManagement(username = obj_attr_list["username"], \
cloud_name = obj_attr_list["cloud_name"])
_cmd = obj_attr_list["additional_discovery"]
_cmd = _cmd.replace("--", " --")
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_cmd)
_extra_attr_list = json.loads(_result_stdout)
for _host_uuid in obj_attr_list["hosts"].split(','):
if obj_attr_list["host_list"][_host_uuid][
"cloud_hostname"] in _extra_attr_list:
obj_attr_list["host_list"][_host_uuid].update(
_extra_attr_list[obj_attr_list["host_list"][
_host_uuid]["cloud_hostname"]])
_status = 0
else:
_status = 0
else:
_status = 0
except Exception, e:
_status = 23
_fmsg = str(e)
def vmdestroy(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list:
obj_attr_list[
"mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
self.common_messages("VM", obj_attr_list, "destroying", 0, '')
if obj_attr_list["cloud_ip"] != "undefined" and obj_attr_list[
"role"] != "check":
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["cloud_ip"], \
priv_key = obj_attr_list["identity"])
_cmd = "~/cb_cleanup.sh; rm ~/cb_*"
_msg = "Shutting down CloudBench Load Manager/Metric Aggregator on "
_msg += "VM \"" + obj_attr_list["name"] + "\" by executing the "
_msg += "command \"" + _cmd + "\""
cbdebug(_msg, True)
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_cmd)
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
_status = 0
except CldOpsException as obj:
_status = obj.status
_fmsg = str(obj.msg)
except Exception as e:
_status = 23
_fmsg = str(e)
finally:
_status, _msg = self.common_messages("VM", obj_attr_list,
"destroyed", _status, _fmsg)
return _status, _msg
def instance_preparation(hostname, depsdict, options) :
'''
TBD
'''
try :
_status = 100
_store = False
_process_manager = False
_cleanup = False
_fmsg = "An error has occurred, but no error message was captured"
if "stores" in depsdict :
for _store in depsdict["stores"] :
_process_manager = ProcessManagement(hostname)
_cmd = depsdict[_store + "_command_check"]
_msg = "Checking accesss from this instance to the CB"
_msg += " Orchestrator's " + _store.capitalize() + " with"
_msg += " \"" + _cmd + "\"..."
cbinfo(_msg)
_status, _x, _y = _process_manager.run_os_command(_cmd)
_msg = "This instance was able to access CB Orchestrator's " + _store.capitalize()
_msg += " with \"" + _cmd + "\" (OK)"
cbinfo(_msg)
if _store.lower() == "filestore" :
print '\n'
_msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb"
print _msg
print '\n'
_msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"]
print _msg
print '\n'
if str(options.addr) != "bypass" :
_cmd = options.wksdir + "/common/cb_cleanup.sh"
_msg = "Running the instance cleanup script \"" + _cmd + "\"..."
cbinfo(_msg)
_process_manager.run_os_command(_cmd)
_cleanup = True
_status = 0
except Exception, e :
_status = 23
_fmsg = str(e)
def configure_port_mapping(self, obj_attr_list, operation):
'''
TBD
'''
_status = 189
_fmsg = "About to configure port mapping"
# LXD does not provide an automated method to expose specific ports
# directly through the host's IP, like Docker does. For now, will
# resort to ssh into the host and start a new "rinetd" instance each
# time a new vmattach is issued.
try:
_proc_man = ProcessManagement(username = "******", \
hostname = obj_attr_list["host_cloud_ip"], \
cloud_name = obj_attr_list["cloud_name"])
if operation == "setup":
_cmd = "echo \"0.0.0.0 " + obj_attr_list[
"prov_cloud_port"] + ' '
_cmd += obj_attr_list["cloud_ip"] + " 22\" > /tmp/"
_cmd += obj_attr_list[
"cloud_vm_name"] + ".rinetd.conf; rinetd -c "
_cmd += "/tmp/" + obj_attr_list[
"cloud_vm_name"] + ".rinetd.conf"
_rexcpt = True
else:
_cmd = "sudo pkill -9 -f 'rinetd -c /tmp/" + obj_attr_list[
"cloud_vm_name"]
_cmd += ".rinetd.conf" + "'; sudo rm -rf /tmp/"
_cmd += obj_attr_list["cloud_vm_name"] + ".rinetd.conf"
_rexcpt = False
_msg = operation.capitalize(
) + " port mapping (" + obj_attr_list["prov_cloud_port"]
_msg += " -> 22) for " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "running on LXD host \"" + obj_attr_list["host_name"] + "\""
cbdebug(_msg, True)
_status, _result_stdout, _fmsg = _proc_man.run_os_command(
_cmd, raise_exception=_rexcpt)
_status = 0
except ProcessManagement.ProcessManagementException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def vmdestroy(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list:
obj_attr_list[
"mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
_msg = "Sending a termination request for " + obj_attr_list[
"name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["cloud_ip"], \
priv_key = obj_attr_list["identity"])
_cmd = "~/cb_cleanup.sh; rm ~/cb_*"
_msg = "Shutting down CloudBench Load Manager/Metric Aggregator on "
_msg += "VM \"" + obj_attr_list["name"] + "\" by executing the "
_msg += "command \"" + _cmd + "\""
cbdebug(_msg, True)
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_cmd)
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
_status = 0
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def check_ifconfig_utility(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking \"ifconfig\" utility....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"ifconfig")
if not _status:
_msg += "OK"
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_bootstrap(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking bootstrap version....."
if access(path[0] + "/3rd_party/bootstrap/package.json", F_OK):
_version = "1.0.0"
_msg += compare_versions('1.0.0', _version)
_status = 0
else:
_status = 1728289
except Exception, e:
_status = 23
def check_passwordless_sudo(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking passwordless sudo for the user \"" + username + "\" ....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"sudo -S ls < /dev/null")
if not _status:
_msg += "Passwordless sudo checked OK"
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_netcat(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking netcat (openbsd) version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"nc -v -w 1 localhost -z 22")
if not _status:
_version = "1.9"
_msg += compare_versions('1.6', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_git_version(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking git version....."
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
"git --version")
if not _status and _result_stdout.count("git version"):
_version = _result_stdout.replace("git version ", '').strip()
_msg += compare_versions('1.6.0', _version)
_status = 0
else:
_status = 1728289
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
def check_custom_gmetad(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_proc_man = ProcessManagement()
_msg = "Checking custom gmetad version....."
if access(path[0] + "/3rd_party/monitor-core/gmetad-python/gmetad.py",
F_OK):
_version = "1.0.0"
_msg += compare_versions('1.0.0', _version)
_status = 0
else:
_status = 1728289
except Exception, e:
_status = 23
print _msg
exit(4)
else:
_msg = "File \"" + _file + "\" IGNORED...."
print _msg
if not len(depsdict):
_msg = "Error: None of the files on the list \"" + str(_file_name_list)
_msg += "\" contained configuration statements"
print _msg
exit(9)
if _cleanup_repos:
if not process_manager:
process_manager = ProcessManagement(hostname)
process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False)
return True
def get_linux_distro():
'''
TBD
'''
_linux_distro_name, _linux_distro_ver, _x = platform.linux_distribution()
if _linux_distro_name.count("Red Hat"):
_distro = "rhel"
elif _linux_distro_name.count("Scientific Linux"):
_distro = "rhel"
def vmccleanup(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
sleep(_wait)
self.common_messages("VMC", obj_attr_list, "cleaning up vms", 0,
'')
_running_instances = True
while _running_instances and _curr_tries < _max_tries:
_running_instances = False
for _endpoint in self.lxdconn:
_proc_man = ProcessManagement(username = "******", \
hostname = _endpoint, \
cloud_name = obj_attr_list["cloud_name"])
_cmd = "sudo pkill -9 -f 'rinetd -c /tmp/cb'; sudo rm -rf /tmp/cb-*.rinetd.conf"
_status, _result_stdout, _fmsg = _proc_man.run_os_command(
_cmd, raise_exception=False)
_container_list = self.lxdconn[_endpoint].containers.all()
for _container in _container_list:
if _container.name.count("cb-" +
obj_attr_list["username"] +
'-' +
obj_attr_list["cloud_name"]):
_running_instances = True
_msg = "Terminating instance: "
_msg += self.generate_random_uuid(
_container.name) + " (" + str(
_container.name) + ")"
cbdebug(_msg, True)
if _container.status == "Running":
_container.stop()
_container.delete()
sleep(_wait)
_curr_tries += 1
if _curr_tries > _max_tries:
_status = 1077
_fmsg = "Some instances on VMC \"" + obj_attr_list[
"name"] + "\""
_fmsg += " could not be removed because they never became active"
_fmsg += ". They will have to be removed manually."
cberr(_msg, True)
else:
_status = 0
except CldOpsException as obj:
_status = obj.status
_fmsg = str(obj.msg)
except LXDError.LXDAPIException as obj:
_status = 18127
_fmsg = str(obj)
except Exception as e:
_status = 23
_fmsg = str(e)
finally:
_status, _msg = self.common_messages("VMC", obj_attr_list,
"cleaned up", _status, _fmsg)
return _status, _msg
def take_action_if_requested(self, obj_type, obj_attr_list, current_step):
'''
TBD
'''
if "staging" not in obj_attr_list:
return
if not obj_attr_list["staging"].count(current_step):
return
if obj_attr_list["staging"] + "_complete" in obj_attr_list:
return
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
if obj_attr_list["staging"] == "pause_" + current_step:
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
# Always subscribe for the VM channel, no matter the object
_sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"],
"VM", "staging",
_max_tries * _wait)
if obj_type == "VM" and obj_attr_list[
"ai"] != "none" and current_step.count("all_vms"):
_target_uuid = obj_attr_list["ai"]
_target_name = obj_attr_list["ai_name"]
_cloud_vm_uuid = obj_attr_list["cloud_vm_uuid"]
else:
_target_uuid = obj_attr_list["uuid"]
_target_name = obj_attr_list["name"]
_cloud_vm_uuid = _target_uuid
self.osci.publish_message(obj_attr_list["cloud_name"], \
obj_type, \
"staging", \
_target_uuid + ";vmready;" + dic2str(obj_attr_list),\
1, \
3600)
_msg = obj_type + ' ' + _cloud_vm_uuid + " ("
_msg += _target_name + ") pausing on attach for continue signal ...."
cbdebug(_msg, True)
for _message in _sub_channel.listen():
_args = str(_message["data"]).split(";")
if len(_args) != 3:
# cbdebug("Message is not for me: " + str(_args))
continue
_id, _status, _info = _args
if (_id == _target_uuid
or _id == _target_name) and _status == "continue":
obj_attr_list[obj_attr_list["staging"] +
"_complete"] = int(time())
if _info.count(":"):
_add_obj_attr_list = str2dic(_info)
obj_attr_list.update(_add_obj_attr_list)
_status = 0
break
_sub_channel.unsubscribe()
_status = 0
elif obj_attr_list["staging"] == "execute_" + current_step:
_proc_man = ProcessManagement(username = obj_attr_list["username"], \
cloud_name = obj_attr_list["cloud_name"])
_json_contents = copy.deepcopy(obj_attr_list)
if obj_type == "AI":
_json_contents["vms"] = {}
_vm_id_list = obj_attr_list["vms"].split(',')
for _vm_id in _vm_id_list:
_vm_uuid = _vm_id.split('|')[0]
_vm_attr_list = self.osci.get_object(
obj_attr_list["cloud_name"], "VM", False, _vm_uuid,
False)
_json_contents["vms"][
_vm_attr_list["uuid"]] = _vm_attr_list
obj_attr_list["execute_json_filename"] = "/tmp/"
obj_attr_list["execute_json_filename"] += obj_attr_list[
"execute_json_filename_prefix"]
obj_attr_list[
"execute_json_filename"] += "_vapp_" + obj_attr_list[
"cloud_name"]
obj_attr_list["execute_json_filename"] += "_" + obj_attr_list[
"name"] + "_"
obj_attr_list[
"execute_json_filename"] += obj_attr_list["uuid"] + ".json"
_json_fh = open(obj_attr_list["execute_json_filename"], 'w')
_json_fh.write(
json.dumps(_json_contents, sort_keys=True, indent=4))
_json_fh.close()
_msg = "JSON contents written to "
_msg += obj_attr_list["execute_json_filename"] + '.'
cbdebug(_msg, True)
_cmd = obj_attr_list["execute_script_name"] + ' '
_cmd += obj_attr_list["execute_json_filename"]
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_cmd)
_msg = "Command \"" + _cmd + "\" executed, with return code " + str(
_status)
cbdebug(_msg, True)
obj_attr_list[obj_attr_list["staging"] + "_complete"] = int(time())
except self.osci.ObjectStoreMgdConnException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def reset(global_objects, soft=True, cloud_name=None):
'''
TBD
'''
try:
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
_username = global_objects["space"]["username"]
_logstore_username = global_objects["logstore"]["username"]
_filestore_username = global_objects["filestore"]["username"]
_filestore_config_file_fn = _stores_wk_dir + '/' + _filestore_username + "_rsync.conf"
_msg = " Killing all processes..."
print(_msg, end=' ')
_proc_man = ProcessManagement()
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cbact")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-api")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-gui")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f ai-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f vm-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f submit-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f capture-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f gmetad.py")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f gtkCBUI_")
print("done")
_proc_man.run_os_command("screen -wipe")
_proc_man.run_os_command(
"rm -rf " +
global_objects["space"]["generated_configurations_dir"] + '/' +
_username + "_cb_lastcloudrc")
_proc_man.run_os_command("rm -rf /tmp/restart_cb*" +
global_objects["logstore"]["username"] + '*')
_proc_man.run_os_command("rm -rf /tmp/" + _username +
"_*-*-*-*-*_avg_acc")
_msg = " Flushing Object Store..."
print(_msg, end=' ')
_rmc = RedisMgdConn(global_objects["objectstore"])
'''
If and only if the object store is being shared and a default 'STARTUP_CLOUD' or cloud on
the command line has been provided,
then we can selectively flush the database
instead of wholly dropping it.
'''
if global_objects["objectstore"]["usage"].lower() == "private":
# Backwards-compatible behavior
cloud_name = None
_rmc.flush_object_store(cloud_name)
print("done")
_msg = " Flushing Log Store..."
print(_msg, end=' ')
if global_objects["logstore"]["usage"].lower() != "shared":
_proc_man.run_os_command("pkill -9 -u " + _logstore_username +
" -f rsyslogd")
_file_list = []
_file_list.append("operations.log")
_file_list.append("report.log")
_file_list.append("submmiter.log")
_file_list.append("loadmanager.log")
_file_list.append("gui.log")
_file_list.append("remotescripts.log")
_file_list.append("messages.log")
_file_list.append("monitor.log")
_file_list.append("subscribe.log")
for _fn in _file_list:
if global_objects["logstore"]["usage"].lower() != "shared":
_proc_man.run_os_command("rm -rf " + _log_dir + '/' +
_logstore_username + '_' + _fn)
_proc_man.run_os_command("touch " + _log_dir + '/' +
_logstore_username + '_' + _fn)
_status, _msg = syslog_logstore_setup(global_objects, "check")
global_objects["logstore"]["just_restarted"] = True
print("done")
#_msg = "Flushing File Store..."
#print _msg,
#_proc_man.run_os_command("sudo pkill -9 -u root -f \"rsync --daemon --config " + _filestore_config_file_fn +"\"", raise_exception = False)
#_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _filestore_username + "_rsyncd.pid")
#_status, _msg = rsync_filestore_setup(global_objects, "check")
#print "done"
if not soft:
_msg = " Flushing Metric Store..."
print(_msg, end=' ')
_mmc = load_metricstore_adapter(global_objects["metricstore"])
_mmc.flush_metric_store(global_objects["mon_defaults"]["username"])
print("done")
print('\n')
_msg = ""
_status = 0
except ProcessManagement.ProcessManagementException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
except MongodbMgdConn.MetricStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
except MysqlMgdConn.MetricStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
except RedisMgdConn.ObjectStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
except Exception as e:
_status = 23
_msg = str(e)
finally:
return _status, _msg
def vmdestroy(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
if "prov_cloud_ip" in obj_attr_list :
try :
_space_attr_list = self.osci.get_object(obj_attr_list["cloud_name"], "GLOBAL", False, "space", False)
_data_dir = _space_attr_list["data_working_dir"] + "/" + obj_attr_list["experiment_id"]
if not access(_data_dir, F_OK) :
makedirs(_data_dir)
_cmd = "scp -i "
_cmd += _space_attr_list["ssh_key_name"]
_cmd += " " + obj_attr_list["login"] + "@" + obj_attr_list["prov_cloud_ip"]
_cmd += ":/home/" + obj_attr_list["login"] + "/nmon/nmon.csv "
_cmd += _data_dir + "/nmon_" + obj_attr_list["cloud_vm_name"] + ".csv"
print _cmd
_proc_man = ProcessManagement()
_proc_man.run_os_command(_cmd)
except Exception, msg :
cbdebug(str(msg), True)
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list :
obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs
if not self.cskconn :
self.connect(obj_attr_list["access"], obj_attr_list["api_key"], \
obj_attr_list["secret_key"])
_zoneid = self.get_zone(obj_attr_list["vmc_name"])
_wait = int(obj_attr_list["update_frequency"])
_instance = self.get_vm_instance(obj_attr_list)
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
if _instance :
_msg = "Sending a termination request for " + obj_attr_list["name"] + ""
_msg += " (instance id " + _instance[u'id'] + ")"
_msg += "...."
cbdebug(_msg, True)
self.cskconn.destroyVirtualMachine({u'id': _instance[u'id']})
sleep(_wait)
while self.is_vm_running(obj_attr_list) :
sleep(_wait)
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
else :
True
# This needs to be changed later. I could not find an easy way to
# find the actual volume id of a given instance
_volumes = self.cskconn.listVolumes({u'zoneid': _zoneid})
if _volumes and len(_volumes) :
for unattachedvol in _volumes :
if unattachedvol[u'state'] == u'Allocated' :
_msg = unattachedvol[u'id'] + ' ' + unattachedvol[u'state']
_msg += "... was deleted"
cbdebug(_msg)
self.cskconn.deleteVolume({u'id':unattachedvol[u'id']})
else:
_msg = unattachedvol[u'id'] + ' ' + unattachedvol[u'state']
_msg += "... still attached and could not be deleted"
cbdebug(_msg)
else :
_msg = "No volumes to remove"
cbdebug(_msg)
_status = 0
def redis_objectstore_setup(global_objects, operation, cloud_name=None):
'''
TBD
'''
_protocol = global_objects["objectstore"]["protocol"]
_hostname = global_objects["objectstore"]["host"]
_databaseid = int(global_objects["objectstore"]["dbid"])
_timeout = float(global_objects["objectstore"]["timout"])
_username = global_objects["objectstore"]["username"]
_usage = global_objects["objectstore"]["usage"].lower()
try:
_instance_dir = global_objects["space"]["instance_dir"]
if operation == "check":
_stores_path = global_objects["space"]["stores_working_dir"]
if not path.exists(_stores_path):
cbdebug("Making stores working directory: " + _stores_path)
mkdir(_stores_path)
if _usage == "shared":
_hostport = int(global_objects["objectstore"]["port"])
_proc_man = ProcessManagement(username="******")
if not pre_check_port(_hostname, _hostport, _protocol):
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
_cmd = "/usr/local/bin/redis-server /etc/redis.conf"
if not _redis_pid:
_msg = "Unable to detect a shared Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = _stores_path + '/' + _username + "_redis.conf"
_cmd = "redis-server " + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
if not _redis_pid:
global_objects["objectstore"][
"port"] = _proc_man.get_free_port(
global_objects["objectstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["objectstore"]["port"])
_config_file_contents = global_objects["objectstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_redis_pid = _proc_man.start_daemon(_cmd)
if not _redis_pid:
_msg = "Unable to detect a private Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["objectstore"][
"port"] = _proc_man.get_port_from_pid(_redis_pid[0])
_hostport = int(global_objects["objectstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \""
_msg += str(_databaseid) + "\" seems to be running."
_status = 0
else:
if not cloud_name:
raise StoreSetupException(
"Name of cloud is required for the 'initialize' mode", 22)
operation = "initialize"
_hostport = int(global_objects["objectstore"]["port"])
_collection_names = [ "reported_management_vm_metric_names", \
"reported_runtime_os_host_metric_names", \
"reported_runtime_os_vm_metric_names", \
"host_management_metrics_header", \
"vm_management_metrics_header", \
"host_runtime_os_metrics_header", \
"vm_runtime_os_metrics_header", \
"vm_runtime_app_metrics_header", \
"trace_header" ]
for _collection_name in _collection_names:
for _component in global_objects["mon_defaults"][
_collection_name].split(','):
if _component.lower() in global_objects["mon_defaults"]:
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name].replace(_component, \
global_objects["mon_defaults"][_component.lower()] + ',')
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')
_rmc = RedisMgdConn(global_objects["objectstore"])
# First we remove the leftovers from previous experiments.
if _rmc.initialize_object_store(cloud_name, global_objects, True):
if not path.exists(_instance_dir):
mkdir(_instance_dir)
for _file_name in listdir(_instance_dir):
_file_name = path.join(_instance_dir, _file_name)
if path.isdir(_file_name):
rmtree(_file_name)
_msg = "Folders (but not data) underneath experiment "
_msg += "directory " + _instance_dir + " were removed."
cbdebug(_msg)
_msg = "The Redis datastore was successfully initialized on server " + _hostname
_msg += ", port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\"."
cbdebug(_msg)
_status = 0
else:
_msg = "The Object Store of the kind \"Redis\" was successfully initialized "
_msg += "on node " + _hostname + ". To change its "
_msg += "attributes/state, use the *alter commands"
_msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
_msg += "and attach this cloud back to this experiment."
cbdebug(_msg)
_status = 0
return _status, _msg
except NetworkException, obj:
_msg = "An Object Store of the kind \"Redis\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid)
_msg += "\" seems to be down: " + str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
def mongodb_metricstore_setup(global_objects, operation="check"):
'''
TBD
'''
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
_cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
if not _mongodb_pid:
_msg = "Unable to detect a shared MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects[
"space"]["stores_working_dir"] + "/mongod.pid"
_proc_man = ProcessManagement(username=_username)
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
if not _mongodb_pid:
global_objects["metricstore"][
"port"] = _proc_man.get_free_port(
global_objects["metricstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["metricstore"]["port"])
_config_file_contents = global_objects["metricstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_mongodb_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _mongodb_pid:
_msg = "Unable to detect a private MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["metricstore"][
"port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
_hostport = int(global_objects["metricstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
else:
operation = "initialize"
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
_msg += "on node: " + str(global_objects["metricstore"])
cbdebug(_msg)
_status = 0
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def execute_command(operation,
depkey,
depsdict,
hostname="127.0.0.1",
username=None,
process_manager=None):
'''
TBD
'''
try:
_status = 100
_msg = "Obtaining command to be executed...."
_cmd = {}
_cmd["configure"] = get_cmdline(depkey, depsdict, "configure")
_cmd["install"] = get_cmdline(depkey, depsdict, "install")
if not process_manager:
process_manager = ProcessManagement(hostname)
_order = depsdict[depkey + "-order"]
if depkey != "sudo" and operation == "configure":
_msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \""
_msg += _cmd[operation] + "\"..."
elif depkey == "sudo" and operation == "configure":
_msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" "
_msg += "by executing the command \"" + _cmd[operation] + "\"..."
else:
_msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \""
_msg += _cmd[operation] + "\"..."
print _msg
_msg = "RESULT: "
if depkey == "repo" and operation == "install":
build_repository_files(depsdict)
_status, _result_stdout, _result_stderr = process_manager.run_os_command(
_cmd[operation], False)
if not _status:
if operation == "install":
_msg += "DONE OK.\n"
else:
_msg += compare_versions(depkey, depsdict,
_result_stdout.strip())
else:
print _result_stderr
_msg += "NOT OK. "
if _msg.count("NOT OK"):
_status = 701
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_result_stderr = str(obj.msg)
_msg += "NOT OK. "
def syslog_logstore_setup(global_objects, operation="check"):
'''
TBD
'''
_hostname = global_objects["logstore"]["hostname"]
_protocol = global_objects["logstore"]["protocol"]
_username = global_objects["logstore"]["username"]
_usage = global_objects["logstore"]["usage"].lower()
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try:
_name, _ip = hostname2ip(_hostname)
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["logstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
_cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid "
if not _rsyslog_pid:
_msg = "Unable to detect a shared rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_proc_man = ProcessManagement(username=_username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid"
if not access(_config_file_fn, F_OK):
# File was deleted, but the rsyslog process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
if not access(_log_dir, W_OK):
# The directory does not even exist, kill any rsyslog processes still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
_proc_man.run_os_command("sudo mkdir -p " + _log_dir +
" && sudo chmod 777 " + _log_dir)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid:
global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\
protocol = "udp")
_hostport = int(global_objects["logstore"]["port"])
_config_file_contents = global_objects["logstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"DOLLAR", '$')
_config_file_contents = _config_file_contents.replace(
"RSYSLOG", "RSYSLOG_")
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username + '_')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_contents = _config_file_contents.replace(
"--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon(_cmd)
if not _rsyslog_pid:
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents:
if _line.count("UDPServerRun"):
global_objects["logstore"]["port"] = _line.split(
)[1]
_hostport = int(global_objects["logstore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsyslog "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc):
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
_network_reachable = False
_curr_tries = 0
if not _network_reachable:
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"]
if str(obj_attr_list["use_jumphost"]).lower() == "false":
_msg += "..."
else:
_msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..."
obj_attr_list["check_boot_complete"] = "run_command_/bin/true"
cbdebug(_msg, True)
self.pending_set(obj_attr_list, _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries:
_start_pooling = int(time())
if "async" not in obj_attr_list or str(
obj_attr_list["async"]).lower() == "false":
if threading.current_thread().abort:
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_"):
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list[
"check_boot_complete"].replace("tcp_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(
int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count("cloud_ping"):
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish network connectivity "
_msg += "through the cloud's API"
cbdebug(_msg)
_vm_is_booted = self.is_vm_alive(obj_attr_list)
elif obj_attr_list["check_boot_complete"].count(
"subscribe_on_"):
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list[
"check_boot_complete"].replace("subscribe_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list[
"cloud_vm_uuid"] + ") has booted by "
_msg += "subscribing to channel \"" + str(
_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM",
"VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(
obj_attr_list["cloud_name"], "VM",
_channel_to_subscribe, _max_tries * _wait)
for _message in _sub_channel.listen():
if str(_message["data"]).count(_string_to_search):
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"],
"VM", "VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_"):
_boot_wait_time = int(
obj_attr_list["check_boot_complete"].replace(
"wait_for_", ''))
_msg = "Assuming that the VM \"" + obj_attr_list[
"cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(
_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time:
sleep(_boot_wait_time)
_vm_is_booted = True
elif obj_attr_list["check_boot_complete"].count(
"run_command_"):
_command_to_run = obj_attr_list[
"check_boot_complete"].replace("run_command_", '')
_command_to_run = _command_to_run.replace("____", ' ')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list[
"cloud_vm_uuid"] + ") has booted by "
_msg += "running the command \"" + str(_command_to_run)
cbdebug(_msg)
if _curr_tries <= _max_tries / 3:
_connection_timeout = int(
obj_attr_list["update_frequency"]) / 2
elif _curr_tries > _max_tries / 3 and _curr_tries < 2 * _max_tries / 3:
_connection_timeout = int(
obj_attr_list["update_frequency"])
obj_attr_list[
"comments"] += "Had to increase ssh timeout. "
else:
_connection_timeout = int(
obj_attr_list["update_frequency"]) * 2
obj_attr_list[
"comments"] += "Had to increase ssh timeout one more time. "
if str(obj_attr_list["use_jumphost"]).lower() == "true":
if "ssh_config_file" in obj_attr_list:
_ssh_conf_file = obj_attr_list["ssh_config_file"]
else:
_ssh_conf_file = None
else:
_ssh_conf_file = None
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["prov_cloud_ip"], \
priv_key = obj_attr_list["identity"], \
config_file = _ssh_conf_file,
connection_timeout = _connection_timeout)
try:
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_command_to_run)
if not _status:
_vm_is_booted = True
else:
_vm_is_booted = False
except:
_vm_is_booted = False
elif obj_attr_list["check_boot_complete"].count(
"snmpget_poll"):
import netsnmp
# Send SNMP GET message. Flag VM as booted if any response at all is recieved
_vm_is_booted = False
try:
_msg = "Opening SNMP session to " + obj_attr_list[
"cloud_ip"]
cbdebug(_msg)
_snmp_wait_time = _wait * 1000000
_snmp_version = int(obj_attr_list["snmp_version"])
_snmp_comm = str(obj_attr_list["snmp_community"])
_snmp_session = netsnmp.Session(Version=_snmp_version, \
DestHost=obj_attr_list["cloud_ip"], \
Community=_snmp_comm, \
Timeout=_snmp_wait_time, Retries=0)
_vars = netsnmp.VarList(
netsnmp.Varbind(obj_attr_list["snmp_variable"],
'0'))
_snmp_response = _snmp_session.get(_vars)
except:
if _snmp_session.ErrorStr:
_msg = "Error in SNMP handler : " + _snmp_session.ErrorStr
else:
_msg = "Unknown error in SNMP handler."
cbdebug(_msg)
_status = 200
raise CldOpsException(_msg, _status)
if (_snmp_response[0] != None):
_vm_is_booted = True
_msg = "SNMP Response: " + str(_snmp_response)
cbdebug(_msg)
else:
_vm_is_booted = False
_msg = "Warning: No valid method specified to determined if VM has booted."
cbdebug(_msg, True)
_pooling_time = int(time()) - _start_pooling
if _pooling_time <= _wait:
_actual_wait = _wait - _pooling_time
else:
_msg = "The time spent on pooling for \"booted\" status (" + str(
_pooling_time)
_msg += " s) is actually longer than the "
_msg += "interval between pooling attempts (" + str(
_wait) + " s)."
cbdebug(_msg, True)
_actual_wait = 0
if _vm_is_booted:
obj_attr_list["mgt_004_network_acessible"] = int(
time()) - time_mark_prc
self.pending_set(obj_attr_list,
"Network accessible now. Continuing...")
_network_reachable = True
break
else:
_msg = "(" + str(
_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(
_actual_wait)
_msg += " seconds and check again."
self.pending_set(obj_attr_list, _msg)
cbdebug(_msg)
sleep(_actual_wait)
_curr_tries += 1
if _curr_tries < _max_tries:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006 and mgt_007 NOT mgt_005
obj_attr_list["mgt_006_instance_preparation"] = "0"
obj_attr_list["mgt_007_application_start"] = "0"
self.pending_set(obj_attr_list, "Application starting up...")
self.get_attr_from_pending(obj_attr_list, "all")
else:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "is not network reachable after " + str(
_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
def rsync_filestore_setup(global_objects, operation="check"):
'''
TBD
'''
_hostname = global_objects["filestore"]["hostname"]
_protocol = global_objects["filestore"]["protocol"]
_username = global_objects["filestore"]["username"]
_port = global_objects["filestore"]["port"]
_usage = global_objects["filestore"]["usage"].lower()
_base_dir = global_objects["space"]["base_dir"]
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try:
_name, _ip = hostname2ip(_hostname)
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["filestore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_rsync_pid = _proc_man.get_pid_from_cmdline(
"rsync --daemon")
_cmd = "rsync --daemon"
if not _rsync_pid:
_msg = "Unable to detect a shared rsync server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_proc_man = ProcessManagement(username=_username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_cmd = "rsync --daemon --config " + _config_file_fn
if not access(_config_file_fn, F_OK):
# File was deleted, but the rsync process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid:
_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir +
'/' + _username + "_rsyncd.pid")
global_objects["filestore"][
"port"] = _proc_man.get_free_port(
global_objects["filestore"]["port"],
protocol="tcp")
_hostport = int(global_objects["filestore"]["port"])
_config_file_contents = global_objects["filestore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"DOLLAR", '$')
_config_file_contents = _config_file_contents.replace(
"REPLEQUAL", '=')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace(
"REPLUSERU", _username + '_')
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username)
_config_file_contents = _config_file_contents.replace(
"REPLBASEDIR", _base_dir)
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_contents = _config_file_contents.replace(
"--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd)
if not _rsyslog_pid:
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents:
if _line.count("port="):
global_objects["filestore"]["port"] = _line.split(
'=')[1]
_hostport = int(
global_objects["filestore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A File Store of the kind \"rsync\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsync "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsync"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def expand_command(cmdline, depsdict, process_manager=False):
'''
TBD
'''
_actual_cmdline = ''
for _command in cmdline.split(';'):
if _command.count("package_install"):
_packages = _command.replace("package_install", '')
if depsdict["cdistkind"] == "ubuntu":
if _packages.count(".deb"):
_command = "sudo dpkg -i PACKAGES; sudo apt-get -f install -y --force-yes --allow-unauthenticated "
else:
_command = "sudo apt-get -q -y --force-yes --allow-unauthenticated -o Dpkg::Options::=\"--force-confnew\" install PACKAGES"
elif depsdict["cdistkind"] == "rhel" or depsdict[
"cdistkind"] == "fedora":
if _packages.count(".rpm"):
_command = "sudo rpm -i PACKAGES"
else:
_command = "sudo yum -y install PACKAGES"
_command = _command.replace("PACKAGES", _packages)
if _command.count("service_restart_enable") or _command.count(
"service_stop_disable"):
if not process_manager:
process_manager = ProcessManagement()
if _command.count("service_restart_enable"):
_services = _command.replace("service_restart_enable", '')
if _command.count("service_stop_disable"):
_services = _command.replace("service_stop_disable", '')
_command = ''
for _service in _services.split():
_status, _result_stdout, _result_stderr = \
process_manager.run_os_command("sudo systemctl status " + _service, \
raise_exception = False)
if not _status:
if cmdline.count("service_restart_enable"):
_service_cmd = "sudo systemctl restart SERVICE; sudo systemctl enable SERVICE"
if cmdline.count("service_stop_disable"):
_service_cmd = "sudo systemctl stop SERVICE; sudo systemctl disable SERVICE"
else:
if cmdline.count("service_restart_enable"):
_service_cmd = "sudo service SERVICE restart;"
if depsdict["cdistkind"] == "rhel" or depsdict[
"cdistkind"] == "fedora":
_service_cmd += "sudo chkconfig SERVICE on;"
elif depsdict["cdistkind"] == "ubuntu":
if os.access("/etc/init/" + _service + ".conf",
os.F_OK):
_service_cmd += "sudo rm -rf /etc/init/SERVICE.override"
else:
_service_cmd += "sudo update-rc.d -f SERVICE defaults"
if cmdline.count("service_stop_disable"):
_service_cmd = "sudo systemctl SERVICE stop; sudo chkconfig SERVICE off;"
if depsdict["cdistkind"] == "rhel" or depsdict[
"cdistkind"] == "fedora":
_service_cmd = "sudo chkconfig SERVICE off;"
elif depsdict["cdistkind"] == "ubuntu":
if os.access("/etc/init/" + _service + ".conf",
os.F_OK):
_service_cmd = "sudo sh -c 'echo manual > /etc/init/SERVICE.override'"
else:
_service_cmd = "sudo update-rc.d -f SERVICE remove"
_service_cmd = _service_cmd.replace("SERVICE", _service) + ';'
_command += _service_cmd
_actual_cmdline += _command + ';'
return _actual_cmdline
def reset(global_objects, soft=True):
'''
TBD
'''
try:
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
_username = global_objects["space"]["username"]
_logstore_username = global_objects["logstore"]["username"]
_filestore_username = global_objects["filestore"]["username"]
_filestore_config_file_fn = _stores_wk_dir + '/' + _filestore_username + "_rsync.conf"
_msg = "Killing all processes..."
print _msg,
_proc_man = ProcessManagement()
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cbact")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-api")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-gui")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f ai-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f vm-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f submit-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f capture-")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f gmetad.py")
_proc_man.run_os_command("pkill -9 -u " + _username + " -f gtkCBUI_")
print "done"
_proc_man.run_os_command("screen -wipe")
_proc_man.run_os_command("rm -rf /tmp/restart_cb*" +
global_objects["logstore"]["username"])
_proc_man.run_os_command("rm -rf /tmp/" + _username +
"_*-*-*-*-*_avg_acc")
_msg = "Flushing Object Store..."
print _msg,
_rmc = RedisMgdConn(global_objects["objectstore"])
_rmc.flush_object_store()
print "done"
_msg = "Flushing Log Store..."
print _msg,
_proc_man.run_os_command("pkill -9 -u " + _logstore_username +
" -f rsyslogd")
_file_list = []
_file_list.append("operations.log")
_file_list.append("report.log")
_file_list.append("submmiter.log")
_file_list.append("loadmanager.log")
_file_list.append("gui.log")
_file_list.append("remotescripts.log")
_file_list.append("messages.log")
_file_list.append("monitor.log")
_file_list.append("subscribe.log")
for _fn in _file_list:
_proc_man.run_os_command("rm -rf " + _log_dir + '/' +
_logstore_username + '_' + _fn)
_proc_man.run_os_command("touch " + _log_dir + '/' +
_logstore_username + '_' + _fn)
_status, _msg = syslog_logstore_setup(global_objects, "check")
global_objects["logstore"]["just_restarted"] = True
print "done\n"
#_msg = "Flushing File Store..."
#print _msg,
#_proc_man.run_os_command("sudo pkill -9 -u root -f \"rsync --daemon --config " + _filestore_config_file_fn +"\"", raise_exception = False)
#_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _filestore_username + "_rsyncd.pid")
#_status, _msg = rsync_filestore_setup(global_objects, "check")
#print "done"
if not soft:
_msg = "Flushing Metric Store..."
print _msg,
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.flush_metric_store(global_objects["mon_defaults"]["username"])
print "done"
_msg = ""
_status = 0
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
def mysql_metricstore_setup(global_objects, operation="check"):
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
_hostport = int(global_objects["metricstore"]["mysql_port"])
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["mysql_port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mysql_pid = _proc_man.get_pid_from_cmdline("mysqld")
if not _mysql_pid:
_msg = "Unable to detect a shared Mysql server daemon running. "
_msg += "Please try to start one."
print(_msg)
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mysqld.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mysqld --defaults-file=" + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_pid = _proc_man.get_pid_from_cmdline(
"mysqld --defaults-file=" + _config_file_fn)
if not _pid:
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_config_file_contents = global_objects["metricstore"][
"mysql_config_string"]
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username)
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
'**', '-')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _pid:
_msg = "Unable to detect a private MysqlDB server daemon running. "
_msg += "You may need to issue $ sudo apt-get install apparmor-utils && sudo aa-complain /usr/bin/mysqld, followed by: " + _cmd + ")"
print(_msg)
exit(8)
else:
global_objects["metricstore"][
"mysql_port"] = _proc_man.get_port_from_pid(_pid[0])
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"Mysql\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
_mmc = MysqlMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"Mysql\" was successfully initialized "
_msg += "on node: " + str(
global_objects["metricstore"]
["host"]) + " " + _protocol + " port " + str(_hostport)
cbdebug(_msg)
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except NetworkException as obj:
_msg = "A Metric Store of the kind \"Mysql\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid) + "\" seems to be down: "
_msg += str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
except MysqlMgdConn.MetricStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except Exception as e:
for line in traceback.format_exc().splitlines():
cberr(line, True)
_status = 23
_msg = str(e)
raise StoreSetupException(_msg, 9)