def pre_check_port(hostname, hostport, protocol):
'''
TBD
'''
_status = 100
_msg = "An error has occurred, but no error message was captured"
try:
_nh_conn = Nethashget(hostname)
_nh_conn.nmap(hostport, protocol)
return True
except NetworkException, obj:
return False
def pre_check_port(hostname, hostport, protocol) :
'''
TBD
'''
_status = 100
_msg = "An error has occurred, but no error message was captured"
try :
_nh_conn = Nethashget(hostname)
_nh_conn.nmap(hostport, protocol)
return True
except NetworkException, obj :
return False
def redis_objectstore_setup(global_objects, operation, cloud_name=None):
'''
TBD
'''
_protocol = global_objects["objectstore"]["protocol"]
_hostname = global_objects["objectstore"]["host"]
_databaseid = int(global_objects["objectstore"]["dbid"])
_timeout = float(global_objects["objectstore"]["timout"])
_username = global_objects["objectstore"]["username"]
_usage = global_objects["objectstore"]["usage"].lower()
try:
_instance_dir = global_objects["space"]["instance_dir"]
if operation == "check":
_stores_path = global_objects["space"]["stores_working_dir"]
if not path.exists(_stores_path):
cbdebug("Making stores working directory: " + _stores_path)
mkdir(_stores_path)
if _usage == "shared":
_hostport = int(global_objects["objectstore"]["port"])
_proc_man = ProcessManagement(username="******")
if not pre_check_port(_hostname, _hostport, _protocol):
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
_cmd = "/usr/local/bin/redis-server /etc/redis.conf"
if not _redis_pid:
_msg = "Unable to detect a shared Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = _stores_path + '/' + _username + "_redis.conf"
_cmd = "redis-server " + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
if not _redis_pid:
global_objects["objectstore"][
"port"] = _proc_man.get_free_port(
global_objects["objectstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["objectstore"]["port"])
_config_file_contents = global_objects["objectstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_redis_pid = _proc_man.start_daemon(_cmd)
if not _redis_pid:
_msg = "Unable to detect a private Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["objectstore"][
"port"] = _proc_man.get_port_from_pid(_redis_pid[0])
_hostport = int(global_objects["objectstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \""
_msg += str(_databaseid) + "\" seems to be running."
_status = 0
else:
if not cloud_name:
raise StoreSetupException(
"Name of cloud is required for the 'initialize' mode", 22)
operation = "initialize"
_hostport = int(global_objects["objectstore"]["port"])
_collection_names = [ "reported_management_vm_metric_names", \
"reported_runtime_os_host_metric_names", \
"reported_runtime_os_vm_metric_names", \
"host_management_metrics_header", \
"vm_management_metrics_header", \
"host_runtime_os_metrics_header", \
"vm_runtime_os_metrics_header", \
"vm_runtime_app_metrics_header", \
"trace_header" ]
for _collection_name in _collection_names:
for _component in global_objects["mon_defaults"][
_collection_name].split(','):
if _component.lower() in global_objects["mon_defaults"]:
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name].replace(_component, \
global_objects["mon_defaults"][_component.lower()] + ',')
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')
_rmc = RedisMgdConn(global_objects["objectstore"])
# First we remove the leftovers from previous experiments.
if _rmc.initialize_object_store(cloud_name, global_objects, True):
if not path.exists(_instance_dir):
mkdir(_instance_dir)
for _file_name in listdir(_instance_dir):
_file_name = path.join(_instance_dir, _file_name)
if path.isdir(_file_name):
rmtree(_file_name)
_msg = "Folders (but not data) underneath experiment "
_msg += "directory " + _instance_dir + " were removed."
cbdebug(_msg)
_msg = "The Redis datastore was successfully initialized on server " + _hostname
_msg += ", port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\"."
cbdebug(_msg)
_status = 0
else:
_msg = "The Object Store of the kind \"Redis\" was successfully initialized "
_msg += "on node " + _hostname + ". To change its "
_msg += "attributes/state, use the *alter commands"
_msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
_msg += "and attach this cloud back to this experiment."
cbdebug(_msg)
_status = 0
return _status, _msg
except NetworkException, obj:
_msg = "An Object Store of the kind \"Redis\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid)
_msg += "\" seems to be down: " + str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
def rsync_filestore_setup(global_objects, operation="check"):
'''
TBD
'''
_hostname = global_objects["filestore"]["hostname"]
_protocol = global_objects["filestore"]["protocol"]
_username = global_objects["filestore"]["username"]
_port = global_objects["filestore"]["port"]
_usage = global_objects["filestore"]["usage"].lower()
_base_dir = global_objects["space"]["base_dir"]
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try:
_name, _ip = hostname2ip(_hostname)
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["filestore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_rsync_pid = _proc_man.get_pid_from_cmdline(
"rsync --daemon")
_cmd = "rsync --daemon"
if not _rsync_pid:
_msg = "Unable to detect a shared rsync server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_proc_man = ProcessManagement(username=_username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_cmd = "rsync --daemon --config " + _config_file_fn
if not access(_config_file_fn, F_OK):
# File was deleted, but the rsync process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid:
_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir +
'/' + _username + "_rsyncd.pid")
global_objects["filestore"][
"port"] = _proc_man.get_free_port(
global_objects["filestore"]["port"],
protocol="tcp")
_hostport = int(global_objects["filestore"]["port"])
_config_file_contents = global_objects["filestore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"DOLLAR", '$')
_config_file_contents = _config_file_contents.replace(
"REPLEQUAL", '=')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace(
"REPLUSERU", _username + '_')
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username)
_config_file_contents = _config_file_contents.replace(
"REPLBASEDIR", _base_dir)
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_contents = _config_file_contents.replace(
"--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd)
if not _rsyslog_pid:
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents:
if _line.count("port="):
global_objects["filestore"]["port"] = _line.split(
'=')[1]
_hostport = int(
global_objects["filestore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A File Store of the kind \"rsync\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsync "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsync"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def mongodb_metricstore_setup(global_objects, operation="check"):
'''
TBD
'''
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
_cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
if not _mongodb_pid:
_msg = "Unable to detect a shared MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects[
"space"]["stores_working_dir"] + "/mongod.pid"
_proc_man = ProcessManagement(username=_username)
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
if not _mongodb_pid:
global_objects["metricstore"][
"port"] = _proc_man.get_free_port(
global_objects["metricstore"]["port"],
protocol="tcp")
_hostport = int(global_objects["metricstore"]["port"])
_config_file_contents = global_objects["metricstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mongod.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_mongodb_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _mongodb_pid:
_msg = "Unable to detect a private MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
global_objects["metricstore"][
"port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
_hostport = int(global_objects["metricstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
else:
operation = "initialize"
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
_msg += "on node: " + str(global_objects["metricstore"])
cbdebug(_msg)
_status = 0
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def syslog_logstore_setup(global_objects, operation="check"):
'''
TBD
'''
_hostname = global_objects["logstore"]["hostname"]
_protocol = global_objects["logstore"]["protocol"]
_username = global_objects["logstore"]["username"]
_usage = global_objects["logstore"]["usage"].lower()
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try:
_name, _ip = hostname2ip(_hostname)
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["logstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
_cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid "
if not _rsyslog_pid:
_msg = "Unable to detect a shared rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_usage = "private"
_proc_man = ProcessManagement(username=_username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid"
if not access(_config_file_fn, F_OK):
# File was deleted, but the rsyslog process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
if not access(_log_dir, W_OK):
# The directory does not even exist, kill any rsyslog processes still dangling
_proc_man.run_os_command("sudo pkill -9 -f " +
_config_file_fn)
_proc_man.run_os_command("sudo mkdir -p " + _log_dir +
" && sudo chmod 777 " + _log_dir)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid:
global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\
protocol = "udp")
_hostport = int(global_objects["logstore"]["port"])
_config_file_contents = global_objects["logstore"][
"config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace(
"DOLLAR", '$')
_config_file_contents = _config_file_contents.replace(
"RSYSLOG", "RSYSLOG_")
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username + '_')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_contents = _config_file_contents.replace(
"--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon(_cmd)
if not _rsyslog_pid:
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else:
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents:
if _line.count("UDPServerRun"):
global_objects["logstore"]["port"] = _line.split(
)[1]
_hostport = int(global_objects["logstore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsyslog "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def redis_objectstore_setup(global_objects, operation, cloud_name = None) :
'''
TBD
'''
_protocol = global_objects["objectstore"]["protocol"]
_hostname = global_objects["objectstore"]["host"]
_databaseid = int(global_objects["objectstore"]["dbid"])
_timeout = float(global_objects["objectstore"]["timout"])
_username = global_objects["objectstore"]["username"]
_usage = global_objects["objectstore"]["usage"].lower()
try :
_instance_dir = global_objects["space"]["instance_dir"]
if operation == "check" :
_stores_path = global_objects["space"]["stores_working_dir"]
if not path.exists(_stores_path) :
cbdebug("Making stores working directory: " + _stores_path)
mkdir(_stores_path)
if _usage == "shared" :
_hostport = int(global_objects["objectstore"]["port"])
_proc_man = ProcessManagement(username = "******")
if not pre_check_port(_hostname, _hostport, _protocol) :
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
_cmd = "/usr/local/bin/redis-server /etc/redis.conf"
if not _redis_pid :
_msg = "Unable to detect a shared Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_config_file_fn = _stores_path + '/' + _username + "_redis.conf"
_cmd = "redis-server " + _config_file_fn
_proc_man = ProcessManagement(username = _username)
_redis_pid = _proc_man.get_pid_from_cmdline("redis-server")
if not _redis_pid :
global_objects["objectstore"]["port"] = _proc_man.get_free_port(global_objects["objectstore"]["port"], protocol = "tcp")
_hostport = int(global_objects["objectstore"]["port"])
_config_file_contents = global_objects["objectstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_redis_pid = _proc_man.start_daemon(_cmd)
if not _redis_pid :
_msg = "Unable to detect a private Redis server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
global_objects["objectstore"]["port"] = _proc_man.get_port_from_pid(_redis_pid[0])
_hostport = int(global_objects["objectstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \""
_msg += str(_databaseid) + "\" seems to be running."
_status = 0
else :
if not cloud_name :
raise StoreSetupException("Name of cloud is required for the 'initialize' mode", 22)
operation = "initialize"
_hostport = int(global_objects["objectstore"]["port"])
_collection_names = [ "reported_management_vm_metric_names", \
"reported_runtime_os_host_metric_names", \
"reported_runtime_os_vm_metric_names", \
"host_management_metrics_header", \
"vm_management_metrics_header", \
"host_runtime_os_metrics_header", \
"vm_runtime_os_metrics_header", \
"vm_runtime_app_metrics_header", \
"trace_header" ]
for _collection_name in _collection_names :
for _component in global_objects["mon_defaults"][_collection_name].split(',') :
if _component.lower() in global_objects["mon_defaults"] :
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name].replace(_component, \
global_objects["mon_defaults"][_component.lower()] + ',')
global_objects["mon_defaults"][_collection_name] = \
global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')
_rmc = RedisMgdConn(global_objects["objectstore"])
# First we remove the leftovers from previous experiments.
if _rmc.initialize_object_store(cloud_name, global_objects, True) :
if not path.exists(_instance_dir) :
mkdir(_instance_dir)
for _file_name in listdir(_instance_dir) :
_file_name = path.join(_instance_dir, _file_name)
if path.isdir(_file_name) :
rmtree(_file_name)
_msg = "Folders (but not data) underneath experiment "
_msg += "directory " + _instance_dir + " were removed."
cbdebug(_msg)
_msg = "The Redis datastore was successfully initialized on server " + _hostname
_msg += ", port " + str(_hostport) + ", database id \"" + str(_databaseid)
_msg += "\"."
cbdebug(_msg)
_status = 0
else :
_msg = "The Object Store of the kind \"Redis\" was successfully initialized "
_msg += "on node " + _hostname + ". To change its "
_msg += "attributes/state, use the *alter commands"
_msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
_msg += "and attach this cloud back to this experiment."
cbdebug(_msg)
_status = 0
return _status, _msg
except NetworkException, obj :
_msg = "An Object Store of the kind \"Redis\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid)
_msg += "\" seems to be down: " + str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) :
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
if not self.get_svm_stub(obj_attr_list) :
_network_reachable = False
else:
_network_reachable = True
_curr_tries = 0
if not _network_reachable :
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"] + "..."
cbdebug(_msg, True)
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", obj_attr_list["uuid"], _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries :
if "async" not in obj_attr_list or obj_attr_list["async"].lower() == "false" :
if threading.current_thread().abort :
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_") :
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count("subscribe_on_") :
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list["cloud_uuid"] + ") has started by "
_msg += "subscribing to channel \"" + str(_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe)
for _message in _sub_channel.listen() :
if str(_message["data"]).count(_string_to_search) :
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_") :
_boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",''))
_msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time :
sleep(_boot_wait_time)
_vm_is_booted = True
else :
_vm_is_booted = False
if _vm_is_booted :
obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", obj_attr_list["uuid"], \
"Network accessible now. Continuing...")
_network_reachable = True
break
else :
_msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(_wait)
_msg += " seconds and check again."
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", \
obj_attr_list["uuid"], \
_msg)
cbdebug(_msg)
sleep(_wait)
_curr_tries += 1
if _curr_tries < _max_tries :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006, NOT mgt_005
obj_attr_list["mgt_006_application_start"] = "0"
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \
obj_attr_list["uuid"], \
"Application starting up...")
else :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
def mongodb_metricstore_setup(global_objects, operation = "check") :
'''
TBD
'''
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
try :
if operation == "check" :
if _usage == "shared" :
_hostport = int(global_objects["metricstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol) :
_proc_man = ProcessManagement(username = "******")
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
_cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
if not _mongodb_pid :
_msg = "Unable to detect a shared MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
_cmd = "mkdir -p " + global_objects["space"]["stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects["space"]["stores_working_dir"] + "/mongod.pid"
_proc_man = ProcessManagement(username = _username)
_mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
if not _mongodb_pid :
global_objects["metricstore"]["port"] = _proc_man.get_free_port(global_objects["metricstore"]["port"], protocol = "tcp")
_hostport = int(global_objects["metricstore"]["port"])
_config_file_contents = global_objects["metricstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace("--", '=')
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_mongodb_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _mongodb_pid :
_msg = "Unable to detect a private MongoDB server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
global_objects["metricstore"]["port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
_hostport = int(global_objects["metricstore"]["port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
else:
operation = "initialize"
_mmc = MongodbMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
_msg += "on node: " + str(global_objects["metricstore"])
cbdebug(_msg)
_status = 0
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def syslog_logstore_setup(global_objects, operation = "check") :
'''
TBD
'''
_hostname = global_objects["logstore"]["hostname"]
_protocol = global_objects["logstore"]["protocol"]
_username = global_objects["logstore"]["username"]
_usage = global_objects["logstore"]["usage"].lower()
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try :
_name, _ip = hostname2ip(_hostname)
if operation == "check" :
if _usage == "shared" :
_hostport = int(global_objects["logstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol) :
_proc_man = ProcessManagement(username = "******")
_rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
_cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid "
if not _rsyslog_pid :
_msg = "Unable to detect a shared rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_proc_man = ProcessManagement(username = _username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid"
if not access(_config_file_fn, F_OK) :
# File was deleted, but the rsyslog process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)
if not access(_log_dir, W_OK) :
# The directory does not even exist, kill any rsyslog processes still dangling
_proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)
_proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid :
global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\
protocol = "udp")
_hostport = int(global_objects["logstore"]["port"])
_config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("DOLLAR", '$')
_config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_")
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace("REPLUSER", _username + '_')
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_contents = _config_file_contents.replace("--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon(_cmd)
if not _rsyslog_pid :
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents :
if _line.count("UDPServerRun") :
global_objects["logstore"]["port"] = _line.split()[1]
_hostport = int(global_objects["logstore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsyslog "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc):
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
_network_reachable = False
_curr_tries = 0
if not _network_reachable:
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"]
if str(obj_attr_list["use_jumphost"]).lower() == "false":
_msg += "..."
else:
_msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..."
obj_attr_list["check_boot_complete"] = "run_command_/bin/true"
cbdebug(_msg, True)
self.pending_set(obj_attr_list, _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries:
_start_pooling = int(time())
if "async" not in obj_attr_list or str(
obj_attr_list["async"]).lower() == "false":
if threading.current_thread().abort:
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_"):
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list[
"check_boot_complete"].replace("tcp_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(
int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count("cloud_ping"):
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish network connectivity "
_msg += "through the cloud's API"
cbdebug(_msg)
_vm_is_booted = self.is_vm_alive(obj_attr_list)
elif obj_attr_list["check_boot_complete"].count(
"subscribe_on_"):
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list[
"check_boot_complete"].replace("subscribe_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list[
"cloud_vm_uuid"] + ") has booted by "
_msg += "subscribing to channel \"" + str(
_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM",
"VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(
obj_attr_list["cloud_name"], "VM",
_channel_to_subscribe, _max_tries * _wait)
for _message in _sub_channel.listen():
if str(_message["data"]).count(_string_to_search):
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"],
"VM", "VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_"):
_boot_wait_time = int(
obj_attr_list["check_boot_complete"].replace(
"wait_for_", ''))
_msg = "Assuming that the VM \"" + obj_attr_list[
"cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(
_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time:
sleep(_boot_wait_time)
_vm_is_booted = True
elif obj_attr_list["check_boot_complete"].count(
"run_command_"):
_command_to_run = obj_attr_list[
"check_boot_complete"].replace("run_command_", '')
_command_to_run = _command_to_run.replace("____", ' ')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list[
"cloud_vm_uuid"] + ") has booted by "
_msg += "running the command \"" + str(_command_to_run)
cbdebug(_msg)
if _curr_tries <= _max_tries / 3:
_connection_timeout = int(
obj_attr_list["update_frequency"]) / 2
elif _curr_tries > _max_tries / 3 and _curr_tries < 2 * _max_tries / 3:
_connection_timeout = int(
obj_attr_list["update_frequency"])
obj_attr_list[
"comments"] += "Had to increase ssh timeout. "
else:
_connection_timeout = int(
obj_attr_list["update_frequency"]) * 2
obj_attr_list[
"comments"] += "Had to increase ssh timeout one more time. "
if str(obj_attr_list["use_jumphost"]).lower() == "true":
if "ssh_config_file" in obj_attr_list:
_ssh_conf_file = obj_attr_list["ssh_config_file"]
else:
_ssh_conf_file = None
else:
_ssh_conf_file = None
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["prov_cloud_ip"], \
priv_key = obj_attr_list["identity"], \
config_file = _ssh_conf_file,
connection_timeout = _connection_timeout)
try:
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_command_to_run)
if not _status:
_vm_is_booted = True
else:
_vm_is_booted = False
except:
_vm_is_booted = False
elif obj_attr_list["check_boot_complete"].count(
"snmpget_poll"):
import netsnmp
# Send SNMP GET message. Flag VM as booted if any response at all is recieved
_vm_is_booted = False
try:
_msg = "Opening SNMP session to " + obj_attr_list[
"cloud_ip"]
cbdebug(_msg)
_snmp_wait_time = _wait * 1000000
_snmp_version = int(obj_attr_list["snmp_version"])
_snmp_comm = str(obj_attr_list["snmp_community"])
_snmp_session = netsnmp.Session(Version=_snmp_version, \
DestHost=obj_attr_list["cloud_ip"], \
Community=_snmp_comm, \
Timeout=_snmp_wait_time, Retries=0)
_vars = netsnmp.VarList(
netsnmp.Varbind(obj_attr_list["snmp_variable"],
'0'))
_snmp_response = _snmp_session.get(_vars)
except:
if _snmp_session.ErrorStr:
_msg = "Error in SNMP handler : " + _snmp_session.ErrorStr
else:
_msg = "Unknown error in SNMP handler."
cbdebug(_msg)
_status = 200
raise CldOpsException(_msg, _status)
if (_snmp_response[0] != None):
_vm_is_booted = True
_msg = "SNMP Response: " + str(_snmp_response)
cbdebug(_msg)
else:
_vm_is_booted = False
_msg = "Warning: No valid method specified to determined if VM has booted."
cbdebug(_msg, True)
_pooling_time = int(time()) - _start_pooling
if _pooling_time <= _wait:
_actual_wait = _wait - _pooling_time
else:
_msg = "The time spent on pooling for \"booted\" status (" + str(
_pooling_time)
_msg += " s) is actually longer than the "
_msg += "interval between pooling attempts (" + str(
_wait) + " s)."
cbdebug(_msg, True)
_actual_wait = 0
if _vm_is_booted:
obj_attr_list["mgt_004_network_acessible"] = int(
time()) - time_mark_prc
self.pending_set(obj_attr_list,
"Network accessible now. Continuing...")
_network_reachable = True
break
else:
_msg = "(" + str(
_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(
_actual_wait)
_msg += " seconds and check again."
self.pending_set(obj_attr_list, _msg)
cbdebug(_msg)
sleep(_actual_wait)
_curr_tries += 1
if _curr_tries < _max_tries:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006 and mgt_007 NOT mgt_005
obj_attr_list["mgt_006_instance_preparation"] = "0"
obj_attr_list["mgt_007_application_start"] = "0"
self.pending_set(obj_attr_list, "Application starting up...")
self.get_attr_from_pending(obj_attr_list, "all")
else:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_vm_uuid"] + ") "
_msg += "is not network reachable after " + str(
_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
def mysql_metricstore_setup(global_objects, operation="check"):
_protocol = global_objects["metricstore"]["protocol"]
_hostname = global_objects["metricstore"]["host"]
_databaseid = global_objects["metricstore"]["database"]
_timeout = float(global_objects["metricstore"]["timeout"])
_username = global_objects["mon_defaults"]["username"]
_usage = global_objects["metricstore"]["usage"].lower()
_hostport = int(global_objects["metricstore"]["mysql_port"])
try:
if operation == "check":
if _usage == "shared":
_hostport = int(global_objects["metricstore"]["mysql_port"])
if not pre_check_port(_hostname, _hostport, _protocol):
_proc_man = ProcessManagement(username="******")
_mysql_pid = _proc_man.get_pid_from_cmdline("mysqld")
if not _mysql_pid:
_msg = "Unable to detect a shared Mysql server daemon running. "
_msg += "Please try to start one."
print(_msg)
exit(8)
else:
_usage = "private"
_config_file_fn = global_objects["space"][
"stores_working_dir"] + '/' + _username + "_mysqld.conf"
_cmd = "mkdir -p " + global_objects["space"][
"stores_working_dir"] + "/logs; mysqld --defaults-file=" + _config_file_fn
_proc_man = ProcessManagement(username=_username)
_pid = _proc_man.get_pid_from_cmdline(
"mysqld --defaults-file=" + _config_file_fn)
if not _pid:
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_config_file_contents = global_objects["metricstore"][
"mysql_config_string"]
_config_file_contents = _config_file_contents.replace(
"REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace(
"REPLUSER", _username)
_config_file_contents = _config_file_contents.replace(
"REPLSTORESWORKINGDIR",
global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(
"--", '=')
_config_file_contents = _config_file_contents.replace(
'**', '-')
_config_file_contents = _config_file_contents.replace(
';', '\n')
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_pid = _proc_man.start_daemon(_cmd)
sleep(5)
if not _pid:
_msg = "Unable to detect a private MysqlDB server daemon running. "
_msg += "You may need to issue $ sudo apt-get install apparmor-utils && sudo aa-complain /usr/bin/mysqld, followed by: " + _cmd + ")"
print(_msg)
exit(8)
else:
global_objects["metricstore"][
"mysql_port"] = _proc_man.get_port_from_pid(_pid[0])
_hostport = int(
global_objects["metricstore"]["mysql_port"])
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Metric Store of the kind \"Mysql\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + ", database id \"" + str(
_databaseid)
_msg += "\" seems to be running."
cbdebug(_msg)
_status = 0
_mmc = MysqlMgdConn(global_objects["metricstore"])
_mmc.initialize_metric_store(_username)
_msg = "The Metric Store of the kind \"Mysql\" was successfully initialized "
_msg += "on node: " + str(
global_objects["metricstore"]
["host"]) + " " + _protocol + " port " + str(_hostport)
cbdebug(_msg)
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except NetworkException as obj:
_msg = "A Metric Store of the kind \"Mysql\" on node "
_msg += _hostname + ", " + _protocol + " port " + str(_hostport)
_msg += ", database id \"" + str(_databaseid) + "\" seems to be down: "
_msg += str(obj.msg) + '.'
cberr(_msg)
raise StoreSetupException(_msg, 8)
except MysqlMgdConn.MetricStoreMgdConnException as obj:
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
except Exception as e:
for line in traceback.format_exc().splitlines():
cberr(line, True)
_status = 23
_msg = str(e)
raise StoreSetupException(_msg, 9)
def syslog_logstore_setup(global_objects, operation = "check") :
'''
TBD
'''
_hostname = global_objects["logstore"]["hostname"]
_protocol = global_objects["logstore"]["protocol"]
_username = global_objects["logstore"]["username"]
_usage = global_objects["logstore"]["usage"].lower()
try :
if operation == "check" :
if _usage == "shared" :
_hostport = int(global_objects["logstore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol) :
_proc_man = ProcessManagement(username = "******")
_rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
_cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid -c 5"
if not _rsyslog_pid :
_msg = "Unable to detect a shared rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf"
_cmd = "rsyslogd -f " + _config_file_fn + " -c 4 " + "-i " + global_objects["space"]["stores_working_dir"] + "/rsyslog.pid"
_proc_man = ProcessManagement(username = _username)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid :
global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"], protocol = "udp")
_hostport = int(global_objects["logstore"]["port"])
_config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("DOLLAR", '$')
_config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_")
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_contents = _config_file_contents.replace("--", ';')
_config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon(_cmd)
if not _rsyslog_pid :
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents :
if _line.count("UDPServerRun") :
global_objects["logstore"]["port"] = _line.split()[1]
_hostport = int(global_objects["logstore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) :
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
if not self.get_svm_stub(obj_attr_list) :
_network_reachable = False
else:
_network_reachable = True
_curr_tries = 0
if not _network_reachable :
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"] + "..."
cbdebug(_msg, True)
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", obj_attr_list["uuid"], _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries :
if "async" not in obj_attr_list or obj_attr_list["async"].lower() == "false" :
if threading.current_thread().abort :
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_") :
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count("subscribe_on_") :
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list["cloud_uuid"] + ") has started by "
_msg += "subscribing to channel \"" + str(_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe)
for _message in _sub_channel.listen() :
if str(_message["data"]).count(_string_to_search) :
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_") :
_boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",''))
_msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time :
sleep(_boot_wait_time)
_vm_is_booted = True
elif obj_attr_list["check_boot_complete"].count("snmpget_poll") :
import netsnmp
# Send SNMP GET message. Flag VM as booted if any response at all is recieved
_vm_is_booted = False
try :
_msg = "Opening SNMP session to " + obj_attr_list["cloud_ip"]
cbdebug(_msg)
_snmp_wait_time = _wait * 1000000
_snmp_version = int(obj_attr_list["snmp_version"])
_snmp_comm = str(obj_attr_list["snmp_community"])
_snmp_session = netsnmp.Session(Version=_snmp_version, DestHost=obj_attr_list["cloud_ip"], \
Community=_snmp_comm, \
Timeout=_snmp_wait_time, Retries=0)
_vars = netsnmp.VarList(netsnmp.Varbind(obj_attr_list["snmp_variable"], '0'))
_snmp_response = _snmp_session.get(_vars)
except :
if _snmp_session.ErrorStr :
_msg = "Error in SNMP handler : " + _snmp_session.ErrorStr
else :
_msg = "Unknown error in SNMP handler."
cbdebug(_msg)
_status = 200
raise CldOpsException(_msg, _status)
if (_snmp_response[0] != None ) :
_vm_is_booted = True
_msg = "SNMP Response: " + str(_snmp_response)
cbdebug(_msg)
else :
_vm_is_booted = False
_msg = "Warning: No valid method specified to determined if VM has booted."
cbdebug(_msg, True)
if _vm_is_booted :
obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", obj_attr_list["uuid"], \
"Network accessible now. Continuing...")
_network_reachable = True
break
else :
_msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(_wait)
_msg += " seconds and check again."
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", \
obj_attr_list["uuid"], \
_msg)
cbdebug(_msg)
sleep(_wait)
_curr_tries += 1
if _curr_tries < _max_tries :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006, NOT mgt_005
obj_attr_list["mgt_006_application_start"] = "0"
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \
obj_attr_list["uuid"], \
"Application starting up...")
else :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") "
_msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
def rsync_filestore_setup(global_objects, operation = "check") :
'''
TBD
'''
_hostname = global_objects["filestore"]["hostname"]
_protocol = global_objects["filestore"]["protocol"]
_username = global_objects["filestore"]["username"]
_port = global_objects["filestore"]["port"]
_usage = global_objects["filestore"]["usage"].lower()
_base_dir = global_objects["space"]["base_dir"]
_stores_wk_dir = global_objects["space"]["stores_working_dir"]
_log_dir = global_objects["space"]["log_dir"]
try :
_name, _ip = hostname2ip(_hostname)
if operation == "check" :
if _usage == "shared" :
_hostport = int(global_objects["filestore"]["port"])
if not pre_check_port(_hostname, _hostport, _protocol) :
_proc_man = ProcessManagement(username = "******")
_rsync_pid = _proc_man.get_pid_from_cmdline("rsync --daemon")
_cmd = "rsync --daemon"
if not _rsync_pid :
_msg = "Unable to detect a shared rsync server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_usage = "private"
_proc_man = ProcessManagement(username = _username)
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_cmd = "rsync --daemon --config " + _config_file_fn
if not access(_config_file_fn, F_OK) :
# File was deleted, but the rsync process is still dangling
_proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)
_rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)
if not _rsyslog_pid :
_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid")
global_objects["filestore"]["port"] = _proc_man.get_free_port(global_objects["filestore"]["port"], protocol = "tcp")
_hostport = int(global_objects["filestore"]["port"])
_config_file_contents = global_objects["filestore"]["config_string"].replace('_', ' ')
_config_file_contents = _config_file_contents.replace("DOLLAR", '$')
_config_file_contents = _config_file_contents.replace("REPLEQUAL", '=')
_config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
_config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir)
_config_file_contents = _config_file_contents.replace("REPLUSERU", _username + '_')
_config_file_contents = _config_file_contents.replace("REPLUSER", _username)
_config_file_contents = _config_file_contents.replace("REPLBASEDIR", _base_dir)
_config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
_config_file_contents = _config_file_contents.replace(';','\n')
_config_file_contents = _config_file_contents.replace("--", ';')
_config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
_config_file_fd = open(_config_file_fn, 'w')
_config_file_fd.write(_config_file_contents)
_config_file_fd.close()
_rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd)
if not _rsyslog_pid :
_msg = "Unable to detect a private rsyslog server daemon running. "
_msg += "Please try to start one (e.g., " + _cmd + ")"
print _msg
exit(8)
else :
_config_file_fd = open(_config_file_fn, 'r')
_config_file_contents = _config_file_fd.readlines()
_config_file_fd.close()
for _line in _config_file_contents :
if _line.count("port=") :
global_objects["filestore"]["port"] = _line.split('=')[1]
_hostport = int(global_objects["filestore"]["port"])
break
_nh_conn = Nethashget(_hostname)
_nh_conn.nmap(_hostport, _protocol)
_msg = "A File Store of the kind \"rsync\" (" + _usage + ") "
_msg += "on node " + _hostname + ", " + _protocol
_msg += " port " + str(_hostport) + " seems to be running."
cbdebug(_msg)
_status = 0
return _status, _msg
except socket.herror:
_status = 1200
_msg = "The IP address \"" + _hostname + "\" - used by the rsync "
_msg += " daemon - is not mapped to a Hostname. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except socket.gaierror:
_status = 1200
_msg = "The Hostname \"" + _hostname + "\" - used by the rsync"
_msg += " daemon - is not mapped to an IP. "
_msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
raise StoreSetupException(_msg, 9)
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_msg = str(obj.msg)
raise StoreSetupException(_msg, 9)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) :
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
_network_reachable = False
_curr_tries = 0
if not _network_reachable :
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"]
if str(obj_attr_list["use_jumphost"]).lower() == "false" :
_msg += "..."
else :
_msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..."
obj_attr_list["check_boot_complete"] = "run_command_/bin/true"
cbdebug(_msg, True)
self.pending_set(obj_attr_list, _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries :
_start_pooling = int(time())
if "async" not in obj_attr_list or str(obj_attr_list["async"]).lower() == "false" :
if threading.current_thread().abort :
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_") :
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count("cloud_ping") :
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish network connectivity "
_msg += "through the cloud's API"
cbdebug(_msg)
_vm_is_booted = self.is_vm_alive(obj_attr_list)
elif obj_attr_list["check_boot_complete"].count("subscribe_on_") :
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by "
_msg += "subscribing to channel \"" + str(_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe, _max_tries * _wait)
for _message in _sub_channel.listen() :
if str(_message["data"]).count(_string_to_search) :
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_") :
_boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",''))
_msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time :
sleep(_boot_wait_time)
_vm_is_booted = True
elif obj_attr_list["check_boot_complete"].count("run_command_") :
_command_to_run = obj_attr_list["check_boot_complete"].replace("run_command_",'')
_command_to_run = _command_to_run.replace("____",' ')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by "
_msg += "running the command \"" + str(_command_to_run)
cbdebug(_msg)
if _curr_tries <= _max_tries/3 :
_connection_timeout = int(obj_attr_list["update_frequency"])/2
elif _curr_tries > _max_tries/3 and _curr_tries < 2*_max_tries/3 :
_connection_timeout = int(obj_attr_list["update_frequency"])
obj_attr_list["comments"] += "Had to increase ssh timeout. "
else :
_connection_timeout = int(obj_attr_list["update_frequency"])*2
obj_attr_list["comments"] += "Had to increase ssh timeout one more time. "
if str(obj_attr_list["use_jumphost"]).lower() == "true" :
if "ssh_config_file" in obj_attr_list:
_ssh_conf_file = obj_attr_list["ssh_config_file"]
else:
_ssh_conf_file = None
else :
_ssh_conf_file = None
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["prov_cloud_ip"], \
priv_key = obj_attr_list["identity"], \
config_file = _ssh_conf_file,
connection_timeout = _connection_timeout)
try :
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(_command_to_run)
if not _status :
_vm_is_booted = True
else :
_vm_is_booted = False
except :
_vm_is_booted = False
elif obj_attr_list["check_boot_complete"].count("snmpget_poll") :
import netsnmp
# Send SNMP GET message. Flag VM as booted if any response at all is recieved
_vm_is_booted = False
try :
_msg = "Opening SNMP session to " + obj_attr_list["cloud_ip"]
cbdebug(_msg)
_snmp_wait_time = _wait * 1000000
_snmp_version = int(obj_attr_list["snmp_version"])
_snmp_comm = str(obj_attr_list["snmp_community"])
_snmp_session = netsnmp.Session(Version=_snmp_version, \
DestHost=obj_attr_list["cloud_ip"], \
Community=_snmp_comm, \
Timeout=_snmp_wait_time, Retries=0)
_vars = netsnmp.VarList(netsnmp.Varbind(obj_attr_list["snmp_variable"], '0'))
_snmp_response = _snmp_session.get(_vars)
except :
if _snmp_session.ErrorStr :
_msg = "Error in SNMP handler : " + _snmp_session.ErrorStr
else :
_msg = "Unknown error in SNMP handler."
cbdebug(_msg)
_status = 200
raise CldOpsException(_msg, _status)
if (_snmp_response[0] != None ) :
_vm_is_booted = True
_msg = "SNMP Response: " + str(_snmp_response)
cbdebug(_msg)
else :
_vm_is_booted = False
_msg = "Warning: No valid method specified to determined if VM has booted."
cbdebug(_msg, True)
_pooling_time = int(time()) - _start_pooling
if _pooling_time <= _wait :
_actual_wait = _wait - _pooling_time
else :
_msg = "The time spent on pooling for \"booted\" status (" + str(_pooling_time)
_msg += " s) is actually longer than the "
_msg += "interval between pooling attempts (" + str(_wait) + " s)."
cbdebug(_msg, True)
_actual_wait = 0
if _vm_is_booted :
obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc
self.pending_set(obj_attr_list, "Network accessible now. Continuing...")
_network_reachable = True
break
else :
_msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(_actual_wait)
_msg += " seconds and check again."
self.pending_set(obj_attr_list, _msg)
cbdebug(_msg)
sleep(_actual_wait)
_curr_tries += 1
if _curr_tries < _max_tries :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006 and mgt_007 NOT mgt_005
obj_attr_list["mgt_006_instance_preparation"] = "0"
obj_attr_list["mgt_007_application_start"] = "0"
self.pending_set(obj_attr_list, "Application starting up...")
self.get_attr_from_pending(obj_attr_list, "all")
else :
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc):
'''
TBD
'''
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
if not self.get_svm_stub(obj_attr_list):
_network_reachable = False
else:
_network_reachable = True
_curr_tries = 0
if not _network_reachable:
_msg = "Trying to establish network connectivity to "
_msg += obj_attr_list["name"] + " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_uuid"] + "), on IP address "
_msg += obj_attr_list["prov_cloud_ip"] + "..."
cbdebug(_msg, True)
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM",
obj_attr_list["uuid"], _msg)
sleep(_wait)
while not _network_reachable and _curr_tries < _max_tries:
if "async" not in obj_attr_list or obj_attr_list[
"async"].lower() == "false":
if threading.current_thread().abort:
_msg = "VM Create Aborting..."
_status = 123
raise CldOpsException(_msg, _status)
if obj_attr_list["check_boot_complete"].count("tcp_on_"):
_nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"])
_port_to_check = obj_attr_list[
"check_boot_complete"].replace("tcp_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted by "
_msg += "attempting to establish a TCP connection to port "
_msg += str(_port_to_check) + " on address "
_msg += obj_attr_list["prov_cloud_ip"]
cbdebug(_msg)
_vm_is_booted = _nh_conn.check_port(
int(_port_to_check), "TCP")
elif obj_attr_list["check_boot_complete"].count(
"subscribe_on_"):
_string_to_search = obj_attr_list["prov_cloud_ip"] + " is "
_string_to_search += "booted"
_channel_to_subscribe = obj_attr_list[
"check_boot_complete"].replace("subscribe_on_", '')
_msg = "Check if the VM \"" + obj_attr_list["name"]
_msg += "\" (" + obj_attr_list[
"cloud_uuid"] + ") has started by "
_msg += "subscribing to channel \"" + str(
_channel_to_subscribe)
_msg += "\" and waiting for the message \""
_msg += _string_to_search + "\"."
cbdebug(_msg)
self.osci.add_to_list(obj_attr_list["cloud_name"], "VM",
"VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
_sub_channel = self.osci.subscribe(
obj_attr_list["cloud_name"], "VM",
_channel_to_subscribe)
for _message in _sub_channel.listen():
if str(_message["data"]).count(_string_to_search):
_vm_is_booted = True
break
_sub_channel.unsubscribe()
self.osci.remove_from_list(obj_attr_list["cloud_name"],
"VM", "VMS_BOOTING",
obj_attr_list["prov_cloud_ip"])
elif obj_attr_list["check_boot_complete"].count("wait_for_"):
_boot_wait_time = int(
obj_attr_list["check_boot_complete"].replace(
"wait_for_", ''))
_msg = "Assuming that the VM \"" + obj_attr_list[
"cloud_name"]
_msg += "\" (" + obj_attr_list["name"] + ") is booted after"
_msg += " waiting for " + str(
_boot_wait_time) + " seconds."
cbdebug(_msg)
if _boot_wait_time:
sleep(_boot_wait_time)
_vm_is_booted = True
elif obj_attr_list["check_boot_complete"].count(
"snmpget_poll"):
import netsnmp
# Send SNMP GET message. Flag VM as booted if any response at all is recieved
_vm_is_booted = False
try:
_msg = "Opening SNMP session to " + obj_attr_list[
"cloud_ip"]
cbdebug(_msg)
_snmp_wait_time = _wait * 1000000
_snmp_version = int(obj_attr_list["snmp_version"])
_snmp_comm = str(obj_attr_list["snmp_community"])
_snmp_session = netsnmp.Session(Version=_snmp_version, DestHost=obj_attr_list["cloud_ip"], \
Community=_snmp_comm, \
Timeout=_snmp_wait_time, Retries=0)
_vars = netsnmp.VarList(
netsnmp.Varbind(obj_attr_list["snmp_variable"],
'0'))
_snmp_response = _snmp_session.get(_vars)
except:
if _snmp_session.ErrorStr:
_msg = "Error in SNMP handler : " + _snmp_session.ErrorStr
else:
_msg = "Unknown error in SNMP handler."
cbdebug(_msg)
_status = 200
raise CldOpsException(_msg, _status)
if (_snmp_response[0] != None):
_vm_is_booted = True
_msg = "SNMP Response: " + str(_snmp_response)
cbdebug(_msg)
else:
_vm_is_booted = False
_msg = "Warning: No valid method specified to determined if VM has booted."
cbdebug(_msg, True)
if _vm_is_booted:
obj_attr_list["mgt_004_network_acessible"] = int(
time()) - time_mark_prc
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", obj_attr_list["uuid"], \
"Network accessible now. Continuing...")
_network_reachable = True
break
else:
_msg = "(" + str(
_curr_tries) + ") " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_uuid"] + ") "
_msg += "still not network reachable. Will wait for " + str(
_wait)
_msg += " seconds and check again."
self.osci.pending_object_set(obj_attr_list["cloud_name"], \
"VM", \
obj_attr_list["uuid"], \
_msg)
cbdebug(_msg)
sleep(_wait)
_curr_tries += 1
if _curr_tries < _max_tries:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_uuid"] + ") "
_msg += "is network reachable (boot process finished successfully)"
cbdebug(_msg)
obj_attr_list["arrival"] = int(time())
# It should be mgt_006, NOT mgt_005
obj_attr_list["mgt_006_application_start"] = "0"
self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \
obj_attr_list["uuid"], \
"Application starting up...")
else:
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list[
"cloud_uuid"] + ") "
_msg += "is not network reachable after " + str(
_max_tries * _wait) + " seconds.... "
_msg += "Giving up."
cberr(_msg, True)
raise CldOpsException(_msg, 89)
