def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) : ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) _network_reachable = False _curr_tries = 0 if not _network_reachable : _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] if str(obj_attr_list["use_jumphost"]).lower() == "false" : _msg += "..." else : _msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..." obj_attr_list["check_boot_complete"] = "run_command_/bin/true" cbdebug(_msg, True) self.pending_set(obj_attr_list, _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries : _start_pooling = int(time()) if "async" not in obj_attr_list or str(obj_attr_list["async"]).lower() == "false" : if threading.current_thread().abort : _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_") : _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("cloud_ping") : _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish network connectivity " _msg += "through the cloud's API" cbdebug(_msg) _vm_is_booted = self.is_vm_alive(obj_attr_list) elif obj_attr_list["check_boot_complete"].count("subscribe_on_") : _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by " _msg += "subscribing to channel \"" + str(_channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe, _max_tries * _wait) for _message in _sub_channel.listen() : if str(_message["data"]).count(_string_to_search) : _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_") : _boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",'')) _msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str(_boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time : sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count("run_command_") : _command_to_run = obj_attr_list["check_boot_complete"].replace("run_command_",'') _command_to_run = _command_to_run.replace("____",' ') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by " _msg += "running the command \"" + str(_command_to_run) cbdebug(_msg) if _curr_tries <= _max_tries/3 : _connection_timeout = int(obj_attr_list["update_frequency"])/2 elif _curr_tries > _max_tries/3 and _curr_tries < 2*_max_tries/3 : _connection_timeout = int(obj_attr_list["update_frequency"]) obj_attr_list["comments"] += "Had to increase ssh timeout. " else : _connection_timeout = int(obj_attr_list["update_frequency"])*2 obj_attr_list["comments"] += "Had to increase ssh timeout one more time. " if str(obj_attr_list["use_jumphost"]).lower() == "true" : if "ssh_config_file" in obj_attr_list: _ssh_conf_file = obj_attr_list["ssh_config_file"] else: _ssh_conf_file = None else : _ssh_conf_file = None _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["prov_cloud_ip"], \ priv_key = obj_attr_list["identity"], \ config_file = _ssh_conf_file, connection_timeout = _connection_timeout) try : _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_command_to_run) if not _status : _vm_is_booted = True else : _vm_is_booted = False except : _vm_is_booted = False elif obj_attr_list["check_boot_complete"].count("snmpget_poll") : import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try : _msg = "Opening SNMP session to " + obj_attr_list["cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, \ DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList(netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except : if _snmp_session.ErrorStr : _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else : _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None ) : _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else : _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) _pooling_time = int(time()) - _start_pooling if _pooling_time <= _wait : _actual_wait = _wait - _pooling_time else : _msg = "The time spent on pooling for \"booted\" status (" + str(_pooling_time) _msg += " s) is actually longer than the " _msg += "interval between pooling attempts (" + str(_wait) + " s)." cbdebug(_msg, True) _actual_wait = 0 if _vm_is_booted : obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc self.pending_set(obj_attr_list, "Network accessible now. Continuing...") _network_reachable = True break else : _msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str(_actual_wait) _msg += " seconds and check again." self.pending_set(obj_attr_list, _msg) cbdebug(_msg) sleep(_actual_wait) _curr_tries += 1 if _curr_tries < _max_tries : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006 and mgt_007 NOT mgt_005 obj_attr_list["mgt_006_instance_preparation"] = "0" obj_attr_list["mgt_007_application_start"] = "0" self.pending_set(obj_attr_list, "Application starting up...") self.get_attr_from_pending(obj_attr_list, "all") else : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc): ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) _network_reachable = False _curr_tries = 0 if not _network_reachable: _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] if str(obj_attr_list["use_jumphost"]).lower() == "false": _msg += "..." else: _msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..." obj_attr_list["check_boot_complete"] = "run_command_/bin/true" cbdebug(_msg, True) self.pending_set(obj_attr_list, _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries: _start_pooling = int(time()) if "async" not in obj_attr_list or str( obj_attr_list["async"]).lower() == "false": if threading.current_thread().abort: _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_"): _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list[ "check_boot_complete"].replace("tcp_on_", '') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port( int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("cloud_ping"): _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish network connectivity " _msg += "through the cloud's API" cbdebug(_msg) _vm_is_booted = self.is_vm_alive(obj_attr_list) elif obj_attr_list["check_boot_complete"].count( "subscribe_on_"): _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list[ "check_boot_complete"].replace("subscribe_on_", '') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list[ "cloud_vm_uuid"] + ") has booted by " _msg += "subscribing to channel \"" + str( _channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe( obj_attr_list["cloud_name"], "VM", _channel_to_subscribe, _max_tries * _wait) for _message in _sub_channel.listen(): if str(_message["data"]).count(_string_to_search): _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_"): _boot_wait_time = int( obj_attr_list["check_boot_complete"].replace( "wait_for_", '')) _msg = "Assuming that the VM \"" + obj_attr_list[ "cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str( _boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time: sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count( "run_command_"): _command_to_run = obj_attr_list[ "check_boot_complete"].replace("run_command_", '') _command_to_run = _command_to_run.replace("____", ' ') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list[ "cloud_vm_uuid"] + ") has booted by " _msg += "running the command \"" + str(_command_to_run) cbdebug(_msg) if _curr_tries <= _max_tries / 3: _connection_timeout = int( obj_attr_list["update_frequency"]) / 2 elif _curr_tries > _max_tries / 3 and _curr_tries < 2 * _max_tries / 3: _connection_timeout = int( obj_attr_list["update_frequency"]) obj_attr_list[ "comments"] += "Had to increase ssh timeout. " else: _connection_timeout = int( obj_attr_list["update_frequency"]) * 2 obj_attr_list[ "comments"] += "Had to increase ssh timeout one more time. " if str(obj_attr_list["use_jumphost"]).lower() == "true": if "ssh_config_file" in obj_attr_list: _ssh_conf_file = obj_attr_list["ssh_config_file"] else: _ssh_conf_file = None else: _ssh_conf_file = None _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["prov_cloud_ip"], \ priv_key = obj_attr_list["identity"], \ config_file = _ssh_conf_file, connection_timeout = _connection_timeout) try: _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _command_to_run) if not _status: _vm_is_booted = True else: _vm_is_booted = False except: _vm_is_booted = False elif obj_attr_list["check_boot_complete"].count( "snmpget_poll"): import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try: _msg = "Opening SNMP session to " + obj_attr_list[ "cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, \ DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList( netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except: if _snmp_session.ErrorStr: _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else: _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None): _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else: _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) _pooling_time = int(time()) - _start_pooling if _pooling_time <= _wait: _actual_wait = _wait - _pooling_time else: _msg = "The time spent on pooling for \"booted\" status (" + str( _pooling_time) _msg += " s) is actually longer than the " _msg += "interval between pooling attempts (" + str( _wait) + " s)." cbdebug(_msg, True) _actual_wait = 0 if _vm_is_booted: obj_attr_list["mgt_004_network_acessible"] = int( time()) - time_mark_prc self.pending_set(obj_attr_list, "Network accessible now. Continuing...") _network_reachable = True break else: _msg = "(" + str( _curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str( _actual_wait) _msg += " seconds and check again." self.pending_set(obj_attr_list, _msg) cbdebug(_msg) sleep(_actual_wait) _curr_tries += 1 if _curr_tries < _max_tries: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006 and mgt_007 NOT mgt_005 obj_attr_list["mgt_006_instance_preparation"] = "0" obj_attr_list["mgt_007_application_start"] = "0" self.pending_set(obj_attr_list, "Application starting up...") self.get_attr_from_pending(obj_attr_list, "all") else: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "is not network reachable after " + str( _max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) : ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) if not self.get_svm_stub(obj_attr_list) : _network_reachable = False else: _network_reachable = True _curr_tries = 0 if not _network_reachable : _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] + "..." cbdebug(_msg, True) self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", obj_attr_list["uuid"], _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries : if "async" not in obj_attr_list or obj_attr_list["async"].lower() == "false" : if threading.current_thread().abort : _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_") : _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("subscribe_on_") : _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_uuid"] + ") has started by " _msg += "subscribing to channel \"" + str(_channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe) for _message in _sub_channel.listen() : if str(_message["data"]).count(_string_to_search) : _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_") : _boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",'')) _msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str(_boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time : sleep(_boot_wait_time) _vm_is_booted = True else : _vm_is_booted = False if _vm_is_booted : obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", obj_attr_list["uuid"], \ "Network accessible now. Continuing...") _network_reachable = True break else : _msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str(_wait) _msg += " seconds and check again." self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", \ obj_attr_list["uuid"], \ _msg) cbdebug(_msg) sleep(_wait) _curr_tries += 1 if _curr_tries < _max_tries : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006, NOT mgt_005 obj_attr_list["mgt_006_application_start"] = "0" self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \ obj_attr_list["uuid"], \ "Application starting up...") else : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) : ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) if not self.get_svm_stub(obj_attr_list) : _network_reachable = False else: _network_reachable = True _curr_tries = 0 if not _network_reachable : _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] + "..." cbdebug(_msg, True) self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", obj_attr_list["uuid"], _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries : if "async" not in obj_attr_list or obj_attr_list["async"].lower() == "false" : if threading.current_thread().abort : _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_") : _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("subscribe_on_") : _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_uuid"] + ") has started by " _msg += "subscribing to channel \"" + str(_channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe) for _message in _sub_channel.listen() : if str(_message["data"]).count(_string_to_search) : _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_") : _boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",'')) _msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str(_boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time : sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count("snmpget_poll") : import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try : _msg = "Opening SNMP session to " + obj_attr_list["cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList(netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except : if _snmp_session.ErrorStr : _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else : _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None ) : _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else : _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) if _vm_is_booted : obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", obj_attr_list["uuid"], \ "Network accessible now. Continuing...") _network_reachable = True break else : _msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str(_wait) _msg += " seconds and check again." self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", \ obj_attr_list["uuid"], \ _msg) cbdebug(_msg) sleep(_wait) _curr_tries += 1 if _curr_tries < _max_tries : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006, NOT mgt_005 obj_attr_list["mgt_006_application_start"] = "0" self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \ obj_attr_list["uuid"], \ "Application starting up...") else : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ") " _msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc): ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) if not self.get_svm_stub(obj_attr_list): _network_reachable = False else: _network_reachable = True _curr_tries = 0 if not _network_reachable: _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] + "..." cbdebug(_msg, True) self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", obj_attr_list["uuid"], _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries: if "async" not in obj_attr_list or obj_attr_list[ "async"].lower() == "false": if threading.current_thread().abort: _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_"): _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list[ "check_boot_complete"].replace("tcp_on_", '') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port( int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count( "subscribe_on_"): _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list[ "check_boot_complete"].replace("subscribe_on_", '') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list[ "cloud_uuid"] + ") has started by " _msg += "subscribing to channel \"" + str( _channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe( obj_attr_list["cloud_name"], "VM", _channel_to_subscribe) for _message in _sub_channel.listen(): if str(_message["data"]).count(_string_to_search): _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_"): _boot_wait_time = int( obj_attr_list["check_boot_complete"].replace( "wait_for_", '')) _msg = "Assuming that the VM \"" + obj_attr_list[ "cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str( _boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time: sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count( "snmpget_poll"): import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try: _msg = "Opening SNMP session to " + obj_attr_list[ "cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList( netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except: if _snmp_session.ErrorStr: _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else: _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None): _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else: _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) if _vm_is_booted: obj_attr_list["mgt_004_network_acessible"] = int( time()) - time_mark_prc self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", obj_attr_list["uuid"], \ "Network accessible now. Continuing...") _network_reachable = True break else: _msg = "(" + str( _curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str( _wait) _msg += " seconds and check again." self.osci.pending_object_set(obj_attr_list["cloud_name"], \ "VM", \ obj_attr_list["uuid"], \ _msg) cbdebug(_msg) sleep(_wait) _curr_tries += 1 if _curr_tries < _max_tries: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006, NOT mgt_005 obj_attr_list["mgt_006_application_start"] = "0" self.osci.pending_object_set(obj_attr_list["cloud_name"], "VM", \ obj_attr_list["uuid"], \ "Application starting up...") else: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_uuid"] + ") " _msg += "is not network reachable after " + str( _max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)