def set_active_cpus(self, active_cpus_tgt_nr):
'''
TBD
'''
_cpu_state = self.get_cpus_state()
_total_cpus = len(_cpu_state)
_active_cpus = []
_inactive_cpus = []
for _cpu_number, _cpu_state in enumerate(_cpu_state) :
if _cpu_state == '1' and _cpu_number > 0 :
_active_cpus.append(_cpu_number)
elif _cpu_state == '0' :
_inactive_cpus.append(_cpu_number)
active_cpus_tgt_nr = int(active_cpus_tgt_nr)
if active_cpus_tgt_nr > _total_cpus :
_msg = "Unable to activate " + str(active_cpus_tgt_nr) + " CPUs on "
_msg += "the guest \"" + self.host + "\" (maximum number of CPUs "
_msg += "for this guest is " + str(_total_cpus) + ")."
cberr(_msg)
raise self.HotplugMgdConnException(_msg, 3)
elif active_cpus_tgt_nr < 1 :
_msg = "At least 1 CPU must be active on the guest "
_msg += "\"" + self.host + "\"."
cberr(_msg)
raise self.HotplugMgdConnException(_msg, 3)
active_cpus_tgt_nr -= 1 #CPU0 is always active
_active_cpus_nr = len(_active_cpus)
if active_cpus_tgt_nr > _active_cpus_nr :
while _active_cpus_nr < active_cpus_tgt_nr :
_cpu_nr = _inactive_cpus.pop()
try :
self.set_cpu_state(_cpu_nr, "active")
_active_cpus_nr += 1
sleep(3)
except self.HotplugMgdConnException :
_msg = "Error while checking active cpus"
raise self.HotplugMgdConnException(_msg, 3)
elif active_cpus_tgt_nr < _active_cpus_nr :
while _active_cpus_nr > active_cpus_tgt_nr :
_cpu_nr = _active_cpus.pop()
try :
self.set_cpu_state(_cpu_nr, "inactive")
_active_cpus_nr -= 1
sleep(3)
except self.HotplugMgdConnException :
_msg = "Error while checking active cpus"
raise self.HotplugMgdConnException(_msg, 3)
else :
_msg = "The number of active CPUs is equal the number of "
_msg += " targeted active CPUs. There is nothing to be done."
cbdebug(_msg)
return True
def check_security_group(self,vmc_name, security_group_name) :
'''
TBD
'''
_security_group_name = False
if security_group_name :
_msg = "Checking if the security group \"" + security_group_name
_msg += "\" is created on VMC " + vmc_name + "...."
cbdebug(_msg, True)
_security_group_found = False
for security_group in self.ec2conn.get_all_security_groups() :
if security_group.name == security_group_name :
_security_group_found = True
if not _security_group_found :
_msg = "ERROR! Please create the security group \""
_msg += security_group_name + "\" in "
_msg += "Amazon EC2 before proceeding."
_fmsg = _msg
cberr(_msg, True)
else :
_security_group_found = True
return _security_group_found
def vmccleanup(self, obj_attr_list):
"""
TBD
"""
try:
_status = 100
self.connect(obj_attr_list["access"], obj_attr_list["credentials"], obj_attr_list["name"])
_pre_existing_instances = False
_reservations = self.ec2conn.get_all_instances()
_running_instances = True
while _running_instances:
_running_instances = False
for _reservation in _reservations:
for _instance in _reservation.instances:
if "Name" in _instance.tags:
if (
_instance.tags[u"Name"].count("cb-" + obj_attr_list["username"])
and _instance.state == u"running"
):
_instance.terminate()
_running_instances = True
sleep(int(obj_attr_list["update_frequency"]))
_msg = "All running instances on the VMC " + obj_attr_list["name"]
_msg += " were terminated"
cbdebug(_msg)
sleep(int(obj_attr_list["update_frequency"]) * 5)
_msg = "Now all EBS volumes belonging to the just terminated "
_msg += "instances on the VMC " + obj_attr_list["name"] + " will "
_msg += "also be removed."
cbdebug(_msg)
_volumes = self.ec2conn.get_all_volumes()
if len(_volumes):
for unattachedvol in _volumes:
if unattachedvol.status == "available":
_msg = unattachedvol.id + " " + unattachedvol.status
_msg += "... was deleted"
cbdebug(_msg)
unattachedvol.delete()
else:
_msg = unattachedvol.id + " " + unattachedvol.status
_msg += "... still attached and could not be deleted"
cbdebug(_msg)
else:
_msg = "No volumes to remove"
cbdebug(_msg)
_status = 0
except AWSException, obj:
_status = int(obj.error_code)
_fmsg = str(obj.error_message)
def connect(self, access_key_id, secret_key, region = "us-east-1") :
'''
TBD
'''
try :
_status = 100
_region_list = regions(aws_access_key_id = access_key_id, \
aws_secret_access_key = secret_key)
_region_info = False
for _idx in range(0,len(_region_list)) :
if _region_list[_idx].name == region :
_region_info = _region_list[_idx]
_region_hostname = _region_info.endpoint
_msg = "Selected region is " + str(_region_info.name)
cbdebug(_msg)
break
if _region_info :
self.ec2conn = _region_info.connect(aws_access_key_id = \
access_key_id, \
aws_secret_access_key = \
secret_key)
_status = 0
else :
_fmsg = "Unknown EC2 region (" + region + ")"
except AWSException, obj:
_status = int(obj.error_code)
_fmsg = str(obj.error_message)
def kill_process(self, cmdline, kill_options = False) :
'''
TBD
'''
_pid = ['X']
while len(_pid) :
_pid = self.get_pid_from_cmdline(cmdline)
_pid = self.get_pid_from_cmdline(cmdline, kill_options)
if len(_pid) :
_pid = _pid[0]
if self.hostname == "127.0.0.1" :
# Not using SIGKILL is insufficient for threaded python processes
# You need to hard-kill them.
_pid = _pid.replace(self.cloud_name,'')
_cmd = "kill -9 " + str(_pid)
self.run_os_command(_cmd)
sleep(1)
_old_pid = _pid
_msg = "A process with the command line \"" + cmdline + "\" (pid "
_msg += str(_old_pid) + ") was killed successfully."
cbdebug(_msg)
return _old_pid
def retriable_run_os_command(self, cmdline, override_hostname = None, total_attempts = 2) :
'''
TBD
'''
_attempts = 0
while _attempts < int(total_attempts) :
_status, _result_stdout, _result_stderr = self.run_os_command(cmdline, override_hostname)
if not _status and _result_stdout and not _result_stdout.count("NOK") :
break
else :
_msg = "Command \"" + cmdline + "\" failed to execute on "
_msg += "hostname " + str(override_hostname) + " after attempt "
_msg += str(_attempts) + '.'
cbdebug(_msg, True)
_attempts += 1
sleep(3)
if _attempts >= int(total_attempts) :
_status = 17368
_fmsg = "Giving up on executing command \"" + cmdline + "\" on hostname "
_fmsg += str(override_hostname) + ". Too many attempts (" + str(_attempts) + ").\n"
#_fmsg += "STDOUT is :\n" + str(_result_stdout) + '\n'
#_fmsg += "STDERR is :\n" + str(_result_stderr) + '\n'
cberr(_fmsg)
return _status, _fmsg, {"status" : _status, "msg" : _msg, "result" : _status}
else :
_status = 0
_msg = "Command \"" + cmdline + "\" executed on hostname "
_msg += str(override_hostname) + " successfully."
cbdebug(_msg)
return _status, _msg, {"status" : _status, "msg" : _msg, "result" : _status}
def vmdestroy(self, obj_attr_list):
"""
TBD
"""
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_wait = int(obj_attr_list["update_frequency"])
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list:
obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = _time_mark_drs - int(
obj_attr_list["mgt_901_deprovisioning_request_originated"]
)
if not self.nodeman:
self.connect(obj_attr_list["access"], obj_attr_list["credentials"], obj_attr_list["vmc_name"])
_wait = int(obj_attr_list["update_frequency"])
_instance = self.get_instances(obj_attr_list, "vm", obj_attr_list["cloud_vm_name"])
if _instance:
_msg = 'Wait until all active transactions for Instance "' + obj_attr_list["name"] + '"'
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += " are finished...."
cbdebug(_msg, True)
while not self.is_vm_running(obj_attr_list) and _curr_tries < _max_tries:
sleep(int(obj_attr_list["update_frequency"]))
_curr_tries += 1
_msg = 'Sending a termination request for Instance "' + obj_attr_list["name"] + '"'
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
self.nodeman.cancel_instance(obj_attr_list["cloud_vm_uuid"])
sleep(_wait)
while self.is_vm_running(obj_attr_list):
sleep(_wait)
else:
True
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = _time_mark_drc - _time_mark_drs
_status = 0
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def retriable_run_os_command(self, cmdline, override_hostname = None, \
total_attempts = 2, retry_interval = 3,
really_execute = True, \
debug_cmd = False, \
raise_exception_on_error = False) :
'''
TBD
'''
_attempts = 0
while _attempts < int(total_attempts) :
try :
_status, _result_stdout, _result_stderr = self.run_os_command(cmdline, \
override_hostname, \
really_execute, \
debug_cmd)
except ProcessManagement.ProcessManagementException, obj :
_status = obj.status
_result_stdout = "NOK"
_result_stderr = obj.msg
if _status and len(_result_stderr) :
_msg = "Command \"" + cmdline + "\" failed to execute on "
_msg += "hostname " + str(override_hostname) + " after attempt "
_msg += str(_attempts) + ". Will try " + str(total_attempts - _attempts)
_msg += " more times."
cbdebug(_msg, True)
_attempts += 1
sleep(retry_interval)
else :
break
def vmdestroy(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list :
obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
_msg = "Sending a termination request for " + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
if obj_attr_list["host"] != "NA" :
self.host_resource_update(obj_attr_list, "destroy")
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
def nodeunregister(self, obj_attr_list) :
'''
TBD
'''
try :
obj_attr_list["uuid"] = "undefined"
_fmsg = "Hypervisor node \"" + obj_attr_list["name"] + "\" ("
_fmsg = obj_attr_list["uuid"] + ") could not be unregistered "
_fmsg += " on this Parallel Libvirt Manager."
self.connect(obj_attr_list["access"])
_status, _msg, _info = self.plmconn.node_unregister(obj_attr_list["name"], obj_attr_list["function"])
if len(_info) :
obj_attr_list.update(_info)
obj_attr_list["uuid"] = obj_attr_list["cloud_uuid"]
obj_attr_list["departure"] = int(time())
cbdebug(_msg, True)
return 0, _msg, obj_attr_list
except PLMException, obj :
status = obj.status
_fmsg = "PLM Exception: " + obj.msg
return status, _fmsg, None
def vmccleanup(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
sleep(_wait)
self.common_messages("VMC", obj_attr_list, "cleaning up vms", 0, '')
_running_instances = True
while _running_instances and _curr_tries < _max_tries :
_running_instances = False
for _endpoint in self.lxdconn :
_proc_man = ProcessManagement(username = "******", \
hostname = _endpoint, \
cloud_name = obj_attr_list["cloud_name"])
_cmd = "sudo pkill -9 -f 'rinetd -c /tmp/cb'; sudo rm -rf /tmp/cb-*.rinetd.conf"
_status, _result_stdout, _fmsg = _proc_man.run_os_command(_cmd, raise_exception=False)
_container_list = self.lxdconn[_endpoint].containers.all()
for _container in _container_list :
if _container.name.count("cb-" + obj_attr_list["username"] + '-' + obj_attr_list["cloud_name"]) :
_running_instances = True
_msg = "Terminating instance: "
_msg += self.generate_random_uuid(_container.name) + " (" + str(_container.name) + ")"
cbdebug(_msg, True)
if _container.status == "Running" :
_container.stop()
_container.delete()
sleep(_wait)
_curr_tries += 1
if _curr_tries > _max_tries :
_status = 1077
_fmsg = "Some instances on VMC \"" + obj_attr_list["name"] + "\""
_fmsg += " could not be removed because they never became active"
_fmsg += ". They will have to be removed manually."
cberr(_msg, True)
else :
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
def retriable_run_os_command(self, cmdline, override_hostname_or_uuid = None, \
total_attempts = 2, retry_interval = 3,
really_execute = True, \
debug_cmd = False, \
raise_exception_on_error = False, \
step = None,
tell_me_if_stderr_contains = False, \
port = 22, \
remaining_time = 100000, \
osci = False, \
get_hostname_using_key = False):
'''
TBD
'''
_attempts = 0
_abort = "no"
_start = int(time())
_spent_time = 0
_override_hostname = "x.x.x.x-invalid"
while _attempts < int(total_attempts) and _abort != "yes" :
try :
if osci is not False and get_hostname_using_key is not False :
_obj_attr_list = osci.get_object(self.cloud_name, "VM", False, override_hostname_or_uuid, False)
_override_hostname = _obj_attr_list[get_hostname_using_key]
else :
_override_hostname = override_hostname_or_uuid
_status, _result_stdout, _result_stderr = self.run_os_command(cmdline, \
_override_hostname, \
really_execute, \
debug_cmd, \
raise_exception_on_error, \
step = step,
tell_me_if_stderr_contains = tell_me_if_stderr_contains, \
port = port)
except ProcessManagement.ProcessManagementException, obj :
if obj.status == "90001" :
raise obj
_status = obj.status
_result_stdout = "NOK"
_result_stderr = obj.msg
if _status and len(_result_stderr) :
_msg = "Command \"" + cmdline + "\" failed to execute on "
_msg += "hostname " + str(_override_hostname) + " after attempt "
_msg += str(_attempts) + ". Will try " + str(total_attempts - _attempts)
_msg += " more times."
cbdebug(_msg, True)
_attempts += 1
sleep(retry_interval)
_spent_time = int(time()) - _start
if _spent_time > remaining_time :
_abort = "yes"
else :
break
def vvdestroy(self, obj_attr_list, identifier):
"""
TBD
"""
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_wait = int(obj_attr_list["update_frequency"])
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
_instance = self.get_instances(obj_attr_list, "vm", obj_attr_list["cloud_vv_name"])
if _instance:
_msg = "Sending a destruction request for the Volume"
_msg += ' previously attached to "'
_msg += obj_attr_list["name"] + '"'
_msg += " (cloud-assigned uuid " + identifier + ")...."
cbdebug(_msg, True)
_operation = (
self.gceconn.disks()
.delete(project=self.project, zone=self.zone, disk=obj_attr_list["cloud_vv_name"])
.execute()
)
self.wait_until_operation(obj_attr_list, _operation)
_status = 0
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def vmcregister(self, obj_attr_list) :
status = 0
msg = ""
try :
cluster_list = obj_attr_list["clusters"].lower().strip().split(",")
region = False
size = False
version = False
worker_size = False
for cluster in cluster_list :
name, region, version, worker_size, nb_workers = cluster.split(":")
if name == obj_attr_list["name"] :
cbdebug("VMC " + name + " in " + region + " using version " + version + " and " + nb_workers + " workers each of size " + worker_size, True)
break
if not region :
return 104, "VMC " + name + " not found in CLUSTERS configuration list. Please correct and try again: " + cluster_list
obj_attr_list["region"] = region
obj_attr_list["nb_workers"] = nb_workers
obj_attr_list["k8s_version"] = version
obj_attr_list["k8s_worker_size"] = worker_size
if "kubeconfig" not in obj_attr_list :
self.access = obj_attr_list["access"]
self.headers = {"Authorization" : "Bearer " + obj_attr_list["credentials"]}
obj_attr_list["kuuid"], obj_attr_list["kubeconfig"] = self.create_cluster(obj_attr_list)
if not obj_attr_list["kuuid"] :
return 458, "vmcregister did not find a UUID, No k8s for you."
status, msg = KubCmds.vmcregister(self, obj_attr_list)
except Exception, e :
for line in traceback.format_exc().splitlines() :
cbwarn(line, True)
status, msg = self.common_messages("VMC", obj_attr_list, "registered", status, msg)
def check_security_group(self, vmc_name, security_group_name):
"""
TBD
"""
_security_group_name = False
if security_group_name:
_msg = 'Checking if the security group "' + security_group_name
_msg += '" is created on VMC ' + vmc_name + "...."
cbdebug(_msg, True)
_security_group_found = False
for security_group in self.gceconn.get_all_security_groups():
if security_group.name == security_group_name:
_security_group_found = True
if not _security_group_found:
_msg = 'ERROR! Please create the security group "'
_msg += security_group_name + '" in '
_msg += "Google CE before proceeding."
_fmsg = _msg
cberr(_msg, True)
else:
_security_group_found = True
return _security_group_found
def connect(self, project, secret_key, zone="us-east1-b"):
"""
TBD
"""
try:
_status = 100
if not self.project:
self.project = project
_credentials = GoogleCredentials.get_application_default()
self.gceconn = build("compute", "v1", credentials=_credentials)
_zone_list = self.gceconn.zones().list(project=self.project).execute()["items"]
_zone_info = False
for _idx in range(0, len(_zone_list)):
if _zone_list[_idx]["description"] == zone:
_zone_info = _zone_list[_idx]
_zone_hostname = _zone_info["region"]
_msg = "Selected zone is " + str(_zone_info["description"])
cbdebug(_msg)
break
if _zone_info:
self.zone = zone
_status = 0
else:
_fmsg = "Unknown GCE zone (" + zone + ")"
except GCEException, obj:
_status = int(obj.error_code)
_fmsg = str(obj.error_message)
def is_vm_running(self, obj_attr_list):
'''
TBD
'''
try :
if "instance_obj" not in obj_attr_list :
_instance = self.get_vm_instance(obj_attr_list)
else :
_instance = obj_attr_list["instance_obj"]
if _instance :
_msg = "is_vm_running reports instance with instance state of " + str(_instance.state)
cbdebug(_msg)
_instance_state = _instance.state
else :
_msg = "is_vm_running reports non-existant instance."
cbdebug(_msg)
_instance_state = "non-existent"
if _instance_state == NodeState.RUNNING :
return True
else :
return False
except Exception, e :
_status = 23
_fmsg = str(e)
raise CldOpsException(_fmsg, _status)
def vmcregister(self, obj_attr_list) :
_fmsg = "none"
try :
_msg = "Attempting to attach a new VMC..."
cbdebug(_msg)
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_prs = int(time())
obj_attr_list["mgt_002_provisioning_request_sent"] = _time_mark_prs - int(obj_attr_list["mgt_001_provisioning_request_originated"])
if "cleanup_on_attach" in obj_attr_list and obj_attr_list["cleanup_on_attach"] == "True" :
_msg = "Cleaning up VMC before attaching it."
cbdebug(_msg)
_status, _fmsg = self.vmccleanup(obj_attr_list)
obj_attr_list["cloud_hostname"] = obj_attr_list["name"]
obj_attr_list["cloud_ip"] = obj_attr_list["name"] + ".digitalocean.com"
obj_attr_list["arrival"] = int(time())
_time_mark_prc = int(time())
obj_attr_list["mgt_003_provisioning_request_completed"] = _time_mark_prc - _time_mark_prs
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
cberr(str(obj))
def start_daemon(self, cmdline, port = None, protocol = None, conditional = False, search_keywords = None) :
'''
TBD
'''
if self.hostname == "127.0.0.1" :
_cmd = cmdline
if port :
_pid, _username = self.get_pid_from_port(port, protocol)
if _pid :
if _username != self.username :
return [ "pnf-" + _pid + '-' + _username ]
if conditional :
_pid = self.get_pid_from_cmdline(_cmd if not search_keywords else search_keywords)
if not _pid :
_status, _a, _b = self.run_os_command(_cmd)
# The process could be a daemon, so there is no point in waiting
# for its completion. However, we do wait a little bit just to be
# able to get the pid of the process, because a daemon would fork
# a new process.
sleep(3)
else :
_status, _a, _b = self.run_os_command(_cmd)
# Same comment
sleep(3)
_pid = self.get_pid_from_cmdline(cmdline if not search_keywords else search_keywords)
_msg = "A process with the command line \"" + cmdline + "\" (pid "
_msg += str(_pid) + ") was started succesfully."
cbdebug(_msg)
return [ _pid ]
def hostname2ip(hostname, raise_exception = False) :
'''
TBD
'''
try :
cbdebug("Looking for host name/IP: " + hostname)
if validIPv4(hostname) :
_x = "ip address"
_hostip = hostname
hostname = gethostbyaddr(hostname)[0]
if hostname.count("in-addr.arpa") :
hostname = hostname.replace(".in-addr.arpa",'')
hostname = hostname.split('.')[0]
else :
_x = "host name"
_hostip = gethostbyname(hostname)
_status = 0
except socket.gaierror :
_status = 1200
_msg = "Error while attempting to resolve the " + _x + " \"" + hostname + "\"."
_msg += " Please make sure this name is resolvable either in /etc/hosts or DNS."
except socket.herror:
_status = 1200
_msg = "Error while attempting to resolve the " + _x + " \"" + hostname + "\"."
_msg += " Please make sure this name is resolvable either in /etc/hosts or DNS."
except Exception, e :
_status = 23
_msg = "Error while attempting to resolve the " + _x + " \"" + hostname + "\":" + str(e)
def vmrunstate(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
if "mgt_201_runstate_request_originated" in obj_attr_list :
_time_mark_rrs = int(time())
obj_attr_list["mgt_202_runstate_request_sent"] = \
_time_mark_rrs - obj_attr_list["mgt_201_runstate_request_originated"]
_time_mark_rrc = int(time())
obj_attr_list["mgt_203_runstate_request_completed"] = \
_time_mark_rrc - _time_mark_rrs
_msg = "VM " + obj_attr_list["name"] + " runstate completed."
cbdebug(_msg)
sleep(5)
_status = 0
except Exception, e :
_status = 23
_fmsg = str(e)
def nmap(self, port = None, protocol = "TCP", reverse = False) :
'''
TBD
'''
try :
if protocol == "TCP" :
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
elif protocol == "UDP" :
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.settimeout(5)
self.socket.connect((self.hostname, self.port if port is None else port))
if not reverse :
return True
else :
_msg = protocol + " port " + str(port) + " on host "
_msg += self.hostname + " is NOT free"
cberr(_msg)
raise NetworkException(str(_msg), "1")
except socket.error, msg :
if not reverse :
_msg = "Unable to connect to " + protocol + " port " + str(port)
_msg += " on host " + self.hostname + ": " + str(msg)
cberr(_msg)
raise NetworkException(str(_msg), "1")
else :
_msg = protocol + " port " + str(port) + " on host "
_msg += self.hostname + ": " + str(msg) + "is free"
cbdebug(_msg)
return True
self.socket.close()
self.socket = None
def vvdestroy(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_wait = int(obj_attr_list["update_frequency"])
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
if "cloud_vv_uuid" in obj_attr_list and obj_attr_list["cloud_vv_uuid"].lower() != "none" :
_msg = "Sending a destruction request for the Volume"
_msg += " previously attached to \""
_msg += obj_attr_list["name"] + "\""
_msg += " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_vv_uuid"] + ")...."
cbdebug(_msg, True)
_volume_detached = False
while not _volume_detached and _curr_tries < _max_tries :
_instance = self.ec2conn.get_all_volumes(volume_ids = [obj_attr_list["cloud_vv_uuid"]])
_volume=_instance[0]
_status = _volume.status
if _status == 'available' :
_volume_detached = True
else :
_msg = " Volume previously attached to \""
_msg += obj_attr_list["name"] + "\""
_msg += " (cloud-assigned uuid "
_msg += obj_attr_list["cloud_vv_uuid"] + ") status "
_msg += "is still \"" + _status + "\". "
_msg += "Will wait " + str(_wait)
_msg += " seconds and try again."
cbdebug(_msg)
sleep(_wait)
_curr_tries += 1
if _curr_tries > _max_tries :
_status = 1077
_fmsg = obj_attr_list["name"] + "\""
_fmsg += " (cloud-assigned uuid "
_fmsg += obj_attr_list["cloud_vv_uuid"] + ") "
_fmsg += "could not be destroyed after " + str(_max_tries * _wait) + " seconds.... "
cberr(_msg)
else :
_volume.delete()
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
def connect(self, access_token) :
# libcloud is totally not thread-safe. bastards.
cbdebug("Checking libcloud connection...")
try :
getattr(catalogs, "digitalocean")
except Exception, e :
cbdebug("Initializing thread local connection.")
catalogs.digitalocean = False
def run_os_command(self, cmdline, override_hostname = None) :
'''
TBD
'''
if override_hostname :
_hostname = override_hostname
else :
_hostname = self.hostname
if _hostname == "127.0.0.1" or _hostname == "0.0.0.0" :
_local = True
else :
_local = False
if _local :
_cmd = cmdline
else :
if self.username :
_username = "******" + self.username + ' '
else :
_username = ''
if self.priv_key :
_priv_key = " -i " + self.priv_key + ' '
else :
_priv_key = ''
_cmd = "ssh " + _priv_key + " -o StrictHostKeyChecking=no "
_cmd += "-o UserKnownHostsFile=/dev/null " + _username
_cmd += _hostname + " \"" + cmdline + "\""
_msg = "running os command: " + _cmd
cbdebug(_msg);
_proc_h = Popen(_cmd, shell=True, stdout=PIPE, stderr=PIPE)
if _proc_h.pid :
if not cmdline.count("--debug_host=localhost") :
_result = _proc_h.communicate()
if _proc_h.returncode and len(_result[1]) and _local :
_msg = "Error while checking for a pid for a process with the "
_msg += "command line \"" + cmdline + "\" (returncode = "
_msg += str(_proc_h.pid) + ") :" + str(_result[1])
raise self.ProcessManagementException(str(_msg), "81918")
else :
_status = 0
_result_stdout = _result[0]
_result_stderr = _result[1]
else :
_status = 0
_result_stdout = ""
_result_stderr = ""
else :
_msg = "Error running the command \"" + cmdline + "\"."
_status = 81713
_result_stdout = ""
_result_stderr = ""
return _status, _result_stdout, _result_stderr
def aiundefine(self, obj_attr_list, current_step):
"""
TBD
"""
_msg = "AI " + obj_attr_list["uuid"] + " was successfully "
_msg += 'undefined on FTCloud "' + obj_attr_list["cloud_name"]
_msg += '".'
cbdebug(_msg, True)
return 0, _msg
def vmcregister(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_prs = int(time())
obj_attr_list[
"mgt_002_provisioning_request_sent"] = _time_mark_prs - int(
obj_attr_list["mgt_001_provisioning_request_originated"])
if "cleanup_on_attach" in obj_attr_list and obj_attr_list[
"cleanup_on_attach"] == "True":
_msg = "Removing all VMs previously created on VMC \""
_msg += obj_attr_list[
"name"] + "\" (only VMs names starting with"
_msg += " \"" + "cb-" + obj_attr_list[
"username"] + '-' + obj_attr_list["cloud_name"]
_msg += "\")....."
cbdebug(_msg, True)
_status, _fmsg = self.vmccleanup(obj_attr_list)
else:
_status = 0
if not _status:
if type(self.plmconn) is type(False):
self.connect(obj_attr_list["access"], \
obj_attr_list["credentials"], \
obj_attr_list["name"])
_status, _msg, _info = self.plmconn.group_register(
obj_attr_list["name"])
_network_address = _info["computenodes"].split(',')[0]
obj_attr_list["cloud_hostname"], obj_attr_list[
"cloud_ip"] = hostname2ip(_network_address)
obj_attr_list["arrival"] = int(time())
if obj_attr_list["discover_hosts"].lower() == "true":
_msg = "Discovering hosts on VMC \"" + obj_attr_list[
"name"] + "\"....."
cbdebug(_msg, True)
_status, _fmsg = self.discover_hosts(
obj_attr_list, _time_mark_prs)
else:
obj_attr_list["hosts"] = ''
obj_attr_list["host_list"] = {}
obj_attr_list["host_count"] = "NA"
_time_mark_prc = int(time())
obj_attr_list[
"mgt_003_provisioning_request_completed"] = _time_mark_prc - _time_mark_prs
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def aiundefine(self, obj_attr_list) :
'''
TBD
'''
_msg = "AI " + obj_attr_list["uuid"] + " was successfully "
_msg += "undefined on TACloud \"" + obj_attr_list["cloud_name"]
_msg += "\"."
cbdebug(_msg, True)
return 0, _msg
def vmccleanup(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
# check for old instances and destroy
self.connect(obj_attr_list["access"], obj_attr_list["credentials"], obj_attr_list["name"])
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
_wait = int(obj_attr_list["update_frequency"])
sleep(_wait)
_running_instances = True
_status = 0
while _running_instances and not _status and _curr_tries < _max_tries :
_running_instances = False
_instance_list = self.query("describe_all_instances", obj_attr_list)
for _instance in _instance_list :
if _instance["instance_tag"].count("cb-" + obj_attr_list["username"]) :
_running_instances = True
if _instance["state"] == "running" :
_msg = "Terminating instance \""
_msg += _instance["instance_id"] + "\""
_msg += " (" + _instance["instance_tag"] + ")"
cbdebug(_msg, True)
_instance["cloud_uuid"] = _instance["gid"]
self.query("terminate_instances", _instance, False)
if _instance["state"] == "starting" :
_msg = "Will wait for instance "
_msg += _instance["instance_id"] + "\""
_msg += " (" + _instance["instance_tag"] + ") to"
_msg += "start and then destroy it."
cbdebug(_msg, True)
_msg = "Some instances are still starting on VMC \"" + obj_attr_list["name"]
_msg += "\". Will wait for " + str(_wait) + " seconds and check again."
sleep(_wait)
_curr_tries += 1
if _curr_tries > _max_tries :
_status = 1077
_fmsg = "Some instances on VMC \"" + obj_attr_list["name"] + "\""
_fmsg += " could not be removed because they never became active"
_fmsg += ". They will have to be removed manually."
cberr(_msg, True)
sleep(int(obj_attr_list["update_frequency"]))
except CldOpsException, obj :
_status = int(obj.error_code)
_fmsg = str(obj.error_message)
def vvdestroy(self, obj_attr_list, identifier) :
'''
TBD
'''
try :
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_wait = int(obj_attr_list["update_frequency"])
_curr_tries = 0
_max_tries = int(obj_attr_list["update_attempts"])
if str(obj_attr_list["cloud_vv_uuid"]).lower() != "none" :
_instance = self.get_instances(obj_attr_list, "vv", identifier)
if _instance :
self.common_messages("VV", obj_attr_list, "destroying", 0, '')
_volume_detached = False
while not _volume_detached and _curr_tries < _max_tries :
_status = _instance.status
if _status == 'available' :
_instance.delete()
_volume_detached = True
else :
_msg = " Volume previously attached to \""
_msg += obj_attr_list["name"] + "\""
_msg += " (cloud-assigned uuid "
_msg += identifier + ") status "
_msg += "is still \"" + _status + "\". "
_msg += "Will wait " + str(_wait)
_msg += " seconds and try again."
cbdebug(_msg)
sleep(_wait)
_curr_tries += 1
if _curr_tries > _max_tries :
_status = 1077
_fmsg = " Volume previously attached to \""
_fmsg += obj_attr_list["name"] + "\""
_fmsg += " (cloud-assigned uuid "
_fmsg += identifier + ") "
_fmsg += "could not be destroyed after " + str(_max_tries * _wait) + " seconds.... "
cberr(_msg)
else :
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
def vmdestroy(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list:
obj_attr_list[
"mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
self.common_messages("VM", obj_attr_list, "destroying", 0, '')
if obj_attr_list["cloud_ip"] != "undefined" and obj_attr_list[
"role"] != "check":
_proc_man = ProcessManagement(username = obj_attr_list["login"], \
cloud_name = obj_attr_list["cloud_name"], \
hostname = obj_attr_list["cloud_ip"], \
priv_key = obj_attr_list["identity"])
_cmd = "~/cb_cleanup.sh; rm ~/cb_*"
_msg = "Shutting down CloudBench Load Manager/Metric Aggregator on "
_msg += "VM \"" + obj_attr_list["name"] + "\" by executing the "
_msg += "command \"" + _cmd + "\""
cbdebug(_msg, True)
_status, _result_stdout, _result_stderr = _proc_man.run_os_command(
_cmd)
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
_status = 0
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def vmrunstate(self, obj_attr_list):
'''
TBD
'''
try:
_status = 100
_ts = obj_attr_list["target_state"]
_cs = obj_attr_list["current_state"]
self.connect(obj_attr_list["access"], obj_attr_list["credentials"], \
obj_attr_list["vmc_name"])
if "mgt_201_runstate_request_originated" in obj_attr_list:
_time_mark_rrs = int(time())
obj_attr_list["mgt_202_runstate_request_sent"] = \
_time_mark_rrs - obj_attr_list["mgt_201_runstate_request_originated"]
self.common_messages("VM", obj_attr_list, "runstate altering", 0,
'')
_instance = self.get_instances(obj_attr_list, "vm", "vmuuid")
if _instance:
if _ts == "fail":
_instance.stop()
elif _ts == "save":
_instance.stop()
elif (_ts == "attached" or _ts == "resume") and _cs == "fail":
_instance.start()
elif (_ts == "attached" or _ts == "restore") and _cs == "save":
_instance.start()
_time_mark_rrc = int(time())
obj_attr_list[
"mgt_203_runstate_request_completed"] = _time_mark_rrc - _time_mark_rrs
_msg = "VM " + obj_attr_list[
"name"] + " runstate request completed."
cbdebug(_msg)
_status = 0
except CldOpsException, obj:
_status = obj.status
_fmsg = str(obj.msg)
def __init__(self, pid, passive, active, background, debug, port,
hostname):
super(APIService, self).__init__()
self._stop = threading.Event()
self.pid = pid
self.abort = False
self.aborted = False
self.port = port
self.hostname = hostname
self.api = API(pid, passive, active, background)
cbdebug("Initializing API Service on port " + str(self.port))
if debug is None:
self.server = AsyncDocXMLRPCServer((self.hostname, int(self.port)),
allow_none=True)
else:
self.server = DocXMLRPCServer((self.hostname, int(self.port)),
allow_none=True)
self.server.abort = False
self.server.aborted = False
self.server.set_server_title("API Service (xmlrpc)")
self.server.set_server_name("API Service (xmlrpc)")
#self.server.register_introspection_functions()
self.api.signatures = {}
for methodtuple in inspect.getmembers(self.api,
predicate=inspect.ismethod):
name = methodtuple[0]
if name in ["__init__", "success", "error"]:
continue
func = getattr(self.api, name)
argspec = inspect.getargspec(func)
spec = argspec[0]
defaults = [] if argspec[3] is None else argspec[3]
num_spec = len(spec)
num_defaults = len(defaults)
diff = num_spec - num_defaults
named = diff - 1
doc = "Usage: "
for x in range(1, diff):
doc += spec[x] + ", "
for x in range(diff, num_spec):
doc += spec[x] + " = " + str(defaults[x - diff]) + ", "
doc = doc[:-2]
self.api.signatures[name] = {"args": spec[1:], "named": named}
self.server.register_function(unwrap_kwargs(func, doc), name)
# self.server.register_instance(self.api)
cbdebug("API Service started")
def vmreplicate_stop(self, obj_attr_list):
'''
TBD
'''
self.connect(obj_attr_list["access"])
kwargs = self.dic_to_rpc_kwargs(self.plmconn, "ft_stop", obj_attr_list)
_status, _fmsg, unused = self.plmconn.ft_stop(**kwargs)
'''
A failure to stop replication is not fatal.
It should not prevent the stub from being destroyed....
'''
_msg = "SVM FT replication for " + obj_attr_list["primary_name"]
_msg += (" could not be stopped: " +
_fmsg) if _status else " was stopped."
cbdebug(_msg, True)
return 0, _msg
def vmrunstate(self, obj_attr_list) :
'''
TBD
'''
try :
_status = 100
_ts = obj_attr_list["target_state"]
_cs = obj_attr_list["current_state"]
self.connect(obj_attr_list["access"], obj_attr_list["api_key"], \
obj_attr_list["secret_key"])
if "mgt_201_runstate_request_originated" in obj_attr_list :
_time_mark_rrs = int(time())
obj_attr_list["mgt_202_runstate_request_sent"] = \
_time_mark_rrs - obj_attr_list["mgt_201_runstate_request_originated"]
_msg = "Sending a runstate change request (" + _ts + " for " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
_instance = self.get_vm_instance(obj_attr_list)
if _instance :
if _ts == "fail" :
self.cskconn.stopVirtualMachine({u'id': _instance[u'id']})
elif _ts == "save" :
self.cskconn.stopVirtualMachine({u'id': _instance[u'id']})
elif (_ts == "attached" or _ts == "resume") and _cs == "fail" :
self.cskconn.startVirtualMachine({u'id': _instance[u'id']})
elif (_ts == "attached" or _ts == "restore") and _cs == "save" :
self.cskconn.startVirtualMachine({u'id': _instance[u'id']})
_time_mark_rrc = int(time())
obj_attr_list["mgt_203_runstate_request_completed"] = _time_mark_rrc - _time_mark_rrs
_msg = "VM " + obj_attr_list["name"] + " runstate request completed."
cbdebug(_msg)
_status = 0
except CldOpsException, obj :
_status = obj.status
_fmsg = str(obj.msg)
def extra_vmccleanup(self, obj_attr_list):
'''
TBD
'''
_wait = int(obj_attr_list["update_frequency"])
_existing_pips = True
while _existing_pips:
_existing_pips = False
for credentials_list in obj_attr_list["credentials"].split(";"):
credentials = credentials_list.split(":")
tenant = credentials[0]
self.common_messages("VMC", obj_attr_list, "cleaning up vvs",
0, '')
obj_attr_list["tenant"] = tenant
_pips = self.get_adapter(credentials_list).ex_list_public_ips(
obj_attr_list["resource_group"])
for _pip in _pips:
if _pip.name.count("cb-" + obj_attr_list["username"] +
"-" + obj_attr_list["cloud_name"]):
try:
cbdebug(
"Destroying: " + _pip.name + " (" + tenant +
")", True)
self.get_adapter(
credentials_list).ex_delete_public_ip(_pip)
except MalformedResponseError as e:
self.dump_httplib_headers(credentials_list)
raise CldOpsException(
"The Cloud's API is misbehaving", 1483)
except Exception as e:
for line in traceback.format_exc().splitlines():
cbwarn(line, True)
self.dump_httplib_headers(credentials_list)
_existing_pips = True
else:
_msg = "Cleaning up " + self.get_description(
) + ". Ignoring Public IP: " + _pip.name
cbdebug(_msg)
if _existing_pips:
_wait = self.backoff(obj_attr_list, _wait)
return True
def cat(self, reset=True):
'''
TBD
'''
result = ''
if (reset and self.socket is not None):
try:
self.socket.close()
except:
True
if reset:
try:
self.connect()
except socket.error as msg:
_msg = "ERROR cannot connect to ganglia gmetad to the port "
_msg += self.port + " on server " + self.hostname + ": "
_msg += str(msg) + '.'
cbdebug(_msg)
self.socket = None
return None
while (1):
_buffer = ''
try:
_buffer = self.socket.recv(1024)
while (_buffer != ''):
result += _buffer
try:
_buffer = self.socket.recv(1024)
except socket.error as err:
print((err, type(err)))
_buffer = ''
if (_buffer == ''):
break
except socket.error as err:
print((err, type(err)))
if (_buffer == ''):
break
self.socket.close()
self.socket = None
return result
def vmcregister(self, obj_attr_list):
status = 0
msg = ""
try:
cluster_list = obj_attr_list["clusters"].lower().strip().split(",")
region = False
size = False
version = False
worker_size = False
for cluster in cluster_list:
name, region, version, worker_size, nb_workers = cluster.split(
":")
if name == obj_attr_list["name"]:
cbdebug(
"VMC " + name + " in " + region + " using version " +
version + " and " + nb_workers +
" workers each of size " + worker_size, True)
break
if not region:
return 104, "VMC " + name + " not found in CLUSTERS configuration list. Please correct and try again: " + cluster_list
obj_attr_list["region"] = region
obj_attr_list["nb_workers"] = nb_workers
obj_attr_list["k8s_version"] = version
obj_attr_list["k8s_worker_size"] = worker_size
if "kubeconfig" not in obj_attr_list:
self.access = obj_attr_list["access"]
self.headers = {
"Authorization": "Bearer " + obj_attr_list["credentials"]
}
obj_attr_list["kuuid"], obj_attr_list[
"kubeconfig"] = self.create_cluster(obj_attr_list)
if not obj_attr_list["kuuid"]:
return 458, "vmcregister did not find a UUID, No k8s for you."
status, msg = KubCmds.vmcregister(self, obj_attr_list)
except Exception as e:
for line in traceback.format_exc().splitlines():
cbwarn(line, True)
status, msg = self.common_messages("VMC", obj_attr_list,
"registered", status, msg)
return status, msg
def vmrunstate(self, obj_attr_list):
_ts = obj_attr_list["target_state"]
_cs = obj_attr_list["current_state"]
self.connect(obj_attr_list["access"])
if "mgt_201_runstate_request_originated" in obj_attr_list:
_time_mark_rrs = int(time())
obj_attr_list["mgt_202_runstate_request_sent"] = \
_time_mark_rrs - obj_attr_list["mgt_201_runstate_request_originated"]
_msg = "Sending a runstate change request (" + _ts + " for " + obj_attr_list[
"name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
kwargs = self.dic_to_rpc_kwargs(self.ftcconn, "suspend", obj_attr_list)
if _ts == "fail":
self.ftcconn.suspend(**kwargs)
elif _ts == "save":
self.ftcconn.save(**kwargs)
elif (_ts == "attached" or _ts == "resume") and _cs == "fail":
self.ftcconn.resume(**kwargs)
elif (_ts == "attached" or _ts == "restore") and _cs == "save":
status, fmsg, result = self.ftcconn.restore(**kwargs)
obj_attr_list.update(result)
self.update_libvirt_variables(obj_attr_list)
if "configured_size" in obj_attr_list:
# For some reason, restored VMs don't maintain their previous
# size properties. Go figure.
self.resize_to_configured_default(obj_attr_list)
_time_mark_rrc = int(time())
obj_attr_list[
"mgt_203_runstate_request_completed"] = _time_mark_rrc - _time_mark_rrs
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "had its runstate successfully "
_msg += "changed on FTCloud \"" + obj_attr_list["cloud_name"]
_msg += "\"."
cbdebug(_msg)
return 0, _msg
def get_pid_from_cmdline(self, cmdline, extra = False) :
'''
TBD
'''
_pid_list = []
_cmd = "sudo ps aux | tr '[:upper:]' '[:lower:]' | grep \"" + cmdline.replace("'", "").lower() + "\" | grep -i \"" + (extra if extra else "") + "\""
if self.cloud_name :
_cmd = _cmd + " | grep \"" + self.cloud_name.lower() + "\""
if self.username :
_cmd = _cmd + " | grep \"" + self.username + "\""
_cmd = _cmd + " | grep -v grep"
if self.hostname != "127.0.0.1" and self.hostname != "localhost" :
_cmd = "ssh " + self.hostname + ' ' + _cmd
_status, _result_stdout, _result_stderr = self.run_os_command(_cmd)
if _status :
_msg = "Error while checking for a pid for a process with the "
_msg += "command line \"" + cmdline + "\" (pid)"
raise self.ProcessManagementException(str(_msg), "81918")
else :
_result_lines = _result_stdout.split('\n')
for _line in _result_lines :
if len(_line) :
if self.username == "" :
_pid = _line.split()[1]
_pid_list.append(_pid)
else :
if _line.count(self.username) :
_pid = _line.split()[1]
_pid_list.append(_pid)
if len(_pid_list) :
_msg = "A pid list for the process with the command line \""
_msg += cmdline + "\" was found."
else :
_msg = "A pid list for the process with the command line \""
_msg += cmdline + "\" was not found."
cbdebug(_msg)
return _pid_list
def disconnect(self) :
try:
if "disconnect" in dir(MysqlMgdConn.catalogs.cbtool["conn"]) :
MysqlMgdConn.catalogs.cbtool["conn"].disconnect()
MysqlMgdConn.catalogs.cbtool["conn"] = False
_msg = "A connection to MySQL running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", was terminated."
cbdebug(_msg)
except mysql.connector.Error as err :
_msg = "Unable to terminate a connection with MySQL "
_msg += "server on host " + self.host + " port "
_msg += str(self.port) + "database " + str(MysqlMgdConn.catalogs.cbtool["database"]) + ": "
_msg += str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 3)
def get_ip_address(self, obj_attr_list):
'''
TBD
'''
self.connect(obj_attr_list["access"])
_status, _fmsg, ip = self.ftcconn.get_ip_address(
obj_attr_list["cloud_mac"])
if ip is None:
cbdebug("ip address not ready for mac " +
obj_attr_list["cloud_mac"])
return False
obj_attr_list["prov_cloud_ip"] = ip
obj_attr_list["run_cloud_ip"] = ip
# NOTE: "cloud_ip" is always equal to "run_cloud_ip"
obj_attr_list["cloud_ip"] = obj_attr_list["run_cloud_ip"]
return True
def get_ip_address(self, obj_attr_list):
'''
TBD
'''
try:
_msg = "Looking of IP address for something."
cbdebug(_msg)
obj_attr_list["cloud_hostname"] = obj_attr_list[
"instance_obj"]._get_public_ips()[0]
obj_attr_list["cloud_ip"] = obj_attr_list[
"instance_obj"]._get_private_ips()[0]
obj_attr_list["prov_cloud_ip"] = obj_attr_list["cloud_ip"]
_msg = "Public IP = " + obj_attr_list["cloud_hostname"]
_msg += " Private IP = " + obj_attr_list["cloud_ip"]
cbdebug(_msg)
return True
except:
return False
def connect(self, access_key_id, secret_key, region = "us-east-1") :
'''
TBD
'''
try :
_status = 100
_region_list = regions(aws_access_key_id = access_key_id, \
aws_secret_access_key = secret_key)
_region_info = False
for _idx in range(0,len(_region_list)) :
if _region_list[_idx].name == region :
_region_info = _region_list[_idx]
_region_hostname = _region_info.endpoint
_msg = "Selected region is " + str(_region_info.name)
cbdebug(_msg)
break
if _region_info :
self.ec2conn = _region_info.connect(aws_access_key_id = \
access_key_id, \
aws_secret_access_key = \
secret_key)
_status = 0
else :
_fmsg = "Unknown " + self.get_description() + " region (" + region + ")"
except AWSException as obj:
_status = int(obj.error_code)
_fmsg = str(obj.error_message)
except Exception as msg :
_fmsg = str(msg)
_status = 23
finally :
if _status :
_msg = self.get_description() + " connection failure: " + _fmsg
cberr(_msg)
raise CldOpsException(_msg, _status)
else :
_msg = self.get_description() + " connection successful."
cbdebug(_msg)
return _status, _msg, _region_hostname
def vmdestroy(self, obj_attr_list):
'''
TBD
'''
_status = 100
_time_mark_drs = int(time())
if "mgt_901_deprovisioning_request_originated" not in obj_attr_list:
obj_attr_list[
"mgt_901_deprovisioning_request_originated"] = _time_mark_drs
obj_attr_list["mgt_902_deprovisioning_request_sent"] = \
_time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"])
self.connect(obj_attr_list["access"])
_msg = "Sending a termination request for " + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
storage_keys = []
for _idx in range(1, int(obj_attr_list["imageids"]) + 1):
_pool_key = "poolbase" + str(_idx)
if _pool_key in obj_attr_list:
storage_keys.append(obj_attr_list[_pool_key])
kwargs = self.dic_to_rpc_kwargs(self.ftcconn, "destroy_instances",
obj_attr_list)
self.ftcconn.destroy_instances(storage_keys, **kwargs)
self.take_action_if_requested("VM", obj_attr_list,
"deprovision_finished")
_time_mark_drc = int(time())
obj_attr_list["mgt_903_deprovisioning_request_completed"] = \
_time_mark_drc - _time_mark_drs
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "was successfully "
_msg += "destroyed on FTCloud \"" + obj_attr_list["cloud_name"]
_msg += "\"."
cbdebug(_msg)
return 0, _msg
def is_vm_ready(self, obj_attr_list) :
'''
TBD
'''
if self.is_vm_running(obj_attr_list) :
self.take_action_if_requested("VM", obj_attr_list, "provision_complete")
if self.get_ip_address(obj_attr_list) :
cbdebug("VM " + obj_attr_list["name"] + " received IP: " + obj_attr_list["cloud_ip"])
obj_attr_list["cloud_hostname"] = "cb-" + obj_attr_list["cloud_ip"].replace('.', '-')
obj_attr_list["last_known_state"] = "running with ip assigned"
return True
else :
obj_attr_list["last_known_state"] = "running with ip unassigned"
return False
else :
obj_attr_list["last_known_state"] = "not running"
return False
def groupregister(self, obj_attr_list):
'''
TBD
'''
try:
self.connect(obj_attr_list["access"])
_status, _msg, _info = self.plmconn.group_register(
obj_attr_list["group_name"])
if _info:
if len(_info):
obj_attr_list.update(_info)
cbdebug(_msg, True)
return _status, _msg, obj_attr_list
except PLMException, obj:
status = obj.status
_fmsg = "PLM Exception: " + obj.msg
return status, _fmsg, None
def noderegister(self, obj_attr_list):
'''
TBD
'''
try:
obj_attr_list["uuid"] = "undefined"
_fmsg = "Hypervisor node \"" + obj_attr_list["name"] + "\" ("
_fmsg = obj_attr_list["uuid"] + ") could not be registered "
_fmsg += " on this Parallel Libvirt Manager."
self.connect(obj_attr_list["access"])
_status, _msg, _info = self.plmconn.node_register(
obj_attr_list["name"], obj_attr_list["function"])
obj_attr_list.update(_info)
obj_attr_list["uuid"] = obj_attr_list["cloud_uuid"]
obj_attr_list["arrival"] = int(time())
# A way to specify an alternative IP address for a hypervisor
# This alternative 'destination' represents a faster NIC
# (such as infiniband) to be used for other types of traffic
if "replication_nodes" in obj_attr_list:
_replication_nodes = obj_attr_list["replication_nodes"]
else:
_replication_nodes = ""
if _replication_nodes.strip() != "":
_rnodes = str2dic(_replication_nodes)
if obj_attr_list["name"] in _rnodes:
obj_attr_list["svm_destination"] = gethostbyname(
_rnodes[obj_attr_list["name"]])
if "svm_destination" not in obj_attr_list:
obj_attr_list["svm_destination"] = obj_attr_list["cloud_ip"]
cbdebug(_msg, True)
return _status, _msg, obj_attr_list
except PLMException, obj:
status = obj.status
_fmsg = "PLM Exception: " + obj.msg
return status, _fmsg, None
def get_ip_address(self, obj_attr_list):
'''
TBD
'''
try:
_private_hostname = '{0}'.format(
obj_attr_list["instance_obj"].private_dns_name)
_private_ip_address = '{0}'.format(
obj_attr_list["instance_obj"].private_ip_address)
_public_hostname = '{0}'.format(
obj_attr_list["instance_obj"].public_dns_name)
_public_hostname, _public_ip_address = hostname2ip(
_public_hostname)
if obj_attr_list["run_netname"] == "private":
obj_attr_list["cloud_hostname"] = _private_hostname
obj_attr_list["run_cloud_ip"] = _private_ip_address
else:
obj_attr_list["cloud_hostname"] = _public_hostname
obj_attr_list["run_cloud_ip"] = _public_ip_address
# NOTE: "cloud_ip" is always equal to "run_cloud_ip"
obj_attr_list["cloud_ip"] = obj_attr_list["run_cloud_ip"]
if str(obj_attr_list["use_vpn_ip"]).lower() == "true" and str(
obj_attr_list["vpn_only"]).lower() == "true":
assert (self.get_attr_from_pending(obj_attr_list))
if "cloud_init_vpn" not in obj_attr_list:
cbdebug("Instance VPN address not yet available.")
return False
cbdebug("Found VPN IP: " + obj_attr_list["cloud_init_vpn"])
obj_attr_list["prov_cloud_ip"] = obj_attr_list[
"cloud_init_vpn"]
else:
if obj_attr_list["prov_netname"] == "private":
obj_attr_list["prov_cloud_ip"] = _private_ip_address
else:
obj_attr_list["prov_cloud_ip"] = _public_ip_address
return True
except:
return False
def start_daemon(self,
cmdline,
port=None,
protocol=None,
conditional=False,
search_keywords=None):
'''
TBD
'''
if self.hostname == "127.0.0.1":
_cmd = cmdline
if port:
_pid, _username = self.get_pid_from_port(port, protocol)
if _pid:
if _username != self.username:
return ["pnf-" + _pid + '-' + _username]
if conditional:
_pid = self.get_pid_from_cmdline(
_cmd if not search_keywords else search_keywords)
if not _pid:
_status, _a, _b = self.run_os_command(_cmd)
# The process could be a daemon, so there is no point in waiting
# for its completion. However, we do wait a little bit just to be
# able to get the pid of the process, because a daemon would fork
# a new process.
sleep(3)
else:
if cmdline.count("cbact") and cmdline.count("--daemon"):
_cmd = "script -qfec \"" + _cmd + "\""
_status, _a, _b = self.run_os_command(_cmd)
# Same comment
sleep(3)
_pid = self.get_pid_from_cmdline(
cmdline if not search_keywords else search_keywords)
_msg = "A process with the command line \"" + cmdline + "\" (pid "
_msg += str(_pid) + ") was started succesfully."
cbdebug(_msg)
return [_pid]
def nodecleanup(self, obj_attr_list):
'''
TBD
'''
try:
_fmsg = "Hypervisor node \"" + obj_attr_list["name"] + "\""
_fmsg = " could not be cleaned-up on this Parallel Libvirt Manager."
self.connect(obj_attr_list["access"])
_status, _msg, _info = self.plmconn.node_cleanup(
obj_attr_list["name"], obj_attr_list["tag"],
obj_attr_list["userid"])
cbdebug(_msg)
return _status, _msg, _info
except PLMException, obj:
status = obj.status
_fmsg = "PLM Exception: " + obj.msg
return status, _fmsg, None
def get_ip_address(self, obj_attr_list):
'''
TBD
'''
try:
obj_attr_list[
"cloud_hostname"] = "vm" + obj_attr_list["name"].split("_")[1]
obj_attr_list["cloud_ip"] = obj_attr_list[
"instance_obj"].private_ips[0]
obj_attr_list["prov_cloud_ip"] = obj_attr_list["cloud_ip"]
_msg = "Public IP = " + obj_attr_list["cloud_hostname"]
_msg += " Private IP = " + obj_attr_list["cloud_ip"]
cbdebug(_msg)
return True
except:
_msg = "Could not retrieve IP addresses for object " + obj_attr_list[
"uuid"]
_msg += "from vCloud Director \"" + obj_attr_list["cloud_name"]
cberr(_msg)
raise CldOpsException(_msg, _status)
def vmcreate(self, obj_attr_list) :
# The only thing we're doing here is recording into MongoDB properly
# exactly which container maps to which physical k8s node, so that we can
# track in the database where the container lived.
status = 0
msg = ""
try :
# Then just call the original function to actually do the create.
status, msg = KubCmds.vmcreate(self, obj_attr_list)
if status == 0 :
_vmc_attr_list = self.osci.get_object(obj_attr_list["cloud_name"], "VMC", False, obj_attr_list["vmc"], False)
for droplet in loads(_vmc_attr_list["droplets"]) :
if droplet["name"] == obj_attr_list["node"] :
obj_attr_list["host_name"] = droplet["id"]
cbdebug("Container " + obj_attr_list["name"] + " sent to Node " + obj_attr_list["node"] + " = " + str(obj_attr_list["host_name"]), True)
obj_attr_list["droplet"] = dumps(droplet)
break
except Exception, e :
for line in traceback.format_exc().splitlines() :
cbwarn(line, True)
def extra_vmc_setup(self, vmc_name, vmc_defaults, vm_defaults,
vm_templates, connection):
'''
TBD
'''
if "OS_TENANT_NAME" in self.connauth_pamap:
vm_defaults["tenant_from_rc"] = self.connauth_pamap[
"OS_TENANT_NAME"]
vm_defaults["access"] = self.access
vmc_defaults["access"] = self.access
if not LibcloudCmds.floating_ip_pools:
cbdebug(
" Caching " + self.get_description() + " Floating IP pools...",
True)
LibcloudCmds.floating_ip_pools = connection.ex_list_floating_ip_pools(
)
return True
def is_vm_ready(self, obj_attr_list):
'''
TBD
'''
if self.is_vm_running(obj_attr_list):
if self.get_ip_address(obj_attr_list):
obj_attr_list["cloud_hostname"] = "cb-" + obj_attr_list[
"cloud_ip"].replace('.', '-')
cbdebug("VM " + obj_attr_list["name"] + " received IP: " +
obj_attr_list["cloud_ip"])
obj_attr_list["last_known_state"] = "running with ip assigned"
return True
else:
obj_attr_list[
"last_known_state"] = "running with ip unassigned"
else:
obj_attr_list["last_known_state"] = "not running"
return False
def get_openvpn_client_ip(self, obj_attr_list):
'''
TBD
'''
if "openvpn_ip" in obj_attr_list:
return True
elif self.osci.pending_object_exists(obj_attr_list["cloud_name"], "VM",
obj_attr_list["uuid"],
"openvpn_ip"):
ip = self.osci.pending_object_get(obj_attr_list["cloud_name"],
"VM", obj_attr_list["uuid"],
"openvpn_ip")
if ip:
cbdebug("Openvpn reported in from client with ip: " + ip)
obj_attr_list["openvpn_ip"] = ip
obj_attr_list["prov_cloud_ip"] = ip
return True
return False
def __init__(self, pid, passive, active, background, debug, port, hostname) :
super(APIService, self).__init__()
self._stop = threading.Event()
self.pid = pid
self.abort = False
self.aborted = False
self.port = port
self.hostname = hostname
self.api = API(pid, passive, active, background)
cbdebug("Initializing API Service on port " + str(self.port))
if debug is None :
self.server = AsyncDocXMLRPCServer((self.hostname, int(self.port)), allow_none = True)
else :
self.server = DocXMLRPCServer((self.hostname, int(self.port)), allow_none = True)
self.server.abort = False
self.server.aborted = False
self.server.set_server_title("API Service (xmlrpc)")
self.server.set_server_name("API Service (xmlrpc)")
#self.server.register_introspection_functions()
self.server.register_instance(self.api)
cbdebug("API Service started")
def vmmigrate(self, obj_attr_list) :
'''
TBD
'''
_time_mark_crs = int(time())
operation = obj_attr_list["mtype"]
obj_attr_list["mgt_502_" + operation + "_request_sent"] = _time_mark_crs - obj_attr_list["mgt_501_" + operation + "_request_originated"]
_msg = "Sending a " + operation + " request for " + obj_attr_list["name"]
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
if obj_attr_list["placement"] != "random" :
self.host_resource_update(obj_attr_list, "migrate")
_time_mark_crc = int(time())
obj_attr_list["mgt_503_" + operation + "_request_completed"] = _time_mark_crc - _time_mark_crs
cbdebug("VM " + obj_attr_list["name"] + " " + operation + " request completed.")
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "was successfully "
_msg += operation + "ed on SimCloud \"" + obj_attr_list["cloud_name"]
_msg += "\"."
cbdebug(_msg)
return 0, _msg
def vmresize(self, obj_attr_list):
'''
TBD
'''
self.connect(obj_attr_list["access"])
_time_mark_crs = int(time())
obj_attr_list[
"mgt_302_resize_request_sent"] = _time_mark_crs - obj_attr_list[
"mgt_301_resize_request_originated"]
_msg = "Sending a resize request for " + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")"
_msg += "...."
cbdebug(_msg, True)
self.vmresize_actual(obj_attr_list)
_time_mark_crc = int(time())
obj_attr_list[
"mgt_303_resize_request_completed"] = _time_mark_crc - _time_mark_crs
cbdebug("VM " + obj_attr_list["name"] + " resize request completed.")
_msg = "" + obj_attr_list["name"] + ""
_msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") "
_msg += "was successfully "
_msg += "resized on FTCloud \"" + obj_attr_list["cloud_name"]
_msg += "\"."
cbdebug(_msg)
return 0, _msg
def vmcregister(self, obj_attr_list):
'''
TBD
'''
_time_mark_prs = int(time())
obj_attr_list[
"mgt_002_provisioning_request_sent"] = _time_mark_prs - int(
obj_attr_list["mgt_001_provisioning_request_originated"])
if "cleanup_on_attach" in obj_attr_list and obj_attr_list[
"cleanup_on_attach"] == "True":
self.vmccleanup(obj_attr_list)
obj_attr_list["cloud_hostname"] = obj_attr_list["name"]
_fmsg = "VMC " + obj_attr_list["uuid"] + " could not be registered "
_fmsg += " on TAcloud \"" + obj_attr_list["cloud_name"] + "\"."
self.connect(obj_attr_list["access"])
_status, _fmsg, info = self.ftcconn.node_register(
obj_attr_list["name"])
obj_attr_list.update(info)
obj_attr_list["arrival"] = int(time())
if obj_attr_list["discover_hosts"].lower() == "true":
self.discover_hosts(obj_attr_list, _time_mark_prs)
else:
obj_attr_list["hosts"] = ''
obj_attr_list["host_list"] = {}
obj_attr_list["host_count"] = "NA"
_time_mark_prc = int(time())
obj_attr_list[
"mgt_003_provisioning_request_completed"] = _time_mark_prc - _time_mark_prs
_msg = "VMC " + obj_attr_list["uuid"] + " was successfully "
_msg += "registered on FTCloud \"" + obj_attr_list["cloud_name"] + "\"."
cbdebug(_msg, True)
return 0, _msg